pub struct LlamaModel { /* private fields */ }Expand description
Llama model implementation
Implementations§
Source§impl LlamaModel
impl LlamaModel
Sourcepub fn from_safetensors(
path: &Path,
device: &Device,
dtype: DType,
) -> Result<Self>
pub fn from_safetensors( path: &Path, device: &Device, dtype: DType, ) -> Result<Self>
Create Llama model from SafeTensors
Sourcepub fn from_weights(
weights: &mut HashMap<String, Tensor>,
device: &Device,
dtype: DType,
) -> Result<Self>
pub fn from_weights( weights: &mut HashMap<String, Tensor>, device: &Device, dtype: DType, ) -> Result<Self>
Create Llama model from pre-loaded weight tensors
Sourcepub fn from_weights_with_config(
weights: &mut HashMap<String, Tensor>,
config: LlamaConfig,
device: &Device,
dtype: DType,
kv_quant_type: KVQuantType,
) -> Result<Self>
pub fn from_weights_with_config( weights: &mut HashMap<String, Tensor>, config: LlamaConfig, device: &Device, dtype: DType, kv_quant_type: KVQuantType, ) -> Result<Self>
Create Llama model with explicit config (allows Qwen models to override) Build model from weights, taking mutable reference to free tensors as they’re processed. This reduces peak memory by ~50% by removing original weight tensors after transposing.
Sourcepub fn stage_from_weights_with_config(
weights: &mut HashMap<String, Tensor>,
config: LlamaConfig,
devices: &LayerDeviceMap,
layer_range: Range<usize>,
dtype: DType,
kv_quant_type: KVQuantType,
) -> Result<Self>
pub fn stage_from_weights_with_config( weights: &mut HashMap<String, Tensor>, config: LlamaConfig, devices: &LayerDeviceMap, layer_range: Range<usize>, dtype: DType, kv_quant_type: KVQuantType, ) -> Result<Self>
Build a single pipeline stage of a Llama model (#314, 2b layer-split).
Loads only global decoder layers [layer_range.start..layer_range.end)
onto their mapped devices (devices.device_for(g)), and gates the
non-layer weights by stage position so middle stages carry none (M-LOAD
seam #1):
is_first(layer_range.start == 0) → keepembed_tokens.is_last(layer_range.end == devices.len()) → keepnorm+lm_head(or the tied transpose).
weights must already contain (at least) this stage’s tensors. Production
stage requests use ModelFactory::create_stage, whose authoritative
HF-index loader supplies only the requested range and role tensors.
Tensors not on the target device are moved with a single .to() at
construction — the only placement cost; the forward path then does zero
intra-stage copies.
State sizing follows the owned layer count, not the global count
(M-LOAD seam #2): the KV-cache manager and self.layers are both sized to
layer_range.len(), and the forward loop indexes them locally.
Sourcepub fn detect_config_from_weights(
weights: &HashMap<String, Tensor>,
) -> Result<LlamaConfig>
pub fn detect_config_from_weights( weights: &HashMap<String, Tensor>, ) -> Result<LlamaConfig>
Detect configuration from weight tensor shapes
Sourcepub fn parse_config(json_str: &str) -> Result<LlamaConfig>
pub fn parse_config(json_str: &str) -> Result<LlamaConfig>
Parse configuration from JSON
Trait Implementations§
Source§impl ModelOperations for LlamaModel
impl ModelOperations for LlamaModel
Source§fn architecture(&self) -> ModelArchitecture
fn architecture(&self) -> ModelArchitecture
Source§fn config(&self) -> &dyn ArchitectureConfig
fn config(&self) -> &dyn ArchitectureConfig
Source§fn forward(&self, input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor>
fn forward(&self, input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor>
Source§fn var_store_mut(&mut self) -> Option<&mut VarStore>
fn var_store_mut(&mut self) -> Option<&mut VarStore>
Source§fn forward_with_cache(&self, input: &Tensor, start_pos: usize) -> Result<Tensor>
fn forward_with_cache(&self, input: &Tensor, start_pos: usize) -> Result<Tensor>
Source§fn forward_with_cache_and_delta(
&self,
input: &Tensor,
start_pos: usize,
delta: Option<&TenantDelta>,
) -> Result<Tensor>
fn forward_with_cache_and_delta( &self, input: &Tensor, start_pos: usize, delta: Option<&TenantDelta>, ) -> Result<Tensor>
Source§fn forward_batched(
&self,
sequences: &mut [(Vec<i64>, usize, Arc<Mutex<KVCacheManager>>)],
delta: Option<&TenantDelta>,
) -> Result<Tensor>
fn forward_batched( &self, sequences: &mut [(Vec<i64>, usize, Arc<Mutex<KVCacheManager>>)], delta: Option<&TenantDelta>, ) -> Result<Tensor>
Source§fn embed_tokens(&self, input_ids: &Tensor) -> Result<Tensor>
fn embed_tokens(&self, input_ids: &Tensor) -> Result<Tensor>
Source§fn forward_from_embeddings(
&self,
embeddings: &Tensor,
start_pos: usize,
) -> Result<Tensor>
fn forward_from_embeddings( &self, embeddings: &Tensor, start_pos: usize, ) -> Result<Tensor>
Source§fn reshape_for_attention(
&self,
tensor: &Tensor,
is_key_value: bool,
) -> Result<Tensor>
fn reshape_for_attention( &self, tensor: &Tensor, is_key_value: bool, ) -> Result<Tensor>
Source§fn decode_layer(
&self,
layer_idx: usize,
hidden_states: &Tensor,
_attention_mask: Option<&Tensor>,
position_ids: Option<&Tensor>,
_past_kv: Option<&LayerKVCache>,
) -> Result<(Tensor, Option<(Tensor, Tensor)>)>
fn decode_layer( &self, layer_idx: usize, hidden_states: &Tensor, _attention_mask: Option<&Tensor>, position_ids: Option<&Tensor>, _past_kv: Option<&LayerKVCache>, ) -> Result<(Tensor, Option<(Tensor, Tensor)>)>
Source§fn decode_layer_with_delta(
&self,
layer_idx: usize,
hidden_states: &Tensor,
_attention_mask: Option<&Tensor>,
position_ids: Option<&Tensor>,
_past_kv: Option<&LayerKVCache>,
delta: &TenantDelta,
) -> Result<(Tensor, Option<(Tensor, Tensor)>)>
fn decode_layer_with_delta( &self, layer_idx: usize, hidden_states: &Tensor, _attention_mask: Option<&Tensor>, position_ids: Option<&Tensor>, _past_kv: Option<&LayerKVCache>, delta: &TenantDelta, ) -> Result<(Tensor, Option<(Tensor, Tensor)>)>
Source§fn forward_layers(
&self,
hidden: &Tensor,
range: Range<usize>,
start_pos: usize,
delta: Option<&TenantDelta>,
) -> Result<Tensor>
fn forward_layers( &self, hidden: &Tensor, range: Range<usize>, start_pos: usize, delta: Option<&TenantDelta>, ) -> Result<Tensor>
Source§fn forward_layers_train(
&self,
hidden: &Tensor,
range: Range<usize>,
delta: Option<&TenantDelta>,
) -> Result<Tensor>
fn forward_layers_train( &self, hidden: &Tensor, range: Range<usize>, delta: Option<&TenantDelta>, ) -> Result<Tensor>
Self::forward_layers — the cross-device
autograd primitive for TTT-on-split (#316, M-TRAIN-COUPLING). Read moreSource§fn apply_final_norm(&self, hidden_states: &Tensor) -> Result<Tensor>
fn apply_final_norm(&self, hidden_states: &Tensor) -> Result<Tensor>
Source§fn lm_head(&self, hidden_states: &Tensor) -> Result<Tensor>
fn lm_head(&self, hidden_states: &Tensor) -> Result<Tensor>
Source§fn num_layers(&self) -> usize
fn num_layers(&self) -> usize
Source§fn apply_rope(&self, _tensor: &Tensor, _position_ids: &Tensor) -> Result<Tensor>
fn apply_rope(&self, _tensor: &Tensor, _position_ids: &Tensor) -> Result<Tensor>
Source§fn normalize(&self, tensor: &Tensor) -> Result<Tensor>
fn normalize(&self, tensor: &Tensor) -> Result<Tensor>
Source§fn get_attention_mask(
&self,
seq_len: usize,
past_kv_len: usize,
) -> Result<Tensor>
fn get_attention_mask( &self, seq_len: usize, past_kv_len: usize, ) -> Result<Tensor>
Source§fn clear_kv_cache(&self)
fn clear_kv_cache(&self)
Source§fn kv_cache_memory_usage(&self) -> usize
fn kv_cache_memory_usage(&self) -> usize
Source§fn set_kv_cache(&mut self, cache: Arc<Mutex<KVCacheManager>>)
fn set_kv_cache(&mut self, cache: Arc<Mutex<KVCacheManager>>)
Source§fn get_kv_cache(&self) -> Option<Arc<Mutex<KVCacheManager>>>
fn get_kv_cache(&self) -> Option<Arc<Mutex<KVCacheManager>>>
Source§fn take_kv_cache(&mut self) -> Option<Arc<Mutex<KVCacheManager>>>
fn take_kv_cache(&mut self) -> Option<Arc<Mutex<KVCacheManager>>>
Source§fn get_tokenizer_config(&self) -> Box<dyn TokenizerConfig>
fn get_tokenizer_config(&self) -> Box<dyn TokenizerConfig>
Source§fn decision_output_embedding_rows(&self, _ids: &[i64]) -> Result<Tensor>
fn decision_output_embedding_rows(&self, _ids: &[i64]) -> Result<Tensor>
Source§fn is_multimodal(&self) -> bool
fn is_multimodal(&self) -> bool
Source§fn prepare_multimodal_inputs(
&self,
_input_ids: &Tensor,
_pixel_values: Option<&Tensor>,
_images_seq_mask: Option<&Tensor>,
_images_emb_mask: Option<&Tensor>,
) -> Result<Tensor>
fn prepare_multimodal_inputs( &self, _input_ids: &Tensor, _pixel_values: Option<&Tensor>, _images_seq_mask: Option<&Tensor>, _images_emb_mask: Option<&Tensor>, ) -> Result<Tensor>
Source§fn encode_vision(&self, _pixel_values: &Tensor) -> Result<Tensor>
fn encode_vision(&self, _pixel_values: &Tensor) -> Result<Tensor>
impl Send for LlamaModel
impl Sync for LlamaModel
Auto Trait Implementations§
impl Freeze for LlamaModel
impl !RefUnwindSafe for LlamaModel
impl Unpin for LlamaModel
impl UnsafeUnpin for LlamaModel
impl !UnwindSafe for LlamaModel
Blanket Implementations§
§impl<T> AnySync for T
impl<T> AnySync for T
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
§impl<T> FutureExt for T
impl<T> FutureExt for T
§fn with_context(self, otel_cx: Context) -> WithContext<Self>
fn with_context(self, otel_cx: Context) -> WithContext<Self>
§fn with_current_context(self) -> WithContext<Self>
fn with_current_context(self) -> WithContext<Self>
§impl<T> Identity for Twhere
T: ?Sized,
impl<T> Identity for Twhere
T: ?Sized,
§impl<T> Instrument for T
impl<T> Instrument for T
§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more