pub struct GemmaModel { /* private fields */ }Expand description
Gemma model implementation
Implementations§
Source§impl GemmaModel
impl GemmaModel
Sourcepub fn from_safetensors(
path: &Path,
device: &Device,
dtype: Kind,
) -> Result<GemmaModel, Error>
pub fn from_safetensors( path: &Path, device: &Device, dtype: Kind, ) -> Result<GemmaModel, Error>
Create Gemma model from SafeTensors
Sourcepub fn from_weights(
weights: &HashMap<String, Tensor>,
device: &Device,
dtype: Kind,
) -> Result<GemmaModel, Error>
pub fn from_weights( weights: &HashMap<String, Tensor>, device: &Device, dtype: Kind, ) -> Result<GemmaModel, Error>
Create Gemma model from pre-loaded weights
Sourcepub fn from_weights_gemma3(
weights: &HashMap<String, Tensor>,
device: &Device,
dtype: Kind,
) -> Result<GemmaModel, Error>
pub fn from_weights_gemma3( weights: &HashMap<String, Tensor>, device: &Device, dtype: Kind, ) -> Result<GemmaModel, Error>
Create Gemma3 model with specific configuration
Source§impl GemmaModel
impl GemmaModel
Sourcepub fn from_varstore(
vs: &VarStore,
device: &Device,
dtype: Kind,
) -> Result<GemmaModel, Error>
pub fn from_varstore( vs: &VarStore, device: &Device, dtype: Kind, ) -> Result<GemmaModel, Error>
Create Gemma model from VarStore weights (proper implementation)
Sourcepub fn load_weights_from_varstore(&mut self, vs: &VarStore) -> Result<(), Error>
pub fn load_weights_from_varstore(&mut self, vs: &VarStore) -> Result<(), Error>
Load model weights from a VarStore (SafeTensors) - legacy method for compatibility
Sourcepub fn simple_forward(&self, input: &Tensor) -> Result<Tensor, Error>
pub fn simple_forward(&self, input: &Tensor) -> Result<Tensor, Error>
Create a simple forward pass that returns valid tensor shapes
Trait Implementations§
Source§impl ModelOperations for GemmaModel
impl ModelOperations for GemmaModel
Source§fn architecture(&self) -> ModelArchitecture
fn architecture(&self) -> ModelArchitecture
Get the architecture type
Source§fn config(&self) -> &dyn ArchitectureConfig
fn config(&self) -> &dyn ArchitectureConfig
Get architecture configuration
Source§fn forward(
&self,
input: &Tensor,
_past_kv: Option<&Tensor>,
) -> Result<Tensor, Error>
fn forward( &self, input: &Tensor, _past_kv: Option<&Tensor>, ) -> Result<Tensor, Error>
Forward pass through the model
Source§fn reshape_for_attention(
&self,
tensor: &Tensor,
is_key_value: bool,
) -> Result<Tensor, Error>
fn reshape_for_attention( &self, tensor: &Tensor, is_key_value: bool, ) -> Result<Tensor, Error>
Reshape tensor for attention computation
Source§fn apply_rope(
&self,
tensor: &Tensor,
position_ids: &Tensor,
) -> Result<Tensor, Error>
fn apply_rope( &self, tensor: &Tensor, position_ids: &Tensor, ) -> Result<Tensor, Error>
Apply RoPE (Rotary Position Embeddings) if supported
Source§fn normalize(&self, tensor: &Tensor) -> Result<Tensor, Error>
fn normalize(&self, tensor: &Tensor) -> Result<Tensor, Error>
Apply architecture-specific normalization (RMSNorm, LayerNorm, etc.)
Source§fn get_attention_mask(
&self,
seq_len: usize,
past_kv_len: usize,
) -> Result<Tensor, Error>
fn get_attention_mask( &self, seq_len: usize, past_kv_len: usize, ) -> Result<Tensor, Error>
Get attention mask for the architecture
Source§fn as_any(&self) -> &(dyn Any + 'static)
fn as_any(&self) -> &(dyn Any + 'static)
Downcast to Any for accessing architecture-specific methods
Source§fn get_tokenizer_config(&self) -> Box<dyn TokenizerConfig>
fn get_tokenizer_config(&self) -> Box<dyn TokenizerConfig>
Get the tokenizer configuration for this model Read more
Stateless full-sequence final-normalized hidden states; no vocabulary logits.
Implementations must use fresh call-local recurrent/KV state.
Stateless decision forward with a cooperative deadline/cancellation check.
Native implementations call the checker between decoder layers.
Source§fn decision_output_embedding_rows(&self, _ids: &[i64]) -> Result<Tensor, Error>
fn decision_output_embedding_rows(&self, _ids: &[i64]) -> Result<Tensor, Error>
Gather original output embedding rows, preserving the LM-head vocabulary axis.
Source§fn var_store_mut(&mut self) -> Option<&mut VarStore>
fn var_store_mut(&mut self) -> Option<&mut VarStore>
Get mutable VarStore for training (if available)
Source§fn forward_with_cache(
&self,
input: &Tensor,
_start_pos: usize,
) -> Result<Tensor, Error>
fn forward_with_cache( &self, input: &Tensor, _start_pos: usize, ) -> Result<Tensor, Error>
Forward pass with position information for KV caching
Source§fn forward_with_cache_and_delta(
&self,
input: &Tensor,
start_pos: usize,
_delta: Option<&TenantDelta>,
) -> Result<Tensor, Error>
fn forward_with_cache_and_delta( &self, input: &Tensor, start_pos: usize, _delta: Option<&TenantDelta>, ) -> Result<Tensor, Error>
Forward pass with KV caching and optional per-tenant delta injection Read more
Source§fn forward_batched(
&self,
_sequences: &mut [(Vec<i64>, usize, Arc<Mutex<RawMutex, KVCacheManager>>)],
_delta: Option<&TenantDelta>,
) -> Result<Tensor, Error>
fn forward_batched( &self, _sequences: &mut [(Vec<i64>, usize, Arc<Mutex<RawMutex, KVCacheManager>>)], _delta: Option<&TenantDelta>, ) -> Result<Tensor, Error>
Batched ragged forward for continuous decode (#329, epic #310). Read more
Source§fn embed_tokens(&self, _input_ids: &Tensor) -> Result<Tensor, Error>
fn embed_tokens(&self, _input_ids: &Tensor) -> Result<Tensor, Error>
Get token embeddings for input IDs
Source§fn forward_from_embeddings(
&self,
_embeddings: &Tensor,
_start_pos: usize,
) -> Result<Tensor, Error>
fn forward_from_embeddings( &self, _embeddings: &Tensor, _start_pos: usize, ) -> Result<Tensor, Error>
Forward pass from pre-computed embeddings (for multimodal models) Read more
Source§fn is_multimodal(&self) -> bool
fn is_multimodal(&self) -> bool
Check if this model is multimodal
Source§fn prepare_multimodal_inputs(
&self,
_input_ids: &Tensor,
_pixel_values: Option<&Tensor>,
_images_seq_mask: Option<&Tensor>,
_images_emb_mask: Option<&Tensor>,
) -> Result<Tensor, Error>
fn prepare_multimodal_inputs( &self, _input_ids: &Tensor, _pixel_values: Option<&Tensor>, _images_seq_mask: Option<&Tensor>, _images_emb_mask: Option<&Tensor>, ) -> Result<Tensor, Error>
Prepare inputs with vision embeddings (for multimodal models) Read more
Source§fn encode_vision(&self, _pixel_values: &Tensor) -> Result<Tensor, Error>
fn encode_vision(&self, _pixel_values: &Tensor) -> Result<Tensor, Error>
Encode images through the vision encoder and return embeddings. Read more
Source§fn decode_layer(
&self,
_layer_idx: usize,
_hidden_states: &Tensor,
_attention_mask: Option<&Tensor>,
_position_ids: Option<&Tensor>,
_past_kv: Option<&LayerKVCache>,
) -> Result<(Tensor, Option<(Tensor, Tensor)>), Error>
fn decode_layer( &self, _layer_idx: usize, _hidden_states: &Tensor, _attention_mask: Option<&Tensor>, _position_ids: Option<&Tensor>, _past_kv: Option<&LayerKVCache>, ) -> Result<(Tensor, Option<(Tensor, Tensor)>), Error>
Decode a single layer (for layer-wise processing)
Source§fn decode_layer_with_delta(
&self,
layer_idx: usize,
hidden_states: &Tensor,
attention_mask: Option<&Tensor>,
position_ids: Option<&Tensor>,
past_kv: Option<&LayerKVCache>,
_delta: &TenantDelta,
) -> Result<(Tensor, Option<(Tensor, Tensor)>), Error>
fn decode_layer_with_delta( &self, layer_idx: usize, hidden_states: &Tensor, attention_mask: Option<&Tensor>, position_ids: Option<&Tensor>, past_kv: Option<&LayerKVCache>, _delta: &TenantDelta, ) -> Result<(Tensor, Option<(Tensor, Tensor)>), Error>
Decode a single layer with per-tenant delta injection (for TTT training path) Read more
Source§fn forward_layers(
&self,
_hidden: &Tensor,
_range: Range<usize>,
_start_pos: usize,
_delta: Option<&TenantDelta>,
) -> Result<Tensor, Error>
fn forward_layers( &self, _hidden: &Tensor, _range: Range<usize>, _start_pos: usize, _delta: Option<&TenantDelta>, ) -> Result<Tensor, Error>
Run a contiguous range of decoder layers — the 2b intra-host pipeline
(layer-split) primitive (#314). Read more
Source§fn forward_layers_train(
&self,
_hidden: &Tensor,
_range: Range<usize>,
_delta: Option<&TenantDelta>,
) -> Result<Tensor, Error>
fn forward_layers_train( &self, _hidden: &Tensor, _range: Range<usize>, _delta: Option<&TenantDelta>, ) -> Result<Tensor, Error>
Training-path sibling of
Self::forward_layers — the cross-device
autograd primitive for TTT-on-split (#316, M-TRAIN-COUPLING). Read moreSource§fn apply_final_norm(&self, _hidden_states: &Tensor) -> Result<Tensor, Error>
fn apply_final_norm(&self, _hidden_states: &Tensor) -> Result<Tensor, Error>
Apply final layer normalization
Source§fn lm_head(&self, _hidden_states: &Tensor) -> Result<Tensor, Error>
fn lm_head(&self, _hidden_states: &Tensor) -> Result<Tensor, Error>
Apply language model head to get logits
Source§fn num_layers(&self) -> usize
fn num_layers(&self) -> usize
Get the number of transformer layers
Source§fn clear_kv_cache(&self)
fn clear_kv_cache(&self)
Clear the KV cache (for new generation)
Source§fn kv_cache_memory_usage(&self) -> usize
fn kv_cache_memory_usage(&self) -> usize
Get KV cache memory usage in bytes
Source§fn set_kv_cache(&mut self, _cache: Arc<Mutex<RawMutex, KVCacheManager>>)
fn set_kv_cache(&mut self, _cache: Arc<Mutex<RawMutex, KVCacheManager>>)
Set an external KV cache (for session-based cache management) Read more
Source§fn get_kv_cache(&self) -> Option<Arc<Mutex<RawMutex, KVCacheManager>>>
fn get_kv_cache(&self) -> Option<Arc<Mutex<RawMutex, KVCacheManager>>>
Get the current KV cache (for external management)
Source§fn take_kv_cache(&mut self) -> Option<Arc<Mutex<RawMutex, KVCacheManager>>>
fn take_kv_cache(&mut self) -> Option<Arc<Mutex<RawMutex, KVCacheManager>>>
Take ownership of the KV cache (removes it from the model)
impl Send for GemmaModel
impl Sync for GemmaModel
Auto Trait Implementations§
impl Freeze for GemmaModel
impl RefUnwindSafe for GemmaModel
impl Unpin for GemmaModel
impl UnsafeUnpin for GemmaModel
impl UnwindSafe for GemmaModel
Blanket Implementations§
§impl<T> AnySync for T
impl<T> AnySync for T
§impl<'a, T, E> AsTaggedExplicit<'a, E> for Twhere
T: 'a,
impl<'a, T, E> AsTaggedExplicit<'a, E> for Twhere
T: 'a,
§impl<'a, T, E> AsTaggedImplicit<'a, E> for Twhere
T: 'a,
impl<'a, T, E> AsTaggedImplicit<'a, E> for Twhere
T: 'a,
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
§impl<T> FutureExt for T
impl<T> FutureExt for T
§fn with_context(self, otel_cx: Context) -> WithContext<Self>
fn with_context(self, otel_cx: Context) -> WithContext<Self>
§fn with_current_context(self) -> WithContext<Self>
fn with_current_context(self) -> WithContext<Self>
§impl<T> Identity for Twhere
T: ?Sized,
impl<T> Identity for Twhere
T: ?Sized,
§impl<T> Instrument for T
impl<T> Instrument for T
§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§impl<T> IntoRequest<T> for T
impl<T> IntoRequest<T> for T
Source§fn into_request(self) -> Request<T>
fn into_request(self) -> Request<T>
Wrap the input message
T in a tonic::Request§impl<T> JaggedSlice<T> for T
impl<T> JaggedSlice<T> for T
§impl<T> Pointable for T
impl<T> Pointable for T
§impl<T> PolicyExt for Twhere
T: ?Sized,
impl<T> PolicyExt for Twhere
T: ?Sized,
§impl<T> ServiceExt for T
impl<T> ServiceExt for T
§fn map_response_body<F>(self, f: F) -> MapResponseBody<Self, F>where
Self: Sized,
fn map_response_body<F>(self, f: F) -> MapResponseBody<Self, F>where
Self: Sized,
Apply a transformation to the response body. Read more
§fn trace_for_http(self) -> Trace<Self, SharedClassifier<ServerErrorsAsFailures>>where
Self: Sized,
fn trace_for_http(self) -> Trace<Self, SharedClassifier<ServerErrorsAsFailures>>where
Self: Sized,
High level tracing that classifies responses using HTTP status codes. Read more
§fn trace_for_grpc(self) -> Trace<Self, SharedClassifier<GrpcErrorsAsFailures>>where
Self: Sized,
fn trace_for_grpc(self) -> Trace<Self, SharedClassifier<GrpcErrorsAsFailures>>where
Self: Sized,
High level tracing that classifies responses using gRPC headers. Read more