Skip to main content

GemmaModel

Struct GemmaModel 

Source
pub struct GemmaModel { /* private fields */ }
Expand description

Gemma model implementation

Implementations§

Source§

impl GemmaModel

Source

pub fn from_safetensors( path: &Path, device: &Device, dtype: Kind, ) -> Result<GemmaModel, Error>

Create Gemma model from SafeTensors

Source

pub fn from_weights( weights: &HashMap<String, Tensor>, device: &Device, dtype: Kind, ) -> Result<GemmaModel, Error>

Create Gemma model from pre-loaded weights

Source

pub fn from_weights_gemma3( weights: &HashMap<String, Tensor>, device: &Device, dtype: Kind, ) -> Result<GemmaModel, Error>

Create Gemma3 model with specific configuration

Source§

impl GemmaModel

Source

pub fn from_varstore( vs: &VarStore, device: &Device, dtype: Kind, ) -> Result<GemmaModel, Error>

Create Gemma model from VarStore weights (proper implementation)

Source

pub fn load_weights_from_varstore(&mut self, vs: &VarStore) -> Result<(), Error>

Load model weights from a VarStore (SafeTensors) - legacy method for compatibility

Source

pub fn simple_forward(&self, input: &Tensor) -> Result<Tensor, Error>

Create a simple forward pass that returns valid tensor shapes

Trait Implementations§

Source§

impl ModelOperations for GemmaModel

Source§

fn architecture(&self) -> ModelArchitecture

Get the architecture type
Source§

fn config(&self) -> &dyn ArchitectureConfig

Get architecture configuration
Source§

fn forward( &self, input: &Tensor, _past_kv: Option<&Tensor>, ) -> Result<Tensor, Error>

Forward pass through the model
Source§

fn reshape_for_attention( &self, tensor: &Tensor, is_key_value: bool, ) -> Result<Tensor, Error>

Reshape tensor for attention computation
Source§

fn apply_rope( &self, tensor: &Tensor, position_ids: &Tensor, ) -> Result<Tensor, Error>

Apply RoPE (Rotary Position Embeddings) if supported
Source§

fn normalize(&self, tensor: &Tensor) -> Result<Tensor, Error>

Apply architecture-specific normalization (RMSNorm, LayerNorm, etc.)
Source§

fn get_attention_mask( &self, seq_len: usize, past_kv_len: usize, ) -> Result<Tensor, Error>

Get attention mask for the architecture
Source§

fn as_any(&self) -> &(dyn Any + 'static)

Downcast to Any for accessing architecture-specific methods
Source§

fn get_tokenizer_config(&self) -> Box<dyn TokenizerConfig>

Get the tokenizer configuration for this model Read more
Source§

fn decision_hidden_states(&self, _input: &Tensor) -> Result<Tensor, Error>

Stateless full-sequence final-normalized hidden states; no vocabulary logits. Implementations must use fresh call-local recurrent/KV state.
Source§

fn decision_hidden_states_checked( &self, input: &Tensor, check: &dyn Fn() -> Result<(), Error>, ) -> Result<Tensor, Error>

Stateless decision forward with a cooperative deadline/cancellation check. Native implementations call the checker between decoder layers.
Source§

fn decision_output_embedding_rows(&self, _ids: &[i64]) -> Result<Tensor, Error>

Gather original output embedding rows, preserving the LM-head vocabulary axis.
Source§

fn var_store(&self) -> Option<&VarStore>

Get VarStore for training (if available)
Source§

fn var_store_mut(&mut self) -> Option<&mut VarStore>

Get mutable VarStore for training (if available)
Source§

fn forward_with_cache( &self, input: &Tensor, _start_pos: usize, ) -> Result<Tensor, Error>

Forward pass with position information for KV caching
Source§

fn forward_with_cache_and_delta( &self, input: &Tensor, start_pos: usize, _delta: Option<&TenantDelta>, ) -> Result<Tensor, Error>

Forward pass with KV caching and optional per-tenant delta injection Read more
Source§

fn forward_batched( &self, _sequences: &mut [(Vec<i64>, usize, Arc<Mutex<RawMutex, KVCacheManager>>)], _delta: Option<&TenantDelta>, ) -> Result<Tensor, Error>

Batched ragged forward for continuous decode (#329, epic #310). Read more
Source§

fn embed_tokens(&self, _input_ids: &Tensor) -> Result<Tensor, Error>

Get token embeddings for input IDs
Source§

fn forward_from_embeddings( &self, _embeddings: &Tensor, _start_pos: usize, ) -> Result<Tensor, Error>

Forward pass from pre-computed embeddings (for multimodal models) Read more
Source§

fn is_multimodal(&self) -> bool

Check if this model is multimodal
Source§

fn prepare_multimodal_inputs( &self, _input_ids: &Tensor, _pixel_values: Option<&Tensor>, _images_seq_mask: Option<&Tensor>, _images_emb_mask: Option<&Tensor>, ) -> Result<Tensor, Error>

Prepare inputs with vision embeddings (for multimodal models) Read more
Source§

fn encode_vision(&self, _pixel_values: &Tensor) -> Result<Tensor, Error>

Encode images through the vision encoder and return embeddings. Read more
Source§

fn decode_layer( &self, _layer_idx: usize, _hidden_states: &Tensor, _attention_mask: Option<&Tensor>, _position_ids: Option<&Tensor>, _past_kv: Option<&LayerKVCache>, ) -> Result<(Tensor, Option<(Tensor, Tensor)>), Error>

Decode a single layer (for layer-wise processing)
Source§

fn decode_layer_with_delta( &self, layer_idx: usize, hidden_states: &Tensor, attention_mask: Option<&Tensor>, position_ids: Option<&Tensor>, past_kv: Option<&LayerKVCache>, _delta: &TenantDelta, ) -> Result<(Tensor, Option<(Tensor, Tensor)>), Error>

Decode a single layer with per-tenant delta injection (for TTT training path) Read more
Source§

fn forward_layers( &self, _hidden: &Tensor, _range: Range<usize>, _start_pos: usize, _delta: Option<&TenantDelta>, ) -> Result<Tensor, Error>

Run a contiguous range of decoder layers — the 2b intra-host pipeline (layer-split) primitive (#314). Read more
Source§

fn forward_layers_train( &self, _hidden: &Tensor, _range: Range<usize>, _delta: Option<&TenantDelta>, ) -> Result<Tensor, Error>

Training-path sibling of Self::forward_layers — the cross-device autograd primitive for TTT-on-split (#316, M-TRAIN-COUPLING). Read more
Source§

fn apply_final_norm(&self, _hidden_states: &Tensor) -> Result<Tensor, Error>

Apply final layer normalization
Source§

fn lm_head(&self, _hidden_states: &Tensor) -> Result<Tensor, Error>

Apply language model head to get logits
Source§

fn num_layers(&self) -> usize

Get the number of transformer layers
Source§

fn clear_kv_cache(&self)

Clear the KV cache (for new generation)
Source§

fn kv_cache_memory_usage(&self) -> usize

Get KV cache memory usage in bytes
Source§

fn set_kv_cache(&mut self, _cache: Arc<Mutex<RawMutex, KVCacheManager>>)

Set an external KV cache (for session-based cache management) Read more
Source§

fn get_kv_cache(&self) -> Option<Arc<Mutex<RawMutex, KVCacheManager>>>

Get the current KV cache (for external management)
Source§

fn take_kv_cache(&mut self) -> Option<Arc<Mutex<RawMutex, KVCacheManager>>>

Take ownership of the KV cache (removes it from the model)
Source§

impl Send for GemmaModel

Source§

impl Sync for GemmaModel

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
§

impl<T> Any for T
where T: Any,

§

fn into_any(self: Box<T>) -> Box<dyn Any>

§

fn into_any_rc(self: Rc<T>) -> Rc<dyn Any>

§

fn type_name(&self) -> &'static str

§

impl<T> AnySync for T
where T: Any + Send + Sync,

§

fn into_any_arc(self: Arc<T>) -> Arc<dyn Any + Sync + Send> ⓘ

§

impl<'a, T, E> AsTaggedExplicit<'a, E> for T
where T: 'a,

§

fn explicit(self, class: Class, tag: u32) -> TaggedParser<'a, Explicit, Self, E>

§

impl<'a, T, E> AsTaggedImplicit<'a, E> for T
where T: 'a,

§

fn implicit( self, class: Class, constructed: bool, tag: u32, ) -> TaggedParser<'a, Implicit, Self, E>

Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

§

impl<T> FutureExt for T

§

fn with_context(self, otel_cx: Context) -> WithContext<Self>

Attaches the provided Context to this type, returning a WithContext wrapper. Read more
§

fn with_current_context(self) -> WithContext<Self>

Attaches the current Context to this type, returning a WithContext wrapper. Read more
§

impl<T, W> HasTypeWitness<W> for T
where W: MakeTypeWitness<Arg = T>, T: ?Sized,

§

const WITNESS: W = W::MAKE

A constant of the type witness
§

impl<T> Identity for T
where T: ?Sized,

§

const TYPE_EQ: TypeEq<T, <T as Identity>::Type> = TypeEq::NEW

Proof that Self is the same type as Self::Type, provides methods for casting between Self and Self::Type.
§

type Type = T

The same type as Self, used to emulate type equality bounds (T == U) with associated type equality constraints (T: Identity<Type = U>).
§

impl<T> Instrument for T

§

fn instrument(self, span: Span) -> Instrumented<Self>

Instruments this type with the provided [Span], returning an Instrumented wrapper. Read more
§

fn in_current_span(self) -> Instrumented<Self>

Instruments this type with the current Span, returning an Instrumented wrapper. Read more
Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> IntoEither for T

Source§

fn into_either(self, into_left: bool) -> Either<Self, Self>

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

impl<T> IntoRequest<T> for T

Source§

fn into_request(self) -> Request<T>

Wrap the input message T in a tonic::Request
§

impl<T> JaggedSlice<T> for T

§

type Index = Index2

§

fn push_into(self, array: &mut Jagged<T>)

§

fn insert_into(self, index: <T as JaggedSlice<T>>::Index, array: &mut Jagged<T>)

§

impl<T> Pointable for T

§

const ALIGN: usize

The alignment of pointer.
§

type Init = T

The type for initializers.
§

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more
§

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more
§

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more
§

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more
§

impl<T> PolicyExt for T
where T: ?Sized,

§

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns [Action::Follow] only if self and other return Action::Follow. Read more
§

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns [Action::Follow] if either self or other returns Action::Follow. Read more
Source§

impl<T> Same for T

Source§

type Output = T

Should always be Self
§

impl<T> ServiceExt for T

§

fn map_response_body<F>(self, f: F) -> MapResponseBody<Self, F>
where Self: Sized,

Apply a transformation to the response body. Read more
§

fn trace_for_http(self) -> Trace<Self, SharedClassifier<ServerErrorsAsFailures>>
where Self: Sized,

High level tracing that classifies responses using HTTP status codes. Read more
§

fn trace_for_grpc(self) -> Trace<Self, SharedClassifier<GrpcErrorsAsFailures>>
where Self: Sized,

High level tracing that classifies responses using gRPC headers. Read more
§

fn follow_redirects(self) -> FollowRedirect<Self>
where Self: Sized,

Follow redirect resposes using the Standard policy. Read more
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = Infallible

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

§

fn vzip(self) -> V

§

impl<T> WithSubscriber for T

§

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a [WithDispatch] wrapper. Read more
§

fn with_current_subscriber(self) -> WithDispatch<Self>

Attaches the current default Subscriber to this type, returning a [WithDispatch] wrapper. Read more
§

impl<T> ErasedDestructor for T
where T: 'static,

§

impl<T> MaybeSend for T
where T: Send,

§

impl<T> MaybeSend for T
where T: Send,

§

impl<T> MaybeSync for T
where T: Sync,

§

impl<T> MaybeSync for T
where T: Sync,

§

impl<T> SendSyncUnwindSafe for T
where T: Send + Sync + UnwindSafe + ?Sized,