pub trait ModelOperations: Send {
Show 34 methods
// Required methods
fn architecture(&self) -> ModelArchitecture;
fn config(&self) -> &dyn ArchitectureConfig;
fn forward(
&self,
input: &Tensor,
past_kv: Option<&Tensor>,
) -> Result<Tensor>;
fn reshape_for_attention(
&self,
tensor: &Tensor,
is_key_value: bool,
) -> Result<Tensor>;
fn apply_rope(
&self,
tensor: &Tensor,
position_ids: &Tensor,
) -> Result<Tensor>;
fn normalize(&self, tensor: &Tensor) -> Result<Tensor>;
fn get_attention_mask(
&self,
seq_len: usize,
past_kv_len: usize,
) -> Result<Tensor>;
fn as_any(&self) -> &dyn Any;
// Provided methods
fn get_tokenizer_config(&self) -> Box<dyn TokenizerConfig> { ... }
fn decision_hidden_states(&self, _input: &Tensor) -> Result<Tensor> { ... }
fn decision_hidden_states_checked(
&self,
input: &Tensor,
check: &dyn Fn() -> Result<()>,
) -> Result<Tensor> { ... }
fn decision_output_embedding_rows(&self, _ids: &[i64]) -> Result<Tensor> { ... }
fn var_store(&self) -> Option<&VarStore> { ... }
fn var_store_mut(&mut self) -> Option<&mut VarStore> { ... }
fn forward_with_cache(
&self,
input: &Tensor,
_start_pos: usize,
) -> Result<Tensor> { ... }
fn forward_with_cache_and_delta(
&self,
input: &Tensor,
start_pos: usize,
_delta: Option<&TenantDelta>,
) -> Result<Tensor> { ... }
fn forward_batched(
&self,
_sequences: &mut [(Vec<i64>, usize, Arc<Mutex<KVCacheManager>>)],
_delta: Option<&TenantDelta>,
) -> Result<Tensor> { ... }
fn embed_tokens(&self, _input_ids: &Tensor) -> Result<Tensor> { ... }
fn forward_from_embeddings(
&self,
_embeddings: &Tensor,
_start_pos: usize,
) -> Result<Tensor> { ... }
fn is_multimodal(&self) -> bool { ... }
fn prepare_multimodal_inputs(
&self,
_input_ids: &Tensor,
_pixel_values: Option<&Tensor>,
_images_seq_mask: Option<&Tensor>,
_images_emb_mask: Option<&Tensor>,
) -> Result<Tensor> { ... }
fn encode_vision(&self, _pixel_values: &Tensor) -> Result<Tensor> { ... }
fn decode_layer(
&self,
_layer_idx: usize,
_hidden_states: &Tensor,
_attention_mask: Option<&Tensor>,
_position_ids: Option<&Tensor>,
_past_kv: Option<&LayerKVCache>,
) -> Result<(Tensor, Option<(Tensor, Tensor)>)> { ... }
fn decode_layer_with_delta(
&self,
layer_idx: usize,
hidden_states: &Tensor,
attention_mask: Option<&Tensor>,
position_ids: Option<&Tensor>,
past_kv: Option<&LayerKVCache>,
_delta: &TenantDelta,
) -> Result<(Tensor, Option<(Tensor, Tensor)>)> { ... }
fn forward_layers(
&self,
_hidden: &Tensor,
_range: Range<usize>,
_start_pos: usize,
_delta: Option<&TenantDelta>,
) -> Result<Tensor> { ... }
fn forward_layers_train(
&self,
_hidden: &Tensor,
_range: Range<usize>,
_delta: Option<&TenantDelta>,
) -> Result<Tensor> { ... }
fn apply_final_norm(&self, _hidden_states: &Tensor) -> Result<Tensor> { ... }
fn lm_head(&self, _hidden_states: &Tensor) -> Result<Tensor> { ... }
fn num_layers(&self) -> usize { ... }
fn clear_kv_cache(&self) { ... }
fn kv_cache_memory_usage(&self) -> usize { ... }
fn set_kv_cache(&mut self, _cache: Arc<Mutex<KVCacheManager>>) { ... }
fn get_kv_cache(&self) -> Option<Arc<Mutex<KVCacheManager>>> { ... }
fn take_kv_cache(&mut self) -> Option<Arc<Mutex<KVCacheManager>>> { ... }
}Expand description
Core trait for architecture-specific model operations
Required Methods§
Sourcefn architecture(&self) -> ModelArchitecture
fn architecture(&self) -> ModelArchitecture
Get the architecture type
Sourcefn config(&self) -> &dyn ArchitectureConfig
fn config(&self) -> &dyn ArchitectureConfig
Get architecture configuration
Sourcefn forward(&self, input: &Tensor, past_kv: Option<&Tensor>) -> Result<Tensor>
fn forward(&self, input: &Tensor, past_kv: Option<&Tensor>) -> Result<Tensor>
Forward pass through the model
Sourcefn reshape_for_attention(
&self,
tensor: &Tensor,
is_key_value: bool,
) -> Result<Tensor>
fn reshape_for_attention( &self, tensor: &Tensor, is_key_value: bool, ) -> Result<Tensor>
Reshape tensor for attention computation
Sourcefn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor>
fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor>
Apply RoPE (Rotary Position Embeddings) if supported
Sourcefn normalize(&self, tensor: &Tensor) -> Result<Tensor>
fn normalize(&self, tensor: &Tensor) -> Result<Tensor>
Apply architecture-specific normalization (RMSNorm, LayerNorm, etc.)
Sourcefn get_attention_mask(
&self,
seq_len: usize,
past_kv_len: usize,
) -> Result<Tensor>
fn get_attention_mask( &self, seq_len: usize, past_kv_len: usize, ) -> Result<Tensor>
Get attention mask for the architecture
Provided Methods§
Sourcefn get_tokenizer_config(&self) -> Box<dyn TokenizerConfig>
fn get_tokenizer_config(&self) -> Box<dyn TokenizerConfig>
Get the tokenizer configuration for this model
Returns a boxed TokenizerConfig trait object that provides model-specific tokenizer configuration logic.
Stateless full-sequence final-normalized hidden states; no vocabulary logits. Implementations must use fresh call-local recurrent/KV state.
Stateless decision forward with a cooperative deadline/cancellation check. Native implementations call the checker between decoder layers.
Sourcefn decision_output_embedding_rows(&self, _ids: &[i64]) -> Result<Tensor>
fn decision_output_embedding_rows(&self, _ids: &[i64]) -> Result<Tensor>
Gather original output embedding rows, preserving the LM-head vocabulary axis.
Sourcefn var_store_mut(&mut self) -> Option<&mut VarStore>
fn var_store_mut(&mut self) -> Option<&mut VarStore>
Get mutable VarStore for training (if available)
Sourcefn forward_with_cache(
&self,
input: &Tensor,
_start_pos: usize,
) -> Result<Tensor>
fn forward_with_cache( &self, input: &Tensor, _start_pos: usize, ) -> Result<Tensor>
Forward pass with position information for KV caching
Sourcefn forward_with_cache_and_delta(
&self,
input: &Tensor,
start_pos: usize,
_delta: Option<&TenantDelta>,
) -> Result<Tensor>
fn forward_with_cache_and_delta( &self, input: &Tensor, start_pos: usize, _delta: Option<&TenantDelta>, ) -> Result<Tensor>
Forward pass with KV caching and optional per-tenant delta injection
When delta is Some, LoRA corrections are injected at each attention layer’s
Q/V projections before KV cache update. This is the unified path for both
base model inference (delta=None) and delta-aware inference (delta=Some).
Default implementation ignores delta and falls back to forward_with_cache().
Only architectures that support delta injection (Llama) need to override this.
Sourcefn forward_batched(
&self,
_sequences: &mut [(Vec<i64>, usize, Arc<Mutex<KVCacheManager>>)],
_delta: Option<&TenantDelta>,
) -> Result<Tensor>
fn forward_batched( &self, _sequences: &mut [(Vec<i64>, usize, Arc<Mutex<KVCacheManager>>)], _delta: Option<&TenantDelta>, ) -> Result<Tensor>
Batched ragged forward for continuous decode (#329, epic #310).
Each entry is (new_token_ids, start_pos, per-sequence KVCacheManager);
all rows share the same query length and the single delta (the scheduler
groups by tenant delta). Returns stacked logits [B, q, vocab]. Per-row
results are equivalent to running each sequence through
forward_with_cache_and_delta serially (CPU-verified merge gate).
Default: unsupported. Only architectures with batched-decode support (Llama in v1) override this; callers fall back to the batch=1 path.
Sourcefn embed_tokens(&self, _input_ids: &Tensor) -> Result<Tensor>
fn embed_tokens(&self, _input_ids: &Tensor) -> Result<Tensor>
Get token embeddings for input IDs
Sourcefn forward_from_embeddings(
&self,
_embeddings: &Tensor,
_start_pos: usize,
) -> Result<Tensor>
fn forward_from_embeddings( &self, _embeddings: &Tensor, _start_pos: usize, ) -> Result<Tensor>
Forward pass from pre-computed embeddings (for multimodal models)
This method allows models to start generation from embeddings instead of token IDs. Useful for multimodal models where vision embeddings are merged with text embeddings.
§Arguments
embeddings- Pre-computed input embeddings [batch_size, seq_len, hidden_size]start_pos- Starting position in KV cache (0 for initial forward)
§Returns
Logits tensor [batch_size, seq_len, vocab_size]
Sourcefn is_multimodal(&self) -> bool
fn is_multimodal(&self) -> bool
Check if this model is multimodal
Sourcefn prepare_multimodal_inputs(
&self,
_input_ids: &Tensor,
_pixel_values: Option<&Tensor>,
_images_seq_mask: Option<&Tensor>,
_images_emb_mask: Option<&Tensor>,
) -> Result<Tensor>
fn prepare_multimodal_inputs( &self, _input_ids: &Tensor, _pixel_values: Option<&Tensor>, _images_seq_mask: Option<&Tensor>, _images_emb_mask: Option<&Tensor>, ) -> Result<Tensor>
Prepare inputs with vision embeddings (for multimodal models)
This method combines text token IDs with vision embeddings for multimodal inference. Only implemented for multimodal architectures.
§Arguments
input_ids- Text token IDs [batch_size, seq_len]pixel_values- Preprocessed images [batch_size, channels, height, width]images_seq_mask- Where to inject vision embeddings [batch_size, seq_len]images_emb_mask- Which vision embeddings to use [batch_size, num_patches]
§Returns
Combined embeddings [batch_size, seq_len, hidden_size]
Sourcefn encode_vision(&self, _pixel_values: &Tensor) -> Result<Tensor>
fn encode_vision(&self, _pixel_values: &Tensor) -> Result<Tensor>
Encode images through the vision encoder and return embeddings.
For multimodal models with a vision encoder (e.g. Janus with SigLIP), this runs the vision encoder forward pass and returns pooled embeddings.
§Arguments
pixel_values- Preprocessed images [batch_size, channels, height, width]
§Returns
Embedding tensor [batch_size, embedding_dim]
Sourcefn decode_layer(
&self,
_layer_idx: usize,
_hidden_states: &Tensor,
_attention_mask: Option<&Tensor>,
_position_ids: Option<&Tensor>,
_past_kv: Option<&LayerKVCache>,
) -> Result<(Tensor, Option<(Tensor, Tensor)>)>
fn decode_layer( &self, _layer_idx: usize, _hidden_states: &Tensor, _attention_mask: Option<&Tensor>, _position_ids: Option<&Tensor>, _past_kv: Option<&LayerKVCache>, ) -> Result<(Tensor, Option<(Tensor, Tensor)>)>
Decode a single layer (for layer-wise processing)
Sourcefn decode_layer_with_delta(
&self,
layer_idx: usize,
hidden_states: &Tensor,
attention_mask: Option<&Tensor>,
position_ids: Option<&Tensor>,
past_kv: Option<&LayerKVCache>,
_delta: &TenantDelta,
) -> Result<(Tensor, Option<(Tensor, Tensor)>)>
fn decode_layer_with_delta( &self, layer_idx: usize, hidden_states: &Tensor, attention_mask: Option<&Tensor>, position_ids: Option<&Tensor>, past_kv: Option<&LayerKVCache>, _delta: &TenantDelta, ) -> Result<(Tensor, Option<(Tensor, Tensor)>)>
Decode a single layer with per-tenant delta injection (for TTT training path)
The delta’s A/B matrices are injected after q_proj/v_proj projections inside the attention layer, creating a differentiable path from loss back to delta parameters. No KV cache is used (training path).
Default implementation ignores the delta and falls back to decode_layer().
Only architectures that support delta injection need to override this.
Sourcefn forward_layers(
&self,
_hidden: &Tensor,
_range: Range<usize>,
_start_pos: usize,
_delta: Option<&TenantDelta>,
) -> Result<Tensor>
fn forward_layers( &self, _hidden: &Tensor, _range: Range<usize>, _start_pos: usize, _delta: Option<&TenantDelta>, ) -> Result<Tensor>
Run a contiguous range of decoder layers — the 2b intra-host pipeline (layer-split) primitive (#314).
This is the missing layer-range runner that complements the already-
exposed embed_tokens / forward_from_embeddings / apply_final_norm /
lm_head / num_layers. Arch-agnostic orchestration composes them:
- stage 0 :
embed_tokens→forward_layers(0..b) - middle :
forward_layers(a..b) - last :
forward_layers(a..N)→apply_final_norm→lm_head
is_first/is_last are implicit in range (range.start == 0 /
range.end == num_layers()); the runner itself only applies decoder
layers — never embeddings, final norm, or the LM head.
§Stage-boundary contract
The only state carried across a stage boundary is hidden + start_pos.
position_ids is recomputed inside from start_pos + seq. Per-layer KV
cache (and any SSM conv/rec state) is stage-local and never
transferred. range is in global layer indices; an implementation
that owns a shard remaps to its local self.layers via the
layer_offset it was constructed with.
§Device placement
Layer g runs on its mapped device; the single cross-device copy is
hidden.to_device(next) inserted only at a boundary where the device
actually changes (zero copies within a stage or when source == dest).
The returned tensor lives on the last owned layer’s device.
§Arguments
hidden-[batch, seq, hidden]; embeddings ifrange.start == 0, otherwise the previous stage’s output.range- global layer indices this stage owns,[a..b).start_pos- KV-cache start position for this forward.delta- optional per-tenant LoRA delta (delta-aware inference).
The default implementation errors; only architectures that support the
pipeline split (Llama; Qwen3.5) override it. The single-device whole-model
forward* paths are unaffected — this method is purely additive.
Sourcefn forward_layers_train(
&self,
_hidden: &Tensor,
_range: Range<usize>,
_delta: Option<&TenantDelta>,
) -> Result<Tensor>
fn forward_layers_train( &self, _hidden: &Tensor, _range: Range<usize>, _delta: Option<&TenantDelta>, ) -> Result<Tensor>
Training-path sibling of Self::forward_layers — the cross-device
autograd primitive for TTT-on-split (#316, M-TRAIN-COUPLING).
TTT is inference-time training that must traverse the same layer
partition inference uses, so the backward pass materializes grads on each
parameter’s own device. This runner builds that autograd graph across the
crate::runtime::device_pool::LayerDeviceMap; the lone stage-boundary
hidden.to_device(next) is autograd-transparent (tch to_device is
differentiable), so gradients flow back through it to the previous stage’s
device.
§How it differs from the inference Self::forward_layers
The inference and training paths are kept deliberately separate (as the whole-model paths already are). The training path:
- uses no KV cache — full causal attention over the entire context;
- pins
start_pos = 0—position_idsare0..seq; - uses fresh, call-local recurrent (SSM) state for hybrid
architectures — the persistent inference
conv/recstate is never read or written, so a TTT step cannot pollute the inference recurrent state (and the split is numerically identical to the whole-model training forward, since per-layer recurrent state never crosses a layer boundary).
Everything else — the global↔local layer remap via layer_offset, the
single boundary copy, per-layer delta injection keyed by global index —
matches Self::forward_layers. The arch-agnostic stage orchestration is
identical (embed_tokens → forward_layers_train(0..b) → … →
forward_layers_train(a..N) → apply_final_norm → lm_head), and the
loss/backward is driven by the caller (e.g. the TTT trainer).
The default implementation errors; only architectures that support the
pipeline split (Llama; Qwen3.5) override it. The single-device whole-model
training path (crate::runtime::TorchEngine::forward_with_delta) is
unaffected — this method is purely additive.
Sourcefn apply_final_norm(&self, _hidden_states: &Tensor) -> Result<Tensor>
fn apply_final_norm(&self, _hidden_states: &Tensor) -> Result<Tensor>
Apply final layer normalization
Sourcefn lm_head(&self, _hidden_states: &Tensor) -> Result<Tensor>
fn lm_head(&self, _hidden_states: &Tensor) -> Result<Tensor>
Apply language model head to get logits
Sourcefn num_layers(&self) -> usize
fn num_layers(&self) -> usize
Get the number of transformer layers
Sourcefn clear_kv_cache(&self)
fn clear_kv_cache(&self)
Clear the KV cache (for new generation)
Sourcefn kv_cache_memory_usage(&self) -> usize
fn kv_cache_memory_usage(&self) -> usize
Get KV cache memory usage in bytes
Sourcefn set_kv_cache(&mut self, _cache: Arc<Mutex<KVCacheManager>>)
fn set_kv_cache(&mut self, _cache: Arc<Mutex<KVCacheManager>>)
Set an external KV cache (for session-based cache management)
This allows a KVCacheRegistry to manage caches externally while the model uses them for inference.
Sourcefn get_kv_cache(&self) -> Option<Arc<Mutex<KVCacheManager>>>
fn get_kv_cache(&self) -> Option<Arc<Mutex<KVCacheManager>>>
Get the current KV cache (for external management)
Sourcefn take_kv_cache(&mut self) -> Option<Arc<Mutex<KVCacheManager>>>
fn take_kv_cache(&mut self) -> Option<Arc<Mutex<KVCacheManager>>>
Take ownership of the KV cache (removes it from the model)