Skip to main content

hyprstream_inference/runtime/architectures/
mod.rs

1//! Architecture-specific model implementations
2//!
3//! This module provides abstractions and implementations for different
4//! transformer architectures (Llama, Gemma, Qwen, etc.) with proper
5//! tensor shape handling and LoRA compatibility.
6
7use anyhow::{anyhow, Result};
8use tch::{nn, Tensor};
9
10pub mod config;
11pub mod gemma;
12pub mod janus;
13pub mod llama;
14pub mod qwen;
15pub mod qwen3_5;
16pub mod qwen3_5_vision;
17pub mod siglip;
18// LoRA adapter moved to lora module
19
20pub use config::{ArchitectureConfig, AttentionConfig};
21// pub use lora_adapter::ArchitectureAwareLoRAAdapter; // Module removed
22
23/// Add a TTT/LoRA delta correction to a base activation, coercing the correction
24/// to the base tensor's dtype first.
25///
26/// The base model may run in BF16 while the delta matrix is forced to fp32
27/// (see `runtime/ttn_profile.rs`). Adding fp32 directly to a bf16 tensor panics
28/// in `addmm_impl_cpu_` ("expected m1 and m2 to have the same dtype").
29///
30/// This helper is the single place that performs the dtype coercion so the fix
31/// cannot be missed when a new architecture adds a delta-injection site.
32/// See #139 (original llama fix) and #440 (qwen3_5 regression).
33#[inline]
34pub fn add_delta(base: &Tensor, correction: Tensor) -> Tensor {
35    base + correction.to_kind(base.kind())
36}
37
38/// Supported model architectures
39#[derive(Debug, Clone, PartialEq)]
40pub enum ModelArchitecture {
41    /// Meta's Llama family (1/2/3)
42    Llama { version: u8 },
43    /// Google's Gemma
44    Gemma,
45    /// Alibaba's Qwen (including Qwen3 with GQA and sparse attention)
46    Qwen {
47        version: u8,
48        /// Qwen3 MoE variant (30B-A3B, 235B-A22B)
49        is_moe: bool,
50        /// Context length (32K, 128K, 262K for Qwen3)
51        context_length: usize,
52    },
53    /// Microsoft's Phi
54    Phi { version: u8 },
55    /// Mistral AI models
56    Mistral,
57    /// BigCode's Starcoder
58    Starcoder,
59    /// TII's Falcon
60    Falcon,
61    /// EleutherAI's GPT-NeoX
62    GPTNeoX,
63    /// OpenAI GPT-OSS (120B/20B MoE models)
64    GPTOSS {
65        /// Total parameters in billions (120 or 20)
66        total_params_b: u16,
67        /// Active parameters in billions (5.1 or 3.6)
68        active_params_b: f32,
69        /// Number of experts in MoE
70        num_experts: usize,
71        /// Experts per token
72        experts_per_token: usize,
73    },
74    /// GPT-J (EleutherAI's 6B model)
75    GPTJ,
76    /// Janus multimodal vision-language model
77    Janus {
78        /// Base language model architecture (usually Llama)
79        base_architecture: Box<ModelArchitecture>,
80        /// Type of vision encoder
81        vision_encoder: VisionEncoderType,
82        /// Whether model supports image generation (VQ-VAE)
83        has_generation: bool,
84    },
85    /// Alibaba Qwen3.5 hybrid GatedDeltaNet/full-attention (dense and MoE variants)
86    Qwen3_5,
87    /// Custom/unknown architecture
88    Custom(String),
89}
90
91/// Types of vision encoders for multimodal models
92#[derive(Debug, Clone, PartialEq)]
93pub enum VisionEncoderType {
94    /// SigLIP vision encoder
95    SigLIP {
96        hidden_size: usize,
97        image_size: usize,
98        patch_size: usize,
99        num_layers: usize,
100    },
101    /// CLIP vision encoder
102    CLIP {
103        hidden_size: usize,
104        image_size: usize,
105        patch_size: usize,
106        num_layers: usize,
107    },
108    /// EVA vision encoder
109    EVA {
110        hidden_size: usize,
111        image_size: usize,
112        patch_size: usize,
113        num_layers: usize,
114    },
115}
116
117impl ModelArchitecture {
118    /// Get human-readable name
119    pub fn name(&self) -> String {
120        match self {
121            Self::Llama { version } => format!("Llama{version}"),
122            Self::Gemma => "Gemma".to_owned(),
123            Self::Qwen {
124                version,
125                is_moe,
126                context_length,
127            } => {
128                if *is_moe {
129                    format!("Qwen{}-MoE-{}K", version, context_length / 1000)
130                } else {
131                    format!("Qwen{}-{}K", version, context_length / 1000)
132                }
133            }
134            Self::Phi { version } => format!("Phi{version}"),
135            Self::Mistral => "Mistral".to_owned(),
136            Self::Starcoder => "Starcoder".to_owned(),
137            Self::Falcon => "Falcon".to_owned(),
138            Self::GPTNeoX => "GPT-NeoX".to_owned(),
139            Self::GPTOSS { total_params_b, .. } => format!("GPT-OSS-{total_params_b}B"),
140            Self::GPTJ => "GPT-J".to_owned(),
141            Self::Qwen3_5 => "Qwen3_5".to_owned(),
142            Self::Janus {
143                vision_encoder,
144                has_generation,
145                ..
146            } => {
147                let encoder_type = match vision_encoder {
148                    VisionEncoderType::SigLIP { .. } => "SigLIP",
149                    VisionEncoderType::CLIP { .. } => "CLIP",
150                    VisionEncoderType::EVA { .. } => "EVA",
151                };
152                if *has_generation {
153                    format!("Janus-{encoder_type}-Gen")
154                } else {
155                    format!("Janus-{encoder_type}")
156                }
157            }
158            Self::Custom(name) => name.clone(),
159        }
160    }
161
162    /// Check if architecture supports multi-query attention
163    pub fn supports_mqa(&self) -> bool {
164        matches!(self, Self::Gemma | Self::Falcon)
165    }
166
167    /// Check if architecture supports grouped-query attention
168    pub fn supports_gqa(&self) -> bool {
169        matches!(
170            self,
171            Self::Llama { version: 3 } |
172            Self::Mistral |
173            Self::Qwen { version: 3, .. } |  // Qwen3 uses GQA
174            Self::GPTOSS { .. } // GPT-OSS uses GQA in MoE
175        )
176    }
177
178    /// Check if architecture uses Mixture of Experts
179    pub fn supports_moe(&self) -> bool {
180        matches!(self, Self::Qwen { is_moe: true, .. } | Self::GPTOSS { .. })
181    }
182
183    /// Check if architecture supports sparse attention
184    pub fn supports_sparse_attention(&self) -> bool {
185        matches!(
186            self,
187            Self::Qwen { version: 3, .. } |  // Qwen3 uses MInference sparse attention
188            Self::GPTOSS { .. } // GPT-OSS optimized for long context
189        )
190    }
191}
192
193/// Core trait for architecture-specific model operations
194pub trait ModelOperations: Send {
195    /// Get the architecture type
196    fn architecture(&self) -> ModelArchitecture;
197
198    /// Get architecture configuration
199    fn config(&self) -> &dyn ArchitectureConfig;
200
201    /// Get the tokenizer configuration for this model
202    ///
203    /// Returns a boxed TokenizerConfig trait object that provides
204    /// model-specific tokenizer configuration logic.
205    fn get_tokenizer_config(&self) -> Box<dyn crate::runtime::tokenizer_config::TokenizerConfig> {
206        use crate::runtime::tokenizer_config::{
207            DefaultTokenizerConfig, GemmaTokenizerConfig, LlamaTokenizerConfig, QwenTokenizerConfig,
208        };
209
210        // Return appropriate config based on architecture
211        match self.architecture() {
212            ModelArchitecture::Qwen { .. } | ModelArchitecture::Qwen3_5 => {
213                Box::new(QwenTokenizerConfig)
214            }
215            ModelArchitecture::Llama { .. } => Box::new(LlamaTokenizerConfig),
216            ModelArchitecture::Gemma => Box::new(GemmaTokenizerConfig),
217            ModelArchitecture::Janus {
218                base_architecture, ..
219            } => {
220                // Use the base architecture's tokenizer config
221                match base_architecture.as_ref() {
222                    ModelArchitecture::Llama { .. } => Box::new(LlamaTokenizerConfig),
223                    ModelArchitecture::Qwen { .. } => Box::new(QwenTokenizerConfig),
224                    _ => Box::new(DefaultTokenizerConfig),
225                }
226            }
227            _ => Box::new(DefaultTokenizerConfig),
228        }
229    }
230
231    /// Forward pass through the model
232    fn forward(&self, input: &Tensor, past_kv: Option<&Tensor>) -> Result<Tensor>;
233
234    /// Stateless full-sequence final-normalized hidden states; no vocabulary logits.
235    /// Implementations must use fresh call-local recurrent/KV state.
236    fn decision_hidden_states(&self, _input: &Tensor) -> Result<Tensor> {
237        Err(anyhow!(
238            "decision hidden states unsupported for this architecture"
239        ))
240    }
241
242    /// Stateless decision forward with a cooperative deadline/cancellation check.
243    /// Native implementations call the checker between decoder layers.
244    fn decision_hidden_states_checked(
245        &self,
246        input: &Tensor,
247        check: &dyn Fn() -> Result<()>,
248    ) -> Result<Tensor> {
249        check()?;
250        self.decision_hidden_states(input)
251    }
252
253    /// Gather original output embedding rows, preserving the LM-head vocabulary axis.
254    fn decision_output_embedding_rows(&self, _ids: &[i64]) -> Result<Tensor> {
255        Err(anyhow!(
256            "decision output embeddings unsupported for this architecture"
257        ))
258    }
259
260    /// Get VarStore for training (if available)
261    fn var_store(&self) -> Option<&nn::VarStore> {
262        None // Default implementation returns None
263    }
264
265    /// Get mutable VarStore for training (if available)
266    fn var_store_mut(&mut self) -> Option<&mut nn::VarStore> {
267        None // Default implementation returns None
268    }
269
270    /// Forward pass with position information for KV caching
271    fn forward_with_cache(&self, input: &Tensor, _start_pos: usize) -> Result<Tensor> {
272        // Default implementation just calls regular forward
273        // Models that support KV caching should override this
274        self.forward(input, None)
275    }
276
277    /// Forward pass with KV caching and optional per-tenant delta injection
278    ///
279    /// When `delta` is `Some`, LoRA corrections are injected at each attention layer's
280    /// Q/V projections before KV cache update. This is the unified path for both
281    /// base model inference (delta=None) and delta-aware inference (delta=Some).
282    ///
283    /// Default implementation ignores delta and falls back to `forward_with_cache()`.
284    /// Only architectures that support delta injection (Llama) need to override this.
285    fn forward_with_cache_and_delta(
286        &self,
287        input: &Tensor,
288        start_pos: usize,
289        _delta: Option<&crate::training::TenantDelta>,
290    ) -> Result<Tensor> {
291        self.forward_with_cache(input, start_pos)
292    }
293
294    /// Batched ragged forward for continuous decode (#329, epic #310).
295    ///
296    /// Each entry is `(new_token_ids, start_pos, per-sequence KVCacheManager)`;
297    /// all rows share the same query length and the single `delta` (the scheduler
298    /// groups by tenant delta). Returns stacked logits `[B, q, vocab]`. Per-row
299    /// results are equivalent to running each sequence through
300    /// `forward_with_cache_and_delta` serially (CPU-verified merge gate).
301    ///
302    /// Default: unsupported. Only architectures with batched-decode support
303    /// (Llama in v1) override this; callers fall back to the batch=1 path.
304    fn forward_batched(
305        &self,
306        _sequences: &mut [(
307            Vec<i64>,
308            usize,
309            std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
310        )],
311        _delta: Option<&crate::training::TenantDelta>,
312    ) -> Result<Tensor> {
313        Err(anyhow!(
314            "continuous batching (forward_batched) is not supported for this architecture"
315        ))
316    }
317
318    /// Get token embeddings for input IDs
319    fn embed_tokens(&self, _input_ids: &Tensor) -> Result<Tensor> {
320        Err(anyhow!(
321            "embed_tokens not implemented for this architecture"
322        ))
323    }
324
325    /// Forward pass from pre-computed embeddings (for multimodal models)
326    ///
327    /// This method allows models to start generation from embeddings instead of token IDs.
328    /// Useful for multimodal models where vision embeddings are merged with text embeddings.
329    ///
330    /// # Arguments
331    /// * `embeddings` - Pre-computed input embeddings [batch_size, seq_len, hidden_size]
332    /// * `start_pos` - Starting position in KV cache (0 for initial forward)
333    ///
334    /// # Returns
335    /// Logits tensor [batch_size, seq_len, vocab_size]
336    fn forward_from_embeddings(&self, _embeddings: &Tensor, _start_pos: usize) -> Result<Tensor> {
337        Err(anyhow!(
338            "forward_from_embeddings not implemented for this architecture"
339        ))
340    }
341
342    /// Check if this model is multimodal
343    fn is_multimodal(&self) -> bool {
344        matches!(self.architecture(), ModelArchitecture::Janus { .. })
345    }
346
347    /// Prepare inputs with vision embeddings (for multimodal models)
348    ///
349    /// This method combines text token IDs with vision embeddings for multimodal inference.
350    /// Only implemented for multimodal architectures.
351    ///
352    /// # Arguments
353    /// * `input_ids` - Text token IDs [batch_size, seq_len]
354    /// * `pixel_values` - Preprocessed images [batch_size, channels, height, width]
355    /// * `images_seq_mask` - Where to inject vision embeddings [batch_size, seq_len]
356    /// * `images_emb_mask` - Which vision embeddings to use [batch_size, num_patches]
357    ///
358    /// # Returns
359    /// Combined embeddings [batch_size, seq_len, hidden_size]
360    fn prepare_multimodal_inputs(
361        &self,
362        _input_ids: &Tensor,
363        _pixel_values: Option<&Tensor>,
364        _images_seq_mask: Option<&Tensor>,
365        _images_emb_mask: Option<&Tensor>,
366    ) -> Result<Tensor> {
367        Err(anyhow!(
368            "prepare_multimodal_inputs not implemented - model is not multimodal"
369        ))
370    }
371
372    /// Encode images through the vision encoder and return embeddings.
373    ///
374    /// For multimodal models with a vision encoder (e.g. Janus with SigLIP),
375    /// this runs the vision encoder forward pass and returns pooled embeddings.
376    ///
377    /// # Arguments
378    /// * `pixel_values` - Preprocessed images [batch_size, channels, height, width]
379    ///
380    /// # Returns
381    /// Embedding tensor [batch_size, embedding_dim]
382    fn encode_vision(&self, _pixel_values: &Tensor) -> Result<Tensor> {
383        Err(anyhow!(
384            "encode_vision not implemented - model has no vision encoder"
385        ))
386    }
387
388    /// Decode a single layer (for layer-wise processing)
389    fn decode_layer(
390        &self,
391        _layer_idx: usize,
392        _hidden_states: &Tensor,
393        _attention_mask: Option<&Tensor>,
394        _position_ids: Option<&Tensor>,
395        _past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
396    ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
397        Err(anyhow!(
398            "decode_layer not implemented for this architecture"
399        ))
400    }
401
402    /// Decode a single layer with per-tenant delta injection (for TTT training path)
403    ///
404    /// The delta's A/B matrices are injected after q_proj/v_proj projections inside
405    /// the attention layer, creating a differentiable path from loss back to delta
406    /// parameters. No KV cache is used (training path).
407    ///
408    /// Default implementation ignores the delta and falls back to `decode_layer()`.
409    /// Only architectures that support delta injection need to override this.
410    fn decode_layer_with_delta(
411        &self,
412        layer_idx: usize,
413        hidden_states: &Tensor,
414        attention_mask: Option<&Tensor>,
415        position_ids: Option<&Tensor>,
416        past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
417        _delta: &crate::training::TenantDelta,
418    ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
419        self.decode_layer(
420            layer_idx,
421            hidden_states,
422            attention_mask,
423            position_ids,
424            past_kv,
425        )
426    }
427
428    /// Run a contiguous range of decoder layers — the 2b intra-host pipeline
429    /// (layer-split) primitive (#314).
430    ///
431    /// This is the missing layer-range runner that complements the already-
432    /// exposed `embed_tokens` / `forward_from_embeddings` / `apply_final_norm` /
433    /// `lm_head` / `num_layers`. Arch-agnostic orchestration composes them:
434    /// - **stage 0** : `embed_tokens` → `forward_layers(0..b)`
435    /// - **middle**  : `forward_layers(a..b)`
436    /// - **last**    : `forward_layers(a..N)` → `apply_final_norm` → `lm_head`
437    ///
438    /// `is_first`/`is_last` are *implicit* in `range` (`range.start == 0` /
439    /// `range.end == num_layers()`); the runner itself only applies decoder
440    /// layers — never embeddings, final norm, or the LM head.
441    ///
442    /// # Stage-boundary contract
443    /// The only state carried across a stage boundary is `hidden` + `start_pos`.
444    /// `position_ids` is recomputed inside from `start_pos` + seq. Per-layer KV
445    /// cache (and any SSM `conv`/`rec` state) is **stage-local and never
446    /// transferred**. `range` is in **global** layer indices; an implementation
447    /// that owns a shard remaps to its local `self.layers` via the
448    /// `layer_offset` it was constructed with.
449    ///
450    /// # Device placement
451    /// Layer `g` runs on its mapped device; the single cross-device copy is
452    /// `hidden.to_device(next)` inserted only at a boundary where the device
453    /// actually changes (zero copies within a stage or when source == dest).
454    /// The returned tensor lives on the **last owned layer's device**.
455    ///
456    /// # Arguments
457    /// * `hidden` - `[batch, seq, hidden]`; embeddings if `range.start == 0`,
458    ///   otherwise the previous stage's output.
459    /// * `range` - global layer indices this stage owns, `[a..b)`.
460    /// * `start_pos` - KV-cache start position for this forward.
461    /// * `delta` - optional per-tenant LoRA delta (delta-aware inference).
462    ///
463    /// The default implementation errors; only architectures that support the
464    /// pipeline split (Llama; Qwen3.5) override it. The single-device whole-model
465    /// `forward*` paths are unaffected — this method is purely additive.
466    fn forward_layers(
467        &self,
468        _hidden: &Tensor,
469        _range: std::ops::Range<usize>,
470        _start_pos: usize,
471        _delta: Option<&crate::training::TenantDelta>,
472    ) -> Result<Tensor> {
473        Err(anyhow!(
474            "forward_layers (pipeline layer-split) not implemented for this architecture"
475        ))
476    }
477
478    /// Training-path sibling of [`Self::forward_layers`] — the cross-device
479    /// autograd primitive for **TTT-on-split** (#316, M-TRAIN-COUPLING).
480    ///
481    /// TTT is inference-time training that must traverse the **same** layer
482    /// partition inference uses, so the backward pass materializes grads on each
483    /// parameter's own device. This runner builds that autograd graph across the
484    /// [`crate::runtime::device_pool::LayerDeviceMap`]; the lone stage-boundary
485    /// `hidden.to_device(next)` is autograd-transparent (tch `to_device` is
486    /// differentiable), so gradients flow back through it to the previous stage's
487    /// device.
488    ///
489    /// # How it differs from the inference [`Self::forward_layers`]
490    /// The inference and training paths are kept deliberately separate (as the
491    /// whole-model paths already are). The training path:
492    /// - uses **no KV cache** — full causal attention over the entire context;
493    /// - pins **`start_pos = 0`** — `position_ids` are `0..seq`;
494    /// - uses **fresh, call-local recurrent (SSM) state** for hybrid
495    ///   architectures — the persistent inference `conv`/`rec` state is never read
496    ///   or written, so a TTT step cannot pollute the inference recurrent state
497    ///   (and the split is numerically identical to the whole-model training
498    ///   forward, since per-layer recurrent state never crosses a layer boundary).
499    ///
500    /// Everything else — the global↔local layer remap via `layer_offset`, the
501    /// single boundary copy, per-layer delta injection keyed by global index —
502    /// matches [`Self::forward_layers`]. The arch-agnostic stage orchestration is
503    /// identical (`embed_tokens` → `forward_layers_train(0..b)` → … →
504    /// `forward_layers_train(a..N)` → `apply_final_norm` → `lm_head`), and the
505    /// loss/backward is driven by the caller (e.g. the TTT trainer).
506    ///
507    /// The default implementation errors; only architectures that support the
508    /// pipeline split (Llama; Qwen3.5) override it. The single-device whole-model
509    /// training path ([`crate::runtime::TorchEngine::forward_with_delta`]) is
510    /// unaffected — this method is purely additive.
511    fn forward_layers_train(
512        &self,
513        _hidden: &Tensor,
514        _range: std::ops::Range<usize>,
515        _delta: Option<&crate::training::TenantDelta>,
516    ) -> Result<Tensor> {
517        Err(anyhow!(
518            "forward_layers_train (pipeline layer-split training) not implemented for this architecture"
519        ))
520    }
521
522    /// Apply final layer normalization
523    fn apply_final_norm(&self, _hidden_states: &Tensor) -> Result<Tensor> {
524        Err(anyhow!(
525            "apply_final_norm not implemented for this architecture"
526        ))
527    }
528
529    /// Apply language model head to get logits
530    fn lm_head(&self, _hidden_states: &Tensor) -> Result<Tensor> {
531        Err(anyhow!("lm_head not implemented for this architecture"))
532    }
533
534    /// Get the number of transformer layers
535    fn num_layers(&self) -> usize {
536        // Default implementation - architectures should override
537        32 // Common default
538    }
539
540    /// Reshape tensor for attention computation
541    fn reshape_for_attention(&self, tensor: &Tensor, is_key_value: bool) -> Result<Tensor>;
542
543    /// Apply RoPE (Rotary Position Embeddings) if supported
544    fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor>;
545
546    /// Apply architecture-specific normalization (RMSNorm, LayerNorm, etc.)
547    fn normalize(&self, tensor: &Tensor) -> Result<Tensor>;
548
549    /// Get attention mask for the architecture
550    fn get_attention_mask(&self, seq_len: usize, past_kv_len: usize) -> Result<Tensor>;
551
552    /// Downcast to Any for accessing architecture-specific methods
553    fn as_any(&self) -> &dyn std::any::Any;
554
555    // ============================================================================
556    // KV Cache Management (for session-based cache isolation)
557    // ============================================================================
558
559    /// Clear the KV cache (for new generation)
560    fn clear_kv_cache(&self) {
561        // Default: no-op for models without KV caching
562    }
563
564    /// Get KV cache memory usage in bytes
565    fn kv_cache_memory_usage(&self) -> usize {
566        0 // Default: no cache
567    }
568
569    /// Set an external KV cache (for session-based cache management)
570    ///
571    /// This allows a KVCacheRegistry to manage caches externally while
572    /// the model uses them for inference.
573    fn set_kv_cache(
574        &mut self,
575        _cache: std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
576    ) {
577        // Default: no-op for models without configurable cache
578    }
579
580    /// Get the current KV cache (for external management)
581    fn get_kv_cache(
582        &self,
583    ) -> Option<std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>> {
584        None // Default: no cache
585    }
586
587    /// Take ownership of the KV cache (removes it from the model)
588    fn take_kv_cache(
589        &mut self,
590    ) -> Option<std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>> {
591        None // Default: no cache
592    }
593}
594
595/// Attention mechanism types
596#[derive(Debug, Clone, PartialEq)]
597pub enum AttentionType {
598    /// Multi-Head Attention (standard)
599    MHA,
600    /// Multi-Query Attention (single KV head)
601    MQA,
602    /// Grouped-Query Attention (multiple query heads per KV head)
603    GQA { num_groups: usize },
604}
605
606#[cfg(test)]
607mod tests {
608    use super::*;
609
610    #[test]
611    fn test_architecture_properties() {
612        assert!(ModelArchitecture::Gemma.supports_mqa());
613        assert!(!ModelArchitecture::Gemma.supports_gqa());
614        assert!(ModelArchitecture::Llama { version: 3 }.supports_gqa());
615    }
616
617    #[test]
618    fn test_architecture_names() {
619        assert_eq!(ModelArchitecture::Llama { version: 2 }.name(), "Llama2");
620        assert_eq!(ModelArchitecture::Gemma.name(), "Gemma");
621        assert_eq!(
622            ModelArchitecture::Custom("MyModel".to_owned()).name(),
623            "MyModel"
624        );
625    }
626
627    /// #139/#440 regression guard: adding an fp32 delta correction to a BF16 base
628    /// activation must NOT panic in addmm/add and must yield a BF16 result.
629    #[test]
630    fn add_delta_coerces_fp32_correction_to_bf16_base() {
631        use tch::{Device, Kind, Tensor};
632        let base = Tensor::ones([2, 4], (Kind::BFloat16, Device::Cpu));
633        let correction = Tensor::ones([2, 4], (Kind::Float, Device::Cpu)) * 0.5;
634        // Without the coercion this panics: "expected m1 and m2 to have the same dtype".
635        let out = add_delta(&base, correction);
636        assert_eq!(out.kind(), Kind::BFloat16, "result must match base dtype");
637        // 1.0 (bf16) + 0.5 (coerced) = 1.5, representable in bf16.
638        assert!((out.double_value(&[0, 0]) - 1.5).abs() < 1e-2);
639    }
640}