hyprstream_inference/runtime/architectures/mod.rs
1//! Architecture-specific model implementations
2//!
3//! This module provides abstractions and implementations for different
4//! transformer architectures (Llama, Gemma, Qwen, etc.) with proper
5//! tensor shape handling and LoRA compatibility.
6
7use anyhow::{anyhow, Result};
8use tch::{nn, Tensor};
9
10pub mod config;
11pub mod gemma;
12pub mod janus;
13pub mod llama;
14pub mod qwen;
15pub mod qwen3_5;
16pub mod qwen3_5_vision;
17pub mod siglip;
18// LoRA adapter moved to lora module
19
20pub use config::{ArchitectureConfig, AttentionConfig};
21// pub use lora_adapter::ArchitectureAwareLoRAAdapter; // Module removed
22
23/// Add a TTT/LoRA delta correction to a base activation, coercing the correction
24/// to the base tensor's dtype first.
25///
26/// The base model may run in BF16 while the delta matrix is forced to fp32
27/// (see `runtime/ttn_profile.rs`). Adding fp32 directly to a bf16 tensor panics
28/// in `addmm_impl_cpu_` ("expected m1 and m2 to have the same dtype").
29///
30/// This helper is the single place that performs the dtype coercion so the fix
31/// cannot be missed when a new architecture adds a delta-injection site.
32/// See #139 (original llama fix) and #440 (qwen3_5 regression).
33#[inline]
34pub fn add_delta(base: &Tensor, correction: Tensor) -> Tensor {
35 base + correction.to_kind(base.kind())
36}
37
38/// Supported model architectures
39#[derive(Debug, Clone, PartialEq)]
40pub enum ModelArchitecture {
41 /// Meta's Llama family (1/2/3)
42 Llama { version: u8 },
43 /// Google's Gemma
44 Gemma,
45 /// Alibaba's Qwen (including Qwen3 with GQA and sparse attention)
46 Qwen {
47 version: u8,
48 /// Qwen3 MoE variant (30B-A3B, 235B-A22B)
49 is_moe: bool,
50 /// Context length (32K, 128K, 262K for Qwen3)
51 context_length: usize,
52 },
53 /// Microsoft's Phi
54 Phi { version: u8 },
55 /// Mistral AI models
56 Mistral,
57 /// BigCode's Starcoder
58 Starcoder,
59 /// TII's Falcon
60 Falcon,
61 /// EleutherAI's GPT-NeoX
62 GPTNeoX,
63 /// OpenAI GPT-OSS (120B/20B MoE models)
64 GPTOSS {
65 /// Total parameters in billions (120 or 20)
66 total_params_b: u16,
67 /// Active parameters in billions (5.1 or 3.6)
68 active_params_b: f32,
69 /// Number of experts in MoE
70 num_experts: usize,
71 /// Experts per token
72 experts_per_token: usize,
73 },
74 /// GPT-J (EleutherAI's 6B model)
75 GPTJ,
76 /// Janus multimodal vision-language model
77 Janus {
78 /// Base language model architecture (usually Llama)
79 base_architecture: Box<ModelArchitecture>,
80 /// Type of vision encoder
81 vision_encoder: VisionEncoderType,
82 /// Whether model supports image generation (VQ-VAE)
83 has_generation: bool,
84 },
85 /// Alibaba Qwen3.5 hybrid GatedDeltaNet/full-attention (dense and MoE variants)
86 Qwen3_5,
87 /// Custom/unknown architecture
88 Custom(String),
89}
90
91/// Types of vision encoders for multimodal models
92#[derive(Debug, Clone, PartialEq)]
93pub enum VisionEncoderType {
94 /// SigLIP vision encoder
95 SigLIP {
96 hidden_size: usize,
97 image_size: usize,
98 patch_size: usize,
99 num_layers: usize,
100 },
101 /// CLIP vision encoder
102 CLIP {
103 hidden_size: usize,
104 image_size: usize,
105 patch_size: usize,
106 num_layers: usize,
107 },
108 /// EVA vision encoder
109 EVA {
110 hidden_size: usize,
111 image_size: usize,
112 patch_size: usize,
113 num_layers: usize,
114 },
115}
116
117impl ModelArchitecture {
118 /// Get human-readable name
119 pub fn name(&self) -> String {
120 match self {
121 Self::Llama { version } => format!("Llama{version}"),
122 Self::Gemma => "Gemma".to_owned(),
123 Self::Qwen {
124 version,
125 is_moe,
126 context_length,
127 } => {
128 if *is_moe {
129 format!("Qwen{}-MoE-{}K", version, context_length / 1000)
130 } else {
131 format!("Qwen{}-{}K", version, context_length / 1000)
132 }
133 }
134 Self::Phi { version } => format!("Phi{version}"),
135 Self::Mistral => "Mistral".to_owned(),
136 Self::Starcoder => "Starcoder".to_owned(),
137 Self::Falcon => "Falcon".to_owned(),
138 Self::GPTNeoX => "GPT-NeoX".to_owned(),
139 Self::GPTOSS { total_params_b, .. } => format!("GPT-OSS-{total_params_b}B"),
140 Self::GPTJ => "GPT-J".to_owned(),
141 Self::Qwen3_5 => "Qwen3_5".to_owned(),
142 Self::Janus {
143 vision_encoder,
144 has_generation,
145 ..
146 } => {
147 let encoder_type = match vision_encoder {
148 VisionEncoderType::SigLIP { .. } => "SigLIP",
149 VisionEncoderType::CLIP { .. } => "CLIP",
150 VisionEncoderType::EVA { .. } => "EVA",
151 };
152 if *has_generation {
153 format!("Janus-{encoder_type}-Gen")
154 } else {
155 format!("Janus-{encoder_type}")
156 }
157 }
158 Self::Custom(name) => name.clone(),
159 }
160 }
161
162 /// Check if architecture supports multi-query attention
163 pub fn supports_mqa(&self) -> bool {
164 matches!(self, Self::Gemma | Self::Falcon)
165 }
166
167 /// Check if architecture supports grouped-query attention
168 pub fn supports_gqa(&self) -> bool {
169 matches!(
170 self,
171 Self::Llama { version: 3 } |
172 Self::Mistral |
173 Self::Qwen { version: 3, .. } | // Qwen3 uses GQA
174 Self::GPTOSS { .. } // GPT-OSS uses GQA in MoE
175 )
176 }
177
178 /// Check if architecture uses Mixture of Experts
179 pub fn supports_moe(&self) -> bool {
180 matches!(self, Self::Qwen { is_moe: true, .. } | Self::GPTOSS { .. })
181 }
182
183 /// Check if architecture supports sparse attention
184 pub fn supports_sparse_attention(&self) -> bool {
185 matches!(
186 self,
187 Self::Qwen { version: 3, .. } | // Qwen3 uses MInference sparse attention
188 Self::GPTOSS { .. } // GPT-OSS optimized for long context
189 )
190 }
191}
192
193/// Core trait for architecture-specific model operations
194pub trait ModelOperations: Send {
195 /// Get the architecture type
196 fn architecture(&self) -> ModelArchitecture;
197
198 /// Get architecture configuration
199 fn config(&self) -> &dyn ArchitectureConfig;
200
201 /// Get the tokenizer configuration for this model
202 ///
203 /// Returns a boxed TokenizerConfig trait object that provides
204 /// model-specific tokenizer configuration logic.
205 fn get_tokenizer_config(&self) -> Box<dyn crate::runtime::tokenizer_config::TokenizerConfig> {
206 use crate::runtime::tokenizer_config::{
207 DefaultTokenizerConfig, GemmaTokenizerConfig, LlamaTokenizerConfig, QwenTokenizerConfig,
208 };
209
210 // Return appropriate config based on architecture
211 match self.architecture() {
212 ModelArchitecture::Qwen { .. } | ModelArchitecture::Qwen3_5 => {
213 Box::new(QwenTokenizerConfig)
214 }
215 ModelArchitecture::Llama { .. } => Box::new(LlamaTokenizerConfig),
216 ModelArchitecture::Gemma => Box::new(GemmaTokenizerConfig),
217 ModelArchitecture::Janus {
218 base_architecture, ..
219 } => {
220 // Use the base architecture's tokenizer config
221 match base_architecture.as_ref() {
222 ModelArchitecture::Llama { .. } => Box::new(LlamaTokenizerConfig),
223 ModelArchitecture::Qwen { .. } => Box::new(QwenTokenizerConfig),
224 _ => Box::new(DefaultTokenizerConfig),
225 }
226 }
227 _ => Box::new(DefaultTokenizerConfig),
228 }
229 }
230
231 /// Forward pass through the model
232 fn forward(&self, input: &Tensor, past_kv: Option<&Tensor>) -> Result<Tensor>;
233
234 /// Stateless full-sequence final-normalized hidden states; no vocabulary logits.
235 /// Implementations must use fresh call-local recurrent/KV state.
236 fn decision_hidden_states(&self, _input: &Tensor) -> Result<Tensor> {
237 Err(anyhow!(
238 "decision hidden states unsupported for this architecture"
239 ))
240 }
241
242 /// Stateless decision forward with a cooperative deadline/cancellation check.
243 /// Native implementations call the checker between decoder layers.
244 fn decision_hidden_states_checked(
245 &self,
246 input: &Tensor,
247 check: &dyn Fn() -> Result<()>,
248 ) -> Result<Tensor> {
249 check()?;
250 self.decision_hidden_states(input)
251 }
252
253 /// Gather original output embedding rows, preserving the LM-head vocabulary axis.
254 fn decision_output_embedding_rows(&self, _ids: &[i64]) -> Result<Tensor> {
255 Err(anyhow!(
256 "decision output embeddings unsupported for this architecture"
257 ))
258 }
259
260 /// Get VarStore for training (if available)
261 fn var_store(&self) -> Option<&nn::VarStore> {
262 None // Default implementation returns None
263 }
264
265 /// Get mutable VarStore for training (if available)
266 fn var_store_mut(&mut self) -> Option<&mut nn::VarStore> {
267 None // Default implementation returns None
268 }
269
270 /// Forward pass with position information for KV caching
271 fn forward_with_cache(&self, input: &Tensor, _start_pos: usize) -> Result<Tensor> {
272 // Default implementation just calls regular forward
273 // Models that support KV caching should override this
274 self.forward(input, None)
275 }
276
277 /// Forward pass with KV caching and optional per-tenant delta injection
278 ///
279 /// When `delta` is `Some`, LoRA corrections are injected at each attention layer's
280 /// Q/V projections before KV cache update. This is the unified path for both
281 /// base model inference (delta=None) and delta-aware inference (delta=Some).
282 ///
283 /// Default implementation ignores delta and falls back to `forward_with_cache()`.
284 /// Only architectures that support delta injection (Llama) need to override this.
285 fn forward_with_cache_and_delta(
286 &self,
287 input: &Tensor,
288 start_pos: usize,
289 _delta: Option<&crate::training::TenantDelta>,
290 ) -> Result<Tensor> {
291 self.forward_with_cache(input, start_pos)
292 }
293
294 /// Batched ragged forward for continuous decode (#329, epic #310).
295 ///
296 /// Each entry is `(new_token_ids, start_pos, per-sequence KVCacheManager)`;
297 /// all rows share the same query length and the single `delta` (the scheduler
298 /// groups by tenant delta). Returns stacked logits `[B, q, vocab]`. Per-row
299 /// results are equivalent to running each sequence through
300 /// `forward_with_cache_and_delta` serially (CPU-verified merge gate).
301 ///
302 /// Default: unsupported. Only architectures with batched-decode support
303 /// (Llama in v1) override this; callers fall back to the batch=1 path.
304 fn forward_batched(
305 &self,
306 _sequences: &mut [(
307 Vec<i64>,
308 usize,
309 std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
310 )],
311 _delta: Option<&crate::training::TenantDelta>,
312 ) -> Result<Tensor> {
313 Err(anyhow!(
314 "continuous batching (forward_batched) is not supported for this architecture"
315 ))
316 }
317
318 /// Get token embeddings for input IDs
319 fn embed_tokens(&self, _input_ids: &Tensor) -> Result<Tensor> {
320 Err(anyhow!(
321 "embed_tokens not implemented for this architecture"
322 ))
323 }
324
325 /// Forward pass from pre-computed embeddings (for multimodal models)
326 ///
327 /// This method allows models to start generation from embeddings instead of token IDs.
328 /// Useful for multimodal models where vision embeddings are merged with text embeddings.
329 ///
330 /// # Arguments
331 /// * `embeddings` - Pre-computed input embeddings [batch_size, seq_len, hidden_size]
332 /// * `start_pos` - Starting position in KV cache (0 for initial forward)
333 ///
334 /// # Returns
335 /// Logits tensor [batch_size, seq_len, vocab_size]
336 fn forward_from_embeddings(&self, _embeddings: &Tensor, _start_pos: usize) -> Result<Tensor> {
337 Err(anyhow!(
338 "forward_from_embeddings not implemented for this architecture"
339 ))
340 }
341
342 /// Check if this model is multimodal
343 fn is_multimodal(&self) -> bool {
344 matches!(self.architecture(), ModelArchitecture::Janus { .. })
345 }
346
347 /// Prepare inputs with vision embeddings (for multimodal models)
348 ///
349 /// This method combines text token IDs with vision embeddings for multimodal inference.
350 /// Only implemented for multimodal architectures.
351 ///
352 /// # Arguments
353 /// * `input_ids` - Text token IDs [batch_size, seq_len]
354 /// * `pixel_values` - Preprocessed images [batch_size, channels, height, width]
355 /// * `images_seq_mask` - Where to inject vision embeddings [batch_size, seq_len]
356 /// * `images_emb_mask` - Which vision embeddings to use [batch_size, num_patches]
357 ///
358 /// # Returns
359 /// Combined embeddings [batch_size, seq_len, hidden_size]
360 fn prepare_multimodal_inputs(
361 &self,
362 _input_ids: &Tensor,
363 _pixel_values: Option<&Tensor>,
364 _images_seq_mask: Option<&Tensor>,
365 _images_emb_mask: Option<&Tensor>,
366 ) -> Result<Tensor> {
367 Err(anyhow!(
368 "prepare_multimodal_inputs not implemented - model is not multimodal"
369 ))
370 }
371
372 /// Encode images through the vision encoder and return embeddings.
373 ///
374 /// For multimodal models with a vision encoder (e.g. Janus with SigLIP),
375 /// this runs the vision encoder forward pass and returns pooled embeddings.
376 ///
377 /// # Arguments
378 /// * `pixel_values` - Preprocessed images [batch_size, channels, height, width]
379 ///
380 /// # Returns
381 /// Embedding tensor [batch_size, embedding_dim]
382 fn encode_vision(&self, _pixel_values: &Tensor) -> Result<Tensor> {
383 Err(anyhow!(
384 "encode_vision not implemented - model has no vision encoder"
385 ))
386 }
387
388 /// Decode a single layer (for layer-wise processing)
389 fn decode_layer(
390 &self,
391 _layer_idx: usize,
392 _hidden_states: &Tensor,
393 _attention_mask: Option<&Tensor>,
394 _position_ids: Option<&Tensor>,
395 _past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
396 ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
397 Err(anyhow!(
398 "decode_layer not implemented for this architecture"
399 ))
400 }
401
402 /// Decode a single layer with per-tenant delta injection (for TTT training path)
403 ///
404 /// The delta's A/B matrices are injected after q_proj/v_proj projections inside
405 /// the attention layer, creating a differentiable path from loss back to delta
406 /// parameters. No KV cache is used (training path).
407 ///
408 /// Default implementation ignores the delta and falls back to `decode_layer()`.
409 /// Only architectures that support delta injection need to override this.
410 fn decode_layer_with_delta(
411 &self,
412 layer_idx: usize,
413 hidden_states: &Tensor,
414 attention_mask: Option<&Tensor>,
415 position_ids: Option<&Tensor>,
416 past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
417 _delta: &crate::training::TenantDelta,
418 ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
419 self.decode_layer(
420 layer_idx,
421 hidden_states,
422 attention_mask,
423 position_ids,
424 past_kv,
425 )
426 }
427
428 /// Run a contiguous range of decoder layers — the 2b intra-host pipeline
429 /// (layer-split) primitive (#314).
430 ///
431 /// This is the missing layer-range runner that complements the already-
432 /// exposed `embed_tokens` / `forward_from_embeddings` / `apply_final_norm` /
433 /// `lm_head` / `num_layers`. Arch-agnostic orchestration composes them:
434 /// - **stage 0** : `embed_tokens` → `forward_layers(0..b)`
435 /// - **middle** : `forward_layers(a..b)`
436 /// - **last** : `forward_layers(a..N)` → `apply_final_norm` → `lm_head`
437 ///
438 /// `is_first`/`is_last` are *implicit* in `range` (`range.start == 0` /
439 /// `range.end == num_layers()`); the runner itself only applies decoder
440 /// layers — never embeddings, final norm, or the LM head.
441 ///
442 /// # Stage-boundary contract
443 /// The only state carried across a stage boundary is `hidden` + `start_pos`.
444 /// `position_ids` is recomputed inside from `start_pos` + seq. Per-layer KV
445 /// cache (and any SSM `conv`/`rec` state) is **stage-local and never
446 /// transferred**. `range` is in **global** layer indices; an implementation
447 /// that owns a shard remaps to its local `self.layers` via the
448 /// `layer_offset` it was constructed with.
449 ///
450 /// # Device placement
451 /// Layer `g` runs on its mapped device; the single cross-device copy is
452 /// `hidden.to_device(next)` inserted only at a boundary where the device
453 /// actually changes (zero copies within a stage or when source == dest).
454 /// The returned tensor lives on the **last owned layer's device**.
455 ///
456 /// # Arguments
457 /// * `hidden` - `[batch, seq, hidden]`; embeddings if `range.start == 0`,
458 /// otherwise the previous stage's output.
459 /// * `range` - global layer indices this stage owns, `[a..b)`.
460 /// * `start_pos` - KV-cache start position for this forward.
461 /// * `delta` - optional per-tenant LoRA delta (delta-aware inference).
462 ///
463 /// The default implementation errors; only architectures that support the
464 /// pipeline split (Llama; Qwen3.5) override it. The single-device whole-model
465 /// `forward*` paths are unaffected — this method is purely additive.
466 fn forward_layers(
467 &self,
468 _hidden: &Tensor,
469 _range: std::ops::Range<usize>,
470 _start_pos: usize,
471 _delta: Option<&crate::training::TenantDelta>,
472 ) -> Result<Tensor> {
473 Err(anyhow!(
474 "forward_layers (pipeline layer-split) not implemented for this architecture"
475 ))
476 }
477
478 /// Training-path sibling of [`Self::forward_layers`] — the cross-device
479 /// autograd primitive for **TTT-on-split** (#316, M-TRAIN-COUPLING).
480 ///
481 /// TTT is inference-time training that must traverse the **same** layer
482 /// partition inference uses, so the backward pass materializes grads on each
483 /// parameter's own device. This runner builds that autograd graph across the
484 /// [`crate::runtime::device_pool::LayerDeviceMap`]; the lone stage-boundary
485 /// `hidden.to_device(next)` is autograd-transparent (tch `to_device` is
486 /// differentiable), so gradients flow back through it to the previous stage's
487 /// device.
488 ///
489 /// # How it differs from the inference [`Self::forward_layers`]
490 /// The inference and training paths are kept deliberately separate (as the
491 /// whole-model paths already are). The training path:
492 /// - uses **no KV cache** — full causal attention over the entire context;
493 /// - pins **`start_pos = 0`** — `position_ids` are `0..seq`;
494 /// - uses **fresh, call-local recurrent (SSM) state** for hybrid
495 /// architectures — the persistent inference `conv`/`rec` state is never read
496 /// or written, so a TTT step cannot pollute the inference recurrent state
497 /// (and the split is numerically identical to the whole-model training
498 /// forward, since per-layer recurrent state never crosses a layer boundary).
499 ///
500 /// Everything else — the global↔local layer remap via `layer_offset`, the
501 /// single boundary copy, per-layer delta injection keyed by global index —
502 /// matches [`Self::forward_layers`]. The arch-agnostic stage orchestration is
503 /// identical (`embed_tokens` → `forward_layers_train(0..b)` → … →
504 /// `forward_layers_train(a..N)` → `apply_final_norm` → `lm_head`), and the
505 /// loss/backward is driven by the caller (e.g. the TTT trainer).
506 ///
507 /// The default implementation errors; only architectures that support the
508 /// pipeline split (Llama; Qwen3.5) override it. The single-device whole-model
509 /// training path ([`crate::runtime::TorchEngine::forward_with_delta`]) is
510 /// unaffected — this method is purely additive.
511 fn forward_layers_train(
512 &self,
513 _hidden: &Tensor,
514 _range: std::ops::Range<usize>,
515 _delta: Option<&crate::training::TenantDelta>,
516 ) -> Result<Tensor> {
517 Err(anyhow!(
518 "forward_layers_train (pipeline layer-split training) not implemented for this architecture"
519 ))
520 }
521
522 /// Apply final layer normalization
523 fn apply_final_norm(&self, _hidden_states: &Tensor) -> Result<Tensor> {
524 Err(anyhow!(
525 "apply_final_norm not implemented for this architecture"
526 ))
527 }
528
529 /// Apply language model head to get logits
530 fn lm_head(&self, _hidden_states: &Tensor) -> Result<Tensor> {
531 Err(anyhow!("lm_head not implemented for this architecture"))
532 }
533
534 /// Get the number of transformer layers
535 fn num_layers(&self) -> usize {
536 // Default implementation - architectures should override
537 32 // Common default
538 }
539
540 /// Reshape tensor for attention computation
541 fn reshape_for_attention(&self, tensor: &Tensor, is_key_value: bool) -> Result<Tensor>;
542
543 /// Apply RoPE (Rotary Position Embeddings) if supported
544 fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor>;
545
546 /// Apply architecture-specific normalization (RMSNorm, LayerNorm, etc.)
547 fn normalize(&self, tensor: &Tensor) -> Result<Tensor>;
548
549 /// Get attention mask for the architecture
550 fn get_attention_mask(&self, seq_len: usize, past_kv_len: usize) -> Result<Tensor>;
551
552 /// Downcast to Any for accessing architecture-specific methods
553 fn as_any(&self) -> &dyn std::any::Any;
554
555 // ============================================================================
556 // KV Cache Management (for session-based cache isolation)
557 // ============================================================================
558
559 /// Clear the KV cache (for new generation)
560 fn clear_kv_cache(&self) {
561 // Default: no-op for models without KV caching
562 }
563
564 /// Get KV cache memory usage in bytes
565 fn kv_cache_memory_usage(&self) -> usize {
566 0 // Default: no cache
567 }
568
569 /// Set an external KV cache (for session-based cache management)
570 ///
571 /// This allows a KVCacheRegistry to manage caches externally while
572 /// the model uses them for inference.
573 fn set_kv_cache(
574 &mut self,
575 _cache: std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
576 ) {
577 // Default: no-op for models without configurable cache
578 }
579
580 /// Get the current KV cache (for external management)
581 fn get_kv_cache(
582 &self,
583 ) -> Option<std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>> {
584 None // Default: no cache
585 }
586
587 /// Take ownership of the KV cache (removes it from the model)
588 fn take_kv_cache(
589 &mut self,
590 ) -> Option<std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>> {
591 None // Default: no cache
592 }
593}
594
595/// Attention mechanism types
596#[derive(Debug, Clone, PartialEq)]
597pub enum AttentionType {
598 /// Multi-Head Attention (standard)
599 MHA,
600 /// Multi-Query Attention (single KV head)
601 MQA,
602 /// Grouped-Query Attention (multiple query heads per KV head)
603 GQA { num_groups: usize },
604}
605
606#[cfg(test)]
607mod tests {
608 use super::*;
609
610 #[test]
611 fn test_architecture_properties() {
612 assert!(ModelArchitecture::Gemma.supports_mqa());
613 assert!(!ModelArchitecture::Gemma.supports_gqa());
614 assert!(ModelArchitecture::Llama { version: 3 }.supports_gqa());
615 }
616
617 #[test]
618 fn test_architecture_names() {
619 assert_eq!(ModelArchitecture::Llama { version: 2 }.name(), "Llama2");
620 assert_eq!(ModelArchitecture::Gemma.name(), "Gemma");
621 assert_eq!(
622 ModelArchitecture::Custom("MyModel".to_owned()).name(),
623 "MyModel"
624 );
625 }
626
627 /// #139/#440 regression guard: adding an fp32 delta correction to a BF16 base
628 /// activation must NOT panic in addmm/add and must yield a BF16 result.
629 #[test]
630 fn add_delta_coerces_fp32_correction_to_bf16_base() {
631 use tch::{Device, Kind, Tensor};
632 let base = Tensor::ones([2, 4], (Kind::BFloat16, Device::Cpu));
633 let correction = Tensor::ones([2, 4], (Kind::Float, Device::Cpu)) * 0.5;
634 // Without the coercion this panics: "expected m1 and m2 to have the same dtype".
635 let out = add_delta(&base, correction);
636 assert_eq!(out.kind(), Kind::BFloat16, "result must match base dtype");
637 // 1.0 (bf16) + 0.5 (coerced) = 1.5, representable in bf16.
638 assert!((out.double_value(&[0, 0]) - 1.5).abs() < 1e-2);
639 }
640}