Skip to main content

hyprstream_inference/runtime/architectures/
gemma.rs

1//! Gemma model implementation with Multi-Query Attention support
2
3use super::{ArchitectureConfig, ModelArchitecture, ModelOperations};
4// use super::lora_adapter::ArchitectureAwareLoRAAdapter; // Module removed
5use crate::runtime::tensor_helpers::{
6    broadcast_add, broadcast_mul, broadcast_sub, dims3, dims4, scalar_tensor, square_tensor,
7};
8use anyhow::{anyhow, Result};
9use tch::{Device, Kind as DType, Tensor};
10use tracing::info;
11// Using tch tensor operations for Gemma architecture
12use std::collections::HashMap;
13use std::path::Path;
14
15/// Gemma model configuration
16#[derive(Debug, Clone)]
17pub struct GemmaConfig {
18    /// Number of attention heads for queries
19    pub num_attention_heads: usize,
20    /// Number of key-value heads (MQA)
21    pub num_key_value_heads: usize,
22    /// Hidden dimension size
23    pub hidden_size: usize,
24    /// Head dimension
25    pub head_dim: usize,
26    /// Intermediate size for FFN
27    pub intermediate_size: usize,
28    /// Maximum position embeddings
29    pub max_position_embeddings: usize,
30    /// RMSNorm epsilon
31    pub rms_norm_eps: f32,
32    /// Vocabulary size
33    pub vocab_size: usize,
34    /// Number of hidden layers
35    pub num_hidden_layers: usize,
36    /// RoPE theta for global attention
37    pub rope_theta: f32,
38    /// RoPE theta for local attention (Gemma3)
39    pub rope_local_base_freq: f32,
40    /// Sliding window size (Gemma3)
41    pub sliding_window: Option<usize>,
42    /// Layer types (sliding_attention or full_attention)
43    pub layer_types: Vec<String>,
44    /// Query pre-attention scalar (QK-norm for Gemma3)
45    pub query_pre_attn_scalar: Option<f32>,
46    /// Hidden activation function
47    pub hidden_activation: String,
48}
49
50impl Default for GemmaConfig {
51    fn default() -> Self {
52        // Gemma-7B configuration
53        Self {
54            num_attention_heads: 16,
55            num_key_value_heads: 4, // MQA with 4 KV heads
56            hidden_size: 3072,
57            head_dim: 256,
58            intermediate_size: 24576,
59            max_position_embeddings: 8192,
60            rms_norm_eps: 1e-6,
61            vocab_size: 256000,
62            num_hidden_layers: 28,
63            rope_theta: 10000.0,
64            rope_local_base_freq: 10000.0,
65            sliding_window: None,
66            layer_types: vec![],
67            query_pre_attn_scalar: None,
68            hidden_activation: "silu".to_owned(),
69        }
70    }
71}
72
73impl ArchitectureConfig for GemmaConfig {
74    fn num_attention_heads(&self) -> usize {
75        self.num_attention_heads
76    }
77
78    fn num_key_value_heads(&self) -> usize {
79        self.num_key_value_heads
80    }
81
82    fn hidden_size(&self) -> usize {
83        self.hidden_size
84    }
85
86    fn intermediate_size(&self) -> usize {
87        self.intermediate_size
88    }
89
90    fn head_dim(&self) -> usize {
91        self.head_dim
92    }
93
94    fn vocab_size(&self) -> usize {
95        self.vocab_size
96    }
97
98    fn max_position_embeddings(&self) -> usize {
99        self.max_position_embeddings
100    }
101
102    fn rope_theta(&self) -> Option<f32> {
103        Some(self.rope_theta)
104    }
105
106    fn rope_dim(&self) -> Option<usize> {
107        Some(self.head_dim)
108    }
109
110    fn layer_norm_eps(&self) -> f32 {
111        self.rms_norm_eps
112    }
113
114    fn use_rms_norm(&self) -> bool {
115        true // Gemma uses RMSNorm
116    }
117}
118
119/// Gemma model implementation
120pub struct GemmaModel {
121    config: GemmaConfig,
122    #[allow(dead_code)]
123    device: Device,
124    #[allow(dead_code)]
125    dtype: DType,
126
127    // Model weights
128    embed_tokens: Option<Tensor>,
129    layers: Vec<GemmaLayer>,
130    norm: Option<RMSNorm>,
131    lm_head: Option<Tensor>,
132}
133
134// SAFETY: Tch tensors are thread-safe when used correctly
135// We ensure no mutable access without proper synchronization
136unsafe impl Send for GemmaModel {}
137unsafe impl Sync for GemmaModel {}
138
139/// Single Gemma transformer layer
140struct GemmaLayer {
141    self_attn: GemmaAttention,
142    mlp: GemmaMLP,
143    input_layernorm: RMSNorm,
144    post_attention_layernorm: RMSNorm,
145}
146
147unsafe impl Send for GemmaLayer {}
148unsafe impl Sync for GemmaLayer {}
149
150/// Gemma attention with Multi-Query Attention
151struct GemmaAttention {
152    q_proj: Tensor,
153    k_proj: Tensor,
154    v_proj: Tensor,
155    o_proj: Tensor,
156    num_heads: usize,
157    num_kv_heads: usize,
158    head_dim: usize,
159    rope_theta: f32,
160    // QK-norm weights (Gemma3)
161    q_norm: Option<Tensor>,
162    k_norm: Option<Tensor>,
163    query_pre_attn_scalar: Option<f32>,
164    // Sliding window (Gemma3)
165    sliding_window: Option<usize>,
166    layer_type: String,
167}
168
169unsafe impl Send for GemmaAttention {}
170unsafe impl Sync for GemmaAttention {}
171
172impl GemmaAttention {
173    /// Apply Multi-Query Attention with optional QK-norm and sliding window
174    fn forward(&self, hidden_states: &Tensor, position_ids: Option<&Tensor>) -> Result<Tensor> {
175        let (batch_size, seq_len, hidden_size) = dims3(hidden_states)?;
176
177        // Reshape for 2D matmul
178        let hidden_states_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
179
180        // Project to Q, K, V - weights are [out, in] so transpose for matmul
181        let q = hidden_states_2d.matmul(&self.q_proj.transpose(0, 1));
182        let k = hidden_states_2d.matmul(&self.k_proj.transpose(0, 1));
183        let v = hidden_states_2d.matmul(&self.v_proj.transpose(0, 1));
184
185        // Reshape for attention
186        // Q: [batch, seq, num_heads, head_dim]
187        let mut q = q.reshape([
188            batch_size,
189            seq_len,
190            self.num_heads as i64,
191            self.head_dim as i64,
192        ]);
193        // K, V: [batch, seq, num_kv_heads, head_dim]
194        let mut k = k.reshape([
195            batch_size,
196            seq_len,
197            self.num_kv_heads as i64,
198            self.head_dim as i64,
199        ]);
200        let v = v.reshape([
201            batch_size,
202            seq_len,
203            self.num_kv_heads as i64,
204            self.head_dim as i64,
205        ]);
206
207        // Apply QK-norm if configured (Gemma3)
208        if let Some(q_norm) = &self.q_norm {
209            q = self.apply_qk_norm(&q, q_norm, self.num_heads)?;
210        }
211        if let Some(k_norm) = &self.k_norm {
212            k = self.apply_qk_norm(&k, k_norm, self.num_kv_heads)?;
213        }
214
215        // Apply RoPE if position_ids provided
216        let (q, k) = if let Some(pos_ids) = position_ids {
217            (self.apply_rope(&q, pos_ids)?, self.apply_rope(&k, pos_ids)?)
218        } else {
219            (q, k)
220        };
221
222        // Apply query pre-attention scalar if configured (Gemma3)
223        // Scale by scalar^(-0.5) which is 1/sqrt(scalar)
224        let q = if let Some(scalar) = self.query_pre_attn_scalar {
225            let scale = scalar.powf(-0.5); // scalar^(-0.5) = 1/sqrt(scalar)
226            let scale_tensor = scalar_tensor(scale, q.device(), q.kind());
227            broadcast_mul(&q, &scale_tensor)?
228        } else {
229            q
230        };
231
232        // Expand K, V for MQA (repeat KV heads to match Q heads)
233        let k = self.expand_kv_for_mqa(&k)?;
234        let v = self.expand_kv_for_mqa(&v)?;
235
236        // Compute attention scores with optional sliding window
237        let scores = self.compute_attention_scores(&q, &k)?;
238
239        // Transpose V for matmul: [batch, seq, heads, dim] -> [batch, heads, seq, dim]
240        let v = v.transpose(1, 2);
241
242        // Apply attention to values: [batch, heads, seq, seq] x [batch, heads, seq, dim]
243        let attn_output = scores.matmul(&v);
244
245        // Transpose back: [batch, heads, seq, dim] -> [batch, seq, heads, dim]
246        let attn_output = attn_output.transpose(1, 2);
247
248        // Reshape and project output - transpose o_proj for matmul
249        let attn_output = attn_output
250            .reshape([batch_size, seq_len, (self.num_heads * self.head_dim) as i64])
251            .reshape([
252                batch_size * seq_len,
253                (self.num_heads * self.head_dim) as i64,
254            ])
255            .matmul(&self.o_proj.transpose(0, 1))
256            .reshape([batch_size, seq_len, hidden_size]);
257
258        Ok(attn_output)
259    }
260
261    /// Expand KV heads to match Q heads for MQA
262    fn expand_kv_for_mqa(&self, kv: &Tensor) -> Result<Tensor> {
263        let (batch_size, seq_len, num_kv_heads, head_dim) = dims4(kv)?;
264        let repeat_factor = (self.num_heads as i64) / num_kv_heads;
265
266        if repeat_factor == 1 {
267            return Ok(kv.shallow_clone());
268        }
269
270        // Repeat KV heads: [batch, seq, num_kv_heads, head_dim] -> [batch, seq, num_heads, head_dim]
271        Ok(kv
272            .unsqueeze(3) // [batch, seq, num_kv_heads, 1, head_dim]
273            .expand(
274                [batch_size, seq_len, num_kv_heads, repeat_factor, head_dim],
275                false,
276            )
277            .reshape([batch_size, seq_len, self.num_heads as i64, head_dim]))
278    }
279
280    /// Apply Rotary Position Embeddings
281    fn apply_rope(&self, tensor: &Tensor, _position_ids: &Tensor) -> Result<Tensor> {
282        // tensor shape: [batch, seq, heads, dim]
283        let (_batch_size, seq_len, _num_heads, head_dim) = dims4(tensor)?;
284
285        // Generate position embeddings
286        let theta = self.rope_theta;
287        let device = tensor.device();
288        let dtype = tensor.kind();
289
290        // Create frequency bands
291        let inv_freq = (0..head_dim / 2)
292            .map(|i| 1.0 / theta.powf(2.0 * i as f32 / head_dim as f32))
293            .collect::<Vec<_>>();
294
295        // Create position indices
296        let _positions =
297            Tensor::arange(seq_len, (DType::Int64, device)).to_dtype(DType::Float, false, false);
298
299        // Compute sin and cos for each position and frequency
300        let mut sin_vals = Vec::new();
301        let mut cos_vals = Vec::new();
302
303        for pos in 0..seq_len {
304            for freq_idx in 0..head_dim / 2 {
305                let angle = pos as f32 * inv_freq[freq_idx as usize];
306                sin_vals.push(angle.sin());
307                cos_vals.push(angle.cos());
308            }
309        }
310
311        // Create sin and cos tensors
312        let sin = Tensor::from_slice(&sin_vals)
313            .reshape([seq_len, head_dim / 2])
314            .to(device)
315            .to_dtype(dtype, false, false);
316        let cos = Tensor::from_slice(&cos_vals)
317            .reshape([seq_len, head_dim / 2])
318            .to(device)
319            .to_dtype(dtype, false, false);
320
321        // Split tensor into two halves for rotation
322        let x1 = tensor.narrow(-1, 0, head_dim / 2);
323        let x2 = tensor.narrow(-1, head_dim / 2, head_dim / 2);
324
325        // Apply rotation
326        let rotated_x1 = broadcast_sub(&broadcast_mul(&x1, &cos)?, &broadcast_mul(&x2, &sin)?)?;
327        let rotated_x2 = broadcast_add(&broadcast_mul(&x1, &sin)?, &broadcast_mul(&x2, &cos)?)?;
328
329        // Concatenate back together
330        Ok(Tensor::cat(&[rotated_x1, rotated_x2], -1))
331    }
332
333    /// Apply QK-norm (Gemma3)
334    fn apply_qk_norm(
335        &self,
336        tensor: &Tensor,
337        norm_weight: &Tensor,
338        num_heads: usize,
339    ) -> Result<Tensor> {
340        // tensor shape: [batch, seq, heads, dim]
341        // norm_weight shape: [heads * dim] or [dim] for single head
342        let (batch_size, seq_len, _tensor_heads, head_dim) = dims4(tensor)?;
343
344        // Reshape norm_weight to match tensor dimensions
345        let norm_weight = if norm_weight.size()[0] == head_dim {
346            // Single head normalization (K in Gemma3)
347            norm_weight
348                .unsqueeze(0) // [1, dim]
349                .expand([num_heads as i64, head_dim], false) // [heads, dim]
350                .reshape([(num_heads * head_dim as usize) as i64])
351        } else {
352            norm_weight.shallow_clone()
353        };
354
355        // Reshape tensor for normalization
356        let tensor_flat =
357            tensor.reshape([batch_size * seq_len, (num_heads * head_dim as usize) as i64]);
358
359        // Apply normalization
360        let normalized = broadcast_mul(&tensor_flat, &norm_weight)?;
361
362        // Reshape back
363        Ok(normalized.reshape([batch_size, seq_len, num_heads as i64, head_dim]))
364    }
365
366    /// Compute scaled dot-product attention scores with optional sliding window
367    fn compute_attention_scores(&self, q: &Tensor, k: &Tensor) -> Result<Tensor> {
368        // If query_pre_attn_scalar is set, queries are already scaled, so use 1.0
369        // Otherwise, use standard scaling of 1/sqrt(head_dim)
370        let scale = if self.query_pre_attn_scalar.is_some() {
371            1.0
372        } else {
373            1.0 / (self.head_dim as f32).sqrt()
374        };
375
376        // Q: [batch, seq, num_heads, head_dim] -> [batch, num_heads, seq, head_dim]
377        let q = q.transpose(1, 2);
378        // K: [batch, seq, num_heads, head_dim] -> [batch, num_heads, seq, head_dim] -> [batch, num_heads, head_dim, seq]
379        let k = k.transpose(1, 2).transpose(2, 3);
380
381        // Compute attention scores: [batch, num_heads, seq, seq]
382        let mut scores = q.matmul(&k) * (scale as f64);
383
384        // Apply sliding window mask if configured (Gemma3)
385        if let Some(window_size) = self.sliding_window {
386            if self.layer_type == "local" {
387                scores = self.apply_sliding_window_mask(&scores, window_size)?;
388            }
389            // Global layers use full attention (no mask)
390        }
391
392        // Apply softmax (maintain dtype consistency)
393        Ok(scores.softmax(-1, scores.kind()))
394    }
395
396    /// Apply sliding window mask for local attention layers
397    fn apply_sliding_window_mask(&self, scores: &Tensor, window_size: usize) -> Result<Tensor> {
398        let (batch_size, num_heads, seq_len, _) = dims4(scores)?;
399        let device = scores.device();
400        let dtype = scores.kind();
401
402        // Create sliding window mask where each position can only attend to window_size previous positions
403        let mut mask_values = vec![0f32; (seq_len * seq_len) as usize];
404        for i in 0..seq_len {
405            for j in 0..seq_len {
406                if j <= i && i - j < window_size as i64 {
407                    mask_values[(i * seq_len + j) as usize] = 0.0; // Can attend
408                } else {
409                    mask_values[(i * seq_len + j) as usize] = -10000.0; // Mask out
410                }
411            }
412        }
413
414        // Create mask tensor and broadcast to match scores shape
415        let mask = Tensor::from_slice(&mask_values)
416            .reshape([seq_len, seq_len])
417            .to(device)
418            .to_dtype(dtype, false, false)
419            .unsqueeze(0) // [1, seq, seq]
420            .unsqueeze(0) // [1, 1, seq, seq]
421            .expand([batch_size, num_heads, seq_len, seq_len], false);
422
423        // Add mask to scores
424        broadcast_add(scores, &mask)
425    }
426}
427
428/// Gemma MLP/FFN layer
429struct GemmaMLP {
430    gate_proj: Tensor,
431    up_proj: Tensor,
432    down_proj: Tensor,
433    activation: String, // "silu" or "gelu_pytorch_tanh"
434}
435
436unsafe impl Send for GemmaMLP {}
437unsafe impl Sync for GemmaMLP {}
438
439impl GemmaMLP {
440    fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {
441        // Handle 3D input: [batch_size, seq_len, hidden_size]
442        let input_dims = hidden_states.size();
443        let needs_reshape = input_dims.len() == 3;
444
445        let (batch_size, seq_len, hidden_size) = if needs_reshape {
446            (input_dims[0], input_dims[1], input_dims[2])
447        } else {
448            (1, 1, input_dims[input_dims.len() - 1])
449        };
450
451        // Reshape to 2D for matmul if needed
452        let hidden_2d = if needs_reshape {
453            hidden_states.reshape([batch_size * seq_len, hidden_size])
454        } else {
455            hidden_states.shallow_clone()
456        };
457
458        // Weights are [out, in] so transpose for matmul
459        let gate_pre_activation = hidden_2d.matmul(&self.gate_proj.transpose(0, 1));
460
461        // Apply activation based on config
462        let gate = match self.activation.as_str() {
463            "gelu_pytorch_tanh" => self.gelu_pytorch_tanh(&gate_pre_activation)?,
464            _ => gate_pre_activation.silu(), // silu is default
465        };
466
467        let up = hidden_2d.matmul(&self.up_proj.transpose(0, 1));
468        let output = (gate * &up).matmul(&self.down_proj.transpose(0, 1));
469
470        // Reshape back to 3D if input was 3D
471        if needs_reshape {
472            let output_hidden_size = output.size()[1];
473            Ok(output.reshape([batch_size, seq_len, output_hidden_size]))
474        } else {
475            Ok(output)
476        }
477    }
478
479    /// GELU activation with PyTorch tanh approximation
480    /// GELU(x) = 0.5 * x * (1 + tanh(sqrt(2/π) * (x + 0.044715 * x^3)))
481    fn gelu_pytorch_tanh(&self, x: &Tensor) -> Result<Tensor> {
482        use std::f32::consts::PI;
483
484        // Constants for the approximation
485        let sqrt_2_over_pi = (2.0_f32 / PI).sqrt();
486        let coeff = 0.044715_f32;
487
488        // x^3
489        let x_cubed = x.pow_tensor_scalar(3.0);
490
491        // x + 0.044715 * x^3
492        let coeff_tensor = scalar_tensor(coeff, x.device(), x.kind());
493        let inner = x + broadcast_mul(&x_cubed, &coeff_tensor)?;
494
495        // sqrt(2/π) * (x + 0.044715 * x^3)
496        let sqrt_tensor = scalar_tensor(sqrt_2_over_pi, x.device(), x.kind());
497        let scaled = broadcast_mul(&inner, &sqrt_tensor)?;
498
499        // tanh(sqrt(2/π) * (x + 0.044715 * x^3))
500        let tanh_result = scaled.tanh();
501
502        // 1 + tanh(...)
503        let one_tensor = scalar_tensor(1.0_f32, x.device(), x.kind());
504        let one_plus_tanh = broadcast_add(&tanh_result, &one_tensor)?;
505
506        // 0.5 * x * (1 + tanh(...))
507        let half_tensor = scalar_tensor(0.5_f32, x.device(), x.kind());
508        broadcast_mul(&(x * &one_plus_tanh), &half_tensor)
509    }
510}
511
512/// RMSNorm implementation for Gemma
513struct RMSNorm {
514    weight: Tensor,
515    eps: f32,
516    add_unit_offset: bool, // For Gemma2/3
517}
518
519unsafe impl Send for RMSNorm {}
520unsafe impl Sync for RMSNorm {}
521
522impl RMSNorm {
523    fn forward(&self, x: &Tensor) -> Result<Tensor> {
524        // Get hidden size from last dimension
525        let hidden_size = x.size()[x.size().len() - 1];
526
527        // Compute RMS normalization
528        // norm_x = sum(x^2) / hidden_size
529        let x2 = square_tensor(x)?;
530        let norm_x = x2.sum_dim_intlist(&[-1i64][..], true, None) * (1.0 / hidden_size as f64);
531
532        // x_normed = x / sqrt(norm_x + eps)
533        let x_normed = x / &(norm_x + self.eps as f64).sqrt();
534
535        // Apply weight with optional unit offset for Gemma2/3
536        if self.add_unit_offset {
537            // Gemma2/3: (weight + 1) * normalized
538            let weight_plus_one = &self.weight + 1.0;
539            Ok(broadcast_mul(&x_normed, &weight_plus_one)?)
540        } else {
541            // Standard: weight * normalized
542            Ok(broadcast_mul(&x_normed, &self.weight)?)
543        }
544    }
545}
546
547impl GemmaModel {
548    /// Create Gemma model from SafeTensors
549    pub fn from_safetensors(path: &Path, device: &Device, dtype: DType) -> Result<Self> {
550        // SafeTensors loading now handled by the safetensors crate
551
552        // Load configuration
553        let config_path = path
554            .parent()
555            .ok_or_else(|| anyhow!("Invalid model path"))?
556            .join("config.json");
557
558        let config = if config_path.exists() {
559            let config_str = std::fs::read_to_string(&config_path)?;
560            let parsed_config = Self::parse_config(&config_str)?;
561            tracing::info!(
562                "Loaded Gemma3 config: hidden_size={}, num_layers={}, vocab_size={}",
563                parsed_config.hidden_size,
564                parsed_config.num_hidden_layers,
565                parsed_config.vocab_size
566            );
567            tracing::info!(
568                "  activation={}, query_pre_attn_scalar={:?}",
569                parsed_config.hidden_activation,
570                parsed_config.query_pre_attn_scalar
571            );
572            parsed_config
573        } else {
574            tracing::warn!("No config.json found, using default Gemma config");
575            GemmaConfig::default()
576        };
577
578        // Load weights from SafeTensors files
579        let mut weights = HashMap::new();
580
581        // Find all .safetensors files in the directory
582        let parent_dir = path.parent().ok_or_else(|| anyhow!("Invalid model path"))?;
583
584        for entry in std::fs::read_dir(parent_dir)? {
585            let entry = entry?;
586            let path = entry.path();
587            if path.extension().and_then(|s| s.to_str()) == Some("safetensors") {
588                tracing::info!("Loading weights from: {:?}", path);
589                // SafeTensors loading handled by TorchEngine now
590                let tensors = std::collections::HashMap::new();
591                for (name, tensor) in tensors {
592                    weights.insert(name, tensor);
593                }
594            }
595        }
596
597        if weights.is_empty() {
598            return Err(anyhow!("No SafeTensors files found in {:?}", parent_dir));
599        }
600
601        tracing::info!("Loaded {} weight tensors", weights.len());
602
603        // Now build the model from weights
604        Self::from_weights_with_config(&weights, config, device, dtype)
605    }
606
607    /// Create Gemma model from pre-loaded weights
608    pub fn from_weights(
609        weights: &HashMap<String, Tensor>,
610        device: &Device,
611        dtype: DType,
612    ) -> Result<Self> {
613        let config = Self::detect_config_from_weights(weights)?;
614        Self::from_weights_with_config(weights, config, device, dtype)
615    }
616
617    /// Create Gemma3 model with specific configuration
618    pub fn from_weights_gemma3(
619        weights: &HashMap<String, Tensor>,
620        device: &Device,
621        dtype: DType,
622    ) -> Result<Self> {
623        let mut config = Self::detect_config_from_weights(weights)?;
624
625        // Override with Gemma3-specific settings
626        config.hidden_activation = "gelu_pytorch_tanh".to_owned();
627        config.query_pre_attn_scalar = Some(256.0);
628        config.sliding_window = Some(512);
629        config.rope_theta = 1000000.0; // Global attention theta
630        config.rope_local_base_freq = 10000.0; // Local attention theta
631
632        tracing::info!("Loading Gemma model");
633        tracing::info!("  - GELU PyTorch tanh activation");
634        tracing::info!("  - Sliding window attention (512 tokens)");
635        tracing::info!("  - QK-norm with scalar {:?}", config.query_pre_attn_scalar);
636        tracing::info!(
637            "  - RoPE theta: global={}, local={}",
638            config.rope_theta,
639            config.rope_local_base_freq
640        );
641
642        Self::from_weights_with_config(weights, config, device, dtype)
643    }
644
645    fn from_weights_with_config(
646        weights: &HashMap<String, Tensor>,
647        config: GemmaConfig,
648        device: &Device,
649        dtype: DType,
650    ) -> Result<Self> {
651        // Extract embeddings.
652        // #405: place top-level weights on `device` here. Without this,
653        // constructing from CPU weights with device=Cuda(0) leaves
654        // embed/lm_head/norm on CPU while layers move to CUDA.
655        let embed_tokens = weights
656            .get("model.embed_tokens.weight")
657            .or_else(|| weights.get("embed_tokens.weight"))
658            .map(|w| w.shallow_clone().to_device(*device));
659
660        // Gemma uses weight tying - lm_head shares weights with embed_tokens
661        // Keep lm_head as [vocab_size, hidden_size], transpose during use
662        let lm_head = weights
663            .get("lm_head.weight")
664            .or_else(|| weights.get("model.lm_head.weight"))
665            .map(|w| w.shallow_clone().to_device(*device));
666
667        if lm_head.is_none() && embed_tokens.is_some() {
668            tracing::info!("Gemma model uses weight tying (lm_head = embed_tokens.T)");
669        }
670
671        // Extract final layer norm
672        // Use add_unit_offset for Gemma3 (detected by presence of query_pre_attn_scalar)
673        let is_gemma3 = config.query_pre_attn_scalar.is_some();
674        let norm = weights
675            .get("model.norm.weight")
676            .or_else(|| weights.get("norm.weight"))
677            .map(|w| RMSNorm {
678                weight: w.shallow_clone().to_device(*device),
679                eps: config.rms_norm_eps,
680                add_unit_offset: is_gemma3,
681            });
682
683        // Build transformer layers
684        let mut layers = Vec::new();
685        for layer_idx in 0..config.num_hidden_layers {
686            if let Some(layer) = Self::build_layer(layer_idx, weights, &config, device)? {
687                layers.push(layer);
688            }
689        }
690
691        Ok(Self {
692            config,
693            device: *device,
694            dtype,
695            embed_tokens,
696            layers,
697            norm,
698            lm_head,
699        })
700    }
701
702    fn detect_config_from_weights(weights: &HashMap<String, Tensor>) -> Result<GemmaConfig> {
703        let mut config = GemmaConfig::default();
704
705        // Get vocab size from embedding
706        if let Some(embed) = weights
707            .get("model.embed_tokens.weight")
708            .or_else(|| weights.get("embed_tokens.weight"))
709        {
710            let shape = embed.size();
711            if shape.len() >= 2 {
712                config.vocab_size = shape[0] as usize;
713                config.hidden_size = shape[1] as usize;
714            }
715        }
716
717        // Count layers
718        let layer_count = weights
719            .keys()
720            .filter(|k| k.contains("layers.") && k.contains(".input_layernorm.weight"))
721            .count();
722        config.num_hidden_layers = layer_count;
723
724        // Detect attention config from q_proj shape
725        if let Some(q_proj) = weights.get("model.layers.0.self_attn.q_proj.weight") {
726            let q_shape = q_proj.size();
727            if q_shape.len() >= 2 {
728                let q_proj_out_dim = q_shape[0]; // PyTorch format [out, in]
729
730                // Detect Gemma3 with 4 heads x 256 dim = 1024
731                if config.vocab_size == 262144 && q_proj_out_dim == 1024 {
732                    config.num_attention_heads = 4;
733                    config.head_dim = 256;
734                    config.num_key_value_heads = 1; // Gemma3 uses 1 KV head
735                } else {
736                    // Try to infer from dimensions
737                    config.head_dim = if q_proj_out_dim % 256 == 0 { 256 } else { 128 };
738                    config.num_attention_heads = (q_proj_out_dim / config.head_dim as i64) as usize;
739
740                    // Check K proj for KV heads
741                    if let Some(k_proj) = weights.get("model.layers.0.self_attn.k_proj.weight") {
742                        let k_proj_out = k_proj.size()[0];
743                        config.num_key_value_heads = (k_proj_out / config.head_dim as i64) as usize;
744                    }
745                }
746            }
747        }
748
749        // Get intermediate size from gate_proj
750        if let Some(gate_proj) = weights.get("model.layers.0.mlp.gate_proj.weight") {
751            config.intermediate_size = gate_proj.size()[0] as usize;
752        }
753
754        Ok(config)
755    }
756
757    fn build_layer(
758        layer_idx: usize,
759        weights: &HashMap<String, Tensor>,
760        config: &GemmaConfig,
761        _device: &Device,
762    ) -> Result<Option<GemmaLayer>> {
763        let prefix = format!("model.layers.{layer_idx}");
764
765        // Check if layer exists
766        if !weights.contains_key(&format!("{prefix}.self_attn.q_proj.weight")) {
767            return Ok(None);
768        }
769
770        // Get attention weights - SafeTensors stores as [out_features, in_features]
771        // Keep them as-is, we'll transpose during matmul
772        let q_proj = weights
773            .get(&format!("{prefix}.self_attn.q_proj.weight"))
774            .ok_or_else(|| anyhow!("Missing q_proj weight"))?
775            .shallow_clone();
776        let k_proj = weights
777            .get(&format!("{prefix}.self_attn.k_proj.weight"))
778            .ok_or_else(|| anyhow!("Missing k_proj weight"))?
779            .shallow_clone();
780        let v_proj = weights
781            .get(&format!("{prefix}.self_attn.v_proj.weight"))
782            .ok_or_else(|| anyhow!("Missing v_proj weight"))?
783            .shallow_clone();
784        let o_proj = weights
785            .get(&format!("{prefix}.self_attn.o_proj.weight"))
786            .ok_or_else(|| anyhow!("Missing o_proj weight"))?
787            .shallow_clone();
788
789        // Check for QK-norm weights (Gemma3)
790        let q_norm = weights
791            .get(&format!("{prefix}.self_attn.q_norm.weight"))
792            .map(tch::Tensor::shallow_clone);
793        let k_norm = weights
794            .get(&format!("{prefix}.self_attn.k_norm.weight"))
795            .map(tch::Tensor::shallow_clone);
796
797        // Determine layer type from config or use default pattern
798        let layer_type = if !config.layer_types.is_empty() && layer_idx < config.layer_types.len() {
799            // Use layer type from config
800            config.layer_types[layer_idx].clone()
801        } else if config.sliding_window.is_some() {
802            // Fallback: Every 6th layer is global, others are local
803            if (layer_idx + 1).is_multiple_of(6) {
804                "global".to_owned()
805            } else {
806                "local".to_owned()
807            }
808        } else {
809            "global".to_owned()
810        };
811
812        // Use different RoPE theta for local vs global
813        let rope_theta = if layer_type == "local" {
814            config.rope_local_base_freq
815        } else {
816            config.rope_theta
817        };
818
819        let sliding_window = if layer_type == "local" {
820            config.sliding_window
821        } else {
822            None
823        };
824
825        let self_attn = GemmaAttention {
826            q_proj,
827            k_proj,
828            v_proj,
829            o_proj,
830            num_heads: config.num_attention_heads,
831            num_kv_heads: config.num_key_value_heads,
832            head_dim: config.head_dim,
833            rope_theta,
834            q_norm,
835            k_norm,
836            query_pre_attn_scalar: config.query_pre_attn_scalar,
837            sliding_window,
838            layer_type,
839        };
840
841        // Build MLP - keep weights as [out, in], transpose during matmul
842        let mlp = GemmaMLP {
843            gate_proj: weights
844                .get(&format!("{prefix}.mlp.gate_proj.weight"))
845                .ok_or_else(|| anyhow!("Missing gate_proj weight"))?
846                .shallow_clone(),
847            up_proj: weights
848                .get(&format!("{prefix}.mlp.up_proj.weight"))
849                .ok_or_else(|| anyhow!("Missing up_proj weight"))?
850                .shallow_clone(),
851            down_proj: weights
852                .get(&format!("{prefix}.mlp.down_proj.weight"))
853                .ok_or_else(|| anyhow!("Missing down_proj weight"))?
854                .shallow_clone(),
855            activation: config.hidden_activation.clone(),
856        };
857
858        // Build layer norms
859        // Use add_unit_offset for Gemma3 (detected by presence of query_pre_attn_scalar)
860        let is_gemma3 = config.query_pre_attn_scalar.is_some();
861
862        let input_layernorm = RMSNorm {
863            weight: weights
864                .get(&format!("{prefix}.input_layernorm.weight"))
865                .ok_or_else(|| anyhow!("Missing input_layernorm weight"))?
866                .shallow_clone(),
867            eps: config.rms_norm_eps,
868            add_unit_offset: is_gemma3,
869        };
870
871        let post_attention_layernorm = RMSNorm {
872            weight: weights
873                .get(&format!("{prefix}.post_attention_layernorm.weight"))
874                .ok_or_else(|| anyhow!("Missing post_attention_layernorm weight"))?
875                .shallow_clone(),
876            eps: config.rms_norm_eps,
877            add_unit_offset: is_gemma3,
878        };
879
880        Ok(Some(GemmaLayer {
881            self_attn,
882            mlp,
883            input_layernorm,
884            post_attention_layernorm,
885        }))
886    }
887
888    /// Parse Gemma configuration from config.json
889    fn parse_config(config_str: &str) -> Result<GemmaConfig> {
890        let config: serde_json::Value = serde_json::from_str(config_str)?;
891
892        // Parse layer_types and convert to local/global
893        let layer_types = config
894            .get("layer_types")
895            .and_then(|v| v.as_array())
896            .map(|arr| {
897                arr.iter()
898                    .map(|v| match v.as_str().unwrap_or("") {
899                        "sliding_attention" => "local".to_owned(),
900                        // "full_attention" and other values default to global
901                        _ => "global".to_owned(),
902                    })
903                    .collect()
904            })
905            .unwrap_or_default();
906
907        // Parse query_pre_attn_scalar - this is an integer in the config
908        let query_pre_attn_scalar = config.get("query_pre_attn_scalar").and_then(|v| {
909            if let Some(int_val) = v.as_u64() {
910                Some(int_val as f32)
911            } else {
912                v.as_f64().map(|f| f as f32)
913            }
914        });
915
916        Ok(GemmaConfig {
917            num_attention_heads: config
918                .get("num_attention_heads")
919                .and_then(serde_json::Value::as_u64)
920                .unwrap_or(4) as usize,
921            num_key_value_heads: config
922                .get("num_key_value_heads")
923                .and_then(serde_json::Value::as_u64)
924                .unwrap_or(1) as usize,
925            hidden_size: config
926                .get("hidden_size")
927                .and_then(serde_json::Value::as_u64)
928                .unwrap_or(640) as usize,
929            head_dim: config
930                .get("head_dim")
931                .and_then(serde_json::Value::as_u64)
932                .unwrap_or(256) as usize,
933            intermediate_size: config
934                .get("intermediate_size")
935                .and_then(serde_json::Value::as_u64)
936                .unwrap_or(2048) as usize,
937            max_position_embeddings: config
938                .get("max_position_embeddings")
939                .and_then(serde_json::Value::as_u64)
940                .unwrap_or(32768) as usize,
941            rms_norm_eps: config
942                .get("rms_norm_eps")
943                .and_then(serde_json::Value::as_f64)
944                .unwrap_or(1e-6) as f32,
945            vocab_size: config
946                .get("vocab_size")
947                .and_then(serde_json::Value::as_u64)
948                .unwrap_or(262144) as usize,
949            num_hidden_layers: config
950                .get("num_hidden_layers")
951                .and_then(serde_json::Value::as_u64)
952                .unwrap_or(18) as usize,
953            rope_theta: config
954                .get("rope_theta")
955                .and_then(serde_json::Value::as_f64)
956                .unwrap_or(1000000.0) as f32,
957            rope_local_base_freq: config
958                .get("rope_local_base_freq")
959                .and_then(serde_json::Value::as_f64)
960                .unwrap_or(10000.0) as f32,
961            sliding_window: config
962                .get("sliding_window")
963                .and_then(serde_json::Value::as_u64)
964                .map(|v| v as usize),
965            layer_types,
966            query_pre_attn_scalar,
967            hidden_activation: config
968                .get("hidden_activation")
969                .and_then(|v| v.as_str())
970                .unwrap_or("silu")
971                .to_owned(),
972        })
973    }
974}
975
976impl ModelOperations for GemmaModel {
977    fn architecture(&self) -> ModelArchitecture {
978        ModelArchitecture::Gemma
979    }
980
981    fn config(&self) -> &dyn ArchitectureConfig {
982        &self.config
983    }
984
985    fn forward(&self, input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor> {
986        // Input should be token IDs with shape [batch_size, seq_len]
987        let mut hidden_states = if let Some(embed) = &self.embed_tokens {
988            // Convert token IDs to embeddings
989            // Get input shape
990            let input_shape = input.size();
991            let batch_size = input_shape[0];
992            let seq_len = if input_shape.len() > 1 {
993                input_shape[1]
994            } else {
995                1
996            };
997
998            // Flatten input for embedding lookup (embedding expects 1D tensor)
999            let flat_input = input.flatten(0, -1);
1000
1001            // Perform embedding lookup using index_select
1002            let embeddings = embed.index_select(0, &flat_input);
1003
1004            // Get the actual hidden size from the embedding result
1005            let emb_dims = embeddings.size();
1006            let hidden_size = emb_dims[emb_dims.len() - 1]; // Last dimension is hidden size
1007
1008            // Reshape back to [batch_size, seq_len, hidden_size]
1009            let mut embeddings = embeddings.reshape([batch_size, seq_len, hidden_size]);
1010
1011            // Apply embedding scaling for Gemma - scale by sqrt(hidden_size)
1012            let scale = (hidden_size as f32).sqrt();
1013            let scale_tensor = Tensor::from(scale)
1014                .to_kind(embeddings.kind())
1015                .to_device(embeddings.device());
1016            embeddings = broadcast_mul(&embeddings, &scale_tensor)?;
1017            tracing::debug!(
1018                "Applied Gemma embedding scaling by sqrt(hidden_size={}) = {}",
1019                hidden_size,
1020                scale
1021            );
1022
1023            embeddings
1024        } else {
1025            // If no embedding layer, assume input is already embedded
1026            input.shallow_clone()
1027        };
1028
1029        // Apply transformer layers
1030        for layer in &self.layers {
1031            let residual = hidden_states.shallow_clone();
1032
1033            // Self-attention block
1034            hidden_states = layer.input_layernorm.forward(&hidden_states)?;
1035            let attn_output = layer.self_attn.forward(&hidden_states, None)?;
1036            hidden_states = residual + attn_output;
1037
1038            // FFN block
1039            let residual = hidden_states.shallow_clone();
1040            hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
1041            let ffn_output = layer.mlp.forward(&hidden_states)?;
1042            hidden_states = residual + ffn_output;
1043        }
1044
1045        // Final layer norm
1046        if let Some(norm) = &self.norm {
1047            hidden_states = norm.forward(&hidden_states)?;
1048        }
1049
1050        // LM head - handle weight tying for Gemma
1051        if let Some(lm_head) = &self.lm_head {
1052            // Use explicit lm_head if available
1053            // lm_head is [vocab_size, hidden_size], need to transpose for matmul
1054            let (batch_size, seq_len, hidden_size) = dims3(&hidden_states)?;
1055            let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
1056            let logits = hidden_2d.matmul(&lm_head.transpose(0, 1));
1057            let vocab_size = logits.size()[1];
1058            hidden_states = logits.reshape([batch_size, seq_len, vocab_size]);
1059        } else if let Some(embed) = &self.embed_tokens {
1060            // Gemma uses weight tying: lm_head = embed_tokens.T
1061            // embed_tokens is [vocab_size, hidden_size], use it directly as lm_head
1062            let (batch_size, seq_len, hidden_size) = dims3(&hidden_states)?;
1063            let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
1064
1065            // For weight tying, embed_tokens already acts as the transposed lm_head
1066            let logits = hidden_2d.matmul(&embed.transpose(0, 1));
1067            let vocab_size = logits.size()[1];
1068            hidden_states = logits.reshape([batch_size, seq_len, vocab_size]);
1069        }
1070
1071        Ok(hidden_states)
1072    }
1073
1074    fn reshape_for_attention(&self, tensor: &Tensor, is_key_value: bool) -> Result<Tensor> {
1075        let (batch_size, seq_len, _hidden_size) = dims3(tensor)?;
1076
1077        if is_key_value {
1078            // For Gemma MQA: reshape to [batch, seq, num_kv_heads, head_dim]
1079            // This is the key fix for the shape mismatch issue!
1080            Ok(tensor.reshape([
1081                batch_size,
1082                seq_len,
1083                self.config.num_key_value_heads as i64,
1084                self.config.head_dim as i64,
1085            ]))
1086        } else {
1087            // For queries: standard reshape
1088            Ok(tensor.reshape([
1089                batch_size,
1090                seq_len,
1091                self.config.num_attention_heads as i64,
1092                self.config.head_dim as i64,
1093            ]))
1094        }
1095    }
1096
1097    fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor> {
1098        // Use the centralized RoPE module with standard base frequency and matching dtype
1099        let mut rope = crate::runtime::rope::RoPE::new_standard_with_dtype(
1100            self.config.head_dim as i64,
1101            8192, // max_seq_len
1102            tensor.device(),
1103            tensor.kind(), // Use same dtype as input tensor
1104        )?;
1105        rope.forward(tensor, Some(position_ids))
1106    }
1107
1108    fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
1109        // Gemma uses RMSNorm
1110        let x2 = square_tensor(tensor)?;
1111        let mean = x2.mean_dim(&[-1i64][..], true, tensor.kind()); // Preserve input precision
1112        let rrms = (mean + self.config.rms_norm_eps as f64).reciprocal().sqrt();
1113        broadcast_mul(tensor, &rrms)
1114    }
1115
1116    fn get_attention_mask(&self, seq_len: usize, past_kv_len: usize) -> Result<Tensor> {
1117        // Create causal attention mask
1118        let total_len = seq_len + past_kv_len;
1119        let mask = Tensor::ones(
1120            [seq_len as i64, total_len as i64],
1121            (DType::BFloat16, self.device),
1122        );
1123
1124        // Apply causal masking
1125        // This is simplified - actual implementation would properly mask future tokens
1126        Ok(mask)
1127    }
1128
1129    fn as_any(&self) -> &dyn std::any::Any {
1130        self
1131    }
1132}
1133
1134impl GemmaModel {
1135    /// Create Gemma model from VarStore weights (proper implementation)
1136    pub fn from_varstore(vs: &tch::nn::VarStore, device: &Device, dtype: DType) -> Result<Self> {
1137        let variables = vs.variables();
1138
1139        if variables.is_empty() {
1140            return Err(anyhow!("No tensors found in VarStore"));
1141        }
1142
1143        info!("Loading {} model tensors", variables.len());
1144
1145        // Convert VarStore variables to HashMap<String, Tensor>
1146        let mut weights = HashMap::new();
1147        for (name, var) in variables.iter() {
1148            weights.insert(name.clone(), var.shallow_clone());
1149        }
1150
1151        // Log some key weights for debugging
1152        if let Some(embed) = weights.get("model.embed_tokens.weight") {
1153            let shape = embed.size();
1154            info!(
1155                "📝 Found embeddings: {} -> [{}, {}]",
1156                "model.embed_tokens.weight", shape[0], shape[1]
1157            );
1158        }
1159
1160        // Count layers
1161        let layer_count = weights
1162            .keys()
1163            .filter(|k| k.contains("layers.") && k.contains(".input_layernorm.weight"))
1164            .count();
1165        info!("📝 Found {} transformer layers", layer_count);
1166
1167        // Use existing from_weights method
1168        Self::from_weights(&weights, device, dtype)
1169    }
1170
1171    /// Load model weights from a VarStore (SafeTensors) - legacy method for compatibility
1172    pub fn load_weights_from_varstore(&mut self, vs: &tch::nn::VarStore) -> Result<()> {
1173        let variables = vs.variables();
1174
1175        info!(
1176            "🔍 Validating {} loaded tensors for Gemma architecture",
1177            variables.len()
1178        );
1179
1180        // For now, just validate basic structure
1181        if variables.is_empty() {
1182            return Err(anyhow!("No tensors found in VarStore"));
1183        }
1184
1185        info!("✅ Gemma architecture ready (use from_varstore for proper loading)");
1186        Ok(())
1187    }
1188
1189    /// Create a simple forward pass that returns valid tensor shapes
1190    pub fn simple_forward(&self, input: &Tensor) -> Result<Tensor> {
1191        let input_shape = input.size();
1192        let batch_size = input_shape[0];
1193        let seq_len = input_shape[1];
1194
1195        // Return random logits with correct shape [batch_size, seq_len, vocab_size]
1196        let logits = Tensor::randn(
1197            [batch_size, seq_len, self.config.vocab_size as i64],
1198            (DType::Float, input.device()),
1199        );
1200
1201        info!(
1202            "🔍 Gemma simple forward: [{}, {}] → [{}, {}, {}]",
1203            batch_size, seq_len, batch_size, seq_len, self.config.vocab_size
1204        );
1205
1206        Ok(logits)
1207    }
1208}
1209
1210// Temporarily disabled - needs updating for new Tensor API
1211#[cfg(test)]
1212#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
1213mod tests {
1214    use super::*;
1215
1216    #[test]
1217    fn test_gemma_reshape_for_mqa() {
1218        let device = Device::Cpu;
1219        let config = GemmaConfig::default();
1220        let model = GemmaModel {
1221            config: config.clone(),
1222            device,
1223            dtype: DType::Float,
1224            embed_tokens: None,
1225            layers: vec![],
1226            norm: None,
1227            lm_head: None,
1228        };
1229
1230        // Test tensor for key/value with shape [1, 21, 1024] (4 * 256)
1231        let kv_tensor = Tensor::randn([1, 21, 1024], (DType::Float, device));
1232
1233        // Reshape for key/value (MQA)
1234        let reshaped = model
1235            .reshape_for_attention(&kv_tensor, true)
1236            .expect("test: reshape kv");
1237        assert_eq!(reshaped.size(), &[1, 21, 4, 256]); // 4 KV heads, 256 head_dim
1238
1239        // Test tensor for query with shape [1, 21, 4096] (16 * 256)
1240        let q_tensor = Tensor::randn([1, 21, 4096], (DType::Float, device));
1241
1242        // Reshape for query
1243        let reshaped = model
1244            .reshape_for_attention(&q_tensor, false)
1245            .expect("test: reshape q");
1246        assert_eq!(reshaped.size(), &[1, 21, 16, 256]); // 16 Q heads, 256 head_dim
1247    }
1248
1249    #[test]
1250    fn test_kv_expansion_for_mqa() {
1251        let device = Device::Cpu;
1252        let attn = GemmaAttention {
1253            q_proj: Tensor::zeros([1024, 4096], (DType::Float, device)),
1254            k_proj: Tensor::zeros([1024, 1024], (DType::Float, device)),
1255            v_proj: Tensor::zeros([1024, 1024], (DType::Float, device)),
1256            o_proj: Tensor::zeros([4096, 1024], (DType::Float, device)),
1257            num_heads: 16,
1258            num_kv_heads: 4,
1259            head_dim: 256,
1260            rope_theta: 10000.0,
1261            q_norm: None,
1262            k_norm: None,
1263            query_pre_attn_scalar: None,
1264            sliding_window: None,
1265            layer_type: "full_attention".to_owned(),
1266        };
1267
1268        // Create KV tensor with 4 heads
1269        let kv = Tensor::randn([2, 10, 4, 256], (DType::Float, device));
1270
1271        // Expand to match 16 query heads
1272        let expanded = attn
1273            .expand_kv_for_mqa(&kv)
1274            .expect("test: expand kv for mqa");
1275        assert_eq!(expanded.size(), &[2, 10, 16, 256]);
1276    }
1277}