1use super::{ArchitectureConfig, ModelArchitecture, ModelOperations};
4use crate::runtime::tensor_helpers::{
6 broadcast_add, broadcast_mul, broadcast_sub, dims3, dims4, scalar_tensor, square_tensor,
7};
8use anyhow::{anyhow, Result};
9use tch::{Device, Kind as DType, Tensor};
10use tracing::info;
11use std::collections::HashMap;
13use std::path::Path;
14
15#[derive(Debug, Clone)]
17pub struct GemmaConfig {
18 pub num_attention_heads: usize,
20 pub num_key_value_heads: usize,
22 pub hidden_size: usize,
24 pub head_dim: usize,
26 pub intermediate_size: usize,
28 pub max_position_embeddings: usize,
30 pub rms_norm_eps: f32,
32 pub vocab_size: usize,
34 pub num_hidden_layers: usize,
36 pub rope_theta: f32,
38 pub rope_local_base_freq: f32,
40 pub sliding_window: Option<usize>,
42 pub layer_types: Vec<String>,
44 pub query_pre_attn_scalar: Option<f32>,
46 pub hidden_activation: String,
48}
49
50impl Default for GemmaConfig {
51 fn default() -> Self {
52 Self {
54 num_attention_heads: 16,
55 num_key_value_heads: 4, hidden_size: 3072,
57 head_dim: 256,
58 intermediate_size: 24576,
59 max_position_embeddings: 8192,
60 rms_norm_eps: 1e-6,
61 vocab_size: 256000,
62 num_hidden_layers: 28,
63 rope_theta: 10000.0,
64 rope_local_base_freq: 10000.0,
65 sliding_window: None,
66 layer_types: vec![],
67 query_pre_attn_scalar: None,
68 hidden_activation: "silu".to_owned(),
69 }
70 }
71}
72
73impl ArchitectureConfig for GemmaConfig {
74 fn num_attention_heads(&self) -> usize {
75 self.num_attention_heads
76 }
77
78 fn num_key_value_heads(&self) -> usize {
79 self.num_key_value_heads
80 }
81
82 fn hidden_size(&self) -> usize {
83 self.hidden_size
84 }
85
86 fn intermediate_size(&self) -> usize {
87 self.intermediate_size
88 }
89
90 fn head_dim(&self) -> usize {
91 self.head_dim
92 }
93
94 fn vocab_size(&self) -> usize {
95 self.vocab_size
96 }
97
98 fn max_position_embeddings(&self) -> usize {
99 self.max_position_embeddings
100 }
101
102 fn rope_theta(&self) -> Option<f32> {
103 Some(self.rope_theta)
104 }
105
106 fn rope_dim(&self) -> Option<usize> {
107 Some(self.head_dim)
108 }
109
110 fn layer_norm_eps(&self) -> f32 {
111 self.rms_norm_eps
112 }
113
114 fn use_rms_norm(&self) -> bool {
115 true }
117}
118
119pub struct GemmaModel {
121 config: GemmaConfig,
122 #[allow(dead_code)]
123 device: Device,
124 #[allow(dead_code)]
125 dtype: DType,
126
127 embed_tokens: Option<Tensor>,
129 layers: Vec<GemmaLayer>,
130 norm: Option<RMSNorm>,
131 lm_head: Option<Tensor>,
132}
133
134unsafe impl Send for GemmaModel {}
137unsafe impl Sync for GemmaModel {}
138
139struct GemmaLayer {
141 self_attn: GemmaAttention,
142 mlp: GemmaMLP,
143 input_layernorm: RMSNorm,
144 post_attention_layernorm: RMSNorm,
145}
146
147unsafe impl Send for GemmaLayer {}
148unsafe impl Sync for GemmaLayer {}
149
150struct GemmaAttention {
152 q_proj: Tensor,
153 k_proj: Tensor,
154 v_proj: Tensor,
155 o_proj: Tensor,
156 num_heads: usize,
157 num_kv_heads: usize,
158 head_dim: usize,
159 rope_theta: f32,
160 q_norm: Option<Tensor>,
162 k_norm: Option<Tensor>,
163 query_pre_attn_scalar: Option<f32>,
164 sliding_window: Option<usize>,
166 layer_type: String,
167}
168
169unsafe impl Send for GemmaAttention {}
170unsafe impl Sync for GemmaAttention {}
171
172impl GemmaAttention {
173 fn forward(&self, hidden_states: &Tensor, position_ids: Option<&Tensor>) -> Result<Tensor> {
175 let (batch_size, seq_len, hidden_size) = dims3(hidden_states)?;
176
177 let hidden_states_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
179
180 let q = hidden_states_2d.matmul(&self.q_proj.transpose(0, 1));
182 let k = hidden_states_2d.matmul(&self.k_proj.transpose(0, 1));
183 let v = hidden_states_2d.matmul(&self.v_proj.transpose(0, 1));
184
185 let mut q = q.reshape([
188 batch_size,
189 seq_len,
190 self.num_heads as i64,
191 self.head_dim as i64,
192 ]);
193 let mut k = k.reshape([
195 batch_size,
196 seq_len,
197 self.num_kv_heads as i64,
198 self.head_dim as i64,
199 ]);
200 let v = v.reshape([
201 batch_size,
202 seq_len,
203 self.num_kv_heads as i64,
204 self.head_dim as i64,
205 ]);
206
207 if let Some(q_norm) = &self.q_norm {
209 q = self.apply_qk_norm(&q, q_norm, self.num_heads)?;
210 }
211 if let Some(k_norm) = &self.k_norm {
212 k = self.apply_qk_norm(&k, k_norm, self.num_kv_heads)?;
213 }
214
215 let (q, k) = if let Some(pos_ids) = position_ids {
217 (self.apply_rope(&q, pos_ids)?, self.apply_rope(&k, pos_ids)?)
218 } else {
219 (q, k)
220 };
221
222 let q = if let Some(scalar) = self.query_pre_attn_scalar {
225 let scale = scalar.powf(-0.5); let scale_tensor = scalar_tensor(scale, q.device(), q.kind());
227 broadcast_mul(&q, &scale_tensor)?
228 } else {
229 q
230 };
231
232 let k = self.expand_kv_for_mqa(&k)?;
234 let v = self.expand_kv_for_mqa(&v)?;
235
236 let scores = self.compute_attention_scores(&q, &k)?;
238
239 let v = v.transpose(1, 2);
241
242 let attn_output = scores.matmul(&v);
244
245 let attn_output = attn_output.transpose(1, 2);
247
248 let attn_output = attn_output
250 .reshape([batch_size, seq_len, (self.num_heads * self.head_dim) as i64])
251 .reshape([
252 batch_size * seq_len,
253 (self.num_heads * self.head_dim) as i64,
254 ])
255 .matmul(&self.o_proj.transpose(0, 1))
256 .reshape([batch_size, seq_len, hidden_size]);
257
258 Ok(attn_output)
259 }
260
261 fn expand_kv_for_mqa(&self, kv: &Tensor) -> Result<Tensor> {
263 let (batch_size, seq_len, num_kv_heads, head_dim) = dims4(kv)?;
264 let repeat_factor = (self.num_heads as i64) / num_kv_heads;
265
266 if repeat_factor == 1 {
267 return Ok(kv.shallow_clone());
268 }
269
270 Ok(kv
272 .unsqueeze(3) .expand(
274 [batch_size, seq_len, num_kv_heads, repeat_factor, head_dim],
275 false,
276 )
277 .reshape([batch_size, seq_len, self.num_heads as i64, head_dim]))
278 }
279
280 fn apply_rope(&self, tensor: &Tensor, _position_ids: &Tensor) -> Result<Tensor> {
282 let (_batch_size, seq_len, _num_heads, head_dim) = dims4(tensor)?;
284
285 let theta = self.rope_theta;
287 let device = tensor.device();
288 let dtype = tensor.kind();
289
290 let inv_freq = (0..head_dim / 2)
292 .map(|i| 1.0 / theta.powf(2.0 * i as f32 / head_dim as f32))
293 .collect::<Vec<_>>();
294
295 let _positions =
297 Tensor::arange(seq_len, (DType::Int64, device)).to_dtype(DType::Float, false, false);
298
299 let mut sin_vals = Vec::new();
301 let mut cos_vals = Vec::new();
302
303 for pos in 0..seq_len {
304 for freq_idx in 0..head_dim / 2 {
305 let angle = pos as f32 * inv_freq[freq_idx as usize];
306 sin_vals.push(angle.sin());
307 cos_vals.push(angle.cos());
308 }
309 }
310
311 let sin = Tensor::from_slice(&sin_vals)
313 .reshape([seq_len, head_dim / 2])
314 .to(device)
315 .to_dtype(dtype, false, false);
316 let cos = Tensor::from_slice(&cos_vals)
317 .reshape([seq_len, head_dim / 2])
318 .to(device)
319 .to_dtype(dtype, false, false);
320
321 let x1 = tensor.narrow(-1, 0, head_dim / 2);
323 let x2 = tensor.narrow(-1, head_dim / 2, head_dim / 2);
324
325 let rotated_x1 = broadcast_sub(&broadcast_mul(&x1, &cos)?, &broadcast_mul(&x2, &sin)?)?;
327 let rotated_x2 = broadcast_add(&broadcast_mul(&x1, &sin)?, &broadcast_mul(&x2, &cos)?)?;
328
329 Ok(Tensor::cat(&[rotated_x1, rotated_x2], -1))
331 }
332
333 fn apply_qk_norm(
335 &self,
336 tensor: &Tensor,
337 norm_weight: &Tensor,
338 num_heads: usize,
339 ) -> Result<Tensor> {
340 let (batch_size, seq_len, _tensor_heads, head_dim) = dims4(tensor)?;
343
344 let norm_weight = if norm_weight.size()[0] == head_dim {
346 norm_weight
348 .unsqueeze(0) .expand([num_heads as i64, head_dim], false) .reshape([(num_heads * head_dim as usize) as i64])
351 } else {
352 norm_weight.shallow_clone()
353 };
354
355 let tensor_flat =
357 tensor.reshape([batch_size * seq_len, (num_heads * head_dim as usize) as i64]);
358
359 let normalized = broadcast_mul(&tensor_flat, &norm_weight)?;
361
362 Ok(normalized.reshape([batch_size, seq_len, num_heads as i64, head_dim]))
364 }
365
366 fn compute_attention_scores(&self, q: &Tensor, k: &Tensor) -> Result<Tensor> {
368 let scale = if self.query_pre_attn_scalar.is_some() {
371 1.0
372 } else {
373 1.0 / (self.head_dim as f32).sqrt()
374 };
375
376 let q = q.transpose(1, 2);
378 let k = k.transpose(1, 2).transpose(2, 3);
380
381 let mut scores = q.matmul(&k) * (scale as f64);
383
384 if let Some(window_size) = self.sliding_window {
386 if self.layer_type == "local" {
387 scores = self.apply_sliding_window_mask(&scores, window_size)?;
388 }
389 }
391
392 Ok(scores.softmax(-1, scores.kind()))
394 }
395
396 fn apply_sliding_window_mask(&self, scores: &Tensor, window_size: usize) -> Result<Tensor> {
398 let (batch_size, num_heads, seq_len, _) = dims4(scores)?;
399 let device = scores.device();
400 let dtype = scores.kind();
401
402 let mut mask_values = vec![0f32; (seq_len * seq_len) as usize];
404 for i in 0..seq_len {
405 for j in 0..seq_len {
406 if j <= i && i - j < window_size as i64 {
407 mask_values[(i * seq_len + j) as usize] = 0.0; } else {
409 mask_values[(i * seq_len + j) as usize] = -10000.0; }
411 }
412 }
413
414 let mask = Tensor::from_slice(&mask_values)
416 .reshape([seq_len, seq_len])
417 .to(device)
418 .to_dtype(dtype, false, false)
419 .unsqueeze(0) .unsqueeze(0) .expand([batch_size, num_heads, seq_len, seq_len], false);
422
423 broadcast_add(scores, &mask)
425 }
426}
427
428struct GemmaMLP {
430 gate_proj: Tensor,
431 up_proj: Tensor,
432 down_proj: Tensor,
433 activation: String, }
435
436unsafe impl Send for GemmaMLP {}
437unsafe impl Sync for GemmaMLP {}
438
439impl GemmaMLP {
440 fn forward(&self, hidden_states: &Tensor) -> Result<Tensor> {
441 let input_dims = hidden_states.size();
443 let needs_reshape = input_dims.len() == 3;
444
445 let (batch_size, seq_len, hidden_size) = if needs_reshape {
446 (input_dims[0], input_dims[1], input_dims[2])
447 } else {
448 (1, 1, input_dims[input_dims.len() - 1])
449 };
450
451 let hidden_2d = if needs_reshape {
453 hidden_states.reshape([batch_size * seq_len, hidden_size])
454 } else {
455 hidden_states.shallow_clone()
456 };
457
458 let gate_pre_activation = hidden_2d.matmul(&self.gate_proj.transpose(0, 1));
460
461 let gate = match self.activation.as_str() {
463 "gelu_pytorch_tanh" => self.gelu_pytorch_tanh(&gate_pre_activation)?,
464 _ => gate_pre_activation.silu(), };
466
467 let up = hidden_2d.matmul(&self.up_proj.transpose(0, 1));
468 let output = (gate * &up).matmul(&self.down_proj.transpose(0, 1));
469
470 if needs_reshape {
472 let output_hidden_size = output.size()[1];
473 Ok(output.reshape([batch_size, seq_len, output_hidden_size]))
474 } else {
475 Ok(output)
476 }
477 }
478
479 fn gelu_pytorch_tanh(&self, x: &Tensor) -> Result<Tensor> {
482 use std::f32::consts::PI;
483
484 let sqrt_2_over_pi = (2.0_f32 / PI).sqrt();
486 let coeff = 0.044715_f32;
487
488 let x_cubed = x.pow_tensor_scalar(3.0);
490
491 let coeff_tensor = scalar_tensor(coeff, x.device(), x.kind());
493 let inner = x + broadcast_mul(&x_cubed, &coeff_tensor)?;
494
495 let sqrt_tensor = scalar_tensor(sqrt_2_over_pi, x.device(), x.kind());
497 let scaled = broadcast_mul(&inner, &sqrt_tensor)?;
498
499 let tanh_result = scaled.tanh();
501
502 let one_tensor = scalar_tensor(1.0_f32, x.device(), x.kind());
504 let one_plus_tanh = broadcast_add(&tanh_result, &one_tensor)?;
505
506 let half_tensor = scalar_tensor(0.5_f32, x.device(), x.kind());
508 broadcast_mul(&(x * &one_plus_tanh), &half_tensor)
509 }
510}
511
512struct RMSNorm {
514 weight: Tensor,
515 eps: f32,
516 add_unit_offset: bool, }
518
519unsafe impl Send for RMSNorm {}
520unsafe impl Sync for RMSNorm {}
521
522impl RMSNorm {
523 fn forward(&self, x: &Tensor) -> Result<Tensor> {
524 let hidden_size = x.size()[x.size().len() - 1];
526
527 let x2 = square_tensor(x)?;
530 let norm_x = x2.sum_dim_intlist(&[-1i64][..], true, None) * (1.0 / hidden_size as f64);
531
532 let x_normed = x / &(norm_x + self.eps as f64).sqrt();
534
535 if self.add_unit_offset {
537 let weight_plus_one = &self.weight + 1.0;
539 Ok(broadcast_mul(&x_normed, &weight_plus_one)?)
540 } else {
541 Ok(broadcast_mul(&x_normed, &self.weight)?)
543 }
544 }
545}
546
547impl GemmaModel {
548 pub fn from_safetensors(path: &Path, device: &Device, dtype: DType) -> Result<Self> {
550 let config_path = path
554 .parent()
555 .ok_or_else(|| anyhow!("Invalid model path"))?
556 .join("config.json");
557
558 let config = if config_path.exists() {
559 let config_str = std::fs::read_to_string(&config_path)?;
560 let parsed_config = Self::parse_config(&config_str)?;
561 tracing::info!(
562 "Loaded Gemma3 config: hidden_size={}, num_layers={}, vocab_size={}",
563 parsed_config.hidden_size,
564 parsed_config.num_hidden_layers,
565 parsed_config.vocab_size
566 );
567 tracing::info!(
568 " activation={}, query_pre_attn_scalar={:?}",
569 parsed_config.hidden_activation,
570 parsed_config.query_pre_attn_scalar
571 );
572 parsed_config
573 } else {
574 tracing::warn!("No config.json found, using default Gemma config");
575 GemmaConfig::default()
576 };
577
578 let mut weights = HashMap::new();
580
581 let parent_dir = path.parent().ok_or_else(|| anyhow!("Invalid model path"))?;
583
584 for entry in std::fs::read_dir(parent_dir)? {
585 let entry = entry?;
586 let path = entry.path();
587 if path.extension().and_then(|s| s.to_str()) == Some("safetensors") {
588 tracing::info!("Loading weights from: {:?}", path);
589 let tensors = std::collections::HashMap::new();
591 for (name, tensor) in tensors {
592 weights.insert(name, tensor);
593 }
594 }
595 }
596
597 if weights.is_empty() {
598 return Err(anyhow!("No SafeTensors files found in {:?}", parent_dir));
599 }
600
601 tracing::info!("Loaded {} weight tensors", weights.len());
602
603 Self::from_weights_with_config(&weights, config, device, dtype)
605 }
606
607 pub fn from_weights(
609 weights: &HashMap<String, Tensor>,
610 device: &Device,
611 dtype: DType,
612 ) -> Result<Self> {
613 let config = Self::detect_config_from_weights(weights)?;
614 Self::from_weights_with_config(weights, config, device, dtype)
615 }
616
617 pub fn from_weights_gemma3(
619 weights: &HashMap<String, Tensor>,
620 device: &Device,
621 dtype: DType,
622 ) -> Result<Self> {
623 let mut config = Self::detect_config_from_weights(weights)?;
624
625 config.hidden_activation = "gelu_pytorch_tanh".to_owned();
627 config.query_pre_attn_scalar = Some(256.0);
628 config.sliding_window = Some(512);
629 config.rope_theta = 1000000.0; config.rope_local_base_freq = 10000.0; tracing::info!("Loading Gemma model");
633 tracing::info!(" - GELU PyTorch tanh activation");
634 tracing::info!(" - Sliding window attention (512 tokens)");
635 tracing::info!(" - QK-norm with scalar {:?}", config.query_pre_attn_scalar);
636 tracing::info!(
637 " - RoPE theta: global={}, local={}",
638 config.rope_theta,
639 config.rope_local_base_freq
640 );
641
642 Self::from_weights_with_config(weights, config, device, dtype)
643 }
644
645 fn from_weights_with_config(
646 weights: &HashMap<String, Tensor>,
647 config: GemmaConfig,
648 device: &Device,
649 dtype: DType,
650 ) -> Result<Self> {
651 let embed_tokens = weights
656 .get("model.embed_tokens.weight")
657 .or_else(|| weights.get("embed_tokens.weight"))
658 .map(|w| w.shallow_clone().to_device(*device));
659
660 let lm_head = weights
663 .get("lm_head.weight")
664 .or_else(|| weights.get("model.lm_head.weight"))
665 .map(|w| w.shallow_clone().to_device(*device));
666
667 if lm_head.is_none() && embed_tokens.is_some() {
668 tracing::info!("Gemma model uses weight tying (lm_head = embed_tokens.T)");
669 }
670
671 let is_gemma3 = config.query_pre_attn_scalar.is_some();
674 let norm = weights
675 .get("model.norm.weight")
676 .or_else(|| weights.get("norm.weight"))
677 .map(|w| RMSNorm {
678 weight: w.shallow_clone().to_device(*device),
679 eps: config.rms_norm_eps,
680 add_unit_offset: is_gemma3,
681 });
682
683 let mut layers = Vec::new();
685 for layer_idx in 0..config.num_hidden_layers {
686 if let Some(layer) = Self::build_layer(layer_idx, weights, &config, device)? {
687 layers.push(layer);
688 }
689 }
690
691 Ok(Self {
692 config,
693 device: *device,
694 dtype,
695 embed_tokens,
696 layers,
697 norm,
698 lm_head,
699 })
700 }
701
702 fn detect_config_from_weights(weights: &HashMap<String, Tensor>) -> Result<GemmaConfig> {
703 let mut config = GemmaConfig::default();
704
705 if let Some(embed) = weights
707 .get("model.embed_tokens.weight")
708 .or_else(|| weights.get("embed_tokens.weight"))
709 {
710 let shape = embed.size();
711 if shape.len() >= 2 {
712 config.vocab_size = shape[0] as usize;
713 config.hidden_size = shape[1] as usize;
714 }
715 }
716
717 let layer_count = weights
719 .keys()
720 .filter(|k| k.contains("layers.") && k.contains(".input_layernorm.weight"))
721 .count();
722 config.num_hidden_layers = layer_count;
723
724 if let Some(q_proj) = weights.get("model.layers.0.self_attn.q_proj.weight") {
726 let q_shape = q_proj.size();
727 if q_shape.len() >= 2 {
728 let q_proj_out_dim = q_shape[0]; if config.vocab_size == 262144 && q_proj_out_dim == 1024 {
732 config.num_attention_heads = 4;
733 config.head_dim = 256;
734 config.num_key_value_heads = 1; } else {
736 config.head_dim = if q_proj_out_dim % 256 == 0 { 256 } else { 128 };
738 config.num_attention_heads = (q_proj_out_dim / config.head_dim as i64) as usize;
739
740 if let Some(k_proj) = weights.get("model.layers.0.self_attn.k_proj.weight") {
742 let k_proj_out = k_proj.size()[0];
743 config.num_key_value_heads = (k_proj_out / config.head_dim as i64) as usize;
744 }
745 }
746 }
747 }
748
749 if let Some(gate_proj) = weights.get("model.layers.0.mlp.gate_proj.weight") {
751 config.intermediate_size = gate_proj.size()[0] as usize;
752 }
753
754 Ok(config)
755 }
756
757 fn build_layer(
758 layer_idx: usize,
759 weights: &HashMap<String, Tensor>,
760 config: &GemmaConfig,
761 _device: &Device,
762 ) -> Result<Option<GemmaLayer>> {
763 let prefix = format!("model.layers.{layer_idx}");
764
765 if !weights.contains_key(&format!("{prefix}.self_attn.q_proj.weight")) {
767 return Ok(None);
768 }
769
770 let q_proj = weights
773 .get(&format!("{prefix}.self_attn.q_proj.weight"))
774 .ok_or_else(|| anyhow!("Missing q_proj weight"))?
775 .shallow_clone();
776 let k_proj = weights
777 .get(&format!("{prefix}.self_attn.k_proj.weight"))
778 .ok_or_else(|| anyhow!("Missing k_proj weight"))?
779 .shallow_clone();
780 let v_proj = weights
781 .get(&format!("{prefix}.self_attn.v_proj.weight"))
782 .ok_or_else(|| anyhow!("Missing v_proj weight"))?
783 .shallow_clone();
784 let o_proj = weights
785 .get(&format!("{prefix}.self_attn.o_proj.weight"))
786 .ok_or_else(|| anyhow!("Missing o_proj weight"))?
787 .shallow_clone();
788
789 let q_norm = weights
791 .get(&format!("{prefix}.self_attn.q_norm.weight"))
792 .map(tch::Tensor::shallow_clone);
793 let k_norm = weights
794 .get(&format!("{prefix}.self_attn.k_norm.weight"))
795 .map(tch::Tensor::shallow_clone);
796
797 let layer_type = if !config.layer_types.is_empty() && layer_idx < config.layer_types.len() {
799 config.layer_types[layer_idx].clone()
801 } else if config.sliding_window.is_some() {
802 if (layer_idx + 1).is_multiple_of(6) {
804 "global".to_owned()
805 } else {
806 "local".to_owned()
807 }
808 } else {
809 "global".to_owned()
810 };
811
812 let rope_theta = if layer_type == "local" {
814 config.rope_local_base_freq
815 } else {
816 config.rope_theta
817 };
818
819 let sliding_window = if layer_type == "local" {
820 config.sliding_window
821 } else {
822 None
823 };
824
825 let self_attn = GemmaAttention {
826 q_proj,
827 k_proj,
828 v_proj,
829 o_proj,
830 num_heads: config.num_attention_heads,
831 num_kv_heads: config.num_key_value_heads,
832 head_dim: config.head_dim,
833 rope_theta,
834 q_norm,
835 k_norm,
836 query_pre_attn_scalar: config.query_pre_attn_scalar,
837 sliding_window,
838 layer_type,
839 };
840
841 let mlp = GemmaMLP {
843 gate_proj: weights
844 .get(&format!("{prefix}.mlp.gate_proj.weight"))
845 .ok_or_else(|| anyhow!("Missing gate_proj weight"))?
846 .shallow_clone(),
847 up_proj: weights
848 .get(&format!("{prefix}.mlp.up_proj.weight"))
849 .ok_or_else(|| anyhow!("Missing up_proj weight"))?
850 .shallow_clone(),
851 down_proj: weights
852 .get(&format!("{prefix}.mlp.down_proj.weight"))
853 .ok_or_else(|| anyhow!("Missing down_proj weight"))?
854 .shallow_clone(),
855 activation: config.hidden_activation.clone(),
856 };
857
858 let is_gemma3 = config.query_pre_attn_scalar.is_some();
861
862 let input_layernorm = RMSNorm {
863 weight: weights
864 .get(&format!("{prefix}.input_layernorm.weight"))
865 .ok_or_else(|| anyhow!("Missing input_layernorm weight"))?
866 .shallow_clone(),
867 eps: config.rms_norm_eps,
868 add_unit_offset: is_gemma3,
869 };
870
871 let post_attention_layernorm = RMSNorm {
872 weight: weights
873 .get(&format!("{prefix}.post_attention_layernorm.weight"))
874 .ok_or_else(|| anyhow!("Missing post_attention_layernorm weight"))?
875 .shallow_clone(),
876 eps: config.rms_norm_eps,
877 add_unit_offset: is_gemma3,
878 };
879
880 Ok(Some(GemmaLayer {
881 self_attn,
882 mlp,
883 input_layernorm,
884 post_attention_layernorm,
885 }))
886 }
887
888 fn parse_config(config_str: &str) -> Result<GemmaConfig> {
890 let config: serde_json::Value = serde_json::from_str(config_str)?;
891
892 let layer_types = config
894 .get("layer_types")
895 .and_then(|v| v.as_array())
896 .map(|arr| {
897 arr.iter()
898 .map(|v| match v.as_str().unwrap_or("") {
899 "sliding_attention" => "local".to_owned(),
900 _ => "global".to_owned(),
902 })
903 .collect()
904 })
905 .unwrap_or_default();
906
907 let query_pre_attn_scalar = config.get("query_pre_attn_scalar").and_then(|v| {
909 if let Some(int_val) = v.as_u64() {
910 Some(int_val as f32)
911 } else {
912 v.as_f64().map(|f| f as f32)
913 }
914 });
915
916 Ok(GemmaConfig {
917 num_attention_heads: config
918 .get("num_attention_heads")
919 .and_then(serde_json::Value::as_u64)
920 .unwrap_or(4) as usize,
921 num_key_value_heads: config
922 .get("num_key_value_heads")
923 .and_then(serde_json::Value::as_u64)
924 .unwrap_or(1) as usize,
925 hidden_size: config
926 .get("hidden_size")
927 .and_then(serde_json::Value::as_u64)
928 .unwrap_or(640) as usize,
929 head_dim: config
930 .get("head_dim")
931 .and_then(serde_json::Value::as_u64)
932 .unwrap_or(256) as usize,
933 intermediate_size: config
934 .get("intermediate_size")
935 .and_then(serde_json::Value::as_u64)
936 .unwrap_or(2048) as usize,
937 max_position_embeddings: config
938 .get("max_position_embeddings")
939 .and_then(serde_json::Value::as_u64)
940 .unwrap_or(32768) as usize,
941 rms_norm_eps: config
942 .get("rms_norm_eps")
943 .and_then(serde_json::Value::as_f64)
944 .unwrap_or(1e-6) as f32,
945 vocab_size: config
946 .get("vocab_size")
947 .and_then(serde_json::Value::as_u64)
948 .unwrap_or(262144) as usize,
949 num_hidden_layers: config
950 .get("num_hidden_layers")
951 .and_then(serde_json::Value::as_u64)
952 .unwrap_or(18) as usize,
953 rope_theta: config
954 .get("rope_theta")
955 .and_then(serde_json::Value::as_f64)
956 .unwrap_or(1000000.0) as f32,
957 rope_local_base_freq: config
958 .get("rope_local_base_freq")
959 .and_then(serde_json::Value::as_f64)
960 .unwrap_or(10000.0) as f32,
961 sliding_window: config
962 .get("sliding_window")
963 .and_then(serde_json::Value::as_u64)
964 .map(|v| v as usize),
965 layer_types,
966 query_pre_attn_scalar,
967 hidden_activation: config
968 .get("hidden_activation")
969 .and_then(|v| v.as_str())
970 .unwrap_or("silu")
971 .to_owned(),
972 })
973 }
974}
975
976impl ModelOperations for GemmaModel {
977 fn architecture(&self) -> ModelArchitecture {
978 ModelArchitecture::Gemma
979 }
980
981 fn config(&self) -> &dyn ArchitectureConfig {
982 &self.config
983 }
984
985 fn forward(&self, input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor> {
986 let mut hidden_states = if let Some(embed) = &self.embed_tokens {
988 let input_shape = input.size();
991 let batch_size = input_shape[0];
992 let seq_len = if input_shape.len() > 1 {
993 input_shape[1]
994 } else {
995 1
996 };
997
998 let flat_input = input.flatten(0, -1);
1000
1001 let embeddings = embed.index_select(0, &flat_input);
1003
1004 let emb_dims = embeddings.size();
1006 let hidden_size = emb_dims[emb_dims.len() - 1]; let mut embeddings = embeddings.reshape([batch_size, seq_len, hidden_size]);
1010
1011 let scale = (hidden_size as f32).sqrt();
1013 let scale_tensor = Tensor::from(scale)
1014 .to_kind(embeddings.kind())
1015 .to_device(embeddings.device());
1016 embeddings = broadcast_mul(&embeddings, &scale_tensor)?;
1017 tracing::debug!(
1018 "Applied Gemma embedding scaling by sqrt(hidden_size={}) = {}",
1019 hidden_size,
1020 scale
1021 );
1022
1023 embeddings
1024 } else {
1025 input.shallow_clone()
1027 };
1028
1029 for layer in &self.layers {
1031 let residual = hidden_states.shallow_clone();
1032
1033 hidden_states = layer.input_layernorm.forward(&hidden_states)?;
1035 let attn_output = layer.self_attn.forward(&hidden_states, None)?;
1036 hidden_states = residual + attn_output;
1037
1038 let residual = hidden_states.shallow_clone();
1040 hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
1041 let ffn_output = layer.mlp.forward(&hidden_states)?;
1042 hidden_states = residual + ffn_output;
1043 }
1044
1045 if let Some(norm) = &self.norm {
1047 hidden_states = norm.forward(&hidden_states)?;
1048 }
1049
1050 if let Some(lm_head) = &self.lm_head {
1052 let (batch_size, seq_len, hidden_size) = dims3(&hidden_states)?;
1055 let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
1056 let logits = hidden_2d.matmul(&lm_head.transpose(0, 1));
1057 let vocab_size = logits.size()[1];
1058 hidden_states = logits.reshape([batch_size, seq_len, vocab_size]);
1059 } else if let Some(embed) = &self.embed_tokens {
1060 let (batch_size, seq_len, hidden_size) = dims3(&hidden_states)?;
1063 let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
1064
1065 let logits = hidden_2d.matmul(&embed.transpose(0, 1));
1067 let vocab_size = logits.size()[1];
1068 hidden_states = logits.reshape([batch_size, seq_len, vocab_size]);
1069 }
1070
1071 Ok(hidden_states)
1072 }
1073
1074 fn reshape_for_attention(&self, tensor: &Tensor, is_key_value: bool) -> Result<Tensor> {
1075 let (batch_size, seq_len, _hidden_size) = dims3(tensor)?;
1076
1077 if is_key_value {
1078 Ok(tensor.reshape([
1081 batch_size,
1082 seq_len,
1083 self.config.num_key_value_heads as i64,
1084 self.config.head_dim as i64,
1085 ]))
1086 } else {
1087 Ok(tensor.reshape([
1089 batch_size,
1090 seq_len,
1091 self.config.num_attention_heads as i64,
1092 self.config.head_dim as i64,
1093 ]))
1094 }
1095 }
1096
1097 fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor> {
1098 let mut rope = crate::runtime::rope::RoPE::new_standard_with_dtype(
1100 self.config.head_dim as i64,
1101 8192, tensor.device(),
1103 tensor.kind(), )?;
1105 rope.forward(tensor, Some(position_ids))
1106 }
1107
1108 fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
1109 let x2 = square_tensor(tensor)?;
1111 let mean = x2.mean_dim(&[-1i64][..], true, tensor.kind()); let rrms = (mean + self.config.rms_norm_eps as f64).reciprocal().sqrt();
1113 broadcast_mul(tensor, &rrms)
1114 }
1115
1116 fn get_attention_mask(&self, seq_len: usize, past_kv_len: usize) -> Result<Tensor> {
1117 let total_len = seq_len + past_kv_len;
1119 let mask = Tensor::ones(
1120 [seq_len as i64, total_len as i64],
1121 (DType::BFloat16, self.device),
1122 );
1123
1124 Ok(mask)
1127 }
1128
1129 fn as_any(&self) -> &dyn std::any::Any {
1130 self
1131 }
1132}
1133
1134impl GemmaModel {
1135 pub fn from_varstore(vs: &tch::nn::VarStore, device: &Device, dtype: DType) -> Result<Self> {
1137 let variables = vs.variables();
1138
1139 if variables.is_empty() {
1140 return Err(anyhow!("No tensors found in VarStore"));
1141 }
1142
1143 info!("Loading {} model tensors", variables.len());
1144
1145 let mut weights = HashMap::new();
1147 for (name, var) in variables.iter() {
1148 weights.insert(name.clone(), var.shallow_clone());
1149 }
1150
1151 if let Some(embed) = weights.get("model.embed_tokens.weight") {
1153 let shape = embed.size();
1154 info!(
1155 "📝 Found embeddings: {} -> [{}, {}]",
1156 "model.embed_tokens.weight", shape[0], shape[1]
1157 );
1158 }
1159
1160 let layer_count = weights
1162 .keys()
1163 .filter(|k| k.contains("layers.") && k.contains(".input_layernorm.weight"))
1164 .count();
1165 info!("📝 Found {} transformer layers", layer_count);
1166
1167 Self::from_weights(&weights, device, dtype)
1169 }
1170
1171 pub fn load_weights_from_varstore(&mut self, vs: &tch::nn::VarStore) -> Result<()> {
1173 let variables = vs.variables();
1174
1175 info!(
1176 "🔍 Validating {} loaded tensors for Gemma architecture",
1177 variables.len()
1178 );
1179
1180 if variables.is_empty() {
1182 return Err(anyhow!("No tensors found in VarStore"));
1183 }
1184
1185 info!("✅ Gemma architecture ready (use from_varstore for proper loading)");
1186 Ok(())
1187 }
1188
1189 pub fn simple_forward(&self, input: &Tensor) -> Result<Tensor> {
1191 let input_shape = input.size();
1192 let batch_size = input_shape[0];
1193 let seq_len = input_shape[1];
1194
1195 let logits = Tensor::randn(
1197 [batch_size, seq_len, self.config.vocab_size as i64],
1198 (DType::Float, input.device()),
1199 );
1200
1201 info!(
1202 "🔍 Gemma simple forward: [{}, {}] → [{}, {}, {}]",
1203 batch_size, seq_len, batch_size, seq_len, self.config.vocab_size
1204 );
1205
1206 Ok(logits)
1207 }
1208}
1209
1210#[cfg(test)]
1212#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
1213mod tests {
1214 use super::*;
1215
1216 #[test]
1217 fn test_gemma_reshape_for_mqa() {
1218 let device = Device::Cpu;
1219 let config = GemmaConfig::default();
1220 let model = GemmaModel {
1221 config: config.clone(),
1222 device,
1223 dtype: DType::Float,
1224 embed_tokens: None,
1225 layers: vec![],
1226 norm: None,
1227 lm_head: None,
1228 };
1229
1230 let kv_tensor = Tensor::randn([1, 21, 1024], (DType::Float, device));
1232
1233 let reshaped = model
1235 .reshape_for_attention(&kv_tensor, true)
1236 .expect("test: reshape kv");
1237 assert_eq!(reshaped.size(), &[1, 21, 4, 256]); let q_tensor = Tensor::randn([1, 21, 4096], (DType::Float, device));
1241
1242 let reshaped = model
1244 .reshape_for_attention(&q_tensor, false)
1245 .expect("test: reshape q");
1246 assert_eq!(reshaped.size(), &[1, 21, 16, 256]); }
1248
1249 #[test]
1250 fn test_kv_expansion_for_mqa() {
1251 let device = Device::Cpu;
1252 let attn = GemmaAttention {
1253 q_proj: Tensor::zeros([1024, 4096], (DType::Float, device)),
1254 k_proj: Tensor::zeros([1024, 1024], (DType::Float, device)),
1255 v_proj: Tensor::zeros([1024, 1024], (DType::Float, device)),
1256 o_proj: Tensor::zeros([4096, 1024], (DType::Float, device)),
1257 num_heads: 16,
1258 num_kv_heads: 4,
1259 head_dim: 256,
1260 rope_theta: 10000.0,
1261 q_norm: None,
1262 k_norm: None,
1263 query_pre_attn_scalar: None,
1264 sliding_window: None,
1265 layer_type: "full_attention".to_owned(),
1266 };
1267
1268 let kv = Tensor::randn([2, 10, 4, 256], (DType::Float, device));
1270
1271 let expanded = attn
1273 .expand_kv_for_mqa(&kv)
1274 .expect("test: expand kv for mqa");
1275 assert_eq!(expanded.size(), &[2, 10, 16, 256]);
1276 }
1277}