1use super::{ArchitectureConfig, ModelArchitecture, ModelOperations};
4use crate::runtime::device_pool::LayerDeviceMap;
6use crate::runtime::rope::RoPE;
7use crate::runtime::tensor_helpers::{
8 broadcast_add, broadcast_mul, dims3, dims4, scalar_tensor, square_tensor,
9};
10use anyhow::{anyhow, Result};
11use std::collections::HashMap;
12use std::path::Path;
13use tch::{nn, Device, Kind as DType, Tensor};
14
15thread_local! {
29 static ROPE_CACHE: std::cell::RefCell<HashMap<(usize, i64, u32), RoPE>> =
30 std::cell::RefCell::new(HashMap::new());
31}
32
33pub(crate) fn clear_rope_cache() {
39 ROPE_CACHE.with(|cache| cache.borrow_mut().clear());
40}
41
42pub(crate) struct LinearProjection {
49 pub(crate) weight: Tensor,
50 pub(crate) bias: Option<Tensor>,
51 pub(crate) scale: Option<Tensor>,
54 pub(crate) scale_v2: Option<Tensor>,
63 pub(crate) rowwise: Option<Box<RowwiseRequant>>,
73 pub(crate) channel_scale: Option<Tensor>,
75 pub(crate) children: Option<Vec<LinearProjection>>,
77}
78
79pub(crate) struct RowwiseRequant {
85 weight: Tensor,
86 scale: Tensor,
87}
88
89impl RowwiseRequant {
90 fn new(weight: Tensor, scale: Tensor) -> Self {
91 #[cfg(test)]
92 {
93 let live = ROWWISE_REQUANT_LIVE.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
94 ROWWISE_REQUANT_PEAK.fetch_max(live, std::sync::atomic::Ordering::SeqCst);
95 }
96 Self { weight, scale }
97 }
98}
99
100impl Drop for RowwiseRequant {
101 fn drop(&mut self) {
102 #[cfg(test)]
103 {
104 ROWWISE_REQUANT_LIVE.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
105 }
106 }
107}
108
109#[cfg(test)]
113static ROWWISE_REQUANT_LIVE: std::sync::atomic::AtomicUsize =
114 std::sync::atomic::AtomicUsize::new(0);
115#[cfg(test)]
116static ROWWISE_REQUANT_PEAK: std::sync::atomic::AtomicUsize =
117 std::sync::atomic::AtomicUsize::new(0);
118#[cfg(test)]
119pub(crate) fn reset_rowwise_requant_peak() {
120 ROWWISE_REQUANT_PEAK.store(
121 ROWWISE_REQUANT_LIVE.load(std::sync::atomic::Ordering::SeqCst),
122 std::sync::atomic::Ordering::SeqCst,
123 );
124}
125#[cfg(test)]
126pub(crate) fn rowwise_requant_live() -> usize {
127 ROWWISE_REQUANT_LIVE.load(std::sync::atomic::Ordering::SeqCst)
128}
129#[cfg(test)]
130pub(crate) fn rowwise_requant_peak() -> usize {
131 ROWWISE_REQUANT_PEAK.load(std::sync::atomic::Ordering::SeqCst)
132}
133
134fn fp8_gemm_enabled() -> bool {
141 static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
142 *ENABLED.get_or_init(crate::config::default_fp8_gemm)
143}
144
145#[cfg(test)]
149static FP8_APPLY_PATH: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
150
151fn quantize_activations_1x128(x: &Tensor) -> (Tensor, Tensor) {
164 const FP8_MAX: f64 = 448.0; let x = x.contiguous();
166 let size = x.size();
167 let (m, k) = (size[0], size[1]);
168 let blocks = k / 128;
169 let x3 = x.to_kind(tch::Kind::Float).view([m, blocks, 128]);
170 let amax = x3.abs().amax(-1, false); let scale = (amax / FP8_MAX).clamp_min(1e-12);
172 let q = (x3 / scale.unsqueeze(-1))
173 .clamp(-FP8_MAX, FP8_MAX)
174 .to_kind(tch::Kind::Float8e4m3fn)
175 .view([m, k]);
176 (q, scale)
177}
178
179fn quantize_activations_rowwise(x: &Tensor) -> (Tensor, Tensor) {
184 const FP8_MAX: f64 = 448.0; let x = x.contiguous();
186 let xf = x.to_kind(tch::Kind::Float);
187 let scale = (xf.abs().amax(-1, false) / FP8_MAX)
188 .clamp_min(1e-12)
189 .unsqueeze(-1)
190 .contiguous(); let q = (xf / &scale)
192 .clamp(-FP8_MAX, FP8_MAX)
193 .to_kind(tch::Kind::Float8e4m3fn);
194 (q, scale)
195}
196
197fn build_rowwise_requant(weight: &Tensor, scale: &Tensor) -> RowwiseRequant {
206 const FP8_MAX: f64 = 448.0;
207 let ws = weight.size(); let ss = scale.size(); let (br, bc) = (ws[0] / ss[0], ws[1] / ss[1]);
210 let w_f = (weight
211 .to_kind(tch::Kind::Float)
212 .view([ss[0], br, ss[1], bc])
213 * scale.to_kind(tch::Kind::Float).view([ss[0], 1, ss[1], 1]))
214 .reshape([ws[0], ws[1]]);
215 let s_b = (w_f.abs().amax(0, false) / FP8_MAX).clamp_min(1e-12); let w_rq = (w_f / s_b.unsqueeze(0))
217 .clamp(-FP8_MAX, FP8_MAX)
218 .to_kind(tch::Kind::Float8e4m3fn)
219 .contiguous();
220 RowwiseRequant::new(w_rq, s_b.view([1, ws[1]]).contiguous())
221}
222
223const RECIPE_BLOCKWISE_1X128: i64 = 4;
226const RECIPE_BLOCKWISE_128X128: i64 = 5;
227const NO_SWIZZLE: i64 = 0;
228
229#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
233enum ScaledMmRecipe {
234 BlockwiseV2,
236 RowwiseV1,
240}
241
242#[derive(Default)]
253struct DeviceLatch(parking_lot::Mutex<std::collections::HashSet<(usize, ScaledMmRecipe)>>);
254
255impl DeviceLatch {
256 fn is_failed(&self, dev: usize, recipe: ScaledMmRecipe) -> bool {
258 self.0.lock().contains(&(dev, recipe))
259 }
260
261 fn mark_failed(&self, dev: usize, recipe: ScaledMmRecipe) -> bool {
265 self.0.lock().insert((dev, recipe))
266 }
267}
268
269static SCALED_MM_FAILED: std::sync::OnceLock<DeviceLatch> = std::sync::OnceLock::new();
270
271fn scaled_mm_latch() -> &'static DeviceLatch {
272 SCALED_MM_FAILED.get_or_init(DeviceLatch::default)
273}
274
275fn pack_scale_b_v2(scale: &Tensor) -> Tensor {
279 let size = scale.size();
280 let kb = size[0];
281 let l4 = (kb + 3) / 4 * 4; let t = scale.to_kind(tch::Kind::Float).transpose(0, 1).contiguous();
284 if l4 == kb {
285 t
286 } else {
287 t.constant_pad_nd([0, l4 - kb]) }
289 .contiguous()
290}
291
292impl LinearProjection {
293 pub(crate) fn apply_fp8_scaled_mm(&self, input: &Tensor) -> Option<Tensor> {
325 if self.weight.kind() != tch::Kind::Float8e4m3fn {
326 return None;
327 }
328 let Device::Cuda(dev_idx) = self.weight.device() else {
329 return None;
330 };
331 let in_dims = input.size();
332 let k = *in_dims.last()?;
333 if k % 128 != 0 {
334 return None;
335 }
336 let m: i64 = in_dims[..in_dims.len() - 1].iter().product();
337 let out = self
338 .try_blockwise_v2(dev_idx, input, m, k)
339 .or_else(|| self.try_rowwise_v1(dev_idx, input, m, k))?;
340 let mut out_dims = in_dims;
341 *out_dims.last_mut()? = self.weight.size()[1];
342 Some(out.reshape(out_dims))
343 }
344
345 fn try_blockwise_v2(&self, dev_idx: usize, input: &Tensor, m: i64, k: i64) -> Option<Tensor> {
359 let scale_v2 = self.scale_v2.as_ref()?;
360 if scaled_mm_latch().is_failed(dev_idx, ScaledMmRecipe::BlockwiseV2) {
361 return None;
362 }
363 let x2 = input.reshape([m, k]);
364 let (q, scale_a) = quantize_activations_1x128(&x2);
365 let mat2 = self.weight.transpose(0, 1).contiguous().transpose(0, 1);
367 let scale_a = scale_a.transpose(0, 1).contiguous().transpose(0, 1);
369 let scale_b = scale_v2.transpose(0, 1);
371 let scale_a_refs = [&scale_a];
372 let scale_b_refs = [&scale_b];
373 let bias = self.bias.as_ref().map(|b| b.to_kind(tch::Kind::BFloat16));
374 q.f_internal_scaled_mm_v2(
375 &mat2,
376 &scale_a_refs,
377 [RECIPE_BLOCKWISE_1X128],
378 [NO_SWIZZLE],
379 &scale_b_refs,
380 [RECIPE_BLOCKWISE_128X128],
381 [NO_SWIZZLE],
382 bias.as_ref(),
383 tch::Kind::BFloat16,
384 [] as [i64; 0],
385 false,
386 )
387 .map_err(|e| {
388 if scaled_mm_latch().mark_failed(dev_idx, ScaledMmRecipe::BlockwiseV2) {
389 tracing::warn!(
390 "FP8 _scaled_mm_v2 (1x128 x 128x128) failed on CUDA device {dev_idx} \
391 (latched off for this device; trying rowwise/lazy fallback): {e}"
392 );
393 }
394 e
395 })
396 .ok()
397 }
398
399 fn try_rowwise_v1(&self, dev_idx: usize, input: &Tensor, m: i64, k: i64) -> Option<Tensor> {
418 let rowwise = self.rowwise.as_deref()?;
419 if scaled_mm_latch().is_failed(dev_idx, ScaledMmRecipe::RowwiseV1) {
420 return None;
421 }
422 let x2 = input.reshape([m, k]);
423 let (q, scale_a) = quantize_activations_rowwise(&x2);
424 let mat2 = rowwise.weight.transpose(0, 1).contiguous().transpose(0, 1);
426 let bias = self.bias.as_ref().map(|b| b.to_kind(tch::Kind::BFloat16));
427 q.f_internal_scaled_mm(
428 &mat2,
429 &scale_a,
430 &rowwise.scale,
431 bias.as_ref(),
432 None::<&Tensor>,
433 tch::Kind::BFloat16,
434 false,
435 )
436 .map_err(|e| {
437 if scaled_mm_latch().mark_failed(dev_idx, ScaledMmRecipe::RowwiseV1) {
438 tracing::warn!(
439 "FP8 rowwise _scaled_mm failed on CUDA device {dev_idx} \
440 (latched off for this device; using lazy dequant there): {e}"
441 );
442 }
443 e
444 })
445 .ok()
446 }
447}
448
449impl LinearProjection {
450 #[inline]
452 pub(crate) fn new(weight: Tensor) -> Self {
453 Self {
454 weight,
455 bias: None,
456 scale: None,
457 scale_v2: None,
458 rowwise: None,
459 channel_scale: None,
460 children: None,
461 }
462 }
463
464 #[inline]
466 pub(crate) fn with_bias(weight: Tensor, bias: Tensor) -> Self {
467 Self {
468 weight,
469 bias: Some(bias),
470 scale: None,
471 scale_v2: None,
472 rowwise: None,
473 channel_scale: None,
474 children: None,
475 }
476 }
477
478 #[inline]
488 pub(crate) fn with_scale(weight: Tensor, scale: Tensor) -> Self {
489 let scale_v2 = pack_scale_b_v2(&scale);
490 let rowwise = if fp8_gemm_enabled() && weight.kind() == tch::Kind::Float8e4m3fn {
491 Some(Box::new(build_rowwise_requant(&weight, &scale)))
492 } else {
493 None
494 };
495 Self {
496 weight,
497 bias: None,
498 scale: Some(scale),
499 scale_v2: Some(scale_v2),
500 rowwise,
501 channel_scale: None,
502 children: None,
503 }
504 }
505
506 #[inline]
509 pub(crate) fn into_device(self, device: Device) -> Self {
510 Self {
511 channel_scale: self.channel_scale.map(|s| s.to_device(device)),
512 children: self
513 .children
514 .map(|ps| ps.into_iter().map(|p| p.into_device(device)).collect()),
515 weight: self.weight.to_device(device),
516 bias: self.bias.map(|b| b.to_device(device)),
517 scale: self.scale.map(|s| s.to_device(device)),
518 scale_v2: self.scale_v2.map(|s| s.to_device(device)),
519 rowwise: self.rowwise.map(|bs| {
520 Box::new(RowwiseRequant::new(
521 bs.weight.to_device(device),
522 bs.scale.to_device(device),
523 ))
524 }),
525 }
526 }
527
528 #[inline]
538 pub(crate) fn apply(&self, input: &Tensor) -> Tensor {
539 if self.children.is_some() || self.channel_scale.is_some() {
540 return self
541 .try_apply(input)
542 .unwrap_or_else(|e| panic!("legacy infallible projection: {e}"));
543 }
544 let fp8_out = if fp8_gemm_enabled() && !input.requires_grad() {
551 self.apply_fp8_scaled_mm(input)
552 } else {
553 None
554 };
555 if let Some(out) = fp8_out {
556 #[cfg(test)]
557 FP8_APPLY_PATH.store(1, std::sync::atomic::Ordering::SeqCst);
558 return out;
559 }
560 #[cfg(test)]
561 FP8_APPLY_PATH.store(2, std::sync::atomic::Ordering::SeqCst);
562 let w = match self.weight.kind() {
563 tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
564 let w_bf16 = self.weight.to_kind(input.kind());
566 if let Some(scale) = &self.scale {
567 let ws = w_bf16.size(); let ss = scale.size(); let block_r = ws[0] / ss[0];
573 let block_c = ws[1] / ss[1];
574 let w_4d = w_bf16.view([ss[0], block_r, ss[1], block_c]);
580 let s_4d = scale.to_kind(input.kind()).view([ss[0], 1, ss[1], 1]);
581 (w_4d * s_4d).reshape([ws[0], ws[1]])
582 } else {
583 w_bf16
584 }
585 }
586 _ => self.weight.shallow_clone(),
587 };
588 let output = input.matmul(&w);
589 match &self.bias {
590 Some(bias) => output + bias,
591 None => output,
592 }
593 }
594
595 pub(crate) fn try_apply(&self, input: &Tensor) -> Result<Tensor> {
600 if let Some(children) = &self.children {
601 let outputs = children
602 .iter()
603 .map(|p| p.try_apply(input))
604 .collect::<Result<Vec<_>>>()?;
605 let output = Tensor::cat(&outputs, -1);
606 return Ok(match &self.bias {
607 Some(bias) => output + bias,
608 None => output,
609 });
610 }
611 if let Some(scale) = &self.channel_scale {
612 let output = super::super::channel_fp8::channel_matmul(input, &self.weight, scale)
613 .map_err(|e| super::super::channel_fp8::Fp8ExecutionError(e.to_string()))?;
614 return Ok(match &self.bias {
615 Some(bias) => output + bias,
616 None => output,
617 });
618 }
619 Ok(self.apply(input))
620 }
621
622 pub(crate) fn take(weights: &mut HashMap<String, Tensor>, key: &str) -> Result<Self> {
629 if let Some(scale) = weights.remove(&format!("{key}_scale")) {
630 let weight = weights
631 .remove(key)
632 .ok_or_else(|| anyhow!("Missing FP8 weight: {key}"))?;
633 super::super::channel_fp8::validate_channel_scale(&weight, &scale)?;
634 anyhow::ensure!(
635 !weights.contains_key(&format!("{key}_scale_inv")),
636 "ambiguous FP8 scale: {key}"
637 );
638 let mut p = Self::new(weight.transpose(0, 1));
639 p.channel_scale = Some(scale.transpose(0, 1));
640 return Ok(p);
641 }
642 let (weight, scale) = Self::take_weight_and_scale(weights, key)?;
643 Ok(match scale {
644 Some(s) => Self::with_scale(weight, s),
645 None => Self::new(weight),
646 })
647 }
648
649 pub(crate) fn take_weight_and_scale(
653 weights: &mut HashMap<String, Tensor>,
654 key: &str,
655 ) -> Result<(Tensor, Option<Tensor>)> {
656 anyhow::ensure!(
657 !weights.contains_key(&format!("{key}_scale")),
658 "channel FP8 cannot enter block-only expert loader: {key}"
659 );
660 let weight = weights
661 .remove(key)
662 .ok_or_else(|| anyhow!("Missing weight tensor: {}", key))?
663 .transpose(0, 1)
664 .contiguous();
665 let scale = weights
667 .remove(&format!("{key}_scale_inv"))
668 .map(|s| s.transpose(0, 1).contiguous());
669 if matches!(
670 weight.kind(),
671 tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2
672 ) {
673 anyhow::ensure!(scale.is_some(), "missing FP8 scale for {key}");
674 let ss = scale.as_ref().map(Tensor::size).unwrap_or_default();
675 let ws = weight.size();
676 anyhow::ensure!(
677 ss.len() == 2
678 && ss.iter().all(|d| *d > 0)
679 && ws.len() == 2
680 && ws[0] % ss[0] == 0
681 && ws[1] % ss[1] == 0,
682 "invalid FP8 block scale for {key}"
683 );
684 }
685 Ok((weight, scale))
686 }
687
688 pub(crate) fn take_with_optional_bias(
691 weights: &mut HashMap<String, Tensor>,
692 weight_key: &str,
693 bias_key: &str,
694 ) -> Result<Self> {
695 let mut proj = Self::take(weights, weight_key)?;
696 if let Some(bias) = weights.get(bias_key).map(tch::Tensor::shallow_clone) {
697 proj.bias = Some(bias);
698 }
699 Ok(proj)
700 }
701
702 #[inline]
704 #[allow(dead_code)]
705 fn out_features(&self) -> i64 {
706 self.weight.size()[1]
707 }
708}
709
710unsafe impl Send for LinearProjection {}
711unsafe impl Sync for LinearProjection {}
712
713#[inline]
715fn calculate_padded_vocab_size(vocab_size: u32) -> u32 {
716 vocab_size.div_ceil(64) * 64
717}
718
719#[derive(Debug, Clone)]
721pub struct LlamaConfig {
722 pub version: u8,
724 pub num_attention_heads: u32,
726 pub num_key_value_heads: u32,
728 pub hidden_size: u32,
730 pub head_dim: u32,
732 pub intermediate_size: u32,
734 pub max_position_embeddings: u32,
736 pub rms_norm_eps: f32,
738 pub vocab_size: u32,
740 pub original_vocab_size: u32,
742 pub num_hidden_layers: u32,
744 pub rope_theta: f32,
746 pub rope_scaling: Option<RopeScaling>,
748 pub hidden_activation: String,
750 pub query_pre_attn_scalar: Option<f32>,
752 pub use_qk_norm: bool,
754 pub scale_embeddings: bool,
756 pub layer_types: Vec<String>,
758 pub rope_local_base_freq: Option<f32>,
760 pub model_architecture: ModelArchitecture,
771}
772
773#[derive(Debug, Clone)]
775pub struct RopeScaling {
776 pub scaling_type: String,
777 pub factor: f32,
778}
779
780impl Default for LlamaConfig {
781 fn default() -> Self {
782 let vocab_size = 32000;
784 Self {
785 version: 2,
786 num_attention_heads: 32,
787 num_key_value_heads: 32, hidden_size: 4096,
789 head_dim: 128,
790 intermediate_size: 11008,
791 max_position_embeddings: 4096,
792 rms_norm_eps: 1e-5,
793 vocab_size,
794 original_vocab_size: vocab_size, num_hidden_layers: 32,
796 rope_theta: 10000.0,
797 rope_scaling: None,
798 hidden_activation: "silu".to_owned(),
799 query_pre_attn_scalar: None,
800 use_qk_norm: false,
801 scale_embeddings: false,
802 layer_types: vec![],
803 rope_local_base_freq: None,
804 model_architecture: ModelArchitecture::Llama { version: 2 },
805 }
806 }
807}
808
809impl LlamaConfig {
810 pub fn llama3_8b() -> Self {
812 let vocab_size = 128256;
813 Self {
814 version: 3,
815 num_attention_heads: 32,
816 num_key_value_heads: 8, hidden_size: 4096,
818 head_dim: 128,
819 intermediate_size: 14336,
820 max_position_embeddings: 8192,
821 rms_norm_eps: 1e-5,
822 vocab_size,
823 original_vocab_size: vocab_size,
824 num_hidden_layers: 32,
825 rope_theta: 500000.0,
826 rope_scaling: Some(RopeScaling {
827 scaling_type: "linear".to_owned(),
828 factor: 8.0,
829 }),
830 hidden_activation: "silu".to_owned(),
831 query_pre_attn_scalar: None,
832 use_qk_norm: false,
833 scale_embeddings: false,
834 layer_types: vec![],
835 rope_local_base_freq: None,
836 model_architecture: ModelArchitecture::Llama { version: 3 },
837 }
838 }
839
840 pub fn llama3_70b() -> Self {
842 let vocab_size = 128256;
843 Self {
844 version: 3,
845 num_attention_heads: 64,
846 num_key_value_heads: 8, hidden_size: 8192,
848 head_dim: 128,
849 intermediate_size: 28672,
850 max_position_embeddings: 8192,
851 rms_norm_eps: 1e-5,
852 vocab_size,
853 original_vocab_size: vocab_size,
854 num_hidden_layers: 80,
855 rope_theta: 500000.0,
856 rope_scaling: Some(RopeScaling {
857 scaling_type: "linear".to_owned(),
858 factor: 8.0,
859 }),
860 hidden_activation: "silu".to_owned(),
861 query_pre_attn_scalar: None,
862 use_qk_norm: false,
863 scale_embeddings: false,
864 layer_types: vec![],
865 rope_local_base_freq: None,
866 model_architecture: ModelArchitecture::Llama { version: 3 },
867 }
868 }
869}
870
871impl ArchitectureConfig for LlamaConfig {
872 fn num_attention_heads(&self) -> usize {
873 self.num_attention_heads as usize
874 }
875
876 fn num_key_value_heads(&self) -> usize {
877 self.num_key_value_heads as usize
878 }
879
880 fn hidden_size(&self) -> usize {
881 self.hidden_size as usize
882 }
883
884 fn intermediate_size(&self) -> usize {
885 self.intermediate_size as usize
886 }
887
888 fn head_dim(&self) -> usize {
889 self.head_dim as usize
890 }
891
892 fn vocab_size(&self) -> usize {
893 self.vocab_size as usize
894 }
895
896 fn max_position_embeddings(&self) -> usize {
897 self.max_position_embeddings as usize
898 }
899
900 fn rope_theta(&self) -> Option<f32> {
901 Some(self.rope_theta)
902 }
903
904 fn rope_dim(&self) -> Option<usize> {
905 Some(self.head_dim as usize)
906 }
907
908 fn layer_norm_eps(&self) -> f32 {
909 self.rms_norm_eps
910 }
911
912 fn use_rms_norm(&self) -> bool {
913 true }
915}
916
917pub struct LlamaModel {
919 config: LlamaConfig,
920 #[allow(dead_code)]
921 device: Device,
922 #[allow(dead_code)]
923 dtype: DType,
924
925 device_map: LayerDeviceMap,
932
933 layer_offset: usize,
938
939 embed_tokens: Option<Tensor>,
941 layers: Vec<LlamaLayer>,
942 norm: Option<RMSNorm>,
943 lm_head: Option<Tensor>,
944
945 lm_head_transposed: Option<Tensor>,
948
949 kv_cache: Option<std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>>,
951
952 vs: Option<nn::VarStore>,
954}
955
956unsafe impl Send for LlamaModel {}
959unsafe impl Sync for LlamaModel {}
960
961struct LlamaLayer {
963 self_attn: LlamaAttention,
964 mlp: LlamaMLP,
965 input_layernorm: RMSNorm,
966 post_attention_layernorm: RMSNorm,
967}
968
969unsafe impl Send for LlamaLayer {}
970unsafe impl Sync for LlamaLayer {}
971
972impl LlamaLayer {
973 #[inline]
976 fn into_device(self, device: Device) -> Self {
977 Self {
978 self_attn: self.self_attn.into_device(device),
979 mlp: self.mlp.into_device(device),
980 input_layernorm: self.input_layernorm.into_device(device),
981 post_attention_layernorm: self.post_attention_layernorm.into_device(device),
982 }
983 }
984}
985
986struct LlamaAttention {
988 q_proj: LinearProjection,
989 k_proj: LinearProjection,
990 v_proj: LinearProjection,
991 o_proj: LinearProjection,
992 num_heads: usize,
993 num_kv_heads: usize,
994 head_dim: usize,
995 rope_theta: f32,
996 #[allow(dead_code)]
997 rope_scaling: Option<RopeScaling>,
998 q_norm: Option<Tensor>,
1000 k_norm: Option<Tensor>,
1001 #[allow(dead_code)]
1002 query_pre_attn_scalar: Option<f32>,
1003 sliding_window: Option<usize>,
1005 layer_type: String, layer_idx: usize,
1007}
1008
1009unsafe impl Send for LlamaAttention {}
1010unsafe impl Sync for LlamaAttention {}
1011
1012impl LlamaAttention {
1013 #[inline]
1015 fn into_device(self, device: Device) -> Self {
1016 Self {
1017 q_proj: self.q_proj.into_device(device),
1018 k_proj: self.k_proj.into_device(device),
1019 v_proj: self.v_proj.into_device(device),
1020 o_proj: self.o_proj.into_device(device),
1021 q_norm: self.q_norm.map(|t| t.to_device(device)),
1022 k_norm: self.k_norm.map(|t| t.to_device(device)),
1023 ..self
1024 }
1025 }
1026
1027 fn forward(
1029 &self,
1030 hidden_states: &Tensor,
1031 position_ids: Option<&Tensor>,
1032 kv_cache: Option<&mut crate::runtime::kv_cache::LayerKVCache>,
1033 start_pos: usize,
1034 delta: Option<&crate::training::TenantDelta>,
1035 ) -> Result<Tensor> {
1036 let (batch_size, seq_len, hidden_size) = dims3(hidden_states)?;
1037
1038 tracing::trace!("LlamaAttention forward: batch_size={}, seq_len={}, hidden_size={}, start_pos={}, has_cache={}",
1039 batch_size, seq_len, hidden_size, start_pos, kv_cache.is_some());
1040
1041 let hidden_states_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
1046 let mut q = self.q_proj.apply(&hidden_states_2d);
1050 let mut k = self.k_proj.apply(&hidden_states_2d);
1051 let mut v = self.v_proj.apply(&hidden_states_2d);
1052
1053 if let Some(delta) = delta {
1057 for (name, proj) in [("q_proj", &mut q), ("k_proj", &mut k), ("v_proj", &mut v)] {
1058 if delta.has_module(name, self.layer_idx) {
1059 let correction = delta.forward_2d(&hidden_states_2d, name, self.layer_idx)?;
1060 if start_pos == 0 && self.layer_idx == 0 {
1061 tracing::info!(
1062 "[TTT] Layer 0 {}: correction_norm={:.6}, proj_norm={:.4}, ratio={:.6}",
1063 name,
1064 correction.norm().double_value(&[]),
1065 proj.norm().double_value(&[]),
1066 correction.norm().double_value(&[])
1067 / (proj.norm().double_value(&[]) + 1e-10)
1068 );
1069 }
1070 let correction_size = correction.size();
1072 *proj = proj.f_add(&correction.to_kind(proj.kind()))
1073 .map_err(|e| anyhow::anyhow!(
1074 "Delta correction shape mismatch at layer {} {}: proj {:?} vs correction {:?}: {}",
1075 self.layer_idx, name, proj.size(), correction_size, e
1076 ))?;
1077 }
1078 }
1079 }
1080
1081 let k_elements = k.size().iter().product::<i64>();
1090 let _v_elements = v.size().iter().product::<i64>();
1091
1092 let kv_heads = if k_elements
1095 == batch_size * seq_len * (self.num_kv_heads as i64) * (self.head_dim as i64)
1096 {
1097 self.num_kv_heads } else if k_elements % (batch_size * seq_len * (self.head_dim as i64)) == 0 {
1099 (k_elements / (batch_size * seq_len * (self.head_dim as i64))) as usize
1101 } else if k_elements % (batch_size * seq_len) == 0 {
1102 let kv_dim = (k_elements / (batch_size * seq_len)) as usize;
1104 if kv_dim == 256 && self.head_dim == 128 {
1105 2 } else if kv_dim == 256 {
1107 8 } else {
1109 self.num_kv_heads }
1111 } else {
1112 self.num_kv_heads };
1114
1115 let mut q = q.reshape([
1120 batch_size,
1121 seq_len,
1122 self.num_heads as i64,
1123 self.head_dim as i64,
1124 ]);
1125 let mut k = k.reshape([batch_size, seq_len, kv_heads as i64, self.head_dim as i64]);
1127 let v = v.reshape([batch_size, seq_len, kv_heads as i64, self.head_dim as i64]);
1128
1129 if let Some(q_norm) = &self.q_norm {
1131 q = self.apply_qk_norm(&q, q_norm, self.num_heads)?;
1132 }
1133 if let Some(k_norm) = &self.k_norm {
1134 k = self.apply_qk_norm(&k, k_norm, kv_heads)?;
1135 }
1136
1137 let (q, k) = if let Some(pos_ids) = position_ids {
1139 (self.apply_rope(&q, pos_ids)?, self.apply_rope(&k, pos_ids)?)
1140 } else {
1141 (q, k)
1142 };
1143
1144 let (k_for_attn, v_for_attn) = if let Some(cache) = kv_cache {
1146 cache.update(&k, &v, start_pos)?;
1148
1149 let (cached_k, cached_v) = cache.get()?;
1151 (cached_k, cached_v)
1152 } else {
1153 (k.shallow_clone(), v.shallow_clone())
1155 };
1156
1157 let (k_expanded, v_expanded) = if kv_heads < self.num_heads {
1159 (
1161 self.expand_kv_for_gqa_with_heads(&k_for_attn, kv_heads)?,
1162 self.expand_kv_for_gqa_with_heads(&v_for_attn, kv_heads)?,
1163 )
1164 } else {
1165 (k_for_attn, v_for_attn)
1166 };
1167
1168 let attn_output = self.compute_attention(&q, &k_expanded, &v_expanded)?;
1177
1178 let attn_output = attn_output.transpose(1, 2).contiguous();
1181
1182 let attn_output =
1184 attn_output.reshape([batch_size, seq_len, (self.num_heads * self.head_dim) as i64]);
1185
1186 let attn_output_2d = attn_output.reshape([
1188 batch_size * seq_len,
1189 (self.num_heads * self.head_dim) as i64,
1190 ]);
1191
1192 let mut attn_output = self.o_proj.apply(&attn_output_2d);
1194
1195 if let Some(delta) = delta {
1197 if delta.has_module("o_proj", self.layer_idx) {
1198 let correction = delta.forward_2d(&attn_output_2d, "o_proj", self.layer_idx)?;
1199 let kind = attn_output.kind();
1200 attn_output += correction.to_kind(kind);
1201 }
1202 }
1203
1204 let attn_output = attn_output.reshape([batch_size, seq_len, hidden_size]);
1206
1207 Ok(attn_output)
1208 }
1209
1210 fn detect_kv_heads(&self, k_elements: i64, rows: i64) -> usize {
1216 if k_elements == rows * (self.num_kv_heads as i64) * (self.head_dim as i64) {
1217 self.num_kv_heads
1218 } else if k_elements % (rows * (self.head_dim as i64)) == 0 {
1219 (k_elements / (rows * (self.head_dim as i64))) as usize
1220 } else if k_elements % rows == 0 {
1221 let kv_dim = (k_elements / rows) as usize;
1222 if kv_dim == 256 && self.head_dim == 128 {
1223 2
1224 } else if kv_dim == 256 {
1225 8
1226 } else {
1227 self.num_kv_heads
1228 }
1229 } else {
1230 self.num_kv_heads
1231 }
1232 }
1233
1234 #[allow(clippy::too_many_arguments)]
1250 fn forward_batched(
1251 &self,
1252 hidden_states: &Tensor,
1253 position_ids: &[Tensor],
1254 kv_caches: &mut [&mut crate::runtime::kv_cache::LayerKVCache],
1255 start_positions: &[usize],
1256 attn_mask: &Tensor,
1257 delta: Option<&crate::training::TenantDelta>,
1258 ) -> Result<Tensor> {
1259 let (batch_size, seq_len, hidden_size) = dims3(hidden_states)?;
1260 debug_assert_eq!(batch_size as usize, kv_caches.len());
1261 debug_assert_eq!(batch_size as usize, position_ids.len());
1262 debug_assert_eq!(batch_size as usize, start_positions.len());
1263
1264 let hidden_states_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
1266 let mut q = self.q_proj.apply(&hidden_states_2d);
1267 let mut k = self.k_proj.apply(&hidden_states_2d);
1268 let mut v = self.v_proj.apply(&hidden_states_2d);
1269
1270 if let Some(delta) = delta {
1272 for (name, proj) in [("q_proj", &mut q), ("k_proj", &mut k), ("v_proj", &mut v)] {
1273 if delta.has_module(name, self.layer_idx) {
1274 let correction = delta.forward_2d(&hidden_states_2d, name, self.layer_idx)?;
1275 let kind = proj.kind();
1276 *proj = proj.f_add(&correction.to_kind(kind)).map_err(|e| {
1277 anyhow::anyhow!(
1278 "Delta correction shape mismatch at layer {} {}: {}",
1279 self.layer_idx,
1280 name,
1281 e
1282 )
1283 })?;
1284 }
1285 }
1286 }
1287
1288 let rows = batch_size * seq_len;
1289 let kv_heads = self.detect_kv_heads(k.size().iter().product::<i64>(), rows);
1290
1291 let mut q = q.reshape([
1292 batch_size,
1293 seq_len,
1294 self.num_heads as i64,
1295 self.head_dim as i64,
1296 ]);
1297 let mut k = k.reshape([batch_size, seq_len, kv_heads as i64, self.head_dim as i64]);
1298 let v = v.reshape([batch_size, seq_len, kv_heads as i64, self.head_dim as i64]);
1299
1300 if let Some(q_norm) = &self.q_norm {
1301 q = self.apply_qk_norm(&q, q_norm, self.num_heads)?;
1302 }
1303 if let Some(k_norm) = &self.k_norm {
1304 k = self.apply_qk_norm(&k, k_norm, kv_heads)?;
1305 }
1306
1307 let mut q_rows: Vec<Tensor> = Vec::with_capacity(batch_size as usize);
1309 let mut k_rows: Vec<Tensor> = Vec::with_capacity(batch_size as usize);
1310 for (r, pos) in position_ids.iter().enumerate() {
1311 let q_r = q.narrow(0, r as i64, 1); let k_r = k.narrow(0, r as i64, 1); q_rows.push(self.apply_rope(&q_r, pos)?);
1314 k_rows.push(self.apply_rope(&k_r, pos)?);
1315 }
1316 let q = Tensor::cat(&q_rows, 0);
1317 let k = Tensor::cat(&k_rows, 0);
1318
1319 let mut row_keys: Vec<Tensor> = Vec::with_capacity(batch_size as usize);
1321 let mut row_values: Vec<Tensor> = Vec::with_capacity(batch_size as usize);
1322 let mut kv_lens: Vec<i64> = Vec::with_capacity(batch_size as usize);
1323 for r in 0..batch_size as usize {
1324 let k_r = k.narrow(0, r as i64, 1).contiguous();
1325 let v_r = v.narrow(0, r as i64, 1).contiguous();
1326 let cache = &mut *kv_caches[r];
1327 cache.update(&k_r, &v_r, start_positions[r])?;
1328 let (ck, cv) = cache.get()?; kv_lens.push(ck.size()[1]);
1330 row_keys.push(ck);
1331 row_values.push(cv);
1332 }
1333
1334 let kv_max = kv_lens.iter().copied().max().unwrap_or(0);
1337 let opt = (k.kind(), k.device());
1338 let pad_row = |t: &Tensor, len: i64| -> Tensor {
1339 if len == kv_max {
1340 t.shallow_clone()
1341 } else {
1342 let mut shape = t.size();
1344 shape[1] = kv_max - len;
1345 let pad = Tensor::zeros(&shape[..], opt);
1346 Tensor::cat(&[t.shallow_clone(), pad], 1)
1347 }
1348 };
1349 let keys_padded: Vec<Tensor> = row_keys
1350 .iter()
1351 .zip(&kv_lens)
1352 .map(|(t, &len)| pad_row(t, len))
1353 .collect();
1354 let values_padded: Vec<Tensor> = row_values
1355 .iter()
1356 .zip(&kv_lens)
1357 .map(|(t, &len)| pad_row(t, len))
1358 .collect();
1359 let k_stacked = Tensor::cat(&keys_padded, 0); let v_stacked = Tensor::cat(&values_padded, 0);
1361
1362 let (k_expanded, v_expanded) = if kv_heads < self.num_heads {
1364 (
1365 self.expand_kv_for_gqa_with_heads(&k_stacked, kv_heads)?,
1366 self.expand_kv_for_gqa_with_heads(&v_stacked, kv_heads)?,
1367 )
1368 } else {
1369 (k_stacked, v_stacked)
1370 };
1371
1372 let attn_output =
1374 self.compute_attention_masked(&q, &k_expanded, &v_expanded, Some(attn_mask))?;
1375
1376 let attn_output = attn_output.transpose(1, 2).contiguous();
1377 let attn_output =
1378 attn_output.reshape([batch_size, seq_len, (self.num_heads * self.head_dim) as i64]);
1379 let attn_output_2d = attn_output.reshape([
1380 batch_size * seq_len,
1381 (self.num_heads * self.head_dim) as i64,
1382 ]);
1383 let mut attn_output = self.o_proj.apply(&attn_output_2d);
1384
1385 if let Some(delta) = delta {
1386 if delta.has_module("o_proj", self.layer_idx) {
1387 let correction = delta.forward_2d(&attn_output_2d, "o_proj", self.layer_idx)?;
1388 let kind = attn_output.kind();
1389 attn_output += correction.to_kind(kind);
1390 }
1391 }
1392
1393 Ok(attn_output.reshape([batch_size, seq_len, hidden_size]))
1394 }
1395
1396 fn expand_kv_for_gqa_with_heads(&self, kv: &Tensor, actual_kv_heads: usize) -> Result<Tensor> {
1398 let (batch_size, seq_len, _detected_heads, head_dim) = dims4(kv)?;
1399 let repeat_factor = self.num_heads / actual_kv_heads;
1400
1401 if repeat_factor == 1 {
1402 return Ok(kv.shallow_clone());
1403 }
1404
1405 Ok(kv
1410 .unsqueeze(3) .expand(
1412 [
1413 batch_size,
1414 seq_len,
1415 actual_kv_heads as i64,
1416 repeat_factor as i64,
1417 head_dim,
1418 ],
1419 false,
1420 )
1421 .reshape([batch_size, seq_len, self.num_heads as i64, head_dim]))
1422 }
1423
1424 fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor> {
1426 ROPE_CACHE.with(|cache| {
1427 let mut cache = cache.borrow_mut();
1428
1429 let key = (
1431 self.layer_idx,
1432 self.head_dim as i64,
1433 (self.rope_theta * 1000.0) as u32, );
1435
1436 if let std::collections::hash_map::Entry::Vacant(e) = cache.entry(key) {
1438 let base = if self.layer_type == "local" {
1440 10000.0
1442 } else {
1443 self.rope_theta as f64
1444 };
1445
1446 tracing::debug!(
1447 "Creating RoPE for layer {} with base={}, head_dim={}",
1448 self.layer_idx,
1449 base,
1450 self.head_dim
1451 );
1452
1453 let rope = RoPE::new_with_dtype(
1455 self.head_dim as i64,
1456 base,
1457 8192, tensor.device(),
1459 tensor.kind(), )
1461 .map_err(|e| {
1462 anyhow!("Failed to create RoPE for layer {}: {}", self.layer_idx, e)
1463 })?;
1464
1465 e.insert(rope);
1466 }
1467
1468 let rope = cache.get_mut(&key).ok_or_else(|| {
1470 anyhow!(
1471 "Internal error: RoPE cache entry missing for layer {}",
1472 self.layer_idx
1473 )
1474 })?;
1475
1476 rope.forward(tensor, Some(position_ids))
1478 })
1479 }
1480
1481 fn apply_qk_norm(
1483 &self,
1484 tensor: &Tensor,
1485 norm_weight: &Tensor,
1486 actual_heads: usize,
1487 ) -> Result<Tensor> {
1488 let (_batch_size, _seq_len, tensor_heads, head_dim) = dims4(tensor)?;
1492
1493 tracing::trace!(
1495 "apply_qk_norm: tensor heads={}, head_dim={}, actual_heads={}, norm_weight shape={:?}",
1496 tensor_heads,
1497 head_dim,
1498 actual_heads,
1499 norm_weight.size()
1500 );
1501
1502 let norm_weight_reshaped = if norm_weight.size().len() == 1 {
1505 let norm_elements = norm_weight.size()[0];
1506 if norm_elements == (actual_heads * head_dim as usize) as i64 {
1510 norm_weight
1512 .reshape([actual_heads as i64, head_dim])
1513 .unsqueeze(0) .unsqueeze(0) } else if norm_elements == head_dim {
1516 norm_weight
1518 .unsqueeze(0) .unsqueeze(0) .unsqueeze(0) } else {
1522 return Err(anyhow!(
1523 "QK-norm weight size {} doesn't match expected size for {} heads with dim {}",
1524 norm_elements,
1525 actual_heads,
1526 head_dim
1527 ));
1528 }
1529 } else {
1530 norm_weight.shallow_clone()
1531 };
1532
1533 let original_dtype = tensor.kind();
1535 let x2 = square_tensor(tensor)?;
1536 let mean = x2.mean_dim(&[-1i64][..], true, original_dtype);
1537 let eps = 1e-6;
1538 let rrms = (mean + eps).reciprocal().sqrt();
1539
1540 broadcast_mul(&broadcast_mul(tensor, &rrms)?, &norm_weight_reshaped)
1542 }
1543
1544 fn compute_attention(&self, q: &Tensor, k: &Tensor, v: &Tensor) -> Result<Tensor> {
1560 self.compute_attention_masked(q, k, v, None)
1561 }
1562
1563 fn compute_attention_masked(
1574 &self,
1575 q: &Tensor,
1576 k: &Tensor,
1577 v: &Tensor,
1578 attn_mask: Option<&Tensor>,
1579 ) -> Result<Tensor> {
1580 const CHUNK: i64 = 1024;
1583
1584 let scale = 1.0 / (self.head_dim as f32).sqrt();
1585 let original_kind = q.kind();
1586
1587 let q = q.transpose(1, 2).contiguous();
1590 let k_t = k.transpose(1, 2).transpose(2, 3).contiguous(); let v = v.transpose(1, 2).contiguous(); let upcast = original_kind != tch::Kind::Float;
1595 let q = if upcast {
1596 q.to_kind(tch::Kind::Float)
1597 } else {
1598 q
1599 };
1600 let k_t = if upcast {
1601 k_t.to_kind(tch::Kind::Float)
1602 } else {
1603 k_t
1604 };
1605 let v = if upcast {
1606 v.to_kind(tch::Kind::Float)
1607 } else {
1608 v
1609 };
1610 let attn_mask = attn_mask.map(|m| {
1612 if m.kind() != tch::Kind::Float {
1613 m.to_kind(tch::Kind::Float)
1614 } else {
1615 m.shallow_clone()
1616 }
1617 });
1618
1619 let q_len = q.size()[2];
1620 let k_len = k_t.size()[3];
1621 let device = q.device();
1622
1623 let single_shot = attn_mask.is_some() || self.sliding_window.is_some() || q_len <= CHUNK;
1627
1628 let attn_output = if single_shot {
1629 let mut scores = q.matmul(&k_t) * (scale as f64);
1631
1632 if let Some(mask) = attn_mask.as_ref() {
1633 scores = broadcast_add(&scores, mask)?;
1636 } else if q_len > 1 {
1637 let mask =
1644 Tensor::ones([q_len, k_len], (tch::Kind::Float, device)).tril(k_len - q_len);
1645 let mask = mask.unsqueeze(0).unsqueeze(0).expand_as(&scores);
1646 scores = scores.masked_fill(&mask.eq(0.0), -10000.0f64);
1647 }
1648
1649 if let Some(window_size) = self.sliding_window {
1651 if self.layer_type == "local" {
1652 scores = self.apply_sliding_window_mask(&scores, window_size)?;
1653 }
1654 }
1655
1656 scores.softmax(-1, tch::Kind::Float).matmul(&v)
1657 } else {
1658 let mut outputs: Vec<Tensor> = Vec::new();
1663 let mut start = 0i64;
1664 while start < q_len {
1665 let cur = (q_len - start).min(CHUNK);
1666 let q_chunk = q.narrow(2, start, cur); let mut scores = q_chunk.matmul(&k_t) * (scale as f64); let mask = Tensor::ones([cur, k_len], (tch::Kind::Float, device))
1669 .tril(k_len - q_len + start);
1670 let mask = mask.unsqueeze(0).unsqueeze(0).expand_as(&scores);
1671 scores = scores.masked_fill(&mask.eq(0.0), -10000.0f64);
1672 outputs.push(scores.softmax(-1, tch::Kind::Float).matmul(&v)); start += cur;
1674 }
1675 Tensor::cat(&outputs, 2) };
1677
1678 Ok(if upcast {
1679 attn_output.to_kind(original_kind)
1680 } else {
1681 attn_output
1682 })
1683 }
1684
1685 fn apply_sliding_window_mask(&self, scores: &Tensor, window_size: usize) -> Result<Tensor> {
1688 let (batch_size, num_heads, seq_len, _) = dims4(scores)?;
1689 let device = scores.device();
1690 let dtype = scores.kind();
1691
1692 let causal = Tensor::ones([seq_len, seq_len], (dtype, device)).tril(0);
1697
1698 let window = Tensor::ones([seq_len, seq_len], (dtype, device)).triu(1 - window_size as i64);
1701
1702 let valid_mask = &causal * &window;
1704
1705 let additive_mask =
1708 (Tensor::ones([seq_len, seq_len], (dtype, device)) - valid_mask) * (-10000.0f64);
1709
1710 let mask = additive_mask
1712 .unsqueeze(0)
1713 .unsqueeze(0)
1714 .expand([batch_size, num_heads, seq_len, seq_len], false);
1715
1716 broadcast_add(scores, &mask)
1718 }
1719}
1720
1721pub(crate) struct LlamaMLP {
1723 pub(crate) gate_proj: LinearProjection,
1724 pub(crate) up_proj: LinearProjection,
1725 pub(crate) down_proj: LinearProjection,
1726 pub(crate) activation: String, pub(crate) layer_idx: usize, }
1729
1730unsafe impl Send for LlamaMLP {}
1731unsafe impl Sync for LlamaMLP {}
1732
1733impl LlamaMLP {
1734 #[inline]
1737 pub(crate) fn into_device(self, device: Device) -> Self {
1738 Self {
1739 gate_proj: self.gate_proj.into_device(device),
1740 up_proj: self.up_proj.into_device(device),
1741 down_proj: self.down_proj.into_device(device),
1742 activation: self.activation,
1743 layer_idx: self.layer_idx,
1744 }
1745 }
1746
1747 pub(crate) fn forward(
1748 &self,
1749 hidden_states: &Tensor,
1750 delta: Option<&crate::training::TenantDelta>,
1751 ) -> Result<Tensor> {
1752 let original_shape = hidden_states.size();
1754 let (batch_size, seq_len, hidden_size) = if original_shape.len() == 3 {
1755 (original_shape[0], original_shape[1], original_shape[2])
1756 } else {
1757 return self.forward_2d(hidden_states, delta);
1759 };
1760
1761 let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
1763
1764 let mut gate_pre = self.gate_proj.try_apply(&hidden_2d)?;
1766 let mut up = self.up_proj.try_apply(&hidden_2d)?;
1767
1768 if let Some(delta) = delta {
1769 if delta.has_module("gate_proj", self.layer_idx) {
1770 let correction = delta.forward_2d(&hidden_2d, "gate_proj", self.layer_idx)?;
1771 let kind = gate_pre.kind();
1772 gate_pre += correction.to_kind(kind);
1773 }
1774 if delta.has_module("up_proj", self.layer_idx) {
1775 let correction = delta.forward_2d(&hidden_2d, "up_proj", self.layer_idx)?;
1776 let kind = up.kind();
1777 up += correction.to_kind(kind);
1778 }
1779 }
1780
1781 let gate = self.apply_activation(&gate_pre)?;
1782 let gated = &gate * &up;
1783
1784 let mut output = self.down_proj.try_apply(&gated)?;
1785 if let Some(delta) = delta {
1786 if delta.has_module("down_proj", self.layer_idx) {
1787 let correction = delta.forward_2d(&gated, "down_proj", self.layer_idx)?;
1788 let kind = output.kind();
1789 output += correction.to_kind(kind);
1790 }
1791 }
1792
1793 let out_size = output.size()[1];
1795 Ok(output.reshape([batch_size, seq_len, out_size]))
1796 }
1797
1798 fn forward_2d(
1799 &self,
1800 hidden_states: &Tensor,
1801 delta: Option<&crate::training::TenantDelta>,
1802 ) -> Result<Tensor> {
1803 let mut gate_pre = self.gate_proj.try_apply(hidden_states)?;
1805 let mut up = self.up_proj.try_apply(hidden_states)?;
1806
1807 if let Some(delta) = delta {
1808 if delta.has_module("gate_proj", self.layer_idx) {
1809 let correction = delta.forward_2d(hidden_states, "gate_proj", self.layer_idx)?;
1810 let kind = gate_pre.kind();
1811 gate_pre += correction.to_kind(kind);
1812 }
1813 if delta.has_module("up_proj", self.layer_idx) {
1814 let correction = delta.forward_2d(hidden_states, "up_proj", self.layer_idx)?;
1815 let kind = up.kind();
1816 up += correction.to_kind(kind);
1817 }
1818 }
1819
1820 let gate = self.apply_activation(&gate_pre)?;
1821 let gated = &gate * &up;
1822
1823 let mut output = self.down_proj.try_apply(&gated)?;
1824 if let Some(delta) = delta {
1825 if delta.has_module("down_proj", self.layer_idx) {
1826 let correction = delta.forward_2d(&gated, "down_proj", self.layer_idx)?;
1827 let kind = output.kind();
1828 output += correction.to_kind(kind);
1829 }
1830 }
1831
1832 Ok(output)
1833 }
1834
1835 fn apply_activation(&self, x: &Tensor) -> Result<Tensor> {
1837 match self.activation.as_str() {
1838 "silu" => Ok(x.silu()),
1839 "gelu_pytorch_tanh" => self.gelu_pytorch_tanh(x),
1840 "gelu" => {
1841 self.gelu_pytorch_tanh(x)
1845 }
1846 _ => {
1847 tracing::warn!(
1848 "Unknown activation '{}', falling back to SiLU",
1849 self.activation
1850 );
1851 Ok(x.silu())
1852 }
1853 }
1854 }
1855
1856 fn gelu_pytorch_tanh(&self, x: &Tensor) -> Result<Tensor> {
1859 use std::f32::consts::PI;
1860
1861 let sqrt_2_over_pi = (2.0_f32 / PI).sqrt();
1863 let coeff = 0.044715_f32;
1864
1865 let x_cubed = x.pow_tensor_scalar(3.0);
1867
1868 let coeff_tensor = scalar_tensor(coeff, x.device(), x.kind());
1870 let inner = x + broadcast_mul(&x_cubed, &coeff_tensor)?;
1871
1872 let sqrt_tensor = scalar_tensor(sqrt_2_over_pi, x.device(), x.kind());
1874 let scaled = broadcast_mul(&inner, &sqrt_tensor)?;
1875
1876 let tanh_result = scaled.tanh();
1878
1879 let one_tensor = scalar_tensor(1.0_f32, x.device(), x.kind());
1881 let one_plus_tanh = broadcast_add(&tanh_result, &one_tensor)?;
1882
1883 let half_tensor = scalar_tensor(0.5_f32, x.device(), x.kind());
1885 broadcast_mul(&(x * &one_plus_tanh), &half_tensor)
1886 }
1887}
1888
1889pub(crate) struct RMSNorm {
1891 pub(crate) weight: Tensor,
1892 pub(crate) eps: f32,
1893}
1894
1895unsafe impl Send for RMSNorm {}
1896unsafe impl Sync for RMSNorm {}
1897
1898impl RMSNorm {
1899 #[inline]
1901 fn into_device(self, device: Device) -> Self {
1902 Self {
1903 weight: self.weight.to_device(device),
1904 eps: self.eps,
1905 }
1906 }
1907}
1908
1909impl RMSNorm {
1910 pub(crate) fn forward(&self, x: &Tensor) -> Result<Tensor> {
1911 let original_dtype = x.kind();
1913 let x2 = square_tensor(x)?;
1914 let mean = x2.mean_dim(&[-1i64][..], true, original_dtype); let rrms = (mean + self.eps as f64).reciprocal().sqrt();
1916
1917 broadcast_mul(&broadcast_mul(x, &rrms)?, &self.weight)
1919 }
1920}
1921
1922impl LlamaModel {
1923 pub fn from_safetensors(path: &Path, device: &Device, dtype: DType) -> Result<Self> {
1925 let config_path = path
1927 .parent()
1928 .ok_or_else(|| anyhow!("Invalid model path"))?
1929 .join("config.json");
1930
1931 let config = if config_path.exists() {
1932 let config_str = std::fs::read_to_string(&config_path)?;
1933 Self::parse_config(&config_str)?
1934 } else {
1935 LlamaConfig::default()
1936 };
1937
1938 let layers = Vec::new();
1940
1941 let device_map =
1944 LayerDeviceMap::single(*device, (config.num_hidden_layers as usize).max(1))?;
1945
1946 Ok(Self {
1947 config,
1948 device: *device,
1949 dtype,
1950 device_map,
1951 layer_offset: 0,
1952 embed_tokens: None,
1953 layers,
1954 norm: None,
1955 lm_head: None,
1956 lm_head_transposed: None,
1957 kv_cache: None, vs: None, })
1960 }
1961
1962 pub fn from_weights(
1964 weights: &mut HashMap<String, Tensor>,
1965 device: &Device,
1966 dtype: DType,
1967 ) -> Result<Self> {
1968 let config = Self::detect_config_from_weights(weights)?;
1970 Self::from_weights_with_config(
1971 weights,
1972 config,
1973 device,
1974 dtype,
1975 hyprstream_rpc_std::model_client::KVQuantType::None,
1976 )
1977 }
1978
1979 pub fn from_weights_with_config(
1983 weights: &mut HashMap<String, Tensor>,
1984 mut config: LlamaConfig,
1985 device: &Device,
1986 dtype: DType,
1987 kv_quant_type: hyprstream_rpc_std::model_client::KVQuantType,
1988 ) -> Result<Self> {
1989 tracing::info!(
1990 "[from_weights_with_config] Received config.max_position_embeddings = {}, kv_quant = {:?}",
1991 config.max_position_embeddings,
1992 kv_quant_type
1993 );
1994
1995 let embed_tokens = weights
2001 .get("model.embed_tokens.weight")
2002 .or_else(|| weights.get("embed_tokens.weight"))
2003 .map(|w| {
2004 let vocab_size = w.size()[0] as u32;
2005 let hidden_size = w.size()[1];
2006
2007 let padded_vocab_size = calculate_padded_vocab_size(vocab_size);
2009
2010 let placed = if padded_vocab_size > vocab_size {
2011 tracing::info!(
2012 "Padding embedding vocab size from {} to {} (multiple of 64)",
2013 vocab_size,
2014 padded_vocab_size
2015 );
2016
2017 let padded = Tensor::zeros(
2019 [padded_vocab_size as i64, hidden_size],
2020 (w.kind(), w.device()),
2021 );
2022
2023 padded.narrow(0, 0, vocab_size as i64).copy_(w);
2025 padded
2026 } else {
2027 w.shallow_clone()
2028 };
2029 placed.to_device(*device)
2030 });
2031
2032 let original_vocab_size = config.vocab_size;
2034 let padded_vocab_size = calculate_padded_vocab_size(original_vocab_size);
2035
2036 config.original_vocab_size = original_vocab_size;
2038
2039 if padded_vocab_size != original_vocab_size {
2040 tracing::info!(
2041 "Updating config vocab_size from {} to padded size {}",
2042 original_vocab_size,
2043 padded_vocab_size
2044 );
2045 config.vocab_size = padded_vocab_size;
2046 }
2047
2048 let lm_head = weights
2051 .get("lm_head.weight")
2052 .or_else(|| weights.get("model.lm_head.weight"))
2053 .map(|w| {
2054 let vocab_size = w.size()[0] as u32;
2056 let hidden_size = w.size()[1];
2057
2058 let padded_vocab_size = calculate_padded_vocab_size(vocab_size);
2061
2062 let padded_w = if padded_vocab_size > vocab_size {
2063 tracing::info!(
2064 "Padding lm_head vocab size from {} to {} (multiple of 64)",
2065 vocab_size,
2066 padded_vocab_size
2067 );
2068
2069 let padded = Tensor::zeros(
2073 [padded_vocab_size as i64, hidden_size],
2074 (w.kind(), w.device()),
2075 );
2076
2077 padded.narrow(0, 0, vocab_size as i64).copy_(w);
2079 padded
2080 } else {
2081 w.shallow_clone()
2082 };
2083
2084 padded_w.transpose(0, 1).contiguous().to_device(*device)
2087 });
2088
2089 let lm_head_transposed = if lm_head.is_none() {
2092 embed_tokens.as_ref().map(|embed| {
2093 tracing::info!(
2094 "Pre-computing lm_head transpose for tied weights (saves ~740MB per forward)"
2095 );
2096 embed.transpose(0, 1).contiguous()
2097 })
2098 } else {
2099 None
2100 };
2101
2102 let norm = weights
2104 .get("model.norm.weight")
2105 .or_else(|| weights.get("norm.weight"))
2106 .map(|w| RMSNorm {
2107 weight: w.shallow_clone().to_device(*device),
2108 eps: config.rms_norm_eps,
2109 });
2110
2111 let mut layers = Vec::new();
2114 for layer_idx in 0..config.num_hidden_layers as usize {
2115 if let Some(layer) = Self::build_layer(layer_idx, weights, &config, device)? {
2116 layers.push(layer);
2117 }
2118 }
2119
2120 let kv_cache = if !layers.is_empty() {
2122 tracing::info!(
2123 "[LlamaModel] Creating KV cache: num_layers={}, max_seq_len={}, quant={:?}",
2124 layers.len(),
2125 config.max_position_embeddings,
2126 kv_quant_type
2127 );
2128 Some(std::sync::Arc::new(parking_lot::Mutex::new(
2129 crate::runtime::kv_cache::KVCacheManager::new(
2130 layers.len(),
2131 config.max_position_embeddings as usize,
2132 kv_quant_type,
2133 ),
2134 )))
2135 } else {
2136 None
2137 };
2138
2139 let device_map = LayerDeviceMap::single(*device, layers.len().max(1))?;
2143
2144 Ok(Self {
2145 config,
2146 device: *device,
2147 dtype,
2148 device_map,
2149 layer_offset: 0,
2150 embed_tokens,
2151 layers,
2152 norm,
2153 lm_head,
2154 lm_head_transposed,
2155 kv_cache,
2156 vs: None, })
2158 }
2159
2160 #[allow(clippy::too_many_arguments)]
2181 pub fn stage_from_weights_with_config(
2182 weights: &mut HashMap<String, Tensor>,
2183 mut config: LlamaConfig,
2184 devices: &LayerDeviceMap,
2185 layer_range: std::ops::Range<usize>,
2186 dtype: DType,
2187 kv_quant_type: hyprstream_rpc_std::model_client::KVQuantType,
2188 ) -> Result<Self> {
2189 let num_global = config.num_hidden_layers as usize;
2190 if devices.len() != num_global {
2191 return Err(anyhow!(
2192 "stage_from_weights: device map covers {} layers but config has {} \
2193 (num_hidden_layers)",
2194 devices.len(),
2195 num_global
2196 ));
2197 }
2198 if layer_range.end > num_global || layer_range.start >= layer_range.end {
2199 return Err(anyhow!(
2200 "stage_from_weights: invalid layer range {:?} for {} layers",
2201 layer_range,
2202 num_global
2203 ));
2204 }
2205
2206 let is_first = layer_range.start == 0;
2207 let is_last = layer_range.end == num_global;
2208 let layer_offset = layer_range.start;
2209 let stage_device = devices.device_for(layer_range.start);
2213
2214 let original_vocab_size = config.vocab_size;
2216 let padded_vocab_size = calculate_padded_vocab_size(original_vocab_size);
2217 config.original_vocab_size = original_vocab_size;
2218 if padded_vocab_size != original_vocab_size {
2219 config.vocab_size = padded_vocab_size;
2220 }
2221
2222 let embed_tokens = if is_first {
2223 let embed = weights
2224 .get("model.embed_tokens.weight")
2225 .or_else(|| weights.get("embed_tokens.weight"))
2226 .map(|w| Self::pad_embedding_to(w, padded_vocab_size, stage_device))
2227 .ok_or_else(|| {
2228 anyhow!("stage_from_weights: first stage requires model.embed_tokens.weight")
2229 })?;
2230 Some(embed)
2231 } else {
2232 None
2233 };
2234
2235 let (lm_head, norm) = if is_last {
2236 let lm_head = weights
2237 .get("lm_head.weight")
2238 .or_else(|| weights.get("model.lm_head.weight"))
2239 .map(|w| {
2240 Self::pad_embedding_to(w, padded_vocab_size, stage_device)
2241 .transpose(0, 1)
2242 .contiguous()
2243 });
2244 let norm = weights
2245 .get("model.norm.weight")
2246 .or_else(|| weights.get("norm.weight"))
2247 .map(|w| RMSNorm {
2248 weight: w.shallow_clone().to_device(stage_device),
2249 eps: config.rms_norm_eps,
2250 });
2251 if norm.is_none() {
2252 return Err(anyhow!(
2253 "stage_from_weights: last stage requires model.norm.weight"
2254 ));
2255 }
2256 (lm_head, norm)
2257 } else {
2258 (None, None)
2259 };
2260
2261 let lm_head_transposed = if is_last && lm_head.is_none() {
2265 embed_tokens
2266 .as_ref()
2267 .map(|embed| embed.transpose(0, 1).contiguous())
2268 } else {
2269 None
2270 };
2271 if is_last && lm_head.is_none() && lm_head_transposed.is_none() {
2272 return Err(anyhow!(
2273 "stage_from_weights: last stage requires lm_head.weight (no tied embedding present)"
2274 ));
2275 }
2276
2277 let mut layers = Vec::with_capacity(layer_range.len());
2279 for g in layer_range.clone() {
2280 let target = devices.device_for(g);
2281 match Self::build_layer(g, weights, &config, &target)? {
2282 Some(layer) => layers.push(layer.into_device(target)),
2283 None => {
2284 return Err(anyhow!(
2285 "stage_from_weights: expected weights for global layer {g} but none found"
2286 ))
2287 }
2288 }
2289 }
2290
2291 let kv_cache = if layers.is_empty() {
2294 None
2295 } else {
2296 Some(std::sync::Arc::new(parking_lot::Mutex::new(
2297 crate::runtime::kv_cache::KVCacheManager::new(
2298 layers.len(),
2299 config.max_position_embeddings as usize,
2300 kv_quant_type,
2301 ),
2302 )))
2303 };
2304
2305 let device_map = devices.clone();
2306
2307 Ok(Self {
2308 config,
2309 device: stage_device,
2310 dtype,
2311 device_map,
2312 layer_offset,
2313 embed_tokens,
2314 layers,
2315 norm,
2316 lm_head,
2317 lm_head_transposed,
2318 kv_cache,
2319 vs: None,
2320 })
2321 }
2322
2323 fn pad_embedding_to(w: &Tensor, padded_vocab: u32, device: Device) -> Tensor {
2326 let vocab_size = w.size()[0] as u32;
2327 let hidden_size = w.size()[1];
2328 let placed = if padded_vocab > vocab_size {
2329 let padded = Tensor::zeros([padded_vocab as i64, hidden_size], (w.kind(), w.device()));
2330 padded.narrow(0, 0, vocab_size as i64).copy_(w);
2331 padded
2332 } else {
2333 w.shallow_clone()
2334 };
2335 placed.to_device(device)
2336 }
2337
2338 pub fn detect_config_from_weights(weights: &HashMap<String, Tensor>) -> Result<LlamaConfig> {
2340 let mut config = LlamaConfig::default();
2342 tracing::info!(
2343 "Starting detect_config_from_weights, default rope_theta={}",
2344 config.rope_theta
2345 );
2346
2347 if let Some(embed) = weights
2349 .get("model.embed_tokens.weight")
2350 .or_else(|| weights.get("embed_tokens.weight"))
2351 {
2352 let shape = embed.size();
2353 if shape.len() >= 2 {
2354 config.vocab_size = shape[0] as u32;
2355 config.original_vocab_size = shape[0] as u32; config.hidden_size = shape[1] as u32;
2357
2358 if config.vocab_size == 262144 {
2360 config.hidden_activation = "gelu_pytorch_tanh".to_owned();
2361 config.use_qk_norm = true;
2362 config.scale_embeddings = true;
2363 config.query_pre_attn_scalar = Some(256.0); config.rope_local_base_freq = Some(10000.0);
2365 config.rope_theta = 1000000.0; tracing::info!("Detected Gemma3 model (vocab_size=262144), configuring with:");
2367 tracing::info!(" - GELU PyTorch tanh activation");
2368 tracing::info!(" - QK-norm enabled");
2369 tracing::info!(" - Embedding scaling enabled");
2370 tracing::info!(" - RoPE theta: global=1000000, local=10000");
2371 }
2372 }
2373 }
2374
2375 let layer_count = weights
2377 .keys()
2378 .filter(|k| k.contains("layers.") && k.contains(".self_attn.q_proj"))
2379 .count();
2380 if layer_count > 0 {
2381 config.num_hidden_layers = layer_count as u32;
2382 }
2383
2384 if let Some(q_proj) = weights.get("model.layers.0.self_attn.q_proj.weight") {
2386 let q_shape = q_proj.size();
2387 if q_shape.len() >= 2 {
2388 config.hidden_size = q_shape[1] as u32;
2391 let q_proj_out_dim = q_shape[0] as usize;
2392
2393 let k_proj_out_dim =
2395 if let Some(k_proj) = weights.get("model.layers.0.self_attn.k_proj.weight") {
2396 k_proj.size()[0] as usize
2397 } else {
2398 q_proj_out_dim };
2400
2401 tracing::debug!(
2402 "Q projection output: {}, K projection output: {}",
2403 q_proj_out_dim,
2404 k_proj_out_dim
2405 );
2406
2407 let possible_head_dims = if config.vocab_size == 262144 {
2410 vec![256, 128, 64, 32]
2412 } else if q_proj_out_dim <= 512 {
2413 vec![32, 64, 48, 40, 128] } else {
2415 vec![128, 64, 80, 96, 160, 256]
2416 };
2417
2418 let mut found_config = false;
2419
2420 for &head_dim in &possible_head_dims {
2421 if q_proj_out_dim.is_multiple_of(head_dim) && k_proj_out_dim % head_dim == 0 {
2422 config.num_attention_heads = (q_proj_out_dim / head_dim) as u32;
2423 config.num_key_value_heads = (k_proj_out_dim / head_dim) as u32;
2424 config.head_dim = head_dim as u32;
2425 found_config = true;
2426 tracing::info!("Detected attention config: Q={} heads, KV={} heads, dim={}, total Q={}",
2427 config.num_attention_heads, config.num_key_value_heads,
2428 config.head_dim, q_proj_out_dim);
2429 break;
2430 }
2431 }
2432
2433 if !found_config {
2434 if q_proj_out_dim == 256 && k_proj_out_dim == 256 {
2437 config.num_attention_heads = 8;
2439 config.num_key_value_heads = 8;
2440 config.head_dim = 32;
2441 tracing::info!("Using common small model config: 8 heads x 32 dim");
2442 } else {
2443 config.num_attention_heads = 8;
2445 config.num_key_value_heads = 8;
2446 config.head_dim = 64;
2447 tracing::warn!("Could not detect attention config from shapes Q:{:?} K:{:?}, using defaults",
2448 q_shape, k_proj_out_dim);
2449 }
2450 }
2451 }
2452 }
2453
2454 tracing::info!(
2455 "Finished detect_config_from_weights, final rope_theta={}, vocab_size={}",
2456 config.rope_theta,
2457 config.vocab_size
2458 );
2459 Ok(config)
2460 }
2461
2462 fn build_layer(
2465 layer_idx: usize,
2466 weights: &mut HashMap<String, Tensor>,
2467 config: &LlamaConfig,
2468 _device: &Device,
2469 ) -> Result<Option<LlamaLayer>> {
2470 let prefix = format!("model.layers.{layer_idx}");
2471
2472 let has_separate_qkv = weights.contains_key(&format!("{prefix}.self_attn.q_proj.weight"));
2474 let has_combined_qkv = weights.contains_key(&format!("{prefix}.self_attn.c_attn.weight"));
2475
2476 if !has_separate_qkv && !has_combined_qkv {
2477 return Ok(None);
2478 }
2479
2480 let (q_proj, k_proj, v_proj) = if has_separate_qkv {
2482 let q_proj = LinearProjection::take_with_optional_bias(
2485 weights,
2486 &format!("{prefix}.self_attn.q_proj.weight"),
2487 &format!("{prefix}.self_attn.q_proj.bias"),
2488 )?;
2489 let k_proj = LinearProjection::take_with_optional_bias(
2490 weights,
2491 &format!("{prefix}.self_attn.k_proj.weight"),
2492 &format!("{prefix}.self_attn.k_proj.bias"),
2493 )?;
2494 let v_proj = LinearProjection::take_with_optional_bias(
2495 weights,
2496 &format!("{prefix}.self_attn.v_proj.weight"),
2497 &format!("{prefix}.self_attn.v_proj.bias"),
2498 )?;
2499
2500 (q_proj, k_proj, v_proj)
2501 } else {
2502 let c_attn_weight = weights
2504 .remove(&format!("{prefix}.self_attn.c_attn.weight"))
2505 .ok_or_else(|| anyhow!("Missing c_attn weight"))?
2506 .transpose(0, 1)
2507 .contiguous();
2508
2509 let total_proj_size = c_attn_weight.size()[1];
2511 let proj_size = total_proj_size / 3;
2512
2513 let q_weight = c_attn_weight.narrow(1, 0, proj_size);
2514 let k_weight = c_attn_weight.narrow(1, proj_size, proj_size);
2515 let v_weight = c_attn_weight.narrow(1, proj_size * 2, proj_size);
2516
2517 let (q_proj, k_proj, v_proj) =
2518 if let Some(bias) = weights.get(&format!("{prefix}.self_attn.c_attn.bias")) {
2519 tracing::debug!(
2520 "Layer {}: Loading combined QKV bias (Qwen-style)",
2521 layer_idx
2522 );
2523 let q_bias = bias.narrow(0, 0, proj_size);
2524 let k_bias = bias.narrow(0, proj_size, proj_size);
2525 let v_bias = bias.narrow(0, proj_size * 2, proj_size);
2526 (
2527 LinearProjection::with_bias(q_weight, q_bias),
2528 LinearProjection::with_bias(k_weight, k_bias),
2529 LinearProjection::with_bias(v_weight, v_bias),
2530 )
2531 } else {
2532 (
2533 LinearProjection::new(q_weight),
2534 LinearProjection::new(k_weight),
2535 LinearProjection::new(v_weight),
2536 )
2537 };
2538
2539 (q_proj, k_proj, v_proj)
2540 };
2541
2542 let q_norm = weights
2544 .get(&format!("{prefix}.self_attn.q_norm.weight"))
2545 .map(tch::Tensor::shallow_clone);
2546 let k_norm = weights
2547 .get(&format!("{prefix}.self_attn.k_norm.weight"))
2548 .map(tch::Tensor::shallow_clone);
2549
2550 if q_norm.is_some() || k_norm.is_some() {
2551 tracing::debug!("Layer {} has QK-norm weights", layer_idx);
2552 }
2553
2554 let layer_type = if !config.layer_types.is_empty() && layer_idx < config.layer_types.len() {
2556 config.layer_types[layer_idx].clone()
2557 } else if config.layer_types.is_empty() && config.use_qk_norm {
2558 if (layer_idx + 1).is_multiple_of(6) {
2560 "global".to_owned()
2561 } else {
2562 "local".to_owned()
2563 }
2564 } else {
2565 "global".to_owned() };
2567
2568 let sliding_window = if config.use_qk_norm && layer_type == "local" {
2570 Some(512) } else {
2572 None
2573 };
2574
2575 let rope_theta = if layer_type == "local" {
2577 config.rope_local_base_freq.unwrap_or(config.rope_theta)
2578 } else {
2579 config.rope_theta
2580 };
2581
2582 let o_proj = if weights.contains_key(&format!("{prefix}.self_attn.o_proj.weight")) {
2585 LinearProjection::take_with_optional_bias(
2586 weights,
2587 &format!("{prefix}.self_attn.o_proj.weight"),
2588 &format!("{prefix}.self_attn.o_proj.bias"),
2589 )?
2590 } else {
2591 LinearProjection::take_with_optional_bias(
2592 weights,
2593 &format!("{prefix}.self_attn.c_proj.weight"),
2594 &format!("{prefix}.self_attn.c_proj.bias"),
2595 )?
2596 };
2597
2598 let self_attn = LlamaAttention {
2599 num_heads: config.num_attention_heads as usize,
2600 num_kv_heads: config.num_key_value_heads as usize,
2601 head_dim: config.head_dim as usize,
2602 q_proj,
2603 k_proj,
2604 v_proj,
2605 o_proj,
2606 rope_theta,
2607 rope_scaling: config.rope_scaling.clone(),
2608 q_norm,
2609 k_norm,
2610 query_pre_attn_scalar: config.query_pre_attn_scalar,
2611 sliding_window,
2612 layer_type,
2613 layer_idx,
2614 };
2615
2616 let gate_proj = LinearProjection::take_with_optional_bias(
2618 weights,
2619 &format!("{prefix}.mlp.gate_proj.weight"),
2620 &format!("{prefix}.mlp.gate_proj.bias"),
2621 )?;
2622 let up_proj = LinearProjection::take_with_optional_bias(
2623 weights,
2624 &format!("{prefix}.mlp.up_proj.weight"),
2625 &format!("{prefix}.mlp.up_proj.bias"),
2626 )?;
2627 let down_proj = LinearProjection::take_with_optional_bias(
2628 weights,
2629 &format!("{prefix}.mlp.down_proj.weight"),
2630 &format!("{prefix}.mlp.down_proj.bias"),
2631 )?;
2632
2633 let mlp = LlamaMLP {
2634 gate_proj,
2635 up_proj,
2636 down_proj,
2637 activation: config.hidden_activation.clone(),
2638 layer_idx,
2639 };
2640
2641 let input_layernorm = RMSNorm {
2643 weight: weights
2644 .get(&format!("{prefix}.input_layernorm.weight"))
2645 .ok_or_else(|| anyhow!("Missing input_layernorm weight"))?
2646 .shallow_clone(),
2647 eps: config.rms_norm_eps,
2648 };
2649
2650 let post_attention_layernorm = RMSNorm {
2651 weight: weights
2652 .get(&format!("{prefix}.post_attention_layernorm.weight"))
2653 .ok_or_else(|| anyhow!("Missing post_attention_layernorm weight"))?
2654 .shallow_clone(),
2655 eps: config.rms_norm_eps,
2656 };
2657
2658 Ok(Some(LlamaLayer {
2659 self_attn,
2660 mlp,
2661 input_layernorm,
2662 post_attention_layernorm,
2663 }))
2664 }
2665
2666 pub fn parse_config(json_str: &str) -> Result<LlamaConfig> {
2668 let json: serde_json::Value = serde_json::from_str(json_str)?;
2669
2670 let require_u64 = |field: &str| -> Result<u64> {
2674 json[field].as_u64().ok_or_else(|| {
2675 anyhow!(
2676 "config.json is missing required field `{field}` \
2677 (or it is not a non-negative integer)"
2678 )
2679 })
2680 };
2681 let num_attention_heads = require_u64("num_attention_heads")? as u32;
2682 let hidden_size = require_u64("hidden_size")? as u32;
2683 let num_hidden_layers = require_u64("num_hidden_layers")? as u32;
2684 if num_attention_heads == 0 || hidden_size == 0 || num_hidden_layers == 0 {
2685 return Err(anyhow!(
2686 "config.json has a zero architectural dimension \
2687 (num_attention_heads={num_attention_heads}, hidden_size={hidden_size}, \
2688 num_hidden_layers={num_hidden_layers})"
2689 ));
2690 }
2691
2692 let version = if json.get("rope_scaling").is_some() {
2694 3
2695 } else if json["num_key_value_heads"] != json["num_attention_heads"] {
2696 2
2697 } else {
2698 1
2699 };
2700
2701 let mut config = LlamaConfig {
2702 version,
2703 num_attention_heads,
2704 num_key_value_heads: json
2707 .get("num_key_value_heads")
2708 .and_then(serde_json::Value::as_u64)
2709 .map(|v| v as u32)
2710 .unwrap_or(num_attention_heads),
2711 hidden_size,
2712 head_dim: json
2713 .get("head_dim")
2714 .and_then(serde_json::Value::as_u64)
2715 .map(|v| v as u32)
2716 .unwrap_or_else(|| {
2717 let heads = num_attention_heads;
2720 let hidden = hidden_size;
2721 if hidden.is_multiple_of(heads * 256) {
2723 256
2724 } else if hidden.is_multiple_of(heads * 128) {
2725 128
2726 } else if hidden.is_multiple_of(heads * 64) {
2727 64
2728 } else {
2729 128 }
2731 }),
2732 intermediate_size: json["intermediate_size"].as_u64().unwrap_or(11008) as u32,
2733 max_position_embeddings: json["max_position_embeddings"].as_u64().unwrap_or(4096)
2734 as u32,
2735 rms_norm_eps: json["rms_norm_eps"].as_f64().unwrap_or(1e-5) as f32,
2736 vocab_size: json["vocab_size"].as_u64().unwrap_or(32000) as u32,
2737 original_vocab_size: json["vocab_size"].as_u64().unwrap_or(32000) as u32, num_hidden_layers,
2739 rope_theta: json
2740 .get("rope_theta")
2741 .and_then(serde_json::Value::as_f64)
2742 .unwrap_or(10000.0) as f32,
2743 rope_scaling: None,
2744 hidden_activation: json
2745 .get("hidden_activation")
2746 .and_then(|v| v.as_str())
2747 .unwrap_or("silu")
2748 .to_owned(),
2749 query_pre_attn_scalar: json
2750 .get("query_pre_attn_scalar")
2751 .and_then(serde_json::Value::as_f64)
2752 .map(|v| v as f32),
2753 use_qk_norm: false, scale_embeddings: false, layer_types: vec![],
2756 rope_local_base_freq: None,
2757 model_architecture: json
2762 .get("model_type")
2763 .and_then(|v| v.as_str())
2764 .map(|mt| {
2765 let mt = mt.to_lowercase();
2766 match mt.as_str() {
2767 "qwen" | "qwen2" | "qwen3" => ModelArchitecture::Qwen {
2768 version: if mt.starts_with("qwen3") {
2769 3
2770 } else if mt.starts_with("qwen2") {
2771 2
2772 } else {
2773 1
2774 },
2775 is_moe: false,
2776 context_length: json["max_position_embeddings"].as_u64().unwrap_or(4096)
2777 as usize,
2778 },
2779 "mistral" => ModelArchitecture::Mistral,
2780 "gemma" => ModelArchitecture::Gemma,
2781 _ => ModelArchitecture::Llama { version: 2 },
2782 }
2783 })
2784 .unwrap_or(ModelArchitecture::Llama { version: 2 }),
2785 };
2786
2787 if let Some(rope_scaling) = json.get("rope_scaling") {
2789 config.rope_scaling = Some(RopeScaling {
2790 scaling_type: rope_scaling["type"].as_str().unwrap_or("linear").to_owned(),
2791 factor: rope_scaling["factor"].as_f64().unwrap_or(8.0) as f32,
2792 });
2793 }
2794
2795 if config.vocab_size == 262144 {
2797 config.hidden_activation = "gelu_pytorch_tanh".to_owned();
2798 config.use_qk_norm = true;
2799 config.scale_embeddings = true;
2800 if config.query_pre_attn_scalar.is_none() {
2801 config.query_pre_attn_scalar = Some(256.0);
2802 }
2803 config.rope_local_base_freq = Some(10000.0);
2804 config.rope_theta = 1000000.0; tracing::info!("Detected Gemma3 model from config.json, applying Gemma3 settings");
2806 }
2807
2808 tracing::info!(
2809 "Parsed Llama config from JSON: rope_theta={}, vocab_size={}",
2810 config.rope_theta,
2811 config.vocab_size
2812 );
2813 Ok(config)
2814 }
2815
2816 fn forward_with_cache_inner(
2822 &self,
2823 input: &Tensor,
2824 start_pos: usize,
2825 delta: Option<&crate::training::TenantDelta>,
2826 ) -> Result<Tensor> {
2827 tracing::trace!("LlamaModel forward_with_cache_inner: input shape={:?}, start_pos={}, has_delta={}, config: hidden_size={}, num_layers={}",
2828 input.size(), start_pos, delta.is_some(), self.config.hidden_size, self.layers.len());
2829
2830 let mut hidden_states = if let Some(embed) = &self.embed_tokens {
2832 let input_shape = input.size();
2834
2835 let batch_size = input_shape[0];
2836 let seq_len = if input_shape.len() > 1 {
2837 input_shape[1]
2838 } else {
2839 1
2840 };
2841
2842 let flat_input = input.flatten(0, -1);
2844
2845 let embeddings = embed.index_select(0, &flat_input);
2847
2848 let emb_dims = embeddings.size();
2850 let hidden_size = emb_dims[emb_dims.len() - 1];
2851
2852 let mut embeddings = embeddings.reshape([batch_size, seq_len, hidden_size]);
2854
2855 if self.config.scale_embeddings {
2857 let scale = (hidden_size as f32).sqrt();
2858 let scale_tensor = Tensor::from_slice(&[scale])
2859 .to_kind(embeddings.kind())
2860 .to_device(embeddings.device());
2861 embeddings = broadcast_mul(&embeddings, &scale_tensor)?;
2862 }
2863
2864 embeddings
2865 } else {
2866 input.shallow_clone()
2868 };
2869
2870 tracing::trace!("Total layers to process: {}", self.layers.len());
2872
2873 let seq_len = hidden_states.size()[1];
2875 let position_ids = if start_pos == 0 {
2876 Tensor::arange(seq_len, (tch::Kind::Int64, hidden_states.device()))
2877 } else {
2878 Tensor::arange_start(
2879 start_pos as i64,
2880 (start_pos + seq_len as usize) as i64,
2881 (tch::Kind::Int64, hidden_states.device()),
2882 )
2883 };
2884 let mut position_ids = position_ids;
2885
2886 let cache_guard = self.kv_cache.as_ref().map(|cache_ref| cache_ref.lock());
2888
2889 for (idx, layer) in self.layers.iter().enumerate() {
2890 let global_idx = self.layer_offset + idx;
2895 let target = self.device_map.device_for(global_idx);
2896 if hidden_states.device() != target {
2897 tracing::debug!(
2898 layer = global_idx,
2899 from = ?hidden_states.device(),
2900 to = ?target,
2901 "Llama cached forward crossing device boundary"
2902 );
2903 hidden_states = hidden_states.to_device(target);
2904 position_ids = position_ids.to_device(target);
2905 }
2906
2907 let residual = hidden_states.shallow_clone();
2908
2909 hidden_states = layer.input_layernorm.forward(&hidden_states)?;
2911
2912 let attn_output = if let Some(ref cache_manager) = cache_guard {
2914 if let Some(result) = cache_manager.with_layer_cache(idx, |layer_cache| {
2915 tracing::trace!(
2916 "Layer {}: Using KV cache, cache_pos={}",
2917 idx,
2918 layer_cache.seq_pos
2919 );
2920 layer.self_attn.forward(
2921 &hidden_states,
2922 Some(&position_ids),
2923 Some(layer_cache),
2924 start_pos,
2925 delta,
2926 )
2927 }) {
2928 result?
2929 } else {
2930 layer.self_attn.forward(
2931 &hidden_states,
2932 Some(&position_ids),
2933 None,
2934 start_pos,
2935 delta,
2936 )?
2937 }
2938 } else {
2939 layer.self_attn.forward(
2940 &hidden_states,
2941 Some(&position_ids),
2942 None,
2943 start_pos,
2944 delta,
2945 )?
2946 };
2947 hidden_states = residual + attn_output;
2948
2949 let residual = hidden_states.shallow_clone();
2951 hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
2952 let ffn_output = layer.mlp.forward(&hidden_states, delta)?;
2953 hidden_states = residual + ffn_output;
2954 }
2955
2956 if hidden_states.device() != self.device {
2961 tracing::debug!(
2962 from = ?hidden_states.device(),
2963 to = ?self.device,
2964 "Llama cached forward returning activation to output device"
2965 );
2966 hidden_states = hidden_states.to_device(self.device);
2967 }
2968
2969 if let Some(norm) = &self.norm {
2971 hidden_states = norm.forward(&hidden_states)?;
2972 }
2973
2974 if let Some(lm_head) = &self.lm_head {
2976 hidden_states = hidden_states.matmul(lm_head);
2977 } else if let Some(output_proj) = &self.lm_head_transposed {
2978 let hs_shape = hidden_states.size();
2979 if hs_shape.len() == 3 {
2980 let (batch_size, seq_len, hidden_size) = (hs_shape[0], hs_shape[1], hs_shape[2]);
2981 let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
2982 let logits_2d = hidden_2d.matmul(output_proj);
2983 hidden_states = logits_2d.reshape([batch_size, seq_len, output_proj.size()[1]]);
2984 } else {
2985 hidden_states = hidden_states.matmul(output_proj);
2986 }
2987 } else {
2988 tracing::warn!("No LM head or embedding weights found - returning hidden states!");
2989 }
2990
2991 let original_vocab_size = self.config.original_vocab_size;
2993 let padded_vocab_size = self.config.vocab_size;
2994 if padded_vocab_size > original_vocab_size && original_vocab_size > 0 {
2995 let logits_shape = hidden_states.size();
2996 let actual_vocab_size = logits_shape[logits_shape.len() - 1] as u32;
2997
2998 if actual_vocab_size == padded_vocab_size {
2999 let mask_start = original_vocab_size as i64;
3000 let mask_count = (padded_vocab_size - original_vocab_size) as i64;
3001
3002 if mask_count > 0 {
3003 let mask_values = Tensor::full(
3004 [mask_count],
3005 -1e10_f64,
3006 (hidden_states.kind(), hidden_states.device()),
3007 );
3008
3009 if logits_shape.len() == 3 {
3010 let batch_size = logits_shape[0];
3011 let seq_len = logits_shape[1];
3012 for b in 0..batch_size {
3013 for s in 0..seq_len {
3014 let slice = hidden_states.select(0, b).select(0, s);
3015 slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3016 }
3017 }
3018 } else if logits_shape.len() == 2 {
3019 let rows = logits_shape[0];
3020 for i in 0..rows {
3021 let slice = hidden_states.select(0, i);
3022 slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3023 }
3024 } else {
3025 hidden_states
3026 .narrow(0, mask_start, mask_count)
3027 .copy_(&mask_values);
3028 }
3029 }
3030 }
3031 }
3032
3033 Ok(hidden_states)
3034 }
3035
3036 fn mask_padded_vocab(&self, logits: &Tensor) {
3042 let original_vocab_size = self.config.original_vocab_size;
3043 let padded_vocab_size = self.config.vocab_size;
3044 if padded_vocab_size <= original_vocab_size || original_vocab_size == 0 {
3045 return;
3046 }
3047 let logits_shape = logits.size();
3048 let actual_vocab_size = logits_shape[logits_shape.len() - 1] as u32;
3049 if actual_vocab_size != padded_vocab_size {
3050 return;
3051 }
3052 let mask_start = original_vocab_size as i64;
3053 let mask_count = (padded_vocab_size - original_vocab_size) as i64;
3054 if mask_count <= 0 {
3055 return;
3056 }
3057 let mask_values = Tensor::full([mask_count], -1e10_f64, (logits.kind(), logits.device()));
3058 match logits_shape.len() {
3059 3 => {
3060 for b in 0..logits_shape[0] {
3061 for s in 0..logits_shape[1] {
3062 let slice = logits.select(0, b).select(0, s);
3063 slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3064 }
3065 }
3066 }
3067 2 => {
3068 for i in 0..logits_shape[0] {
3069 let slice = logits.select(0, i);
3070 slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3071 }
3072 }
3073 _ => {
3074 logits.narrow(0, mask_start, mask_count).copy_(&mask_values);
3075 }
3076 }
3077 }
3078
3079 fn forward_batched_impl(
3093 &self,
3094 sequences: &mut [(
3095 Vec<i64>,
3096 usize,
3097 std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
3098 )],
3099 delta: Option<&crate::training::TenantDelta>,
3100 ) -> Result<Tensor> {
3101 let batch_size = sequences.len();
3102 if batch_size == 0 {
3103 return Err(anyhow!("forward_batched: empty batch"));
3104 }
3105 let q_len = sequences[0].0.len();
3106 if q_len == 0 {
3107 return Err(anyhow!("forward_batched: empty sequence"));
3108 }
3109 if sequences.iter().any(|(toks, ..)| toks.len() != q_len) {
3110 return Err(anyhow!(
3111 "forward_batched: ragged query lengths are not supported in v1 (all rows must share q_len)"
3112 ));
3113 }
3114 let device = self.device;
3115
3116 let flat_ids: Vec<i64> = sequences
3118 .iter()
3119 .flat_map(|(t, ..)| t.iter().copied())
3120 .collect();
3121 let input_ids = Tensor::from_slice(&flat_ids)
3122 .reshape([batch_size as i64, q_len as i64])
3123 .to_device(device);
3124 let mut hidden_states = self.embed_tokens(&input_ids)?;
3125
3126 let start_positions: Vec<usize> = sequences.iter().map(|&(_, sp, _)| sp).collect();
3128 let position_ids: Vec<Tensor> = start_positions
3129 .iter()
3130 .map(|&sp| {
3131 Tensor::arange_start(sp as i64, (sp + q_len) as i64, (tch::Kind::Int64, device))
3132 })
3133 .collect();
3134 let kv_lens: Vec<i64> = start_positions
3136 .iter()
3137 .map(|&sp| (sp + q_len) as i64)
3138 .collect();
3139 let kv_max = kv_lens.iter().copied().max().unwrap_or(0);
3140
3141 let neg = -1.0e9_f64;
3145 let mask = Tensor::zeros(
3146 [batch_size as i64, 1, q_len as i64, kv_max],
3147 (tch::Kind::Float, device),
3148 );
3149 {
3150 let cols = Tensor::arange(kv_max, (tch::Kind::Int64, device));
3152 for r in 0..batch_size {
3153 let sp = start_positions[r] as i64;
3154 let kv_len = kv_lens[r];
3155 for i in 0..q_len as i64 {
3156 let allowed_upto = sp + i; let valid = cols.le(allowed_upto).logical_and(&cols.lt(kv_len));
3159 let row_mask = valid.logical_not().to_kind(tch::Kind::Float) * neg; mask.select(0, r as i64)
3161 .select(0, 0)
3162 .select(0, i)
3163 .copy_(&row_mask);
3164 }
3165 }
3166 }
3167
3168 let guards: Vec<_> = sequences.iter().map(|(_, _, c)| c.lock()).collect();
3171
3172 let mut position_refs: Vec<Tensor> = position_ids; let mut mask = mask;
3174
3175 for (idx, layer) in self.layers.iter().enumerate() {
3176 let global_idx = self.layer_offset + idx;
3180 let target = self.device_map.device_for(global_idx);
3181 if hidden_states.device() != target {
3182 tracing::debug!(
3183 layer = global_idx,
3184 from = ?hidden_states.device(),
3185 to = ?target,
3186 "Llama batched forward crossing device boundary"
3187 );
3188 hidden_states = hidden_states.to_device(target);
3189 for positions in &mut position_refs {
3190 *positions = positions.to_device(target);
3191 }
3192 mask = mask.to_device(target);
3193 }
3194
3195 let residual = hidden_states.shallow_clone();
3196 hidden_states = layer.input_layernorm.forward(&hidden_states)?;
3197
3198 let attn_output = {
3202 let mut layer_caches: Vec<
3203 dashmap::mapref::one::RefMut<usize, crate::runtime::kv_cache::LayerKVCache>,
3204 > = Vec::with_capacity(batch_size);
3205 for g in guards.iter() {
3206 let cell = g
3207 .layer_cache_ref(idx)
3208 .ok_or_else(|| anyhow!("forward_batched: missing layer cache {idx}"))?;
3209 layer_caches.push(cell);
3210 }
3211 let mut cache_refs: Vec<&mut crate::runtime::kv_cache::LayerKVCache> =
3212 layer_caches.iter_mut().map(|c| &mut **c).collect();
3213 layer.self_attn.forward_batched(
3214 &hidden_states,
3215 &position_refs,
3216 &mut cache_refs,
3217 &start_positions,
3218 &mask,
3219 delta,
3220 )?
3221 };
3222 hidden_states = residual + attn_output;
3223
3224 let residual = hidden_states.shallow_clone();
3225 hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
3226 let ffn_output = layer.mlp.forward(&hidden_states, delta)?;
3227 hidden_states = residual + ffn_output;
3228 }
3229 drop(guards);
3230
3231 if hidden_states.device() != self.device {
3232 tracing::debug!(
3233 from = ?hidden_states.device(),
3234 to = ?self.device,
3235 "Llama batched forward returning activation to output device"
3236 );
3237 hidden_states = hidden_states.to_device(self.device);
3238 }
3239
3240 hidden_states = self.apply_final_norm(&hidden_states)?;
3241 let logits = self.lm_head(&hidden_states)?;
3242 self.mask_padded_vocab(&logits);
3243 Ok(logits)
3244 }
3245
3246 fn forward_layers_inner(
3255 &self,
3256 hidden: &Tensor,
3257 range: std::ops::Range<usize>,
3258 start_pos: usize,
3259 delta: Option<&crate::training::TenantDelta>,
3260 ) -> Result<Tensor> {
3261 let num_global = self.config.num_hidden_layers as usize;
3262 if range.end > num_global || range.start >= range.end {
3263 return Err(anyhow!(
3264 "forward_layers: invalid global range {range:?} for {num_global} layers"
3265 ));
3266 }
3267 let owned = self.layer_offset..self.layer_offset + self.layers.len();
3269 if range.start < owned.start || range.end > owned.end {
3270 return Err(anyhow!(
3271 "forward_layers: range {range:?} not within this stage's owned layers {owned:?}"
3272 ));
3273 }
3274
3275 let mut hidden_states = hidden.shallow_clone();
3276
3277 let seq_len = hidden_states.size()[1];
3281 let position_ids = if start_pos == 0 {
3282 Tensor::arange(seq_len, (tch::Kind::Int64, hidden_states.device()))
3283 } else {
3284 Tensor::arange_start(
3285 start_pos as i64,
3286 (start_pos + seq_len as usize) as i64,
3287 (tch::Kind::Int64, hidden_states.device()),
3288 )
3289 };
3290 let mut position_ids = position_ids;
3292
3293 let cache_guard = self.kv_cache.as_ref().map(|cache_ref| cache_ref.lock());
3294
3295 for g in range {
3296 let local_idx = g - self.layer_offset;
3297 let layer = &self.layers[local_idx];
3298
3299 let target = self.device_map.device_for(g);
3302 if hidden_states.device() != target {
3303 hidden_states = hidden_states.to_device(target);
3304 position_ids = position_ids.to_device(target);
3305 }
3306
3307 let residual = hidden_states.shallow_clone();
3308 hidden_states = layer.input_layernorm.forward(&hidden_states)?;
3309
3310 let attn_output = if let Some(ref cache_manager) = cache_guard {
3311 if let Some(result) = cache_manager.with_layer_cache(local_idx, |layer_cache| {
3312 layer.self_attn.forward(
3313 &hidden_states,
3314 Some(&position_ids),
3315 Some(layer_cache),
3316 start_pos,
3317 delta,
3318 )
3319 }) {
3320 result?
3321 } else {
3322 layer.self_attn.forward(
3323 &hidden_states,
3324 Some(&position_ids),
3325 None,
3326 start_pos,
3327 delta,
3328 )?
3329 }
3330 } else {
3331 layer.self_attn.forward(
3332 &hidden_states,
3333 Some(&position_ids),
3334 None,
3335 start_pos,
3336 delta,
3337 )?
3338 };
3339 hidden_states = residual + attn_output;
3340
3341 let residual = hidden_states.shallow_clone();
3342 hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
3343 let ffn_output = layer.mlp.forward(&hidden_states, delta)?;
3344 hidden_states = residual + ffn_output;
3345 }
3346
3347 Ok(hidden_states)
3348 }
3349
3350 fn forward_layers_train_inner(
3362 &self,
3363 hidden: &Tensor,
3364 range: std::ops::Range<usize>,
3365 delta: Option<&crate::training::TenantDelta>,
3366 ) -> Result<Tensor> {
3367 let num_global = self.config.num_hidden_layers as usize;
3368 if range.end > num_global || range.start >= range.end {
3369 return Err(anyhow!(
3370 "forward_layers_train: invalid global range {range:?} for {num_global} layers"
3371 ));
3372 }
3373 let owned = self.layer_offset..self.layer_offset + self.layers.len();
3374 if range.start < owned.start || range.end > owned.end {
3375 return Err(anyhow!(
3376 "forward_layers_train: range {range:?} not within this stage's owned layers {owned:?}"
3377 ));
3378 }
3379
3380 let mut hidden_states = hidden.shallow_clone();
3381
3382 let seq_len = hidden_states.size()[1];
3385 let mut position_ids = Tensor::arange(seq_len, (tch::Kind::Int64, hidden_states.device()));
3386
3387 for g in range {
3391 let local_idx = g - self.layer_offset;
3392 let layer = &self.layers[local_idx];
3393
3394 let target = self.device_map.device_for(g);
3397 if hidden_states.device() != target {
3398 hidden_states = hidden_states.to_device(target);
3399 position_ids = position_ids.to_device(target);
3400 }
3401
3402 let residual = hidden_states.shallow_clone();
3403 hidden_states = layer.input_layernorm.forward(&hidden_states)?;
3404 let attn_output = layer.self_attn.forward(
3405 &hidden_states,
3406 Some(&position_ids),
3407 None, 0, delta,
3410 )?;
3411 hidden_states = residual + attn_output;
3412
3413 let residual = hidden_states.shallow_clone();
3414 hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
3415 let ffn_output = layer.mlp.forward(&hidden_states, delta)?;
3416 hidden_states = residual + ffn_output;
3417 }
3418
3419 Ok(hidden_states)
3420 }
3421}
3422
3423impl ModelOperations for LlamaModel {
3424 fn architecture(&self) -> ModelArchitecture {
3425 match &self.config.model_architecture {
3430 ModelArchitecture::Llama { .. } | ModelArchitecture::Custom(_) => {
3431 ModelArchitecture::Llama {
3432 version: self.config.version,
3433 }
3434 }
3435 other => other.clone(),
3436 }
3437 }
3438
3439 fn config(&self) -> &dyn ArchitectureConfig {
3440 &self.config
3441 }
3442
3443 fn forward(&self, input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor> {
3444 self.forward_with_cache(input, 0)
3446 }
3447
3448 fn var_store(&self) -> Option<&nn::VarStore> {
3449 self.vs.as_ref()
3450 }
3451
3452 fn var_store_mut(&mut self) -> Option<&mut nn::VarStore> {
3453 self.vs.as_mut()
3454 }
3455
3456 fn forward_with_cache(&self, input: &Tensor, start_pos: usize) -> Result<Tensor> {
3457 self.forward_with_cache_inner(input, start_pos, None)
3458 }
3459
3460 fn forward_with_cache_and_delta(
3461 &self,
3462 input: &Tensor,
3463 start_pos: usize,
3464 delta: Option<&crate::training::TenantDelta>,
3465 ) -> Result<Tensor> {
3466 self.forward_with_cache_inner(input, start_pos, delta)
3467 }
3468
3469 fn forward_batched(
3470 &self,
3471 sequences: &mut [(
3472 Vec<i64>,
3473 usize,
3474 std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
3475 )],
3476 delta: Option<&crate::training::TenantDelta>,
3477 ) -> Result<Tensor> {
3478 self.forward_batched_impl(sequences, delta)
3479 }
3480
3481 fn embed_tokens(&self, input_ids: &Tensor) -> Result<Tensor> {
3482 if let Some(embed) = &self.embed_tokens {
3483 let input_shape = input_ids.size();
3485 let batch_size = input_shape[0];
3486 let seq_len = if input_shape.len() > 1 {
3487 input_shape[1]
3488 } else {
3489 1
3490 };
3491
3492 let flat_input = input_ids.flatten(0, -1);
3494
3495 let embeddings = embed.index_select(0, &flat_input);
3497
3498 let emb_dims = embeddings.size();
3500 let hidden_size = emb_dims[emb_dims.len() - 1];
3501
3502 let mut embeddings = embeddings.reshape([batch_size, seq_len, hidden_size]);
3504
3505 if self.config.scale_embeddings {
3507 let scale = (hidden_size as f32).sqrt();
3508 let scale_tensor = Tensor::from_slice(&[scale])
3509 .to_kind(embeddings.kind())
3510 .to_device(embeddings.device());
3511 embeddings = broadcast_mul(&embeddings, &scale_tensor)?;
3512 }
3513
3514 Ok(embeddings)
3515 } else {
3516 Err(anyhow!("No embedding layer available in model"))
3517 }
3518 }
3519
3520 fn forward_from_embeddings(&self, embeddings: &Tensor, start_pos: usize) -> Result<Tensor> {
3521 tracing::trace!(
3522 "LlamaModel forward_from_embeddings: embeddings shape={:?}, start_pos={}",
3523 embeddings.size(),
3524 start_pos
3525 );
3526
3527 let mut hidden_states = embeddings.shallow_clone();
3529
3530 let seq_len = hidden_states.size()[1];
3532 let position_ids = if start_pos == 0 {
3533 Tensor::arange(seq_len, (tch::Kind::Int64, hidden_states.device()))
3534 } else {
3535 Tensor::arange_start(
3536 start_pos as i64,
3537 (start_pos + seq_len as usize) as i64,
3538 (tch::Kind::Int64, hidden_states.device()),
3539 )
3540 };
3541
3542 for (idx, layer) in self.layers.iter().enumerate() {
3543 let residual = hidden_states.shallow_clone();
3544
3545 hidden_states = layer.input_layernorm.forward(&hidden_states)?;
3547
3548 let attn_output = if let Some(cache_ref) = self.kv_cache.as_ref() {
3550 let cache_manager = cache_ref.lock();
3551 if let Some(result) = cache_manager.with_layer_cache(idx, |layer_cache| {
3552 layer.self_attn.forward(
3553 &hidden_states,
3554 Some(&position_ids),
3555 Some(layer_cache),
3556 start_pos,
3557 None,
3558 )
3559 }) {
3560 result?
3561 } else {
3562 layer.self_attn.forward(
3563 &hidden_states,
3564 Some(&position_ids),
3565 None,
3566 start_pos,
3567 None,
3568 )?
3569 }
3570 } else {
3571 layer.self_attn.forward(
3572 &hidden_states,
3573 Some(&position_ids),
3574 None,
3575 start_pos,
3576 None,
3577 )?
3578 };
3579 hidden_states = residual + attn_output;
3580
3581 let residual = hidden_states.shallow_clone();
3583 hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
3584 let ffn_output = layer.mlp.forward(&hidden_states, None)?;
3585 hidden_states = residual + ffn_output;
3586 }
3587
3588 if let Some(norm) = &self.norm {
3590 hidden_states = norm.forward(&hidden_states)?;
3591 }
3592
3593 if let Some(lm_head) = &self.lm_head {
3595 hidden_states = hidden_states.matmul(lm_head);
3596 } else if let Some(output_proj) = &self.lm_head_transposed {
3597 let hs_shape = hidden_states.size();
3599 if hs_shape.len() == 3 {
3600 let (batch_size, seq_len, hidden_size) = (hs_shape[0], hs_shape[1], hs_shape[2]);
3601 let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
3602 let logits_2d = hidden_2d.matmul(output_proj);
3603 hidden_states = logits_2d.reshape([batch_size, seq_len, -1]);
3604 } else {
3605 hidden_states = hidden_states.matmul(output_proj);
3606 }
3607 }
3608
3609 let padded_vocab_size = self.config.vocab_size;
3611 let original_vocab_size = self.config.original_vocab_size;
3612 if padded_vocab_size > original_vocab_size && original_vocab_size > 0 {
3613 let logits_shape = hidden_states.size();
3614 let actual_vocab_size = logits_shape[logits_shape.len() - 1] as u32;
3615
3616 if actual_vocab_size == padded_vocab_size {
3617 let mask_start = original_vocab_size as i64;
3618 let mask_count = (padded_vocab_size - original_vocab_size) as i64;
3619
3620 if mask_count > 0 {
3621 let mask_values = Tensor::full(
3622 [mask_count],
3623 -1e10_f64,
3624 (hidden_states.kind(), hidden_states.device()),
3625 );
3626
3627 if logits_shape.len() == 3 {
3628 let batch_size = logits_shape[0];
3629 let seq_len = logits_shape[1];
3630 for b in 0..batch_size {
3631 for s in 0..seq_len {
3632 let slice = hidden_states.select(0, b).select(0, s);
3633 slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3634 }
3635 }
3636 } else if logits_shape.len() == 2 {
3637 let rows = logits_shape[0];
3638 for i in 0..rows {
3639 let slice = hidden_states.select(0, i);
3640 slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3641 }
3642 } else {
3643 hidden_states
3644 .narrow(0, mask_start, mask_count)
3645 .copy_(&mask_values);
3646 }
3647 }
3648 }
3649 }
3650
3651 Ok(hidden_states)
3652 }
3653
3654 fn reshape_for_attention(&self, tensor: &Tensor, is_key_value: bool) -> Result<Tensor> {
3655 let (batch_size, seq_len, _hidden_size) = dims3(tensor)?;
3656
3657 if is_key_value {
3658 Ok(tensor.reshape([
3660 batch_size,
3661 seq_len,
3662 self.config.num_key_value_heads as i64,
3663 self.config.head_dim as i64,
3664 ]))
3665 } else {
3666 Ok(tensor.reshape([
3668 batch_size,
3669 seq_len,
3670 self.config.num_attention_heads as i64,
3671 self.config.head_dim as i64,
3672 ]))
3673 }
3674 }
3675
3676 fn decode_layer(
3677 &self,
3678 layer_idx: usize,
3679 hidden_states: &Tensor,
3680 _attention_mask: Option<&Tensor>,
3681 position_ids: Option<&Tensor>,
3682 _past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
3683 ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
3684 let layer = self.layers.get(layer_idx).ok_or_else(|| {
3685 anyhow!(
3686 "Layer index {} out of range (model has {} layers)",
3687 layer_idx,
3688 self.layers.len()
3689 )
3690 })?;
3691
3692 let residual = hidden_states.shallow_clone();
3693
3694 let normed = layer.input_layernorm.forward(hidden_states)?;
3696
3697 let attn_output = if let Some(cache_ref) = self.kv_cache.as_ref() {
3699 let cache_manager = cache_ref.lock();
3700 if let Some(result) = cache_manager.with_layer_cache(layer_idx, |layer_cache| {
3701 layer.self_attn.forward(
3702 &normed,
3703 position_ids,
3704 Some(layer_cache),
3705 0, None,
3707 )
3708 }) {
3709 result?
3710 } else {
3711 layer
3712 .self_attn
3713 .forward(&normed, position_ids, None, 0, None)?
3714 }
3715 } else {
3716 layer
3717 .self_attn
3718 .forward(&normed, position_ids, None, 0, None)?
3719 };
3720
3721 let hidden_states = residual + attn_output;
3722
3723 let residual = hidden_states.shallow_clone();
3725 let normed = layer.post_attention_layernorm.forward(&hidden_states)?;
3726 let ffn_output = layer.mlp.forward(&normed, None)?;
3727 let hidden_states = residual + ffn_output;
3728
3729 Ok((hidden_states, None))
3731 }
3732
3733 fn decode_layer_with_delta(
3734 &self,
3735 layer_idx: usize,
3736 hidden_states: &Tensor,
3737 _attention_mask: Option<&Tensor>,
3738 position_ids: Option<&Tensor>,
3739 _past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
3740 delta: &crate::training::TenantDelta,
3741 ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
3742 let layer = self.layers.get(layer_idx).ok_or_else(|| {
3743 anyhow!(
3744 "Layer index {} out of range (model has {} layers)",
3745 layer_idx,
3746 self.layers.len()
3747 )
3748 })?;
3749
3750 let residual = hidden_states.shallow_clone();
3751
3752 let normed = layer.input_layernorm.forward(hidden_states)?;
3754 let attn_output = layer
3755 .self_attn
3756 .forward(&normed, position_ids, None, 0, Some(delta))?;
3757 let hidden_states = residual + attn_output;
3758
3759 let residual = hidden_states.shallow_clone();
3761 let normed = layer.post_attention_layernorm.forward(&hidden_states)?;
3762 let ffn_output = layer.mlp.forward(&normed, Some(delta))?;
3763 let hidden_states = residual + ffn_output;
3764
3765 Ok((hidden_states, None))
3766 }
3767
3768 fn forward_layers(
3769 &self,
3770 hidden: &Tensor,
3771 range: std::ops::Range<usize>,
3772 start_pos: usize,
3773 delta: Option<&crate::training::TenantDelta>,
3774 ) -> Result<Tensor> {
3775 self.forward_layers_inner(hidden, range, start_pos, delta)
3776 }
3777
3778 fn forward_layers_train(
3779 &self,
3780 hidden: &Tensor,
3781 range: std::ops::Range<usize>,
3782 delta: Option<&crate::training::TenantDelta>,
3783 ) -> Result<Tensor> {
3784 self.forward_layers_train_inner(hidden, range, delta)
3785 }
3786
3787 fn apply_final_norm(&self, hidden_states: &Tensor) -> Result<Tensor> {
3788 if let Some(norm) = &self.norm {
3789 norm.forward(hidden_states)
3790 } else {
3791 Ok(hidden_states.shallow_clone())
3792 }
3793 }
3794
3795 fn lm_head(&self, hidden_states: &Tensor) -> Result<Tensor> {
3796 if let Some(lm_head) = &self.lm_head {
3797 let lm_head = lm_head.to_kind(hidden_states.kind());
3802 Ok(hidden_states
3803 .f_matmul(&lm_head)
3804 .map_err(|e| anyhow!("lm_head matmul failed: {}", e))?)
3805 } else if let Some(output_proj) = &self.lm_head_transposed {
3806 let output_proj = output_proj.to_kind(hidden_states.kind());
3807 let hs_shape = hidden_states.size();
3808 if hs_shape.len() == 3 {
3809 let (batch_size, seq_len, hidden_size) = (hs_shape[0], hs_shape[1], hs_shape[2]);
3810 let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
3811 let logits_2d = hidden_2d
3812 .f_matmul(&output_proj)
3813 .map_err(|e| anyhow!("lm_head transposed matmul failed: {}", e))?;
3814 Ok(logits_2d.reshape([batch_size, seq_len, -1]))
3815 } else {
3816 Ok(hidden_states
3817 .f_matmul(&output_proj)
3818 .map_err(|e| anyhow!("lm_head transposed matmul failed: {}", e))?)
3819 }
3820 } else {
3821 Err(anyhow!("No LM head or embedding weights found"))
3822 }
3823 }
3824
3825 fn num_layers(&self) -> usize {
3826 self.config.num_hidden_layers as usize
3827 }
3828
3829 fn apply_rope(&self, _tensor: &Tensor, _position_ids: &Tensor) -> Result<Tensor> {
3830 Err(anyhow!(
3833 "RoPE should be applied in attention layers, not at model level"
3834 ))
3835 }
3836
3837 fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
3838 let original_dtype = tensor.kind();
3840 let x2 = square_tensor(tensor)?;
3841 let mean = x2.mean_dim(&[-1i64][..], true, original_dtype);
3842 let rrms = (mean + self.config.rms_norm_eps as f64).reciprocal().sqrt();
3843 broadcast_mul(tensor, &rrms)
3844 }
3845
3846 fn get_attention_mask(&self, seq_len: usize, past_kv_len: usize) -> Result<Tensor> {
3847 let total_len = seq_len + past_kv_len;
3849 let mask = Tensor::ones(
3850 [seq_len as i64, total_len as i64],
3851 (self.dtype, self.device),
3852 );
3853
3854 Ok(mask)
3856 }
3857
3858 fn as_any(&self) -> &dyn std::any::Any {
3859 self
3860 }
3861
3862 fn clear_kv_cache(&self) {
3867 if let Some(cache_ref) = self.kv_cache.as_ref() {
3868 let mut cache_manager = cache_ref.lock();
3869 cache_manager.clear_all();
3870 }
3871 }
3872
3873 fn kv_cache_memory_usage(&self) -> usize {
3874 self.kv_cache
3875 .as_ref()
3876 .map(|cache_ref| cache_ref.lock().memory_usage())
3877 .unwrap_or(0)
3878 }
3879
3880 fn set_kv_cache(
3881 &mut self,
3882 cache: std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
3883 ) {
3884 self.kv_cache = Some(cache);
3885 }
3886
3887 fn get_kv_cache(
3888 &self,
3889 ) -> Option<std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>> {
3890 self.kv_cache.clone()
3891 }
3892
3893 fn take_kv_cache(
3894 &mut self,
3895 ) -> Option<std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>> {
3896 self.kv_cache.take()
3897 }
3898}
3899
3900#[cfg(test)]
3903#[allow(clippy::unwrap_used, clippy::expect_used)]
3904mod pipeline_tests {
3905 use super::*;
3906 use crate::runtime::device_pool::LayerDeviceMap;
3907 use hyprstream_rpc_std::model_client::KVQuantType;
3908
3909 const HIDDEN: i64 = 16;
3910 const HEADS: i64 = 2;
3911 const HEAD_DIM: i64 = 8; const INTER: i64 = 32;
3913 const VOCAB: i64 = 48; const LAYERS: usize = 4;
3915
3916 fn tiny_config() -> LlamaConfig {
3917 LlamaConfig {
3918 version: 2,
3919 num_attention_heads: HEADS as u32,
3920 num_key_value_heads: HEADS as u32,
3921 hidden_size: HIDDEN as u32,
3922 head_dim: HEAD_DIM as u32,
3923 intermediate_size: INTER as u32,
3924 max_position_embeddings: 64,
3925 rms_norm_eps: 1e-5,
3926 vocab_size: VOCAB as u32,
3927 original_vocab_size: VOCAB as u32,
3928 num_hidden_layers: LAYERS as u32,
3929 rope_theta: 10000.0,
3930 rope_scaling: None,
3931 hidden_activation: "silu".to_owned(),
3932 query_pre_attn_scalar: None,
3933 use_qk_norm: false,
3934 scale_embeddings: false,
3935 layer_types: vec![],
3936 rope_local_base_freq: None,
3937 model_architecture: ModelArchitecture::Llama { version: 2 },
3938 }
3939 }
3940
3941 fn tiny_weights() -> HashMap<String, Tensor> {
3949 let opt = (DType::Float, Device::Cpu);
3950 let mut w = HashMap::new();
3951 let randn = |dims: &[i64]| {
3952 let n: i64 = dims.iter().product();
3953 (Tensor::arange(n, opt) * 0.017).sin().reshape(dims) * 0.05
3955 };
3956
3957 w.insert(
3958 "model.embed_tokens.weight".to_owned(),
3959 randn(&[VOCAB, HIDDEN]),
3960 );
3961 w.insert("model.norm.weight".to_owned(), Tensor::ones([HIDDEN], opt));
3962 w.insert("lm_head.weight".to_owned(), randn(&[VOCAB, HIDDEN]));
3963
3964 for i in 0..LAYERS {
3965 let p = format!("model.layers.{i}");
3966 w.insert(
3968 format!("{p}.self_attn.q_proj.weight"),
3969 randn(&[HEADS * HEAD_DIM, HIDDEN]),
3970 );
3971 w.insert(
3972 format!("{p}.self_attn.k_proj.weight"),
3973 randn(&[HEADS * HEAD_DIM, HIDDEN]),
3974 );
3975 w.insert(
3976 format!("{p}.self_attn.v_proj.weight"),
3977 randn(&[HEADS * HEAD_DIM, HIDDEN]),
3978 );
3979 w.insert(
3980 format!("{p}.self_attn.o_proj.weight"),
3981 randn(&[HIDDEN, HEADS * HEAD_DIM]),
3982 );
3983 w.insert(format!("{p}.mlp.gate_proj.weight"), randn(&[INTER, HIDDEN]));
3985 w.insert(format!("{p}.mlp.up_proj.weight"), randn(&[INTER, HIDDEN]));
3986 w.insert(format!("{p}.mlp.down_proj.weight"), randn(&[HIDDEN, INTER]));
3987 w.insert(
3989 format!("{p}.input_layernorm.weight"),
3990 Tensor::ones([HIDDEN], opt),
3991 );
3992 w.insert(
3993 format!("{p}.post_attention_layernorm.weight"),
3994 Tensor::ones([HIDDEN], opt),
3995 );
3996 }
3997 w
3998 }
3999
4000 fn whole_model() -> LlamaModel {
4001 let mut w = tiny_weights();
4002 LlamaModel::from_weights_with_config(
4003 &mut w,
4004 tiny_config(),
4005 &Device::Cpu,
4006 DType::Float,
4007 KVQuantType::None,
4008 )
4009 .unwrap()
4010 }
4011
4012 fn orchestrated_logits(m: &LlamaModel, input: &Tensor) -> Tensor {
4015 let emb = m.embed_tokens(input).unwrap();
4016 let h = m.forward_layers(&emb, 0..m.num_layers(), 0, None).unwrap();
4017 let h = m.apply_final_norm(&h).unwrap();
4018 m.lm_head(&h).unwrap()
4019 }
4020
4021 #[test]
4022 fn forward_layers_full_range_matches_whole_model_forward() {
4023 let input = Tensor::from_slice(&[1i64, 5, 9, 2]).reshape([1, 4]);
4026
4027 let reference = whole_model();
4029 let ref_logits = reference.forward_with_cache(&input, 0).unwrap();
4030
4031 let piped = whole_model();
4035 let pipe_logits = orchestrated_logits(&piped, &input);
4036
4037 let orig = reference.config.original_vocab_size as i64;
4038 let ref_c = ref_logits.narrow(2, 0, orig);
4039 let pipe_c = pipe_logits.narrow(2, 0, orig);
4040 let max_diff = (&ref_c - &pipe_c).abs().max().double_value(&[]);
4041 assert!(
4042 ref_c.allclose(&pipe_c, 1e-4, 1e-4, false),
4043 "forward_layers(0..N) over a single-device map must equal whole-model forward \
4044 (max_diff={max_diff}, ref_shape={:?})",
4045 ref_c.size()
4046 );
4047 assert!(piped.device_map.is_single_device());
4049 assert_eq!(piped.layer_offset, 0);
4050 }
4051
4052 #[test]
4053 fn staged_split_equals_whole_model() {
4054 let input = Tensor::from_slice(&[3i64, 7, 1, 4]).reshape([1, 4]);
4060
4061 let reference = whole_model();
4062 let ref_logits = reference.forward_with_cache(&input, 0).unwrap();
4063
4064 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4066 let split = LAYERS / 2;
4067
4068 let mut w0 = tiny_weights();
4069 let stage0 = LlamaModel::stage_from_weights_with_config(
4070 &mut w0,
4071 tiny_config(),
4072 &map,
4073 0..split,
4074 DType::Float,
4075 KVQuantType::None,
4076 )
4077 .unwrap();
4078 assert_eq!(stage0.layer_offset, 0);
4079 assert!(stage0.embed_tokens.is_some(), "first stage keeps embed");
4080 assert!(
4081 stage0.norm.is_none() && stage0.lm_head.is_none(),
4082 "first stage has no head"
4083 );
4084
4085 let mut w1 = tiny_weights();
4086 let stage1 = LlamaModel::stage_from_weights_with_config(
4087 &mut w1,
4088 tiny_config(),
4089 &map,
4090 split..LAYERS,
4091 DType::Float,
4092 KVQuantType::None,
4093 )
4094 .unwrap();
4095 assert_eq!(stage1.layer_offset, split);
4096 assert!(stage1.embed_tokens.is_none(), "last stage has no embed");
4097 assert!(stage1.norm.is_some(), "last stage keeps final norm");
4098
4099 let emb = stage0.embed_tokens(&input).unwrap();
4101 let h0 = stage0.forward_layers(&emb, 0..split, 0, None).unwrap();
4102 let h1 = stage1.forward_layers(&h0, split..LAYERS, 0, None).unwrap();
4103 let h1 = stage1.apply_final_norm(&h1).unwrap();
4104 let logits = stage1.lm_head(&h1).unwrap();
4105
4106 let orig = reference.config.original_vocab_size as i64;
4107 let ref_c = ref_logits.narrow(2, 0, orig);
4108 let pipe_c = logits.narrow(2, 0, orig);
4109 let max_diff = (&ref_c - &pipe_c).abs().max().double_value(&[]);
4112 assert!(
4113 ref_c.allclose(&pipe_c, 1e-4, 1e-4, false),
4114 "two-stage split (same device) must equal whole-model forward (max_diff={max_diff})"
4115 );
4116 assert!(stage1.lm_head_transposed.is_none());
4118 }
4119
4120 #[test]
4121 fn forward_layers_rejects_out_of_window_range() {
4122 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4124 let mut w = tiny_weights();
4125 let stage = LlamaModel::stage_from_weights_with_config(
4126 &mut w,
4127 tiny_config(),
4128 &map,
4129 2..LAYERS,
4130 DType::Float,
4131 KVQuantType::None,
4132 )
4133 .unwrap();
4134 let emb = Tensor::randn([1, 3, HIDDEN], (DType::Float, Device::Cpu));
4135 assert!(
4136 stage.forward_layers(&emb, 0..2, 0, None).is_err(),
4137 "range below window"
4138 );
4139 assert!(
4140 stage.forward_layers(&emb, 2..LAYERS, 0, None).is_ok(),
4141 "owned range ok"
4142 );
4143 }
4144
4145 #[test]
4146 fn boundary_copy_skipped_when_same_device() {
4147 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4151 for g in 1..LAYERS {
4152 assert!(!map.is_boundary(Device::Cpu, g));
4153 }
4154 }
4155
4156 #[test]
4157 fn stage_loader_rejects_missing_required_weights() {
4158 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4160 let mut w = tiny_weights();
4161 w.remove("model.norm.weight");
4162 let result = LlamaModel::stage_from_weights_with_config(
4163 &mut w,
4164 tiny_config(),
4165 &map,
4166 0..LAYERS,
4167 DType::Float,
4168 KVQuantType::None,
4169 );
4170 match result {
4171 Ok(_) => panic!("last stage without norm weight must error"),
4172 Err(e) => assert!(e.to_string().contains("norm"), "got: {e}"),
4173 }
4174 }
4175
4176 #[test]
4177 fn parse_config_requires_core_dims() {
4178 let ok = r#"{"num_attention_heads":2,"hidden_size":16,"num_hidden_layers":4}"#;
4180 assert!(LlamaModel::parse_config(ok).is_ok());
4181 let missing = r#"{"hidden_size":16,"num_hidden_layers":4}"#;
4182 let err = LlamaModel::parse_config(missing).unwrap_err();
4183 assert!(
4184 err.to_string().contains("num_attention_heads"),
4185 "got: {err}"
4186 );
4187 }
4188
4189 use crate::training::tenant_delta::{TenantDelta, TenantDeltaConfig};
4194
4195 fn tiny_delta() -> TenantDelta {
4199 let mut dims = std::collections::HashMap::new();
4200 dims.insert("q_proj".to_owned(), (HIDDEN as usize, HIDDEN as usize));
4201 dims.insert("v_proj".to_owned(), (HIDDEN as usize, HIDDEN as usize));
4202 let cfg = TenantDeltaConfig {
4203 rank: 2,
4204 alpha: 2.0,
4205 target_modules: vec!["q_proj".to_owned(), "v_proj".to_owned()],
4206 ..Default::default()
4207 };
4208 let delta = TenantDelta::new(&cfg, &dims, Device::Cpu, LAYERS).unwrap();
4209 let key_offset = |key: &str| -> i64 { key.bytes().map(|b| b as i64).sum::<i64>() % 17 };
4214 let _g = tch::no_grad_guard();
4215 for (k, a) in delta.lora_a.iter() {
4216 let n: i64 = a.size().iter().product();
4217 let vals = (Tensor::arange(n, (DType::Float, Device::Cpu)) + key_offset(k)).sin() * 0.1;
4218 a.shallow_clone().copy_(&vals.reshape(a.size()));
4220 }
4221 for (k, b) in delta.lora_b.iter() {
4222 let n: i64 = b.size().iter().product();
4223 let vals =
4224 (Tensor::arange(n, (DType::Float, Device::Cpu)) + key_offset(k) + 7).cos() * 0.1;
4225 b.shallow_clone().copy_(&vals.reshape(b.size()));
4226 }
4227 delta
4228 }
4229
4230 fn grad_snapshot(delta: &TenantDelta) -> std::collections::HashMap<String, f64> {
4232 let mut out = std::collections::HashMap::new();
4233 for (k, a) in &delta.lora_a {
4234 assert!(a.grad().defined(), "A grad undefined for {k}");
4235 out.insert(format!("A:{k}"), a.grad().norm().double_value(&[]));
4236 }
4237 for (k, b) in &delta.lora_b {
4238 assert!(b.grad().defined(), "B grad undefined for {k}");
4239 out.insert(format!("B:{k}"), b.grad().norm().double_value(&[]));
4240 }
4241 out
4242 }
4243
4244 #[test]
4249 fn ttt_split_autograd_matches_whole_model_grads() {
4250 use crate::training::pipeline::{compute_ntp_loss_split, TrainStage};
4251
4252 let input = Tensor::from_slice(&[3i64, 7, 1, 4, 9, 2]).reshape([1, 6]);
4253
4254 let whole = whole_model();
4256 let whole_delta = tiny_delta();
4257 let whole_stage = [TrainStage {
4258 model: &whole,
4259 range: 0..LAYERS,
4260 }];
4261 let loss_whole = compute_ntp_loss_split(&whole_stage, &input, Some(&whole_delta)).unwrap();
4262 loss_whole.backward();
4263 let whole_grads = grad_snapshot(&whole_delta);
4264 whole_delta.zero_grad();
4265
4266 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4269 let split = LAYERS / 2;
4270 let mut w0 = tiny_weights();
4271 let stage0 = LlamaModel::stage_from_weights_with_config(
4272 &mut w0,
4273 tiny_config(),
4274 &map,
4275 0..split,
4276 DType::Float,
4277 KVQuantType::None,
4278 )
4279 .unwrap();
4280 let mut w1 = tiny_weights();
4281 let stage1 = LlamaModel::stage_from_weights_with_config(
4282 &mut w1,
4283 tiny_config(),
4284 &map,
4285 split..LAYERS,
4286 DType::Float,
4287 KVQuantType::None,
4288 )
4289 .unwrap();
4290 let split_delta = tiny_delta();
4291 let stages = [
4292 TrainStage {
4293 model: &stage0,
4294 range: 0..split,
4295 },
4296 TrainStage {
4297 model: &stage1,
4298 range: split..LAYERS,
4299 },
4300 ];
4301 let loss_split = compute_ntp_loss_split(&stages, &input, Some(&split_delta)).unwrap();
4302 loss_split.backward();
4303 let split_grads = grad_snapshot(&split_delta);
4304
4305 let lw = loss_whole.double_value(&[]);
4307 let ls = loss_split.double_value(&[]);
4308 assert!(
4309 (lw - ls).abs() < 1e-4,
4310 "loss diverged: whole={lw} split={ls}"
4311 );
4312
4313 assert_eq!(whole_grads.len(), split_grads.len());
4315 for (k, &gw) in &whole_grads {
4316 let gs = *split_grads
4317 .get(k)
4318 .unwrap_or_else(|| panic!("missing grad {k}"));
4319 assert!(
4320 (gw - gs).abs() <= 1e-4 + 1e-4 * gw.abs(),
4321 "grad norm diverged for {k}: whole={gw} split={gs}"
4322 );
4323 assert!(
4326 gw > 0.0,
4327 "grad for {k} is zero — autograd path not exercised"
4328 );
4329 }
4330 }
4331
4332 use crate::runtime::kv_cache::KVCacheManager;
4337 use parking_lot::Mutex;
4338 use std::sync::Arc;
4339
4340 fn fresh_cache() -> Arc<Mutex<KVCacheManager>> {
4341 Arc::new(Mutex::new(KVCacheManager::new(
4342 LAYERS,
4343 64,
4344 KVQuantType::None,
4345 )))
4346 }
4347
4348 fn serial_decode_logits(prompt: &[i64], next_tok: i64) -> Tensor {
4351 let model = whole_model();
4352 let cache = Arc::new(Mutex::new(KVCacheManager::new(
4353 LAYERS,
4354 64,
4355 KVQuantType::None,
4356 )));
4357 let model = {
4359 let mut m = model;
4360 <LlamaModel as ModelOperations>::set_kv_cache(&mut m, cache);
4361 m
4362 };
4363 let prompt_t = Tensor::from_slice(prompt).reshape([1, prompt.len() as i64]);
4365 let _ = model.forward_with_cache(&prompt_t, 0).unwrap();
4366 let dec = Tensor::from_slice(&[next_tok]).reshape([1, 1]);
4368 let logits = model.forward_with_cache(&dec, prompt.len()).unwrap();
4369 logits.select(0, 0).select(0, 0)
4371 }
4372
4373 #[test]
4374 fn batched_ragged_decode_matches_serial() {
4375 let prompts: [&[i64]; 3] = [&[1, 5, 9, 2], &[7, 3], &[4, 8, 6, 1, 0]];
4379 let next: [i64; 3] = [11, 13, 6];
4380
4381 let ref_logits: Vec<Tensor> = prompts
4383 .iter()
4384 .zip(&next)
4385 .map(|(p, &t)| serial_decode_logits(p, t))
4386 .collect();
4387
4388 let model = whole_model();
4390 let mut seqs: Vec<(Vec<i64>, usize, Arc<Mutex<KVCacheManager>>)> = prompts
4391 .iter()
4392 .map(|_| (Vec::new(), 0usize, fresh_cache()))
4393 .collect();
4394
4395 for (i, p) in prompts.iter().enumerate() {
4398 let mut one = vec![(p.to_vec(), 0usize, seqs[i].2.clone())];
4399 let _ = model.forward_batched(&mut one, None).unwrap();
4400 }
4401
4402 for (i, &t) in next.iter().enumerate() {
4404 seqs[i].0 = vec![t];
4405 seqs[i].1 = prompts[i].len();
4406 }
4407 let batched = model.forward_batched(&mut seqs, None).unwrap(); let orig = model.config.original_vocab_size as i64;
4410 for (i, ref_l) in ref_logits.iter().enumerate() {
4411 let b_l = batched.select(0, i as i64).select(0, 0); let r_c = ref_l.narrow(0, 0, orig);
4413 let b_c = b_l.narrow(0, 0, orig);
4414 let max_diff = (&r_c - &b_c).abs().max().double_value(&[]);
4415 assert!(
4416 r_c.allclose(&b_c, 1e-4, 1e-4, false),
4417 "batched decode row {i} diverged from serial (max_diff={max_diff})"
4418 );
4419 assert_eq!(
4421 r_c.argmax(0, false).int64_value(&[]),
4422 b_c.argmax(0, false).int64_value(&[]),
4423 "row {i} argmax token differs",
4424 );
4425 }
4426 }
4427
4428 #[test]
4429 fn batched_single_row_matches_serial_prefill() {
4430 let prompt: [i64; 4] = [1, 5, 9, 2];
4433 let next = 11i64;
4434 let ref_l = serial_decode_logits(&prompt, next);
4435
4436 let model = whole_model();
4437 let cache = fresh_cache();
4438 let mut prefill = vec![(prompt.to_vec(), 0usize, cache.clone())];
4439 let _ = model.forward_batched(&mut prefill, None).unwrap();
4440 let mut dec = vec![(vec![next], prompt.len(), cache)];
4441 let logits = model.forward_batched(&mut dec, None).unwrap();
4442 let b_l = logits.select(0, 0).select(0, 0);
4443
4444 let orig = model.config.original_vocab_size as i64;
4445 let r_c = ref_l.narrow(0, 0, orig);
4446 let b_c = b_l.narrow(0, 0, orig);
4447 let max_diff = (&r_c - &b_c).abs().max().double_value(&[]);
4448 assert!(
4449 r_c.allclose(&b_c, 1e-4, 1e-4, false),
4450 "single-row batched decode diverged from serial (max_diff={max_diff})"
4451 );
4452 }
4453
4454 #[test]
4462 fn cached_two_token_forward_matches_serial_decode_steps() {
4463 let prompt: [i64; 4] = [1, 5, 9, 2];
4464
4465 let serial = whole_model();
4467 let prompt_t = Tensor::from_slice(&prompt).reshape([1, 4]);
4468 let _ = serial.forward_with_cache(&prompt_t, 0).unwrap();
4469 let ref1 = serial
4470 .forward_with_cache(&Tensor::from_slice(&[11i64]).reshape([1, 1]), 4)
4471 .unwrap();
4472 let ref2 = serial
4473 .forward_with_cache(&Tensor::from_slice(&[13i64]).reshape([1, 1]), 5)
4474 .unwrap();
4475
4476 let cached = whole_model();
4478 let _ = cached.forward_with_cache(&prompt_t, 0).unwrap();
4479 let logits2 = cached
4480 .forward_with_cache(&Tensor::from_slice(&[11i64, 13]).reshape([1, 2]), 4)
4481 .unwrap();
4482
4483 let orig = serial.config.original_vocab_size as i64;
4484 for (row, ref_l) in [(0i64, &ref1), (1, &ref2)] {
4485 let got = logits2.select(1, row).select(0, 0).narrow(0, 0, orig);
4486 let want = ref_l.select(0, 0).select(0, 0).narrow(0, 0, orig);
4487 let max_diff = (&got - &want).abs().max().double_value(&[]);
4488 assert!(
4489 got.allclose(&want, 1e-4, 1e-4, false),
4490 "cached 2-token forward row {row} diverged from the serial decode step \
4491 (max_diff={max_diff}); the causal mask must keep k_len - q_len history, \
4492 not tril(0)"
4493 );
4494 assert_eq!(
4495 got.argmax(-1, false).int64_value(&[]),
4496 want.argmax(-1, false).int64_value(&[]),
4497 "row {row} argmax differs",
4498 );
4499 }
4500 }
4501
4502 #[test]
4505 fn forward_layers_train_rejects_out_of_window_range() {
4506 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4507 let mut w = tiny_weights();
4508 let stage = LlamaModel::stage_from_weights_with_config(
4509 &mut w,
4510 tiny_config(),
4511 &map,
4512 2..LAYERS,
4513 DType::Float,
4514 KVQuantType::None,
4515 )
4516 .unwrap();
4517 let emb = Tensor::randn([1, 3, HIDDEN], (DType::Float, Device::Cpu));
4518 assert!(
4519 stage.forward_layers_train(&emb, 0..2, None).is_err(),
4520 "range below window"
4521 );
4522 assert!(
4523 stage.forward_layers_train(&emb, 2..LAYERS, None).is_ok(),
4524 "owned range ok"
4525 );
4526 }
4527}
4528
4529#[cfg(test)]
4533#[allow(
4534 clippy::unwrap_used,
4535 clippy::expect_used,
4536 clippy::print_stdout,
4537 clippy::print_stderr
4538)]
4539mod fp8_gemm_tests {
4540 use super::*;
4541 use std::process::Command;
4542 use tch::Kind;
4543
4544 const FP8: Kind = Kind::Float8e4m3fn;
4545 const OPT: (Kind, Device) = (Kind::Float, Device::Cpu);
4546
4547 fn det(dims: &[i64], seed: i64) -> Tensor {
4550 let n: i64 = dims.iter().product();
4551 ((Tensor::arange(n, OPT) + seed).sin() * 2.0).reshape(dims)
4552 }
4553
4554 fn max_rel_err(a: &Tensor, b: &Tensor) -> f64 {
4555 let a = a.to_kind(Kind::Double);
4556 let b = b.to_kind(Kind::Double);
4557 let num = (&a - &b).abs();
4558 let den = b.abs().clamp_min(1e-6);
4559 (num / den).max().double_value(&[])
4560 }
4561
4562 fn rel_frob_err(a: &Tensor, b: &Tensor) -> f64 {
4565 let a = a.to_kind(Kind::Double);
4566 let b = b.to_kind(Kind::Double);
4567 let num = (&a - &b).norm().double_value(&[]);
4568 let den = b.norm().double_value(&[]).max(1e-12);
4569 num / den
4570 }
4571
4572 fn mean_cuda_us(iterations: u32, mut op: impl FnMut()) -> f64 {
4573 tch::Cuda::synchronize(0);
4574 let started = std::time::Instant::now();
4575 for _ in 0..iterations {
4576 op();
4577 tch::Cuda::synchronize(0);
4578 }
4579 started.elapsed().as_micros() as f64 / f64::from(iterations)
4580 }
4581
4582 fn nvidia_cuda_compute_capability(device: usize) -> Option<(u32, u32)> {
4586 if tch::utils::has_hip() {
4587 eprintln!("skipping NVIDIA FP8 capability test on ROCm/HIP backend");
4588 return None;
4589 }
4590 let gpu = std::env::var("CUDA_VISIBLE_DEVICES")
4591 .ok()
4592 .and_then(|visible| {
4593 visible
4594 .split(',')
4595 .nth(device)
4596 .map(str::trim)
4597 .map(str::to_owned)
4598 })
4599 .filter(|id| !id.is_empty())
4600 .unwrap_or_else(|| device.to_string());
4601 let output = match Command::new("nvidia-smi")
4602 .arg(format!("--id={gpu}"))
4603 .arg("--query-gpu=compute_cap")
4604 .arg("--format=csv,noheader,nounits")
4605 .output()
4606 {
4607 Ok(output) if output.status.success() => output,
4608 Ok(output) => {
4609 eprintln!("skipping NVIDIA FP8 capability test: nvidia-smi failed for CUDA device {device}: {}", String::from_utf8_lossy(&output.stderr).trim());
4610 return None;
4611 }
4612 Err(error) => {
4613 eprintln!("skipping NVIDIA FP8 capability test: cannot run nvidia-smi: {error}");
4614 return None;
4615 }
4616 };
4617 let capability = match String::from_utf8(output.stdout) {
4618 Ok(capability) => capability,
4619 Err(error) => {
4620 eprintln!(
4621 "skipping NVIDIA FP8 capability test: non-UTF-8 capability output: {error}"
4622 );
4623 return None;
4624 }
4625 };
4626 let capability = capability.trim();
4627 let Some((major, minor)) = capability.split_once('.') else {
4628 eprintln!(
4629 "skipping NVIDIA FP8 capability test: malformed compute capability {capability:?}"
4630 );
4631 return None;
4632 };
4633 match (major.parse(), minor.parse()) {
4634 (Ok(major), Ok(minor)) => Some((major, minor)),
4635 _ => {
4636 eprintln!("skipping NVIDIA FP8 capability test: non-numeric compute capability {capability:?}");
4637 None
4638 }
4639 }
4640 }
4641
4642 fn det_rand(dims: &[i64], seed: i64) -> Tensor {
4647 let n: i64 = dims.iter().product();
4648 let idx = Tensor::arange(n, (Kind::Double, Device::Cpu)) + seed;
4649 let h = (idx * 12.9898).sin() * 43758.5453;
4650 ((&h - h.floor()) * 2.0 - 1.0)
4651 .to_kind(Kind::Float)
4652 .reshape(dims)
4653 }
4654
4655 #[test]
4665 fn scaled_mm_cpu_capability_contract() {
4666 let (m, k, n) = (8, 128, 64);
4667 let a = det(&[m, k], 1).to_kind(FP8);
4668 let b = det(&[k, n], 2).to_kind(FP8);
4669 let one = Tensor::ones([1, 1], OPT);
4670 let out = a
4672 .f_internal_scaled_mm(
4673 &b,
4674 &one,
4675 &one,
4676 None::<&Tensor>,
4677 None::<&Tensor>,
4678 Kind::BFloat16,
4679 false,
4680 )
4681 .expect("v1 tensorwise e4m3 must work on CPU");
4682 assert_eq!(out.size(), [m, n]);
4683 assert_eq!(out.kind(), Kind::BFloat16);
4684 let expect = a.to_kind(Kind::Float).matmul(&b.to_kind(Kind::Float));
4686 assert!(
4687 max_rel_err(&out, &expect) < 1e-2,
4688 "v1 tensorwise result diverged from plain matmul of dequantized inputs"
4689 );
4690 let sa = Tensor::ones([m, 1], OPT);
4692 let sb = Tensor::ones([1, n], OPT);
4693 let err = a
4694 .f_internal_scaled_mm(
4695 &b,
4696 &sa,
4697 &sb,
4698 None::<&Tensor>,
4699 None::<&Tensor>,
4700 Kind::BFloat16,
4701 false,
4702 )
4703 .expect_err("v1 rowwise must be rejected on CPU");
4704 assert!(
4705 err.to_string().contains("per-tensor scaling"),
4706 "unexpected: {err}"
4707 );
4708 let sa_refs = [&sa];
4710 let sb_refs = [&sb];
4711 let err = a
4712 .f_internal_scaled_mm_v2(
4713 &b,
4714 &sa_refs,
4715 [RECIPE_BLOCKWISE_1X128],
4716 [NO_SWIZZLE],
4717 &sb_refs,
4718 [RECIPE_BLOCKWISE_128X128],
4719 [NO_SWIZZLE],
4720 None::<&Tensor>,
4721 Kind::BFloat16,
4722 [] as [i64; 0],
4723 false,
4724 )
4725 .expect_err("v2 must be unavailable on CPU");
4726 assert!(
4727 err.to_string().contains("_scaled_mm_v2"),
4728 "unexpected: {err}"
4729 );
4730 }
4731
4732 #[test]
4737 fn quantize_activations_1x128_roundtrip() {
4738 let x = det(&[7, 256], 3);
4739 let (q, scale) = quantize_activations_1x128(&x);
4740 assert_eq!(q.kind(), FP8);
4741 assert_eq!(q.size(), x.size());
4742 assert_eq!(scale.kind(), Kind::Float);
4743 assert_eq!(scale.size(), [7, 2]);
4744 let xq = (q.to_kind(Kind::Float).view([7, 2, 128]) * scale.unsqueeze(-1)).view([7, 256]);
4746 let rel = max_rel_err(&xq, &x);
4747 assert!(
4753 rel <= 0.0625,
4754 "roundtrip max rel err {rel} exceeds e4m3 ulp bound"
4755 );
4756 assert!(q.to_kind(Kind::Float).abs().max().double_value(&[]) <= 448.0);
4757 }
4758
4759 #[test]
4770 fn fp8_blockwise_matmul_error_envelope() {
4771 let (m, k, n) = (8, 256, 256);
4772 let w = det_rand(&[k, n], 4) * 0.05; let x = det_rand(&[m, k], 5);
4774
4775 let w4 = w.view([k / 128, 128, n / 128, 128]);
4777 let ws = w4.abs().amax(-1, false).amax(1, false).clamp_min(1e-12) / 448.0; let wq = (w4 / ws.view([k / 128, 1, n / 128, 1]))
4779 .clamp(-448.0, 448.0)
4780 .to_kind(FP8);
4781 let w_deq = (wq.to_kind(Kind::Float) * ws.view([k / 128, 1, n / 128, 1])).view([k, n]);
4782
4783 let (q, sa) = quantize_activations_1x128(&x);
4785 let x_deq =
4786 (q.to_kind(Kind::Float).view([m, k / 128, 128]) * sa.unsqueeze(-1)).view([m, k]);
4787 let out_sim = x_deq.matmul(&w_deq);
4788
4789 let out_lazy = x
4791 .to_kind(Kind::BFloat16)
4792 .matmul(&w_deq.to_kind(Kind::BFloat16))
4793 .to_kind(Kind::Float);
4794 let rel_vs_lazy = rel_frob_err(&out_sim, &out_lazy);
4795 let out_ref = x.matmul(&w);
4797 let rel_vs_ref = rel_frob_err(&out_sim, &out_ref);
4798 assert!(
4803 rel_vs_ref < 0.10,
4804 "sim vs full-precision rel err {rel_vs_ref}"
4805 );
4806 assert!(
4807 rel_vs_lazy < 0.10,
4808 "sim vs lazy-dequant rel err {rel_vs_lazy}"
4809 );
4810 println!(
4811 "error envelope (frob rel): x_deq={:.4} w_deq={:.4} sim_vs_lazy={:.4} sim_vs_ref={:.4}",
4812 rel_frob_err(&x_deq, &x),
4813 rel_frob_err(&w_deq, &w),
4814 rel_vs_lazy,
4815 rel_vs_ref
4816 );
4817 }
4818
4819 #[test]
4831 fn fp8_rowwise_requant_error_envelope() {
4832 let (m, k, n) = (8, 256, 256);
4833 let mut worst_lazy = 0.0f64;
4834 let mut worst_ref = 0.0f64;
4835 for &(seed, spread) in &[(4i64, 1.0f64), (17, 8.0), (101, 64.0)] {
4836 let row_factors = Tensor::ones([k, 1], OPT);
4840 let small = Tensor::full([k / 2, 1], 1.0 / spread, OPT);
4841 row_factors.narrow(0, k / 2, k / 2).copy_(&small);
4842 let w = det_rand(&[k, n], seed) * 0.05 * row_factors;
4843 let x = det_rand(&[m, k], seed + 1);
4844
4845 let w4 = w.view([k / 128, 128, n / 128, 128]);
4847 let ws = w4.abs().amax(-1, false).amax(1, false).clamp_min(1e-12) / 448.0;
4848 let wq = (w4 / ws.view([k / 128, 1, n / 128, 1]))
4849 .clamp(-448.0, 448.0)
4850 .to_kind(FP8);
4851 let w_deq_block =
4852 (wq.to_kind(Kind::Float) * ws.view([k / 128, 1, n / 128, 1])).view([k, n]);
4853
4854 let rowwise = build_rowwise_requant(&wq.view([k, n]), &ws);
4856 assert_eq!(rowwise.scale.size(), [1, n]);
4857
4858 let (q, s_a) = quantize_activations_rowwise(&x);
4860 assert_eq!(s_a.size(), [m, 1]);
4861 let x_deq = q.to_kind(Kind::Float) * s_a;
4862 let w_deq = rowwise.weight.to_kind(Kind::Float) * &rowwise.scale;
4863 let out_sim = x_deq.matmul(&w_deq);
4864
4865 let out_lazy = x
4866 .to_kind(Kind::BFloat16)
4867 .matmul(&w_deq_block.to_kind(Kind::BFloat16))
4868 .to_kind(Kind::Float);
4869 let rel_vs_lazy = rel_frob_err(&out_sim, &out_lazy);
4870 let out_ref = x.matmul(&w);
4871 let rel_vs_ref = rel_frob_err(&out_sim, &out_ref);
4872 worst_lazy = worst_lazy.max(rel_vs_lazy);
4873 worst_ref = worst_ref.max(rel_vs_ref);
4874 assert!(
4875 rel_vs_lazy < 0.12,
4876 "rowwise seed={seed} spread={spread} vs lazy err {rel_vs_lazy}"
4877 );
4878 assert!(
4879 rel_vs_ref < 0.12,
4880 "rowwise seed={seed} spread={spread} vs reference err {rel_vs_ref}"
4881 );
4882 println!(
4883 "rowwise envelope seed={seed} spread={spread}: sim_vs_lazy={rel_vs_lazy:.4} sim_vs_ref={rel_vs_ref:.4}"
4884 );
4885 }
4886 println!("rowwise envelope worst: sim_vs_lazy={worst_lazy:.4} sim_vs_ref={worst_ref:.4}");
4887 }
4888
4889 #[test]
4893 fn fp8_scale_v2_packed_layout() {
4894 let scale = det(&[2, 3], 9);
4896 let packed = pack_scale_b_v2(&scale);
4897 assert_eq!(packed.size(), [3, 4]);
4898 assert_eq!(packed.kind(), Kind::Float);
4899 assert!(packed.is_contiguous());
4900 let expect = scale.transpose(0, 1);
4902 assert!(packed.narrow(1, 0, 2).allclose(&expect, 0.0, 0.0, false));
4903 assert_eq!(packed.narrow(1, 2, 2).abs().max().double_value(&[]), 0.0);
4905 let passed = packed.transpose(0, 1);
4907 assert_eq!(passed.size(), [4, 3]);
4908 assert_eq!(passed.stride(), [1, 4]);
4909 let packed4 = pack_scale_b_v2(&det(&[4, 2], 10));
4911 assert_eq!(packed4.size(), [2, 4]);
4912 let proj = LinearProjection::with_scale(det(&[256, 384], 11).to_kind(FP8), det(&[2, 3], 9));
4914 assert_eq!(proj.scale_v2.as_ref().unwrap().size(), [3, 4]);
4915 }
4916
4917 #[test]
4923 fn device_latch_isolation() {
4924 use ScaledMmRecipe::{BlockwiseV2, RowwiseV1};
4925 let latch = DeviceLatch::default();
4926 assert!(!latch.is_failed(0, BlockwiseV2) && !latch.is_failed(1, BlockwiseV2));
4927 assert!(latch.mark_failed(0, BlockwiseV2));
4929 assert!(!latch.mark_failed(0, BlockwiseV2));
4930 assert!(latch.is_failed(0, BlockwiseV2));
4932 assert!(!latch.is_failed(1, BlockwiseV2));
4933 assert!(!latch.is_failed(0, RowwiseV1));
4934 assert!(latch.mark_failed(0, RowwiseV1));
4936 assert!(latch.is_failed(0, RowwiseV1));
4937 assert!(latch.mark_failed(1, BlockwiseV2));
4939 assert!(latch.is_failed(1, BlockwiseV2));
4940 }
4941
4942 #[test]
4954 fn failing_recipe_latches_off_without_warning_spam() {
4955 let latch = DeviceLatch::default();
4956 let attempts = std::cell::Cell::new(0);
4957 let warnings = std::cell::Cell::new(0);
4958 let arm = |latch: &DeviceLatch| -> Option<()> {
4962 if latch.is_failed(0, ScaledMmRecipe::BlockwiseV2) {
4963 return None; }
4965 attempts.set(attempts.get() + 1);
4966 let res: Result<(), &str> = Err("SM120 does not support blockwise v2");
4967 res.inspect_err(|_| {
4968 if latch.mark_failed(0, ScaledMmRecipe::BlockwiseV2) {
4969 warnings.set(warnings.get() + 1); }
4971 })
4972 .ok()
4973 };
4974 for _ in 0..8 {
4975 assert!(arm(&latch).is_none());
4976 }
4977 assert_eq!(
4978 attempts.get(),
4979 1,
4980 "kernel must only be attempted before the latch trips"
4981 );
4982 assert_eq!(warnings.get(), 1, "warn-once per (device, recipe)");
4983 assert!(latch.is_failed(0, ScaledMmRecipe::BlockwiseV2));
4984 assert!(!latch.is_failed(0, ScaledMmRecipe::RowwiseV1));
4986 }
4987
4988 #[test]
4991 fn fp8_scaled_mm_declines_on_cpu() {
4992 let w = det(&[128, 256], 6).clamp(-2.0, 2.0).to_kind(FP8); let scale = Tensor::ones([1, 2], OPT); let proj = LinearProjection::with_scale(w, scale);
4995 let x = det(&[3, 128], 7).to_kind(Kind::BFloat16);
4996 assert!(
4997 proj.apply_fp8_scaled_mm(&x).is_none(),
4998 "CPU must decline the scaled_mm path"
4999 );
5000 let out = proj.apply(&x);
5002 assert_eq!(out.size(), [3, 256]);
5003 }
5004
5005 #[test]
5009 fn fp8_gate_probe() {
5010 let Ok(mode) = std::env::var("HYPRSTREAM_FP8_GATE_PROBE") else {
5011 return;
5012 };
5013 let gate_on = std::env::var("HYPRSTREAM_FP8_GEMM").as_deref() == Ok("1");
5014 let dequant_on = std::env::var("HYPRSTREAM_FP8_DEQUANT_LOAD").as_deref() == Ok("1");
5015 assert_eq!(
5016 fp8_gemm_enabled(),
5017 gate_on,
5018 "FP8 gate env was not read independently"
5019 );
5020 assert_eq!(
5021 crate::config::default_fp8_dequant_load(),
5022 dequant_on,
5023 "load-time dequant gate changed independently of its own env"
5024 );
5025
5026 let w = det(&[256, 256], 31).clamp(-2.0, 2.0).to_kind(FP8);
5027 let p = LinearProjection::with_scale(w, Tensor::ones([2, 2], OPT));
5028 let x_cpu = det(&[2, 256], 32).to_kind(Kind::BFloat16);
5029 let _ = p.apply(&x_cpu);
5030 if mode == "cpu" {
5031 assert_eq!(
5034 FP8_APPLY_PATH.load(std::sync::atomic::Ordering::SeqCst),
5035 2,
5036 "CPU probe unexpectedly selected a scaled-mm path"
5037 );
5038 println!("causal gate probe: CPU lazy fallback observed");
5039 return;
5040 }
5041
5042 assert_eq!(mode, "sm120", "unknown FP8 gate probe mode");
5043 assert!(
5044 gate_on && !dequant_on,
5045 "SM120 probe must be FP8-on/dequant-off"
5046 );
5047 assert!(tch::Cuda::is_available(), "SM120 probe requires CUDA");
5048 let dev = Device::Cuda(0);
5049 let p = p.into_device(dev);
5050 let x_train = x_cpu.to_device(dev).set_requires_grad(true);
5051 let out_train = p.apply(&x_train);
5052 assert_eq!(
5053 FP8_APPLY_PATH.load(std::sync::atomic::Ordering::SeqCst),
5054 2,
5055 "autograd-capable input unexpectedly selected scaled-mm"
5056 );
5057 out_train.sum(Kind::Float).backward();
5058 assert!(
5059 x_train.grad().defined(),
5060 "lazy matmul did not retain a backward path"
5061 );
5062
5063 let x = x_cpu.to_device(dev);
5064 let out = p.apply(&x);
5065 assert_eq!(out.size(), [2, 256]);
5066 assert_eq!(
5067 FP8_APPLY_PATH.load(std::sync::atomic::Ordering::SeqCst),
5068 1,
5069 "production apply did not return the gated scaled-mm result"
5070 );
5071 assert!(
5072 scaled_mm_latch().is_failed(0, ScaledMmRecipe::BlockwiseV2),
5073 "SM120 blockwise recipe did not latch off"
5074 );
5075 assert!(
5076 !scaled_mm_latch().is_failed(0, ScaledMmRecipe::RowwiseV1),
5077 "SM120 rowwise recipe latched despite libtorch 2.11"
5078 );
5079 println!(
5080 "causal gate probe: FP8 on selected rowwise after blockwise latch; dequant gate off"
5081 );
5082 }
5083
5084 #[test]
5085 fn fp8_e5m2_no_requant_probe() {
5086 if std::env::var_os("HYPRSTREAM_FP8_E5M2_PROBE").is_none() {
5087 return;
5088 }
5089 assert!(
5090 fp8_gemm_enabled(),
5091 "E5M2 allocation probe requires FP8 gate on"
5092 );
5093 reset_rowwise_requant_peak();
5094 let _projection = LinearProjection::with_scale(
5095 det(&[256, 256], 41).to_kind(Kind::Float8e5m2),
5096 Tensor::ones([2, 2], OPT),
5097 );
5098 assert_eq!(
5099 rowwise_requant_live(),
5100 0,
5101 "unsupported E5M2 construction retained a rowwise allocation"
5102 );
5103 assert_eq!(
5104 rowwise_requant_peak(),
5105 0,
5106 "unsupported E5M2 construction invoked rowwise requantization"
5107 );
5108 println!("causal allocation probe: E5M2 did not construct rowwise requantization");
5109 }
5110
5111 #[test]
5117 fn fp8_gate_causal_subprocesses() {
5118 fn run_probe(name: &str, fp8: &str, dequant: &str, probe: &str, marker: &str) {
5119 let exe = std::env::current_exe().expect("test executable path");
5120 let output = Command::new(exe)
5121 .arg("--exact")
5122 .arg(format!(
5123 "runtime::architectures::llama::fp8_gemm_tests::{name}"
5124 ))
5125 .arg("--nocapture")
5126 .env("HYPRSTREAM_FP8_GEMM", fp8)
5127 .env("HYPRSTREAM_FP8_DEQUANT_LOAD", dequant)
5128 .env("HYPRSTREAM_FP8_GATE_PROBE", probe)
5129 .output()
5130 .expect("spawn FP8 gate probe");
5131 let transcript = format!(
5132 "{}{}",
5133 String::from_utf8_lossy(&output.stdout),
5134 String::from_utf8_lossy(&output.stderr)
5135 );
5136 assert!(
5137 output.status.success(),
5138 "FP8 gate probe {probe} failed: {}\n{transcript}",
5139 output.status
5140 );
5141 assert!(
5142 transcript.contains("1 passed"),
5143 "FP8 gate probe {probe} did not execute exactly one test:\n{transcript}"
5144 );
5145 assert!(
5146 transcript.contains(marker),
5147 "FP8 gate probe {probe} missed expected marker {marker:?}:\n{transcript}"
5148 );
5149 }
5150
5151 let cpu_marker = "causal gate probe: CPU lazy fallback observed";
5152 run_probe("fp8_gate_probe", "0", "0", "cpu", cpu_marker);
5153 run_probe("fp8_gate_probe", "1", "0", "cpu", cpu_marker);
5154 run_probe("fp8_gate_probe", "0", "1", "cpu", cpu_marker);
5155 let exe = std::env::current_exe().expect("test executable path");
5156 let output = Command::new(&exe)
5157 .arg("--exact")
5158 .arg("runtime::architectures::llama::fp8_gemm_tests::fp8_e5m2_no_requant_probe")
5159 .arg("--nocapture")
5160 .env("HYPRSTREAM_FP8_GEMM", "1")
5161 .env("HYPRSTREAM_FP8_E5M2_PROBE", "1")
5162 .output()
5163 .expect("spawn E5M2 allocation probe");
5164 let transcript = format!(
5165 "{}{}",
5166 String::from_utf8_lossy(&output.stdout),
5167 String::from_utf8_lossy(&output.stderr)
5168 );
5169 assert!(
5170 output.status.success(),
5171 "E5M2 allocation probe failed: {}\n{transcript}",
5172 output.status
5173 );
5174 assert!(
5175 transcript.contains("1 passed")
5176 && transcript.contains(
5177 "causal allocation probe: E5M2 did not construct rowwise requantization"
5178 ),
5179 "E5M2 allocation probe did not execute its expected child:\n{transcript}"
5180 );
5181
5182 if tch::Cuda::is_available() && matches!(nvidia_cuda_compute_capability(0), Some((12, _))) {
5183 run_probe("fp8_gate_probe", "1", "0", "sm120", "causal gate probe: FP8 on selected rowwise after blockwise latch; dequant gate off");
5184 }
5185
5186 let output = Command::new(exe)
5190 .arg("--exact")
5191 .arg("runtime::architectures::qwen3_5::pipeline_tests::fp8_fused_projection_metadata_probe")
5192 .arg("--nocapture")
5193 .env("HYPRSTREAM_FP8_GEMM", "1")
5194 .env("HYPRSTREAM_FP8_DEQUANT_LOAD", "0")
5195 .env("HYPRSTREAM_QWEN_FP8_PROBE", "1")
5196 .output()
5197 .expect("spawn Qwen FP8 metadata probe");
5198 let transcript = format!(
5199 "{}{}",
5200 String::from_utf8_lossy(&output.stdout),
5201 String::from_utf8_lossy(&output.stderr)
5202 );
5203 assert!(
5204 output.status.success(),
5205 "Qwen fused metadata probe failed: {}\n{transcript}",
5206 output.status
5207 );
5208 assert!(transcript.contains("1 passed") && transcript.contains("Qwen fused FP8 allocation probe: source rowwise copies dropped before fused allocation"),
5209 "Qwen fused allocation probe did not execute its expected child:\n{transcript}");
5210 }
5211
5212 #[test]
5219 fn fp8_scaled_mm_kernel_cuda() {
5220 if !tch::Cuda::is_available() {
5221 eprintln!("skipping: no CUDA device");
5222 return;
5223 }
5224 let dev = Device::Cuda(0);
5225 let (m, k, n) = (8, 256, 256);
5226 let w = det_rand(&[k, n], 4) * 0.05;
5227 let w4 = w.view([k / 128, 128, n / 128, 128]);
5228 let ws = w4.abs().amax(-1, false).amax(1, false).clamp_min(1e-12) / 448.0;
5229 let wq = (w4 / ws.view([k / 128, 1, n / 128, 1]))
5230 .clamp(-448.0, 448.0)
5231 .to_kind(FP8)
5232 .view([k, n]);
5233 let mut proj = LinearProjection::with_scale(wq, ws.view([k / 128, n / 128]));
5236 proj.rowwise = Some(Box::new(build_rowwise_requant(
5237 &proj.weight,
5238 proj.scale.as_ref().unwrap(),
5239 )));
5240 assert_eq!(proj.scale_v2.as_ref().unwrap().size(), [n / 128, 4]);
5242 let proj = proj.into_device(dev);
5243 let x = det_rand(&[m, k], 5).to_kind(Kind::BFloat16).to_device(dev);
5244
5245 let out_lazy = proj.apply(&x); let Some((cc_major, cc_minor)) = nvidia_cuda_compute_capability(0) else {
5247 return;
5248 };
5249 println!("CUDA device 0 compute capability: {cc_major}.{cc_minor}");
5250
5251 if cc_major == 9 {
5252 let out = proj
5255 .try_blockwise_v2(0, &x, m, k)
5256 .expect("SM90/Hopper must execute the blockwise v2 FP8 kernel");
5257 assert!(
5258 !scaled_mm_latch().is_failed(0, ScaledMmRecipe::BlockwiseV2),
5259 "blockwise v2 latched despite a successful SM90 kernel call"
5260 );
5261 let rel = rel_frob_err(&out, &out_lazy);
5262 println!("blockwise v2 (SM90) vs lazy-dequant frob rel err: {rel:.4}");
5263 assert!(rel < 0.30, "blockwise v2 diverged on SM90: {rel}");
5264 return;
5265 }
5266 if cc_major < 10 {
5267 eprintln!(
5271 "skipping architecture-specific FP8 recipe assertion on SM{cc_major}{cc_minor}"
5272 );
5273 return;
5274 }
5275
5276 assert!(
5280 proj.try_blockwise_v2(0, &x, m, k).is_none(),
5281 "blockwise v2 must reject on SM100+ before rowwise runs"
5282 );
5283 assert!(
5284 scaled_mm_latch().is_failed(0, ScaledMmRecipe::BlockwiseV2),
5285 "blockwise v2 neither ran nor latched off on this CUDA device"
5286 );
5287 assert!(proj.try_blockwise_v2(0, &x, m, k).is_none());
5289
5290 let out = proj.try_rowwise_v1(0, &x, m, k).expect(
5291 "libtorch 2.11 must execute the SM100+ rowwise kernel instead of latching to lazy fallback",
5292 );
5293 assert!(
5294 !scaled_mm_latch().is_failed(0, ScaledMmRecipe::RowwiseV1),
5295 "rowwise v1 latched despite a successful SM100+ kernel call"
5296 );
5297 let rel = rel_frob_err(&out, &out_lazy);
5298 println!("rowwise v1 (SM{cc_major}{cc_minor}) vs lazy-dequant frob rel err: {rel:.4}");
5299 assert!(rel < 0.12, "rowwise v1 diverged: {rel}");
5302
5303 let (bench_m, bench_k, bench_n) = (1, 5120, 5120);
5306 let bench_w = det_rand(&[bench_k, bench_n], 14) * 0.05;
5307 let bench_w4 = bench_w.view([bench_k / 128, 128, bench_n / 128, 128]);
5308 let bench_scale = bench_w4
5309 .abs()
5310 .amax(-1, false)
5311 .amax(1, false)
5312 .clamp_min(1e-12)
5313 / 448.0;
5314 let bench_wq = (bench_w4 / bench_scale.view([bench_k / 128, 1, bench_n / 128, 1]))
5315 .clamp(-448.0, 448.0)
5316 .to_kind(FP8)
5317 .view([bench_k, bench_n]);
5318 let mut bench_proj = LinearProjection::with_scale(bench_wq, bench_scale);
5319 bench_proj.rowwise = Some(Box::new(build_rowwise_requant(
5320 &bench_proj.weight,
5321 bench_proj.scale.as_ref().unwrap(),
5322 )));
5323 let bench_proj = bench_proj.into_device(dev);
5324 let bench_x = det_rand(&[bench_m, bench_k], 15)
5325 .to_kind(Kind::BFloat16)
5326 .to_device(dev);
5327 let iters = 20;
5328 let lazy_us = mean_cuda_us(iters, || {
5329 let _ = std::hint::black_box(bench_proj.apply(&bench_x));
5330 });
5331 let rowwise_us = mean_cuda_us(iters, || {
5332 let _ = std::hint::black_box(
5333 bench_proj
5334 .try_rowwise_v1(0, &bench_x, bench_m, bench_k)
5335 .expect("rowwise benchmark kernel must run"),
5336 );
5337 });
5338 println!(
5339 "SM120 FP8 projection [{bench_m},{bench_k}]x[{bench_k},{bench_n}]: \
5340 lazy_dequant={lazy_us:.0}us ({:.2} iter/s), rowwise={rowwise_us:.0}us ({:.2} iter/s), ratio={:.3}x",
5341 1_000_000.0 / lazy_us,
5342 1_000_000.0 / rowwise_us,
5343 lazy_us / rowwise_us,
5344 );
5345 }
5346
5347 #[test]
5353 #[ignore = "microbenchmark — run explicitly; GPU numbers need the CUDA box"]
5354 fn bench_fp8_gemm_vs_lazy_dequant() {
5355 let (m, k, n) = (1, 5120, 5120);
5356 let w = Tensor::zeros([k, n], OPT).clamp(-1.0, 1.0).to_kind(FP8);
5357 let scale = Tensor::ones([k / 128, n / 128], OPT);
5358 let proj = LinearProjection::with_scale(w, scale);
5359 let x = det(&[m, k], 8).to_kind(Kind::BFloat16);
5360 let iters = 20;
5361
5362 let t0 = std::time::Instant::now();
5363 for _ in 0..iters {
5364 let _ = proj.apply(&x);
5365 }
5366 let lazy_us = t0.elapsed().as_micros() as f64 / iters as f64;
5367
5368 let x2 = x.reshape([m, k]);
5369 let t0 = std::time::Instant::now();
5370 for _ in 0..iters {
5371 let _ = quantize_activations_1x128(&x2);
5372 }
5373 let quant_us = t0.elapsed().as_micros() as f64 / iters as f64;
5374
5375 let one = Tensor::ones([1, 1], OPT);
5377 let (q, _) = quantize_activations_1x128(&x2);
5378 let t0 = std::time::Instant::now();
5379 for _ in 0..iters {
5380 let _ = q.internal_scaled_mm(
5381 &proj.weight,
5382 &one,
5383 &one,
5384 None::<&Tensor>,
5385 None::<&Tensor>,
5386 Kind::BFloat16,
5387 false,
5388 );
5389 }
5390 let smm_us = t0.elapsed().as_micros() as f64 / iters as f64;
5391
5392 println!(
5393 "shape [{m},{k}]x[{k},{n}] CPU: lazy_dequant={lazy_us:.0}us \
5394 act_quant={quant_us:.0}us v1_tensorwise_scaled_mm={smm_us:.0}us"
5395 );
5396 }
5397}