1#![allow(
3 dead_code,
4 unused_variables,
5 clippy::redundant_closure,
6 clippy::assign_op_pattern,
7 clippy::needless_borrows_for_generic_args,
8 clippy::redundant_closure_for_method_calls
9)]
10use super::config::ArchitectureConfig;
16use super::llama::{LinearProjection, LlamaMLP};
17use super::qwen3_5_vision::{Qwen3_5VisionConfig, Qwen3_5VisionEncoder};
18use super::{ModelArchitecture, ModelOperations};
19use crate::runtime::device_pool::LayerDeviceMap;
20use crate::runtime::kv_cache::KVCacheManager;
21use crate::runtime::model_config::ModelConfig;
22use crate::runtime::rope::RoPE;
23use crate::runtime::tensor_helpers::{dims3, dims4};
24use anyhow::{anyhow, Result};
25use hyprstream_rpc_std::model_client::KVQuantType;
26use std::collections::HashMap;
27use std::sync::Arc;
28use tch::{Device, Kind, Tensor};
29use tracing::info;
30
31thread_local! {
41 static ROPE_CACHE: std::cell::RefCell<HashMap<(usize, u32, i64), RoPE>> =
42 std::cell::RefCell::new(HashMap::new());
43}
44
45pub(crate) fn clear_rope_cache() {
48 ROPE_CACHE.with(|cache| cache.borrow_mut().clear());
49}
50
51pub struct Qwen3_5TextConfig {
57 pub hidden_size: u32,
58 pub num_hidden_layers: u32,
59 pub num_attention_heads: u32,
60 pub num_key_value_heads: u32,
61 pub head_dim: u32,
62 pub intermediate_size: u32,
63 pub vocab_size: u32,
64 pub max_position_embeddings: u32,
65 pub rms_norm_eps: f32,
66 pub rope_theta: f32,
67 pub partial_rotary_factor: f32,
69 pub rotary_dim: usize,
71 pub layer_types: Vec<String>,
73 pub linear_conv_kernel_dim: usize,
75 pub linear_key_head_dim: usize,
76 pub linear_value_head_dim: usize,
77 pub linear_num_key_heads: usize,
78 pub linear_num_value_heads: usize,
79 pub is_moe: bool,
81 pub num_experts: usize,
82 pub num_experts_per_tok: usize,
83 pub moe_intermediate_size: usize,
84 pub shared_expert_intermediate_size: usize,
85 pub has_vision: bool,
87 pub vision_out_hidden_size: usize,
88}
89
90impl ArchitectureConfig for Qwen3_5TextConfig {
91 fn num_attention_heads(&self) -> usize {
92 self.num_attention_heads as usize
93 }
94 fn num_key_value_heads(&self) -> usize {
95 self.num_key_value_heads as usize
96 }
97 fn hidden_size(&self) -> usize {
98 self.hidden_size as usize
99 }
100 fn intermediate_size(&self) -> usize {
101 self.intermediate_size as usize
102 }
103 fn head_dim(&self) -> usize {
104 self.head_dim as usize
105 }
106 fn vocab_size(&self) -> usize {
107 self.vocab_size as usize
108 }
109 fn max_position_embeddings(&self) -> usize {
110 self.max_position_embeddings as usize
111 }
112 fn rope_theta(&self) -> Option<f32> {
113 Some(self.rope_theta)
114 }
115 fn rope_dim(&self) -> Option<usize> {
116 Some(self.rotary_dim)
117 }
118 fn layer_norm_eps(&self) -> f32 {
119 self.rms_norm_eps
120 }
121 fn use_rms_norm(&self) -> bool {
122 true
123 }
124}
125
126impl Qwen3_5TextConfig {
127 pub fn from_model_config(cfg: &ModelConfig, max_pos: usize) -> Self {
128 let partial_rotary_factor = cfg.partial_rotary_factor.unwrap_or(0.25);
129 let rotary_dim = ((cfg.head_dim as f32) * partial_rotary_factor) as usize;
130
131 let layer_types = if cfg.layer_types.is_empty() {
133 (0..cfg.num_hidden_layers)
134 .map(|i| {
135 if (i + 1) % 4 == 0 {
136 "full_attention".to_owned()
137 } else {
138 "linear_attention".to_owned()
139 }
140 })
141 .collect()
142 } else {
143 cfg.layer_types.clone()
144 };
145
146 let moe_int_size = if cfg.moe_intermediate_size > 0 {
148 cfg.moe_intermediate_size
149 } else {
150 cfg.intermediate_size
151 };
152 let shared_int_size = if cfg.shared_expert_intermediate_size > 0 {
153 cfg.shared_expert_intermediate_size
154 } else {
155 cfg.intermediate_size
156 };
157
158 Self {
159 hidden_size: cfg.hidden_size as u32,
160 num_hidden_layers: cfg.num_hidden_layers as u32,
161 num_attention_heads: cfg.num_attention_heads as u32,
162 num_key_value_heads: cfg.num_key_value_heads as u32,
163 head_dim: cfg.head_dim as u32,
164 intermediate_size: cfg.intermediate_size as u32,
165 vocab_size: cfg.vocab_size as u32,
166 max_position_embeddings: max_pos as u32,
167 rms_norm_eps: cfg.rms_norm_eps,
168 rope_theta: cfg.rope_theta,
169 partial_rotary_factor,
170 rotary_dim,
171 layer_types,
172 linear_conv_kernel_dim: if cfg.linear_conv_kernel_dim > 0 {
173 cfg.linear_conv_kernel_dim
174 } else {
175 4
176 },
177 linear_key_head_dim: if cfg.linear_key_head_dim > 0 {
178 cfg.linear_key_head_dim
179 } else {
180 128
181 },
182 linear_value_head_dim: if cfg.linear_value_head_dim > 0 {
183 cfg.linear_value_head_dim
184 } else {
185 128
186 },
187 linear_num_key_heads: if cfg.linear_num_key_heads > 0 {
188 cfg.linear_num_key_heads
189 } else {
190 16
191 },
192 linear_num_value_heads: if cfg.linear_num_value_heads > 0 {
193 cfg.linear_num_value_heads
194 } else {
195 32
196 },
197 is_moe: cfg.is_moe,
198 num_experts: cfg.num_experts,
199 num_experts_per_tok: cfg.num_experts_per_tok,
200 moe_intermediate_size: moe_int_size,
201 shared_expert_intermediate_size: shared_int_size,
202 has_vision: cfg.has_vision,
203 vision_out_hidden_size: cfg.vision_out_hidden_size,
204 }
205 }
206}
207
208fn cat_projs(projs: Vec<LinearProjection>) -> LinearProjection {
220 if projs
221 .iter()
222 .any(|p| p.channel_scale.is_some() || p.children.is_some())
223 {
224 let k = projs[0].weight.size()[0];
226 let n: i64 = projs.iter().map(|p| p.weight.size()[1]).sum();
227 let mut fused = LinearProjection::new(
228 Tensor::zeros([1, 1], (projs[0].weight.kind(), projs[0].weight.device()))
229 .expand([k, n], false),
230 );
231 fused.children = Some(projs);
232 return fused;
233 }
234 #[inline]
235 fn is_fp8(k: tch::Kind) -> bool {
236 matches!(k, tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2)
237 }
238
239 let all_fp8 = projs.iter().all(|p| is_fp8(p.weight.kind()));
240 let any_fp8 = projs.iter().any(|p| is_fp8(p.weight.kind()));
241
242 if all_fp8 {
243 let mut weights = Vec::with_capacity(projs.len());
246 let mut scales = Vec::with_capacity(projs.len());
247 for LinearProjection { weight, scale, .. } in projs {
248 weights.push(weight);
249 scales.push(scale);
250 }
251 let weight_refs: Vec<&Tensor> = weights.iter().collect();
252 let fused_weight = Tensor::cat(&weight_refs, 1);
253 let scale = if scales.iter().any(Option::is_some) {
254 #[allow(clippy::expect_used)] let scale_refs: Vec<&Tensor> = scales
256 .iter()
257 .map(|scale| scale.as_ref().expect("FP8 projection missing scale"))
258 .collect();
259 Some(Tensor::cat(&scale_refs, 1))
260 } else {
261 None
262 };
263 match scale {
264 Some(s) => LinearProjection::with_scale(fused_weight, s),
265 None => LinearProjection::new(fused_weight),
266 }
267 } else if any_fp8 {
268 let weights: Vec<Tensor> = projs
271 .into_iter()
272 .map(|p| {
273 if is_fp8(p.weight.kind()) {
274 let w_bf = p.weight.to_kind(tch::Kind::BFloat16);
275 if let Some(s) = p.scale {
276 let ws = w_bf.size();
277 let ss = s.size();
278 let br = ws[0] / ss[0];
279 let bc = ws[1] / ss[1];
280 let w_4d = w_bf.view([ss[0], br, ss[1], bc]);
281 let s_4d = s.to_kind(tch::Kind::BFloat16).view([ss[0], 1, ss[1], 1]);
282 (w_4d * s_4d).reshape([ws[0], ws[1]])
283 } else {
284 w_bf
285 }
286 } else {
287 p.weight
288 }
289 })
290 .collect();
291 let weight_refs: Vec<&Tensor> = weights.iter().collect();
292 LinearProjection::new(Tensor::cat(&weight_refs, 1))
293 } else {
294 let weight_refs: Vec<&Tensor> = projs.iter().map(|p| &p.weight).collect();
296 LinearProjection::new(Tensor::cat(&weight_refs, 1))
297 }
298}
299
300fn dequant_batched(w: &Tensor, scale: Option<&Tensor>) -> Tensor {
304 match w.kind() {
305 tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
306 let w_bf = w.to_kind(tch::Kind::BFloat16);
307 if let Some(s) = scale {
308 let ws = w_bf.size(); let ss = s.size(); let br = ws[1] / ss[1];
311 let bc = ws[2] / ss[2];
312 let w_5d = w_bf.view([ss[0], ss[1], br, ss[2], bc]);
313 let s_5d = s
314 .to_kind(tch::Kind::BFloat16)
315 .view([ss[0], ss[1], 1, ss[2], 1]);
316 (&w_5d * &s_5d).reshape([ws[0], ws[1], ws[2]])
317 } else {
318 w_bf
319 }
320 }
321 _ => w.shallow_clone(),
322 }
323}
324
325struct Qwen3_5RMSNorm {
331 weight: Tensor,
332 weight_f32: Tensor,
334 eps: f32,
335}
336
337impl Qwen3_5RMSNorm {
338 fn new(weight: Tensor, eps: f32) -> Self {
339 let weight_f32 = weight.to_kind(Kind::Float);
340 Self {
341 weight,
342 weight_f32,
343 eps,
344 }
345 }
346
347 fn forward(&self, x: &Tensor) -> Result<Tensor> {
348 let orig = x.kind();
349 let x_f = x.to_kind(Kind::Float);
350 let mean_sq = (&x_f * &x_f).mean_dim(&[-1i64][..], true, Kind::Float);
351 let rrms = (mean_sq + self.eps as f64).rsqrt();
352 let normed = &x_f * rrms;
353 let one_plus_w = &self.weight_f32 + 1.0f64;
355 Ok((normed * one_plus_w).to_kind(orig))
356 }
357
358 #[inline]
361 fn into_device(self, device: Device) -> Self {
362 Self {
363 weight: self.weight.to_device(device),
364 weight_f32: self.weight_f32.to_device(device),
365 eps: self.eps,
366 }
367 }
368}
369
370unsafe impl Send for Qwen3_5RMSNorm {}
371unsafe impl Sync for Qwen3_5RMSNorm {}
372
373struct RMSNormGated {
375 weight: Tensor,
376 weight_f32: Tensor,
378 eps: f32,
379}
380
381impl RMSNormGated {
382 fn new(weight: Tensor, eps: f32) -> Self {
383 let weight_f32 = weight.to_kind(Kind::Float);
384 Self {
385 weight,
386 weight_f32,
387 eps,
388 }
389 }
390
391 fn forward(&self, x: &Tensor, gate: &Tensor) -> Result<Tensor> {
392 let orig = x.kind();
393 let x_f = if x.kind() == Kind::Float {
394 x.shallow_clone()
395 } else {
396 x.to_kind(Kind::Float)
397 };
398 let mean_sq = (&x_f * &x_f).mean_dim(&[-1i64][..], true, Kind::Float);
399 let rrms = (mean_sq + self.eps as f64).rsqrt();
400 let normed = &x_f * rrms;
401 let g = gate.to_kind(Kind::Float).silu();
402 Ok((normed * &self.weight_f32 * g).to_kind(orig))
403 }
404
405 #[inline]
407 fn into_device(self, device: Device) -> Self {
408 Self {
409 weight: self.weight.to_device(device),
410 weight_f32: self.weight_f32.to_device(device),
411 eps: self.eps,
412 }
413 }
414}
415
416unsafe impl Send for RMSNormGated {}
417unsafe impl Sync for RMSNormGated {}
418
419#[inline]
421fn l2_normalize(x: &Tensor) -> Tensor {
422 let sum_sq = (x * x).sum_dim_intlist(&[-1i64][..], true, Kind::Float);
426 x * (sum_sq + 1e-6f64).rsqrt()
427}
428
429fn chunked_delta_rule(
435 q: &Tensor, k: &Tensor, v: &Tensor, g: &Tensor, beta: &Tensor, initial_state: Option<&Tensor>,
441 device: tch::Device,
442) -> Result<(Tensor, Tensor)> {
443 const CHUNK_SIZE: i64 = 64;
444
445 let batch = q.size()[0];
446 let nv = q.size()[1];
447 let seq = q.size()[2];
448 let hk = q.size()[3];
449 let hv = v.size()[3];
450
451 let pad = (CHUNK_SIZE - seq % CHUNK_SIZE) % CHUNK_SIZE;
453 let seq_p = seq + pad;
454
455 let pad_tensor = |t: &Tensor, extra_dims: i64| -> Tensor {
456 if pad == 0 {
457 return t.shallow_clone();
458 }
459 let z = match extra_dims {
460 0 => Tensor::zeros([batch, nv, pad], (Kind::Float, device)),
461 _ => Tensor::zeros([batch, nv, pad, extra_dims], (Kind::Float, device)),
462 };
463 Tensor::cat(&[t, &z], 2)
464 };
465
466 let q = pad_tensor(q, hk);
467 let k = pad_tensor(k, hk);
468 let v = pad_tensor(v, hv);
469 let g = pad_tensor(g, 0);
470 let beta = pad_tensor(beta, 0);
471
472 let v_beta = &v * beta.unsqueeze(-1);
473 let k_beta = &k * beta.unsqueeze(-1);
474
475 let nc = seq_p / CHUNK_SIZE;
476
477 let reshape5 = |t: &Tensor, d: i64| t.reshape([batch, nv, nc, CHUNK_SIZE, d]);
479 let q = reshape5(&q, hk);
480 let k = reshape5(&k, hk);
481 let v = reshape5(&v, hv);
482 let k_beta = reshape5(&k_beta, hk);
483 let v_beta = reshape5(&v_beta, hv);
484 let g = g.reshape([batch, nv, nc, CHUNK_SIZE]);
486
487 let g = g.cumsum(-1, Kind::Float);
489
490 let g_t = g.unsqueeze(-1); let g_s = g.unsqueeze(-2); let decay_mask = (g_t - g_s).exp().tril(0); let mask0 = Tensor::ones([CHUNK_SIZE, CHUNK_SIZE], (Kind::Bool, device)).triu(0);
498 let mask1 = Tensor::ones([CHUNK_SIZE, CHUNK_SIZE], (Kind::Bool, device)).triu(1);
500
501 let attn0 = -(k_beta.matmul(&k.transpose(-1, -2)) * &decay_mask).masked_fill(&mask0, 0.0f64);
504
505 let c = CHUNK_SIZE;
517 let mut rows: Vec<Tensor> = Vec::with_capacity(c as usize);
519 rows.push(attn0.narrow(-2, 0, 1).squeeze_dim(-2)); for i in 1..c {
521 let a_row = attn0.narrow(-2, i, 1).narrow(-1, 0, i).squeeze_dim(-2); let sub = Tensor::stack(&rows, -2).narrow(-1, 0, i); let correction = a_row.unsqueeze(-2).matmul(&sub).squeeze_dim(-2); let corrected = &a_row + &correction; let pad_cols = c - i;
529 let zeros = Tensor::zeros([batch, nv, nc, pad_cols], (Kind::Float, device));
530 rows.push(Tensor::cat(&[corrected, zeros], -1)); }
532 let attn = Tensor::stack(&rows, -2); let eye = Tensor::eye(CHUNK_SIZE, (Kind::Float, device))
536 .unsqueeze(0)
537 .unsqueeze(0)
538 .unsqueeze(0); let attn = attn + eye;
540
541 let value_t = attn.matmul(&v_beta);
543 let k_cumdecay = attn.matmul(&(k_beta * g.exp().unsqueeze(-1)));
545
546 let mut state = initial_state
547 .map(|s| s.to_kind(Kind::Float))
548 .unwrap_or_else(|| Tensor::zeros([batch, nv, hk, hv], (Kind::Float, device)));
549 let mut chunks_out = Vec::with_capacity(nc as usize);
550
551 for i in 0..nc {
552 let q_i = q.select(2, i); let k_i = k.select(2, i); let v_i = value_t.select(2, i); let g_i = g.select(2, i); let dm_i = decay_mask.select(2, i); let kcd_i = k_cumdecay.select(2, i); let attn_inner = (q_i.matmul(&k_i.transpose(-1, -2)) * &dm_i).masked_fill(&mask1, 0.0f64);
561
562 let v_prime = kcd_i.matmul(&state); let v_new = v_i - v_prime;
565
566 let attn_inter = (q_i * g_i.exp().unsqueeze(-1)).matmul(&state); let chunk_out = attn_inter + attn_inner.matmul(&v_new);
570 chunks_out.push(chunk_out);
571
572 let g_last = g_i.narrow(-1, CHUNK_SIZE - 1, 1); let decay = g_last.exp().unsqueeze(-1); let g_diff = (g_last - &g_i).exp().unsqueeze(-1); let update = (k_i * g_diff).transpose(-1, -2).matmul(&v_new); state = &state * decay + update;
578 }
579
580 let out = Tensor::stack(&chunks_out, 2) .reshape([batch, nv, seq_p, hv])
586 .narrow(2, 0, seq)
587 .contiguous();
588
589 Ok((out, state))
590}
591
592struct GatedDeltaNetLayer {
597 in_proj_all: LinearProjection,
600 proj_split: [i64; 4],
602 conv1d_weight: Tensor, a_log: Tensor, dt_bias: Tensor, neg_a_exp_f32: Tensor, dt_bias_f32: Tensor, norm: RMSNormGated,
611 out_proj: LinearProjection, num_k_heads: usize,
614 num_v_heads: usize,
615 head_k_dim: usize,
616 head_v_dim: usize,
617 key_dim: usize, value_dim: usize, conv_dim: usize, kernel_size: usize,
621 layer_idx: usize,
622}
623
624unsafe impl Send for GatedDeltaNetLayer {}
625unsafe impl Sync for GatedDeltaNetLayer {}
626
627impl GatedDeltaNetLayer {
628 fn load(
629 weights: &mut HashMap<String, Tensor>,
630 prefix: &str,
631 cfg: &Qwen3_5TextConfig,
632 layer_idx: usize,
633 ) -> Result<Self> {
634 let num_k_heads = cfg.linear_num_key_heads;
635 let num_v_heads = cfg.linear_num_value_heads;
636 let head_k_dim = cfg.linear_key_head_dim;
637 let head_v_dim = cfg.linear_value_head_dim;
638 let key_dim = num_k_heads * head_k_dim;
639 let value_dim = num_v_heads * head_v_dim;
640 let conv_dim = key_dim * 2 + value_dim;
641 let kernel_size = cfg.linear_conv_kernel_dim;
642
643 let mut get = |name: &str| -> Result<Tensor> {
644 let full = format!("{prefix}.{name}");
645 weights
646 .remove(&full)
647 .ok_or_else(|| anyhow!("Missing weight: {}", full))
648 };
649
650 let conv1d_weight = get("conv1d.weight")?;
651 let a_log = get("A_log")?;
652 let dt_bias = get("dt_bias")?;
653 let norm_weight = get("norm.weight")?;
654
655 let a_log_sq = a_log.squeeze();
656 let dt_bias_sq = dt_bias.squeeze();
657 let neg_a_exp_f32 = -(a_log_sq
659 .to_kind(Kind::Float)
660 .exp()
661 .unsqueeze(0)
662 .unsqueeze(0)); let dt_bias_f32 = dt_bias_sq.to_kind(Kind::Float).unsqueeze(0).unsqueeze(0); let w_qkv = LinearProjection::take(weights, &format!("{prefix}.in_proj_qkv.weight"))?;
669 let w_z = LinearProjection::take(weights, &format!("{prefix}.in_proj_z.weight"))?;
670 let w_b = LinearProjection::take(weights, &format!("{prefix}.in_proj_b.weight"))?;
671 let w_a = LinearProjection::take(weights, &format!("{prefix}.in_proj_a.weight"))?;
672 let in_proj_all = cat_projs(vec![w_qkv, w_z, w_b, w_a]);
673 let proj_split = [
674 conv_dim as i64,
675 value_dim as i64,
676 num_v_heads as i64,
677 num_v_heads as i64,
678 ];
679
680 Ok(Self {
681 in_proj_all,
682 proj_split,
683 conv1d_weight,
684 a_log: a_log_sq,
685 dt_bias: dt_bias_sq,
686 neg_a_exp_f32,
687 dt_bias_f32,
688 norm: RMSNormGated::new(norm_weight, 1e-6),
689 out_proj: LinearProjection::take(weights, &format!("{prefix}.out_proj.weight"))?,
690 num_k_heads,
691 num_v_heads,
692 head_k_dim,
693 head_v_dim,
694 key_dim,
695 value_dim,
696 conv_dim,
697 kernel_size,
698 layer_idx,
699 })
700 }
701
702 fn forward(
703 &self,
704 hidden: &Tensor,
705 conv_state: &mut Option<Tensor>,
706 rec_state: &mut Option<Tensor>,
707 delta: Option<(&crate::training::TenantDelta, usize)>,
708 ) -> Result<Tensor> {
709 let (batch, seq, _) = dims3(hidden)?;
710 let device = hidden.device();
711 let dtype = hidden.kind();
712
713 let h2 = hidden.reshape([batch * seq, hidden.size()[2]]);
715 let all_proj = self.in_proj_all.try_apply(&h2)?; let [cd, vd, nv_d, _] = self.proj_split;
717 let mut off = 0i64;
718 let mixed_qkv = all_proj
719 .narrow(1, off, cd)
720 .contiguous()
721 .reshape([batch, seq, cd]);
722 off += cd;
723 let z = all_proj
724 .narrow(1, off, vd)
725 .contiguous()
726 .reshape([batch, seq, vd]);
727 off += vd;
728 let b = all_proj
729 .narrow(1, off, nv_d)
730 .contiguous()
731 .reshape([batch, seq, nv_d]);
732 off += nv_d;
733 let a_in = all_proj
734 .narrow(1, off, nv_d)
735 .contiguous()
736 .reshape([batch, seq, nv_d]);
737
738 let conv_dim = self.conv_dim as i64;
741 let ks = self.kernel_size as i64;
742
743 let x_t = mixed_qkv.permute([0, 2, 1]); let (x_for_conv, new_conv_state) = match conv_state {
747 Some(ref cs) => {
748 let padded = Tensor::cat(&[cs, &x_t], 2); let total = padded.size()[2];
752 let new_state = padded.narrow(2, total - (ks - 1), ks - 1).contiguous();
753 (padded, new_state)
754 }
755 None => {
756 let pad = Tensor::zeros([batch, conv_dim, ks - 1], (dtype, device));
758 let padded = Tensor::cat(&[&pad, &x_t], 2);
759 let new_state = padded.narrow(2, seq, ks - 1).contiguous();
762 (padded, new_state)
763 }
764 };
765 *conv_state = Some(new_conv_state);
766
767 let conv_out = x_for_conv.conv1d(
769 &self.conv1d_weight,
770 None::<&Tensor>,
771 &[1i64], &[0i64], &[1i64], conv_dim, ); let conv_out = conv_out.silu().permute([0, 2, 1]); let key_dim = self.key_dim as i64;
780 let val_dim = self.value_dim as i64;
781 let q_raw = conv_out.narrow(2, 0, key_dim);
782 let k_raw = conv_out.narrow(2, key_dim, key_dim);
783 let v_raw = conv_out.narrow(2, key_dim * 2, val_dim);
784
785 let q = q_raw.reshape([batch, seq, self.num_k_heads as i64, self.head_k_dim as i64]);
787 let k = k_raw.reshape([batch, seq, self.num_k_heads as i64, self.head_k_dim as i64]);
788 let v = v_raw.reshape([batch, seq, self.num_v_heads as i64, self.head_v_dim as i64]);
789
790 let q = l2_normalize(&q.to_kind(Kind::Float));
792 let k = l2_normalize(&k.to_kind(Kind::Float));
793 let scale = 1.0 / (self.head_k_dim as f64).sqrt();
794 let q = q * scale;
795
796 let a_in_f = a_in.to_kind(Kind::Float); let softplus_input = a_in_f + &self.dt_bias_f32; let g = &self.neg_a_exp_f32 * softplus_input.softplus(); let nv = self.num_v_heads as i64;
806 let hk = self.head_k_dim as i64;
807 let hv = self.head_v_dim as i64;
808
809 let state_init = rec_state
810 .take()
811 .unwrap_or_else(|| Tensor::zeros([batch, nv, hk, hv], (Kind::Float, device)));
812
813 let b_f = b.to_kind(Kind::Float).sigmoid(); let v_f = v.to_kind(Kind::Float); let k_for_state = if self.num_k_heads == self.num_v_heads {
821 k.to_kind(Kind::Float)
822 } else {
823 let repeats = (self.num_v_heads / self.num_k_heads) as i64;
824 k.repeat_interleave_self_int(repeats, 2, None)
825 .to_kind(Kind::Float)
826 };
827 let q_for_state = if self.num_k_heads == self.num_v_heads {
829 q.shallow_clone()
830 } else {
831 let repeats = (self.num_v_heads / self.num_k_heads) as i64;
832 q.repeat_interleave_self_int(repeats, 2, None)
833 };
834
835 let out = if seq == 1 {
839 let g_t = g.narrow(1, 0, 1).squeeze_dim(1);
841 let b_t = b_f.narrow(1, 0, 1).squeeze_dim(1);
842 let k_t = k_for_state.narrow(1, 0, 1).squeeze_dim(1);
843 let v_t = v_f.narrow(1, 0, 1).squeeze_dim(1);
844 let q_t = q_for_state.narrow(1, 0, 1).squeeze_dim(1);
845
846 let decay = g_t.exp().reshape([batch, nv, 1, 1]);
849 let state = &state_init * &decay;
850 let state_3d = state.reshape([batch * nv, hk, hv]);
852 let k_col = k_t.reshape([batch * nv, 1, hk]);
853 let kv_mem = k_col.bmm(&state_3d).reshape([batch, nv, hv]);
854 let delta = (v_t - kv_mem) * b_t.unsqueeze(-1);
855 let state =
857 state.reshape([batch, nv, hk, hv]) + k_t.unsqueeze(-1) * delta.unsqueeze(-2);
858 let state_3d = state.reshape([batch * nv, hk, hv]);
860 let q_col = q_t.reshape([batch * nv, 1, hk]);
861 let out_t = q_col.bmm(&state_3d).reshape([batch, nv, hv]);
862
863 *rec_state = Some(state);
864 out_t.unsqueeze(1).reshape([batch, 1, -1])
866 } else {
867 let q_h = q_for_state.permute([0, 2, 1, 3]); let k_h = k_for_state.permute([0, 2, 1, 3]);
871 let v_h = v_f.permute([0, 2, 1, 3]); let g_h = g.permute([0, 2, 1]); let b_h = b_f.permute([0, 2, 1]); let (out_h, new_state) =
878 chunked_delta_rule(&q_h, &k_h, &v_h, &g_h, &b_h, Some(&state_init), device)?;
879 *rec_state = Some(new_state);
881
882 out_h
885 .permute([0, 2, 1, 3])
886 .contiguous()
887 .reshape([batch, seq, -1])
888 };
889
890 let actual_vd = out.size()[2];
894 let actual_hv = self.norm.weight.size()[0]; let actual_nv = actual_vd / actual_hv;
896 let z_f = z.narrow(2, 0, actual_vd).contiguous(); let out_4d = out.reshape([batch, seq, actual_nv, actual_hv]);
899 let z_4d = z_f.reshape([batch, seq, actual_nv, actual_hv]);
900 let normed = self.norm.forward(&out_4d, &z_4d)?; let normed = if normed.kind() != dtype {
903 normed.to_kind(dtype)
904 } else {
905 normed
906 };
907 let flat = normed.reshape([batch * seq, actual_vd]);
908
909 let projected = self.out_proj.try_apply(&flat)?;
910 let projected = if let Some((delta, layer_idx)) = delta {
914 if delta.has_module("o_proj", layer_idx) {
915 super::add_delta(&projected, delta.forward_2d(&flat, "o_proj", layer_idx)?)
918 } else {
919 projected
920 }
921 } else {
922 projected
923 };
924 Ok(projected.reshape([batch, seq, self.hidden_size_from_out_proj()]))
925 }
926
927 fn hidden_size_from_out_proj(&self) -> i64 {
928 self.out_proj.weight.size()[1]
930 }
931
932 #[inline]
936 fn into_device(self, device: Device) -> Self {
937 Self {
938 in_proj_all: self.in_proj_all.into_device(device),
939 conv1d_weight: self.conv1d_weight.to_device(device),
940 a_log: self.a_log.to_device(device),
941 dt_bias: self.dt_bias.to_device(device),
942 neg_a_exp_f32: self.neg_a_exp_f32.to_device(device),
943 dt_bias_f32: self.dt_bias_f32.to_device(device),
944 norm: self.norm.into_device(device),
945 out_proj: self.out_proj.into_device(device),
946 ..self
947 }
948 }
949}
950
951struct Qwen3_5FullAttention {
956 qkv_proj: LinearProjection, qkv_split: [i64; 3], o_proj: LinearProjection, q_norm: Qwen3_5RMSNorm, k_norm: Qwen3_5RMSNorm,
961 num_heads: usize,
962 num_kv_heads: usize,
963 head_dim: usize,
964 rotary_dim: usize,
965 rope_theta: f32,
966 max_pos: usize,
967 layer_idx: usize,
968}
969
970unsafe impl Send for Qwen3_5FullAttention {}
971unsafe impl Sync for Qwen3_5FullAttention {}
972
973impl Qwen3_5FullAttention {
974 fn load(
975 weights: &mut HashMap<String, Tensor>,
976 prefix: &str,
977 cfg: &Qwen3_5TextConfig,
978 layer_idx: usize,
979 ) -> Result<Self> {
980 let q_norm_weight = weights
981 .remove(&format!("{prefix}.q_norm.weight"))
982 .ok_or_else(|| anyhow!("Missing {prefix}.q_norm.weight"))?;
983 let k_norm_weight = weights
984 .remove(&format!("{prefix}.k_norm.weight"))
985 .ok_or_else(|| anyhow!("Missing {prefix}.k_norm.weight"))?;
986
987 let q_proj = LinearProjection::take(weights, &format!("{prefix}.q_proj.weight"))?;
990 let k_proj = LinearProjection::take(weights, &format!("{prefix}.k_proj.weight"))?;
991 let v_proj = LinearProjection::take(weights, &format!("{prefix}.v_proj.weight"))?;
992 let q_dim = q_proj.weight.size()[1];
993 let k_dim = k_proj.weight.size()[1];
994 let v_dim = v_proj.weight.size()[1];
995 let fused = cat_projs(vec![q_proj, k_proj, v_proj]);
996
997 Ok(Self {
998 qkv_proj: fused,
999 qkv_split: [q_dim, k_dim, v_dim],
1000 o_proj: LinearProjection::take(weights, &format!("{prefix}.o_proj.weight"))?,
1001 q_norm: Qwen3_5RMSNorm::new(q_norm_weight, cfg.rms_norm_eps),
1002 k_norm: Qwen3_5RMSNorm::new(k_norm_weight, cfg.rms_norm_eps),
1003 num_heads: cfg.num_attention_heads as usize,
1004 num_kv_heads: cfg.num_key_value_heads as usize,
1005 head_dim: cfg.head_dim as usize,
1006 rotary_dim: cfg.rotary_dim,
1007 rope_theta: cfg.rope_theta,
1008 max_pos: cfg.max_position_embeddings as usize,
1009 layer_idx,
1010 })
1011 }
1012
1013 #[inline]
1015 fn into_device(self, device: Device) -> Self {
1016 Self {
1017 qkv_proj: self.qkv_proj.into_device(device),
1018 o_proj: self.o_proj.into_device(device),
1019 q_norm: self.q_norm.into_device(device),
1020 k_norm: self.k_norm.into_device(device),
1021 ..self
1022 }
1023 }
1024
1025 fn forward(
1026 &self,
1027 hidden: &Tensor,
1028 position_ids: Option<&Tensor>,
1029 kv_cache: Option<&mut crate::runtime::kv_cache::LayerKVCache>,
1030 start_pos: usize,
1031 delta: Option<(&crate::training::TenantDelta, usize)>,
1032 ) -> Result<Tensor> {
1033 let (batch, seq, _) = dims3(hidden)?;
1034 let device = hidden.device();
1035 let dtype = hidden.kind();
1036 let h2 = hidden.reshape([batch * seq, hidden.size()[2]]);
1037
1038 let qkv = self.qkv_proj.try_apply(&h2)?;
1040 let [q_dim, k_dim, v_dim] = self.qkv_split;
1041
1042 let (q_out, v_out) = if let Some((d, layer_idx)) = delta {
1047 let q_raw = qkv.narrow(1, 0, q_dim);
1048 let v_raw = qkv.narrow(1, q_dim + k_dim, v_dim);
1049
1050 let q_out = if d.has_module("q_proj", layer_idx) {
1051 let q_half = q_dim / 2;
1053 let q_part = q_raw.narrow(1, 0, q_half);
1054 let gate_part = q_raw.narrow(1, q_half, q_dim - q_half);
1055 let q_correction = d.forward_2d(&h2, "q_proj", layer_idx)?;
1056 let q_correction = if q_correction.size()[1] == q_half {
1059 q_correction
1060 } else {
1061 q_correction.narrow(1, 0, q_half)
1062 };
1063 let corrected_q = super::add_delta(&q_part, q_correction);
1065 Tensor::cat(&[corrected_q, gate_part], 1)
1066 } else {
1067 q_raw
1068 };
1069
1070 let v_out = if d.has_module("v_proj", layer_idx) {
1071 super::add_delta(&v_raw, d.forward_2d(&h2, "v_proj", layer_idx)?)
1073 } else {
1074 v_raw
1075 };
1076 (q_out, v_out)
1077 } else {
1078 (qkv.narrow(1, 0, q_dim), qkv.narrow(1, q_dim + k_dim, v_dim))
1079 };
1080 let k_out = qkv.narrow(1, q_dim, k_dim);
1081
1082 let nh = self.num_heads as i64;
1083 let nkv = self.num_kv_heads as i64;
1084 let hd = self.head_dim as i64;
1085
1086 let q_full = q_out.reshape([batch, seq, nh, hd * 2]);
1089 let query = q_full.narrow(3, 0, hd); let gate = q_full.narrow(3, hd, hd); let mut k = k_out.reshape([batch, seq, nkv, hd]);
1094 let v = v_out.reshape([batch, seq, nkv, hd]);
1095
1096 let query = self.apply_qknorm(&query, &self.q_norm, self.num_heads)?;
1098 k = self.apply_qknorm(&k, &self.k_norm, self.num_kv_heads)?;
1099
1100 let rd = self.rotary_dim as i64;
1102 let (query, k) = if rd > 0 {
1103 let query_rot = self.apply_rope(
1104 &query.narrow(3, 0, rd),
1105 position_ids,
1106 start_pos,
1107 device,
1108 dtype,
1109 )?;
1110 let query_pass = query.narrow(3, rd, hd - rd);
1111 let query = Tensor::cat(&[query_rot, query_pass], 3);
1112
1113 let k_rot =
1114 self.apply_rope(&k.narrow(3, 0, rd), position_ids, start_pos, device, dtype)?;
1115 let k_pass = k.narrow(3, rd, hd - rd);
1116 let k = Tensor::cat(&[k_rot, k_pass], 3);
1117 (query, k)
1118 } else {
1119 (query, k)
1120 };
1121
1122 let (k_full, v_full) = if let Some(cache) = kv_cache {
1124 cache.update(&k, &v, start_pos)?;
1125 cache.get()?
1126 } else {
1127 (k.shallow_clone(), v.shallow_clone())
1128 };
1129
1130 let groups = self.num_heads / self.num_kv_heads;
1132 let k_exp = k_full.repeat_interleave_self_int(groups as i64, 2, None);
1133 let v_exp = v_full.repeat_interleave_self_int(groups as i64, 2, None);
1134
1135 const ATTN_CHUNK: i64 = 1024;
1143 let scale = 1.0 / (self.head_dim as f64).sqrt();
1144 let q_p = query
1145 .permute([0, 2, 1, 3])
1146 .to_kind(Kind::Float)
1147 .contiguous();
1148 let k_p = k_exp
1149 .permute([0, 2, 3, 1])
1150 .to_kind(Kind::Float)
1151 .contiguous();
1152 let v_p = v_exp
1153 .permute([0, 2, 1, 3])
1154 .to_kind(Kind::Float)
1155 .contiguous();
1156
1157 let q_len = q_p.size()[2];
1158 let kv_len = k_p.size()[3];
1159 let ctx = if q_len <= ATTN_CHUNK {
1160 let mut scores = q_p.matmul(&k_p) * scale; if q_len > 1 {
1162 let mask =
1167 Tensor::ones([q_len, kv_len], (Kind::Float, device)).tril(kv_len - q_len);
1168 let mask = mask.unsqueeze(0).unsqueeze(0).expand_as(&scores);
1169 scores = scores.masked_fill(&mask.eq(0.0), -10000.0f64);
1170 }
1171 let attn = scores.softmax(-1, Kind::Float).to_kind(dtype);
1172 attn.matmul(&v_p.to_kind(dtype)) } else {
1174 let v_d = v_p.to_kind(dtype);
1179 let mut outs: Vec<Tensor> = Vec::new();
1180 let mut start = 0i64;
1181 while start < q_len {
1182 let cur = (q_len - start).min(ATTN_CHUNK);
1183 let q_chunk = q_p.narrow(2, start, cur); let mut scores = q_chunk.matmul(&k_p) * scale; let mask =
1186 Tensor::ones([cur, kv_len], (Kind::Float, device)).tril(kv_len - q_len + start);
1187 let mask = mask.unsqueeze(0).unsqueeze(0).expand_as(&scores);
1188 scores = scores.masked_fill(&mask.eq(0.0), -10000.0f64);
1189 let attn = scores.softmax(-1, Kind::Float).to_kind(dtype);
1190 outs.push(attn.matmul(&v_d)); start += cur;
1192 }
1193 Tensor::cat(&outs, 2) };
1195 let ctx = ctx.permute([0, 2, 1, 3]).contiguous(); let ctx_flat = ctx.reshape([batch * seq, nh * hd]);
1199 let gate_flat = gate.reshape([batch * seq, nh * hd]).sigmoid();
1200 let ctx_gated = ctx_flat * gate_flat;
1201
1202 let ctx_gated = if let Some((d, layer_idx)) = delta {
1204 if d.has_module("o_proj", layer_idx) {
1205 super::add_delta(&ctx_gated, d.forward_2d(&ctx_gated, "o_proj", layer_idx)?)
1207 } else {
1208 ctx_gated
1209 }
1210 } else {
1211 ctx_gated
1212 };
1213
1214 let out = self.o_proj.try_apply(&ctx_gated)?;
1215 Ok(out.reshape([batch, seq, -1]))
1216 }
1217
1218 fn apply_qknorm(&self, x: &Tensor, norm: &Qwen3_5RMSNorm, _nheads: usize) -> Result<Tensor> {
1219 let orig_shape = x.size();
1221 let (batch, seq, heads, hd) = dims4(x)?;
1222 let flat = x.reshape([batch * seq * heads, hd]);
1223 let orig = flat.kind();
1224 let flat_f = flat.to_kind(Kind::Float);
1225 let mean_sq = (&flat_f * &flat_f).mean_dim(&[-1i64][..], true, Kind::Float);
1226 let rrms = (mean_sq + norm.eps as f64).rsqrt();
1227 let normed = &flat_f * rrms;
1228 let one_plus_w = norm.weight_f32.reshape([1, hd]) + 1.0f64;
1230 let normed_scaled = normed * one_plus_w;
1231 Ok(normed_scaled.to_kind(orig).reshape(orig_shape))
1232 }
1233
1234 fn apply_rope(
1235 &self,
1236 x: &Tensor,
1237 position_ids: Option<&Tensor>,
1238 start_pos: usize,
1239 device: Device,
1240 dtype: Kind,
1241 ) -> Result<Tensor> {
1242 let device_key: i64 = match device {
1247 Device::Cuda(i) => i as i64,
1248 _ => -1,
1249 };
1250
1251 let rd = self.rotary_dim as i64;
1252 let seq = x.size()[1];
1253 let pos_ids = position_ids.map(|p| p.shallow_clone()).unwrap_or_else(|| {
1254 Tensor::arange_start(
1255 start_pos as i64,
1256 start_pos as i64 + seq,
1257 (Kind::Int64, device),
1258 )
1259 });
1260
1261 ROPE_CACHE.with(|cache| {
1262 let mut cache = cache.borrow_mut();
1263 let key = (self.layer_idx, self.rope_theta.to_bits(), device_key);
1264 let rope = match cache.entry(key) {
1265 std::collections::hash_map::Entry::Occupied(e) => e.into_mut(),
1266 std::collections::hash_map::Entry::Vacant(e) => e.insert(RoPE::new_with_dtype(
1267 rd,
1268 self.rope_theta as f64,
1269 self.max_pos as i64,
1270 device,
1271 dtype,
1272 )?),
1273 };
1274 rope.forward(x, Some(&pos_ids))
1275 })
1276 }
1277}
1278
1279struct Qwen3_5Expert {
1284 gate_proj: LinearProjection,
1285 up_proj: LinearProjection,
1286 down_proj: LinearProjection,
1287}
1288
1289impl Qwen3_5Expert {
1290 fn forward(&self, x: &Tensor) -> Result<Tensor> {
1291 let gate = self.gate_proj.try_apply(x)?.silu();
1292 let up = self.up_proj.try_apply(x)?;
1293 self.down_proj.try_apply(&(gate * up))
1294 }
1295
1296 #[inline]
1298 fn into_device(self, device: Device) -> Self {
1299 Self {
1300 gate_proj: self.gate_proj.into_device(device),
1301 up_proj: self.up_proj.into_device(device),
1302 down_proj: self.down_proj.into_device(device),
1303 }
1304 }
1305}
1306
1307unsafe impl Send for Qwen3_5Expert {}
1308unsafe impl Sync for Qwen3_5Expert {}
1309
1310struct Qwen3_5SparseMoE {
1311 gate: LinearProjection, expert_gate_w: Tensor, expert_gate_scale: Option<Tensor>, expert_up_w: Tensor, expert_up_scale: Option<Tensor>,
1319 expert_down_w: Tensor, expert_down_scale: Option<Tensor>,
1321 shared_expert: Qwen3_5Expert,
1322 shared_expert_gate: Option<LinearProjection>,
1325 num_experts_per_tok: usize,
1326}
1327
1328impl Qwen3_5SparseMoE {
1329 fn forward(&self, x: &Tensor) -> Result<Tensor> {
1330 let batch = x.size()[0];
1332 let hidden = x.size()[1];
1333 let k = self.num_experts_per_tok as i64;
1334 let dtype = x.kind();
1335
1336 let logits = self.gate.try_apply(x)?; let (top_logits, top_ids) = logits.topk(k, -1, true, true);
1340 let top_weights = top_logits.softmax(-1, dtype); let top_ids_flat = top_ids.reshape([-1]);
1345
1346 let gate_sel = dequant_batched(
1348 &self.expert_gate_w.index_select(0, &top_ids_flat),
1349 self.expert_gate_scale
1350 .as_ref()
1351 .map(|s| s.index_select(0, &top_ids_flat))
1352 .as_ref(),
1353 ); let up_sel = dequant_batched(
1355 &self.expert_up_w.index_select(0, &top_ids_flat),
1356 self.expert_up_scale
1357 .as_ref()
1358 .map(|s| s.index_select(0, &top_ids_flat))
1359 .as_ref(),
1360 );
1361
1362 let x_rep = if x.kind() != tch::Kind::BFloat16 {
1364 x.repeat_interleave_self_int(k, 0, None)
1365 .to_kind(tch::Kind::BFloat16)
1366 } else {
1367 x.repeat_interleave_self_int(k, 0, None)
1368 };
1369
1370 let moe_int = gate_sel.size()[2];
1372 let gate_up_sel = Tensor::cat(&[&gate_sel, &up_sel], 2); let x_3d = x_rep.unsqueeze(1); let gate_up = x_3d.bmm(&gate_up_sel).squeeze_dim(1); let gate_out = gate_up.narrow(1, 0, moe_int).silu();
1376 let up_out = gate_up.narrow(1, moe_int, moe_int);
1377
1378 let down_sel = dequant_batched(
1380 &self.expert_down_w.index_select(0, &top_ids_flat),
1381 self.expert_down_scale
1382 .as_ref()
1383 .map(|s| s.index_select(0, &top_ids_flat))
1384 .as_ref(),
1385 ); let expert_out = (gate_out * up_out)
1387 .unsqueeze(1)
1388 .bmm(&down_sel)
1389 .squeeze_dim(1);
1390
1391 let expert_out = expert_out.reshape([batch, k, hidden]);
1393 let output = (expert_out * top_weights.unsqueeze(-1).to_kind(tch::Kind::BFloat16))
1394 .sum_dim_intlist(&[1i64][..], false, None);
1395 let output = if dtype != tch::Kind::BFloat16 {
1396 output.to_kind(dtype)
1397 } else {
1398 output
1399 };
1400
1401 let shared_out = self.shared_expert.forward(x)?;
1402 let shared_out = if let Some(gate_proj) = &self.shared_expert_gate {
1403 let gate = gate_proj.try_apply(x)?.sigmoid();
1405 shared_out * gate
1406 } else {
1407 shared_out
1408 };
1409
1410 Ok(output + shared_out)
1411 }
1412
1413 #[inline]
1416 fn into_device(self, device: Device) -> Self {
1417 Self {
1418 gate: self.gate.into_device(device),
1419 expert_gate_w: self.expert_gate_w.to_device(device),
1420 expert_gate_scale: self.expert_gate_scale.map(|s| s.to_device(device)),
1421 expert_up_w: self.expert_up_w.to_device(device),
1422 expert_up_scale: self.expert_up_scale.map(|s| s.to_device(device)),
1423 expert_down_w: self.expert_down_w.to_device(device),
1424 expert_down_scale: self.expert_down_scale.map(|s| s.to_device(device)),
1425 shared_expert: self.shared_expert.into_device(device),
1426 shared_expert_gate: self.shared_expert_gate.map(|g| g.into_device(device)),
1427 num_experts_per_tok: self.num_experts_per_tok,
1428 }
1429 }
1430}
1431
1432unsafe impl Send for Qwen3_5SparseMoE {}
1433unsafe impl Sync for Qwen3_5SparseMoE {}
1434
1435enum Qwen3_5Mlp {
1440 Dense(Box<LlamaMLP>),
1441 Sparse(Box<Qwen3_5SparseMoE>),
1442}
1443
1444impl Qwen3_5Mlp {
1445 fn forward(&self, x: &Tensor) -> Result<Tensor> {
1446 match self {
1447 Qwen3_5Mlp::Dense(mlp) => mlp.forward(x, None),
1448 Qwen3_5Mlp::Sparse(moe) => {
1449 let orig = x.size();
1450 let batch = orig[0];
1451 let seq = orig[1];
1452 let h = orig[2];
1453 let flat = x.reshape([batch * seq, h]);
1454 let out = moe.forward(&flat)?;
1455 Ok(out.reshape(orig))
1456 }
1457 }
1458 }
1459
1460 #[inline]
1462 fn into_device(self, device: Device) -> Self {
1463 match self {
1464 Qwen3_5Mlp::Dense(mlp) => Qwen3_5Mlp::Dense(Box::new((*mlp).into_device(device))),
1465 Qwen3_5Mlp::Sparse(moe) => Qwen3_5Mlp::Sparse(Box::new((*moe).into_device(device))),
1466 }
1467 }
1468}
1469
1470unsafe impl Send for Qwen3_5Mlp {}
1471unsafe impl Sync for Qwen3_5Mlp {}
1472
1473enum LayerMixer {
1478 LinearAttn(GatedDeltaNetLayer),
1479 FullAttn(Qwen3_5FullAttention),
1480}
1481
1482impl LayerMixer {
1483 #[inline]
1485 fn into_device(self, device: Device) -> Self {
1486 match self {
1487 LayerMixer::LinearAttn(gdn) => LayerMixer::LinearAttn(gdn.into_device(device)),
1488 LayerMixer::FullAttn(attn) => LayerMixer::FullAttn(attn.into_device(device)),
1489 }
1490 }
1491}
1492
1493unsafe impl Send for LayerMixer {}
1494unsafe impl Sync for LayerMixer {}
1495
1496struct Qwen3_5Layer {
1497 mixer: LayerMixer,
1498 mlp: Qwen3_5Mlp,
1499 input_layernorm: Qwen3_5RMSNorm,
1500 post_attention_layernorm: Qwen3_5RMSNorm,
1501}
1502
1503impl Qwen3_5Layer {
1504 #[inline]
1507 fn into_device(self, device: Device) -> Self {
1508 Self {
1509 mixer: self.mixer.into_device(device),
1510 mlp: self.mlp.into_device(device),
1511 input_layernorm: self.input_layernorm.into_device(device),
1512 post_attention_layernorm: self.post_attention_layernorm.into_device(device),
1513 }
1514 }
1515}
1516
1517unsafe impl Send for Qwen3_5Layer {}
1518unsafe impl Sync for Qwen3_5Layer {}
1519
1520struct Qwen3_5MtpHead {
1547 pre_fc_norm_hidden: Qwen3_5RMSNorm,
1548 pre_fc_norm_embedding: Qwen3_5RMSNorm,
1549 fc: LinearProjection, attn: Qwen3_5FullAttention,
1551 mlp: LlamaMLP, input_layernorm: Qwen3_5RMSNorm,
1553 post_attention_layernorm: Qwen3_5RMSNorm,
1554 final_norm: Qwen3_5RMSNorm, }
1556
1557unsafe impl Send for Qwen3_5MtpHead {}
1558unsafe impl Sync for Qwen3_5MtpHead {}
1559
1560impl Qwen3_5MtpHead {
1561 fn load(weights: &mut HashMap<String, Tensor>, cfg: &Qwen3_5TextConfig) -> Result<Self> {
1564 fn take_norm(
1565 weights: &mut HashMap<String, Tensor>,
1566 name: &str,
1567 eps: f32,
1568 ) -> Result<Qwen3_5RMSNorm> {
1569 let w = weights
1570 .remove(name)
1571 .ok_or_else(|| anyhow!("Missing weight: {name}"))?;
1572 Ok(Qwen3_5RMSNorm::new(w, eps))
1573 }
1574
1575 let layer_prefix = "mtp.layers.0";
1576 let attn = Qwen3_5FullAttention::load(
1580 weights,
1581 &format!("{layer_prefix}.self_attn"),
1582 cfg,
1583 cfg.num_hidden_layers as usize,
1584 )?;
1585 let mlp_prefix = format!("{layer_prefix}.mlp");
1586 let mlp = LlamaMLP {
1587 gate_proj: LinearProjection::take(weights, &format!("{mlp_prefix}.gate_proj.weight"))?,
1588 up_proj: LinearProjection::take(weights, &format!("{mlp_prefix}.up_proj.weight"))?,
1589 down_proj: LinearProjection::take(weights, &format!("{mlp_prefix}.down_proj.weight"))?,
1590 activation: "silu".to_owned(),
1591 layer_idx: cfg.num_hidden_layers as usize,
1592 };
1593 let eps = cfg.rms_norm_eps;
1594
1595 Ok(Self {
1596 pre_fc_norm_hidden: take_norm(weights, "mtp.pre_fc_norm_hidden.weight", eps)?,
1597 pre_fc_norm_embedding: take_norm(weights, "mtp.pre_fc_norm_embedding.weight", eps)?,
1598 fc: LinearProjection::take(weights, "mtp.fc.weight")?,
1599 attn,
1600 mlp,
1601 input_layernorm: take_norm(
1602 weights,
1603 &format!("{layer_prefix}.input_layernorm.weight"),
1604 eps,
1605 )?,
1606 post_attention_layernorm: take_norm(
1607 weights,
1608 &format!("{layer_prefix}.post_attention_layernorm.weight"),
1609 eps,
1610 )?,
1611 final_norm: take_norm(weights, "mtp.norm.weight", eps)?,
1612 })
1613 }
1614}
1615
1616pub struct Qwen3_5Model {
1621 config: Qwen3_5TextConfig,
1622 device: Device,
1623 dtype: Kind,
1624
1625 device_map: LayerDeviceMap,
1632
1633 layer_offset: usize,
1638
1639 embed_tokens: Tensor,
1640 layers: Vec<Qwen3_5Layer>,
1641 norm: Qwen3_5RMSNorm,
1642 lm_head: Option<Tensor>, lm_head_transposed: Option<Tensor>, conv_states: Arc<parking_lot::Mutex<Vec<Option<Tensor>>>>,
1648 rec_states: Arc<parking_lot::Mutex<Vec<Option<Tensor>>>>,
1649 kv_cache: Option<Arc<parking_lot::Mutex<KVCacheManager>>>,
1650 mtp: Option<Qwen3_5MtpHead>,
1653 vision_encoder: Option<Qwen3_5VisionEncoder>,
1655 vision_projector: Option<LinearProjection>,
1657}
1658
1659unsafe impl Send for Qwen3_5Model {}
1660unsafe impl Sync for Qwen3_5Model {}
1661
1662impl Qwen3_5Model {
1663 pub fn from_weights(
1667 weights: &mut HashMap<String, Tensor>,
1668 cfg: Qwen3_5TextConfig,
1669 vision_cfg: Option<Qwen3_5VisionConfig>,
1670 device: &Device,
1671 dtype: Kind,
1672 kv_quant_type: KVQuantType,
1673 speculative_decoding: bool,
1674 ) -> Result<Self> {
1675 let needs_remap = weights
1679 .keys()
1680 .any(|k| k.starts_with("model.language_model."));
1681 if needs_remap {
1682 info!("Remapping Qwen3.5 weight prefixes (model.language_model.* → *, model.visual.* → visual.*)");
1683 let remapped: HashMap<String, Tensor> = weights
1684 .drain()
1685 .map(|(k, v)| {
1686 let new_key = if let Some(rest) = k.strip_prefix("model.language_model.") {
1689 format!("model.{rest}")
1690 } else if let Some(rest) = k.strip_prefix("model.visual.") {
1691 format!("visual.{rest}")
1692 } else {
1693 k
1694 };
1695 (new_key, v)
1696 })
1697 .collect();
1698 *weights = remapped;
1699 }
1700
1701 let has_vision = weights.keys().any(|k| k.starts_with("visual."));
1703 let (vision_encoder, vision_projector) = if has_vision {
1704 if let Some(vcfg) = vision_cfg {
1705 info!(
1706 "Loading Qwen3.5 vision encoder (out_hidden_size={})",
1707 vcfg.out_hidden_size
1708 );
1709 let out_hidden = vcfg.out_hidden_size;
1710 let text_hidden = cfg.hidden_size;
1711 match Qwen3_5VisionEncoder::from_weights(weights, vcfg) {
1713 Ok(enc) => {
1714 let proj_key = if weights.contains_key("visual_projector.weight") {
1715 Some("visual_projector.weight")
1716 } else if weights.contains_key("vision_projector.weight") {
1717 Some("vision_projector.weight")
1718 } else {
1719 None
1720 };
1721 let projector = if let Some(key) = proj_key {
1722 Some(LinearProjection::take(weights, key)?)
1723 } else {
1724 info!("No vision_projector weights found; vision output will be zeros");
1725 let w = Tensor::zeros(
1726 [out_hidden as i64, text_hidden as i64],
1727 (dtype, *device),
1728 );
1729 Some(LinearProjection::new(w))
1730 };
1731 (Some(enc), projector)
1732 }
1733 Err(e) => {
1734 info!("Vision encoder load skipped (key naming mismatch, text-only mode): {e}");
1735 weights.retain(|k, _| !k.starts_with("visual."));
1736 (None, None)
1737 }
1738 }
1739 } else {
1740 weights.retain(|k, _| !k.starts_with("visual."));
1742 (None, None)
1743 }
1744 } else {
1745 (None, None)
1746 };
1747
1748 let mtp = if !speculative_decoding || kv_quant_type != KVQuantType::None {
1752 weights.retain(|k, _| !k.starts_with("mtp."));
1756 None
1757 } else if weights.keys().any(|k| k.starts_with("mtp.")) {
1758 if cfg.is_moe {
1759 info!("MTP head present but MoE MTP is not yet supported; skipping mtp.* weights (non-speculative decode)");
1760 weights.retain(|k, _| !k.starts_with("mtp."));
1761 None
1762 } else if weights.keys().any(|k| k.starts_with("mtp.layers.1.")) {
1763 info!("Multi-layer MTP (mtp_num_hidden_layers > 1) is not supported; skipping mtp.* weights");
1764 weights.retain(|k, _| !k.starts_with("mtp."));
1765 None
1766 } else {
1767 match Qwen3_5MtpHead::load(weights, &cfg) {
1768 Ok(head) => {
1769 info!(
1770 "Loaded Qwen3.5 MTP draft head (self-speculative decoding available)"
1771 );
1772 Some(head)
1773 }
1774 Err(e) => {
1775 info!(
1776 "MTP head load failed ({e}); continuing without speculative decoding"
1777 );
1778 weights.retain(|k, _| !k.starts_with("mtp."));
1779 None
1780 }
1781 }
1782 }
1783 } else {
1784 None
1785 };
1786
1787 let embed = weights
1792 .remove("model.embed_tokens.weight")
1793 .ok_or_else(|| anyhow!("Missing model.embed_tokens.weight"))?
1794 .to_device(*device);
1795 let norm_w = weights
1796 .remove("model.norm.weight")
1797 .ok_or_else(|| anyhow!("Missing model.norm.weight"))?
1798 .to_device(*device);
1799 let lm_head_w = if let Some(w) = weights.remove("lm_head.weight") {
1800 match w.kind() {
1804 tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
1805 let w_bf = w.to_kind(tch::Kind::BFloat16);
1806 if let Some(s) = weights.remove("lm_head.weight_scale_inv") {
1807 let ws = w_bf.size(); let ss = s.size(); let br = ws[0] / ss[0];
1811 let bc = ws[1] / ss[1];
1812 let w_4d = w_bf.view([ss[0], br, ss[1], bc]);
1813 let s_4d = s
1814 .to_kind(tch::Kind::BFloat16)
1815 .view([ss[0], 1i64, ss[1], 1i64]);
1816 Some((w_4d * s_4d).reshape([ws[0], ws[1]]).to_device(*device))
1817 } else {
1818 Some(w_bf.to_device(*device))
1819 }
1820 }
1821 _ => Some(w.to_device(*device)),
1822 }
1823 } else {
1824 None
1825 };
1826
1827 let num_layers = cfg.num_hidden_layers as usize;
1828 let mut layers = Vec::with_capacity(num_layers);
1829
1830 for idx in 0..num_layers {
1831 layers.push(Self::build_layer(weights, &cfg, idx)?);
1832 }
1833
1834 let lm_head_transposed = if lm_head_w.is_none() {
1835 let embed_bf16 = match embed.kind() {
1837 tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
1838 embed.to_kind(tch::Kind::BFloat16)
1839 }
1840 _ => embed.shallow_clone(),
1841 };
1842 Some(embed_bf16.tr().contiguous())
1843 } else {
1844 None
1845 };
1846
1847 let conv_states = Arc::new(parking_lot::Mutex::new(
1848 (0..num_layers).map(|_| None::<Tensor>).collect::<Vec<_>>(),
1849 ));
1850 let rec_states = Arc::new(parking_lot::Mutex::new(
1851 (0..num_layers).map(|_| None::<Tensor>).collect::<Vec<_>>(),
1852 ));
1853
1854 let kv_cache = Some(Arc::new(parking_lot::Mutex::new(
1859 crate::runtime::kv_cache::KVCacheManager::new(
1860 num_layers + usize::from(mtp.is_some()),
1861 cfg.max_position_embeddings as usize,
1862 kv_quant_type,
1863 ),
1864 )));
1865
1866 let device_map = LayerDeviceMap::single(*device, num_layers.max(1))?;
1870
1871 Ok(Self {
1872 config: cfg,
1873 device: *device,
1874 dtype,
1875 device_map,
1876 layer_offset: 0,
1877 embed_tokens: embed,
1878 layers,
1879 norm: Qwen3_5RMSNorm::new(norm_w, 1e-6),
1880 lm_head: lm_head_w,
1881 lm_head_transposed,
1882 conv_states,
1883 rec_states,
1884 kv_cache,
1885 mtp,
1886 vision_encoder,
1887 vision_projector,
1888 })
1889 }
1890
1891 fn load_mlp(
1892 weights: &mut HashMap<String, Tensor>,
1893 prefix: &str,
1894 cfg: &Qwen3_5TextConfig,
1895 layer_idx: usize,
1896 ) -> Result<Qwen3_5Mlp> {
1897 if cfg.is_moe {
1898 let gate_proj = LinearProjection::take(weights, &format!("{prefix}.gate.weight"))?;
1900
1901 let n = cfg.num_experts;
1906 let mut gate_w_vecs: Vec<Tensor> = Vec::with_capacity(n);
1907 let mut gate_s_vecs: Vec<Tensor> = Vec::with_capacity(n);
1908 let mut up_w_vecs: Vec<Tensor> = Vec::with_capacity(n);
1909 let mut up_s_vecs: Vec<Tensor> = Vec::with_capacity(n);
1910 let mut down_w_vecs: Vec<Tensor> = Vec::with_capacity(n);
1911 let mut down_s_vecs: Vec<Tensor> = Vec::with_capacity(n);
1912 for e in 0..n {
1913 let ep = format!("{prefix}.experts.{e}");
1914 let (g, gs) = LinearProjection::take_weight_and_scale(
1917 weights,
1918 &format!("{ep}.gate_proj.weight"),
1919 )?;
1920 let (u, us) = LinearProjection::take_weight_and_scale(
1921 weights,
1922 &format!("{ep}.up_proj.weight"),
1923 )?;
1924 let (d, ds) = LinearProjection::take_weight_and_scale(
1925 weights,
1926 &format!("{ep}.down_proj.weight"),
1927 )?;
1928 gate_w_vecs.push(g.unsqueeze(0));
1929 if let Some(s) = gs {
1930 gate_s_vecs.push(s.unsqueeze(0));
1931 }
1932 up_w_vecs.push(u.unsqueeze(0));
1933 if let Some(s) = us {
1934 up_s_vecs.push(s.unsqueeze(0));
1935 }
1936 down_w_vecs.push(d.unsqueeze(0));
1937 if let Some(s) = ds {
1938 down_s_vecs.push(s.unsqueeze(0));
1939 }
1940 }
1941 let expert_gate_w = Tensor::cat(&gate_w_vecs.iter().collect::<Vec<_>>(), 0);
1943 let expert_gate_scale = (!gate_s_vecs.is_empty())
1944 .then(|| Tensor::cat(&gate_s_vecs.iter().collect::<Vec<_>>(), 0));
1945 let expert_up_w = Tensor::cat(&up_w_vecs.iter().collect::<Vec<_>>(), 0);
1946 let expert_up_scale = (!up_s_vecs.is_empty())
1947 .then(|| Tensor::cat(&up_s_vecs.iter().collect::<Vec<_>>(), 0));
1948 let expert_down_w = Tensor::cat(&down_w_vecs.iter().collect::<Vec<_>>(), 0);
1949 let expert_down_scale = (!down_s_vecs.is_empty())
1950 .then(|| Tensor::cat(&down_s_vecs.iter().collect::<Vec<_>>(), 0));
1951
1952 let sp = format!("{prefix}.shared_expert");
1953 let shared = Qwen3_5Expert {
1954 gate_proj: LinearProjection::take(weights, &format!("{sp}.gate_proj.weight"))?,
1955 up_proj: LinearProjection::take(weights, &format!("{sp}.up_proj.weight"))?,
1956 down_proj: LinearProjection::take(weights, &format!("{sp}.down_proj.weight"))?,
1957 };
1958 let shared_expert_gate =
1960 LinearProjection::take(weights, &format!("{prefix}.shared_expert_gate.weight"))
1961 .ok();
1962
1963 Ok(Qwen3_5Mlp::Sparse(Box::new(Qwen3_5SparseMoE {
1964 gate: gate_proj,
1965 expert_gate_w,
1966 expert_gate_scale,
1967 expert_up_w,
1968 expert_up_scale,
1969 expert_down_w,
1970 expert_down_scale,
1971 shared_expert: shared,
1972 shared_expert_gate,
1973 num_experts_per_tok: cfg.num_experts_per_tok,
1974 })))
1975 } else {
1976 Ok(Qwen3_5Mlp::Dense(Box::new(LlamaMLP {
1977 gate_proj: LinearProjection::take(weights, &format!("{prefix}.gate_proj.weight"))?,
1978 up_proj: LinearProjection::take(weights, &format!("{prefix}.up_proj.weight"))?,
1979 down_proj: LinearProjection::take(weights, &format!("{prefix}.down_proj.weight"))?,
1980 activation: "silu".to_owned(),
1981 layer_idx,
1982 })))
1983 }
1984 }
1985
1986 fn build_layer(
1992 weights: &mut HashMap<String, Tensor>,
1993 cfg: &Qwen3_5TextConfig,
1994 g: usize,
1995 ) -> Result<Qwen3_5Layer> {
1996 let layer_prefix = format!("model.layers.{g}");
1997
1998 let input_norm_w = weights
1999 .remove(&format!("{layer_prefix}.input_layernorm.weight"))
2000 .ok_or_else(|| anyhow!("Missing {layer_prefix}.input_layernorm.weight"))?;
2001 let post_norm_w = weights
2002 .remove(&format!("{layer_prefix}.post_attention_layernorm.weight"))
2003 .ok_or_else(|| anyhow!("Missing {layer_prefix}.post_attention_layernorm.weight"))?;
2004
2005 let layer_type = cfg
2009 .layer_types
2010 .get(g)
2011 .map(|s| s.as_str())
2012 .unwrap_or("linear_attention");
2013 info!("Loading layer {g}: {layer_type}");
2014
2015 let mixer = if layer_type == "full_attention" {
2016 let attn_prefix = format!("{layer_prefix}.self_attn");
2017 LayerMixer::FullAttn(Qwen3_5FullAttention::load(weights, &attn_prefix, cfg, g)?)
2018 } else {
2019 let lin_prefix = format!("{layer_prefix}.linear_attn");
2020 LayerMixer::LinearAttn(GatedDeltaNetLayer::load(weights, &lin_prefix, cfg, g)?)
2021 };
2022
2023 let mlp = Self::load_mlp(weights, &format!("{layer_prefix}.mlp"), cfg, g)?;
2024
2025 Ok(Qwen3_5Layer {
2026 mixer,
2027 mlp,
2028 input_layernorm: Qwen3_5RMSNorm::new(input_norm_w, cfg.rms_norm_eps),
2029 post_attention_layernorm: Qwen3_5RMSNorm::new(post_norm_w, cfg.rms_norm_eps),
2030 })
2031 }
2032
2033 #[allow(clippy::too_many_arguments)]
2055 pub fn stage_from_weights_with_config(
2056 weights: &mut HashMap<String, Tensor>,
2057 cfg: Qwen3_5TextConfig,
2058 devices: &LayerDeviceMap,
2059 layer_range: std::ops::Range<usize>,
2060 dtype: Kind,
2061 kv_quant_type: KVQuantType,
2062 ) -> Result<Self> {
2063 let num_global = cfg.num_hidden_layers as usize;
2064 if devices.len() != num_global {
2065 return Err(anyhow!(
2066 "stage_from_weights: device map covers {} layers but config has {} \
2067 (num_hidden_layers)",
2068 devices.len(),
2069 num_global
2070 ));
2071 }
2072 if layer_range.end > num_global || layer_range.start >= layer_range.end {
2073 return Err(anyhow!(
2074 "stage_from_weights: invalid layer range {:?} for {} layers",
2075 layer_range,
2076 num_global
2077 ));
2078 }
2079
2080 let is_first = layer_range.start == 0;
2081 let is_last = layer_range.end == num_global;
2082 let layer_offset = layer_range.start;
2083 let stage_device = devices.device_for(layer_range.start);
2087
2088 let embed = if is_first {
2090 weights
2091 .remove("model.embed_tokens.weight")
2092 .map(|w| w.to_device(stage_device))
2093 .ok_or_else(|| {
2094 anyhow!("stage_from_weights: first stage requires model.embed_tokens.weight")
2095 })?
2096 } else {
2097 Tensor::zeros([1, cfg.hidden_size as i64], (dtype, stage_device))
2100 };
2101
2102 let (norm, lm_head_w) = if is_last {
2103 let norm_w = weights
2104 .remove("model.norm.weight")
2105 .map(|w| w.to_device(stage_device))
2106 .ok_or_else(|| {
2107 anyhow!("stage_from_weights: last stage requires model.norm.weight")
2108 })?;
2109 let lm_head_w = weights.remove("lm_head.weight").map(|w| {
2110 match w.kind() {
2113 tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
2114 let w_bf = w.to_kind(tch::Kind::BFloat16);
2115 if let Some(s) = weights.remove("lm_head.weight_scale_inv") {
2116 let ws = w_bf.size();
2117 let ss = s.size();
2118 let br = ws[0] / ss[0];
2119 let bc = ws[1] / ss[1];
2120 let w_4d = w_bf.view([ss[0], br, ss[1], bc]);
2121 let s_4d = s
2122 .to_kind(tch::Kind::BFloat16)
2123 .view([ss[0], 1i64, ss[1], 1i64]);
2124 (w_4d * s_4d)
2125 .reshape([ws[0], ws[1]])
2126 .to_device(stage_device)
2127 } else {
2128 w_bf.to_device(stage_device)
2129 }
2130 }
2131 _ => w.to_device(stage_device),
2132 }
2133 });
2134 (Some(Qwen3_5RMSNorm::new(norm_w, 1e-6)), lm_head_w)
2135 } else {
2136 (None, None)
2137 };
2138
2139 let lm_head_transposed = if is_last && lm_head_w.is_none() {
2143 if !is_first {
2144 return Err(anyhow!(
2145 "stage_from_weights: last stage requires lm_head.weight (no tied embedding present)"
2146 ));
2147 }
2148 let embed_bf16 = match embed.kind() {
2149 tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
2150 embed.to_kind(tch::Kind::BFloat16)
2151 }
2152 _ => embed.shallow_clone(),
2153 };
2154 Some(embed_bf16.tr().contiguous())
2155 } else {
2156 None
2157 };
2158
2159 let mut layers = Vec::with_capacity(layer_range.len());
2161 for g in layer_range.clone() {
2162 let target = devices.device_for(g);
2163 let layer = Self::build_layer(weights, &cfg, g)?;
2164 layers.push(layer.into_device(target));
2165 }
2166
2167 let owned = layers.len();
2170 let conv_states = Arc::new(parking_lot::Mutex::new(
2171 (0..owned).map(|_| None::<Tensor>).collect::<Vec<_>>(),
2172 ));
2173 let rec_states = Arc::new(parking_lot::Mutex::new(
2174 (0..owned).map(|_| None::<Tensor>).collect::<Vec<_>>(),
2175 ));
2176 let kv_cache = Some(Arc::new(parking_lot::Mutex::new(
2177 crate::runtime::kv_cache::KVCacheManager::new(
2178 owned,
2179 cfg.max_position_embeddings as usize,
2180 kv_quant_type,
2181 ),
2182 )));
2183
2184 let lm_head_norm = norm.unwrap_or_else(|| {
2185 Qwen3_5RMSNorm::new(Tensor::zeros([1], (dtype, stage_device)), cfg.rms_norm_eps)
2188 });
2189
2190 let device_map = devices.clone();
2191
2192 Ok(Self {
2193 config: cfg,
2194 device: stage_device,
2195 dtype,
2196 device_map,
2197 layer_offset,
2198 embed_tokens: embed,
2199 layers,
2200 norm: lm_head_norm,
2201 lm_head: lm_head_w,
2202 lm_head_transposed,
2203 conv_states,
2204 rec_states,
2205 kv_cache,
2206 mtp: None,
2209 vision_encoder: None,
2210 vision_projector: None,
2211 })
2212 }
2213
2214 fn forward_inner(
2219 &self,
2220 input_ids: Option<&Tensor>,
2221 embeddings: Option<&Tensor>,
2222 start_pos: usize,
2223 delta: Option<&crate::training::TenantDelta>,
2224 ) -> Result<Tensor> {
2225 let mut hidden = if let Some(embeds) = embeddings {
2226 embeds.shallow_clone()
2227 } else {
2228 let ids = input_ids.ok_or_else(|| anyhow!("No input"))?;
2229 let flat = ids.flatten(0, -1);
2230 let emb = self.embed_tokens.index_select(0, &flat);
2231 let emb_shape = emb.size();
2232 let hidden_sz = emb_shape[emb_shape.len() - 1];
2233 let id_shape = ids.size();
2234 let batch = id_shape[0];
2235 let seq = id_shape[1];
2236 emb.reshape([batch, seq, hidden_sz])
2237 };
2238
2239 let (_, seq) = (hidden.size()[0], hidden.size()[1]);
2240
2241 let position_ids = Tensor::arange_start(
2247 start_pos as i64,
2248 start_pos as i64 + seq,
2249 (Kind::Int64, hidden.device()),
2250 );
2251
2252 let mut conv_guard = self.conv_states.lock();
2253 let mut rec_guard = self.rec_states.lock();
2254
2255 for (idx, layer) in self.layers.iter().enumerate() {
2256 let residual = hidden.shallow_clone();
2257
2258 hidden = layer.input_layernorm.forward(&hidden)?;
2259
2260 let delta_arg = delta.map(|d| (d, idx));
2261
2262 let mixer_out = match &layer.mixer {
2263 LayerMixer::LinearAttn(gdn) => gdn.forward(
2264 &hidden,
2265 &mut conv_guard[idx],
2266 &mut rec_guard[idx],
2267 delta_arg,
2268 )?,
2269 LayerMixer::FullAttn(attn) => {
2270 if let Some(ref cache_arc) = self.kv_cache {
2271 let kv = cache_arc.lock();
2272 kv.with_layer_cache(idx, |lc| {
2273 attn.forward(
2274 &hidden,
2275 Some(&position_ids),
2276 Some(lc),
2277 start_pos,
2278 delta_arg,
2279 )
2280 })
2281 .ok_or_else(|| anyhow!("No KV cache for layer {idx}"))??
2282 } else {
2283 attn.forward(&hidden, Some(&position_ids), None, start_pos, delta_arg)?
2284 }
2285 }
2286 };
2287
2288 hidden = residual + &mixer_out;
2289
2290 let residual2 = hidden.shallow_clone();
2291 hidden = layer.post_attention_layernorm.forward(&hidden)?;
2292 hidden = layer.mlp.forward(&hidden)?;
2293 hidden = residual2 + &hidden;
2294 }
2295 Ok(hidden)
2296 }
2297
2298 fn forward_layers_inner(
2313 &self,
2314 hidden: &Tensor,
2315 range: std::ops::Range<usize>,
2316 start_pos: usize,
2317 delta: Option<&crate::training::TenantDelta>,
2318 ) -> Result<Tensor> {
2319 let num_global = self.config.num_hidden_layers as usize;
2320 if range.end > num_global || range.start >= range.end {
2321 return Err(anyhow!(
2322 "forward_layers: invalid global range {range:?} for {num_global} layers"
2323 ));
2324 }
2325 let owned = self.layer_offset..self.layer_offset + self.layers.len();
2327 if range.start < owned.start || range.end > owned.end {
2328 return Err(anyhow!(
2329 "forward_layers: range {range:?} not within this stage's owned layers {owned:?}"
2330 ));
2331 }
2332
2333 let mut hidden = hidden.shallow_clone();
2334
2335 let seq = hidden.size()[1];
2340 let mut position_ids = Tensor::arange_start(
2341 start_pos as i64,
2342 start_pos as i64 + seq,
2343 (Kind::Int64, hidden.device()),
2344 );
2345
2346 let mut conv_guard = self.conv_states.lock();
2347 let mut rec_guard = self.rec_states.lock();
2348
2349 for g in range {
2350 let local_idx = g - self.layer_offset;
2351
2352 let target = self.device_map.device_for(g);
2355 if hidden.device() != target {
2356 hidden = hidden.to_device(target);
2357 position_ids = position_ids.to_device(target);
2358 }
2359
2360 let layer = &self.layers[local_idx];
2361 let residual = hidden.shallow_clone();
2362 hidden = layer.input_layernorm.forward(&hidden)?;
2363
2364 let delta_arg = delta.map(|d| (d, g));
2367
2368 let mixer_out = match &layer.mixer {
2369 LayerMixer::LinearAttn(gdn) => {
2370 gdn.forward(
2372 &hidden,
2373 &mut conv_guard[local_idx],
2374 &mut rec_guard[local_idx],
2375 delta_arg,
2376 )?
2377 }
2378 LayerMixer::FullAttn(attn) => {
2379 if let Some(ref cache_arc) = self.kv_cache {
2380 let kv = cache_arc.lock();
2381 kv.with_layer_cache(local_idx, |lc| {
2382 attn.forward(
2383 &hidden,
2384 Some(&position_ids),
2385 Some(lc),
2386 start_pos,
2387 delta_arg,
2388 )
2389 })
2390 .ok_or_else(|| anyhow!("No KV cache for local layer {local_idx}"))??
2391 } else {
2392 attn.forward(&hidden, Some(&position_ids), None, start_pos, delta_arg)?
2393 }
2394 }
2395 };
2396
2397 hidden = residual + &mixer_out;
2398
2399 let residual2 = hidden.shallow_clone();
2400 hidden = layer.post_attention_layernorm.forward(&hidden)?;
2401 hidden = layer.mlp.forward(&hidden)?;
2402 hidden = residual2 + &hidden;
2403 }
2404 Ok(hidden)
2405 }
2406
2407 fn forward_layers_train_inner(
2427 &self,
2428 hidden: &Tensor,
2429 range: std::ops::Range<usize>,
2430 delta: Option<&crate::training::TenantDelta>,
2431 ) -> Result<Tensor> {
2432 self.forward_layers_stateless_inner(hidden, range, delta, None)
2433 }
2434
2435 fn forward_layers_stateless_inner(
2436 &self,
2437 hidden: &Tensor,
2438 range: std::ops::Range<usize>,
2439 delta: Option<&crate::training::TenantDelta>,
2440 check: Option<&dyn Fn() -> Result<()>>,
2441 ) -> Result<Tensor> {
2442 let num_global = self.config.num_hidden_layers as usize;
2443 if range.end > num_global || range.start >= range.end {
2444 return Err(anyhow!(
2445 "forward_layers_train: invalid global range {range:?} for {num_global} layers"
2446 ));
2447 }
2448 let owned = self.layer_offset..self.layer_offset + self.layers.len();
2449 if range.start < owned.start || range.end > owned.end {
2450 return Err(anyhow!(
2451 "forward_layers_train: range {range:?} not within this stage's owned layers {owned:?}"
2452 ));
2453 }
2454
2455 let mut hidden = hidden.shallow_clone();
2456
2457 let seq = hidden.size()[1];
2460 let mut position_ids = Tensor::arange(seq, (Kind::Int64, hidden.device()));
2461
2462 let mut conv_local: Vec<Option<Tensor>> = (0..self.layers.len()).map(|_| None).collect();
2466 let mut rec_local: Vec<Option<Tensor>> = (0..self.layers.len()).map(|_| None).collect();
2467
2468 for g in range {
2469 if let Some(check) = check {
2470 check()?;
2471 }
2472 let local_idx = g - self.layer_offset;
2473
2474 let target = self.device_map.device_for(g);
2477 if hidden.device() != target {
2478 hidden = hidden.to_device(target);
2479 position_ids = position_ids.to_device(target);
2480 }
2481
2482 let layer = &self.layers[local_idx];
2483 let residual = hidden.shallow_clone();
2484 hidden = layer.input_layernorm.forward(&hidden)?;
2485
2486 let delta_arg = delta.map(|d| (d, g));
2489
2490 let mixer_out = match &layer.mixer {
2491 LayerMixer::LinearAttn(gdn) => gdn.forward(
2492 &hidden,
2493 &mut conv_local[local_idx],
2494 &mut rec_local[local_idx],
2495 delta_arg,
2496 )?,
2497 LayerMixer::FullAttn(attn) => {
2498 attn.forward(&hidden, Some(&position_ids), None, 0, delta_arg)?
2500 }
2501 };
2502
2503 hidden = residual + &mixer_out;
2504
2505 let residual2 = hidden.shallow_clone();
2506 hidden = layer.post_attention_layernorm.forward(&hidden)?;
2507 hidden = layer.mlp.forward(&hidden)?;
2508 hidden = residual2 + &hidden;
2509 }
2510 if let Some(check) = check {
2511 check()?;
2512 }
2513 Ok(hidden)
2514 }
2515
2516 fn encode_vision_features(&self, pixel_values: &Tensor) -> Result<Tensor> {
2521 let enc = self
2522 .vision_encoder
2523 .as_ref()
2524 .ok_or_else(|| anyhow!("No vision encoder loaded"))?;
2525 let proj = self
2526 .vision_projector
2527 .as_ref()
2528 .ok_or_else(|| anyhow!("No vision projector loaded"))?;
2529
2530 let features = enc.forward(pixel_values)?;
2532 proj.try_apply(&features)
2534 }
2535
2536 fn prepare_inputs_embeds(
2541 &self,
2542 input_ids: &Tensor,
2543 pixel_values: Option<&Tensor>,
2544 images_seq_mask: Option<&Tensor>,
2545 _images_emb_mask: Option<&Tensor>,
2546 ) -> Result<Tensor> {
2547 let flat = input_ids.flatten(0, -1);
2549 let text_emb = self.embed_tokens.index_select(0, &flat);
2550 let ids_shape = input_ids.size();
2551 let hidden_sz = text_emb
2552 .size()
2553 .last()
2554 .copied()
2555 .unwrap_or(self.config.hidden_size as i64);
2556 let combined = text_emb.reshape([ids_shape[0], ids_shape[1], hidden_sz]);
2557
2558 if let (Some(pv), Some(mask)) = (pixel_values, images_seq_mask) {
2560 let vision_feats = self.encode_vision_features(pv)?; let (batch, seq, hs) = dims3(&combined)?;
2564 for b in 0..batch {
2565 let mask_b = mask.select(0, b).to_kind(Kind::Bool); let positions: Vec<i64> = (0..seq)
2567 .filter(|&t| mask_b.double_value(&[t]) != 0.0)
2568 .collect();
2569 if positions.is_empty() {
2570 continue;
2571 }
2572 let n_img = positions.len() as i64;
2574 let vf_slice = vision_feats.narrow(0, 0, n_img.min(vision_feats.size()[0]));
2575 for (img_idx, &pos) in positions.iter().enumerate() {
2576 if (img_idx as i64) < vf_slice.size()[0] {
2577 let vf = vf_slice.select(0, img_idx as i64); combined
2579 .select(0, b)
2580 .select(0, pos)
2581 .copy_(&vf.to_kind(self.dtype));
2582 }
2583 }
2584 }
2585 }
2586
2587 Ok(combined)
2588 }
2589
2590 pub fn text_config(&self) -> &Qwen3_5TextConfig {
2592 &self.config
2593 }
2594
2595 pub fn snapshot_ssm_states(&self) -> (Vec<Option<Tensor>>, Vec<Option<Tensor>>) {
2600 let conv_snapshot = {
2601 let guard = self.conv_states.lock();
2602 guard
2603 .iter()
2604 .map(|opt| opt.as_ref().map(|t| t.copy()))
2605 .collect()
2606 };
2607 let rec_snapshot = {
2608 let guard = self.rec_states.lock();
2609 guard
2610 .iter()
2611 .map(|opt| opt.as_ref().map(|t| t.copy()))
2612 .collect()
2613 };
2614 (conv_snapshot, rec_snapshot)
2615 }
2616
2617 pub fn restore_ssm_states(
2619 &self,
2620 conv_snapshot: Vec<Option<Tensor>>,
2621 rec_snapshot: Vec<Option<Tensor>>,
2622 ) {
2623 *self.conv_states.lock() = conv_snapshot;
2624 *self.rec_states.lock() = rec_snapshot;
2625 }
2626
2627 fn decode_layer_impl(
2638 &self,
2639 layer_idx: usize,
2640 hidden_states: &Tensor,
2641 position_ids: Option<&Tensor>,
2642 delta: Option<&crate::training::TenantDelta>,
2643 ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
2644 let layer = self
2645 .layers
2646 .get(layer_idx)
2647 .ok_or_else(|| anyhow!("Layer {layer_idx} out of range"))?;
2648 let residual = hidden_states.shallow_clone();
2649 let normed = layer.input_layernorm.forward(hidden_states)?;
2650
2651 let delta_arg = delta.map(|d| (d, layer_idx));
2652
2653 let mixer_out = match &layer.mixer {
2654 LayerMixer::LinearAttn(gdn) => {
2655 let mut conv = self.conv_states.lock();
2656 let mut rec = self.rec_states.lock();
2657 gdn.forward(
2658 &normed,
2659 &mut conv[layer_idx],
2660 &mut rec[layer_idx],
2661 delta_arg,
2662 )?
2663 }
2664 LayerMixer::FullAttn(attn) => {
2665 attn.forward(&normed, position_ids, None, 0, delta_arg)?
2666 }
2667 };
2668
2669 let hidden = &residual + &mixer_out;
2670 let residual2 = hidden.shallow_clone();
2671 let normed2 = layer.post_attention_layernorm.forward(&hidden)?;
2672 let ffn_out = layer.mlp.forward(&normed2)?;
2673 Ok((&residual2 + &ffn_out, None))
2674 }
2675
2676 fn lm_head_apply(&self, hidden: &Tensor) -> Result<Tensor> {
2677 let (batch, seq, hs) = dims3(hidden)?;
2678 let flat = hidden.reshape([batch * seq, hs]);
2679 let logits = if let Some(ref lm_head) = self.lm_head {
2680 flat.matmul(&lm_head.tr())
2682 } else if let Some(ref lm_t) = self.lm_head_transposed {
2683 flat.matmul(lm_t)
2684 } else {
2685 return Err(anyhow!("No lm_head weight"));
2686 };
2687
2688 Ok(logits.reshape([batch, seq, -1]))
2689 }
2690
2691 pub fn has_mtp(&self) -> bool {
2697 self.mtp.is_some()
2698 }
2699
2700 fn mtp_kv_slot(&self) -> usize {
2702 self.config.num_hidden_layers as usize
2703 }
2704
2705 pub fn forward_with_cache_hidden(
2713 &self,
2714 input: &Tensor,
2715 start_pos: usize,
2716 last_logits_only: bool,
2717 ) -> Result<(Tensor, Tensor)> {
2718 let hidden = self.forward_inner(Some(input), None, start_pos, None)?;
2719 let seq_len = hidden.size()[1];
2720 anyhow::ensure!(
2721 seq_len > 0,
2722 "forward_with_cache_hidden: empty token sequence (seq_len=0)"
2723 );
2724 let logits = if last_logits_only {
2725 let last = hidden.narrow(1, seq_len - 1, 1);
2726 let normed = self.norm.forward(&last)?;
2727 self.lm_head_apply(&normed)?
2728 } else {
2729 let normed = self.norm.forward(&hidden)?;
2730 self.lm_head_apply(&normed)?
2731 };
2732 Ok((logits, hidden))
2733 }
2734
2735 pub fn mtp_forward(&self, token_ids: &Tensor, hiddens: &Tensor, base: usize) -> Result<Tensor> {
2746 let mtp = self
2747 .mtp
2748 .as_ref()
2749 .ok_or_else(|| anyhow!("MTP head not loaded (dense checkpoints only)"))?;
2750 let (batch, seq, hidden_sz) = dims3(hiddens)?;
2751 anyhow::ensure!(seq > 0, "mtp_forward: empty slot sequence");
2752 anyhow::ensure!(
2753 token_ids.size() == [batch, seq],
2754 "mtp_forward: token_ids shape {:?} does not match hiddens shape {:?}",
2755 token_ids.size(),
2756 hiddens.size()
2757 );
2758
2759 let emb = self.embed_tokens(token_ids)?;
2760 let e = mtp.pre_fc_norm_embedding.forward(&emb)?;
2761 let h = mtp.pre_fc_norm_hidden.forward(hiddens)?;
2762 let x = Tensor::cat(&[e, h], -1).reshape([batch * seq, 2 * hidden_sz]);
2764 let x = mtp.fc.try_apply(&x)?.reshape([batch, seq, hidden_sz]);
2765
2766 let position_ids =
2767 Tensor::arange_start(base as i64, base as i64 + seq, (Kind::Int64, x.device()));
2768
2769 let residual = x.shallow_clone();
2771 let normed = mtp.input_layernorm.forward(&x)?;
2772 let attn_out = if let Some(ref cache_arc) = self.kv_cache {
2773 let kv = cache_arc.lock();
2774 kv.with_layer_cache(self.mtp_kv_slot(), |lc| {
2775 mtp.attn
2776 .forward(&normed, Some(&position_ids), Some(lc), base, None)
2777 })
2778 .ok_or_else(|| anyhow!("No KV cache for MTP slot {}", self.mtp_kv_slot()))??
2779 } else {
2780 mtp.attn
2781 .forward(&normed, Some(&position_ids), None, base, None)?
2782 };
2783 let x = residual + attn_out;
2784
2785 let residual2 = x.shallow_clone();
2786 let normed2 = mtp.post_attention_layernorm.forward(&x)?;
2787 let mlp_out = mtp.mlp.forward(&normed2, None)?;
2788 let x = residual2 + mlp_out;
2789
2790 let last = x.narrow(1, seq - 1, 1);
2792 let normed = mtp.final_norm.forward(&last)?;
2793 self.lm_head_apply(&normed)
2794 }
2795}
2796
2797impl ModelOperations for Qwen3_5Model {
2802 fn architecture(&self) -> ModelArchitecture {
2803 ModelArchitecture::Qwen3_5
2804 }
2805
2806 fn config(&self) -> &dyn ArchitectureConfig {
2807 &self.config
2808 }
2809
2810 fn decision_hidden_states(&self, input: &Tensor) -> Result<Tensor> {
2811 self.decision_hidden_states_checked(input, &|| Ok(()))
2812 }
2813
2814 fn decision_hidden_states_checked(
2815 &self,
2816 input: &Tensor,
2817 check: &dyn Fn() -> Result<()>,
2818 ) -> Result<Tensor> {
2819 check()?;
2820 anyhow::ensure!(
2821 input.kind() == Kind::Int64
2822 && input.dim() == 2
2823 && input.size()[0] == 1
2824 && input.size()[1] > 0,
2825 "decision input must be [1,tokens]"
2826 );
2827 anyhow::ensure!(
2828 self.layer_offset == 0 && self.layers.len() == self.config.num_hidden_layers as usize,
2829 "decision inference requires the complete backbone"
2830 );
2831 anyhow::ensure!(
2832 input.min().int64_value(&[]) >= 0
2833 && input.max().int64_value(&[]) < self.embed_tokens.size()[0],
2834 "decision token ID outside embedding vocabulary"
2835 );
2836 let embedded = self.embed_tokens(input)?;
2837 let hidden = self.forward_layers_stateless_inner(
2838 &embedded,
2839 0..self.layers.len(),
2840 None,
2841 Some(check),
2842 )?;
2843 Ok(self.norm.forward(&hidden)?.squeeze_dim(0))
2844 }
2845
2846 fn decision_output_embedding_rows(&self, ids: &[i64]) -> Result<Tensor> {
2847 let (weight, transposed) = match (&self.lm_head, &self.lm_head_transposed) {
2848 (Some(w), _) => (w, false),
2849 (None, Some(w)) => (w, true),
2850 _ => return Err(anyhow!("missing output embedding table")),
2851 };
2852 anyhow::ensure!(
2853 !matches!(weight.kind(), Kind::Float8e4m3fn | Kind::Float8e5m2),
2854 "decision output rows require preserved floating output table"
2855 );
2856 let vocab = weight.size()[if transposed { 1 } else { 0 }];
2857 anyhow::ensure!(
2858 ids.iter().all(|id| *id >= 0 && *id < vocab),
2859 "output token ID outside vocabulary"
2860 );
2861 let indices = Tensor::from_slice(ids).to_device(weight.device());
2862 Ok(if transposed {
2863 weight.index_select(1, &indices).transpose(0, 1)
2864 } else {
2865 weight.index_select(0, &indices)
2866 })
2867 }
2868
2869 fn forward(&self, input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor> {
2870 let hidden = self.forward_inner(Some(input), None, 0, None)?;
2871 let normed = self.norm.forward(&hidden)?;
2872 self.lm_head_apply(&normed)
2873 }
2874
2875 fn forward_with_cache(&self, input: &Tensor, start_pos: usize) -> Result<Tensor> {
2876 let hidden = self.forward_inner(Some(input), None, start_pos, None)?;
2877 let seq_len = hidden.size()[1];
2878 anyhow::ensure!(
2879 seq_len > 0,
2880 "forward_with_cache: empty token sequence (seq_len=0)"
2881 );
2882 let last = hidden.narrow(1, seq_len - 1, 1);
2885 let normed = self.norm.forward(&last)?;
2886 self.lm_head_apply(&normed)
2887 }
2888
2889 fn forward_from_embeddings(&self, embeddings: &Tensor, start_pos: usize) -> Result<Tensor> {
2890 let hidden = self.forward_inner(None, Some(embeddings), start_pos, None)?;
2891 let seq_len = hidden.size()[1];
2892 anyhow::ensure!(
2893 seq_len > 0,
2894 "forward_from_embeddings: empty embedding sequence (seq_len=0)"
2895 );
2896 let last = hidden.narrow(1, seq_len - 1, 1);
2897 let normed = self.norm.forward(&last)?;
2898 self.lm_head_apply(&normed)
2899 }
2900
2901 fn forward_with_cache_and_delta(
2902 &self,
2903 input: &Tensor,
2904 start_pos: usize,
2905 delta: Option<&crate::training::TenantDelta>,
2906 ) -> Result<Tensor> {
2907 let hidden = self.forward_inner(Some(input), None, start_pos, delta)?;
2908 let seq_len = hidden.size()[1];
2909 anyhow::ensure!(
2910 seq_len > 0,
2911 "forward_with_cache_and_delta: empty token sequence (seq_len=0)"
2912 );
2913 let last = hidden.narrow(1, seq_len - 1, 1);
2914 let normed = self.norm.forward(&last)?;
2915 self.lm_head_apply(&normed)
2916 }
2917
2918 fn embed_tokens(&self, input_ids: &Tensor) -> Result<Tensor> {
2919 let flat = input_ids.flatten(0, -1);
2920 let emb = self.embed_tokens.index_select(0, &flat);
2921 let emb_shape = emb.size();
2922 let hidden_sz = emb_shape[emb_shape.len() - 1];
2923 let ids_shape = input_ids.size();
2924 Ok(emb.reshape([ids_shape[0], ids_shape[1], hidden_sz]))
2925 }
2926
2927 fn apply_final_norm(&self, hidden_states: &Tensor) -> Result<Tensor> {
2928 self.norm.forward(hidden_states)
2929 }
2930
2931 fn lm_head(&self, hidden_states: &Tensor) -> Result<Tensor> {
2932 self.lm_head_apply(hidden_states)
2933 }
2934
2935 fn num_layers(&self) -> usize {
2936 self.config.num_hidden_layers as usize
2937 }
2938
2939 fn forward_layers(
2940 &self,
2941 hidden: &Tensor,
2942 range: std::ops::Range<usize>,
2943 start_pos: usize,
2944 delta: Option<&crate::training::TenantDelta>,
2945 ) -> Result<Tensor> {
2946 self.forward_layers_inner(hidden, range, start_pos, delta)
2947 }
2948
2949 fn forward_layers_train(
2950 &self,
2951 hidden: &Tensor,
2952 range: std::ops::Range<usize>,
2953 delta: Option<&crate::training::TenantDelta>,
2954 ) -> Result<Tensor> {
2955 self.forward_layers_train_inner(hidden, range, delta)
2956 }
2957
2958 fn decode_layer(
2959 &self,
2960 layer_idx: usize,
2961 hidden_states: &Tensor,
2962 _attention_mask: Option<&Tensor>,
2963 position_ids: Option<&Tensor>,
2964 _past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
2965 ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
2966 self.decode_layer_impl(layer_idx, hidden_states, position_ids, None)
2967 }
2968
2969 fn decode_layer_with_delta(
2970 &self,
2971 layer_idx: usize,
2972 hidden_states: &Tensor,
2973 _attention_mask: Option<&Tensor>,
2974 position_ids: Option<&Tensor>,
2975 _past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
2976 delta: &crate::training::TenantDelta,
2977 ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
2978 self.decode_layer_impl(layer_idx, hidden_states, position_ids, Some(delta))
2979 }
2980
2981 fn clear_kv_cache(&self) {
2982 let mut conv = self.conv_states.lock();
2983 let mut rec = self.rec_states.lock();
2984 for s in conv.iter_mut() {
2985 *s = None;
2986 }
2987 for s in rec.iter_mut() {
2988 *s = None;
2989 }
2990 if let Some(ref cache) = self.kv_cache {
2991 cache.lock().clear_all();
2992 }
2993 }
2994
2995 fn set_kv_cache(&mut self, cache: Arc<parking_lot::Mutex<KVCacheManager>>) {
2996 if self.mtp.is_some() {
3000 cache.lock().ensure_layer_cache(self.mtp_kv_slot());
3001 }
3002 self.kv_cache = Some(cache);
3003 }
3004
3005 fn get_kv_cache(&self) -> Option<Arc<parking_lot::Mutex<KVCacheManager>>> {
3006 self.kv_cache.clone()
3007 }
3008
3009 fn take_kv_cache(&mut self) -> Option<Arc<parking_lot::Mutex<KVCacheManager>>> {
3010 self.kv_cache.take()
3011 }
3012
3013 fn reshape_for_attention(&self, tensor: &Tensor, _is_key_value: bool) -> Result<Tensor> {
3014 Ok(tensor.shallow_clone())
3015 }
3016
3017 fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor> {
3018 let rd = self.config.rotary_dim as i64;
3019 let device = tensor.device();
3020 let dtype = tensor.kind();
3021 let mut rope = RoPE::new_with_dtype(
3022 rd,
3023 self.config.rope_theta as f64,
3024 self.config.max_position_embeddings as i64,
3025 device,
3026 dtype,
3027 )?;
3028 rope.forward(tensor, Some(position_ids))
3029 }
3030
3031 fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
3032 self.norm.forward(tensor)
3033 }
3034
3035 fn get_attention_mask(&self, seq_len: usize, _past_kv_len: usize) -> Result<Tensor> {
3036 let sq = seq_len as i64;
3037 Ok(Tensor::ones([sq, sq], (Kind::Float, self.device)).tril(0))
3038 }
3039
3040 fn is_multimodal(&self) -> bool {
3041 self.vision_encoder.is_some()
3042 }
3043
3044 fn prepare_multimodal_inputs(
3045 &self,
3046 input_ids: &Tensor,
3047 pixel_values: Option<&Tensor>,
3048 images_seq_mask: Option<&Tensor>,
3049 images_emb_mask: Option<&Tensor>,
3050 ) -> Result<Tensor> {
3051 self.prepare_inputs_embeds(input_ids, pixel_values, images_seq_mask, images_emb_mask)
3052 }
3053
3054 fn encode_vision(&self, pixel_values: &Tensor) -> Result<Tensor> {
3055 self.encode_vision_features(pixel_values)
3056 }
3057
3058 fn as_any(&self) -> &dyn std::any::Any {
3059 self
3060 }
3061}
3062
3063#[cfg(test)]
3064#[allow(clippy::unwrap_used, clippy::expect_used)]
3065mod pipeline_tests {
3066 use super::*;
3075 use crate::runtime::device_pool::LayerDeviceMap;
3076
3077 #[test]
3081 fn gdn_l2_normalize_matches_additive_epsilon_reference() {
3082 let input = Tensor::from_slice(&[3.0f32, 4.0, -2.0, 1.0]).reshape([2, 2]);
3083 let expected = Tensor::from_slice(&[
3084 (3.0f64 / (25.0f64 + 1e-6).sqrt()) as f32,
3085 (4.0f64 / (25.0f64 + 1e-6).sqrt()) as f32,
3086 (-2.0f64 / (5.0f64 + 1e-6).sqrt()) as f32,
3087 (1.0f64 / (5.0f64 + 1e-6).sqrt()) as f32,
3088 ])
3089 .reshape([2, 2]);
3090 assert!(l2_normalize(&input).allclose(&expected, 1e-7, 1e-7, false));
3091 }
3092
3093 #[test]
3094 fn gdn_l2_normalize_small_vectors_do_not_clamp_norm() {
3095 let input = Tensor::from_slice(&[3e-4f32, 4e-4, 1e-3, 0.0]).reshape([1, 2, 2]);
3096 let a = 3e-4f32 as f64;
3097 let b = 4e-4f32 as f64;
3098 let c = 1e-3f32 as f64;
3099 let denominator = (a * a + b * b + 1e-6).sqrt();
3100 let expected = Tensor::from_slice(&[
3101 (a / denominator) as f32,
3102 (b / denominator) as f32,
3103 (c / (c * c + 1e-6).sqrt()) as f32,
3104 0.0,
3105 ])
3106 .reshape([1, 2, 2]);
3107 let output = l2_normalize(&input);
3108 assert_eq!(output.size(), input.size());
3109 assert!(output.allclose(&expected, 1e-7, 1e-7, false));
3110 let old_clamped = &input
3111 / input
3112 .norm_scalaropt_dim(2.0, &[-1i64][..], true)
3113 .clamp_min(1e-3);
3114 assert!((&output - old_clamped).abs().max().double_value(&[]) > 0.1);
3115 }
3116
3117 #[test]
3118 fn gdn_l2_normalize_zero_vectors_remain_finite_zero() {
3119 let input = Tensor::zeros([2, 3, 4], (Kind::Float, Device::Cpu));
3120 let output = l2_normalize(&input);
3121 assert_eq!(output.size(), input.size());
3122 assert_eq!(output.kind(), Kind::Float);
3123 assert_ne!(output.isfinite().all().int64_value(&[]), 0);
3124 assert_eq!(output.abs().max().double_value(&[]), 0.0);
3125 }
3126
3127 pub(super) const HIDDEN: i64 = 16;
3128 pub(super) const HEADS: i64 = 2;
3129 pub(super) const KV_HEADS: i64 = 2;
3130 pub(super) const HEAD_DIM: i64 = 8;
3131 pub(super) const INTER: i64 = 32;
3132 pub(super) const VOCAB: i64 = 48;
3133 const LAYERS: usize = 4;
3134 const LIN_K_HEADS: usize = 1;
3136 const LIN_V_HEADS: usize = 1;
3137 const LIN_K_DIM: usize = 4;
3138 const LIN_V_DIM: usize = 4;
3139 const CONV_KERNEL: usize = 4;
3140
3141 #[test]
3147 #[allow(clippy::print_stderr)]
3148 fn fp8_fused_projection_metadata_probe() {
3149 if std::env::var_os("HYPRSTREAM_QWEN_FP8_PROBE").is_none() {
3150 return;
3151 }
3152 assert!(crate::config::default_fp8_gemm());
3153 let w1 = Tensor::zeros([256, 256], (Kind::Float, Device::Cpu)).to_kind(Kind::Float8e4m3fn);
3154 let w2 = Tensor::zeros([256, 256], (Kind::Float, Device::Cpu)).to_kind(Kind::Float8e4m3fn);
3155 let s1 = Tensor::ones([2, 2], (Kind::Float, Device::Cpu));
3156 let s2 = Tensor::ones([2, 2], (Kind::Float, Device::Cpu));
3157 crate::runtime::architectures::llama::reset_rowwise_requant_peak();
3158 let fused = cat_projs(vec![
3159 LinearProjection::with_scale(w1, s1),
3160 LinearProjection::with_scale(w2, s2),
3161 ]);
3162 assert_eq!(fused.weight.size(), [256, 512]);
3163 assert_eq!(
3164 fused.scale.as_ref().expect("fused FP8 scale").size(),
3165 [2, 4]
3166 );
3167 assert_eq!(
3168 fused.scale_v2.as_ref().expect("fused packed scale").size(),
3169 [4, 4]
3170 );
3171 assert_eq!(
3172 crate::runtime::architectures::llama::rowwise_requant_live(),
3173 1,
3174 "fusion did not leave exactly its one derived rowwise allocation alive"
3175 );
3176 assert_eq!(
3177 crate::runtime::architectures::llama::rowwise_requant_peak(),
3178 2,
3179 "source rowwise allocations overlapped with fused rowwise construction"
3180 );
3181 eprintln!("Qwen fused FP8 allocation probe: source rowwise copies dropped before fused allocation");
3182 }
3183
3184 #[test]
3187 fn fp8_moe_loading_avoids_transient_requant() {
3188 if std::env::var_os("HYPRSTREAM_QWEN_MOE_LOADING_PROBE").is_none() {
3189 let output = std::process::Command::new(std::env::current_exe().unwrap())
3190 .args(["--exact", "runtime::architectures::qwen3_5::pipeline_tests::fp8_moe_loading_avoids_transient_requant", "--nocapture"])
3191 .env("HYPRSTREAM_FP8_GEMM", "1")
3192 .env("HYPRSTREAM_FP8_DEQUANT_LOAD", "0")
3193 .env("HYPRSTREAM_QWEN_MOE_LOADING_PROBE", "1")
3194 .output().expect("spawn MoE allocation probe");
3195 let transcript = format!(
3196 "{}{}",
3197 String::from_utf8_lossy(&output.stdout),
3198 String::from_utf8_lossy(&output.stderr)
3199 );
3200 assert!(
3201 output.status.success() && transcript.contains("1 passed"),
3202 "MoE allocation probe did not pass: {transcript}"
3203 );
3204 return;
3205 }
3206 use crate::runtime::architectures::llama::{
3207 reset_rowwise_requant_peak, rowwise_requant_live, rowwise_requant_peak,
3208 };
3209 assert!(crate::config::default_fp8_gemm());
3210 let mut cfg = tiny_config();
3211 cfg.is_moe = true;
3212 cfg.num_experts = 3;
3213 cfg.num_experts_per_tok = 1;
3214 let mut weights = HashMap::new();
3215 weights.insert(
3216 "mlp.gate.weight".to_owned(),
3217 Tensor::zeros([3, 128], (Kind::Float, Device::Cpu)),
3218 );
3219 for (name, shape) in [
3220 ("gate_proj", [256, 128]),
3221 ("up_proj", [256, 128]),
3222 ("down_proj", [128, 256]),
3223 ] {
3224 weights.insert(
3225 format!("mlp.shared_expert.{name}.weight"),
3226 Tensor::zeros(shape, (Kind::Float, Device::Cpu)),
3227 );
3228 }
3229 let mut expected = Vec::new();
3230 for e in 0..cfg.num_experts {
3231 for (name, shape) in [
3232 ("gate_proj", [256, 128]),
3233 ("up_proj", [256, 128]),
3234 ("down_proj", [128, 256]),
3235 ] {
3236 let key = format!("mlp.experts.{e}.{name}.weight");
3237 let weight = (Tensor::arange(shape[0] * shape[1], (Kind::Float, Device::Cpu))
3238 * 0.01
3239 + e as f64)
3240 .sin()
3241 .reshape(shape)
3242 .to_kind(Kind::Float8e4m3fn);
3243 let scale = Tensor::from_slice(&[0.5f32 + e as f32, 1.5 + e as f32])
3244 .reshape([shape[0] / 128, shape[1] / 128]);
3245 expected.push((
3246 name,
3247 e as i64,
3248 weight.transpose(0, 1).to_kind(Kind::Float),
3249 scale.transpose(0, 1).contiguous(),
3250 ));
3251 weights.insert(key.clone(), weight);
3252 weights.insert(format!("{key}_scale_inv"), scale);
3253 }
3254 }
3255 reset_rowwise_requant_peak();
3256 let Qwen3_5Mlp::Sparse(loaded) =
3257 Qwen3_5Model::load_mlp(&mut weights, "mlp", &cfg, 0).unwrap()
3258 else {
3259 panic!("fixture must load sparse experts");
3260 };
3261 assert!(weights.is_empty(), "loader must consume checkpoint tensors");
3262 assert_eq!(rowwise_requant_live(), 0);
3263 assert_eq!(
3264 rowwise_requant_peak(),
3265 0,
3266 "transient batched experts must never construct discarded rowwise copies"
3267 );
3268 for (name, e, weight, scale) in expected {
3269 let (stacked_w, stacked_s) = match name {
3270 "gate_proj" => (&loaded.expert_gate_w, &loaded.expert_gate_scale),
3271 "up_proj" => (&loaded.expert_up_w, &loaded.expert_up_scale),
3272 _ => (&loaded.expert_down_w, &loaded.expert_down_scale),
3273 };
3274 assert_eq!(stacked_w.kind(), Kind::Float8e4m3fn);
3275 assert!(stacked_w.get(e).to_kind(Kind::Float).equal(&weight));
3276 assert!(stacked_s.as_ref().unwrap().get(e).equal(&scale));
3277 }
3278 let retained = LinearProjection::with_scale(
3281 Tensor::zeros([128, 256], (Kind::Float, Device::Cpu)).to_kind(Kind::Float8e4m3fn),
3282 Tensor::ones([1, 2], (Kind::Float, Device::Cpu)),
3283 );
3284 assert!(retained.scale_v2.is_some());
3285 assert_eq!(rowwise_requant_live(), 1);
3286 assert_eq!(rowwise_requant_peak(), 1);
3287 }
3288
3289 #[test]
3290 fn gdn_chunked_continuation_preserves_recurrent_history() {
3291 let mut weights = tiny_weights();
3292 let layer = GatedDeltaNetLayer::load(
3293 &mut weights,
3294 "model.layers.0.linear_attn",
3295 &tiny_config(),
3296 0,
3297 )
3298 .unwrap();
3299 let input = (Tensor::arange(2 * HIDDEN, (Kind::Float, Device::Cpu)) * 0.13)
3300 .sin()
3301 .reshape([1, 2, HIDDEN]);
3302 let initial = Tensor::full(
3303 [1, LIN_V_HEADS as i64, LIN_K_DIM as i64, LIN_V_DIM as i64],
3304 3.0,
3305 (Kind::Float, Device::Cpu),
3306 );
3307 let conv = Tensor::zeros(
3308 [1, layer.conv_dim as i64, layer.kernel_size as i64 - 1],
3309 (Kind::Float, Device::Cpu),
3310 );
3311 let mut serial_conv = Some(conv.copy());
3312 let mut serial_outputs = Vec::new();
3313 let mut serial_rec = Some(initial.copy());
3314 for position in 0..2 {
3315 serial_outputs.push(
3316 layer
3317 .forward(
3318 &input.narrow(1, position, 1),
3319 &mut serial_conv,
3320 &mut serial_rec,
3321 None,
3322 )
3323 .unwrap(),
3324 );
3325 }
3326 let mut chunk_conv = Some(conv.copy());
3327 let mut chunk_rec = Some(initial);
3328 let chunk_output = layer
3329 .forward(&input, &mut chunk_conv, &mut chunk_rec, None)
3330 .unwrap();
3331 let mut zero_conv = Some(conv);
3332 let mut zero_rec = None;
3333 let _ = layer
3334 .forward(&input, &mut zero_conv, &mut zero_rec, None)
3335 .unwrap();
3336 let expected = serial_rec.unwrap();
3337 assert!(
3338 !expected.allclose(&zero_rec.unwrap(), 1e-4, 1e-4, false),
3339 "fixture must distinguish preserved history from an empty recurrent state"
3340 );
3341 assert!(
3342 chunk_output.allclose(&Tensor::cat(&serial_outputs, 1), 1e-4, 1e-4, false),
3343 "chunked outputs must match serial continuation"
3344 );
3345 let actual = chunk_rec.unwrap();
3346 assert!(
3347 actual.allclose(&expected, 1e-4, 1e-4, false),
3348 "chunked continuation lost recurrent history; max difference {}",
3349 (&actual - &expected).abs().max().double_value(&[])
3350 );
3351 }
3352
3353 #[test]
3354 fn gdn_short_prefill_keeps_full_convolution_history() {
3355 let mut weights = tiny_weights();
3356 let layer = GatedDeltaNetLayer::load(
3357 &mut weights,
3358 "model.layers.0.linear_attn",
3359 &tiny_config(),
3360 0,
3361 )
3362 .unwrap();
3363 let input = Tensor::ones([1, 1, HIDDEN], (Kind::Float, Device::Cpu));
3364 let mut conv = None;
3365 let mut recurrent = None;
3366 for _ in 0..2 {
3367 let _ = layer
3368 .forward(&input, &mut conv, &mut recurrent, None)
3369 .unwrap();
3370 assert_eq!(
3371 conv.as_ref().unwrap().size()[2],
3372 layer.kernel_size as i64 - 1
3373 );
3374 }
3375 }
3376
3377 pub(super) fn tiny_config() -> Qwen3_5TextConfig {
3378 let layer_types = (0..LAYERS)
3383 .map(|i| {
3384 if (i + 1) % 4 == 0 {
3385 "full_attention".to_owned()
3386 } else {
3387 "linear_attention".to_owned()
3388 }
3389 })
3390 .collect();
3391 let rotary_dim = ((HEAD_DIM as f32) * 0.25) as usize; Qwen3_5TextConfig {
3393 hidden_size: HIDDEN as u32,
3394 num_hidden_layers: LAYERS as u32,
3395 num_attention_heads: HEADS as u32,
3396 num_key_value_heads: KV_HEADS as u32,
3397 head_dim: HEAD_DIM as u32,
3398 intermediate_size: INTER as u32,
3399 vocab_size: VOCAB as u32,
3400 max_position_embeddings: 64,
3401 rms_norm_eps: 1e-5,
3402 rope_theta: 10000.0,
3403 partial_rotary_factor: 0.25,
3404 rotary_dim,
3405 layer_types,
3406 linear_conv_kernel_dim: CONV_KERNEL,
3407 linear_key_head_dim: LIN_K_DIM,
3408 linear_value_head_dim: LIN_V_DIM,
3409 linear_num_key_heads: LIN_K_HEADS,
3410 linear_num_value_heads: LIN_V_HEADS,
3411 is_moe: false,
3412 num_experts: 0,
3413 num_experts_per_tok: 0,
3414 moe_intermediate_size: 0,
3415 shared_expert_intermediate_size: 0,
3416 has_vision: false,
3417 vision_out_hidden_size: 0,
3418 }
3419 }
3420
3421 pub(super) fn tiny_weights() -> HashMap<String, Tensor> {
3427 let opt = (Kind::Float, Device::Cpu);
3428 let mut w = HashMap::new();
3429 let mut seed: i64 = 0;
3432 let mut pat = |dims: &[i64]| -> Tensor {
3433 let n: i64 = dims.iter().product();
3434 seed += 7;
3435 (Tensor::arange(n, opt) * 0.017 + seed as f64 * 0.013)
3436 .sin()
3437 .reshape(dims)
3438 * 0.05
3439 };
3440
3441 w.insert(
3442 "model.embed_tokens.weight".to_owned(),
3443 pat(&[VOCAB, HIDDEN]),
3444 );
3445 w.insert("model.norm.weight".to_owned(), pat(&[HIDDEN]));
3446 w.insert("lm_head.weight".to_owned(), pat(&[VOCAB, HIDDEN]));
3447
3448 let key_dim = (LIN_K_HEADS * LIN_K_DIM) as i64;
3449 let val_dim = (LIN_V_HEADS * LIN_V_DIM) as i64;
3450 let conv_dim = key_dim * 2 + val_dim;
3451 let nv = LIN_V_HEADS as i64;
3452
3453 for i in 0..LAYERS {
3454 let p = format!("model.layers.{i}");
3455 w.insert(format!("{p}.input_layernorm.weight"), pat(&[HIDDEN]));
3456 w.insert(
3457 format!("{p}.post_attention_layernorm.weight"),
3458 pat(&[HIDDEN]),
3459 );
3460
3461 let is_full = (i + 1) % 4 == 0;
3462 if is_full {
3463 let ap = format!("{p}.self_attn");
3465 w.insert(
3466 format!("{ap}.q_proj.weight"),
3467 pat(&[HEADS * HEAD_DIM * 2, HIDDEN]),
3468 );
3469 w.insert(
3470 format!("{ap}.k_proj.weight"),
3471 pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
3472 );
3473 w.insert(
3474 format!("{ap}.v_proj.weight"),
3475 pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
3476 );
3477 w.insert(
3478 format!("{ap}.o_proj.weight"),
3479 pat(&[HIDDEN, HEADS * HEAD_DIM]),
3480 );
3481 w.insert(format!("{ap}.q_norm.weight"), pat(&[HEAD_DIM]));
3482 w.insert(format!("{ap}.k_norm.weight"), pat(&[HEAD_DIM]));
3483 } else {
3484 let lp = format!("{p}.linear_attn");
3486 w.insert(format!("{lp}.in_proj_qkv.weight"), pat(&[conv_dim, HIDDEN]));
3487 w.insert(format!("{lp}.in_proj_z.weight"), pat(&[val_dim, HIDDEN]));
3488 w.insert(format!("{lp}.in_proj_b.weight"), pat(&[nv, HIDDEN]));
3489 w.insert(format!("{lp}.in_proj_a.weight"), pat(&[nv, HIDDEN]));
3490 w.insert(
3492 format!("{lp}.conv1d.weight"),
3493 pat(&[conv_dim, 1, CONV_KERNEL as i64]),
3494 );
3495 w.insert(format!("{lp}.A_log"), pat(&[nv]));
3496 w.insert(format!("{lp}.dt_bias"), pat(&[nv]));
3497 w.insert(format!("{lp}.norm.weight"), pat(&[LIN_V_DIM as i64]));
3498 w.insert(format!("{lp}.out_proj.weight"), pat(&[HIDDEN, val_dim]));
3499 }
3500
3501 let mp = format!("{p}.mlp");
3503 w.insert(format!("{mp}.gate_proj.weight"), pat(&[INTER, HIDDEN]));
3504 w.insert(format!("{mp}.up_proj.weight"), pat(&[INTER, HIDDEN]));
3505 w.insert(format!("{mp}.down_proj.weight"), pat(&[HIDDEN, INTER]));
3506 }
3507 w
3508 }
3509
3510 #[test]
3511 fn channel_fp8_fusion_preserves_quantized_and_ignored_boundaries() {
3512 let input = Tensor::from_slice(&[448f32, 1.0625, -4.0]).reshape([1, 3]);
3513 let make = || {
3514 let mut w = HashMap::from([
3515 (
3516 "p.weight".to_owned(),
3517 Tensor::ones([2, 3], (Kind::Float, Device::Cpu)).to_kind(Kind::Float8e4m3fn),
3518 ),
3519 (
3520 "p.weight_scale".to_owned(),
3521 Tensor::from_slice(&[0.25f32, 0.5]).reshape([2, 1]),
3522 ),
3523 ]);
3524 LinearProjection::take(&mut w, "p.weight").unwrap()
3525 };
3526 let ignored = || LinearProjection::new(Tensor::ones([3, 1], (Kind::Float, Device::Cpu)));
3527 let expected = Tensor::cat(&[make().apply(&input), ignored().apply(&input)], -1);
3528 let fused = cat_projs(vec![make(), ignored()]);
3529 assert!(fused.children.is_some());
3530 assert_eq!(fused.weight.numel(), 9); assert!(fused.apply(&input).allclose(&expected, 0.0, 0.0, false));
3532 assert_eq!(fused.apply(&input).double_value(&[0, 2]), 445.0625);
3533 }
3534
3535 #[test]
3536 fn decision_hidden_and_output_rows_are_fresh_and_match_full_forward() {
3537 let model = whole_model();
3538 let input = Tensor::from_slice(&[1i64, 5, 9, 2]).reshape([1, 4]);
3539 let hidden = model.decision_hidden_states(&input).unwrap();
3540 assert_eq!(hidden.size(), [4, HIDDEN]);
3541 let expected_logits = whole_model().forward(&input, None).unwrap().squeeze_dim(0);
3542 let rows = model.decision_output_embedding_rows(&[0, 5, 0]).unwrap();
3543 let logits = hidden.matmul(&rows.transpose(0, 1));
3544 let expected = expected_logits.index_select(1, &Tensor::from_slice(&[0i64, 5, 0]));
3545 assert!(logits.allclose(&expected, 1e-4, 1e-4, false));
3546 let _ = model
3548 .forward_with_cache(&Tensor::from_slice(&[7i64, 4]).reshape([1, 2]), 0)
3549 .unwrap();
3550 let hidden_again = model.decision_hidden_states(&input).unwrap();
3551 assert!(hidden_again.allclose(&hidden, 0.0, 0.0, false));
3552 assert!(model.decision_output_embedding_rows(&[-1]).is_err());
3553 assert!(model.decision_output_embedding_rows(&[VOCAB]).is_err());
3554 assert!(model
3555 .decision_hidden_states(&Tensor::from_slice(&[VOCAB]).reshape([1, 1]))
3556 .is_err());
3557 assert!(model
3558 .decision_hidden_states(&Tensor::zeros([1, 0], (Kind::Int64, Device::Cpu)))
3559 .is_err());
3560 }
3561
3562 #[test]
3563 fn decision_cancel_between_layers_preserves_next_request() {
3564 let model = whole_model();
3565 let input = Tensor::from_slice(&[1i64, 5, 9, 2]).reshape([1, 4]);
3566 let baseline = model.decision_hidden_states(&input).unwrap();
3567 let checks = std::cell::Cell::new(0usize);
3568 let cancel = || {
3569 let calls = checks.get() + 1;
3570 checks.set(calls);
3571 if calls == 3 {
3574 Err(anyhow!("request cancelled at decoder boundary"))
3575 } else {
3576 Ok(())
3577 }
3578 };
3579 let failure = model
3580 .decision_hidden_states_checked(&input, &cancel)
3581 .unwrap_err();
3582 assert!(failure.to_string().contains("request cancelled"));
3583 assert_eq!(checks.get(), 3);
3584 let next = model.decision_hidden_states(&input).unwrap();
3585 assert!(
3586 next.allclose(&baseline, 0.0, 0.0, false),
3587 "cancelled request leaked recurrent/KV state"
3588 );
3589 }
3590
3591 fn whole_model() -> Qwen3_5Model {
3592 let mut w = tiny_weights();
3593 Qwen3_5Model::from_weights(
3594 &mut w,
3595 tiny_config(),
3596 None,
3597 &Device::Cpu,
3598 Kind::Float,
3599 KVQuantType::None,
3600 false,
3601 )
3602 .unwrap()
3603 }
3604
3605 fn orchestrated_logits(m: &Qwen3_5Model, input: &Tensor) -> Tensor {
3608 let emb = m.embed_tokens(input).unwrap();
3609 let h = m.forward_layers(&emb, 0..m.num_layers(), 0, None).unwrap();
3610 let h = m.apply_final_norm(&h).unwrap();
3611 m.lm_head(&h).unwrap()
3612 }
3613
3614 #[test]
3615 fn forward_layers_full_range_matches_whole_model_forward() {
3616 let input = Tensor::from_slice(&[1i64, 5, 9, 2]).reshape([1, 4]);
3619
3620 let reference = whole_model();
3623 let ref_logits = reference.forward(&input, None).unwrap();
3624
3625 let piped = whole_model();
3628 let pipe_logits = orchestrated_logits(&piped, &input);
3629
3630 let max_diff = (&ref_logits - &pipe_logits).abs().max().double_value(&[]);
3631 assert!(
3632 ref_logits.allclose(&pipe_logits, 1e-4, 1e-4, false),
3633 "forward_layers(0..N) over a single-device map must equal whole-model forward \
3634 (max_diff={max_diff}, shape={:?})",
3635 ref_logits.size()
3636 );
3637 assert!(piped.device_map.is_single_device());
3638 assert_eq!(piped.layer_offset, 0);
3639 }
3640
3641 #[test]
3642 fn staged_split_equals_whole_model() {
3643 let input = Tensor::from_slice(&[3i64, 7, 1, 4]).reshape([1, 4]);
3651
3652 let reference = whole_model();
3653 let ref_logits = reference.forward(&input, None).unwrap();
3654
3655 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
3656 let split = LAYERS / 2;
3657
3658 let mut w0 = tiny_weights();
3659 let stage0 = Qwen3_5Model::stage_from_weights_with_config(
3660 &mut w0,
3661 tiny_config(),
3662 &map,
3663 0..split,
3664 Kind::Float,
3665 KVQuantType::None,
3666 )
3667 .unwrap();
3668 assert_eq!(stage0.layer_offset, 0);
3669 assert_eq!(stage0.layers.len(), split, "stage0 owns exactly its layers");
3670 assert!(
3671 stage0.lm_head.is_none(),
3672 "first (non-last) stage has no head"
3673 );
3674 assert_eq!(stage0.conv_states.lock().len(), split);
3676 assert_eq!(stage0.rec_states.lock().len(), split);
3677
3678 let mut w1 = tiny_weights();
3679 let stage1 = Qwen3_5Model::stage_from_weights_with_config(
3680 &mut w1,
3681 tiny_config(),
3682 &map,
3683 split..LAYERS,
3684 Kind::Float,
3685 KVQuantType::None,
3686 )
3687 .unwrap();
3688 assert_eq!(stage1.layer_offset, split);
3689 assert_eq!(stage1.layers.len(), LAYERS - split);
3690 assert_eq!(stage1.conv_states.lock().len(), LAYERS - split);
3691
3692 let emb = stage0.embed_tokens(&input).unwrap();
3695 let h0 = stage0.forward_layers(&emb, 0..split, 0, None).unwrap();
3696 let h1 = stage1.forward_layers(&h0, split..LAYERS, 0, None).unwrap();
3697 let h1 = stage1.apply_final_norm(&h1).unwrap();
3698 let logits = stage1.lm_head(&h1).unwrap();
3699
3700 let max_diff = (&ref_logits - &logits).abs().max().double_value(&[]);
3701 assert!(
3702 ref_logits.allclose(&logits, 1e-4, 1e-4, false),
3703 "two-stage split (same device) must equal whole-model forward \
3704 (max_diff={max_diff}); a mismatch here means the stage-local SSM state \
3705 threading is wrong"
3706 );
3707 }
3708
3709 #[test]
3710 fn forward_layers_rejects_out_of_window_range() {
3711 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
3713 let mut w = tiny_weights();
3714 let stage = Qwen3_5Model::stage_from_weights_with_config(
3715 &mut w,
3716 tiny_config(),
3717 &map,
3718 2..LAYERS,
3719 Kind::Float,
3720 KVQuantType::None,
3721 )
3722 .unwrap();
3723 let emb = Tensor::randn([1, 3, HIDDEN], (Kind::Float, Device::Cpu));
3724 assert!(
3725 stage.forward_layers(&emb, 0..2, 0, None).is_err(),
3726 "range below window"
3727 );
3728 assert!(
3729 stage.forward_layers(&emb, 2..LAYERS, 0, None).is_ok(),
3730 "owned range ok"
3731 );
3732 }
3733
3734 #[test]
3735 fn stage_loader_rejects_missing_required_weights() {
3736 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
3738 let mut w = tiny_weights();
3739 w.remove("model.norm.weight");
3740 let result = Qwen3_5Model::stage_from_weights_with_config(
3741 &mut w,
3742 tiny_config(),
3743 &map,
3744 0..LAYERS,
3745 Kind::Float,
3746 KVQuantType::None,
3747 );
3748 match result {
3749 Ok(_) => panic!("last stage without norm weight must error"),
3750 Err(e) => assert!(e.to_string().contains("norm"), "got: {e}"),
3751 }
3752 }
3753
3754 use crate::training::tenant_delta::{TenantDelta, TenantDeltaConfig};
3760
3761 fn tiny_delta() -> TenantDelta {
3767 let hidden = HIDDEN as usize;
3768 let gdn_in = LIN_V_HEADS * LIN_V_DIM; let full_in = (HEADS * HEAD_DIM) as usize; let mut per_layer: std::collections::HashMap<
3771 usize,
3772 std::collections::HashMap<String, (usize, usize)>,
3773 > = std::collections::HashMap::new();
3774 for i in 0..LAYERS {
3775 let is_full = (i + 1) % 4 == 0;
3776 let mut m = std::collections::HashMap::new();
3777 let in_dim = if is_full { full_in } else { gdn_in };
3778 m.insert("o_proj".to_owned(), (in_dim, hidden));
3779 per_layer.insert(i, m);
3780 }
3781 let cfg = TenantDeltaConfig {
3782 rank: 2,
3783 alpha: 2.0,
3784 target_modules: vec!["o_proj".to_owned()],
3785 ..Default::default()
3786 };
3787 let flat: std::collections::HashMap<String, (usize, usize)> =
3788 std::collections::HashMap::new();
3789 let delta = TenantDelta::new_with_per_layer_dims(
3790 &cfg,
3791 &flat,
3792 Device::Cpu,
3793 LAYERS,
3794 Some(&per_layer),
3795 )
3796 .unwrap();
3797
3798 let key_offset = |key: &str| -> i64 { key.bytes().map(|b| b as i64).sum::<i64>() % 17 };
3801 let _g = tch::no_grad_guard();
3802 for (k, a) in delta.lora_a.iter() {
3803 let n: i64 = a.size().iter().product();
3804 let vals = (Tensor::arange(n, (Kind::Float, Device::Cpu)) + key_offset(k)).sin() * 0.1;
3805 a.shallow_clone().copy_(&vals.reshape(a.size()));
3807 }
3808 for (k, b) in delta.lora_b.iter() {
3809 let n: i64 = b.size().iter().product();
3810 let vals =
3811 (Tensor::arange(n, (Kind::Float, Device::Cpu)) + key_offset(k) + 7).cos() * 0.1;
3812 b.shallow_clone().copy_(&vals.reshape(b.size()));
3813 }
3814 delta
3815 }
3816
3817 fn grad_snapshot(delta: &TenantDelta) -> std::collections::HashMap<String, f64> {
3818 let mut out = std::collections::HashMap::new();
3819 for (k, a) in &delta.lora_a {
3820 assert!(a.grad().defined(), "A grad undefined for {k}");
3821 out.insert(format!("A:{k}"), a.grad().norm().double_value(&[]));
3822 }
3823 for (k, b) in &delta.lora_b {
3824 assert!(b.grad().defined(), "B grad undefined for {k}");
3825 out.insert(format!("B:{k}"), b.grad().norm().double_value(&[]));
3826 }
3827 out
3828 }
3829
3830 #[test]
3837 fn ttt_split_autograd_matches_whole_model_grads() {
3838 use crate::training::pipeline::{compute_ntp_loss_split, TrainStage};
3839
3840 let input = Tensor::from_slice(&[3i64, 7, 1, 4, 9, 2]).reshape([1, 6]);
3841
3842 let whole = whole_model();
3844 let whole_delta = tiny_delta();
3845 let whole_stage = [TrainStage {
3846 model: &whole,
3847 range: 0..LAYERS,
3848 }];
3849 let loss_whole = compute_ntp_loss_split(&whole_stage, &input, Some(&whole_delta)).unwrap();
3850 loss_whole.backward();
3851 let whole_grads = grad_snapshot(&whole_delta);
3852 whole_delta.zero_grad();
3853
3854 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
3856 let split = LAYERS / 2;
3857 let mut w0 = tiny_weights();
3858 let stage0 = Qwen3_5Model::stage_from_weights_with_config(
3859 &mut w0,
3860 tiny_config(),
3861 &map,
3862 0..split,
3863 Kind::Float,
3864 KVQuantType::None,
3865 )
3866 .unwrap();
3867 let mut w1 = tiny_weights();
3868 let stage1 = Qwen3_5Model::stage_from_weights_with_config(
3869 &mut w1,
3870 tiny_config(),
3871 &map,
3872 split..LAYERS,
3873 Kind::Float,
3874 KVQuantType::None,
3875 )
3876 .unwrap();
3877 let split_delta = tiny_delta();
3878 let stages = [
3879 TrainStage {
3880 model: &stage0,
3881 range: 0..split,
3882 },
3883 TrainStage {
3884 model: &stage1,
3885 range: split..LAYERS,
3886 },
3887 ];
3888 let loss_split = compute_ntp_loss_split(&stages, &input, Some(&split_delta)).unwrap();
3889 loss_split.backward();
3890 let split_grads = grad_snapshot(&split_delta);
3891
3892 let lw = loss_whole.double_value(&[]);
3893 let ls = loss_split.double_value(&[]);
3894 assert!(
3895 (lw - ls).abs() < 1e-4,
3896 "loss diverged: whole={lw} split={ls}"
3897 );
3898
3899 assert_eq!(whole_grads.len(), split_grads.len());
3900 for (k, &gw) in &whole_grads {
3901 let gs = *split_grads
3902 .get(k)
3903 .unwrap_or_else(|| panic!("missing grad {k}"));
3904 assert!(
3905 (gw - gs).abs() <= 1e-4 + 1e-4 * gw.abs(),
3906 "grad norm diverged for {k}: whole={gw} split={gs}"
3907 );
3908 assert!(
3909 gw > 0.0,
3910 "grad for {k} is zero — autograd path not exercised"
3911 );
3912 }
3913 }
3914
3915 #[test]
3924 fn forward_with_bf16_base_and_fp32_delta_does_not_panic() {
3925 let _g = tch::no_grad_guard();
3926 let dtype = Kind::BFloat16;
3927
3928 let mut w = tiny_weights();
3930 for t in w.values_mut() {
3931 *t = t.to_kind(dtype);
3932 }
3933 let model = Qwen3_5Model::from_weights(
3934 &mut w,
3935 tiny_config(),
3936 None,
3937 &Device::Cpu,
3938 dtype,
3939 KVQuantType::None,
3940 false,
3941 )
3942 .unwrap();
3943
3944 let delta = tiny_delta();
3947
3948 let input = Tensor::from_slice(&[3i64, 7, 1, 4, 9, 2]).reshape([1, 6]);
3949 let emb = model.embed_tokens(&input).unwrap();
3950 assert_eq!(emb.kind(), dtype, "bf16 embeddings");
3951
3952 let out = model
3954 .forward_layers(&emb, 0..LAYERS, 0, Some(&delta))
3955 .expect("bf16 base + fp32 delta forward must not error");
3956 assert_eq!(out.kind(), dtype, "delta-injected forward must stay bf16");
3958 assert!(
3959 out.isfinite().all().int64_value(&[]) == 1,
3960 "output must be finite"
3961 );
3962 }
3963
3964 #[test]
3965 fn forward_layers_train_rejects_out_of_window_range() {
3966 let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
3967 let mut w = tiny_weights();
3968 let stage = Qwen3_5Model::stage_from_weights_with_config(
3969 &mut w,
3970 tiny_config(),
3971 &map,
3972 2..LAYERS,
3973 Kind::Float,
3974 KVQuantType::None,
3975 )
3976 .unwrap();
3977 let emb = Tensor::randn([1, 3, HIDDEN], (Kind::Float, Device::Cpu));
3978 assert!(
3979 stage.forward_layers_train(&emb, 0..2, None).is_err(),
3980 "range below window"
3981 );
3982 assert!(
3983 stage.forward_layers_train(&emb, 2..LAYERS, None).is_ok(),
3984 "owned range ok"
3985 );
3986 }
3987
3988 use parking_lot::Mutex;
3995
3996 fn session_cache() -> Arc<Mutex<KVCacheManager>> {
3997 Arc::new(Mutex::new(KVCacheManager::new(
3998 LAYERS,
3999 64,
4000 KVQuantType::None,
4001 )))
4002 }
4003
4004 fn greedy_next(logits: &Tensor) -> i64 {
4006 logits
4007 .select(0, 0)
4008 .select(0, 0)
4009 .argmax(0, false)
4010 .int64_value(&[])
4011 }
4012
4013 fn run_turn(
4019 model: &Qwen3_5Model,
4020 prompt: &[i64],
4021 prefill_start: usize,
4022 n_decode: usize,
4023 ) -> (
4024 Vec<i64>,
4025 Vec<Tensor>,
4026 (Vec<Option<Tensor>>, Vec<Option<Tensor>>),
4027 ) {
4028 let _g = tch::no_grad_guard();
4029 let prefill = Tensor::from_slice(&prompt[prefill_start..])
4030 .reshape([1, (prompt.len() - prefill_start) as i64]);
4031 let logits = model.forward_with_cache(&prefill, prefill_start).unwrap();
4032 let ssm = model.snapshot_ssm_states();
4035
4036 let mut out = Vec::with_capacity(n_decode);
4037 let mut step_logits = Vec::with_capacity(n_decode);
4038 let mut next = greedy_next(&logits);
4039 out.push(next);
4040 step_logits.push(logits.select(0, 0).select(0, 0));
4041 for pos in prompt.len()..prompt.len() + n_decode - 1 {
4042 let step = Tensor::from_slice(&[next]).reshape([1, 1]);
4043 let logits = model.forward_with_cache(&step, pos).unwrap();
4044 next = greedy_next(&logits);
4045 out.push(next);
4046 step_logits.push(logits.select(0, 0).select(0, 0));
4047 }
4048 (out, step_logits, ssm)
4049 }
4050
4051 fn assert_logits_match(actual: &[Tensor], expected: &[Tensor], context: &str) {
4053 assert_eq!(actual.len(), expected.len(), "{context}: step count");
4054 for (i, (a, e)) in actual.iter().zip(expected).enumerate() {
4055 let max_diff = (a - e).abs().max().double_value(&[]);
4056 assert!(
4057 a.allclose(e, 1e-5, 1e-5, false),
4058 "{context}: decode step {i} logits diverged (max_diff={max_diff})"
4059 );
4060 }
4061 }
4062
4063 #[test]
4072 fn session_prefix_hit_without_ssm_rewind_diverges_from_fresh_cache() {
4073 let prompt1 = [1i64, 5, 9, 2, 7, 3];
4074 let suffix = [8i64, 4];
4075 let prompt2: Vec<i64> = prompt1.iter().chain(&suffix).copied().collect();
4076
4077 let model = whole_model();
4079 let cache = session_cache();
4080 let mut m = model;
4081 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache.clone());
4082 let model = m;
4083
4084 let (_gen1, _logits1, _ssm1) = run_turn(&model, &prompt1, 0, 3);
4085 cache.lock().set_cached_tokens(prompt1.to_vec());
4086
4087 let prefix_len = cache.lock().prefix_match_len(&prompt2);
4088 assert_eq!(prefix_len, prompt1.len());
4089 cache.lock().truncate_to(prefix_len);
4090 let (_gen2_stale, logits2_stale, _ssm2) = run_turn(&model, &prompt2, prefix_len, 4);
4091
4092 let ref_model = whole_model();
4094 let ref_cache = session_cache();
4095 let mut m = ref_model;
4096 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4097 let ref_model = m;
4098 let (_gen2_ref, logits2_ref, _ssm_ref) = run_turn(&ref_model, &prompt2, 0, 4);
4099
4100 let max_diff: f64 = logits2_stale
4101 .iter()
4102 .zip(&logits2_ref)
4103 .map(|(a, e)| (a - e).abs().max().double_value(&[]))
4104 .fold(0.0, f64::max);
4105 assert!(
4106 max_diff > 1e-5,
4107 "stale-state fixture must diverge from fresh cache (max_diff={max_diff}); \
4108 otherwise the rewind regression test proves nothing"
4109 );
4110 }
4111
4112 #[test]
4121 fn session_prefix_hit_restores_ssm_state_matches_fresh_cache() {
4122 let prompt1 = [1i64, 5, 9, 2, 7, 3];
4123 let suffix = [8i64, 4];
4124 let prompt2: Vec<i64> = prompt1.iter().chain(&suffix).copied().collect();
4125
4126 let model = whole_model();
4128 let cache = session_cache();
4129 let mut m = model;
4130 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache.clone());
4131 let model = m;
4132
4133 let (_gen1, _logits1, ssm1) = run_turn(&model, &prompt1, 0, 3);
4134 cache
4135 .lock()
4136 .set_cached_tokens_with_ssm(prompt1.to_vec(), Some(ssm1));
4137
4138 let prefix_len = cache.lock().prefix_match_len(&prompt2);
4141 assert_eq!(prefix_len, prompt1.len());
4142 let start = crate::runtime::torch_engine::rewind_session_state(&model, prefix_len);
4143 assert_eq!(start, prompt1.len(), "full cached prefix must be reusable");
4144 let (gen2, logits2, _ssm2) = run_turn(&model, &prompt2, start, 4);
4145
4146 let oracle_model = whole_model();
4150 let oracle_cache = session_cache();
4151 let mut m = oracle_model;
4152 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, oracle_cache);
4153 let oracle_model = m;
4154 {
4155 let _g = tch::no_grad_guard();
4156 let prefill1 = Tensor::from_slice(&prompt1).reshape([1, prompt1.len() as i64]);
4157 let _ = oracle_model.forward_with_cache(&prefill1, 0).unwrap();
4158 }
4159 let (gen2_oracle, logits2_oracle, _ssm_o) =
4160 run_turn(&oracle_model, &prompt2, prompt1.len(), 4);
4161
4162 assert_eq!(
4163 gen2, gen2_oracle,
4164 "rewound run must match split-prefill oracle"
4165 );
4166 assert_logits_match(
4167 &logits2,
4168 &logits2_oracle,
4169 "rewound run vs split-prefill oracle",
4170 );
4171
4172 let ref_model = whole_model();
4177 let ref_cache = session_cache();
4178 let mut m = ref_model;
4179 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4180 let ref_model = m;
4181 let (gen2_ref, _logits2_ref, _ssm_ref) = run_turn(&ref_model, &prompt2, 0, 4);
4182 assert_eq!(
4183 gen2, gen2_ref,
4184 "prefix-hit decode must match a fresh-cache run token-for-token"
4185 );
4186 }
4187
4188 #[test]
4193 fn session_partial_prefix_match_forces_full_recompute() {
4194 let prompt1 = [1i64, 5, 9, 2, 7, 3];
4195 let prompt2 = [1i64, 5, 9, 4, 8, 6, 2];
4197
4198 let model = whole_model();
4199 let cache = session_cache();
4200 let mut m = model;
4201 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache.clone());
4202 let model = m;
4203
4204 let (_gen1, _logits1, ssm1) = run_turn(&model, &prompt1, 0, 3);
4205 cache
4206 .lock()
4207 .set_cached_tokens_with_ssm(prompt1.to_vec(), Some(ssm1));
4208
4209 let prefix_len = cache.lock().prefix_match_len(&prompt2);
4210 assert!(0 < prefix_len && prefix_len < prompt1.len());
4211 let start = crate::runtime::torch_engine::rewind_session_state(&model, prefix_len);
4212 assert_eq!(start, 0, "partial prefix match must force full recompute");
4213 assert!(model.conv_states.lock().iter().all(Option::is_none));
4216 assert!(model.rec_states.lock().iter().all(Option::is_none));
4217
4218 let (gen2, logits2, _ssm2) = run_turn(&model, &prompt2, 0, 4);
4219
4220 let ref_model = whole_model();
4221 let ref_cache = session_cache();
4222 let mut m = ref_model;
4223 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4224 let ref_model = m;
4225 let (gen2_ref, logits2_ref, _ssm_ref) = run_turn(&ref_model, &prompt2, 0, 4);
4226
4227 assert_eq!(gen2, gen2_ref);
4228 assert_logits_match(&logits2, &logits2_ref, "recomputed prefill vs fresh cache");
4229 }
4230
4231 #[test]
4235 fn clear_kv_cache_fully_resets_ssm_state() {
4236 let prompt = [1i64, 5, 9, 2, 7, 3];
4237
4238 let model = whole_model();
4239 let cache = session_cache();
4240 let mut m = model;
4241 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache);
4242 let model = m;
4243
4244 let _ = run_turn(&model, &prompt, 0, 2);
4245 assert!(model.conv_states.lock().iter().any(Option::is_some));
4246 assert!(model.rec_states.lock().iter().any(Option::is_some));
4247
4248 <Qwen3_5Model as ModelOperations>::clear_kv_cache(&model);
4249 assert!(
4250 model.conv_states.lock().iter().all(Option::is_none),
4251 "conv state must be reset on cache clear"
4252 );
4253 assert!(
4254 model.rec_states.lock().iter().all(Option::is_none),
4255 "rec state must be reset on cache clear"
4256 );
4257
4258 let (gen, logits, _ssm) = run_turn(&model, &prompt, 0, 3);
4260 let ref_model = whole_model();
4261 let ref_cache = session_cache();
4262 let mut m = ref_model;
4263 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4264 let ref_model = m;
4265 let (gen_ref, logits_ref, _ssm_ref) = run_turn(&ref_model, &prompt, 0, 3);
4266 assert_eq!(gen, gen_ref);
4267 assert_logits_match(&logits, &logits_ref, "post-clear run vs fresh model");
4268 }
4269
4270 #[test]
4283 fn session_exact_prompt_hit_restored_reforward_diverges_from_fresh_cache() {
4284 let prompt: Vec<i64> = [1i64, 5, 9, 2, 7, 3, 8, 4]
4287 .iter()
4288 .cycle()
4289 .take(24)
4290 .copied()
4291 .collect();
4292
4293 let model = whole_model();
4295 let cache = session_cache();
4296 let mut m = model;
4297 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache.clone());
4298 let model = m;
4299
4300 let (_gen1, _logits1, ssm1) = run_turn(&model, &prompt, 0, 3);
4301 cache
4302 .lock()
4303 .set_cached_tokens_with_ssm(prompt.clone(), Some(ssm1));
4304
4305 let row_logits = |m: &Qwen3_5Model| -> Tensor {
4309 let _g = tch::no_grad_guard();
4310 let ids = Tensor::from_slice(&prompt).reshape([1, prompt.len() as i64]);
4311 let emb = m.embed_tokens(&ids).unwrap();
4312 let h = m.forward_layers(&emb, 0..m.num_layers(), 0, None).unwrap();
4313 let h = m.apply_final_norm(&h).unwrap();
4314 m.lm_head(&h).unwrap() };
4316 let prefix_len = cache.lock().prefix_match_len(&prompt);
4317 assert_eq!(prefix_len, prompt.len(), "exact repeat must fully match");
4318 let start = crate::runtime::torch_engine::rewind_session_state(&model, prefix_len);
4319 assert_eq!(
4320 start,
4321 prompt.len(),
4322 "rewind alone still reports the full hit"
4323 );
4324 let broken_rows = row_logits(&model);
4325
4326 let ref_model = whole_model();
4328 let ref_cache = session_cache();
4329 let mut m = ref_model;
4330 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4331 let ref_model = m;
4332 let fresh_rows = row_logits(&ref_model);
4333
4334 let first_row_diff = (&broken_rows.select(1, 0).reshape([-1i64])
4335 - &fresh_rows.select(1, 0).reshape([-1i64]))
4336 .abs()
4337 .max()
4338 .double_value(&[]);
4339 assert!(
4340 first_row_diff > 1e-4,
4341 "restore-then-reforward on an exact repeat must condition the first row on the \
4342 previous turn's recurrent state and diverge from a fresh prefill \
4343 (first-row max_diff={first_row_diff}); otherwise the cleared-recompute \
4344 regression test proves nothing"
4345 );
4346 }
4347
4348 #[test]
4357 fn session_exact_prompt_repeat_forces_cleared_recompute() {
4358 let prompt = [1i64, 5, 9, 2, 7, 3];
4359
4360 let model = whole_model();
4361 let cache = session_cache();
4362 let mut m = model;
4363 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache.clone());
4364 let model = m;
4365
4366 let (_gen1, _logits1, ssm1) = run_turn(&model, &prompt, 0, 3);
4367 cache
4368 .lock()
4369 .set_cached_tokens_with_ssm(prompt.to_vec(), Some(ssm1));
4370
4371 let prefix_len = cache.lock().prefix_match_len(&prompt);
4372 assert_eq!(prefix_len, prompt.len(), "exact repeat must fully match");
4373 let start = crate::runtime::torch_engine::resolve_session_prefill_start(
4374 &model,
4375 prefix_len,
4376 prompt.len(),
4377 );
4378 assert_eq!(start, 0, "exact-prompt repeat must force full recompute");
4379 assert!(model.conv_states.lock().iter().all(Option::is_none));
4382 assert!(model.rec_states.lock().iter().all(Option::is_none));
4383
4384 let (gen2, logits2, _ssm2) = run_turn(&model, &prompt, start, 4);
4385
4386 let ref_model = whole_model();
4387 let ref_cache = session_cache();
4388 let mut m = ref_model;
4389 <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4390 let ref_model = m;
4391 let (gen2_ref, logits2_ref, _ssm_ref) = run_turn(&ref_model, &prompt, 0, 4);
4392
4393 assert_eq!(gen2, gen2_ref);
4394 assert_logits_match(
4395 &logits2,
4396 &logits2_ref,
4397 "recomputed exact repeat vs fresh cache",
4398 );
4399 }
4400 #[test]
4412 fn cached_two_token_forward_matches_serial_decode_steps() {
4413 let allattn_config = || {
4414 let mut cfg = tiny_config();
4415 cfg.layer_types = (0..cfg.num_hidden_layers)
4416 .map(|_| "full_attention".to_owned())
4417 .collect();
4418 cfg
4419 };
4420 let build = || {
4421 let mut w = tiny_weights();
4422 let cfg = allattn_config();
4423 let opt = (Kind::Float, Device::Cpu);
4426 let mut seed: i64 = 20_000;
4427 let mut pat = |dims: &[i64]| -> Tensor {
4428 let n: i64 = dims.iter().product();
4429 seed += 7;
4430 (Tensor::arange(n, opt) * 0.017 + seed as f64 * 0.013)
4431 .sin()
4432 .reshape(dims)
4433 * 0.05
4434 };
4435 for i in 0..cfg.num_hidden_layers as usize {
4436 if (i + 1) % 4 == 0 {
4437 continue; }
4439 let p = format!("model.layers.{i}");
4440 w.retain(|k, _| !k.starts_with(&format!("{p}.linear_attn.")));
4441 let ap = format!("{p}.self_attn");
4442 w.insert(
4443 format!("{ap}.q_proj.weight"),
4444 pat(&[HEADS * HEAD_DIM * 2, HIDDEN]),
4445 );
4446 w.insert(
4447 format!("{ap}.k_proj.weight"),
4448 pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
4449 );
4450 w.insert(
4451 format!("{ap}.v_proj.weight"),
4452 pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
4453 );
4454 w.insert(
4455 format!("{ap}.o_proj.weight"),
4456 pat(&[HIDDEN, HEADS * HEAD_DIM]),
4457 );
4458 w.insert(format!("{ap}.q_norm.weight"), pat(&[HEAD_DIM]));
4459 w.insert(format!("{ap}.k_norm.weight"), pat(&[HEAD_DIM]));
4460 }
4461 Qwen3_5Model::from_weights(
4462 &mut w,
4463 allattn_config(),
4464 None,
4465 &Device::Cpu,
4466 Kind::Float,
4467 KVQuantType::None,
4468 false,
4469 )
4470 .unwrap()
4471 };
4472
4473 let prompt: [i64; 6] = [1, 5, 9, 2, 7, 3];
4474 let prompt_t = Tensor::from_slice(&prompt).reshape([1, prompt.len() as i64]);
4475 let pos = prompt.len();
4476
4477 let serial = build();
4479 let _ = serial.forward_with_cache(&prompt_t, 0).unwrap();
4480 let step1 = Tensor::from_slice(&[11i64]).reshape([1, 1]);
4481 let ref1 = serial.forward_with_cache(&step1, pos).unwrap(); let step2 = Tensor::from_slice(&[13i64]).reshape([1, 1]);
4483 let ref2 = serial.forward_with_cache(&step2, pos + 1).unwrap();
4484
4485 let cached = build();
4489 let _ = cached.forward_with_cache(&prompt_t, 0).unwrap();
4490 let pair = Tensor::from_slice(&[11i64, 13]).reshape([1, 2]);
4491 let emb = cached.embed_tokens(&pair).unwrap();
4492 let h = cached
4493 .forward_layers(&emb, 0..cached.num_layers(), pos, None)
4494 .unwrap();
4495 let h = cached.apply_final_norm(&h).unwrap();
4496 let logits2 = cached.lm_head(&h).unwrap(); for (row, ref_l) in [(0i64, &ref1), (1, &ref2)] {
4499 let got = logits2.select(1, row).reshape([-1i64]);
4500 let want = ref_l.reshape([-1i64]);
4501 let max_diff = (&got - &want).abs().max().double_value(&[]);
4502 assert!(
4503 got.allclose(&want, 1e-4, 1e-4, false),
4504 "cached 2-token forward row {row} diverged from the serial decode step \
4505 (max_diff={max_diff}); the causal mask must keep kv_len - q_len history, \
4506 not tril(0)"
4507 );
4508 assert_eq!(
4509 got.argmax(-1, false).int64_value(&[]),
4510 want.argmax(-1, false).int64_value(&[]),
4511 "row {row} argmax differs",
4512 );
4513 }
4514 }
4515}
4516
4517#[cfg(test)]
4518#[allow(clippy::unwrap_used, clippy::expect_used)]
4519pub(crate) mod mtp_tests {
4520 use super::pipeline_tests::{
4533 tiny_config, tiny_weights, HEADS, HEAD_DIM, HIDDEN, INTER, KV_HEADS, VOCAB,
4534 };
4535 use super::*;
4536
4537 fn tiny_weights_mtp() -> HashMap<String, Tensor> {
4539 let mut w = tiny_weights();
4540 let opt = (Kind::Float, Device::Cpu);
4541 let mut seed: i64 = 10_000;
4543 let mut pat = |dims: &[i64]| -> Tensor {
4544 let n: i64 = dims.iter().product();
4545 seed += 7;
4546 (Tensor::arange(n, opt) * 0.017 + seed as f64 * 0.013)
4547 .sin()
4548 .reshape(dims)
4549 * 0.05
4550 };
4551
4552 w.insert("mtp.pre_fc_norm_hidden.weight".to_owned(), pat(&[HIDDEN]));
4553 w.insert(
4554 "mtp.pre_fc_norm_embedding.weight".to_owned(),
4555 pat(&[HIDDEN]),
4556 );
4557 w.insert("mtp.fc.weight".to_owned(), pat(&[HIDDEN, 2 * HIDDEN]));
4558 let lp = "mtp.layers.0";
4559 w.insert(format!("{lp}.input_layernorm.weight"), pat(&[HIDDEN]));
4560 w.insert(
4561 format!("{lp}.post_attention_layernorm.weight"),
4562 pat(&[HIDDEN]),
4563 );
4564 let ap = format!("{lp}.self_attn");
4565 w.insert(
4566 format!("{ap}.q_proj.weight"),
4567 pat(&[HEADS * HEAD_DIM * 2, HIDDEN]),
4568 );
4569 w.insert(
4570 format!("{ap}.k_proj.weight"),
4571 pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
4572 );
4573 w.insert(
4574 format!("{ap}.v_proj.weight"),
4575 pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
4576 );
4577 w.insert(
4578 format!("{ap}.o_proj.weight"),
4579 pat(&[HIDDEN, HEADS * HEAD_DIM]),
4580 );
4581 w.insert(format!("{ap}.q_norm.weight"), pat(&[HEAD_DIM]));
4582 w.insert(format!("{ap}.k_norm.weight"), pat(&[HEAD_DIM]));
4583 let mp = format!("{lp}.mlp");
4584 w.insert(format!("{mp}.gate_proj.weight"), pat(&[INTER, HIDDEN]));
4585 w.insert(format!("{mp}.up_proj.weight"), pat(&[INTER, HIDDEN]));
4586 w.insert(format!("{mp}.down_proj.weight"), pat(&[HIDDEN, INTER]));
4587 w.insert("mtp.norm.weight".to_owned(), pat(&[HIDDEN]));
4588 w
4589 }
4590
4591 pub(crate) fn whole_model_mtp() -> Qwen3_5Model {
4592 let mut w = tiny_weights_mtp();
4593 Qwen3_5Model::from_weights(
4594 &mut w,
4595 tiny_config(),
4596 None,
4597 &Device::Cpu,
4598 Kind::Float,
4599 KVQuantType::None,
4600 true,
4601 )
4602 .unwrap()
4603 }
4604
4605 #[test]
4606 fn mtp_head_obeys_explicit_toml_runtime_setting() {
4607 for enabled in [false, true] {
4608 let configured = crate::config::RuntimeConfig {
4609 speculative_decoding: enabled,
4610 use_gpu: false,
4611 ..Default::default()
4612 };
4613 let toml = toml::to_string(&configured).unwrap();
4614 let runtime: crate::config::RuntimeConfig = toml::from_str(&toml).unwrap();
4615 let mut weights = tiny_weights_mtp();
4616 let model = Qwen3_5Model::from_weights(
4617 &mut weights,
4618 tiny_config(),
4619 None,
4620 &Device::Cpu,
4621 Kind::Float,
4622 KVQuantType::None,
4623 runtime.speculative_decoding,
4624 )
4625 .unwrap();
4626 assert_eq!(model.has_mtp(), enabled);
4627 assert!(!weights.keys().any(|name| name.starts_with("mtp.")));
4628 }
4629 }
4630
4631 #[test]
4632 fn mtp_quantized_direct_model_drops_head() {
4633 for kv in [
4634 KVQuantType::None,
4635 KVQuantType::Int8,
4636 KVQuantType::Nf4,
4637 KVQuantType::Fp4,
4638 ] {
4639 let mut weights = tiny_weights_mtp();
4640 let model = Qwen3_5Model::from_weights(
4641 &mut weights,
4642 tiny_config(),
4643 None,
4644 &Device::Cpu,
4645 Kind::Float,
4646 kv,
4647 true,
4648 )
4649 .unwrap();
4650 assert_eq!(model.has_mtp(), kv == KVQuantType::None, "KV mode {kv:?}");
4651 assert!(!weights.keys().any(|name| name.starts_with("mtp.")));
4652 }
4653 }
4654
4655 fn argmax1(logits: &Tensor) -> i64 {
4657 logits.reshape([-1i64]).argmax(-1, false).int64_value(&[])
4658 }
4659
4660 fn serial_generate(
4664 model: &Qwen3_5Model,
4665 prompt: &[i64],
4666 gen_tokens: usize,
4667 prefill_from: usize,
4668 ) -> Vec<i64> {
4669 let _g = tch::no_grad_guard();
4670 let (slice, start_pos) = if prefill_from > 0 {
4671 (&prompt[prefill_from..], prefill_from)
4672 } else {
4673 (prompt, 0)
4674 };
4675 let prompt_t = Tensor::from_slice(slice).reshape([1, slice.len() as i64]);
4676 let logits = model.forward_with_cache(&prompt_t, start_pos).unwrap();
4677 let mut out = Vec::with_capacity(gen_tokens);
4678 let mut last = argmax1(&logits);
4679 out.push(last);
4680 let mut pos = prompt.len();
4681 while out.len() < gen_tokens {
4682 let dec = Tensor::from_slice(&[last]).reshape([1, 1]);
4683 let logits = model.forward_with_cache(&dec, pos).unwrap();
4684 last = argmax1(&logits);
4685 out.push(last);
4686 pos += 1;
4687 }
4688 out
4689 }
4690
4691 #[derive(Clone, Copy, PartialEq, Eq)]
4692 enum DraftMode {
4693 Natural,
4695 ForceReject,
4698 Oracle,
4701 }
4702
4703 fn spec_generate(
4706 model: &Qwen3_5Model,
4707 prompt: &[i64],
4708 gen_tokens: usize,
4709 prefill_from: usize,
4710 mode: DraftMode,
4711 oracle: Option<&[i64]>,
4712 ) -> (Vec<i64>, u64, u64) {
4713 let _g = tch::no_grad_guard();
4714 let (slice, start_pos) = if prefill_from > 0 {
4715 (&prompt[prefill_from..], prefill_from)
4716 } else {
4717 (prompt, 0)
4718 };
4719
4720 let prompt_t = Tensor::from_slice(slice).reshape([1, slice.len() as i64]);
4722 let (logits, hidden) = model
4723 .forward_with_cache_hidden(&prompt_t, start_pos, true)
4724 .unwrap();
4725 let mut out: Vec<i64> = Vec::with_capacity(gen_tokens);
4726 let mut last = argmax1(&logits);
4727 out.push(last);
4728 let mut mtp_tokens: Vec<i64> = slice[1..].to_vec();
4729 mtp_tokens.push(last);
4730 let mtp_in = Tensor::from_slice(&mtp_tokens).reshape([1, mtp_tokens.len() as i64]);
4731 let mut pending = argmax1(&model.mtp_forward(&mtp_in, &hidden, start_pos).unwrap());
4732
4733 let mut pos = prompt.len();
4734 let (mut accepted, mut rejected) = (0u64, 0u64);
4735 while out.len() < gen_tokens {
4736 let draft = match mode {
4737 DraftMode::Natural => pending,
4738 DraftMode::ForceReject => (pending + 1) % VOCAB,
4739 DraftMode::Oracle => {
4740 oracle.expect("oracle mode needs the serial output")[out.len()]
4741 }
4742 };
4743
4744 let snapshot = model.snapshot_ssm_states();
4745 let verify_in = Tensor::from_slice(&[last, draft]).reshape([1, 2]);
4746 let (logits, hidden) = model
4747 .forward_with_cache_hidden(&verify_in, pos, false)
4748 .unwrap();
4749 let v1 = argmax1(&logits.select(1, 0));
4750
4751 if v1 == draft {
4752 let v2 = argmax1(&logits.select(1, 1));
4754 let mtp_in = Tensor::from_slice(&[v1, v2]).reshape([1, 2]);
4755 pending = argmax1(&model.mtp_forward(&mtp_in, &hidden, pos).unwrap());
4756 out.push(v1);
4757 out.push(v2);
4758 last = v2;
4759 pos += 2;
4760 accepted += 1;
4761 } else {
4762 model.restore_ssm_states(snapshot.0, snapshot.1);
4765 model.get_kv_cache().unwrap().lock().truncate_to(pos + 1);
4766 let re_in = Tensor::from_slice(&[last]).reshape([1, 1]);
4767 let (_, hidden2) = model.forward_with_cache_hidden(&re_in, pos, true).unwrap();
4768 let mtp_in = Tensor::from_slice(&[v1]).reshape([1, 1]);
4769 pending = argmax1(&model.mtp_forward(&mtp_in, &hidden2, pos).unwrap());
4770 out.push(v1);
4771 last = v1;
4772 pos += 1;
4773 rejected += 1;
4774 }
4775 }
4776 out.truncate(gen_tokens);
4777 (out, accepted, rejected)
4778 }
4779
4780 const PROMPT: [i64; 6] = [1, 5, 9, 2, 7, 3];
4781 const GEN: usize = 12;
4782
4783 #[test]
4785 fn mtp_decode_matches_serial() {
4786 let reference = serial_generate(&whole_model_mtp(), &PROMPT, GEN, 0);
4787 let (spec, accepted, rejected) = spec_generate(
4788 &whole_model_mtp(),
4789 &PROMPT,
4790 GEN,
4791 0,
4792 DraftMode::Natural,
4793 None,
4794 );
4795 assert_eq!(
4796 spec, reference,
4797 "speculative output diverged from serial greedy"
4798 );
4799 assert!(accepted + rejected > 0, "no speculative rounds ran");
4800 }
4801
4802 #[test]
4805 fn mtp_decode_forced_reject_matches_serial() {
4806 let reference = serial_generate(&whole_model_mtp(), &PROMPT, GEN, 0);
4807 let (spec, _accepted, rejected) = spec_generate(
4808 &whole_model_mtp(),
4809 &PROMPT,
4810 GEN,
4811 0,
4812 DraftMode::ForceReject,
4813 None,
4814 );
4815 assert_eq!(
4816 spec, reference,
4817 "forced-reject rewind diverged from serial greedy"
4818 );
4819 assert!(
4820 rejected > 0,
4821 "forced-reject mode never rejected — rewind path untested"
4822 );
4823 }
4824
4825 #[test]
4828 fn mtp_decode_oracle_accept_matches_serial() {
4829 let reference = serial_generate(&whole_model_mtp(), &PROMPT, GEN, 0);
4830 let (spec, accepted, _rejected) = spec_generate(
4831 &whole_model_mtp(),
4832 &PROMPT,
4833 GEN,
4834 0,
4835 DraftMode::Oracle,
4836 Some(&reference),
4837 );
4838 assert_eq!(
4839 spec, reference,
4840 "oracle-accept output diverged from serial greedy"
4841 );
4842 assert!(
4843 accepted > 0,
4844 "oracle drafts never accepted — accept path untested"
4845 );
4846 }
4847
4848 #[test]
4863 fn mtp_decode_falls_back_to_serial_at_context_boundary() {
4864 let _g = tch::no_grad_guard();
4865 const CAP: usize = 12; let build = || {
4867 let mut w = tiny_weights_mtp();
4868 let mut model = Qwen3_5Model::from_weights(
4869 &mut w,
4870 tiny_config(),
4871 None,
4872 &Device::Cpu,
4873 Kind::Float,
4874 KVQuantType::None,
4875 true,
4876 )
4877 .unwrap();
4878 let layers = model.num_layers();
4879 model.set_kv_cache(std::sync::Arc::new(parking_lot::Mutex::new(
4880 crate::runtime::kv_cache::KVCacheManager::new(layers + 1, CAP, KVQuantType::None),
4881 )));
4882 model
4883 };
4884
4885 let serial_until_error = |model: &Qwen3_5Model| -> (Vec<i64>, Option<String>) {
4887 let prompt_t = Tensor::from_slice(&PROMPT).reshape([1, PROMPT.len() as i64]);
4888 let mut out = Vec::new();
4889 match model.forward_with_cache(&prompt_t, 0) {
4890 Ok(l) => out.push(argmax1(&l)),
4891 Err(e) => return (out, Some(e.to_string())),
4892 }
4893 let mut pos = PROMPT.len();
4894 loop {
4895 let dec = Tensor::from_slice(&[*out.last().unwrap()]).reshape([1, 1]);
4896 match model.forward_with_cache(&dec, pos) {
4897 Ok(l) => {
4898 out.push(argmax1(&l));
4899 pos += 1;
4900 }
4901 Err(e) => return (out, Some(e.to_string())),
4902 }
4903 }
4904 };
4905
4906 let spec_until_error = |model: &Qwen3_5Model,
4910 fallback_enabled: bool|
4911 -> (Vec<i64>, u64, Option<String>) {
4912 let prompt_t = Tensor::from_slice(&PROMPT).reshape([1, PROMPT.len() as i64]);
4913 let mut out: Vec<i64> = Vec::new();
4914 let mut fallbacks = 0u64;
4915 let (logits, hidden) = match model.forward_with_cache_hidden(&prompt_t, 0, true) {
4916 Ok(v) => v,
4917 Err(e) => return (out, fallbacks, Some(e.to_string())),
4918 };
4919 let mut last = argmax1(&logits);
4920 out.push(last);
4921 let mut mtp_tokens: Vec<i64> = PROMPT[1..].to_vec();
4922 mtp_tokens.push(last);
4923 let mtp_in = Tensor::from_slice(&mtp_tokens).reshape([1, mtp_tokens.len() as i64]);
4924 let mut pending = argmax1(&model.mtp_forward(&mtp_in, &hidden, 0).unwrap());
4925
4926 let mut pos = PROMPT.len();
4927 loop {
4928 if fallback_enabled && pos + 2 > CAP {
4929 let dec = Tensor::from_slice(&[last]).reshape([1, 1]);
4931 match model.forward_with_cache(&dec, pos) {
4932 Ok(l) => {
4933 last = argmax1(&l);
4934 out.push(last);
4935 pos += 1;
4936 fallbacks += 1;
4937 }
4938 Err(e) => return (out, fallbacks, Some(e.to_string())),
4939 }
4940 continue;
4941 }
4942 let draft = (pending + 1) % VOCAB; let snapshot = model.snapshot_ssm_states();
4944 let verify_in = Tensor::from_slice(&[last, draft]).reshape([1, 2]);
4945 let (logits, hidden) = match model.forward_with_cache_hidden(&verify_in, pos, false)
4946 {
4947 Ok(v) => v,
4948 Err(e) => return (out, fallbacks, Some(e.to_string())),
4949 };
4950 let v1 = argmax1(&logits.select(1, 0));
4951 if v1 == draft {
4952 let v2 = argmax1(&logits.select(1, 1));
4954 let mtp_in = Tensor::from_slice(&[v1, v2]).reshape([1, 2]);
4955 pending = argmax1(&model.mtp_forward(&mtp_in, &hidden, pos).unwrap());
4956 out.push(v1);
4957 out.push(v2);
4958 last = v2;
4959 pos += 2;
4960 } else {
4961 model.restore_ssm_states(snapshot.0, snapshot.1);
4962 model.get_kv_cache().unwrap().lock().truncate_to(pos + 1);
4963 let re_in = Tensor::from_slice(&[last]).reshape([1, 1]);
4964 let (_, hidden2) = model.forward_with_cache_hidden(&re_in, pos, true).unwrap();
4965 let mtp_in = Tensor::from_slice(&[v1]).reshape([1, 1]);
4966 pending = argmax1(&model.mtp_forward(&mtp_in, &hidden2, pos).unwrap());
4967 out.push(v1);
4968 last = v1;
4969 pos += 1;
4970 }
4971 }
4972 };
4973
4974 let (serial_out, serial_err) = serial_until_error(&build());
4975 assert!(
4976 serial_err
4977 .as_deref()
4978 .is_some_and(|e| e.contains("Cache overflow")),
4979 "serial driver must end at the context boundary, got {serial_err:?}"
4980 );
4981 assert_eq!(
4982 serial_out.len(),
4983 7,
4984 "prefill token + forwards at pos 6..=11"
4985 );
4986
4987 let (old_out, _, old_err) = spec_until_error(&build(), false);
4990 assert!(
4991 old_err
4992 .as_deref()
4993 .is_some_and(|e| e.contains("Cache overflow")),
4994 "pre-fix behavior must overflow, got {old_err:?}"
4995 );
4996 assert!(
4997 old_out.len() < serial_out.len(),
4998 "pre-fix driver must lose the final token (old={}, serial={})",
4999 old_out.len(),
5000 serial_out.len()
5001 );
5002
5003 let (spec_out, fallbacks, spec_err) = spec_until_error(&build(), true);
5006 assert_eq!(
5007 spec_out, serial_out,
5008 "boundary fallback diverged from serial"
5009 );
5010 assert!(
5011 fallbacks >= 1,
5012 "serial fallback never engaged at the boundary"
5013 );
5014 assert!(
5015 spec_err
5016 .as_deref()
5017 .is_some_and(|e| e.contains("Cache overflow")),
5018 "spec driver must end at the true boundary like serial, got {spec_err:?}"
5019 );
5020 }
5021
5022 #[test]
5028 fn mtp_decode_matches_serial_across_prefix_cache_hit() {
5029 let model_serial = whole_model_mtp();
5030 let model_spec = whole_model_mtp();
5031
5032 let s1 = serial_generate(&model_serial, &PROMPT, GEN, 0);
5034 let (p1, _, _) = spec_generate(&model_spec, &PROMPT, GEN, 0, DraftMode::Natural, None);
5035 assert_eq!(p1, s1, "turn 1: speculative diverged from serial");
5036
5037 let prefix_len = PROMPT.len();
5042 for m in [&model_serial, &model_spec] {
5043 m.get_kv_cache().unwrap().lock().truncate_to(prefix_len);
5044 let n = m.num_layers();
5045 let empty_conv: Vec<Option<Tensor>> = (0..n).map(|_| None).collect();
5047 let empty_rec: Vec<Option<Tensor>> = (0..n).map(|_| None).collect();
5048 m.restore_ssm_states(empty_conv, empty_rec);
5049 }
5050
5051 let prompt2: [i64; 9] = [1, 5, 9, 2, 7, 3, 4, 8, 6];
5053 let s2 = serial_generate(&model_serial, &prompt2, GEN, prefix_len);
5054 let (p2, _, _) = spec_generate(
5055 &model_spec,
5056 &prompt2,
5057 GEN,
5058 prefix_len,
5059 DraftMode::Natural,
5060 None,
5061 );
5062 assert_eq!(
5063 p2, s2,
5064 "turn 2 (prefix hit): speculative diverged from serial"
5065 );
5066 }
5067
5068 #[test]
5071 fn mtp_head_absent_falls_back_to_serial() {
5072 let mut w = tiny_weights();
5073 let model = Qwen3_5Model::from_weights(
5074 &mut w,
5075 tiny_config(),
5076 None,
5077 &Device::Cpu,
5078 Kind::Float,
5079 KVQuantType::None,
5080 true,
5081 )
5082 .unwrap();
5083 assert!(
5084 !model.has_mtp(),
5085 "model without mtp.* weights must not speculate"
5086 );
5087 }
5088
5089 #[test]
5101 fn cached_two_token_forward_matches_serial_decode_steps() {
5102 let _g = tch::no_grad_guard();
5103 let allattn_config = || {
5104 let mut cfg = tiny_config();
5105 cfg.layer_types = (0..cfg.num_hidden_layers)
5106 .map(|_| "full_attention".to_owned())
5107 .collect();
5108 cfg
5109 };
5110 let build = || {
5111 let mut w = tiny_weights_mtp();
5112 let cfg = allattn_config();
5113 let opt = (Kind::Float, Device::Cpu);
5115 let mut seed: i64 = 20_000;
5116 let mut pat = |dims: &[i64]| -> Tensor {
5117 let n: i64 = dims.iter().product();
5118 seed += 7;
5119 (Tensor::arange(n, opt) * 0.017 + seed as f64 * 0.013)
5120 .sin()
5121 .reshape(dims)
5122 * 0.05
5123 };
5124 for i in 0..cfg.num_hidden_layers as usize {
5125 if (i + 1) % 4 == 0 {
5126 continue; }
5128 let p = format!("model.layers.{i}");
5129 w.retain(|k, _| !k.starts_with(&format!("{p}.linear_attn.")));
5130 let ap = format!("{p}.self_attn");
5131 w.insert(
5132 format!("{ap}.q_proj.weight"),
5133 pat(&[HEADS * HEAD_DIM * 2, HIDDEN]),
5134 );
5135 w.insert(
5136 format!("{ap}.k_proj.weight"),
5137 pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
5138 );
5139 w.insert(
5140 format!("{ap}.v_proj.weight"),
5141 pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
5142 );
5143 w.insert(
5144 format!("{ap}.o_proj.weight"),
5145 pat(&[HIDDEN, HEADS * HEAD_DIM]),
5146 );
5147 w.insert(format!("{ap}.q_norm.weight"), pat(&[HEAD_DIM]));
5148 w.insert(format!("{ap}.k_norm.weight"), pat(&[HEAD_DIM]));
5149 }
5150 Qwen3_5Model::from_weights(
5151 &mut w,
5152 allattn_config(),
5153 None,
5154 &Device::Cpu,
5155 Kind::Float,
5156 KVQuantType::None,
5157 true,
5158 )
5159 .unwrap()
5160 };
5161
5162 let serial = build();
5164 let prompt_t = Tensor::from_slice(&PROMPT).reshape([1, PROMPT.len() as i64]);
5165 let _ = serial.forward_with_cache(&prompt_t, 0).unwrap();
5166 let pos = PROMPT.len();
5167 let step1 = Tensor::from_slice(&[11i64]).reshape([1, 1]);
5168 let ref1 = serial.forward_with_cache(&step1, pos).unwrap(); let step2 = Tensor::from_slice(&[13i64]).reshape([1, 1]);
5170 let ref2 = serial.forward_with_cache(&step2, pos + 1).unwrap();
5171
5172 let cached = build();
5174 let _ = cached.forward_with_cache(&prompt_t, 0).unwrap();
5175 let pair = Tensor::from_slice(&[11i64, 13]).reshape([1, 2]);
5176 let (logits2, _) = cached.forward_with_cache_hidden(&pair, pos, false).unwrap();
5177
5178 for (row, ref_l) in [(0i64, &ref1), (1, &ref2)] {
5179 let got = logits2.select(1, row).reshape([-1i64]);
5180 let want = ref_l.reshape([-1i64]);
5181 let max_diff = (&got - &want).abs().max().double_value(&[]);
5182 assert!(
5183 got.allclose(&want, 1e-4, 1e-4, false),
5184 "cached 2-token forward row {row} diverged from the serial decode step \
5185 (max_diff={max_diff}); the causal mask must keep kv_len - q_len history, \
5186 not tril(0)"
5187 );
5188 assert_eq!(
5189 got.argmax(-1, false).int64_value(&[]),
5190 want.argmax(-1, false).int64_value(&[]),
5191 "row {row} argmax differs",
5192 );
5193 }
5194 }
5195}