Skip to main content

hyprstream_inference/runtime/architectures/
qwen3_5.rs

1//! Qwen3.5 hybrid SSM/full-attention model implementation
2#![allow(
3    dead_code,
4    unused_variables,
5    clippy::redundant_closure,
6    clippy::assign_op_pattern,
7    clippy::needless_borrows_for_generic_args,
8    clippy::redundant_closure_for_method_calls
9)]
10//!
11//! Qwen3.5 interleaves Gated DeltaNet (linear SSM attention) and standard GQA layers.
12//! Every 4th layer (by default) is full attention; the rest are GatedDeltaNet.
13//! Supports both dense (qwen3_5_text) and MoE (qwen3_5_moe) variants.
14
15use super::config::ArchitectureConfig;
16use super::llama::{LinearProjection, LlamaMLP};
17use super::qwen3_5_vision::{Qwen3_5VisionConfig, Qwen3_5VisionEncoder};
18use super::{ModelArchitecture, ModelOperations};
19use crate::runtime::device_pool::LayerDeviceMap;
20use crate::runtime::kv_cache::KVCacheManager;
21use crate::runtime::model_config::ModelConfig;
22use crate::runtime::rope::RoPE;
23use crate::runtime::tensor_helpers::{dims3, dims4};
24use anyhow::{anyhow, Result};
25use hyprstream_rpc_std::model_client::KVQuantType;
26use std::collections::HashMap;
27use std::sync::Arc;
28use tch::{Device, Kind, Tensor};
29use tracing::info;
30
31// Thread-local cache of RoPE instances (sin/cos tables on the compute device),
32// keyed by (layer_idx, rope_theta bits, device ordinal). See the equivalent
33// `llama::ROPE_CACHE` for the full rationale.
34//
35// Teardown (#429): entries hold real CUDA tensors on GPU runs and survive the
36// engine's `Drop` (they are thread-local). `clear_rope_cache` drops them on the
37// service thread at shutdown, before libtorch's atexit handler tears down the
38// CUDA context — otherwise the TLS destructor frees them during process-exit
39// cleanup and aborts with `c10::Error: invalid device pointer`.
40thread_local! {
41    static ROPE_CACHE: std::cell::RefCell<HashMap<(usize, u32, i64), RoPE>> =
42        std::cell::RefCell::new(HashMap::new());
43}
44
45/// Drop every cached RoPE instance on the *calling* thread. Idempotent; call on
46/// the inference service thread at shutdown. See [`ROPE_CACHE`].
47pub(crate) fn clear_rope_cache() {
48    ROPE_CACHE.with(|cache| cache.borrow_mut().clear());
49}
50
51// ============================================================================
52// Config
53// ============================================================================
54
55/// Qwen3.5 text backbone configuration
56pub struct Qwen3_5TextConfig {
57    pub hidden_size: u32,
58    pub num_hidden_layers: u32,
59    pub num_attention_heads: u32,
60    pub num_key_value_heads: u32,
61    pub head_dim: u32,
62    pub intermediate_size: u32,
63    pub vocab_size: u32,
64    pub max_position_embeddings: u32,
65    pub rms_norm_eps: f32,
66    pub rope_theta: f32,
67    /// Fraction of head_dim to rotate (0.25 → rotary_dim = head_dim/4)
68    pub partial_rotary_factor: f32,
69    /// Derived: head_dim * partial_rotary_factor
70    pub rotary_dim: usize,
71    /// Per-layer type: "linear_attention" or "full_attention"
72    pub layer_types: Vec<String>,
73    // Linear attention (GatedDeltaNet) dimensions
74    pub linear_conv_kernel_dim: usize,
75    pub linear_key_head_dim: usize,
76    pub linear_value_head_dim: usize,
77    pub linear_num_key_heads: usize,
78    pub linear_num_value_heads: usize,
79    // MoE
80    pub is_moe: bool,
81    pub num_experts: usize,
82    pub num_experts_per_tok: usize,
83    pub moe_intermediate_size: usize,
84    pub shared_expert_intermediate_size: usize,
85    // Vision
86    pub has_vision: bool,
87    pub vision_out_hidden_size: usize,
88}
89
90impl ArchitectureConfig for Qwen3_5TextConfig {
91    fn num_attention_heads(&self) -> usize {
92        self.num_attention_heads as usize
93    }
94    fn num_key_value_heads(&self) -> usize {
95        self.num_key_value_heads as usize
96    }
97    fn hidden_size(&self) -> usize {
98        self.hidden_size as usize
99    }
100    fn intermediate_size(&self) -> usize {
101        self.intermediate_size as usize
102    }
103    fn head_dim(&self) -> usize {
104        self.head_dim as usize
105    }
106    fn vocab_size(&self) -> usize {
107        self.vocab_size as usize
108    }
109    fn max_position_embeddings(&self) -> usize {
110        self.max_position_embeddings as usize
111    }
112    fn rope_theta(&self) -> Option<f32> {
113        Some(self.rope_theta)
114    }
115    fn rope_dim(&self) -> Option<usize> {
116        Some(self.rotary_dim)
117    }
118    fn layer_norm_eps(&self) -> f32 {
119        self.rms_norm_eps
120    }
121    fn use_rms_norm(&self) -> bool {
122        true
123    }
124}
125
126impl Qwen3_5TextConfig {
127    pub fn from_model_config(cfg: &ModelConfig, max_pos: usize) -> Self {
128        let partial_rotary_factor = cfg.partial_rotary_factor.unwrap_or(0.25);
129        let rotary_dim = ((cfg.head_dim as f32) * partial_rotary_factor) as usize;
130
131        // Derive layer_types if not present (text-only checkpoint may lack them)
132        let layer_types = if cfg.layer_types.is_empty() {
133            (0..cfg.num_hidden_layers)
134                .map(|i| {
135                    if (i + 1) % 4 == 0 {
136                        "full_attention".to_owned()
137                    } else {
138                        "linear_attention".to_owned()
139                    }
140                })
141                .collect()
142        } else {
143            cfg.layer_types.clone()
144        };
145
146        // MoE: use config.moe_intermediate_size if set, else fall back to intermediate_size
147        let moe_int_size = if cfg.moe_intermediate_size > 0 {
148            cfg.moe_intermediate_size
149        } else {
150            cfg.intermediate_size
151        };
152        let shared_int_size = if cfg.shared_expert_intermediate_size > 0 {
153            cfg.shared_expert_intermediate_size
154        } else {
155            cfg.intermediate_size
156        };
157
158        Self {
159            hidden_size: cfg.hidden_size as u32,
160            num_hidden_layers: cfg.num_hidden_layers as u32,
161            num_attention_heads: cfg.num_attention_heads as u32,
162            num_key_value_heads: cfg.num_key_value_heads as u32,
163            head_dim: cfg.head_dim as u32,
164            intermediate_size: cfg.intermediate_size as u32,
165            vocab_size: cfg.vocab_size as u32,
166            max_position_embeddings: max_pos as u32,
167            rms_norm_eps: cfg.rms_norm_eps,
168            rope_theta: cfg.rope_theta,
169            partial_rotary_factor,
170            rotary_dim,
171            layer_types,
172            linear_conv_kernel_dim: if cfg.linear_conv_kernel_dim > 0 {
173                cfg.linear_conv_kernel_dim
174            } else {
175                4
176            },
177            linear_key_head_dim: if cfg.linear_key_head_dim > 0 {
178                cfg.linear_key_head_dim
179            } else {
180                128
181            },
182            linear_value_head_dim: if cfg.linear_value_head_dim > 0 {
183                cfg.linear_value_head_dim
184            } else {
185                128
186            },
187            linear_num_key_heads: if cfg.linear_num_key_heads > 0 {
188                cfg.linear_num_key_heads
189            } else {
190                16
191            },
192            linear_num_value_heads: if cfg.linear_num_value_heads > 0 {
193                cfg.linear_num_value_heads
194            } else {
195                32
196            },
197            is_moe: cfg.is_moe,
198            num_experts: cfg.num_experts,
199            num_experts_per_tok: cfg.num_experts_per_tok,
200            moe_intermediate_size: moe_int_size,
201            shared_expert_intermediate_size: shared_int_size,
202            has_vision: cfg.has_vision,
203            vision_out_hidden_size: cfg.vision_out_hidden_size,
204        }
205    }
206}
207
208// ============================================================================
209// FP8 helpers
210// ============================================================================
211
212/// Fuse multiple `LinearProjection`s into one by concatenating along the output dimension (dim 1).
213///
214/// If all projections are the same dtype (all FP8 or all BF16), the weight and scale tensors
215/// are concatenated directly — no BF16 conversion at load time.
216///
217/// If dtypes are mixed (e.g. FP8 + BF16 in the same layer), FP8 weights are dequantized to
218/// BF16 first so `cat` can proceed. This only occurs for small projection types in hybrid layers.
219fn cat_projs(projs: Vec<LinearProjection>) -> LinearProjection {
220    if projs
221        .iter()
222        .any(|p| p.channel_scale.is_some() || p.children.is_some())
223    {
224        // Child weights retain their original axes, activation QDQ and ignore policy.
225        let k = projs[0].weight.size()[0];
226        let n: i64 = projs.iter().map(|p| p.weight.size()[1]).sum();
227        let mut fused = LinearProjection::new(
228            Tensor::zeros([1, 1], (projs[0].weight.kind(), projs[0].weight.device()))
229                .expand([k, n], false),
230        );
231        fused.children = Some(projs);
232        return fused;
233    }
234    #[inline]
235    fn is_fp8(k: tch::Kind) -> bool {
236        matches!(k, tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2)
237    }
238
239    let all_fp8 = projs.iter().all(|p| is_fp8(p.weight.kind()));
240    let any_fp8 = projs.iter().any(|p| is_fp8(p.weight.kind()));
241
242    if all_fp8 {
243        // Consume sources before rebuilding derived metadata, dropping each
244        // source rowwise copy before the fused copy is allocated.
245        let mut weights = Vec::with_capacity(projs.len());
246        let mut scales = Vec::with_capacity(projs.len());
247        for LinearProjection { weight, scale, .. } in projs {
248            weights.push(weight);
249            scales.push(scale);
250        }
251        let weight_refs: Vec<&Tensor> = weights.iter().collect();
252        let fused_weight = Tensor::cat(&weight_refs, 1);
253        let scale = if scales.iter().any(Option::is_some) {
254            #[allow(clippy::expect_used)] // all_fp8 invariant: every proj has a scale
255            let scale_refs: Vec<&Tensor> = scales
256                .iter()
257                .map(|scale| scale.as_ref().expect("FP8 projection missing scale"))
258                .collect();
259            Some(Tensor::cat(&scale_refs, 1))
260        } else {
261            None
262        };
263        match scale {
264            Some(s) => LinearProjection::with_scale(fused_weight, s),
265            None => LinearProjection::new(fused_weight),
266        }
267    } else if any_fp8 {
268        // Mixed FP8/BF16: dequantize FP8 projections to BF16 before catting.
269        // Occurs for small gating projections in hybrid layers; memory impact is minor.
270        let weights: Vec<Tensor> = projs
271            .into_iter()
272            .map(|p| {
273                if is_fp8(p.weight.kind()) {
274                    let w_bf = p.weight.to_kind(tch::Kind::BFloat16);
275                    if let Some(s) = p.scale {
276                        let ws = w_bf.size();
277                        let ss = s.size();
278                        let br = ws[0] / ss[0];
279                        let bc = ws[1] / ss[1];
280                        let w_4d = w_bf.view([ss[0], br, ss[1], bc]);
281                        let s_4d = s.to_kind(tch::Kind::BFloat16).view([ss[0], 1, ss[1], 1]);
282                        (w_4d * s_4d).reshape([ws[0], ws[1]])
283                    } else {
284                        w_bf
285                    }
286                } else {
287                    p.weight
288                }
289            })
290            .collect();
291        let weight_refs: Vec<&Tensor> = weights.iter().collect();
292        LinearProjection::new(Tensor::cat(&weight_refs, 1))
293    } else {
294        // All non-FP8: cat directly.
295        let weight_refs: Vec<&Tensor> = projs.iter().map(|p| &p.weight).collect();
296        LinearProjection::new(Tensor::cat(&weight_refs, 1))
297    }
298}
299
300/// Dequantize a batched FP8 tensor `[k, in, out]` with optional scale `[k, in/bs, out/bs]`.
301/// Used after `index_select` on stacked MoE expert weights.
302/// Non-FP8 tensors are returned as-is (shallow clone).
303fn dequant_batched(w: &Tensor, scale: Option<&Tensor>) -> Tensor {
304    match w.kind() {
305        tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
306            let w_bf = w.to_kind(tch::Kind::BFloat16);
307            if let Some(s) = scale {
308                let ws = w_bf.size(); // [k, in, out]
309                let ss = s.size(); // [k, in/bs, out/bs]
310                let br = ws[1] / ss[1];
311                let bc = ws[2] / ss[2];
312                let w_5d = w_bf.view([ss[0], ss[1], br, ss[2], bc]);
313                let s_5d = s
314                    .to_kind(tch::Kind::BFloat16)
315                    .view([ss[0], ss[1], 1, ss[2], 1]);
316                (&w_5d * &s_5d).reshape([ws[0], ws[1], ws[2]])
317            } else {
318                w_bf
319            }
320        }
321        _ => w.shallow_clone(),
322    }
323}
324
325// ============================================================================
326// Normalization helpers
327// ============================================================================
328
329/// Qwen3.5 RMSNorm: weight initialized to 0, formula norm(x_f32) * (1 + weight)
330struct Qwen3_5RMSNorm {
331    weight: Tensor,
332    /// Precomputed weight cast to float — avoids one dtype conversion per forward call.
333    weight_f32: Tensor,
334    eps: f32,
335}
336
337impl Qwen3_5RMSNorm {
338    fn new(weight: Tensor, eps: f32) -> Self {
339        let weight_f32 = weight.to_kind(Kind::Float);
340        Self {
341            weight,
342            weight_f32,
343            eps,
344        }
345    }
346
347    fn forward(&self, x: &Tensor) -> Result<Tensor> {
348        let orig = x.kind();
349        let x_f = x.to_kind(Kind::Float);
350        let mean_sq = (&x_f * &x_f).mean_dim(&[-1i64][..], true, Kind::Float);
351        let rrms = (mean_sq + self.eps as f64).rsqrt();
352        let normed = &x_f * rrms;
353        // Checkpoint weights are near 0 (mean≈0.028) — model uses (1+w)*normed formula
354        let one_plus_w = &self.weight_f32 + 1.0f64;
355        Ok((normed * one_plus_w).to_kind(orig))
356    }
357
358    /// Move both weight copies to `device` (#314 pipeline placement). No-op when
359    /// the tensors already live on `device`.
360    #[inline]
361    fn into_device(self, device: Device) -> Self {
362        Self {
363            weight: self.weight.to_device(device),
364            weight_f32: self.weight_f32.to_device(device),
365            eps: self.eps,
366        }
367    }
368}
369
370unsafe impl Send for Qwen3_5RMSNorm {}
371unsafe impl Sync for Qwen3_5RMSNorm {}
372
373/// Gated RMSNorm used for GatedDeltaNet output: norm(x) * weight * silu(gate)
374struct RMSNormGated {
375    weight: Tensor,
376    /// Precomputed `weight.to_kind(Float)` — avoids 1 cast kernel per forward call.
377    weight_f32: Tensor,
378    eps: f32,
379}
380
381impl RMSNormGated {
382    fn new(weight: Tensor, eps: f32) -> Self {
383        let weight_f32 = weight.to_kind(Kind::Float);
384        Self {
385            weight,
386            weight_f32,
387            eps,
388        }
389    }
390
391    fn forward(&self, x: &Tensor, gate: &Tensor) -> Result<Tensor> {
392        let orig = x.kind();
393        let x_f = if x.kind() == Kind::Float {
394            x.shallow_clone()
395        } else {
396            x.to_kind(Kind::Float)
397        };
398        let mean_sq = (&x_f * &x_f).mean_dim(&[-1i64][..], true, Kind::Float);
399        let rrms = (mean_sq + self.eps as f64).rsqrt();
400        let normed = &x_f * rrms;
401        let g = gate.to_kind(Kind::Float).silu();
402        Ok((normed * &self.weight_f32 * g).to_kind(orig))
403    }
404
405    /// Move both weight copies to `device` (#314 pipeline placement).
406    #[inline]
407    fn into_device(self, device: Device) -> Self {
408        Self {
409            weight: self.weight.to_device(device),
410            weight_f32: self.weight_f32.to_device(device),
411            eps: self.eps,
412        }
413    }
414}
415
416unsafe impl Send for RMSNormGated {}
417unsafe impl Sync for RMSNormGated {}
418
419/// L2 normalization along last dim (used on Q and K before delta rule)
420#[inline]
421fn l2_normalize(x: &Tensor) -> Tensor {
422    // Match transformers 5.10.2 Qwen3.5 l2norm: epsilon is additive
423    // inside rsqrt, not a lower bound on the norm. The distinction matters
424    // for small query/key vectors and propagates through recurrent state.
425    let sum_sq = (x * x).sum_dim_intlist(&[-1i64][..], true, Kind::Float);
426    x * (sum_sq + 1e-6f64).rsqrt()
427}
428
429/// Chunked GatedDeltaNet prefill (translation of `torch_chunk_gated_delta_rule`).
430///
431/// All inputs in [batch, num_v_heads, seq, head_dim] layout (heads second, already in f32).
432/// `q` is already L2-normalized and scaled by 1/sqrt(head_k_dim).
433/// Returns (output [batch, num_v_heads, seq, head_v_dim], final state [batch, nv, hk, hv]).
434fn chunked_delta_rule(
435    q: &Tensor,    // [B, nv, T, hk]  L2-normed, scaled
436    k: &Tensor,    // [B, nv, T, hk]  L2-normed
437    v: &Tensor,    // [B, nv, T, hv]
438    g: &Tensor,    // [B, nv, T]      log-decay (negative)
439    beta: &Tensor, // [B, nv, T]      in (0,1)
440    initial_state: Option<&Tensor>,
441    device: tch::Device,
442) -> Result<(Tensor, Tensor)> {
443    const CHUNK_SIZE: i64 = 64;
444
445    let batch = q.size()[0];
446    let nv = q.size()[1];
447    let seq = q.size()[2];
448    let hk = q.size()[3];
449    let hv = v.size()[3];
450
451    // Pad sequence length to multiple of CHUNK_SIZE
452    let pad = (CHUNK_SIZE - seq % CHUNK_SIZE) % CHUNK_SIZE;
453    let seq_p = seq + pad;
454
455    let pad_tensor = |t: &Tensor, extra_dims: i64| -> Tensor {
456        if pad == 0 {
457            return t.shallow_clone();
458        }
459        let z = match extra_dims {
460            0 => Tensor::zeros([batch, nv, pad], (Kind::Float, device)),
461            _ => Tensor::zeros([batch, nv, pad, extra_dims], (Kind::Float, device)),
462        };
463        Tensor::cat(&[t, &z], 2)
464    };
465
466    let q = pad_tensor(q, hk);
467    let k = pad_tensor(k, hk);
468    let v = pad_tensor(v, hv);
469    let g = pad_tensor(g, 0);
470    let beta = pad_tensor(beta, 0);
471
472    let v_beta = &v * beta.unsqueeze(-1);
473    let k_beta = &k * beta.unsqueeze(-1);
474
475    let nc = seq_p / CHUNK_SIZE;
476
477    // Reshape to [B, nv, num_chunks, chunk_size, D]
478    let reshape5 = |t: &Tensor, d: i64| t.reshape([batch, nv, nc, CHUNK_SIZE, d]);
479    let q = reshape5(&q, hk);
480    let k = reshape5(&k, hk);
481    let v = reshape5(&v, hv);
482    let k_beta = reshape5(&k_beta, hk);
483    let v_beta = reshape5(&v_beta, hv);
484    // g: [B, nv, num_chunks, chunk_size]
485    let g = g.reshape([batch, nv, nc, CHUNK_SIZE]);
486
487    // Cumulative decay within each chunk
488    let g = g.cumsum(-1, Kind::Float);
489
490    // decay_mask[t, s] = exp(g[t] - g[s]) for t >= s, 0 for t < s
491    // Must exp first, then tril — otherwise upper tri becomes exp(0)=1 instead of 0
492    let g_t = g.unsqueeze(-1); // [..., C, 1]
493    let g_s = g.unsqueeze(-2); // [..., 1, C]
494    let decay_mask = (g_t - g_s).exp().tril(0); // [B, nv, nc, C, C]
495
496    // Upper triangular mask (diagonal=0 inclusive) for masking future positions
497    let mask0 = Tensor::ones([CHUNK_SIZE, CHUNK_SIZE], (Kind::Bool, device)).triu(0);
498    // Upper triangular excluding diagonal (diagonal=1) for within-chunk attn
499    let mask1 = Tensor::ones([CHUNK_SIZE, CHUNK_SIZE], (Kind::Bool, device)).triu(1);
500
501    // attn = -(k_beta @ k^T * decay_mask).masked_fill(mask0, 0)
502    // [B, nv, nc, C, C]
503    let attn0 = -(k_beta.matmul(&k.transpose(-1, -2)) * &decay_mask).masked_fill(&mask0, 0.0f64);
504
505    // Recursive delta-rule correction (forward substitution over chunk positions).
506    //
507    // Row `i` of the corrected lower-triangular matrix is
508    //   T[i, :i] = A[i, :i] + A[i, :i] @ T[:i, :i]
509    // i.e. each row depends on the ALREADY-corrected rows above it. The original
510    // implementation built this with an in-place `attn[..i].copy_(row)`, which is
511    // unsafe for autograd (it mutates a tensor saved for backward → "variable
512    // needed for gradient computation modified by an inplace operation"). On the
513    // inference path this ran under `no_grad`; on the TTT-on-split training path
514    // (#316) the graph is live, so we build the rows out-of-place and stack them.
515    // Numerically identical to the in-place form; only the graph differs.
516    let c = CHUNK_SIZE;
517    // Each entry is a full-width row [B, nv, nc, C]; row 0 is unchanged.
518    let mut rows: Vec<Tensor> = Vec::with_capacity(c as usize);
519    rows.push(attn0.narrow(-2, 0, 1).squeeze_dim(-2)); // [B, nv, nc, C]
520    for i in 1..c {
521        // Original row i, columns [0, i) (columns >= i are 0 from mask0).
522        let a_row = attn0.narrow(-2, i, 1).narrow(-1, 0, i).squeeze_dim(-2); // [B, nv, nc, i]
523                                                                             // T[:i, :i] — stack the already-corrected rows, take their first i cols.
524        let sub = Tensor::stack(&rows, -2).narrow(-1, 0, i); // [B, nv, nc, i, i]
525        let correction = a_row.unsqueeze(-2).matmul(&sub).squeeze_dim(-2); // [B, nv, nc, i]
526        let corrected = &a_row + &correction; // [B, nv, nc, i]
527                                              // Re-pad to full width C (columns [i, C) are 0, matching mask0).
528        let pad_cols = c - i;
529        let zeros = Tensor::zeros([batch, nv, nc, pad_cols], (Kind::Float, device));
530        rows.push(Tensor::cat(&[corrected, zeros], -1)); // [B, nv, nc, C]
531    }
532    let attn = Tensor::stack(&rows, -2); // [B, nv, nc, C, C]
533
534    // Add identity
535    let eye = Tensor::eye(CHUNK_SIZE, (Kind::Float, device))
536        .unsqueeze(0)
537        .unsqueeze(0)
538        .unsqueeze(0); // [1, 1, 1, C, C]
539    let attn = attn + eye;
540
541    // value_transformed = attn @ v_beta  [B, nv, nc, C, hv]
542    let value_t = attn.matmul(&v_beta);
543    // k_cumdecay = attn @ (k_beta * g.exp()) [B, nv, nc, C, hk]
544    let k_cumdecay = attn.matmul(&(k_beta * g.exp().unsqueeze(-1)));
545
546    let mut state = initial_state
547        .map(|s| s.to_kind(Kind::Float))
548        .unwrap_or_else(|| Tensor::zeros([batch, nv, hk, hv], (Kind::Float, device)));
549    let mut chunks_out = Vec::with_capacity(nc as usize);
550
551    for i in 0..nc {
552        let q_i = q.select(2, i); // [B, nv, C, hk]
553        let k_i = k.select(2, i); // [B, nv, C, hk]
554        let v_i = value_t.select(2, i); // [B, nv, C, hv]
555        let g_i = g.select(2, i); // [B, nv, C]
556        let dm_i = decay_mask.select(2, i); // [B, nv, C, C]
557        let kcd_i = k_cumdecay.select(2, i); // [B, nv, C, hk]
558
559        // Within-chunk attention (causal, exclude future)
560        let attn_inner = (q_i.matmul(&k_i.transpose(-1, -2)) * &dm_i).masked_fill(&mask1, 0.0f64);
561
562        // Cross-chunk contribution from previous state
563        let v_prime = kcd_i.matmul(&state); // [B, nv, C, hv]
564        let v_new = v_i - v_prime;
565
566        // Inter-chunk: q scaled by cumulative g × state
567        let attn_inter = (q_i * g_i.exp().unsqueeze(-1)).matmul(&state); // [B, nv, C, hv]
568
569        let chunk_out = attn_inter + attn_inner.matmul(&v_new);
570        chunks_out.push(chunk_out);
571
572        // Update state for next chunk
573        let g_last = g_i.narrow(-1, CHUNK_SIZE - 1, 1); // [B, nv, 1]
574        let decay = g_last.exp().unsqueeze(-1); // [B, nv, 1, 1]
575        let g_diff = (g_last - &g_i).exp().unsqueeze(-1); // [B, nv, C, 1]
576        let update = (k_i * g_diff).transpose(-1, -2).matmul(&v_new); // [B, nv, hk, hv]
577        state = &state * decay + update;
578    }
579
580    // Stack chunks: [B, nv, nc, C, hv] → flatten → trim padding → contiguous
581    // The .contiguous() after narrow is critical: without it, the resulting tensor has
582    // padded strides (stride[0] = nv*seq_p*hv instead of nv*seq*hv) which causes
583    // libtorch to miscalculate buffer sizes during subsequent permute+reshape operations.
584    let out = Tensor::stack(&chunks_out, 2) // [B, nv, nc, C, hv]
585        .reshape([batch, nv, seq_p, hv])
586        .narrow(2, 0, seq)
587        .contiguous();
588
589    Ok((out, state))
590}
591
592// ============================================================================
593// GatedDeltaNet (linear attention / SSM layer)
594// ============================================================================
595
596struct GatedDeltaNetLayer {
597    // Fused input projection: hidden → (conv_dim + value_dim + 2*num_v_heads)
598    // Single matmul instead of 4 — reduces kernel launches by 75% for this step.
599    in_proj_all: LinearProjection,
600    // Split offsets (conv_dim, value_dim, num_v_heads, num_v_heads)
601    proj_split: [i64; 4],
602    // Depthwise conv1d on mixed QKV
603    conv1d_weight: Tensor, // [conv_dim, 1, kernel_size]
604    // Decay and dt bias (stored in Float to avoid per-step casting)
605    a_log: Tensor,         // [num_v_heads] original dtype
606    dt_bias: Tensor,       // [num_v_heads] original dtype
607    neg_a_exp_f32: Tensor, // [1, 1, num_v_heads] precomputed -exp(a_log) in Float
608    dt_bias_f32: Tensor,   // [1, 1, num_v_heads] precomputed in Float
609    // Output norm and projection
610    norm: RMSNormGated,
611    out_proj: LinearProjection, // value_dim → hidden
612    // Dimensions
613    num_k_heads: usize,
614    num_v_heads: usize,
615    head_k_dim: usize,
616    head_v_dim: usize,
617    key_dim: usize,   // num_k_heads * head_k_dim
618    value_dim: usize, // num_v_heads * head_v_dim
619    conv_dim: usize,  // key_dim*2 + value_dim
620    kernel_size: usize,
621    layer_idx: usize,
622}
623
624unsafe impl Send for GatedDeltaNetLayer {}
625unsafe impl Sync for GatedDeltaNetLayer {}
626
627impl GatedDeltaNetLayer {
628    fn load(
629        weights: &mut HashMap<String, Tensor>,
630        prefix: &str,
631        cfg: &Qwen3_5TextConfig,
632        layer_idx: usize,
633    ) -> Result<Self> {
634        let num_k_heads = cfg.linear_num_key_heads;
635        let num_v_heads = cfg.linear_num_value_heads;
636        let head_k_dim = cfg.linear_key_head_dim;
637        let head_v_dim = cfg.linear_value_head_dim;
638        let key_dim = num_k_heads * head_k_dim;
639        let value_dim = num_v_heads * head_v_dim;
640        let conv_dim = key_dim * 2 + value_dim;
641        let kernel_size = cfg.linear_conv_kernel_dim;
642
643        let mut get = |name: &str| -> Result<Tensor> {
644            let full = format!("{prefix}.{name}");
645            weights
646                .remove(&full)
647                .ok_or_else(|| anyhow!("Missing weight: {}", full))
648        };
649
650        let conv1d_weight = get("conv1d.weight")?;
651        let a_log = get("A_log")?;
652        let dt_bias = get("dt_bias")?;
653        let norm_weight = get("norm.weight")?;
654
655        let a_log_sq = a_log.squeeze();
656        let dt_bias_sq = dt_bias.squeeze();
657        // Precompute float versions with broadcast shape [1, 1, nv] for decode path
658        let neg_a_exp_f32 = -(a_log_sq
659            .to_kind(Kind::Float)
660            .exp()
661            .unsqueeze(0)
662            .unsqueeze(0)); // [1, 1, nv] already negated
663        let dt_bias_f32 = dt_bias_sq.to_kind(Kind::Float).unsqueeze(0).unsqueeze(0); // [1, 1, nv]
664
665        // Fuse 4 input projections into 1 matmul.
666        // Cat along output dim (dim 1, weight is [in, out] after take()), preserving FP8+scale.
667        // LinearProjection::apply() dequantizes lazily at forward time — no BF16 copy at load.
668        let w_qkv = LinearProjection::take(weights, &format!("{prefix}.in_proj_qkv.weight"))?;
669        let w_z = LinearProjection::take(weights, &format!("{prefix}.in_proj_z.weight"))?;
670        let w_b = LinearProjection::take(weights, &format!("{prefix}.in_proj_b.weight"))?;
671        let w_a = LinearProjection::take(weights, &format!("{prefix}.in_proj_a.weight"))?;
672        let in_proj_all = cat_projs(vec![w_qkv, w_z, w_b, w_a]);
673        let proj_split = [
674            conv_dim as i64,
675            value_dim as i64,
676            num_v_heads as i64,
677            num_v_heads as i64,
678        ];
679
680        Ok(Self {
681            in_proj_all,
682            proj_split,
683            conv1d_weight,
684            a_log: a_log_sq,
685            dt_bias: dt_bias_sq,
686            neg_a_exp_f32,
687            dt_bias_f32,
688            norm: RMSNormGated::new(norm_weight, 1e-6),
689            out_proj: LinearProjection::take(weights, &format!("{prefix}.out_proj.weight"))?,
690            num_k_heads,
691            num_v_heads,
692            head_k_dim,
693            head_v_dim,
694            key_dim,
695            value_dim,
696            conv_dim,
697            kernel_size,
698            layer_idx,
699        })
700    }
701
702    fn forward(
703        &self,
704        hidden: &Tensor,
705        conv_state: &mut Option<Tensor>,
706        rec_state: &mut Option<Tensor>,
707        delta: Option<(&crate::training::TenantDelta, usize)>,
708    ) -> Result<Tensor> {
709        let (batch, seq, _) = dims3(hidden)?;
710        let device = hidden.device();
711        let dtype = hidden.kind();
712
713        // Fused input projection: 1 matmul for all 4 projections.
714        let h2 = hidden.reshape([batch * seq, hidden.size()[2]]);
715        let all_proj = self.in_proj_all.try_apply(&h2)?; // [batch*seq, conv_dim+val_dim+2*nv]
716        let [cd, vd, nv_d, _] = self.proj_split;
717        let mut off = 0i64;
718        let mixed_qkv = all_proj
719            .narrow(1, off, cd)
720            .contiguous()
721            .reshape([batch, seq, cd]);
722        off += cd;
723        let z = all_proj
724            .narrow(1, off, vd)
725            .contiguous()
726            .reshape([batch, seq, vd]);
727        off += vd;
728        let b = all_proj
729            .narrow(1, off, nv_d)
730            .contiguous()
731            .reshape([batch, seq, nv_d]);
732        off += nv_d;
733        let a_in = all_proj
734            .narrow(1, off, nv_d)
735            .contiguous()
736            .reshape([batch, seq, nv_d]);
737
738        // Depthwise causal conv1d on mixed_qkv
739        // mixed_qkv: [batch, seq, conv_dim] → need [batch, conv_dim, seq] for conv1d
740        let conv_dim = self.conv_dim as i64;
741        let ks = self.kernel_size as i64;
742
743        // Maintain conv state: [batch, conv_dim, kernel_size-1]
744        let x_t = mixed_qkv.permute([0, 2, 1]); // [batch, conv_dim, seq]
745
746        let (x_for_conv, new_conv_state) = match conv_state {
747            Some(ref cs) => {
748                // Prepend previous state for causal context
749                let padded = Tensor::cat(&[cs, &x_t], 2); // [batch, conv_dim, prev+seq]
750                                                          // Keep last kernel_size-1 frames as new state
751                let total = padded.size()[2];
752                let new_state = padded.narrow(2, total - (ks - 1), ks - 1).contiguous();
753                (padded, new_state)
754            }
755            None => {
756                // First call: zero-pad left by kernel_size-1
757                let pad = Tensor::zeros([batch, conv_dim, ks - 1], (dtype, device));
758                let padded = Tensor::cat(&[&pad, &x_t], 2);
759                // Retain the left padding when the first prompt is shorter
760                // than the convolution history, so the next decode has all taps.
761                let new_state = padded.narrow(2, seq, ks - 1).contiguous();
762                (padded, new_state)
763            }
764        };
765        *conv_state = Some(new_conv_state);
766
767        // conv1d with groups=conv_dim (depthwise), no padding (already padded manually)
768        let conv_out = x_for_conv.conv1d(
769            &self.conv1d_weight,
770            None::<&Tensor>,
771            &[1i64],  // stride=1
772            &[0i64],  // padding=0 (handled manually)
773            &[1i64],  // dilation=1
774            conv_dim, // groups
775        ); // [batch, conv_dim, seq]
776        let conv_out = conv_out.silu().permute([0, 2, 1]); // [batch, seq, conv_dim]
777
778        // Split conv_out into q, k, v
779        let key_dim = self.key_dim as i64;
780        let val_dim = self.value_dim as i64;
781        let q_raw = conv_out.narrow(2, 0, key_dim);
782        let k_raw = conv_out.narrow(2, key_dim, key_dim);
783        let v_raw = conv_out.narrow(2, key_dim * 2, val_dim);
784
785        // Reshape to [batch, seq, num_heads, head_dim]
786        let q = q_raw.reshape([batch, seq, self.num_k_heads as i64, self.head_k_dim as i64]);
787        let k = k_raw.reshape([batch, seq, self.num_k_heads as i64, self.head_k_dim as i64]);
788        let v = v_raw.reshape([batch, seq, self.num_v_heads as i64, self.head_v_dim as i64]);
789
790        // L2 normalize Q and K before delta rule
791        let q = l2_normalize(&q.to_kind(Kind::Float));
792        let k = l2_normalize(&k.to_kind(Kind::Float));
793        let scale = 1.0 / (self.head_k_dim as f64).sqrt();
794        let q = q * scale;
795
796        // Compute decay g = A * dt where A = -exp(A_log), dt = softplus(a_in + dt_bias)
797        // A_log stores log(-A) so A = -exp(A_log) is negative; g = A * dt is negative log-decay
798        // Use precomputed float tensors — avoids to_kind + exp + unsqueeze per step
799        let a_in_f = a_in.to_kind(Kind::Float); // [batch, seq, num_v_heads]
800        let softplus_input = a_in_f + &self.dt_bias_f32; // [1,1,nv] broadcasts
801        let g = &self.neg_a_exp_f32 * softplus_input.softplus(); // [batch, seq, nv]
802
803        // Recurrent delta rule
804        // state shape: [batch, num_v_heads, head_k_dim, head_v_dim]
805        let nv = self.num_v_heads as i64;
806        let hk = self.head_k_dim as i64;
807        let hv = self.head_v_dim as i64;
808
809        let state_init = rec_state
810            .take()
811            .unwrap_or_else(|| Tensor::zeros([batch, nv, hk, hv], (Kind::Float, device)));
812
813        let b_f = b.to_kind(Kind::Float).sigmoid(); // [batch, seq, num_v_heads]
814        let v_f = v.to_kind(Kind::Float); // [batch, seq, num_v_heads, head_v_dim]
815
816        // For num_k_heads != num_v_heads, each k head is shared by
817        // (num_v_heads / num_k_heads) consecutive v heads (GQA-style interleave).
818        // Must use repeat_interleave (not repeat/tile) so head i maps to v heads
819        // [i*r, ..., i*r + r-1] rather than [i, i+num_k, ...].
820        let k_for_state = if self.num_k_heads == self.num_v_heads {
821            k.to_kind(Kind::Float)
822        } else {
823            let repeats = (self.num_v_heads / self.num_k_heads) as i64;
824            k.repeat_interleave_self_int(repeats, 2, None)
825                .to_kind(Kind::Float)
826        };
827        // q similarly expanded with repeat_interleave
828        let q_for_state = if self.num_k_heads == self.num_v_heads {
829            q.shallow_clone()
830        } else {
831            let repeats = (self.num_v_heads / self.num_k_heads) as i64;
832            q.repeat_interleave_self_int(repeats, 2, None)
833        };
834
835        // Choose recurrent path based on sequence length:
836        // - seq == 1  (decode): token-by-token recurrent loop
837        // - seq > 1   (prefill): chunked algorithm for O(seq · chunk) vs O(seq²)
838        let out = if seq == 1 {
839            // --- Decode path: single-token recurrent step ---
840            let g_t = g.narrow(1, 0, 1).squeeze_dim(1);
841            let b_t = b_f.narrow(1, 0, 1).squeeze_dim(1);
842            let k_t = k_for_state.narrow(1, 0, 1).squeeze_dim(1);
843            let v_t = v_f.narrow(1, 0, 1).squeeze_dim(1);
844            let q_t = q_for_state.narrow(1, 0, 1).squeeze_dim(1);
845
846            // state: [B, nv, hk, hv]; k_t, q_t: [B, nv, hk]; v_t, b_t: [B, nv, hv]
847            // In-place state decay (Step 9)
848            let decay = g_t.exp().reshape([batch, nv, 1, 1]);
849            let state = &state_init * &decay;
850            // BMM-based readout: state[B*nv, hk, hv] × k_t[B*nv, 1, hk]^T → [B*nv, 1, hv]
851            let state_3d = state.reshape([batch * nv, hk, hv]);
852            let k_col = k_t.reshape([batch * nv, 1, hk]);
853            let kv_mem = k_col.bmm(&state_3d).reshape([batch, nv, hv]);
854            let delta = (v_t - kv_mem) * b_t.unsqueeze(-1);
855            // outer product update: state[h,k,v] += k[h,k] * delta[h,v]
856            let state =
857                state.reshape([batch, nv, hk, hv]) + k_t.unsqueeze(-1) * delta.unsqueeze(-2);
858            // BMM-based output readout
859            let state_3d = state.reshape([batch * nv, hk, hv]);
860            let q_col = q_t.reshape([batch * nv, 1, hk]);
861            let out_t = q_col.bmm(&state_3d).reshape([batch, nv, hv]);
862
863            *rec_state = Some(state);
864            // out_t: [batch, nv, hv] → [batch, 1, nv*hv]
865            out_t.unsqueeze(1).reshape([batch, 1, -1])
866        } else {
867            // --- Prefill path: chunked GatedDeltaNet ---
868            // Reorder to [batch, nv, seq, dim] for chunked_delta_rule
869            let q_h = q_for_state.permute([0, 2, 1, 3]); // [B, nv, T, hk]
870            let k_h = k_for_state.permute([0, 2, 1, 3]);
871            let v_h = v_f.permute([0, 2, 1, 3]); // [B, nv, T, hv]
872            let g_h = g.permute([0, 2, 1]); // [B, nv, T]
873            let b_h = b_f.permute([0, 2, 1]); // [B, nv, T]
874
875            // The previous state was moved out above. Use that owned state
876            // for cached multi-token continuation, just as single-token decode does.
877            let (out_h, new_state) =
878                chunked_delta_rule(&q_h, &k_h, &v_h, &g_h, &b_h, Some(&state_init), device)?;
879            // out_h: [B, nv, T, hv]
880            *rec_state = Some(new_state);
881
882            // Transpose back to [B, T, nv, hv] and flatten to [B, T, nv*hv]
883            // Flatten [B, nv, T, hv] → [B, T, nv*hv]
884            out_h
885                .permute([0, 2, 1, 3])
886                .contiguous()
887                .reshape([batch, seq, -1])
888        };
889
890        // Gated norm and output projection
891        // RMSNormGated weight is [head_v_dim]; reshape to per-head before norm.
892        // Derive actual value dim from out tensor (may differ from config for small models).
893        let actual_vd = out.size()[2];
894        let actual_hv = self.norm.weight.size()[0]; // norm weight shape = [head_v_dim]
895        let actual_nv = actual_vd / actual_hv;
896        let z_f = z.narrow(2, 0, actual_vd).contiguous(); // trim z to match actual value dim
897
898        let out_4d = out.reshape([batch, seq, actual_nv, actual_hv]);
899        let z_4d = z_f.reshape([batch, seq, actual_nv, actual_hv]);
900        let normed = self.norm.forward(&out_4d, &z_4d)?; // out_4d may be F32 from recurrent step; RMSNormGated handles both
901                                                         // Ensure output matches model dtype for out_proj matmul (weight is BF16)
902        let normed = if normed.kind() != dtype {
903            normed.to_kind(dtype)
904        } else {
905            normed
906        };
907        let flat = normed.reshape([batch * seq, actual_vd]);
908
909        let projected = self.out_proj.try_apply(&flat)?;
910        // Standard LoRA: add correction in the OUTPUT space of out_proj (y = Wx + BAx).
911        // Inject after out_proj so correction shape [B*seq, hidden_size] matches projected,
912        // regardless of whether val_dim == hidden_size. (Issue 6 + standard LoRA semantics)
913        let projected = if let Some((delta, layer_idx)) = delta {
914            if delta.has_module("o_proj", layer_idx) {
915                // forward_2d(flat): [B*seq, val_dim] → [B*seq, hidden_size]
916                // #139/#440: coerce fp32 delta to base dtype before adding.
917                super::add_delta(&projected, delta.forward_2d(&flat, "o_proj", layer_idx)?)
918            } else {
919                projected
920            }
921        } else {
922            projected
923        };
924        Ok(projected.reshape([batch, seq, self.hidden_size_from_out_proj()]))
925    }
926
927    fn hidden_size_from_out_proj(&self) -> i64 {
928        // weight is stored [in_features, out_features] after transpose at load; size()[1] = out_features = hidden_size
929        self.out_proj.weight.size()[1]
930    }
931
932    /// Move every owned tensor to `device` (#314 pipeline placement). The SSM
933    /// runtime state (`conv`/`rec`) is NOT part of the layer — it is stage-local
934    /// and threaded separately — so only the static weights move here.
935    #[inline]
936    fn into_device(self, device: Device) -> Self {
937        Self {
938            in_proj_all: self.in_proj_all.into_device(device),
939            conv1d_weight: self.conv1d_weight.to_device(device),
940            a_log: self.a_log.to_device(device),
941            dt_bias: self.dt_bias.to_device(device),
942            neg_a_exp_f32: self.neg_a_exp_f32.to_device(device),
943            dt_bias_f32: self.dt_bias_f32.to_device(device),
944            norm: self.norm.into_device(device),
945            out_proj: self.out_proj.into_device(device),
946            ..self
947        }
948    }
949}
950
951// ============================================================================
952// Full attention layer (GQA + output gate + partial RoPE)
953// ============================================================================
954
955struct Qwen3_5FullAttention {
956    qkv_proj: LinearProjection, // fused: hidden → (q_out + k_out + v_out)
957    qkv_split: [i64; 3],        // [q_dim, k_dim, v_dim] for narrow splits
958    o_proj: LinearProjection,   // in: num_heads * head_dim
959    q_norm: Qwen3_5RMSNorm,     // per-head, weight dim = head_dim
960    k_norm: Qwen3_5RMSNorm,
961    num_heads: usize,
962    num_kv_heads: usize,
963    head_dim: usize,
964    rotary_dim: usize,
965    rope_theta: f32,
966    max_pos: usize,
967    layer_idx: usize,
968}
969
970unsafe impl Send for Qwen3_5FullAttention {}
971unsafe impl Sync for Qwen3_5FullAttention {}
972
973impl Qwen3_5FullAttention {
974    fn load(
975        weights: &mut HashMap<String, Tensor>,
976        prefix: &str,
977        cfg: &Qwen3_5TextConfig,
978        layer_idx: usize,
979    ) -> Result<Self> {
980        let q_norm_weight = weights
981            .remove(&format!("{prefix}.q_norm.weight"))
982            .ok_or_else(|| anyhow!("Missing {prefix}.q_norm.weight"))?;
983        let k_norm_weight = weights
984            .remove(&format!("{prefix}.k_norm.weight"))
985            .ok_or_else(|| anyhow!("Missing {prefix}.k_norm.weight"))?;
986
987        // Fuse Q/K/V projections into single matmul, preserving FP8+scale.
988        // LinearProjection::apply() dequantizes lazily at forward time — no BF16 copy at load.
989        let q_proj = LinearProjection::take(weights, &format!("{prefix}.q_proj.weight"))?;
990        let k_proj = LinearProjection::take(weights, &format!("{prefix}.k_proj.weight"))?;
991        let v_proj = LinearProjection::take(weights, &format!("{prefix}.v_proj.weight"))?;
992        let q_dim = q_proj.weight.size()[1];
993        let k_dim = k_proj.weight.size()[1];
994        let v_dim = v_proj.weight.size()[1];
995        let fused = cat_projs(vec![q_proj, k_proj, v_proj]);
996
997        Ok(Self {
998            qkv_proj: fused,
999            qkv_split: [q_dim, k_dim, v_dim],
1000            o_proj: LinearProjection::take(weights, &format!("{prefix}.o_proj.weight"))?,
1001            q_norm: Qwen3_5RMSNorm::new(q_norm_weight, cfg.rms_norm_eps),
1002            k_norm: Qwen3_5RMSNorm::new(k_norm_weight, cfg.rms_norm_eps),
1003            num_heads: cfg.num_attention_heads as usize,
1004            num_kv_heads: cfg.num_key_value_heads as usize,
1005            head_dim: cfg.head_dim as usize,
1006            rotary_dim: cfg.rotary_dim,
1007            rope_theta: cfg.rope_theta,
1008            max_pos: cfg.max_position_embeddings as usize,
1009            layer_idx,
1010        })
1011    }
1012
1013    /// Move all projection + QK-norm weights to `device` (#314 pipeline placement).
1014    #[inline]
1015    fn into_device(self, device: Device) -> Self {
1016        Self {
1017            qkv_proj: self.qkv_proj.into_device(device),
1018            o_proj: self.o_proj.into_device(device),
1019            q_norm: self.q_norm.into_device(device),
1020            k_norm: self.k_norm.into_device(device),
1021            ..self
1022        }
1023    }
1024
1025    fn forward(
1026        &self,
1027        hidden: &Tensor,
1028        position_ids: Option<&Tensor>,
1029        kv_cache: Option<&mut crate::runtime::kv_cache::LayerKVCache>,
1030        start_pos: usize,
1031        delta: Option<(&crate::training::TenantDelta, usize)>,
1032    ) -> Result<Tensor> {
1033        let (batch, seq, _) = dims3(hidden)?;
1034        let device = hidden.device();
1035        let dtype = hidden.kind();
1036        let h2 = hidden.reshape([batch * seq, hidden.size()[2]]);
1037
1038        // Fused QKV projection: single matmul, then split
1039        let qkv = self.qkv_proj.try_apply(&h2)?;
1040        let [q_dim, k_dim, v_dim] = self.qkv_split;
1041
1042        // Delta injection on q_proj and v_proj (out-of-place to avoid aliasing, C2 fix)
1043        // q_out layout: [B*seq, q_dim] where q_dim = num_heads * head_dim * 2
1044        //   first half = Q, second half = gate
1045        // Must apply correction to Q-half only, then reassemble with gate half
1046        let (q_out, v_out) = if let Some((d, layer_idx)) = delta {
1047            let q_raw = qkv.narrow(1, 0, q_dim);
1048            let v_raw = qkv.narrow(1, q_dim + k_dim, v_dim);
1049
1050            let q_out = if d.has_module("q_proj", layer_idx) {
1051                // q_dim = num_heads * head_dim * 2; split into Q and gate halves
1052                let q_half = q_dim / 2;
1053                let q_part = q_raw.narrow(1, 0, q_half);
1054                let gate_part = q_raw.narrow(1, q_half, q_dim - q_half);
1055                let q_correction = d.forward_2d(&h2, "q_proj", layer_idx)?;
1056                // q_correction shape: [B*seq, q_half] (delta target is the pre-gate Q only)
1057                // If correction has full q_dim, only apply to q-half
1058                let q_correction = if q_correction.size()[1] == q_half {
1059                    q_correction
1060                } else {
1061                    q_correction.narrow(1, 0, q_half)
1062                };
1063                // #139/#440: coerce fp32 delta to base dtype before adding (after narrow).
1064                let corrected_q = super::add_delta(&q_part, q_correction);
1065                Tensor::cat(&[corrected_q, gate_part], 1)
1066            } else {
1067                q_raw
1068            };
1069
1070            let v_out = if d.has_module("v_proj", layer_idx) {
1071                // #139/#440: coerce fp32 delta to base dtype before adding.
1072                super::add_delta(&v_raw, d.forward_2d(&h2, "v_proj", layer_idx)?)
1073            } else {
1074                v_raw
1075            };
1076            (q_out, v_out)
1077        } else {
1078            (qkv.narrow(1, 0, q_dim), qkv.narrow(1, q_dim + k_dim, v_dim))
1079        };
1080        let k_out = qkv.narrow(1, q_dim, k_dim);
1081
1082        let nh = self.num_heads as i64;
1083        let nkv = self.num_kv_heads as i64;
1084        let hd = self.head_dim as i64;
1085
1086        // Split query and gate (from doubled q_proj output)
1087        // Reference: gate.reshape(batch, seq, num_heads * head_dim), then * sigmoid(gate) before o_proj
1088        let q_full = q_out.reshape([batch, seq, nh, hd * 2]);
1089        let query = q_full.narrow(3, 0, hd); // [batch, seq, nh, head_dim]
1090        let gate = q_full.narrow(3, hd, hd); // [batch, seq, nh, head_dim]
1091                                             // gate will be applied as sigmoid(gate.reshape(batch, seq, nh*hd)) * attn_output BEFORE o_proj
1092
1093        let mut k = k_out.reshape([batch, seq, nkv, hd]);
1094        let v = v_out.reshape([batch, seq, nkv, hd]);
1095
1096        // Per-head QK norm
1097        let query = self.apply_qknorm(&query, &self.q_norm, self.num_heads)?;
1098        k = self.apply_qknorm(&k, &self.k_norm, self.num_kv_heads)?;
1099
1100        // Partial RoPE: rotate first rotary_dim dims, pass through rest
1101        let rd = self.rotary_dim as i64;
1102        let (query, k) = if rd > 0 {
1103            let query_rot = self.apply_rope(
1104                &query.narrow(3, 0, rd),
1105                position_ids,
1106                start_pos,
1107                device,
1108                dtype,
1109            )?;
1110            let query_pass = query.narrow(3, rd, hd - rd);
1111            let query = Tensor::cat(&[query_rot, query_pass], 3);
1112
1113            let k_rot =
1114                self.apply_rope(&k.narrow(3, 0, rd), position_ids, start_pos, device, dtype)?;
1115            let k_pass = k.narrow(3, rd, hd - rd);
1116            let k = Tensor::cat(&[k_rot, k_pass], 3);
1117            (query, k)
1118        } else {
1119            (query, k)
1120        };
1121
1122        // KV cache
1123        let (k_full, v_full) = if let Some(cache) = kv_cache {
1124            cache.update(&k, &v, start_pos)?;
1125            cache.get()?
1126        } else {
1127            (k.shallow_clone(), v.shallow_clone())
1128        };
1129
1130        // Expand KV for GQA
1131        let groups = self.num_heads / self.num_kv_heads;
1132        let k_exp = k_full.repeat_interleave_self_int(groups as i64, 2, None);
1133        let v_exp = v_full.repeat_interleave_self_int(groups as i64, 2, None);
1134
1135        // Manual attention in f32 for numerical stability. For long prefills the
1136        // query axis is processed in chunks so the [q_seq, kv_seq] FP32 score and
1137        // softmax tensors are never materialized at full size — peak VRAM stays
1138        // O(ATTN_CHUNK * kv_seq) rather than O(q_seq * kv_seq). An 8K-token prompt
1139        // otherwise pushes these full-attn layers past 13 GB, and the CUDA caching
1140        // allocator retains that peak, OOMing the next request. The decode path
1141        // (q_seq == 1) and short prompts take the single-shot branch unchanged.
1142        const ATTN_CHUNK: i64 = 1024;
1143        let scale = 1.0 / (self.head_dim as f64).sqrt();
1144        let q_p = query
1145            .permute([0, 2, 1, 3])
1146            .to_kind(Kind::Float)
1147            .contiguous();
1148        let k_p = k_exp
1149            .permute([0, 2, 3, 1])
1150            .to_kind(Kind::Float)
1151            .contiguous();
1152        let v_p = v_exp
1153            .permute([0, 2, 1, 3])
1154            .to_kind(Kind::Float)
1155            .contiguous();
1156
1157        let q_len = q_p.size()[2];
1158        let kv_len = k_p.size()[3];
1159        let ctx = if q_len <= ATTN_CHUNK {
1160            let mut scores = q_p.matmul(&k_p) * scale; // [batch, heads, q_seq, kv_seq]
1161            if q_len > 1 {
1162                // Causal mask for a possibly-cached forward: query row i sits at
1163                // absolute position (kv_len - q_len) + i, so it may attend keys
1164                // 0..=(kv_len - q_len + i). tril(0) would mask out all history
1165                // whenever start_pos > 0 (partial prefill, speculative verify).
1166                let mask =
1167                    Tensor::ones([q_len, kv_len], (Kind::Float, device)).tril(kv_len - q_len);
1168                let mask = mask.unsqueeze(0).unsqueeze(0).expand_as(&scores);
1169                scores = scores.masked_fill(&mask.eq(0.0), -10000.0f64);
1170            }
1171            let attn = scores.softmax(-1, Kind::Float).to_kind(dtype);
1172            attn.matmul(&v_p.to_kind(dtype)) // [batch, heads, q_seq, head_dim]
1173        } else {
1174            // Chunk over the query axis. Per-chunk causal mask
1175            // `tril(kv_len - q_len + start)` is the exact restriction of the
1176            // full causal mask (history offset + row index) to rows
1177            // [start, start+cur).
1178            let v_d = v_p.to_kind(dtype);
1179            let mut outs: Vec<Tensor> = Vec::new();
1180            let mut start = 0i64;
1181            while start < q_len {
1182                let cur = (q_len - start).min(ATTN_CHUNK);
1183                let q_chunk = q_p.narrow(2, start, cur); // [batch, heads, cur, head_dim]
1184                let mut scores = q_chunk.matmul(&k_p) * scale; // [batch, heads, cur, kv_seq]
1185                let mask =
1186                    Tensor::ones([cur, kv_len], (Kind::Float, device)).tril(kv_len - q_len + start);
1187                let mask = mask.unsqueeze(0).unsqueeze(0).expand_as(&scores);
1188                scores = scores.masked_fill(&mask.eq(0.0), -10000.0f64);
1189                let attn = scores.softmax(-1, Kind::Float).to_kind(dtype);
1190                outs.push(attn.matmul(&v_d)); // [batch, heads, cur, head_dim]
1191                start += cur;
1192            }
1193            Tensor::cat(&outs, 2) // [batch, heads, q_seq, head_dim]
1194        };
1195        let ctx = ctx.permute([0, 2, 1, 3]).contiguous(); // [batch, seq, heads, head_dim]
1196
1197        // Apply output gate: gate_flat[batch*seq, nh*hd] * sigmoid(gate) BEFORE o_proj
1198        let ctx_flat = ctx.reshape([batch * seq, nh * hd]);
1199        let gate_flat = gate.reshape([batch * seq, nh * hd]).sigmoid();
1200        let ctx_gated = ctx_flat * gate_flat;
1201
1202        // Delta injection on o_proj (after gate, before linear projection)
1203        let ctx_gated = if let Some((d, layer_idx)) = delta {
1204            if d.has_module("o_proj", layer_idx) {
1205                // #139/#440: coerce fp32 delta to base dtype before adding.
1206                super::add_delta(&ctx_gated, d.forward_2d(&ctx_gated, "o_proj", layer_idx)?)
1207            } else {
1208                ctx_gated
1209            }
1210        } else {
1211            ctx_gated
1212        };
1213
1214        let out = self.o_proj.try_apply(&ctx_gated)?;
1215        Ok(out.reshape([batch, seq, -1]))
1216    }
1217
1218    fn apply_qknorm(&self, x: &Tensor, norm: &Qwen3_5RMSNorm, _nheads: usize) -> Result<Tensor> {
1219        // x: [batch, seq, heads, head_dim]
1220        let orig_shape = x.size();
1221        let (batch, seq, heads, hd) = dims4(x)?;
1222        let flat = x.reshape([batch * seq * heads, hd]);
1223        let orig = flat.kind();
1224        let flat_f = flat.to_kind(Kind::Float);
1225        let mean_sq = (&flat_f * &flat_f).mean_dim(&[-1i64][..], true, Kind::Float);
1226        let rrms = (mean_sq + norm.eps as f64).rsqrt();
1227        let normed = &flat_f * rrms;
1228        // q_norm/k_norm weights are near-0 init — use (1+w)*normed formula (same as Qwen3_5RMSNorm)
1229        let one_plus_w = norm.weight_f32.reshape([1, hd]) + 1.0f64;
1230        let normed_scaled = normed * one_plus_w;
1231        Ok(normed_scaled.to_kind(orig).reshape(orig_shape))
1232    }
1233
1234    fn apply_rope(
1235        &self,
1236        x: &Tensor,
1237        position_ids: Option<&Tensor>,
1238        start_pos: usize,
1239        device: Device,
1240        dtype: Kind,
1241    ) -> Result<Tensor> {
1242        // Thread-local RoPE cache: reuses sin/cos tables across decode steps.
1243        // Saves ~9 GPU kernels per call (generate_embeddings) after the first token.
1244        // Same pattern as LlamaModel::apply_rope. (The static lives at module scope;
1245        // see [`clear_rope_cache`] for the teardown story.)
1246        let device_key: i64 = match device {
1247            Device::Cuda(i) => i as i64,
1248            _ => -1,
1249        };
1250
1251        let rd = self.rotary_dim as i64;
1252        let seq = x.size()[1];
1253        let pos_ids = position_ids.map(|p| p.shallow_clone()).unwrap_or_else(|| {
1254            Tensor::arange_start(
1255                start_pos as i64,
1256                start_pos as i64 + seq,
1257                (Kind::Int64, device),
1258            )
1259        });
1260
1261        ROPE_CACHE.with(|cache| {
1262            let mut cache = cache.borrow_mut();
1263            let key = (self.layer_idx, self.rope_theta.to_bits(), device_key);
1264            let rope = match cache.entry(key) {
1265                std::collections::hash_map::Entry::Occupied(e) => e.into_mut(),
1266                std::collections::hash_map::Entry::Vacant(e) => e.insert(RoPE::new_with_dtype(
1267                    rd,
1268                    self.rope_theta as f64,
1269                    self.max_pos as i64,
1270                    device,
1271                    dtype,
1272                )?),
1273            };
1274            rope.forward(x, Some(&pos_ids))
1275        })
1276    }
1277}
1278
1279// ============================================================================
1280// MoE (Sparse MLP)
1281// ============================================================================
1282
1283struct Qwen3_5Expert {
1284    gate_proj: LinearProjection,
1285    up_proj: LinearProjection,
1286    down_proj: LinearProjection,
1287}
1288
1289impl Qwen3_5Expert {
1290    fn forward(&self, x: &Tensor) -> Result<Tensor> {
1291        let gate = self.gate_proj.try_apply(x)?.silu();
1292        let up = self.up_proj.try_apply(x)?;
1293        self.down_proj.try_apply(&(gate * up))
1294    }
1295
1296    /// Move all projection weights to `device` (#314 pipeline placement).
1297    #[inline]
1298    fn into_device(self, device: Device) -> Self {
1299        Self {
1300            gate_proj: self.gate_proj.into_device(device),
1301            up_proj: self.up_proj.into_device(device),
1302            down_proj: self.down_proj.into_device(device),
1303        }
1304    }
1305}
1306
1307unsafe impl Send for Qwen3_5Expert {}
1308unsafe impl Sync for Qwen3_5Expert {}
1309
1310struct Qwen3_5SparseMoE {
1311    gate: LinearProjection, // router: [hidden, num_experts]
1312    // Stacked expert weights — [num_experts, in_dim, out_dim], kept in FP8 to save VRAM.
1313    // Companion scales — [num_experts, in/128, out/128], BF16. None for non-FP8 weights.
1314    // Dequantized lazily after index_select (only k=8 experts per forward, not all 256).
1315    expert_gate_w: Tensor, // [num_experts, hidden, moe_int]  FP8 or BF16
1316    expert_gate_scale: Option<Tensor>, // [num_experts, hidden/128, moe_int/128]
1317    expert_up_w: Tensor,   // [num_experts, hidden, moe_int]
1318    expert_up_scale: Option<Tensor>,
1319    expert_down_w: Tensor, // [num_experts, moe_int, hidden]
1320    expert_down_scale: Option<Tensor>,
1321    shared_expert: Qwen3_5Expert,
1322    /// Optional gating weight for shared expert: [hidden, 1] (transposed from [1, hidden]).
1323    /// When present: shared_out *= sigmoid(x @ shared_expert_gate)
1324    shared_expert_gate: Option<LinearProjection>,
1325    num_experts_per_tok: usize,
1326}
1327
1328impl Qwen3_5SparseMoE {
1329    fn forward(&self, x: &Tensor) -> Result<Tensor> {
1330        // x: [batch, hidden]
1331        let batch = x.size()[0];
1332        let hidden = x.size()[1];
1333        let k = self.num_experts_per_tok as i64;
1334        let dtype = x.kind();
1335
1336        // Router — topK on raw logits, then softmax over selected experts only.
1337        // Avoids softmax over all 256 experts + renormalize div+sum.
1338        let logits = self.gate.try_apply(x)?; // [batch, num_experts]
1339        let (top_logits, top_ids) = logits.topk(k, -1, true, true);
1340        let top_weights = top_logits.softmax(-1, dtype); // softmax over k=8 only
1341
1342        // Gather expert weights on GPU — no CPU sync.
1343        // top_ids: [batch, k] → [batch*k]
1344        let top_ids_flat = top_ids.reshape([-1]);
1345
1346        // Select k experts and dequantize FP8 → BF16 (only k=8 of 256, not all at once).
1347        let gate_sel = dequant_batched(
1348            &self.expert_gate_w.index_select(0, &top_ids_flat),
1349            self.expert_gate_scale
1350                .as_ref()
1351                .map(|s| s.index_select(0, &top_ids_flat))
1352                .as_ref(),
1353        ); // [batch*k, hidden, moe_int] BF16
1354        let up_sel = dequant_batched(
1355            &self.expert_up_w.index_select(0, &top_ids_flat),
1356            self.expert_up_scale
1357                .as_ref()
1358                .map(|s| s.index_select(0, &top_ids_flat))
1359                .as_ref(),
1360        );
1361
1362        // Repeat input k times: [batch*k, hidden]
1363        let x_rep = if x.kind() != tch::Kind::BFloat16 {
1364            x.repeat_interleave_self_int(k, 0, None)
1365                .to_kind(tch::Kind::BFloat16)
1366        } else {
1367            x.repeat_interleave_self_int(k, 0, None)
1368        };
1369
1370        // Fused gate+up BMM: cat weights along output dim, single BMM, then split
1371        let moe_int = gate_sel.size()[2];
1372        let gate_up_sel = Tensor::cat(&[&gate_sel, &up_sel], 2); // [batch*k, hidden, 2*moe_int]
1373        let x_3d = x_rep.unsqueeze(1); // [batch*k, 1, hidden]
1374        let gate_up = x_3d.bmm(&gate_up_sel).squeeze_dim(1); // [batch*k, 2*moe_int]
1375        let gate_out = gate_up.narrow(1, 0, moe_int).silu();
1376        let up_out = gate_up.narrow(1, moe_int, moe_int);
1377
1378        // [batch*k, 1, moe_int] × [batch*k, moe_int, hidden] → [batch*k, hidden]
1379        let down_sel = dequant_batched(
1380            &self.expert_down_w.index_select(0, &top_ids_flat),
1381            self.expert_down_scale
1382                .as_ref()
1383                .map(|s| s.index_select(0, &top_ids_flat))
1384                .as_ref(),
1385        ); // [batch*k, moe_int, hidden] BF16
1386        let expert_out = (gate_out * up_out)
1387            .unsqueeze(1)
1388            .bmm(&down_sel)
1389            .squeeze_dim(1);
1390
1391        // Weighted sum: [batch, k, hidden], then reduce over k.
1392        let expert_out = expert_out.reshape([batch, k, hidden]);
1393        let output = (expert_out * top_weights.unsqueeze(-1).to_kind(tch::Kind::BFloat16))
1394            .sum_dim_intlist(&[1i64][..], false, None);
1395        let output = if dtype != tch::Kind::BFloat16 {
1396            output.to_kind(dtype)
1397        } else {
1398            output
1399        };
1400
1401        let shared_out = self.shared_expert.forward(x)?;
1402        let shared_out = if let Some(gate_proj) = &self.shared_expert_gate {
1403            // sigmoid(x @ gate_proj): [batch, 1], broadcast over hidden dim
1404            let gate = gate_proj.try_apply(x)?.sigmoid();
1405            shared_out * gate
1406        } else {
1407            shared_out
1408        };
1409
1410        Ok(output + shared_out)
1411    }
1412
1413    /// Move the router, stacked expert weights/scales, and shared expert to
1414    /// `device` (#314 pipeline placement).
1415    #[inline]
1416    fn into_device(self, device: Device) -> Self {
1417        Self {
1418            gate: self.gate.into_device(device),
1419            expert_gate_w: self.expert_gate_w.to_device(device),
1420            expert_gate_scale: self.expert_gate_scale.map(|s| s.to_device(device)),
1421            expert_up_w: self.expert_up_w.to_device(device),
1422            expert_up_scale: self.expert_up_scale.map(|s| s.to_device(device)),
1423            expert_down_w: self.expert_down_w.to_device(device),
1424            expert_down_scale: self.expert_down_scale.map(|s| s.to_device(device)),
1425            shared_expert: self.shared_expert.into_device(device),
1426            shared_expert_gate: self.shared_expert_gate.map(|g| g.into_device(device)),
1427            num_experts_per_tok: self.num_experts_per_tok,
1428        }
1429    }
1430}
1431
1432unsafe impl Send for Qwen3_5SparseMoE {}
1433unsafe impl Sync for Qwen3_5SparseMoE {}
1434
1435// ============================================================================
1436// MLP enum (Dense or Sparse)
1437// ============================================================================
1438
1439enum Qwen3_5Mlp {
1440    Dense(Box<LlamaMLP>),
1441    Sparse(Box<Qwen3_5SparseMoE>),
1442}
1443
1444impl Qwen3_5Mlp {
1445    fn forward(&self, x: &Tensor) -> Result<Tensor> {
1446        match self {
1447            Qwen3_5Mlp::Dense(mlp) => mlp.forward(x, None),
1448            Qwen3_5Mlp::Sparse(moe) => {
1449                let orig = x.size();
1450                let batch = orig[0];
1451                let seq = orig[1];
1452                let h = orig[2];
1453                let flat = x.reshape([batch * seq, h]);
1454                let out = moe.forward(&flat)?;
1455                Ok(out.reshape(orig))
1456            }
1457        }
1458    }
1459
1460    /// Move the dense or sparse MLP weights to `device` (#314 pipeline placement).
1461    #[inline]
1462    fn into_device(self, device: Device) -> Self {
1463        match self {
1464            Qwen3_5Mlp::Dense(mlp) => Qwen3_5Mlp::Dense(Box::new((*mlp).into_device(device))),
1465            Qwen3_5Mlp::Sparse(moe) => Qwen3_5Mlp::Sparse(Box::new((*moe).into_device(device))),
1466        }
1467    }
1468}
1469
1470unsafe impl Send for Qwen3_5Mlp {}
1471unsafe impl Sync for Qwen3_5Mlp {}
1472
1473// ============================================================================
1474// Transformer layer
1475// ============================================================================
1476
1477enum LayerMixer {
1478    LinearAttn(GatedDeltaNetLayer),
1479    FullAttn(Qwen3_5FullAttention),
1480}
1481
1482impl LayerMixer {
1483    /// Move the mixer's weights to `device` (#314 pipeline placement).
1484    #[inline]
1485    fn into_device(self, device: Device) -> Self {
1486        match self {
1487            LayerMixer::LinearAttn(gdn) => LayerMixer::LinearAttn(gdn.into_device(device)),
1488            LayerMixer::FullAttn(attn) => LayerMixer::FullAttn(attn.into_device(device)),
1489        }
1490    }
1491}
1492
1493unsafe impl Send for LayerMixer {}
1494unsafe impl Sync for LayerMixer {}
1495
1496struct Qwen3_5Layer {
1497    mixer: LayerMixer,
1498    mlp: Qwen3_5Mlp,
1499    input_layernorm: Qwen3_5RMSNorm,
1500    post_attention_layernorm: Qwen3_5RMSNorm,
1501}
1502
1503impl Qwen3_5Layer {
1504    /// Move every weight in this layer (mixer + MLP + both norms) to `device`
1505    /// (#314 pipeline placement). A no-op per tensor already resident on `device`.
1506    #[inline]
1507    fn into_device(self, device: Device) -> Self {
1508        Self {
1509            mixer: self.mixer.into_device(device),
1510            mlp: self.mlp.into_device(device),
1511            input_layernorm: self.input_layernorm.into_device(device),
1512            post_attention_layernorm: self.post_attention_layernorm.into_device(device),
1513        }
1514    }
1515}
1516
1517unsafe impl Send for Qwen3_5Layer {}
1518unsafe impl Sync for Qwen3_5Layer {}
1519
1520// ============================================================================
1521// MTP (multi-token prediction) head — self-speculative draft module
1522// ============================================================================
1523
1524/// Qwen3.5/3.8 dense checkpoints ship a 1-layer MTP head that predicts token
1525/// t+2 from the pair (main-model hidden after token t, embedding of token t+1):
1526///
1527/// ```text
1528/// e = pre_fc_norm_embedding(embed(token_ids[i]))
1529/// h = pre_fc_norm_hidden(hiddens[i])
1530/// x = fc(cat([e, h], -1))            # embedding FIRST, hidden second
1531/// x = mtp_layer(x)                   # full-attention decoder block (never GDN)
1532/// logits = lm_head(mtp.norm(x))      # shared embed_tokens/lm_head
1533/// ```
1534///
1535/// Slot `i` runs at the RoPE/KV position of the main-model token whose hidden
1536/// state it consumes (`base + i`). The block has its own KV cache stored in the
1537/// shared [`KVCacheManager`] at layer index `num_hidden_layers` (right after the
1538/// main layers), so manager-wide `clear_all`/`truncate_to`/session swaps cover
1539/// it automatically. FP8 checkpoints carry `_scale_inv` block scales for the 7
1540/// attention/MLP linears (handled lazily by [`LinearProjection`]) but NOT for
1541/// `mtp.fc`, which stays BF16.
1542///
1543/// v1 is dense-only: MoE checkpoints carry a full MoE MLP in the MTP block
1544/// (~785 tensors) which is a deliberate follow-up — the loader falls back to
1545/// non-speculative decode there.
1546struct Qwen3_5MtpHead {
1547    pre_fc_norm_hidden: Qwen3_5RMSNorm,
1548    pre_fc_norm_embedding: Qwen3_5RMSNorm,
1549    fc: LinearProjection, // in: 2*hidden, out: hidden (no bias)
1550    attn: Qwen3_5FullAttention,
1551    mlp: LlamaMLP, // dense SwiGLU
1552    input_layernorm: Qwen3_5RMSNorm,
1553    post_attention_layernorm: Qwen3_5RMSNorm,
1554    final_norm: Qwen3_5RMSNorm, // mtp.norm — before the shared lm_head
1555}
1556
1557unsafe impl Send for Qwen3_5MtpHead {}
1558unsafe impl Sync for Qwen3_5MtpHead {}
1559
1560impl Qwen3_5MtpHead {
1561    /// Load the dense MTP head, consuming all `mtp.*` keys from `weights`.
1562    /// Caller must have already rejected MoE configs (checked by `from_weights`).
1563    fn load(weights: &mut HashMap<String, Tensor>, cfg: &Qwen3_5TextConfig) -> Result<Self> {
1564        fn take_norm(
1565            weights: &mut HashMap<String, Tensor>,
1566            name: &str,
1567            eps: f32,
1568        ) -> Result<Qwen3_5RMSNorm> {
1569            let w = weights
1570                .remove(name)
1571                .ok_or_else(|| anyhow!("Missing weight: {name}"))?;
1572            Ok(Qwen3_5RMSNorm::new(w, eps))
1573        }
1574
1575        let layer_prefix = "mtp.layers.0";
1576        // The MTP block is always a full-attention layer (never GatedDeltaNet).
1577        // Give it layer_idx = num_hidden_layers so its RoPE cache entry and KV
1578        // slot sit right after the main layers.
1579        let attn = Qwen3_5FullAttention::load(
1580            weights,
1581            &format!("{layer_prefix}.self_attn"),
1582            cfg,
1583            cfg.num_hidden_layers as usize,
1584        )?;
1585        let mlp_prefix = format!("{layer_prefix}.mlp");
1586        let mlp = LlamaMLP {
1587            gate_proj: LinearProjection::take(weights, &format!("{mlp_prefix}.gate_proj.weight"))?,
1588            up_proj: LinearProjection::take(weights, &format!("{mlp_prefix}.up_proj.weight"))?,
1589            down_proj: LinearProjection::take(weights, &format!("{mlp_prefix}.down_proj.weight"))?,
1590            activation: "silu".to_owned(),
1591            layer_idx: cfg.num_hidden_layers as usize,
1592        };
1593        let eps = cfg.rms_norm_eps;
1594
1595        Ok(Self {
1596            pre_fc_norm_hidden: take_norm(weights, "mtp.pre_fc_norm_hidden.weight", eps)?,
1597            pre_fc_norm_embedding: take_norm(weights, "mtp.pre_fc_norm_embedding.weight", eps)?,
1598            fc: LinearProjection::take(weights, "mtp.fc.weight")?,
1599            attn,
1600            mlp,
1601            input_layernorm: take_norm(
1602                weights,
1603                &format!("{layer_prefix}.input_layernorm.weight"),
1604                eps,
1605            )?,
1606            post_attention_layernorm: take_norm(
1607                weights,
1608                &format!("{layer_prefix}.post_attention_layernorm.weight"),
1609                eps,
1610            )?,
1611            final_norm: take_norm(weights, "mtp.norm.weight", eps)?,
1612        })
1613    }
1614}
1615
1616// ============================================================================
1617// Top-level model
1618// ============================================================================
1619
1620pub struct Qwen3_5Model {
1621    config: Qwen3_5TextConfig,
1622    device: Device,
1623    dtype: Kind,
1624
1625    /// Per-global-layer device assignment (#314, 2b pipeline). For the unsplit
1626    /// fast path this is a single-device map of length `num_hidden_layers`
1627    /// (`is_single_device() == true`); for a pipeline shard it is the *global*
1628    /// map (still length `num_hidden_layers`) and `layer_offset` selects the owned
1629    /// window. Drives both per-layer placement at construction and the lone
1630    /// boundary `to_device` in `forward_layers`. Mirrors `LlamaModel`.
1631    device_map: LayerDeviceMap,
1632
1633    /// Global index of `self.layers[0]`. `0` for a whole model; `a` for a shard
1634    /// owning global layers `[a..a+self.layers.len())`. Used to remap a global
1635    /// layer index to its local slot in `self.layers` / KV / conv-rec state, and
1636    /// to pick the correct per-global-index `layer_types` entry at load.
1637    layer_offset: usize,
1638
1639    embed_tokens: Tensor,
1640    layers: Vec<Qwen3_5Layer>,
1641    norm: Qwen3_5RMSNorm,
1642    lm_head: Option<Tensor>,            // None = tied to embed_tokens
1643    lm_head_transposed: Option<Tensor>, // pre-transposed tied weights
1644    // Interior-mutable SSM state (required because forward_with_cache takes &self).
1645    // Sized to the OWNED layer count (== self.layers.len()), indexed by LOCAL
1646    // layer index. Stage-local: never transferred across a pipeline boundary.
1647    conv_states: Arc<parking_lot::Mutex<Vec<Option<Tensor>>>>,
1648    rec_states: Arc<parking_lot::Mutex<Vec<Option<Tensor>>>>,
1649    kv_cache: Option<Arc<parking_lot::Mutex<KVCacheManager>>>,
1650    /// MTP draft head (dense checkpoints only). Its KV cache lives in the shared
1651    /// manager at layer index `num_hidden_layers` (see [`Qwen3_5MtpHead`]).
1652    mtp: Option<Qwen3_5MtpHead>,
1653    // Vision encoder (optional — None for text-only weights)
1654    vision_encoder: Option<Qwen3_5VisionEncoder>,
1655    // Linear projection: vision out_hidden_size → text hidden_size
1656    vision_projector: Option<LinearProjection>,
1657}
1658
1659unsafe impl Send for Qwen3_5Model {}
1660unsafe impl Sync for Qwen3_5Model {}
1661
1662impl Qwen3_5Model {
1663    /// Construct from caller-owned tensors. Disabled speculation discards any
1664    /// MTP tensors already supplied; use ModelFactory to filter them before
1665    /// materialization and avoid their temporary device allocation as well.
1666    pub fn from_weights(
1667        weights: &mut HashMap<String, Tensor>,
1668        cfg: Qwen3_5TextConfig,
1669        vision_cfg: Option<Qwen3_5VisionConfig>,
1670        device: &Device,
1671        dtype: Kind,
1672        kv_quant_type: KVQuantType,
1673        speculative_decoding: bool,
1674    ) -> Result<Self> {
1675        // Normalize weight key prefixes:
1676        // Qwen3.5 Instruct weights use "model.language_model." and "model.visual." prefixes
1677        // Strip these so the rest of the loader sees the expected flat names
1678        let needs_remap = weights
1679            .keys()
1680            .any(|k| k.starts_with("model.language_model."));
1681        if needs_remap {
1682            info!("Remapping Qwen3.5 weight prefixes (model.language_model.* → *, model.visual.* → visual.*)");
1683            let remapped: HashMap<String, Tensor> = weights
1684                .drain()
1685                .map(|(k, v)| {
1686                    // model.language_model.X → model.X  (restores expected prefix)
1687                    // model.visual.X        → visual.X  (handled by vision loader)
1688                    let new_key = if let Some(rest) = k.strip_prefix("model.language_model.") {
1689                        format!("model.{rest}")
1690                    } else if let Some(rest) = k.strip_prefix("model.visual.") {
1691                        format!("visual.{rest}")
1692                    } else {
1693                        k
1694                    };
1695                    (new_key, v)
1696                })
1697                .collect();
1698            *weights = remapped;
1699        }
1700
1701        // Load vision encoder if visual weights are present
1702        let has_vision = weights.keys().any(|k| k.starts_with("visual."));
1703        let (vision_encoder, vision_projector) = if has_vision {
1704            if let Some(vcfg) = vision_cfg {
1705                info!(
1706                    "Loading Qwen3.5 vision encoder (out_hidden_size={})",
1707                    vcfg.out_hidden_size
1708                );
1709                let out_hidden = vcfg.out_hidden_size;
1710                let text_hidden = cfg.hidden_size;
1711                // Best-effort: vision key naming varies across model sizes; skip on error
1712                match Qwen3_5VisionEncoder::from_weights(weights, vcfg) {
1713                    Ok(enc) => {
1714                        let proj_key = if weights.contains_key("visual_projector.weight") {
1715                            Some("visual_projector.weight")
1716                        } else if weights.contains_key("vision_projector.weight") {
1717                            Some("vision_projector.weight")
1718                        } else {
1719                            None
1720                        };
1721                        let projector = if let Some(key) = proj_key {
1722                            Some(LinearProjection::take(weights, key)?)
1723                        } else {
1724                            info!("No vision_projector weights found; vision output will be zeros");
1725                            let w = Tensor::zeros(
1726                                [out_hidden as i64, text_hidden as i64],
1727                                (dtype, *device),
1728                            );
1729                            Some(LinearProjection::new(w))
1730                        };
1731                        (Some(enc), projector)
1732                    }
1733                    Err(e) => {
1734                        info!("Vision encoder load skipped (key naming mismatch, text-only mode): {e}");
1735                        weights.retain(|k, _| !k.starts_with("visual."));
1736                        (None, None)
1737                    }
1738                }
1739            } else {
1740                // No vision config provided — discard visual weights
1741                weights.retain(|k, _| !k.starts_with("visual."));
1742                (None, None)
1743            }
1744        } else {
1745            (None, None)
1746        };
1747
1748        // MTP (multi-token prediction) head: 1-layer self-speculative draft module.
1749        // v1 loads dense checkpoints only; MoE MTP blocks (~785 tensors) are a
1750        // documented follow-up — fall back to non-speculative decode there.
1751        let mtp = if !speculative_decoding || kv_quant_type != KVQuantType::None {
1752            // Factory loading filters these before device allocation. Direct
1753            // callers may supply already-materialized tensors; discard them
1754            // here too rather than retaining a disabled/unusable draft head.
1755            weights.retain(|k, _| !k.starts_with("mtp."));
1756            None
1757        } else if weights.keys().any(|k| k.starts_with("mtp.")) {
1758            if cfg.is_moe {
1759                info!("MTP head present but MoE MTP is not yet supported; skipping mtp.* weights (non-speculative decode)");
1760                weights.retain(|k, _| !k.starts_with("mtp."));
1761                None
1762            } else if weights.keys().any(|k| k.starts_with("mtp.layers.1.")) {
1763                info!("Multi-layer MTP (mtp_num_hidden_layers > 1) is not supported; skipping mtp.* weights");
1764                weights.retain(|k, _| !k.starts_with("mtp."));
1765                None
1766            } else {
1767                match Qwen3_5MtpHead::load(weights, &cfg) {
1768                    Ok(head) => {
1769                        info!(
1770                            "Loaded Qwen3.5 MTP draft head (self-speculative decoding available)"
1771                        );
1772                        Some(head)
1773                    }
1774                    Err(e) => {
1775                        info!(
1776                            "MTP head load failed ({e}); continuing without speculative decoding"
1777                        );
1778                        weights.retain(|k, _| !k.starts_with("mtp."));
1779                        None
1780                    }
1781                }
1782            }
1783        } else {
1784            None
1785        };
1786
1787        // #405: place top-level weights on `device`, mirroring the
1788        // `into_device` move that `stage_from_weights_with_config` applies.
1789        // Without this, constructing from CPU weights with device=Cuda(0)
1790        // leaves embed/norm/lm_head on CPU — a mixed-device model.
1791        let embed = weights
1792            .remove("model.embed_tokens.weight")
1793            .ok_or_else(|| anyhow!("Missing model.embed_tokens.weight"))?
1794            .to_device(*device);
1795        let norm_w = weights
1796            .remove("model.norm.weight")
1797            .ok_or_else(|| anyhow!("Missing model.norm.weight"))?
1798            .to_device(*device);
1799        let lm_head_w = if let Some(w) = weights.remove("lm_head.weight") {
1800            // Cast FP8 lm_head to BF16, applying the companion block scale if present.
1801            // Without this the raw FP8 values (~448) are used instead of the true weights (~0.09),
1802            // which gives completely wrong logit distributions.
1803            match w.kind() {
1804                tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
1805                    let w_bf = w.to_kind(tch::Kind::BFloat16);
1806                    if let Some(s) = weights.remove("lm_head.weight_scale_inv") {
1807                        // scale stored as [vocab/128, hidden/128]; apply block-wise
1808                        let ws = w_bf.size(); // [vocab, hidden]
1809                        let ss = s.size(); // [vocab/128, hidden/128]
1810                        let br = ws[0] / ss[0];
1811                        let bc = ws[1] / ss[1];
1812                        let w_4d = w_bf.view([ss[0], br, ss[1], bc]);
1813                        let s_4d = s
1814                            .to_kind(tch::Kind::BFloat16)
1815                            .view([ss[0], 1i64, ss[1], 1i64]);
1816                        Some((w_4d * s_4d).reshape([ws[0], ws[1]]).to_device(*device))
1817                    } else {
1818                        Some(w_bf.to_device(*device))
1819                    }
1820                }
1821                _ => Some(w.to_device(*device)),
1822            }
1823        } else {
1824            None
1825        };
1826
1827        let num_layers = cfg.num_hidden_layers as usize;
1828        let mut layers = Vec::with_capacity(num_layers);
1829
1830        for idx in 0..num_layers {
1831            layers.push(Self::build_layer(weights, &cfg, idx)?);
1832        }
1833
1834        let lm_head_transposed = if lm_head_w.is_none() {
1835            // embed may be FP8; cast to BF16 for matmul
1836            let embed_bf16 = match embed.kind() {
1837                tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
1838                    embed.to_kind(tch::Kind::BFloat16)
1839                }
1840                _ => embed.shallow_clone(),
1841            };
1842            Some(embed_bf16.tr().contiguous())
1843        } else {
1844            None
1845        };
1846
1847        let conv_states = Arc::new(parking_lot::Mutex::new(
1848            (0..num_layers).map(|_| None::<Tensor>).collect::<Vec<_>>(),
1849        ));
1850        let rec_states = Arc::new(parking_lot::Mutex::new(
1851            (0..num_layers).map(|_| None::<Tensor>).collect::<Vec<_>>(),
1852        ));
1853
1854        // Create KV cache for full-attention layers (same pattern as LlamaModel).
1855        // When an MTP head is loaded, one extra layer slot (index num_layers)
1856        // holds the MTP block's KV, so manager-wide clear/truncate/session-swap
1857        // cover it without special casing.
1858        let kv_cache = Some(Arc::new(parking_lot::Mutex::new(
1859            crate::runtime::kv_cache::KVCacheManager::new(
1860                num_layers + usize::from(mtp.is_some()),
1861                cfg.max_position_embeddings as usize,
1862                kv_quant_type,
1863            ),
1864        )));
1865
1866        // Unsplit fast path: every layer on the one device. This keeps the
1867        // whole-model forward byte-identical — `forward_layers` over this
1868        // single-device map performs zero cross-device copies (#314).
1869        let device_map = LayerDeviceMap::single(*device, num_layers.max(1))?;
1870
1871        Ok(Self {
1872            config: cfg,
1873            device: *device,
1874            dtype,
1875            device_map,
1876            layer_offset: 0,
1877            embed_tokens: embed,
1878            layers,
1879            norm: Qwen3_5RMSNorm::new(norm_w, 1e-6),
1880            lm_head: lm_head_w,
1881            lm_head_transposed,
1882            conv_states,
1883            rec_states,
1884            kv_cache,
1885            mtp,
1886            vision_encoder,
1887            vision_projector,
1888        })
1889    }
1890
1891    fn load_mlp(
1892        weights: &mut HashMap<String, Tensor>,
1893        prefix: &str,
1894        cfg: &Qwen3_5TextConfig,
1895        layer_idx: usize,
1896    ) -> Result<Qwen3_5Mlp> {
1897        if cfg.is_moe {
1898            // gate: [num_experts, hidden] → [hidden, num_experts]
1899            let gate_proj = LinearProjection::take(weights, &format!("{prefix}.gate.weight"))?;
1900
1901            // Stack expert weights as [num_experts, in_dim, out_dim] for GPU-native BMM dispatch.
1902            // Keep FP8 weights as-is; stack companion scales as [num_experts, in/128, out/128].
1903            // Dequantization happens lazily after index_select in forward() — only k=8 experts,
1904            // not all 256, keeping peak VRAM at FP8 size (~35 GB) instead of BF16 (~70 GB).
1905            let n = cfg.num_experts;
1906            let mut gate_w_vecs: Vec<Tensor> = Vec::with_capacity(n);
1907            let mut gate_s_vecs: Vec<Tensor> = Vec::with_capacity(n);
1908            let mut up_w_vecs: Vec<Tensor> = Vec::with_capacity(n);
1909            let mut up_s_vecs: Vec<Tensor> = Vec::with_capacity(n);
1910            let mut down_w_vecs: Vec<Tensor> = Vec::with_capacity(n);
1911            let mut down_s_vecs: Vec<Tensor> = Vec::with_capacity(n);
1912            for e in 0..n {
1913                let ep = format!("{prefix}.experts.{e}");
1914                // These tensors go straight into batched stacks, never apply().
1915                // Avoid constructing per-projection acceleration metadata.
1916                let (g, gs) = LinearProjection::take_weight_and_scale(
1917                    weights,
1918                    &format!("{ep}.gate_proj.weight"),
1919                )?;
1920                let (u, us) = LinearProjection::take_weight_and_scale(
1921                    weights,
1922                    &format!("{ep}.up_proj.weight"),
1923                )?;
1924                let (d, ds) = LinearProjection::take_weight_and_scale(
1925                    weights,
1926                    &format!("{ep}.down_proj.weight"),
1927                )?;
1928                gate_w_vecs.push(g.unsqueeze(0));
1929                if let Some(s) = gs {
1930                    gate_s_vecs.push(s.unsqueeze(0));
1931                }
1932                up_w_vecs.push(u.unsqueeze(0));
1933                if let Some(s) = us {
1934                    up_s_vecs.push(s.unsqueeze(0));
1935                }
1936                down_w_vecs.push(d.unsqueeze(0));
1937                if let Some(s) = ds {
1938                    down_s_vecs.push(s.unsqueeze(0));
1939                }
1940            }
1941            // Stack: [num_experts, in, out] (FP8) + optional [num_experts, in/128, out/128] scale
1942            let expert_gate_w = Tensor::cat(&gate_w_vecs.iter().collect::<Vec<_>>(), 0);
1943            let expert_gate_scale = (!gate_s_vecs.is_empty())
1944                .then(|| Tensor::cat(&gate_s_vecs.iter().collect::<Vec<_>>(), 0));
1945            let expert_up_w = Tensor::cat(&up_w_vecs.iter().collect::<Vec<_>>(), 0);
1946            let expert_up_scale = (!up_s_vecs.is_empty())
1947                .then(|| Tensor::cat(&up_s_vecs.iter().collect::<Vec<_>>(), 0));
1948            let expert_down_w = Tensor::cat(&down_w_vecs.iter().collect::<Vec<_>>(), 0);
1949            let expert_down_scale = (!down_s_vecs.is_empty())
1950                .then(|| Tensor::cat(&down_s_vecs.iter().collect::<Vec<_>>(), 0));
1951
1952            let sp = format!("{prefix}.shared_expert");
1953            let shared = Qwen3_5Expert {
1954                gate_proj: LinearProjection::take(weights, &format!("{sp}.gate_proj.weight"))?,
1955                up_proj: LinearProjection::take(weights, &format!("{sp}.up_proj.weight"))?,
1956                down_proj: LinearProjection::take(weights, &format!("{sp}.down_proj.weight"))?,
1957            };
1958            // Optional gating scalar for shared expert (present in 35B FP8, not all models).
1959            let shared_expert_gate =
1960                LinearProjection::take(weights, &format!("{prefix}.shared_expert_gate.weight"))
1961                    .ok();
1962
1963            Ok(Qwen3_5Mlp::Sparse(Box::new(Qwen3_5SparseMoE {
1964                gate: gate_proj,
1965                expert_gate_w,
1966                expert_gate_scale,
1967                expert_up_w,
1968                expert_up_scale,
1969                expert_down_w,
1970                expert_down_scale,
1971                shared_expert: shared,
1972                shared_expert_gate,
1973                num_experts_per_tok: cfg.num_experts_per_tok,
1974            })))
1975        } else {
1976            Ok(Qwen3_5Mlp::Dense(Box::new(LlamaMLP {
1977                gate_proj: LinearProjection::take(weights, &format!("{prefix}.gate_proj.weight"))?,
1978                up_proj: LinearProjection::take(weights, &format!("{prefix}.up_proj.weight"))?,
1979                down_proj: LinearProjection::take(weights, &format!("{prefix}.down_proj.weight"))?,
1980                activation: "silu".to_owned(),
1981                layer_idx,
1982            })))
1983        }
1984    }
1985
1986    /// Build a single decoder layer for **global** index `g`, selecting the
1987    /// hybrid mixer (GatedDeltaNet vs full attention) by the *global* per-layer
1988    /// type (`cfg.layer_types[g]`). Shared by the whole-model loader and the
1989    /// pipeline stage loader — the latter passes the global index so a shard that
1990    /// owns `[a..b)` still picks the correct type for each layer it owns.
1991    fn build_layer(
1992        weights: &mut HashMap<String, Tensor>,
1993        cfg: &Qwen3_5TextConfig,
1994        g: usize,
1995    ) -> Result<Qwen3_5Layer> {
1996        let layer_prefix = format!("model.layers.{g}");
1997
1998        let input_norm_w = weights
1999            .remove(&format!("{layer_prefix}.input_layernorm.weight"))
2000            .ok_or_else(|| anyhow!("Missing {layer_prefix}.input_layernorm.weight"))?;
2001        let post_norm_w = weights
2002            .remove(&format!("{layer_prefix}.post_attention_layernorm.weight"))
2003            .ok_or_else(|| anyhow!("Missing {layer_prefix}.post_attention_layernorm.weight"))?;
2004
2005        // Hybrid layer type is selected by GLOBAL index: every Nth layer is full
2006        // attention, the rest GatedDeltaNet. A stage owning [a..b) must consult
2007        // the global index `g`, not a local one, to load the right mixer.
2008        let layer_type = cfg
2009            .layer_types
2010            .get(g)
2011            .map(|s| s.as_str())
2012            .unwrap_or("linear_attention");
2013        info!("Loading layer {g}: {layer_type}");
2014
2015        let mixer = if layer_type == "full_attention" {
2016            let attn_prefix = format!("{layer_prefix}.self_attn");
2017            LayerMixer::FullAttn(Qwen3_5FullAttention::load(weights, &attn_prefix, cfg, g)?)
2018        } else {
2019            let lin_prefix = format!("{layer_prefix}.linear_attn");
2020            LayerMixer::LinearAttn(GatedDeltaNetLayer::load(weights, &lin_prefix, cfg, g)?)
2021        };
2022
2023        let mlp = Self::load_mlp(weights, &format!("{layer_prefix}.mlp"), cfg, g)?;
2024
2025        Ok(Qwen3_5Layer {
2026            mixer,
2027            mlp,
2028            input_layernorm: Qwen3_5RMSNorm::new(input_norm_w, cfg.rms_norm_eps),
2029            post_attention_layernorm: Qwen3_5RMSNorm::new(post_norm_w, cfg.rms_norm_eps),
2030        })
2031    }
2032
2033    /// Build a **single pipeline stage** of a Qwen3.5 model (#314, 2b layer-split).
2034    ///
2035    /// Mirrors `LlamaModel::stage_from_weights_with_config`, plus Qwen3.5's
2036    /// specifics:
2037    /// - **Hybrid layer type** is chosen by the *global* layer index inside
2038    ///   [`Self::build_layer`] (`cfg.layer_types[g]`), so a shard owning `[a..b)`
2039    ///   loads the correct GDN/full-attn mixer per global layer.
2040    /// - **SSM state** (`conv_states`/`rec_states`) is sized to the OWNED layer
2041    ///   count (`layer_range.len()`), indexed by the LOCAL layer index, and is
2042    ///   stage-local — never transferred across a pipeline boundary (M-LOAD seam
2043    ///   #2). Only `hidden` + `start_pos` cross a boundary.
2044    ///
2045    /// Non-layer weights are gated by stage position (M-LOAD seam #1):
2046    /// - `is_first` (`layer_range.start == 0`)  → keep `embed_tokens`.
2047    /// - `is_last`  (`layer_range.end == num_hidden_layers`) → keep `norm` +
2048    ///   `lm_head` (or the tied transpose).
2049    ///
2050    /// Each owned layer is placed on its mapped device (`devices.device_for(g)`)
2051    /// via the `into_device` builders — the only placement cost; the forward path
2052    /// then performs zero intra-stage copies. Vision weights are out of scope for
2053    /// a pipeline stage (text backbone split only).
2054    #[allow(clippy::too_many_arguments)]
2055    pub fn stage_from_weights_with_config(
2056        weights: &mut HashMap<String, Tensor>,
2057        cfg: Qwen3_5TextConfig,
2058        devices: &LayerDeviceMap,
2059        layer_range: std::ops::Range<usize>,
2060        dtype: Kind,
2061        kv_quant_type: KVQuantType,
2062    ) -> Result<Self> {
2063        let num_global = cfg.num_hidden_layers as usize;
2064        if devices.len() != num_global {
2065            return Err(anyhow!(
2066                "stage_from_weights: device map covers {} layers but config has {} \
2067                 (num_hidden_layers)",
2068                devices.len(),
2069                num_global
2070            ));
2071        }
2072        if layer_range.end > num_global || layer_range.start >= layer_range.end {
2073            return Err(anyhow!(
2074                "stage_from_weights: invalid layer range {:?} for {} layers",
2075                layer_range,
2076                num_global
2077            ));
2078        }
2079
2080        let is_first = layer_range.start == 0;
2081        let is_last = layer_range.end == num_global;
2082        let layer_offset = layer_range.start;
2083        // Stage device = device of this stage's first owned layer (its inputs and
2084        // KV/SSM state live here). `embed_tokens`, when present, lives on the first
2085        // stage's first device, which is exactly this when is_first.
2086        let stage_device = devices.device_for(layer_range.start);
2087
2088        // --- Non-layer weights, gated by stage position (M-LOAD seam #1) ---
2089        let embed = if is_first {
2090            weights
2091                .remove("model.embed_tokens.weight")
2092                .map(|w| w.to_device(stage_device))
2093                .ok_or_else(|| {
2094                    anyhow!("stage_from_weights: first stage requires model.embed_tokens.weight")
2095                })?
2096        } else {
2097            // Middle/last stage owns no embedding; a placeholder is never read
2098            // (embed_tokens()/forward_inner from-ids paths run only on stage 0).
2099            Tensor::zeros([1, cfg.hidden_size as i64], (dtype, stage_device))
2100        };
2101
2102        let (norm, lm_head_w) = if is_last {
2103            let norm_w = weights
2104                .remove("model.norm.weight")
2105                .map(|w| w.to_device(stage_device))
2106                .ok_or_else(|| {
2107                    anyhow!("stage_from_weights: last stage requires model.norm.weight")
2108                })?;
2109            let lm_head_w = weights.remove("lm_head.weight").map(|w| {
2110                // Cast FP8 lm_head to BF16 (applying companion scale), mirroring
2111                // from_weights. On the test path (plain f32) this is a no-op clone.
2112                match w.kind() {
2113                    tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
2114                        let w_bf = w.to_kind(tch::Kind::BFloat16);
2115                        if let Some(s) = weights.remove("lm_head.weight_scale_inv") {
2116                            let ws = w_bf.size();
2117                            let ss = s.size();
2118                            let br = ws[0] / ss[0];
2119                            let bc = ws[1] / ss[1];
2120                            let w_4d = w_bf.view([ss[0], br, ss[1], bc]);
2121                            let s_4d = s
2122                                .to_kind(tch::Kind::BFloat16)
2123                                .view([ss[0], 1i64, ss[1], 1i64]);
2124                            (w_4d * s_4d)
2125                                .reshape([ws[0], ws[1]])
2126                                .to_device(stage_device)
2127                        } else {
2128                            w_bf.to_device(stage_device)
2129                        }
2130                    }
2131                    _ => w.to_device(stage_device),
2132                }
2133            });
2134            (Some(Qwen3_5RMSNorm::new(norm_w, 1e-6)), lm_head_w)
2135        } else {
2136            (None, None)
2137        };
2138
2139        // Tied lm_head: only on the last stage, and only if it also holds the
2140        // embedding (single-stage model). A multi-stage last shard must ship an
2141        // explicit lm_head since it does not own embed_tokens.
2142        let lm_head_transposed = if is_last && lm_head_w.is_none() {
2143            if !is_first {
2144                return Err(anyhow!(
2145                    "stage_from_weights: last stage requires lm_head.weight (no tied embedding present)"
2146                ));
2147            }
2148            let embed_bf16 = match embed.kind() {
2149                tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
2150                    embed.to_kind(tch::Kind::BFloat16)
2151                }
2152                _ => embed.shallow_clone(),
2153            };
2154            Some(embed_bf16.tr().contiguous())
2155        } else {
2156            None
2157        };
2158
2159        // --- Owned decoder layers, each on its mapped device (per-layer .to) ---
2160        let mut layers = Vec::with_capacity(layer_range.len());
2161        for g in layer_range.clone() {
2162            let target = devices.device_for(g);
2163            let layer = Self::build_layer(weights, &cfg, g)?;
2164            layers.push(layer.into_device(target));
2165        }
2166
2167        // SSM + KV state sized to the OWNED layer count (M-LOAD seam #2); indexed
2168        // LOCALLY in the forward loop. Stage-local — never crosses a boundary.
2169        let owned = layers.len();
2170        let conv_states = Arc::new(parking_lot::Mutex::new(
2171            (0..owned).map(|_| None::<Tensor>).collect::<Vec<_>>(),
2172        ));
2173        let rec_states = Arc::new(parking_lot::Mutex::new(
2174            (0..owned).map(|_| None::<Tensor>).collect::<Vec<_>>(),
2175        ));
2176        let kv_cache = Some(Arc::new(parking_lot::Mutex::new(
2177            crate::runtime::kv_cache::KVCacheManager::new(
2178                owned,
2179                cfg.max_position_embeddings as usize,
2180                kv_quant_type,
2181            ),
2182        )));
2183
2184        let lm_head_norm = norm.unwrap_or_else(|| {
2185            // Middle/first stages have no final norm. A placeholder is never
2186            // invoked (apply_final_norm runs only on the last stage); sized [1].
2187            Qwen3_5RMSNorm::new(Tensor::zeros([1], (dtype, stage_device)), cfg.rms_norm_eps)
2188        });
2189
2190        let device_map = devices.clone();
2191
2192        Ok(Self {
2193            config: cfg,
2194            device: stage_device,
2195            dtype,
2196            device_map,
2197            layer_offset,
2198            embed_tokens: embed,
2199            layers,
2200            norm: lm_head_norm,
2201            lm_head: lm_head_w,
2202            lm_head_transposed,
2203            conv_states,
2204            rec_states,
2205            kv_cache,
2206            // Pipeline stages never own the MTP head (v1: whole-model loads only),
2207            // so speculative decoding is unavailable on multi-device splits.
2208            mtp: None,
2209            vision_encoder: None,
2210            vision_projector: None,
2211        })
2212    }
2213
2214    /// Unified forward inner: replaces the old `forward_inner` to avoid
2215    /// a deadlock that would occur if a no-delta wrapper acquired conv/rec
2216    /// locks and then delegated to a delta version that also tried to acquire them.
2217    /// (C5 deadlock fix: single implementation, delta=None for inference path)
2218    fn forward_inner(
2219        &self,
2220        input_ids: Option<&Tensor>,
2221        embeddings: Option<&Tensor>,
2222        start_pos: usize,
2223        delta: Option<&crate::training::TenantDelta>,
2224    ) -> Result<Tensor> {
2225        let mut hidden = if let Some(embeds) = embeddings {
2226            embeds.shallow_clone()
2227        } else {
2228            let ids = input_ids.ok_or_else(|| anyhow!("No input"))?;
2229            let flat = ids.flatten(0, -1);
2230            let emb = self.embed_tokens.index_select(0, &flat);
2231            let emb_shape = emb.size();
2232            let hidden_sz = emb_shape[emb_shape.len() - 1];
2233            let id_shape = ids.size();
2234            let batch = id_shape[0];
2235            let seq = id_shape[1];
2236            emb.reshape([batch, seq, hidden_sz])
2237        };
2238
2239        let (_, seq) = (hidden.size()[0], hidden.size()[1]);
2240
2241        // Position IDs for full-attention RoPE. Build on the HIDDEN-state device,
2242        // not `self.device` — under a pipeline split (#314) the first owned layer
2243        // may have been moved off `self.device`, and a mismatched position_ids
2244        // device would fault at SDPA. (llama already does this; see
2245        // forward_with_cache_inner.) On the single-device path this is identical.
2246        let position_ids = Tensor::arange_start(
2247            start_pos as i64,
2248            start_pos as i64 + seq,
2249            (Kind::Int64, hidden.device()),
2250        );
2251
2252        let mut conv_guard = self.conv_states.lock();
2253        let mut rec_guard = self.rec_states.lock();
2254
2255        for (idx, layer) in self.layers.iter().enumerate() {
2256            let residual = hidden.shallow_clone();
2257
2258            hidden = layer.input_layernorm.forward(&hidden)?;
2259
2260            let delta_arg = delta.map(|d| (d, idx));
2261
2262            let mixer_out = match &layer.mixer {
2263                LayerMixer::LinearAttn(gdn) => gdn.forward(
2264                    &hidden,
2265                    &mut conv_guard[idx],
2266                    &mut rec_guard[idx],
2267                    delta_arg,
2268                )?,
2269                LayerMixer::FullAttn(attn) => {
2270                    if let Some(ref cache_arc) = self.kv_cache {
2271                        let kv = cache_arc.lock();
2272                        kv.with_layer_cache(idx, |lc| {
2273                            attn.forward(
2274                                &hidden,
2275                                Some(&position_ids),
2276                                Some(lc),
2277                                start_pos,
2278                                delta_arg,
2279                            )
2280                        })
2281                        .ok_or_else(|| anyhow!("No KV cache for layer {idx}"))??
2282                    } else {
2283                        attn.forward(&hidden, Some(&position_ids), None, start_pos, delta_arg)?
2284                    }
2285                }
2286            };
2287
2288            hidden = residual + &mixer_out;
2289
2290            let residual2 = hidden.shallow_clone();
2291            hidden = layer.post_attention_layernorm.forward(&hidden)?;
2292            hidden = layer.mlp.forward(&hidden)?;
2293            hidden = residual2 + &hidden;
2294        }
2295        Ok(hidden)
2296    }
2297
2298    /// Run global decoder layers `[range.start..range.end)` — the 2b pipeline
2299    /// layer-range runner (#314). See the trait docs for the stage contract.
2300    ///
2301    /// Mirrors `LlamaModel::forward_layers_inner`, plus Qwen3.5's hybrid mixers
2302    /// and second (SSM) state vector:
2303    /// - Global layer `g` is remapped to its local slot `g - layer_offset` for
2304    ///   `self.layers`, the KV cache, and the `conv`/`rec` SSM state (all sized to
2305    ///   the owned layer count).
2306    /// - Per-layer GDN-vs-full-attention dispatch is intrinsic to the loaded
2307    ///   `LayerMixer`, so no global `layer_types` lookup is needed at runtime.
2308    /// - The lone cross-device copy is `hidden.to_device(next)` (carrying
2309    ///   `position_ids` with it), inserted only when the next layer's mapped device
2310    ///   differs from where `hidden` currently lives. SSM `conv`/`rec` state is
2311    ///   stage-local and never transferred across a boundary.
2312    fn forward_layers_inner(
2313        &self,
2314        hidden: &Tensor,
2315        range: std::ops::Range<usize>,
2316        start_pos: usize,
2317        delta: Option<&crate::training::TenantDelta>,
2318    ) -> Result<Tensor> {
2319        let num_global = self.config.num_hidden_layers as usize;
2320        if range.end > num_global || range.start >= range.end {
2321            return Err(anyhow!(
2322                "forward_layers: invalid global range {range:?} for {num_global} layers"
2323            ));
2324        }
2325        // The owned window must contain the requested range.
2326        let owned = self.layer_offset..self.layer_offset + self.layers.len();
2327        if range.start < owned.start || range.end > owned.end {
2328            return Err(anyhow!(
2329                "forward_layers: range {range:?} not within this stage's owned layers {owned:?}"
2330            ));
2331        }
2332
2333        let mut hidden = hidden.shallow_clone();
2334
2335        // position_ids recomputed from start_pos + seq (never carried across a
2336        // boundary as state). Built on the *input* device (the first owned
2337        // layer's), then moved alongside `hidden` at a device boundary so RoPE /
2338        // SDPA always see a matching device.
2339        let seq = hidden.size()[1];
2340        let mut position_ids = Tensor::arange_start(
2341            start_pos as i64,
2342            start_pos as i64 + seq,
2343            (Kind::Int64, hidden.device()),
2344        );
2345
2346        let mut conv_guard = self.conv_states.lock();
2347        let mut rec_guard = self.rec_states.lock();
2348
2349        for g in range {
2350            let local_idx = g - self.layer_offset;
2351
2352            // The single cross-device transfer: only when this layer's device
2353            // differs from where `hidden` currently is (zero-copy otherwise).
2354            let target = self.device_map.device_for(g);
2355            if hidden.device() != target {
2356                hidden = hidden.to_device(target);
2357                position_ids = position_ids.to_device(target);
2358            }
2359
2360            let layer = &self.layers[local_idx];
2361            let residual = hidden.shallow_clone();
2362            hidden = layer.input_layernorm.forward(&hidden)?;
2363
2364            // Delta module lookup uses the GLOBAL layer index (per-layer LoRA is
2365            // keyed by global layer, matching whole-model inference).
2366            let delta_arg = delta.map(|d| (d, g));
2367
2368            let mixer_out = match &layer.mixer {
2369                LayerMixer::LinearAttn(gdn) => {
2370                    // SSM conv/rec state indexed LOCALLY; stage-local, not moved.
2371                    gdn.forward(
2372                        &hidden,
2373                        &mut conv_guard[local_idx],
2374                        &mut rec_guard[local_idx],
2375                        delta_arg,
2376                    )?
2377                }
2378                LayerMixer::FullAttn(attn) => {
2379                    if let Some(ref cache_arc) = self.kv_cache {
2380                        let kv = cache_arc.lock();
2381                        kv.with_layer_cache(local_idx, |lc| {
2382                            attn.forward(
2383                                &hidden,
2384                                Some(&position_ids),
2385                                Some(lc),
2386                                start_pos,
2387                                delta_arg,
2388                            )
2389                        })
2390                        .ok_or_else(|| anyhow!("No KV cache for local layer {local_idx}"))??
2391                    } else {
2392                        attn.forward(&hidden, Some(&position_ids), None, start_pos, delta_arg)?
2393                    }
2394                }
2395            };
2396
2397            hidden = residual + &mixer_out;
2398
2399            let residual2 = hidden.shallow_clone();
2400            hidden = layer.post_attention_layernorm.forward(&hidden)?;
2401            hidden = layer.mlp.forward(&hidden)?;
2402            hidden = residual2 + &hidden;
2403        }
2404        Ok(hidden)
2405    }
2406
2407    /// Training-path sibling of [`Self::forward_layers_inner`] — the cross-device
2408    /// autograd primitive for TTT-on-split (#316). See the trait
2409    /// `forward_layers_train` docs for the contract.
2410    ///
2411    /// Differs from the inference runner in three deliberate ways:
2412    /// - **no KV cache** — full-attention layers run with full causal attention
2413    ///   over the entire context (`start_pos = 0`, no cache);
2414    /// - **`start_pos = 0`** — `position_ids = 0..seq`;
2415    /// - **fresh, call-local recurrent (SSM) state** — `conv`/`rec` start at
2416    ///   `None` for every owned layer and are never written back to
2417    ///   `self.conv_states` / `self.rec_states`. This keeps a TTT step from
2418    ///   polluting the persistent inference recurrent state, and makes the split
2419    ///   numerically identical to the whole-model training forward (per-layer
2420    ///   recurrent state never crosses a layer boundary, so partitioning the layer
2421    ///   range is exact).
2422    ///
2423    /// The lone cross-device `hidden.to_device(next)` (carrying `position_ids`) is
2424    /// autograd-transparent, so `loss.backward()` materializes grads on each
2425    /// parameter's own device.
2426    fn forward_layers_train_inner(
2427        &self,
2428        hidden: &Tensor,
2429        range: std::ops::Range<usize>,
2430        delta: Option<&crate::training::TenantDelta>,
2431    ) -> Result<Tensor> {
2432        self.forward_layers_stateless_inner(hidden, range, delta, None)
2433    }
2434
2435    fn forward_layers_stateless_inner(
2436        &self,
2437        hidden: &Tensor,
2438        range: std::ops::Range<usize>,
2439        delta: Option<&crate::training::TenantDelta>,
2440        check: Option<&dyn Fn() -> Result<()>>,
2441    ) -> Result<Tensor> {
2442        let num_global = self.config.num_hidden_layers as usize;
2443        if range.end > num_global || range.start >= range.end {
2444            return Err(anyhow!(
2445                "forward_layers_train: invalid global range {range:?} for {num_global} layers"
2446            ));
2447        }
2448        let owned = self.layer_offset..self.layer_offset + self.layers.len();
2449        if range.start < owned.start || range.end > owned.end {
2450            return Err(anyhow!(
2451                "forward_layers_train: range {range:?} not within this stage's owned layers {owned:?}"
2452            ));
2453        }
2454
2455        let mut hidden = hidden.shallow_clone();
2456
2457        // Training path: start_pos is always 0 → position_ids = 0..seq, built on
2458        // the input device and moved alongside `hidden` at each device boundary.
2459        let seq = hidden.size()[1];
2460        let mut position_ids = Tensor::arange(seq, (Kind::Int64, hidden.device()));
2461
2462        // Fresh, call-local SSM state — never touches the persistent inference
2463        // `conv_states`/`rec_states`. Sized to the OWNED layer count and indexed
2464        // locally, mirroring the persistent vectors' sizing.
2465        let mut conv_local: Vec<Option<Tensor>> = (0..self.layers.len()).map(|_| None).collect();
2466        let mut rec_local: Vec<Option<Tensor>> = (0..self.layers.len()).map(|_| None).collect();
2467
2468        for g in range {
2469            if let Some(check) = check {
2470                check()?;
2471            }
2472            let local_idx = g - self.layer_offset;
2473
2474            // The single cross-device transfer (autograd-transparent): only when
2475            // this layer's mapped device differs from where `hidden` currently is.
2476            let target = self.device_map.device_for(g);
2477            if hidden.device() != target {
2478                hidden = hidden.to_device(target);
2479                position_ids = position_ids.to_device(target);
2480            }
2481
2482            let layer = &self.layers[local_idx];
2483            let residual = hidden.shallow_clone();
2484            hidden = layer.input_layernorm.forward(&hidden)?;
2485
2486            // Delta module lookup uses the GLOBAL layer index (per-layer LoRA is
2487            // keyed by global layer, matching whole-model training).
2488            let delta_arg = delta.map(|d| (d, g));
2489
2490            let mixer_out = match &layer.mixer {
2491                LayerMixer::LinearAttn(gdn) => gdn.forward(
2492                    &hidden,
2493                    &mut conv_local[local_idx],
2494                    &mut rec_local[local_idx],
2495                    delta_arg,
2496                )?,
2497                LayerMixer::FullAttn(attn) => {
2498                    // No KV cache (training): full causal attention, start_pos = 0.
2499                    attn.forward(&hidden, Some(&position_ids), None, 0, delta_arg)?
2500                }
2501            };
2502
2503            hidden = residual + &mixer_out;
2504
2505            let residual2 = hidden.shallow_clone();
2506            hidden = layer.post_attention_layernorm.forward(&hidden)?;
2507            hidden = layer.mlp.forward(&hidden)?;
2508            hidden = residual2 + &hidden;
2509        }
2510        if let Some(check) = check {
2511            check()?;
2512        }
2513        Ok(hidden)
2514    }
2515
2516    /// Encode images through the vision encoder and project to text hidden space.
2517    ///
2518    /// pixel_values: [B, C, H, W]
2519    /// Returns: [total_image_patches, hidden_size]
2520    fn encode_vision_features(&self, pixel_values: &Tensor) -> Result<Tensor> {
2521        let enc = self
2522            .vision_encoder
2523            .as_ref()
2524            .ok_or_else(|| anyhow!("No vision encoder loaded"))?;
2525        let proj = self
2526            .vision_projector
2527            .as_ref()
2528            .ok_or_else(|| anyhow!("No vision projector loaded"))?;
2529
2530        // [B * out_patches, out_hidden_size]
2531        let features = enc.forward(pixel_values)?;
2532        // Project to text hidden size: [B * out_patches, hidden_size]
2533        proj.try_apply(&features)
2534    }
2535
2536    /// Build combined embeddings by replacing image placeholder tokens with vision features.
2537    ///
2538    /// image_token_id identifies placeholder positions in input_ids.
2539    /// pixel_values provides image patches for each placeholder sequence.
2540    fn prepare_inputs_embeds(
2541        &self,
2542        input_ids: &Tensor,
2543        pixel_values: Option<&Tensor>,
2544        images_seq_mask: Option<&Tensor>,
2545        _images_emb_mask: Option<&Tensor>,
2546    ) -> Result<Tensor> {
2547        // Get text embeddings for all tokens
2548        let flat = input_ids.flatten(0, -1);
2549        let text_emb = self.embed_tokens.index_select(0, &flat);
2550        let ids_shape = input_ids.size();
2551        let hidden_sz = text_emb
2552            .size()
2553            .last()
2554            .copied()
2555            .unwrap_or(self.config.hidden_size as i64);
2556        let combined = text_emb.reshape([ids_shape[0], ids_shape[1], hidden_sz]);
2557
2558        // If we have pixel_values and a sequence mask, inject vision embeddings
2559        if let (Some(pv), Some(mask)) = (pixel_values, images_seq_mask) {
2560            let vision_feats = self.encode_vision_features(pv)?; // [N_img, hidden]
2561                                                                 // mask: [batch, seq] bool tensor marking where to inject
2562                                                                 // Flatten and scatter vision features into combined at masked positions
2563            let (batch, seq, hs) = dims3(&combined)?;
2564            for b in 0..batch {
2565                let mask_b = mask.select(0, b).to_kind(Kind::Bool); // [seq]
2566                let positions: Vec<i64> = (0..seq)
2567                    .filter(|&t| mask_b.double_value(&[t]) != 0.0)
2568                    .collect();
2569                if positions.is_empty() {
2570                    continue;
2571                }
2572                // Slice vision_feats to match the number of masked positions
2573                let n_img = positions.len() as i64;
2574                let vf_slice = vision_feats.narrow(0, 0, n_img.min(vision_feats.size()[0]));
2575                for (img_idx, &pos) in positions.iter().enumerate() {
2576                    if (img_idx as i64) < vf_slice.size()[0] {
2577                        let vf = vf_slice.select(0, img_idx as i64); // [hidden]
2578                        combined
2579                            .select(0, b)
2580                            .select(0, pos)
2581                            .copy_(&vf.to_kind(self.dtype));
2582                    }
2583                }
2584            }
2585        }
2586
2587        Ok(combined)
2588    }
2589
2590    /// Expose the Qwen3.5 text config for external callers (e.g., get_per_layer_lora_dims).
2591    pub fn text_config(&self) -> &Qwen3_5TextConfig {
2592        &self.config
2593    }
2594
2595    /// Deep-copy current SSM states for TTT snapshot/restore (C4 fix: must use .copy(), not .shallow_clone()).
2596    ///
2597    /// Returns (conv_snapshot, rec_snapshot). Each tensor is deep-copied so that
2598    /// subsequent SSM mutations during TTT adaptation do not corrupt the snapshot.
2599    pub fn snapshot_ssm_states(&self) -> (Vec<Option<Tensor>>, Vec<Option<Tensor>>) {
2600        let conv_snapshot = {
2601            let guard = self.conv_states.lock();
2602            guard
2603                .iter()
2604                .map(|opt| opt.as_ref().map(|t| t.copy()))
2605                .collect()
2606        };
2607        let rec_snapshot = {
2608            let guard = self.rec_states.lock();
2609            guard
2610                .iter()
2611                .map(|opt| opt.as_ref().map(|t| t.copy()))
2612                .collect()
2613        };
2614        (conv_snapshot, rec_snapshot)
2615    }
2616
2617    /// Restore SSM states from a snapshot produced by `snapshot_ssm_states`.
2618    pub fn restore_ssm_states(
2619        &self,
2620        conv_snapshot: Vec<Option<Tensor>>,
2621        rec_snapshot: Vec<Option<Tensor>>,
2622    ) {
2623        *self.conv_states.lock() = conv_snapshot;
2624        *self.rec_states.lock() = rec_snapshot;
2625    }
2626
2627    /// Shared implementation for `decode_layer` and `decode_layer_with_delta`.
2628    ///
2629    /// **Training path only.** Full-attention layers run with `start_pos=0` and no KV
2630    /// cache, which is correct for the TTT gradient loop (full causal mask over the entire
2631    /// context). Do not use this method for incremental autoregressive inference — use
2632    /// `forward_with_cache_and_delta` / `forward_with_cache` instead.
2633    ///
2634    /// GDN layers acquire `conv_states` / `rec_states` locks per call (not held across
2635    /// layers), which is safe because `forward_with_delta` in TorchEngine never holds
2636    /// those locks itself.
2637    fn decode_layer_impl(
2638        &self,
2639        layer_idx: usize,
2640        hidden_states: &Tensor,
2641        position_ids: Option<&Tensor>,
2642        delta: Option<&crate::training::TenantDelta>,
2643    ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
2644        let layer = self
2645            .layers
2646            .get(layer_idx)
2647            .ok_or_else(|| anyhow!("Layer {layer_idx} out of range"))?;
2648        let residual = hidden_states.shallow_clone();
2649        let normed = layer.input_layernorm.forward(hidden_states)?;
2650
2651        let delta_arg = delta.map(|d| (d, layer_idx));
2652
2653        let mixer_out = match &layer.mixer {
2654            LayerMixer::LinearAttn(gdn) => {
2655                let mut conv = self.conv_states.lock();
2656                let mut rec = self.rec_states.lock();
2657                gdn.forward(
2658                    &normed,
2659                    &mut conv[layer_idx],
2660                    &mut rec[layer_idx],
2661                    delta_arg,
2662                )?
2663            }
2664            LayerMixer::FullAttn(attn) => {
2665                attn.forward(&normed, position_ids, None, 0, delta_arg)?
2666            }
2667        };
2668
2669        let hidden = &residual + &mixer_out;
2670        let residual2 = hidden.shallow_clone();
2671        let normed2 = layer.post_attention_layernorm.forward(&hidden)?;
2672        let ffn_out = layer.mlp.forward(&normed2)?;
2673        Ok((&residual2 + &ffn_out, None))
2674    }
2675
2676    fn lm_head_apply(&self, hidden: &Tensor) -> Result<Tensor> {
2677        let (batch, seq, hs) = dims3(hidden)?;
2678        let flat = hidden.reshape([batch * seq, hs]);
2679        let logits = if let Some(ref lm_head) = self.lm_head {
2680            // lm_head stored as [vocab, hidden], need to transpose for matmul
2681            flat.matmul(&lm_head.tr())
2682        } else if let Some(ref lm_t) = self.lm_head_transposed {
2683            flat.matmul(lm_t)
2684        } else {
2685            return Err(anyhow!("No lm_head weight"));
2686        };
2687
2688        Ok(logits.reshape([batch, seq, -1]))
2689    }
2690
2691    // ========================================================================
2692    // MTP self-speculative decoding support
2693    // ========================================================================
2694
2695    /// Whether a usable MTP draft head was loaded (dense checkpoints only).
2696    pub fn has_mtp(&self) -> bool {
2697        self.mtp.is_some()
2698    }
2699
2700    /// KV-cache slot of the MTP block: right after the main layers.
2701    fn mtp_kv_slot(&self) -> usize {
2702        self.config.num_hidden_layers as usize
2703    }
2704
2705    /// Like [`ModelOperations::forward_with_cache`] but also returns the
2706    /// pre-final-norm hidden states for ALL forwarded positions — the MTP head
2707    /// consumes them as its `h` input. With `last_logits_only` the LM head runs
2708    /// over the last position only (the #201 prefill optimization); otherwise
2709    /// every position's logits are returned (the 2-token verify step needs both).
2710    ///
2711    /// Returns `(logits [1, seq|1, vocab], hidden [1, seq, hidden])`.
2712    pub fn forward_with_cache_hidden(
2713        &self,
2714        input: &Tensor,
2715        start_pos: usize,
2716        last_logits_only: bool,
2717    ) -> Result<(Tensor, Tensor)> {
2718        let hidden = self.forward_inner(Some(input), None, start_pos, None)?;
2719        let seq_len = hidden.size()[1];
2720        anyhow::ensure!(
2721            seq_len > 0,
2722            "forward_with_cache_hidden: empty token sequence (seq_len=0)"
2723        );
2724        let logits = if last_logits_only {
2725            let last = hidden.narrow(1, seq_len - 1, 1);
2726            let normed = self.norm.forward(&last)?;
2727            self.lm_head_apply(&normed)?
2728        } else {
2729            let normed = self.norm.forward(&hidden)?;
2730            self.lm_head_apply(&normed)?
2731        };
2732        Ok((logits, hidden))
2733    }
2734
2735    /// Run the MTP draft head over `seq` consecutive slots.
2736    ///
2737    /// Slot `i` pairs `hiddens[.., i, ..]` (the main model's pre-norm hidden
2738    /// after some token x_t) with `embed(token_ids[.., i])` (the next token
2739    /// x_{t+1}) at absolute RoPE/KV position `base + i`, predicting x_{t+2}.
2740    /// Every slot's KV is written to the MTP cache slot (keeping it contiguous
2741    /// with prior drafts); only the LAST slot's logits are returned — that is
2742    /// the draft for the position after the last consumed token.
2743    ///
2744    /// Returns logits `[1, 1, vocab]`.
2745    pub fn mtp_forward(&self, token_ids: &Tensor, hiddens: &Tensor, base: usize) -> Result<Tensor> {
2746        let mtp = self
2747            .mtp
2748            .as_ref()
2749            .ok_or_else(|| anyhow!("MTP head not loaded (dense checkpoints only)"))?;
2750        let (batch, seq, hidden_sz) = dims3(hiddens)?;
2751        anyhow::ensure!(seq > 0, "mtp_forward: empty slot sequence");
2752        anyhow::ensure!(
2753            token_ids.size() == [batch, seq],
2754            "mtp_forward: token_ids shape {:?} does not match hiddens shape {:?}",
2755            token_ids.size(),
2756            hiddens.size()
2757        );
2758
2759        let emb = self.embed_tokens(token_ids)?;
2760        let e = mtp.pre_fc_norm_embedding.forward(&emb)?;
2761        let h = mtp.pre_fc_norm_hidden.forward(hiddens)?;
2762        // Embedding FIRST, hidden second (checkpoint fc weight layout).
2763        let x = Tensor::cat(&[e, h], -1).reshape([batch * seq, 2 * hidden_sz]);
2764        let x = mtp.fc.try_apply(&x)?.reshape([batch, seq, hidden_sz]);
2765
2766        let position_ids =
2767            Tensor::arange_start(base as i64, base as i64 + seq, (Kind::Int64, x.device()));
2768
2769        // Decoder block (pre-norm attention + pre-norm MLP, dense only).
2770        let residual = x.shallow_clone();
2771        let normed = mtp.input_layernorm.forward(&x)?;
2772        let attn_out = if let Some(ref cache_arc) = self.kv_cache {
2773            let kv = cache_arc.lock();
2774            kv.with_layer_cache(self.mtp_kv_slot(), |lc| {
2775                mtp.attn
2776                    .forward(&normed, Some(&position_ids), Some(lc), base, None)
2777            })
2778            .ok_or_else(|| anyhow!("No KV cache for MTP slot {}", self.mtp_kv_slot()))??
2779        } else {
2780            mtp.attn
2781                .forward(&normed, Some(&position_ids), None, base, None)?
2782        };
2783        let x = residual + attn_out;
2784
2785        let residual2 = x.shallow_clone();
2786        let normed2 = mtp.post_attention_layernorm.forward(&x)?;
2787        let mlp_out = mtp.mlp.forward(&normed2, None)?;
2788        let x = residual2 + mlp_out;
2789
2790        // Only the last slot produces a draft; earlier slots just fill MTP KV.
2791        let last = x.narrow(1, seq - 1, 1);
2792        let normed = mtp.final_norm.forward(&last)?;
2793        self.lm_head_apply(&normed)
2794    }
2795}
2796
2797// ============================================================================
2798// ModelOperations impl
2799// ============================================================================
2800
2801impl ModelOperations for Qwen3_5Model {
2802    fn architecture(&self) -> ModelArchitecture {
2803        ModelArchitecture::Qwen3_5
2804    }
2805
2806    fn config(&self) -> &dyn ArchitectureConfig {
2807        &self.config
2808    }
2809
2810    fn decision_hidden_states(&self, input: &Tensor) -> Result<Tensor> {
2811        self.decision_hidden_states_checked(input, &|| Ok(()))
2812    }
2813
2814    fn decision_hidden_states_checked(
2815        &self,
2816        input: &Tensor,
2817        check: &dyn Fn() -> Result<()>,
2818    ) -> Result<Tensor> {
2819        check()?;
2820        anyhow::ensure!(
2821            input.kind() == Kind::Int64
2822                && input.dim() == 2
2823                && input.size()[0] == 1
2824                && input.size()[1] > 0,
2825            "decision input must be [1,tokens]"
2826        );
2827        anyhow::ensure!(
2828            self.layer_offset == 0 && self.layers.len() == self.config.num_hidden_layers as usize,
2829            "decision inference requires the complete backbone"
2830        );
2831        anyhow::ensure!(
2832            input.min().int64_value(&[]) >= 0
2833                && input.max().int64_value(&[]) < self.embed_tokens.size()[0],
2834            "decision token ID outside embedding vocabulary"
2835        );
2836        let embedded = self.embed_tokens(input)?;
2837        let hidden = self.forward_layers_stateless_inner(
2838            &embedded,
2839            0..self.layers.len(),
2840            None,
2841            Some(check),
2842        )?;
2843        Ok(self.norm.forward(&hidden)?.squeeze_dim(0))
2844    }
2845
2846    fn decision_output_embedding_rows(&self, ids: &[i64]) -> Result<Tensor> {
2847        let (weight, transposed) = match (&self.lm_head, &self.lm_head_transposed) {
2848            (Some(w), _) => (w, false),
2849            (None, Some(w)) => (w, true),
2850            _ => return Err(anyhow!("missing output embedding table")),
2851        };
2852        anyhow::ensure!(
2853            !matches!(weight.kind(), Kind::Float8e4m3fn | Kind::Float8e5m2),
2854            "decision output rows require preserved floating output table"
2855        );
2856        let vocab = weight.size()[if transposed { 1 } else { 0 }];
2857        anyhow::ensure!(
2858            ids.iter().all(|id| *id >= 0 && *id < vocab),
2859            "output token ID outside vocabulary"
2860        );
2861        let indices = Tensor::from_slice(ids).to_device(weight.device());
2862        Ok(if transposed {
2863            weight.index_select(1, &indices).transpose(0, 1)
2864        } else {
2865            weight.index_select(0, &indices)
2866        })
2867    }
2868
2869    fn forward(&self, input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor> {
2870        let hidden = self.forward_inner(Some(input), None, 0, None)?;
2871        let normed = self.norm.forward(&hidden)?;
2872        self.lm_head_apply(&normed)
2873    }
2874
2875    fn forward_with_cache(&self, input: &Tensor, start_pos: usize) -> Result<Tensor> {
2876        let hidden = self.forward_inner(Some(input), None, start_pos, None)?;
2877        let seq_len = hidden.size()[1];
2878        anyhow::ensure!(
2879            seq_len > 0,
2880            "forward_with_cache: empty token sequence (seq_len=0)"
2881        );
2882        // Prefill optimization (#201): only run the expensive LM-head matmul over the
2883        // last position — the caller always discards all but the last token's logits.
2884        let last = hidden.narrow(1, seq_len - 1, 1);
2885        let normed = self.norm.forward(&last)?;
2886        self.lm_head_apply(&normed)
2887    }
2888
2889    fn forward_from_embeddings(&self, embeddings: &Tensor, start_pos: usize) -> Result<Tensor> {
2890        let hidden = self.forward_inner(None, Some(embeddings), start_pos, None)?;
2891        let seq_len = hidden.size()[1];
2892        anyhow::ensure!(
2893            seq_len > 0,
2894            "forward_from_embeddings: empty embedding sequence (seq_len=0)"
2895        );
2896        let last = hidden.narrow(1, seq_len - 1, 1);
2897        let normed = self.norm.forward(&last)?;
2898        self.lm_head_apply(&normed)
2899    }
2900
2901    fn forward_with_cache_and_delta(
2902        &self,
2903        input: &Tensor,
2904        start_pos: usize,
2905        delta: Option<&crate::training::TenantDelta>,
2906    ) -> Result<Tensor> {
2907        let hidden = self.forward_inner(Some(input), None, start_pos, delta)?;
2908        let seq_len = hidden.size()[1];
2909        anyhow::ensure!(
2910            seq_len > 0,
2911            "forward_with_cache_and_delta: empty token sequence (seq_len=0)"
2912        );
2913        let last = hidden.narrow(1, seq_len - 1, 1);
2914        let normed = self.norm.forward(&last)?;
2915        self.lm_head_apply(&normed)
2916    }
2917
2918    fn embed_tokens(&self, input_ids: &Tensor) -> Result<Tensor> {
2919        let flat = input_ids.flatten(0, -1);
2920        let emb = self.embed_tokens.index_select(0, &flat);
2921        let emb_shape = emb.size();
2922        let hidden_sz = emb_shape[emb_shape.len() - 1];
2923        let ids_shape = input_ids.size();
2924        Ok(emb.reshape([ids_shape[0], ids_shape[1], hidden_sz]))
2925    }
2926
2927    fn apply_final_norm(&self, hidden_states: &Tensor) -> Result<Tensor> {
2928        self.norm.forward(hidden_states)
2929    }
2930
2931    fn lm_head(&self, hidden_states: &Tensor) -> Result<Tensor> {
2932        self.lm_head_apply(hidden_states)
2933    }
2934
2935    fn num_layers(&self) -> usize {
2936        self.config.num_hidden_layers as usize
2937    }
2938
2939    fn forward_layers(
2940        &self,
2941        hidden: &Tensor,
2942        range: std::ops::Range<usize>,
2943        start_pos: usize,
2944        delta: Option<&crate::training::TenantDelta>,
2945    ) -> Result<Tensor> {
2946        self.forward_layers_inner(hidden, range, start_pos, delta)
2947    }
2948
2949    fn forward_layers_train(
2950        &self,
2951        hidden: &Tensor,
2952        range: std::ops::Range<usize>,
2953        delta: Option<&crate::training::TenantDelta>,
2954    ) -> Result<Tensor> {
2955        self.forward_layers_train_inner(hidden, range, delta)
2956    }
2957
2958    fn decode_layer(
2959        &self,
2960        layer_idx: usize,
2961        hidden_states: &Tensor,
2962        _attention_mask: Option<&Tensor>,
2963        position_ids: Option<&Tensor>,
2964        _past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
2965    ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
2966        self.decode_layer_impl(layer_idx, hidden_states, position_ids, None)
2967    }
2968
2969    fn decode_layer_with_delta(
2970        &self,
2971        layer_idx: usize,
2972        hidden_states: &Tensor,
2973        _attention_mask: Option<&Tensor>,
2974        position_ids: Option<&Tensor>,
2975        _past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
2976        delta: &crate::training::TenantDelta,
2977    ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
2978        self.decode_layer_impl(layer_idx, hidden_states, position_ids, Some(delta))
2979    }
2980
2981    fn clear_kv_cache(&self) {
2982        let mut conv = self.conv_states.lock();
2983        let mut rec = self.rec_states.lock();
2984        for s in conv.iter_mut() {
2985            *s = None;
2986        }
2987        for s in rec.iter_mut() {
2988            *s = None;
2989        }
2990        if let Some(ref cache) = self.kv_cache {
2991            cache.lock().clear_all();
2992        }
2993    }
2994
2995    fn set_kv_cache(&mut self, cache: Arc<parking_lot::Mutex<KVCacheManager>>) {
2996        // Session caches are created from the registry's layer count, which may
2997        // predate the model install and lack the MTP slot. Insert it on swap so
2998        // `mtp_forward` never finds its slot missing.
2999        if self.mtp.is_some() {
3000            cache.lock().ensure_layer_cache(self.mtp_kv_slot());
3001        }
3002        self.kv_cache = Some(cache);
3003    }
3004
3005    fn get_kv_cache(&self) -> Option<Arc<parking_lot::Mutex<KVCacheManager>>> {
3006        self.kv_cache.clone()
3007    }
3008
3009    fn take_kv_cache(&mut self) -> Option<Arc<parking_lot::Mutex<KVCacheManager>>> {
3010        self.kv_cache.take()
3011    }
3012
3013    fn reshape_for_attention(&self, tensor: &Tensor, _is_key_value: bool) -> Result<Tensor> {
3014        Ok(tensor.shallow_clone())
3015    }
3016
3017    fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor> {
3018        let rd = self.config.rotary_dim as i64;
3019        let device = tensor.device();
3020        let dtype = tensor.kind();
3021        let mut rope = RoPE::new_with_dtype(
3022            rd,
3023            self.config.rope_theta as f64,
3024            self.config.max_position_embeddings as i64,
3025            device,
3026            dtype,
3027        )?;
3028        rope.forward(tensor, Some(position_ids))
3029    }
3030
3031    fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
3032        self.norm.forward(tensor)
3033    }
3034
3035    fn get_attention_mask(&self, seq_len: usize, _past_kv_len: usize) -> Result<Tensor> {
3036        let sq = seq_len as i64;
3037        Ok(Tensor::ones([sq, sq], (Kind::Float, self.device)).tril(0))
3038    }
3039
3040    fn is_multimodal(&self) -> bool {
3041        self.vision_encoder.is_some()
3042    }
3043
3044    fn prepare_multimodal_inputs(
3045        &self,
3046        input_ids: &Tensor,
3047        pixel_values: Option<&Tensor>,
3048        images_seq_mask: Option<&Tensor>,
3049        images_emb_mask: Option<&Tensor>,
3050    ) -> Result<Tensor> {
3051        self.prepare_inputs_embeds(input_ids, pixel_values, images_seq_mask, images_emb_mask)
3052    }
3053
3054    fn encode_vision(&self, pixel_values: &Tensor) -> Result<Tensor> {
3055        self.encode_vision_features(pixel_values)
3056    }
3057
3058    fn as_any(&self) -> &dyn std::any::Any {
3059        self
3060    }
3061}
3062
3063#[cfg(test)]
3064#[allow(clippy::unwrap_used, clippy::expect_used)]
3065mod pipeline_tests {
3066    //! CPU equivalence tests for the 2b pipeline layer-split (#314).
3067    //!
3068    //! These prove that running qwen3_5's hybrid (GatedDeltaNet + full-attention)
3069    //! decoder stack via `forward_layers` — whole-range AND split into two stages
3070    //! over an all-CPU [`LayerDeviceMap`] — reproduces the whole-model forward to
3071    //! float-reassociation tolerance. The split must hold even though each GDN
3072    //! layer carries its own stage-local SSM (`conv`/`rec`) state: if the SSM
3073    //! state threading were wrong, the two-stage result would diverge.
3074    use super::*;
3075    use crate::runtime::device_pool::LayerDeviceMap;
3076
3077    // Pinned reference: transformers 5.10.2 models/qwen3_5/modeling_qwen3_5.py
3078    // l2norm computes x * rsqrt(sum(x*x) + 1e-6). Scalar f64 evaluation supplies
3079    // an independent numerical reference; these are not model outputs.
3080    #[test]
3081    fn gdn_l2_normalize_matches_additive_epsilon_reference() {
3082        let input = Tensor::from_slice(&[3.0f32, 4.0, -2.0, 1.0]).reshape([2, 2]);
3083        let expected = Tensor::from_slice(&[
3084            (3.0f64 / (25.0f64 + 1e-6).sqrt()) as f32,
3085            (4.0f64 / (25.0f64 + 1e-6).sqrt()) as f32,
3086            (-2.0f64 / (5.0f64 + 1e-6).sqrt()) as f32,
3087            (1.0f64 / (5.0f64 + 1e-6).sqrt()) as f32,
3088        ])
3089        .reshape([2, 2]);
3090        assert!(l2_normalize(&input).allclose(&expected, 1e-7, 1e-7, false));
3091    }
3092
3093    #[test]
3094    fn gdn_l2_normalize_small_vectors_do_not_clamp_norm() {
3095        let input = Tensor::from_slice(&[3e-4f32, 4e-4, 1e-3, 0.0]).reshape([1, 2, 2]);
3096        let a = 3e-4f32 as f64;
3097        let b = 4e-4f32 as f64;
3098        let c = 1e-3f32 as f64;
3099        let denominator = (a * a + b * b + 1e-6).sqrt();
3100        let expected = Tensor::from_slice(&[
3101            (a / denominator) as f32,
3102            (b / denominator) as f32,
3103            (c / (c * c + 1e-6).sqrt()) as f32,
3104            0.0,
3105        ])
3106        .reshape([1, 2, 2]);
3107        let output = l2_normalize(&input);
3108        assert_eq!(output.size(), input.size());
3109        assert!(output.allclose(&expected, 1e-7, 1e-7, false));
3110        let old_clamped = &input
3111            / input
3112                .norm_scalaropt_dim(2.0, &[-1i64][..], true)
3113                .clamp_min(1e-3);
3114        assert!((&output - old_clamped).abs().max().double_value(&[]) > 0.1);
3115    }
3116
3117    #[test]
3118    fn gdn_l2_normalize_zero_vectors_remain_finite_zero() {
3119        let input = Tensor::zeros([2, 3, 4], (Kind::Float, Device::Cpu));
3120        let output = l2_normalize(&input);
3121        assert_eq!(output.size(), input.size());
3122        assert_eq!(output.kind(), Kind::Float);
3123        assert_ne!(output.isfinite().all().int64_value(&[]), 0);
3124        assert_eq!(output.abs().max().double_value(&[]), 0.0);
3125    }
3126
3127    pub(super) const HIDDEN: i64 = 16;
3128    pub(super) const HEADS: i64 = 2;
3129    pub(super) const KV_HEADS: i64 = 2;
3130    pub(super) const HEAD_DIM: i64 = 8;
3131    pub(super) const INTER: i64 = 32;
3132    pub(super) const VOCAB: i64 = 48;
3133    const LAYERS: usize = 4;
3134    // GatedDeltaNet dims (kept tiny + symmetric: num_k_heads == num_v_heads).
3135    const LIN_K_HEADS: usize = 1;
3136    const LIN_V_HEADS: usize = 1;
3137    const LIN_K_DIM: usize = 4;
3138    const LIN_V_DIM: usize = 4;
3139    const CONV_KERNEL: usize = 4;
3140
3141    /// Child-process probe for the FP8 causal gate test. Starting with
3142    /// `HYPRSTREAM_FP8_GEMM=1` is essential because `with_scale` caches that
3143    /// env decision for the process. This is an ownership/allocation probe:
3144    /// it verifies source rowwise pairs are released before fusion builds the
3145    /// fused pair, rather than merely checking a field exists afterwards.
3146    #[test]
3147    #[allow(clippy::print_stderr)]
3148    fn fp8_fused_projection_metadata_probe() {
3149        if std::env::var_os("HYPRSTREAM_QWEN_FP8_PROBE").is_none() {
3150            return;
3151        }
3152        assert!(crate::config::default_fp8_gemm());
3153        let w1 = Tensor::zeros([256, 256], (Kind::Float, Device::Cpu)).to_kind(Kind::Float8e4m3fn);
3154        let w2 = Tensor::zeros([256, 256], (Kind::Float, Device::Cpu)).to_kind(Kind::Float8e4m3fn);
3155        let s1 = Tensor::ones([2, 2], (Kind::Float, Device::Cpu));
3156        let s2 = Tensor::ones([2, 2], (Kind::Float, Device::Cpu));
3157        crate::runtime::architectures::llama::reset_rowwise_requant_peak();
3158        let fused = cat_projs(vec![
3159            LinearProjection::with_scale(w1, s1),
3160            LinearProjection::with_scale(w2, s2),
3161        ]);
3162        assert_eq!(fused.weight.size(), [256, 512]);
3163        assert_eq!(
3164            fused.scale.as_ref().expect("fused FP8 scale").size(),
3165            [2, 4]
3166        );
3167        assert_eq!(
3168            fused.scale_v2.as_ref().expect("fused packed scale").size(),
3169            [4, 4]
3170        );
3171        assert_eq!(
3172            crate::runtime::architectures::llama::rowwise_requant_live(),
3173            1,
3174            "fusion did not leave exactly its one derived rowwise allocation alive"
3175        );
3176        assert_eq!(
3177            crate::runtime::architectures::llama::rowwise_requant_peak(),
3178            2,
3179            "source rowwise allocations overlapped with fused rowwise construction"
3180        );
3181        eprintln!("Qwen fused FP8 allocation probe: source rowwise copies dropped before fused allocation");
3182    }
3183
3184    /// Exercise the actual expert loader in a fresh process: the FP8 gate is
3185    /// cached, and allocation witnesses must not include parallel tests.
3186    #[test]
3187    fn fp8_moe_loading_avoids_transient_requant() {
3188        if std::env::var_os("HYPRSTREAM_QWEN_MOE_LOADING_PROBE").is_none() {
3189            let output = std::process::Command::new(std::env::current_exe().unwrap())
3190                .args(["--exact", "runtime::architectures::qwen3_5::pipeline_tests::fp8_moe_loading_avoids_transient_requant", "--nocapture"])
3191                .env("HYPRSTREAM_FP8_GEMM", "1")
3192                .env("HYPRSTREAM_FP8_DEQUANT_LOAD", "0")
3193                .env("HYPRSTREAM_QWEN_MOE_LOADING_PROBE", "1")
3194                .output().expect("spawn MoE allocation probe");
3195            let transcript = format!(
3196                "{}{}",
3197                String::from_utf8_lossy(&output.stdout),
3198                String::from_utf8_lossy(&output.stderr)
3199            );
3200            assert!(
3201                output.status.success() && transcript.contains("1 passed"),
3202                "MoE allocation probe did not pass: {transcript}"
3203            );
3204            return;
3205        }
3206        use crate::runtime::architectures::llama::{
3207            reset_rowwise_requant_peak, rowwise_requant_live, rowwise_requant_peak,
3208        };
3209        assert!(crate::config::default_fp8_gemm());
3210        let mut cfg = tiny_config();
3211        cfg.is_moe = true;
3212        cfg.num_experts = 3;
3213        cfg.num_experts_per_tok = 1;
3214        let mut weights = HashMap::new();
3215        weights.insert(
3216            "mlp.gate.weight".to_owned(),
3217            Tensor::zeros([3, 128], (Kind::Float, Device::Cpu)),
3218        );
3219        for (name, shape) in [
3220            ("gate_proj", [256, 128]),
3221            ("up_proj", [256, 128]),
3222            ("down_proj", [128, 256]),
3223        ] {
3224            weights.insert(
3225                format!("mlp.shared_expert.{name}.weight"),
3226                Tensor::zeros(shape, (Kind::Float, Device::Cpu)),
3227            );
3228        }
3229        let mut expected = Vec::new();
3230        for e in 0..cfg.num_experts {
3231            for (name, shape) in [
3232                ("gate_proj", [256, 128]),
3233                ("up_proj", [256, 128]),
3234                ("down_proj", [128, 256]),
3235            ] {
3236                let key = format!("mlp.experts.{e}.{name}.weight");
3237                let weight = (Tensor::arange(shape[0] * shape[1], (Kind::Float, Device::Cpu))
3238                    * 0.01
3239                    + e as f64)
3240                    .sin()
3241                    .reshape(shape)
3242                    .to_kind(Kind::Float8e4m3fn);
3243                let scale = Tensor::from_slice(&[0.5f32 + e as f32, 1.5 + e as f32])
3244                    .reshape([shape[0] / 128, shape[1] / 128]);
3245                expected.push((
3246                    name,
3247                    e as i64,
3248                    weight.transpose(0, 1).to_kind(Kind::Float),
3249                    scale.transpose(0, 1).contiguous(),
3250                ));
3251                weights.insert(key.clone(), weight);
3252                weights.insert(format!("{key}_scale_inv"), scale);
3253            }
3254        }
3255        reset_rowwise_requant_peak();
3256        let Qwen3_5Mlp::Sparse(loaded) =
3257            Qwen3_5Model::load_mlp(&mut weights, "mlp", &cfg, 0).unwrap()
3258        else {
3259            panic!("fixture must load sparse experts");
3260        };
3261        assert!(weights.is_empty(), "loader must consume checkpoint tensors");
3262        assert_eq!(rowwise_requant_live(), 0);
3263        assert_eq!(
3264            rowwise_requant_peak(),
3265            0,
3266            "transient batched experts must never construct discarded rowwise copies"
3267        );
3268        for (name, e, weight, scale) in expected {
3269            let (stacked_w, stacked_s) = match name {
3270                "gate_proj" => (&loaded.expert_gate_w, &loaded.expert_gate_scale),
3271                "up_proj" => (&loaded.expert_up_w, &loaded.expert_up_scale),
3272                _ => (&loaded.expert_down_w, &loaded.expert_down_scale),
3273            };
3274            assert_eq!(stacked_w.kind(), Kind::Float8e4m3fn);
3275            assert!(stacked_w.get(e).to_kind(Kind::Float).equal(&weight));
3276            assert!(stacked_s.as_ref().unwrap().get(e).equal(&scale));
3277        }
3278        // A normal retained projection still builds the accelerated metadata;
3279        // the zero expert peak must not be obtained by disabling the gate.
3280        let retained = LinearProjection::with_scale(
3281            Tensor::zeros([128, 256], (Kind::Float, Device::Cpu)).to_kind(Kind::Float8e4m3fn),
3282            Tensor::ones([1, 2], (Kind::Float, Device::Cpu)),
3283        );
3284        assert!(retained.scale_v2.is_some());
3285        assert_eq!(rowwise_requant_live(), 1);
3286        assert_eq!(rowwise_requant_peak(), 1);
3287    }
3288
3289    #[test]
3290    fn gdn_chunked_continuation_preserves_recurrent_history() {
3291        let mut weights = tiny_weights();
3292        let layer = GatedDeltaNetLayer::load(
3293            &mut weights,
3294            "model.layers.0.linear_attn",
3295            &tiny_config(),
3296            0,
3297        )
3298        .unwrap();
3299        let input = (Tensor::arange(2 * HIDDEN, (Kind::Float, Device::Cpu)) * 0.13)
3300            .sin()
3301            .reshape([1, 2, HIDDEN]);
3302        let initial = Tensor::full(
3303            [1, LIN_V_HEADS as i64, LIN_K_DIM as i64, LIN_V_DIM as i64],
3304            3.0,
3305            (Kind::Float, Device::Cpu),
3306        );
3307        let conv = Tensor::zeros(
3308            [1, layer.conv_dim as i64, layer.kernel_size as i64 - 1],
3309            (Kind::Float, Device::Cpu),
3310        );
3311        let mut serial_conv = Some(conv.copy());
3312        let mut serial_outputs = Vec::new();
3313        let mut serial_rec = Some(initial.copy());
3314        for position in 0..2 {
3315            serial_outputs.push(
3316                layer
3317                    .forward(
3318                        &input.narrow(1, position, 1),
3319                        &mut serial_conv,
3320                        &mut serial_rec,
3321                        None,
3322                    )
3323                    .unwrap(),
3324            );
3325        }
3326        let mut chunk_conv = Some(conv.copy());
3327        let mut chunk_rec = Some(initial);
3328        let chunk_output = layer
3329            .forward(&input, &mut chunk_conv, &mut chunk_rec, None)
3330            .unwrap();
3331        let mut zero_conv = Some(conv);
3332        let mut zero_rec = None;
3333        let _ = layer
3334            .forward(&input, &mut zero_conv, &mut zero_rec, None)
3335            .unwrap();
3336        let expected = serial_rec.unwrap();
3337        assert!(
3338            !expected.allclose(&zero_rec.unwrap(), 1e-4, 1e-4, false),
3339            "fixture must distinguish preserved history from an empty recurrent state"
3340        );
3341        assert!(
3342            chunk_output.allclose(&Tensor::cat(&serial_outputs, 1), 1e-4, 1e-4, false),
3343            "chunked outputs must match serial continuation"
3344        );
3345        let actual = chunk_rec.unwrap();
3346        assert!(
3347            actual.allclose(&expected, 1e-4, 1e-4, false),
3348            "chunked continuation lost recurrent history; max difference {}",
3349            (&actual - &expected).abs().max().double_value(&[])
3350        );
3351    }
3352
3353    #[test]
3354    fn gdn_short_prefill_keeps_full_convolution_history() {
3355        let mut weights = tiny_weights();
3356        let layer = GatedDeltaNetLayer::load(
3357            &mut weights,
3358            "model.layers.0.linear_attn",
3359            &tiny_config(),
3360            0,
3361        )
3362        .unwrap();
3363        let input = Tensor::ones([1, 1, HIDDEN], (Kind::Float, Device::Cpu));
3364        let mut conv = None;
3365        let mut recurrent = None;
3366        for _ in 0..2 {
3367            let _ = layer
3368                .forward(&input, &mut conv, &mut recurrent, None)
3369                .unwrap();
3370            assert_eq!(
3371                conv.as_ref().unwrap().size()[2],
3372                layer.kernel_size as i64 - 1
3373            );
3374        }
3375    }
3376
3377    pub(super) fn tiny_config() -> Qwen3_5TextConfig {
3378        // Default hybrid pattern: layer (i+1)%4==0 is full attention, rest GDN.
3379        // With LAYERS=4 → [GDN, GDN, GDN, FullAttn]; a split at 2 puts the full-
3380        // attention layer in the second stage, exercising both mixer kinds across
3381        // the stage boundary.
3382        let layer_types = (0..LAYERS)
3383            .map(|i| {
3384                if (i + 1) % 4 == 0 {
3385                    "full_attention".to_owned()
3386                } else {
3387                    "linear_attention".to_owned()
3388                }
3389            })
3390            .collect();
3391        let rotary_dim = ((HEAD_DIM as f32) * 0.25) as usize; // 2
3392        Qwen3_5TextConfig {
3393            hidden_size: HIDDEN as u32,
3394            num_hidden_layers: LAYERS as u32,
3395            num_attention_heads: HEADS as u32,
3396            num_key_value_heads: KV_HEADS as u32,
3397            head_dim: HEAD_DIM as u32,
3398            intermediate_size: INTER as u32,
3399            vocab_size: VOCAB as u32,
3400            max_position_embeddings: 64,
3401            rms_norm_eps: 1e-5,
3402            rope_theta: 10000.0,
3403            partial_rotary_factor: 0.25,
3404            rotary_dim,
3405            layer_types,
3406            linear_conv_kernel_dim: CONV_KERNEL,
3407            linear_key_head_dim: LIN_K_DIM,
3408            linear_value_head_dim: LIN_V_DIM,
3409            linear_num_key_heads: LIN_K_HEADS,
3410            linear_num_value_heads: LIN_V_HEADS,
3411            is_moe: false,
3412            num_experts: 0,
3413            num_experts_per_tok: 0,
3414            moe_intermediate_size: 0,
3415            shared_expert_intermediate_size: 0,
3416            has_vision: false,
3417            vision_out_hidden_size: 0,
3418        }
3419    }
3420
3421    /// Deterministic, RNG-free weights on CPU/f32. tch's global RNG is shared
3422    /// across parallel tests, so a `randn`-based build could interleave and
3423    /// diverge between the whole-model and staged paths; a fixed sin-of-index
3424    /// pattern is byte-identical regardless of scheduling. HF stores projections
3425    /// as `[out, in]` (LinearProjection::take transposes to `[in, out]`).
3426    pub(super) fn tiny_weights() -> HashMap<String, Tensor> {
3427        let opt = (Kind::Float, Device::Cpu);
3428        let mut w = HashMap::new();
3429        // Bounded small values in roughly [-0.05, 0.05], offset per-tensor so
3430        // different weights are not identical.
3431        let mut seed: i64 = 0;
3432        let mut pat = |dims: &[i64]| -> Tensor {
3433            let n: i64 = dims.iter().product();
3434            seed += 7;
3435            (Tensor::arange(n, opt) * 0.017 + seed as f64 * 0.013)
3436                .sin()
3437                .reshape(dims)
3438                * 0.05
3439        };
3440
3441        w.insert(
3442            "model.embed_tokens.weight".to_owned(),
3443            pat(&[VOCAB, HIDDEN]),
3444        );
3445        w.insert("model.norm.weight".to_owned(), pat(&[HIDDEN]));
3446        w.insert("lm_head.weight".to_owned(), pat(&[VOCAB, HIDDEN]));
3447
3448        let key_dim = (LIN_K_HEADS * LIN_K_DIM) as i64;
3449        let val_dim = (LIN_V_HEADS * LIN_V_DIM) as i64;
3450        let conv_dim = key_dim * 2 + val_dim;
3451        let nv = LIN_V_HEADS as i64;
3452
3453        for i in 0..LAYERS {
3454            let p = format!("model.layers.{i}");
3455            w.insert(format!("{p}.input_layernorm.weight"), pat(&[HIDDEN]));
3456            w.insert(
3457                format!("{p}.post_attention_layernorm.weight"),
3458                pat(&[HIDDEN]),
3459            );
3460
3461            let is_full = (i + 1) % 4 == 0;
3462            if is_full {
3463                // Full attention: q_proj out = nh*hd*2 (q + gate), k/v out = nkv*hd.
3464                let ap = format!("{p}.self_attn");
3465                w.insert(
3466                    format!("{ap}.q_proj.weight"),
3467                    pat(&[HEADS * HEAD_DIM * 2, HIDDEN]),
3468                );
3469                w.insert(
3470                    format!("{ap}.k_proj.weight"),
3471                    pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
3472                );
3473                w.insert(
3474                    format!("{ap}.v_proj.weight"),
3475                    pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
3476                );
3477                w.insert(
3478                    format!("{ap}.o_proj.weight"),
3479                    pat(&[HIDDEN, HEADS * HEAD_DIM]),
3480                );
3481                w.insert(format!("{ap}.q_norm.weight"), pat(&[HEAD_DIM]));
3482                w.insert(format!("{ap}.k_norm.weight"), pat(&[HEAD_DIM]));
3483            } else {
3484                // GatedDeltaNet: 4 input projections fused at load.
3485                let lp = format!("{p}.linear_attn");
3486                w.insert(format!("{lp}.in_proj_qkv.weight"), pat(&[conv_dim, HIDDEN]));
3487                w.insert(format!("{lp}.in_proj_z.weight"), pat(&[val_dim, HIDDEN]));
3488                w.insert(format!("{lp}.in_proj_b.weight"), pat(&[nv, HIDDEN]));
3489                w.insert(format!("{lp}.in_proj_a.weight"), pat(&[nv, HIDDEN]));
3490                // conv1d_weight stays raw [conv_dim, 1, kernel_size] (not transposed).
3491                w.insert(
3492                    format!("{lp}.conv1d.weight"),
3493                    pat(&[conv_dim, 1, CONV_KERNEL as i64]),
3494                );
3495                w.insert(format!("{lp}.A_log"), pat(&[nv]));
3496                w.insert(format!("{lp}.dt_bias"), pat(&[nv]));
3497                w.insert(format!("{lp}.norm.weight"), pat(&[LIN_V_DIM as i64]));
3498                w.insert(format!("{lp}.out_proj.weight"), pat(&[HIDDEN, val_dim]));
3499            }
3500
3501            // Dense MLP.
3502            let mp = format!("{p}.mlp");
3503            w.insert(format!("{mp}.gate_proj.weight"), pat(&[INTER, HIDDEN]));
3504            w.insert(format!("{mp}.up_proj.weight"), pat(&[INTER, HIDDEN]));
3505            w.insert(format!("{mp}.down_proj.weight"), pat(&[HIDDEN, INTER]));
3506        }
3507        w
3508    }
3509
3510    #[test]
3511    fn channel_fp8_fusion_preserves_quantized_and_ignored_boundaries() {
3512        let input = Tensor::from_slice(&[448f32, 1.0625, -4.0]).reshape([1, 3]);
3513        let make = || {
3514            let mut w = HashMap::from([
3515                (
3516                    "p.weight".to_owned(),
3517                    Tensor::ones([2, 3], (Kind::Float, Device::Cpu)).to_kind(Kind::Float8e4m3fn),
3518                ),
3519                (
3520                    "p.weight_scale".to_owned(),
3521                    Tensor::from_slice(&[0.25f32, 0.5]).reshape([2, 1]),
3522                ),
3523            ]);
3524            LinearProjection::take(&mut w, "p.weight").unwrap()
3525        };
3526        let ignored = || LinearProjection::new(Tensor::ones([3, 1], (Kind::Float, Device::Cpu)));
3527        let expected = Tensor::cat(&[make().apply(&input), ignored().apply(&input)], -1);
3528        let fused = cat_projs(vec![make(), ignored()]);
3529        assert!(fused.children.is_some());
3530        assert_eq!(fused.weight.numel(), 9); // expanded metadata view, not materialized weights
3531        assert!(fused.apply(&input).allclose(&expected, 0.0, 0.0, false));
3532        assert_eq!(fused.apply(&input).double_value(&[0, 2]), 445.0625);
3533    }
3534
3535    #[test]
3536    fn decision_hidden_and_output_rows_are_fresh_and_match_full_forward() {
3537        let model = whole_model();
3538        let input = Tensor::from_slice(&[1i64, 5, 9, 2]).reshape([1, 4]);
3539        let hidden = model.decision_hidden_states(&input).unwrap();
3540        assert_eq!(hidden.size(), [4, HIDDEN]);
3541        let expected_logits = whole_model().forward(&input, None).unwrap().squeeze_dim(0);
3542        let rows = model.decision_output_embedding_rows(&[0, 5, 0]).unwrap();
3543        let logits = hidden.matmul(&rows.transpose(0, 1));
3544        let expected = expected_logits.index_select(1, &Tensor::from_slice(&[0i64, 5, 0]));
3545        assert!(logits.allclose(&expected, 1e-4, 1e-4, false));
3546        // Contaminate the persistent cache with unrelated generation.
3547        let _ = model
3548            .forward_with_cache(&Tensor::from_slice(&[7i64, 4]).reshape([1, 2]), 0)
3549            .unwrap();
3550        let hidden_again = model.decision_hidden_states(&input).unwrap();
3551        assert!(hidden_again.allclose(&hidden, 0.0, 0.0, false));
3552        assert!(model.decision_output_embedding_rows(&[-1]).is_err());
3553        assert!(model.decision_output_embedding_rows(&[VOCAB]).is_err());
3554        assert!(model
3555            .decision_hidden_states(&Tensor::from_slice(&[VOCAB]).reshape([1, 1]))
3556            .is_err());
3557        assert!(model
3558            .decision_hidden_states(&Tensor::zeros([1, 0], (Kind::Int64, Device::Cpu)))
3559            .is_err());
3560    }
3561
3562    #[test]
3563    fn decision_cancel_between_layers_preserves_next_request() {
3564        let model = whole_model();
3565        let input = Tensor::from_slice(&[1i64, 5, 9, 2]).reshape([1, 4]);
3566        let baseline = model.decision_hidden_states(&input).unwrap();
3567        let checks = std::cell::Cell::new(0usize);
3568        let cancel = || {
3569            let calls = checks.get() + 1;
3570            checks.set(calls);
3571            // Initial admission, layer0, then abort before layer1: exactly one
3572            // full decoder layer has completed when cancellation is observed.
3573            if calls == 3 {
3574                Err(anyhow!("request cancelled at decoder boundary"))
3575            } else {
3576                Ok(())
3577            }
3578        };
3579        let failure = model
3580            .decision_hidden_states_checked(&input, &cancel)
3581            .unwrap_err();
3582        assert!(failure.to_string().contains("request cancelled"));
3583        assert_eq!(checks.get(), 3);
3584        let next = model.decision_hidden_states(&input).unwrap();
3585        assert!(
3586            next.allclose(&baseline, 0.0, 0.0, false),
3587            "cancelled request leaked recurrent/KV state"
3588        );
3589    }
3590
3591    fn whole_model() -> Qwen3_5Model {
3592        let mut w = tiny_weights();
3593        Qwen3_5Model::from_weights(
3594            &mut w,
3595            tiny_config(),
3596            None,
3597            &Device::Cpu,
3598            Kind::Float,
3599            KVQuantType::None,
3600            false,
3601        )
3602        .unwrap()
3603    }
3604
3605    /// Drive the pipeline path on one model the way an engine would:
3606    /// embed → forward_layers(0..N) → final norm → lm_head.
3607    fn orchestrated_logits(m: &Qwen3_5Model, input: &Tensor) -> Tensor {
3608        let emb = m.embed_tokens(input).unwrap();
3609        let h = m.forward_layers(&emb, 0..m.num_layers(), 0, None).unwrap();
3610        let h = m.apply_final_norm(&h).unwrap();
3611        m.lm_head(&h).unwrap()
3612    }
3613
3614    #[test]
3615    fn forward_layers_full_range_matches_whole_model_forward() {
3616        // An all-CPU LayerDeviceMap means forward_layers(0..N) must equal the
3617        // whole-model single-device forward (the byte-identity equivalence test).
3618        let input = Tensor::from_slice(&[1i64, 5, 9, 2]).reshape([1, 4]);
3619
3620        // Reference: whole-model forward over the full sequence (NOT
3621        // forward_with_cache, which only returns the last position's logits).
3622        let reference = whole_model();
3623        let ref_logits = reference.forward(&input, None).unwrap();
3624
3625        // Pipeline path on a fresh model (independent SSM/KV state), full range
3626        // over the implicit single-device map.
3627        let piped = whole_model();
3628        let pipe_logits = orchestrated_logits(&piped, &input);
3629
3630        let max_diff = (&ref_logits - &pipe_logits).abs().max().double_value(&[]);
3631        assert!(
3632            ref_logits.allclose(&pipe_logits, 1e-4, 1e-4, false),
3633            "forward_layers(0..N) over a single-device map must equal whole-model forward \
3634             (max_diff={max_diff}, shape={:?})",
3635            ref_logits.size()
3636        );
3637        assert!(piped.device_map.is_single_device());
3638        assert_eq!(piped.layer_offset, 0);
3639    }
3640
3641    #[test]
3642    fn staged_split_equals_whole_model() {
3643        // Build TWO stages over an all-CPU 2-way split and run them as a pipeline:
3644        // stage0: embed → forward_layers(0..k); stage1: forward_layers(k..N) →
3645        // norm → lm_head. The split point (k=2) puts the full-attention layer in
3646        // stage1, so this exercises the global↔local remap, is_first/is_last
3647        // gating, per-global-index hybrid layer selection, and — critically —
3648        // that each stage's GDN SSM state is sized to its OWNED layer count and
3649        // indexed locally. Result must equal the whole-model forward.
3650        let input = Tensor::from_slice(&[3i64, 7, 1, 4]).reshape([1, 4]);
3651
3652        let reference = whole_model();
3653        let ref_logits = reference.forward(&input, None).unwrap();
3654
3655        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
3656        let split = LAYERS / 2;
3657
3658        let mut w0 = tiny_weights();
3659        let stage0 = Qwen3_5Model::stage_from_weights_with_config(
3660            &mut w0,
3661            tiny_config(),
3662            &map,
3663            0..split,
3664            Kind::Float,
3665            KVQuantType::None,
3666        )
3667        .unwrap();
3668        assert_eq!(stage0.layer_offset, 0);
3669        assert_eq!(stage0.layers.len(), split, "stage0 owns exactly its layers");
3670        assert!(
3671            stage0.lm_head.is_none(),
3672            "first (non-last) stage has no head"
3673        );
3674        // SSM state sized to OWNED count, not global.
3675        assert_eq!(stage0.conv_states.lock().len(), split);
3676        assert_eq!(stage0.rec_states.lock().len(), split);
3677
3678        let mut w1 = tiny_weights();
3679        let stage1 = Qwen3_5Model::stage_from_weights_with_config(
3680            &mut w1,
3681            tiny_config(),
3682            &map,
3683            split..LAYERS,
3684            Kind::Float,
3685            KVQuantType::None,
3686        )
3687        .unwrap();
3688        assert_eq!(stage1.layer_offset, split);
3689        assert_eq!(stage1.layers.len(), LAYERS - split);
3690        assert_eq!(stage1.conv_states.lock().len(), LAYERS - split);
3691
3692        // Drive the pipeline. Only `hidden` crosses the (same-device) boundary;
3693        // each stage's SSM state stays local.
3694        let emb = stage0.embed_tokens(&input).unwrap();
3695        let h0 = stage0.forward_layers(&emb, 0..split, 0, None).unwrap();
3696        let h1 = stage1.forward_layers(&h0, split..LAYERS, 0, None).unwrap();
3697        let h1 = stage1.apply_final_norm(&h1).unwrap();
3698        let logits = stage1.lm_head(&h1).unwrap();
3699
3700        let max_diff = (&ref_logits - &logits).abs().max().double_value(&[]);
3701        assert!(
3702            ref_logits.allclose(&logits, 1e-4, 1e-4, false),
3703            "two-stage split (same device) must equal whole-model forward \
3704             (max_diff={max_diff}); a mismatch here means the stage-local SSM state \
3705             threading is wrong"
3706        );
3707    }
3708
3709    #[test]
3710    fn forward_layers_rejects_out_of_window_range() {
3711        // A stage that owns [2..4) must reject a range outside its window.
3712        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
3713        let mut w = tiny_weights();
3714        let stage = Qwen3_5Model::stage_from_weights_with_config(
3715            &mut w,
3716            tiny_config(),
3717            &map,
3718            2..LAYERS,
3719            Kind::Float,
3720            KVQuantType::None,
3721        )
3722        .unwrap();
3723        let emb = Tensor::randn([1, 3, HIDDEN], (Kind::Float, Device::Cpu));
3724        assert!(
3725            stage.forward_layers(&emb, 0..2, 0, None).is_err(),
3726            "range below window"
3727        );
3728        assert!(
3729            stage.forward_layers(&emb, 2..LAYERS, 0, None).is_ok(),
3730            "owned range ok"
3731        );
3732    }
3733
3734    #[test]
3735    fn stage_loader_rejects_missing_required_weights() {
3736        // Last stage with no norm weight must error (M-LOAD seam #1 gating).
3737        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
3738        let mut w = tiny_weights();
3739        w.remove("model.norm.weight");
3740        let result = Qwen3_5Model::stage_from_weights_with_config(
3741            &mut w,
3742            tiny_config(),
3743            &map,
3744            0..LAYERS,
3745            Kind::Float,
3746            KVQuantType::None,
3747        );
3748        match result {
3749            Ok(_) => panic!("last stage without norm weight must error"),
3750            Err(e) => assert!(e.to_string().contains("norm"), "got: {e}"),
3751        }
3752    }
3753
3754    // ========================================================================
3755    // #316 — TTT-on-split cross-device autograd equivalence (CPU-verifiable),
3756    // exercising the hybrid GDN + full-attention stack across a stage boundary.
3757    // ========================================================================
3758
3759    use crate::training::tenant_delta::{TenantDelta, TenantDeltaConfig};
3760
3761    /// Per-layer o_proj LoRA delta on CPU. o_proj is the one module BOTH mixer
3762    /// kinds inject (GDN out_proj: in=val_dim; full-attn: in=nh*hd), so its dims
3763    /// differ per layer type — supplied via per_layer_dims (mirrors
3764    /// `TorchEngine::get_per_layer_lora_dims`). Seeds deterministic non-zero A and
3765    /// B so gradients w.r.t. both are non-trivial.
3766    fn tiny_delta() -> TenantDelta {
3767        let hidden = HIDDEN as usize;
3768        let gdn_in = LIN_V_HEADS * LIN_V_DIM; // out_proj input dim for GDN layers
3769        let full_in = (HEADS * HEAD_DIM) as usize; // o_proj input dim for full-attn layers
3770        let mut per_layer: std::collections::HashMap<
3771            usize,
3772            std::collections::HashMap<String, (usize, usize)>,
3773        > = std::collections::HashMap::new();
3774        for i in 0..LAYERS {
3775            let is_full = (i + 1) % 4 == 0;
3776            let mut m = std::collections::HashMap::new();
3777            let in_dim = if is_full { full_in } else { gdn_in };
3778            m.insert("o_proj".to_owned(), (in_dim, hidden));
3779            per_layer.insert(i, m);
3780        }
3781        let cfg = TenantDeltaConfig {
3782            rank: 2,
3783            alpha: 2.0,
3784            target_modules: vec!["o_proj".to_owned()],
3785            ..Default::default()
3786        };
3787        let flat: std::collections::HashMap<String, (usize, usize)> =
3788            std::collections::HashMap::new();
3789        let delta = TenantDelta::new_with_per_layer_dims(
3790            &cfg,
3791            &flat,
3792            Device::Cpu,
3793            LAYERS,
3794            Some(&per_layer),
3795        )
3796        .unwrap();
3797
3798        // Seed by a STABLE per-key offset (HashMap iteration order is not
3799        // deterministic, so independent builds must agree key-for-key).
3800        let key_offset = |key: &str| -> i64 { key.bytes().map(|b| b as i64).sum::<i64>() % 17 };
3801        let _g = tch::no_grad_guard();
3802        for (k, a) in delta.lora_a.iter() {
3803            let n: i64 = a.size().iter().product();
3804            let vals = (Tensor::arange(n, (Kind::Float, Device::Cpu)) + key_offset(k)).sin() * 0.1;
3805            // copy_ needs &mut; shallow_clone shares storage so the delta param is mutated.
3806            a.shallow_clone().copy_(&vals.reshape(a.size()));
3807        }
3808        for (k, b) in delta.lora_b.iter() {
3809            let n: i64 = b.size().iter().product();
3810            let vals =
3811                (Tensor::arange(n, (Kind::Float, Device::Cpu)) + key_offset(k) + 7).cos() * 0.1;
3812            b.shallow_clone().copy_(&vals.reshape(b.size()));
3813        }
3814        delta
3815    }
3816
3817    fn grad_snapshot(delta: &TenantDelta) -> std::collections::HashMap<String, f64> {
3818        let mut out = std::collections::HashMap::new();
3819        for (k, a) in &delta.lora_a {
3820            assert!(a.grad().defined(), "A grad undefined for {k}");
3821            out.insert(format!("A:{k}"), a.grad().norm().double_value(&[]));
3822        }
3823        for (k, b) in &delta.lora_b {
3824            assert!(b.grad().defined(), "B grad undefined for {k}");
3825            out.insert(format!("B:{k}"), b.grad().norm().double_value(&[]));
3826        }
3827        out
3828    }
3829
3830    /// The #316 correctness guardrail for the hybrid architecture: a TTT/training
3831    /// step (forward_layers_train → NTP loss → backward) over a two-stage CPU split
3832    /// must produce the SAME delta gradients as the whole-model training forward,
3833    /// within ~1e-4. The split point (k=2) keeps the full-attention layer in stage1
3834    /// and GDN layers in stage0, so this proves cross-device autograd is correct
3835    /// across both mixer kinds AND the (fresh, stage-local) SSM state threading.
3836    #[test]
3837    fn ttt_split_autograd_matches_whole_model_grads() {
3838        use crate::training::pipeline::{compute_ntp_loss_split, TrainStage};
3839
3840        let input = Tensor::from_slice(&[3i64, 7, 1, 4, 9, 2]).reshape([1, 6]);
3841
3842        // (a) whole-model training forward over a single-device map.
3843        let whole = whole_model();
3844        let whole_delta = tiny_delta();
3845        let whole_stage = [TrainStage {
3846            model: &whole,
3847            range: 0..LAYERS,
3848        }];
3849        let loss_whole = compute_ntp_loss_split(&whole_stage, &input, Some(&whole_delta)).unwrap();
3850        loss_whole.backward();
3851        let whole_grads = grad_snapshot(&whole_delta);
3852        whole_delta.zero_grad();
3853
3854        // (b) two-stage split over an all-CPU map.
3855        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
3856        let split = LAYERS / 2;
3857        let mut w0 = tiny_weights();
3858        let stage0 = Qwen3_5Model::stage_from_weights_with_config(
3859            &mut w0,
3860            tiny_config(),
3861            &map,
3862            0..split,
3863            Kind::Float,
3864            KVQuantType::None,
3865        )
3866        .unwrap();
3867        let mut w1 = tiny_weights();
3868        let stage1 = Qwen3_5Model::stage_from_weights_with_config(
3869            &mut w1,
3870            tiny_config(),
3871            &map,
3872            split..LAYERS,
3873            Kind::Float,
3874            KVQuantType::None,
3875        )
3876        .unwrap();
3877        let split_delta = tiny_delta();
3878        let stages = [
3879            TrainStage {
3880                model: &stage0,
3881                range: 0..split,
3882            },
3883            TrainStage {
3884                model: &stage1,
3885                range: split..LAYERS,
3886            },
3887        ];
3888        let loss_split = compute_ntp_loss_split(&stages, &input, Some(&split_delta)).unwrap();
3889        loss_split.backward();
3890        let split_grads = grad_snapshot(&split_delta);
3891
3892        let lw = loss_whole.double_value(&[]);
3893        let ls = loss_split.double_value(&[]);
3894        assert!(
3895            (lw - ls).abs() < 1e-4,
3896            "loss diverged: whole={lw} split={ls}"
3897        );
3898
3899        assert_eq!(whole_grads.len(), split_grads.len());
3900        for (k, &gw) in &whole_grads {
3901            let gs = *split_grads
3902                .get(k)
3903                .unwrap_or_else(|| panic!("missing grad {k}"));
3904            assert!(
3905                (gw - gs).abs() <= 1e-4 + 1e-4 * gw.abs(),
3906                "grad norm diverged for {k}: whole={gw} split={gs}"
3907            );
3908            assert!(
3909                gw > 0.0,
3910                "grad for {k} is zero — autograd path not exercised"
3911            );
3912        }
3913    }
3914
3915    /// #440 regression guard: BF16 base model + fp32 TTT/LoRA delta must NOT panic
3916    /// in the delta-injection path (the #139 class of dtype mismatch).
3917    ///
3918    /// The base model is built BF16 (`Kind::BFloat16`) while `tiny_delta()` builds
3919    /// an fp32 delta (matching `ttn_profile.rs` forcing `Kind::Float`). Before the
3920    /// #440 fix the qwen3_5 delta adds (`q_proj`/`v_proj`/`o_proj`) added fp32
3921    /// directly to bf16 activations → `addmm_impl_cpu_` panic. This drives a full
3922    /// forward through both mixer kinds (GDN + full-attn) with the delta active.
3923    #[test]
3924    fn forward_with_bf16_base_and_fp32_delta_does_not_panic() {
3925        let _g = tch::no_grad_guard();
3926        let dtype = Kind::BFloat16;
3927
3928        // Build the model in BF16 (cast tiny_weights to bf16 first).
3929        let mut w = tiny_weights();
3930        for t in w.values_mut() {
3931            *t = t.to_kind(dtype);
3932        }
3933        let model = Qwen3_5Model::from_weights(
3934            &mut w,
3935            tiny_config(),
3936            None,
3937            &Device::Cpu,
3938            dtype,
3939            KVQuantType::None,
3940            false,
3941        )
3942        .unwrap();
3943
3944        // fp32 delta (tiny_delta builds Kind::Float A/B), exercising o_proj on both
3945        // mixer kinds across all layers.
3946        let delta = tiny_delta();
3947
3948        let input = Tensor::from_slice(&[3i64, 7, 1, 4, 9, 2]).reshape([1, 6]);
3949        let emb = model.embed_tokens(&input).unwrap();
3950        assert_eq!(emb.kind(), dtype, "bf16 embeddings");
3951
3952        // The forward with delta must not panic on bf16 + fp32 mismatch...
3953        let out = model
3954            .forward_layers(&emb, 0..LAYERS, 0, Some(&delta))
3955            .expect("bf16 base + fp32 delta forward must not error");
3956        // ...and must remain in the base (bf16) dtype.
3957        assert_eq!(out.kind(), dtype, "delta-injected forward must stay bf16");
3958        assert!(
3959            out.isfinite().all().int64_value(&[]) == 1,
3960            "output must be finite"
3961        );
3962    }
3963
3964    #[test]
3965    fn forward_layers_train_rejects_out_of_window_range() {
3966        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
3967        let mut w = tiny_weights();
3968        let stage = Qwen3_5Model::stage_from_weights_with_config(
3969            &mut w,
3970            tiny_config(),
3971            &map,
3972            2..LAYERS,
3973            Kind::Float,
3974            KVQuantType::None,
3975        )
3976        .unwrap();
3977        let emb = Tensor::randn([1, 3, HIDDEN], (Kind::Float, Device::Cpu));
3978        assert!(
3979            stage.forward_layers_train(&emb, 0..2, None).is_err(),
3980            "range below window"
3981        );
3982        assert!(
3983            stage.forward_layers_train(&emb, 2..LAYERS, None).is_ok(),
3984            "owned range ok"
3985        );
3986    }
3987
3988    // ========================================================================
3989    // Session prefix-cache rewind: GDN conv/rec state must be rewound alongside
3990    // the KV cache on a prefix hit, or multi-turn sessions decode with the
3991    // previous turn's recurrent state.
3992    // ========================================================================
3993
3994    use parking_lot::Mutex;
3995
3996    fn session_cache() -> Arc<Mutex<KVCacheManager>> {
3997        Arc::new(Mutex::new(KVCacheManager::new(
3998            LAYERS,
3999            64,
4000            KVQuantType::None,
4001        )))
4002    }
4003
4004    /// Greedy argmax of a `[1, 1, vocab]` logits tensor.
4005    fn greedy_next(logits: &Tensor) -> i64 {
4006        logits
4007            .select(0, 0)
4008            .select(0, 0)
4009            .argmax(0, false)
4010            .int64_value(&[])
4011    }
4012
4013    /// Drive one session turn the way `TextStream` does: prefill
4014    /// `prompt[prefill_start..]` at `prefill_start`, then greedily decode
4015    /// `n_decode` tokens. Returns the generated token IDs, the per-step decode
4016    /// logits (`[vocab]` each), and the SSM (conv/rec) snapshot as of
4017    /// end-of-prefill.
4018    fn run_turn(
4019        model: &Qwen3_5Model,
4020        prompt: &[i64],
4021        prefill_start: usize,
4022        n_decode: usize,
4023    ) -> (
4024        Vec<i64>,
4025        Vec<Tensor>,
4026        (Vec<Option<Tensor>>, Vec<Option<Tensor>>),
4027    ) {
4028        let _g = tch::no_grad_guard();
4029        let prefill = Tensor::from_slice(&prompt[prefill_start..])
4030            .reshape([1, (prompt.len() - prefill_start) as i64]);
4031        let logits = model.forward_with_cache(&prefill, prefill_start).unwrap();
4032        // Engine captures the SSM snapshot here (TextStream::sample_next_token),
4033        // before decode advances the recurrent state.
4034        let ssm = model.snapshot_ssm_states();
4035
4036        let mut out = Vec::with_capacity(n_decode);
4037        let mut step_logits = Vec::with_capacity(n_decode);
4038        let mut next = greedy_next(&logits);
4039        out.push(next);
4040        step_logits.push(logits.select(0, 0).select(0, 0));
4041        for pos in prompt.len()..prompt.len() + n_decode - 1 {
4042            let step = Tensor::from_slice(&[next]).reshape([1, 1]);
4043            let logits = model.forward_with_cache(&step, pos).unwrap();
4044            next = greedy_next(&logits);
4045            out.push(next);
4046            step_logits.push(logits.select(0, 0).select(0, 0));
4047        }
4048        (out, step_logits, ssm)
4049    }
4050
4051    /// Assert two per-step logit traces match step-for-step.
4052    fn assert_logits_match(actual: &[Tensor], expected: &[Tensor], context: &str) {
4053        assert_eq!(actual.len(), expected.len(), "{context}: step count");
4054        for (i, (a, e)) in actual.iter().zip(expected).enumerate() {
4055            let max_diff = (a - e).abs().max().double_value(&[]);
4056            assert!(
4057                a.allclose(e, 1e-5, 1e-5, false),
4058                "{context}: decode step {i} logits diverged (max_diff={max_diff})"
4059            );
4060        }
4061    }
4062
4063    /// Bug characterization (non-vacuity guard for the regression test below):
4064    /// the pre-fix hit path — `truncate_to` only, no SSM rewind — leaves the
4065    /// GDN conv/rec state reflecting turn 1's generation, and turn 2's decode
4066    /// MUST diverge from a fresh-cache reference. If this ever stops
4067    /// diverging, the fixture no longer exercises the bug and
4068    /// `session_prefix_hit_restores_ssm_state_matches_fresh_cache` is vacuous.
4069    /// (Logits are compared directly: with tiny random weights the argmax can
4070    /// coincide even when the state is wrong.)
4071    #[test]
4072    fn session_prefix_hit_without_ssm_rewind_diverges_from_fresh_cache() {
4073        let prompt1 = [1i64, 5, 9, 2, 7, 3];
4074        let suffix = [8i64, 4];
4075        let prompt2: Vec<i64> = prompt1.iter().chain(&suffix).copied().collect();
4076
4077        // Session model: turn 1, then turn 2 with the old truncate-only hit path.
4078        let model = whole_model();
4079        let cache = session_cache();
4080        let mut m = model;
4081        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache.clone());
4082        let model = m;
4083
4084        let (_gen1, _logits1, _ssm1) = run_turn(&model, &prompt1, 0, 3);
4085        cache.lock().set_cached_tokens(prompt1.to_vec());
4086
4087        let prefix_len = cache.lock().prefix_match_len(&prompt2);
4088        assert_eq!(prefix_len, prompt1.len());
4089        cache.lock().truncate_to(prefix_len);
4090        let (_gen2_stale, logits2_stale, _ssm2) = run_turn(&model, &prompt2, prefix_len, 4);
4091
4092        // Reference: fresh model + fresh cache, full prefill of turn 2's prompt.
4093        let ref_model = whole_model();
4094        let ref_cache = session_cache();
4095        let mut m = ref_model;
4096        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4097        let ref_model = m;
4098        let (_gen2_ref, logits2_ref, _ssm_ref) = run_turn(&ref_model, &prompt2, 0, 4);
4099
4100        let max_diff: f64 = logits2_stale
4101            .iter()
4102            .zip(&logits2_ref)
4103            .map(|(a, e)| (a - e).abs().max().double_value(&[]))
4104            .fold(0.0, f64::max);
4105        assert!(
4106            max_diff > 1e-5,
4107            "stale-state fixture must diverge from fresh cache (max_diff={max_diff}); \
4108             otherwise the rewind regression test proves nothing"
4109        );
4110    }
4111
4112    /// Regression test: a session prefix-cache hit must rewind the GDN conv/rec
4113    /// state (via the end-of-prefill snapshot saved with the cached tokens)
4114    /// alongside the KV truncation. The rewound run must reproduce, to
4115    /// bit-exact tolerance, the oracle computation "prefill the cached prompt,
4116    /// then continue with the new suffix" (split-prefill ground truth), and
4117    /// its greedy tokens must match a monolithic fresh-cache prefill
4118    /// token-for-token. Exercises the production rewind seam
4119    /// `torch_engine::rewind_session_state`.
4120    #[test]
4121    fn session_prefix_hit_restores_ssm_state_matches_fresh_cache() {
4122        let prompt1 = [1i64, 5, 9, 2, 7, 3];
4123        let suffix = [8i64, 4];
4124        let prompt2: Vec<i64> = prompt1.iter().chain(&suffix).copied().collect();
4125
4126        // Session model: turn 1, save tokens + end-of-prefill SSM snapshot.
4127        let model = whole_model();
4128        let cache = session_cache();
4129        let mut m = model;
4130        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache.clone());
4131        let model = m;
4132
4133        let (_gen1, _logits1, ssm1) = run_turn(&model, &prompt1, 0, 3);
4134        cache
4135            .lock()
4136            .set_cached_tokens_with_ssm(prompt1.to_vec(), Some(ssm1));
4137
4138        // Turn 2: full cached prefix matches — the production rewind must
4139        // restore the snapshot and reuse the prefix.
4140        let prefix_len = cache.lock().prefix_match_len(&prompt2);
4141        assert_eq!(prefix_len, prompt1.len());
4142        let start = crate::runtime::torch_engine::rewind_session_state(&model, prefix_len);
4143        assert_eq!(start, prompt1.len(), "full cached prefix must be reusable");
4144        let (gen2, logits2, _ssm2) = run_turn(&model, &prompt2, start, 4);
4145
4146        // Oracle: fresh model, genuinely prefill prompt1 (no decode), then
4147        // continue with the suffix — the exact computation the rewind is
4148        // supposed to reproduce.
4149        let oracle_model = whole_model();
4150        let oracle_cache = session_cache();
4151        let mut m = oracle_model;
4152        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, oracle_cache);
4153        let oracle_model = m;
4154        {
4155            let _g = tch::no_grad_guard();
4156            let prefill1 = Tensor::from_slice(&prompt1).reshape([1, prompt1.len() as i64]);
4157            let _ = oracle_model.forward_with_cache(&prefill1, 0).unwrap();
4158        }
4159        let (gen2_oracle, logits2_oracle, _ssm_o) =
4160            run_turn(&oracle_model, &prompt2, prompt1.len(), 4);
4161
4162        assert_eq!(
4163            gen2, gen2_oracle,
4164            "rewound run must match split-prefill oracle"
4165        );
4166        assert_logits_match(
4167            &logits2,
4168            &logits2_oracle,
4169            "rewound run vs split-prefill oracle",
4170        );
4171
4172        // Contract-level check: greedy tokens must also match a monolithic
4173        // fresh-cache prefill of turn 2's full prompt. (Logits are NOT
4174        // compared here: split vs monolithic prefill legitimately differ by
4175        // float reassociation, exactly as for pure-attention KV prefix reuse.)
4176        let ref_model = whole_model();
4177        let ref_cache = session_cache();
4178        let mut m = ref_model;
4179        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4180        let ref_model = m;
4181        let (gen2_ref, _logits2_ref, _ssm_ref) = run_turn(&ref_model, &prompt2, 0, 4);
4182        assert_eq!(
4183            gen2, gen2_ref,
4184            "prefix-hit decode must match a fresh-cache run token-for-token"
4185        );
4186    }
4187
4188    /// A PARTIAL prefix match (divergence inside the cached sequence) cannot
4189    /// rewind the Markovian recurrent state to an intermediate position, so
4190    /// the production rewind must discard the cache and force a full recompute
4191    /// — which must then match a fresh-cache reference exactly.
4192    #[test]
4193    fn session_partial_prefix_match_forces_full_recompute() {
4194        let prompt1 = [1i64, 5, 9, 2, 7, 3];
4195        // Shares only the first 3 tokens with prompt1.
4196        let prompt2 = [1i64, 5, 9, 4, 8, 6, 2];
4197
4198        let model = whole_model();
4199        let cache = session_cache();
4200        let mut m = model;
4201        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache.clone());
4202        let model = m;
4203
4204        let (_gen1, _logits1, ssm1) = run_turn(&model, &prompt1, 0, 3);
4205        cache
4206            .lock()
4207            .set_cached_tokens_with_ssm(prompt1.to_vec(), Some(ssm1));
4208
4209        let prefix_len = cache.lock().prefix_match_len(&prompt2);
4210        assert!(0 < prefix_len && prefix_len < prompt1.len());
4211        let start = crate::runtime::torch_engine::rewind_session_state(&model, prefix_len);
4212        assert_eq!(start, 0, "partial prefix match must force full recompute");
4213        // The rewind must also have reset the live SSM state (no leakage of
4214        // turn 1's recurrent state into the recomputed prefill).
4215        assert!(model.conv_states.lock().iter().all(Option::is_none));
4216        assert!(model.rec_states.lock().iter().all(Option::is_none));
4217
4218        let (gen2, logits2, _ssm2) = run_turn(&model, &prompt2, 0, 4);
4219
4220        let ref_model = whole_model();
4221        let ref_cache = session_cache();
4222        let mut m = ref_model;
4223        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4224        let ref_model = m;
4225        let (gen2_ref, logits2_ref, _ssm_ref) = run_turn(&ref_model, &prompt2, 0, 4);
4226
4227        assert_eq!(gen2, gen2_ref);
4228        assert_logits_match(&logits2, &logits2_ref, "recomputed prefill vs fresh cache");
4229    }
4230
4231    /// Prefix MISS / stateless reset guard: `clear_kv_cache` (what the engine
4232    /// calls on a miss and for non-session requests) must fully reset the GDN
4233    /// conv/rec state, so no recurrent state leaks across requests.
4234    #[test]
4235    fn clear_kv_cache_fully_resets_ssm_state() {
4236        let prompt = [1i64, 5, 9, 2, 7, 3];
4237
4238        let model = whole_model();
4239        let cache = session_cache();
4240        let mut m = model;
4241        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache);
4242        let model = m;
4243
4244        let _ = run_turn(&model, &prompt, 0, 2);
4245        assert!(model.conv_states.lock().iter().any(Option::is_some));
4246        assert!(model.rec_states.lock().iter().any(Option::is_some));
4247
4248        <Qwen3_5Model as ModelOperations>::clear_kv_cache(&model);
4249        assert!(
4250            model.conv_states.lock().iter().all(Option::is_none),
4251            "conv state must be reset on cache clear"
4252        );
4253        assert!(
4254            model.rec_states.lock().iter().all(Option::is_none),
4255            "rec state must be reset on cache clear"
4256        );
4257
4258        // And a post-clear run must match a fresh-model run exactly.
4259        let (gen, logits, _ssm) = run_turn(&model, &prompt, 0, 3);
4260        let ref_model = whole_model();
4261        let ref_cache = session_cache();
4262        let mut m = ref_model;
4263        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4264        let ref_model = m;
4265        let (gen_ref, logits_ref, _ssm_ref) = run_turn(&ref_model, &prompt, 0, 3);
4266        assert_eq!(gen, gen_ref);
4267        assert_logits_match(&logits, &logits_ref, "post-clear run vs fresh model");
4268    }
4269
4270    /// Bug characterization (non-vacuity guard for the exact-repeat regression
4271    /// below): routing an EXACT-prompt repeat (`prefix_len == prompt_len`)
4272    /// through the snapshot restore re-runs the FULL prompt with the GDN
4273    /// recurrence starting from the previous turn's end-of-prefill state
4274    /// instead of zero — this is what the pre-fix hit path effectively did,
4275    /// because `sample_next_token` falls through to a full prefill when no
4276    /// suffix remains to prefill. The first re-forwarded row is then
4277    /// conditioned on turn 1's recurrent state and MUST diverge from a
4278    /// fresh-cache prefill. (The divergence decays across rows as the
4279    /// delta-rule recurrence forgets the restored state, so it is asserted on
4280    /// the FIRST row; the last row lands near the fixture's float-reassociation
4281    /// noise floor and proves nothing.)
4282    #[test]
4283    fn session_exact_prompt_hit_restored_reforward_diverges_from_fresh_cache() {
4284        // Long enough cached history that the restored end-of-prefill state is
4285        // appreciable at the first re-forwarded row.
4286        let prompt: Vec<i64> = [1i64, 5, 9, 2, 7, 3, 8, 4]
4287            .iter()
4288            .cycle()
4289            .take(24)
4290            .copied()
4291            .collect();
4292
4293        // Session model: turn 1, save tokens + end-of-prefill SSM snapshot.
4294        let model = whole_model();
4295        let cache = session_cache();
4296        let mut m = model;
4297        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache.clone());
4298        let model = m;
4299
4300        let (_gen1, _logits1, ssm1) = run_turn(&model, &prompt, 0, 3);
4301        cache
4302            .lock()
4303            .set_cached_tokens_with_ssm(prompt.clone(), Some(ssm1));
4304
4305        // The pre-fix exact-hit path: restore the snapshot, then re-run the
4306        // whole prompt at position 0. Row-wise logits of that re-forward (row
4307        // i is conditioned on the recurrence as of position i).
4308        let row_logits = |m: &Qwen3_5Model| -> Tensor {
4309            let _g = tch::no_grad_guard();
4310            let ids = Tensor::from_slice(&prompt).reshape([1, prompt.len() as i64]);
4311            let emb = m.embed_tokens(&ids).unwrap();
4312            let h = m.forward_layers(&emb, 0..m.num_layers(), 0, None).unwrap();
4313            let h = m.apply_final_norm(&h).unwrap();
4314            m.lm_head(&h).unwrap() // [1, seq, V]
4315        };
4316        let prefix_len = cache.lock().prefix_match_len(&prompt);
4317        assert_eq!(prefix_len, prompt.len(), "exact repeat must fully match");
4318        let start = crate::runtime::torch_engine::rewind_session_state(&model, prefix_len);
4319        assert_eq!(
4320            start,
4321            prompt.len(),
4322            "rewind alone still reports the full hit"
4323        );
4324        let broken_rows = row_logits(&model);
4325
4326        // Reference: fresh model + fresh cache, same row-wise prefill.
4327        let ref_model = whole_model();
4328        let ref_cache = session_cache();
4329        let mut m = ref_model;
4330        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4331        let ref_model = m;
4332        let fresh_rows = row_logits(&ref_model);
4333
4334        let first_row_diff = (&broken_rows.select(1, 0).reshape([-1i64])
4335            - &fresh_rows.select(1, 0).reshape([-1i64]))
4336            .abs()
4337            .max()
4338            .double_value(&[]);
4339        assert!(
4340            first_row_diff > 1e-4,
4341            "restore-then-reforward on an exact repeat must condition the first row on the \
4342             previous turn's recurrent state and diverge from a fresh prefill \
4343             (first-row max_diff={first_row_diff}); otherwise the cleared-recompute \
4344             regression test proves nothing"
4345        );
4346    }
4347
4348    /// Regression test (exact-prompt repeat): when the new prompt is IDENTICAL
4349    /// to the cached one, the production reuse decision must NOT route the hit
4350    /// through the end-of-prefill snapshot restore — there is no suffix left
4351    /// to prefill, and sampling the first token needs a fresh forward, which
4352    /// would double-advance the restored GDN recurrence (see the
4353    /// characterization test above). `resolve_session_prefill_start` must
4354    /// clear the cache and return 0, and the recomputed turn must then match
4355    /// a fresh-cache reference token-for-token and logit-for-logit.
4356    #[test]
4357    fn session_exact_prompt_repeat_forces_cleared_recompute() {
4358        let prompt = [1i64, 5, 9, 2, 7, 3];
4359
4360        let model = whole_model();
4361        let cache = session_cache();
4362        let mut m = model;
4363        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, cache.clone());
4364        let model = m;
4365
4366        let (_gen1, _logits1, ssm1) = run_turn(&model, &prompt, 0, 3);
4367        cache
4368            .lock()
4369            .set_cached_tokens_with_ssm(prompt.to_vec(), Some(ssm1));
4370
4371        let prefix_len = cache.lock().prefix_match_len(&prompt);
4372        assert_eq!(prefix_len, prompt.len(), "exact repeat must fully match");
4373        let start = crate::runtime::torch_engine::resolve_session_prefill_start(
4374            &model,
4375            prefix_len,
4376            prompt.len(),
4377        );
4378        assert_eq!(start, 0, "exact-prompt repeat must force full recompute");
4379        // Cleared state: neither the restored snapshot nor turn 1's live GDN
4380        // recurrence may leak into the recomputed prefill.
4381        assert!(model.conv_states.lock().iter().all(Option::is_none));
4382        assert!(model.rec_states.lock().iter().all(Option::is_none));
4383
4384        let (gen2, logits2, _ssm2) = run_turn(&model, &prompt, start, 4);
4385
4386        let ref_model = whole_model();
4387        let ref_cache = session_cache();
4388        let mut m = ref_model;
4389        <Qwen3_5Model as ModelOperations>::set_kv_cache(&mut m, ref_cache);
4390        let ref_model = m;
4391        let (gen2_ref, logits2_ref, _ssm_ref) = run_turn(&ref_model, &prompt, 0, 4);
4392
4393        assert_eq!(gen2, gen2_ref);
4394        assert_logits_match(
4395            &logits2,
4396            &logits2_ref,
4397            "recomputed exact repeat vs fresh cache",
4398        );
4399    }
4400    /// Regression guard for the causal-mask offset in cached multi-token
4401    /// forwards: with `q_len > 1` and `start_pos > 0` the mask must be
4402    /// `tril(kv_len - q_len)`, not `tril(0)` — otherwise every cached row
4403    /// attends only to the first `i+1` keys and all prompt history is masked
4404    /// out. This is exactly the serial partial-prefill shape taken on a session
4405    /// prefix-cache hit with a >=2-token suffix. Compares LOGITS (not just
4406    /// argmax) at fp tolerance.
4407    ///
4408    /// Uses an all-full-attention variant of the tiny model so the only
4409    /// divergence source is attention itself (the hybrid GDN stack adds ~1e-3
4410    /// chunked-vs-recurrent kernel noise, too loose for this guard).
4411    #[test]
4412    fn cached_two_token_forward_matches_serial_decode_steps() {
4413        let allattn_config = || {
4414            let mut cfg = tiny_config();
4415            cfg.layer_types = (0..cfg.num_hidden_layers)
4416                .map(|_| "full_attention".to_owned())
4417                .collect();
4418            cfg
4419        };
4420        let build = || {
4421            let mut w = tiny_weights();
4422            let cfg = allattn_config();
4423            // Replace GDN mixers with full-attention ones (deterministic pattern,
4424            // same style as tiny_weights).
4425            let opt = (Kind::Float, Device::Cpu);
4426            let mut seed: i64 = 20_000;
4427            let mut pat = |dims: &[i64]| -> Tensor {
4428                let n: i64 = dims.iter().product();
4429                seed += 7;
4430                (Tensor::arange(n, opt) * 0.017 + seed as f64 * 0.013)
4431                    .sin()
4432                    .reshape(dims)
4433                    * 0.05
4434            };
4435            for i in 0..cfg.num_hidden_layers as usize {
4436                if (i + 1) % 4 == 0 {
4437                    continue; // already full-attention in tiny_weights
4438                }
4439                let p = format!("model.layers.{i}");
4440                w.retain(|k, _| !k.starts_with(&format!("{p}.linear_attn.")));
4441                let ap = format!("{p}.self_attn");
4442                w.insert(
4443                    format!("{ap}.q_proj.weight"),
4444                    pat(&[HEADS * HEAD_DIM * 2, HIDDEN]),
4445                );
4446                w.insert(
4447                    format!("{ap}.k_proj.weight"),
4448                    pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
4449                );
4450                w.insert(
4451                    format!("{ap}.v_proj.weight"),
4452                    pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
4453                );
4454                w.insert(
4455                    format!("{ap}.o_proj.weight"),
4456                    pat(&[HIDDEN, HEADS * HEAD_DIM]),
4457                );
4458                w.insert(format!("{ap}.q_norm.weight"), pat(&[HEAD_DIM]));
4459                w.insert(format!("{ap}.k_norm.weight"), pat(&[HEAD_DIM]));
4460            }
4461            Qwen3_5Model::from_weights(
4462                &mut w,
4463                allattn_config(),
4464                None,
4465                &Device::Cpu,
4466                Kind::Float,
4467                KVQuantType::None,
4468                false,
4469            )
4470            .unwrap()
4471        };
4472
4473        let prompt: [i64; 6] = [1, 5, 9, 2, 7, 3];
4474        let prompt_t = Tensor::from_slice(&prompt).reshape([1, prompt.len() as i64]);
4475        let pos = prompt.len();
4476
4477        // Serial reference: prefill, then two one-token decode steps.
4478        let serial = build();
4479        let _ = serial.forward_with_cache(&prompt_t, 0).unwrap();
4480        let step1 = Tensor::from_slice(&[11i64]).reshape([1, 1]);
4481        let ref1 = serial.forward_with_cache(&step1, pos).unwrap(); // [1, 1, V]
4482        let step2 = Tensor::from_slice(&[13i64]).reshape([1, 1]);
4483        let ref2 = serial.forward_with_cache(&step2, pos + 1).unwrap();
4484
4485        // Cached two-token forward at start_pos=pos (partial-prefill shape).
4486        // forward_with_cache returns last-row logits only (#201), so go through
4487        // the forward_layers orchestration to get BOTH rows.
4488        let cached = build();
4489        let _ = cached.forward_with_cache(&prompt_t, 0).unwrap();
4490        let pair = Tensor::from_slice(&[11i64, 13]).reshape([1, 2]);
4491        let emb = cached.embed_tokens(&pair).unwrap();
4492        let h = cached
4493            .forward_layers(&emb, 0..cached.num_layers(), pos, None)
4494            .unwrap();
4495        let h = cached.apply_final_norm(&h).unwrap();
4496        let logits2 = cached.lm_head(&h).unwrap(); // [1, 2, V]
4497
4498        for (row, ref_l) in [(0i64, &ref1), (1, &ref2)] {
4499            let got = logits2.select(1, row).reshape([-1i64]);
4500            let want = ref_l.reshape([-1i64]);
4501            let max_diff = (&got - &want).abs().max().double_value(&[]);
4502            assert!(
4503                got.allclose(&want, 1e-4, 1e-4, false),
4504                "cached 2-token forward row {row} diverged from the serial decode step \
4505                 (max_diff={max_diff}); the causal mask must keep kv_len - q_len history, \
4506                 not tril(0)"
4507            );
4508            assert_eq!(
4509                got.argmax(-1, false).int64_value(&[]),
4510                want.argmax(-1, false).int64_value(&[]),
4511                "row {row} argmax differs",
4512            );
4513        }
4514    }
4515}
4516
4517#[cfg(test)]
4518#[allow(clippy::unwrap_used, clippy::expect_used)]
4519pub(crate) mod mtp_tests {
4520    //! Correctness gate for MTP self-speculative decoding (k=1, greedy).
4521    //!
4522    //! Speculative greedy output must equal the serial (non-speculative) greedy
4523    //! output token-for-token on a tiny deterministic dense Qwen3.5 model on
4524    //! CPU — including across a session prefix-cache hit, under forced draft
4525    //! rejection (the KV/SSM rewind path), and under oracle drafts (the accept
4526    //! path). The drivers below mirror `TextStream::sample_speculative_tokens`
4527    //! and `sample_next_token` at the model level (plain argmax, no repeat
4528    //! penalty), the same style as `batched_ragged_decode_matches_serial`.
4529    //!
4530    //! With near-random weights the MTP head's natural acceptance is ~0 — that
4531    //! is fine: correctness is the gate here, acceptance is measured on GPU.
4532    use super::pipeline_tests::{
4533        tiny_config, tiny_weights, HEADS, HEAD_DIM, HIDDEN, INTER, KV_HEADS, VOCAB,
4534    };
4535    use super::*;
4536
4537    /// tiny_weights plus the 15 dense `mtp.*` tensors (HF `[out, in]` layout).
4538    fn tiny_weights_mtp() -> HashMap<String, Tensor> {
4539        let mut w = tiny_weights();
4540        let opt = (Kind::Float, Device::Cpu);
4541        // Same deterministic sin-of-index pattern as tiny_weights, distinct offset.
4542        let mut seed: i64 = 10_000;
4543        let mut pat = |dims: &[i64]| -> Tensor {
4544            let n: i64 = dims.iter().product();
4545            seed += 7;
4546            (Tensor::arange(n, opt) * 0.017 + seed as f64 * 0.013)
4547                .sin()
4548                .reshape(dims)
4549                * 0.05
4550        };
4551
4552        w.insert("mtp.pre_fc_norm_hidden.weight".to_owned(), pat(&[HIDDEN]));
4553        w.insert(
4554            "mtp.pre_fc_norm_embedding.weight".to_owned(),
4555            pat(&[HIDDEN]),
4556        );
4557        w.insert("mtp.fc.weight".to_owned(), pat(&[HIDDEN, 2 * HIDDEN]));
4558        let lp = "mtp.layers.0";
4559        w.insert(format!("{lp}.input_layernorm.weight"), pat(&[HIDDEN]));
4560        w.insert(
4561            format!("{lp}.post_attention_layernorm.weight"),
4562            pat(&[HIDDEN]),
4563        );
4564        let ap = format!("{lp}.self_attn");
4565        w.insert(
4566            format!("{ap}.q_proj.weight"),
4567            pat(&[HEADS * HEAD_DIM * 2, HIDDEN]),
4568        );
4569        w.insert(
4570            format!("{ap}.k_proj.weight"),
4571            pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
4572        );
4573        w.insert(
4574            format!("{ap}.v_proj.weight"),
4575            pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
4576        );
4577        w.insert(
4578            format!("{ap}.o_proj.weight"),
4579            pat(&[HIDDEN, HEADS * HEAD_DIM]),
4580        );
4581        w.insert(format!("{ap}.q_norm.weight"), pat(&[HEAD_DIM]));
4582        w.insert(format!("{ap}.k_norm.weight"), pat(&[HEAD_DIM]));
4583        let mp = format!("{lp}.mlp");
4584        w.insert(format!("{mp}.gate_proj.weight"), pat(&[INTER, HIDDEN]));
4585        w.insert(format!("{mp}.up_proj.weight"), pat(&[INTER, HIDDEN]));
4586        w.insert(format!("{mp}.down_proj.weight"), pat(&[HIDDEN, INTER]));
4587        w.insert("mtp.norm.weight".to_owned(), pat(&[HIDDEN]));
4588        w
4589    }
4590
4591    pub(crate) fn whole_model_mtp() -> Qwen3_5Model {
4592        let mut w = tiny_weights_mtp();
4593        Qwen3_5Model::from_weights(
4594            &mut w,
4595            tiny_config(),
4596            None,
4597            &Device::Cpu,
4598            Kind::Float,
4599            KVQuantType::None,
4600            true,
4601        )
4602        .unwrap()
4603    }
4604
4605    #[test]
4606    fn mtp_head_obeys_explicit_toml_runtime_setting() {
4607        for enabled in [false, true] {
4608            let configured = crate::config::RuntimeConfig {
4609                speculative_decoding: enabled,
4610                use_gpu: false,
4611                ..Default::default()
4612            };
4613            let toml = toml::to_string(&configured).unwrap();
4614            let runtime: crate::config::RuntimeConfig = toml::from_str(&toml).unwrap();
4615            let mut weights = tiny_weights_mtp();
4616            let model = Qwen3_5Model::from_weights(
4617                &mut weights,
4618                tiny_config(),
4619                None,
4620                &Device::Cpu,
4621                Kind::Float,
4622                KVQuantType::None,
4623                runtime.speculative_decoding,
4624            )
4625            .unwrap();
4626            assert_eq!(model.has_mtp(), enabled);
4627            assert!(!weights.keys().any(|name| name.starts_with("mtp.")));
4628        }
4629    }
4630
4631    #[test]
4632    fn mtp_quantized_direct_model_drops_head() {
4633        for kv in [
4634            KVQuantType::None,
4635            KVQuantType::Int8,
4636            KVQuantType::Nf4,
4637            KVQuantType::Fp4,
4638        ] {
4639            let mut weights = tiny_weights_mtp();
4640            let model = Qwen3_5Model::from_weights(
4641                &mut weights,
4642                tiny_config(),
4643                None,
4644                &Device::Cpu,
4645                Kind::Float,
4646                kv,
4647                true,
4648            )
4649            .unwrap();
4650            assert_eq!(model.has_mtp(), kv == KVQuantType::None, "KV mode {kv:?}");
4651            assert!(!weights.keys().any(|name| name.starts_with("mtp.")));
4652        }
4653    }
4654
4655    /// Greedy argmax of a single position's logits (any leading dims).
4656    fn argmax1(logits: &Tensor) -> i64 {
4657        logits.reshape([-1i64]).argmax(-1, false).int64_value(&[])
4658    }
4659
4660    /// Serial reference: the trusted batch=1 path (prefill + one-token decode
4661    /// steps), plain argmax. `prefill_from` > 0 simulates a session
4662    /// prefix-cache hit: only the prompt suffix is forwarded.
4663    fn serial_generate(
4664        model: &Qwen3_5Model,
4665        prompt: &[i64],
4666        gen_tokens: usize,
4667        prefill_from: usize,
4668    ) -> Vec<i64> {
4669        let _g = tch::no_grad_guard();
4670        let (slice, start_pos) = if prefill_from > 0 {
4671            (&prompt[prefill_from..], prefill_from)
4672        } else {
4673            (prompt, 0)
4674        };
4675        let prompt_t = Tensor::from_slice(slice).reshape([1, slice.len() as i64]);
4676        let logits = model.forward_with_cache(&prompt_t, start_pos).unwrap();
4677        let mut out = Vec::with_capacity(gen_tokens);
4678        let mut last = argmax1(&logits);
4679        out.push(last);
4680        let mut pos = prompt.len();
4681        while out.len() < gen_tokens {
4682            let dec = Tensor::from_slice(&[last]).reshape([1, 1]);
4683            let logits = model.forward_with_cache(&dec, pos).unwrap();
4684            last = argmax1(&logits);
4685            out.push(last);
4686            pos += 1;
4687        }
4688        out
4689    }
4690
4691    #[derive(Clone, Copy, PartialEq, Eq)]
4692    enum DraftMode {
4693        /// Use the MTP head's own drafts (near-random weights → ~0 acceptance).
4694        Natural,
4695        /// Corrupt every draft before the verify forward, forcing the reject
4696        /// rewind path each round.
4697        ForceReject,
4698        /// Draft the true next token (from a precomputed serial run), forcing
4699        /// the accept path each round.
4700        Oracle,
4701    }
4702
4703    /// Speculative driver mirroring `TextStream::sample_speculative_tokens` at
4704    /// the model level. Returns (tokens, accepted, rejected).
4705    fn spec_generate(
4706        model: &Qwen3_5Model,
4707        prompt: &[i64],
4708        gen_tokens: usize,
4709        prefill_from: usize,
4710        mode: DraftMode,
4711        oracle: Option<&[i64]>,
4712    ) -> (Vec<i64>, u64, u64) {
4713        let _g = tch::no_grad_guard();
4714        let (slice, start_pos) = if prefill_from > 0 {
4715            (&prompt[prefill_from..], prefill_from)
4716        } else {
4717            (prompt, 0)
4718        };
4719
4720        // Prefill with hidden capture, then MTP prompt prefill + first draft.
4721        let prompt_t = Tensor::from_slice(slice).reshape([1, slice.len() as i64]);
4722        let (logits, hidden) = model
4723            .forward_with_cache_hidden(&prompt_t, start_pos, true)
4724            .unwrap();
4725        let mut out: Vec<i64> = Vec::with_capacity(gen_tokens);
4726        let mut last = argmax1(&logits);
4727        out.push(last);
4728        let mut mtp_tokens: Vec<i64> = slice[1..].to_vec();
4729        mtp_tokens.push(last);
4730        let mtp_in = Tensor::from_slice(&mtp_tokens).reshape([1, mtp_tokens.len() as i64]);
4731        let mut pending = argmax1(&model.mtp_forward(&mtp_in, &hidden, start_pos).unwrap());
4732
4733        let mut pos = prompt.len();
4734        let (mut accepted, mut rejected) = (0u64, 0u64);
4735        while out.len() < gen_tokens {
4736            let draft = match mode {
4737                DraftMode::Natural => pending,
4738                DraftMode::ForceReject => (pending + 1) % VOCAB,
4739                DraftMode::Oracle => {
4740                    oracle.expect("oracle mode needs the serial output")[out.len()]
4741                }
4742            };
4743
4744            let snapshot = model.snapshot_ssm_states();
4745            let verify_in = Tensor::from_slice(&[last, draft]).reshape([1, 2]);
4746            let (logits, hidden) = model
4747                .forward_with_cache_hidden(&verify_in, pos, false)
4748                .unwrap();
4749            let v1 = argmax1(&logits.select(1, 0));
4750
4751            if v1 == draft {
4752                // Accept: emit draft + bonus token, re-draft from both new slots.
4753                let v2 = argmax1(&logits.select(1, 1));
4754                let mtp_in = Tensor::from_slice(&[v1, v2]).reshape([1, 2]);
4755                pending = argmax1(&model.mtp_forward(&mtp_in, &hidden, pos).unwrap());
4756                out.push(v1);
4757                out.push(v2);
4758                last = v2;
4759                pos += 2;
4760                accepted += 1;
4761            } else {
4762                // Reject: rewind SSM + KV, re-forward the verified token alone
4763                // to re-sync GDN state, emit only the correction token.
4764                model.restore_ssm_states(snapshot.0, snapshot.1);
4765                model.get_kv_cache().unwrap().lock().truncate_to(pos + 1);
4766                let re_in = Tensor::from_slice(&[last]).reshape([1, 1]);
4767                let (_, hidden2) = model.forward_with_cache_hidden(&re_in, pos, true).unwrap();
4768                let mtp_in = Tensor::from_slice(&[v1]).reshape([1, 1]);
4769                pending = argmax1(&model.mtp_forward(&mtp_in, &hidden2, pos).unwrap());
4770                out.push(v1);
4771                last = v1;
4772                pos += 1;
4773                rejected += 1;
4774            }
4775        }
4776        out.truncate(gen_tokens);
4777        (out, accepted, rejected)
4778    }
4779
4780    const PROMPT: [i64; 6] = [1, 5, 9, 2, 7, 3];
4781    const GEN: usize = 12;
4782
4783    /// The core gate: natural MTP drafts, speculative == serial token-for-token.
4784    #[test]
4785    fn mtp_decode_matches_serial() {
4786        let reference = serial_generate(&whole_model_mtp(), &PROMPT, GEN, 0);
4787        let (spec, accepted, rejected) = spec_generate(
4788            &whole_model_mtp(),
4789            &PROMPT,
4790            GEN,
4791            0,
4792            DraftMode::Natural,
4793            None,
4794        );
4795        assert_eq!(
4796            spec, reference,
4797            "speculative output diverged from serial greedy"
4798        );
4799        assert!(accepted + rejected > 0, "no speculative rounds ran");
4800    }
4801
4802    /// Forced-reject: every draft is corrupted before verify, so every round
4803    /// exercises the SSM-restore + KV-truncate + re-forward rewind path.
4804    #[test]
4805    fn mtp_decode_forced_reject_matches_serial() {
4806        let reference = serial_generate(&whole_model_mtp(), &PROMPT, GEN, 0);
4807        let (spec, _accepted, rejected) = spec_generate(
4808            &whole_model_mtp(),
4809            &PROMPT,
4810            GEN,
4811            0,
4812            DraftMode::ForceReject,
4813            None,
4814        );
4815        assert_eq!(
4816            spec, reference,
4817            "forced-reject rewind diverged from serial greedy"
4818        );
4819        assert!(
4820            rejected > 0,
4821            "forced-reject mode never rejected — rewind path untested"
4822        );
4823    }
4824
4825    /// Oracle drafts: acceptance is ~100%, exercising the 2-token accept path
4826    /// (bonus token emission + two-slot re-draft).
4827    #[test]
4828    fn mtp_decode_oracle_accept_matches_serial() {
4829        let reference = serial_generate(&whole_model_mtp(), &PROMPT, GEN, 0);
4830        let (spec, accepted, _rejected) = spec_generate(
4831            &whole_model_mtp(),
4832            &PROMPT,
4833            GEN,
4834            0,
4835            DraftMode::Oracle,
4836            Some(&reference),
4837        );
4838        assert_eq!(
4839            spec, reference,
4840            "oracle-accept output diverged from serial greedy"
4841        );
4842        assert!(
4843            accepted > 0,
4844            "oracle drafts never accepted — accept path untested"
4845        );
4846    }
4847
4848    /// Context-boundary regression: when only one KV slot remains
4849    /// (`kv_position == max_seq_len - 1`), the 2-token verify forward would
4850    /// exceed LayerKVCache::update's hard `end_pos <= max_seq_len` bound and
4851    /// kill the stream with a cache-overflow error. The speculative driver must
4852    /// fall back to a serial single-token step for that round (only that
4853    /// round), yielding the same final token the serial path produces.
4854    ///
4855    /// Mirrors `TextStream::sample_speculative_tokens`' boundary check at the
4856    /// model level with forced-reject drafts (deterministic positions: every
4857    /// round advances exactly one slot, so a round provably starts at
4858    /// `cap - 1`). The KV manager is swapped for a small-capacity one so the
4859    /// KV bound — not the RoPE table range (both default to
4860    /// max_position_embeddings) — is the operative limit, matching production
4861    /// with a `max_context` override.
4862    #[test]
4863    fn mtp_decode_falls_back_to_serial_at_context_boundary() {
4864        let _g = tch::no_grad_guard();
4865        const CAP: usize = 12; // prompt (6) + exactly 6 forwards fit
4866        let build = || {
4867            let mut w = tiny_weights_mtp();
4868            let mut model = Qwen3_5Model::from_weights(
4869                &mut w,
4870                tiny_config(),
4871                None,
4872                &Device::Cpu,
4873                Kind::Float,
4874                KVQuantType::None,
4875                true,
4876            )
4877            .unwrap();
4878            let layers = model.num_layers();
4879            model.set_kv_cache(std::sync::Arc::new(parking_lot::Mutex::new(
4880                crate::runtime::kv_cache::KVCacheManager::new(layers + 1, CAP, KVQuantType::None),
4881            )));
4882            model
4883        };
4884
4885        // Serial reference: decode until the cache overflows; keep the tokens.
4886        let serial_until_error = |model: &Qwen3_5Model| -> (Vec<i64>, Option<String>) {
4887            let prompt_t = Tensor::from_slice(&PROMPT).reshape([1, PROMPT.len() as i64]);
4888            let mut out = Vec::new();
4889            match model.forward_with_cache(&prompt_t, 0) {
4890                Ok(l) => out.push(argmax1(&l)),
4891                Err(e) => return (out, Some(e.to_string())),
4892            }
4893            let mut pos = PROMPT.len();
4894            loop {
4895                let dec = Tensor::from_slice(&[*out.last().unwrap()]).reshape([1, 1]);
4896                match model.forward_with_cache(&dec, pos) {
4897                    Ok(l) => {
4898                        out.push(argmax1(&l));
4899                        pos += 1;
4900                    }
4901                    Err(e) => return (out, Some(e.to_string())),
4902                }
4903            }
4904        };
4905
4906        // Speculative driver with the boundary fallback, mirroring
4907        // `sample_speculative_tokens` (forced-reject drafts for determinism).
4908        // `fallback_enabled = false` reproduces the pre-fix behavior.
4909        let spec_until_error = |model: &Qwen3_5Model,
4910                                fallback_enabled: bool|
4911         -> (Vec<i64>, u64, Option<String>) {
4912            let prompt_t = Tensor::from_slice(&PROMPT).reshape([1, PROMPT.len() as i64]);
4913            let mut out: Vec<i64> = Vec::new();
4914            let mut fallbacks = 0u64;
4915            let (logits, hidden) = match model.forward_with_cache_hidden(&prompt_t, 0, true) {
4916                Ok(v) => v,
4917                Err(e) => return (out, fallbacks, Some(e.to_string())),
4918            };
4919            let mut last = argmax1(&logits);
4920            out.push(last);
4921            let mut mtp_tokens: Vec<i64> = PROMPT[1..].to_vec();
4922            mtp_tokens.push(last);
4923            let mtp_in = Tensor::from_slice(&mtp_tokens).reshape([1, mtp_tokens.len() as i64]);
4924            let mut pending = argmax1(&model.mtp_forward(&mtp_in, &hidden, 0).unwrap());
4925
4926            let mut pos = PROMPT.len();
4927            loop {
4928                if fallback_enabled && pos + 2 > CAP {
4929                    // Boundary fallback: serial single-token step, this round only.
4930                    let dec = Tensor::from_slice(&[last]).reshape([1, 1]);
4931                    match model.forward_with_cache(&dec, pos) {
4932                        Ok(l) => {
4933                            last = argmax1(&l);
4934                            out.push(last);
4935                            pos += 1;
4936                            fallbacks += 1;
4937                        }
4938                        Err(e) => return (out, fallbacks, Some(e.to_string())),
4939                    }
4940                    continue;
4941                }
4942                let draft = (pending + 1) % VOCAB; // forced reject
4943                let snapshot = model.snapshot_ssm_states();
4944                let verify_in = Tensor::from_slice(&[last, draft]).reshape([1, 2]);
4945                let (logits, hidden) = match model.forward_with_cache_hidden(&verify_in, pos, false)
4946                {
4947                    Ok(v) => v,
4948                    Err(e) => return (out, fallbacks, Some(e.to_string())),
4949                };
4950                let v1 = argmax1(&logits.select(1, 0));
4951                if v1 == draft {
4952                    // Accidental accept: same handling as the accept path.
4953                    let v2 = argmax1(&logits.select(1, 1));
4954                    let mtp_in = Tensor::from_slice(&[v1, v2]).reshape([1, 2]);
4955                    pending = argmax1(&model.mtp_forward(&mtp_in, &hidden, pos).unwrap());
4956                    out.push(v1);
4957                    out.push(v2);
4958                    last = v2;
4959                    pos += 2;
4960                } else {
4961                    model.restore_ssm_states(snapshot.0, snapshot.1);
4962                    model.get_kv_cache().unwrap().lock().truncate_to(pos + 1);
4963                    let re_in = Tensor::from_slice(&[last]).reshape([1, 1]);
4964                    let (_, hidden2) = model.forward_with_cache_hidden(&re_in, pos, true).unwrap();
4965                    let mtp_in = Tensor::from_slice(&[v1]).reshape([1, 1]);
4966                    pending = argmax1(&model.mtp_forward(&mtp_in, &hidden2, pos).unwrap());
4967                    out.push(v1);
4968                    last = v1;
4969                    pos += 1;
4970                }
4971            }
4972        };
4973
4974        let (serial_out, serial_err) = serial_until_error(&build());
4975        assert!(
4976            serial_err
4977                .as_deref()
4978                .is_some_and(|e| e.contains("Cache overflow")),
4979            "serial driver must end at the context boundary, got {serial_err:?}"
4980        );
4981        assert_eq!(
4982            serial_out.len(),
4983            7,
4984            "prefill token + forwards at pos 6..=11"
4985        );
4986
4987        // Pre-fix behavior (no fallback): the 2-token verify at pos = CAP-1
4988        // overflows and the stream dies one token earlier than serial.
4989        let (old_out, _, old_err) = spec_until_error(&build(), false);
4990        assert!(
4991            old_err
4992                .as_deref()
4993                .is_some_and(|e| e.contains("Cache overflow")),
4994            "pre-fix behavior must overflow, got {old_err:?}"
4995        );
4996        assert!(
4997            old_out.len() < serial_out.len(),
4998            "pre-fix driver must lose the final token (old={}, serial={})",
4999            old_out.len(),
5000            serial_out.len()
5001        );
5002
5003        // Fixed behavior: the boundary round falls back to a serial step and
5004        // yields the same tokens as serial, erroring only at the true boundary.
5005        let (spec_out, fallbacks, spec_err) = spec_until_error(&build(), true);
5006        assert_eq!(
5007            spec_out, serial_out,
5008            "boundary fallback diverged from serial"
5009        );
5010        assert!(
5011            fallbacks >= 1,
5012            "serial fallback never engaged at the boundary"
5013        );
5014        assert!(
5015            spec_err
5016                .as_deref()
5017                .is_some_and(|e| e.contains("Cache overflow")),
5018            "spec driver must end at the true boundary like serial, got {spec_err:?}"
5019        );
5020    }
5021
5022    /// Same gate across a session prefix-cache hit: turn 2 shares turn 1's
5023    /// prompt prefix, so only the suffix is prefilled. Both drivers get the
5024    /// identical treatment (KV truncated to the prefix; GDN conv/rec state is
5025    /// NOT restored by a prefix hit — reset to empty on both sides here), so
5026    /// speculative must still equal serial token-for-token.
5027    #[test]
5028    fn mtp_decode_matches_serial_across_prefix_cache_hit() {
5029        let model_serial = whole_model_mtp();
5030        let model_spec = whole_model_mtp();
5031
5032        // Turn 1: full prefill + generation.
5033        let s1 = serial_generate(&model_serial, &PROMPT, GEN, 0);
5034        let (p1, _, _) = spec_generate(&model_spec, &PROMPT, GEN, 0, DraftMode::Natural, None);
5035        assert_eq!(p1, s1, "turn 1: speculative diverged from serial");
5036
5037        // Session save/restore: the engine saves prompt tokens only, so the
5038        // next turn's prefix hit truncates KV to the prompt prefix. A prefix
5039        // hit does not restore GDN SSM state; reset it identically on both
5040        // drivers so both see the same (KV-only) resumed state.
5041        let prefix_len = PROMPT.len();
5042        for m in [&model_serial, &model_spec] {
5043            m.get_kv_cache().unwrap().lock().truncate_to(prefix_len);
5044            let n = m.num_layers();
5045            // Tensor is not Clone — build the two empty state vectors separately.
5046            let empty_conv: Vec<Option<Tensor>> = (0..n).map(|_| None).collect();
5047            let empty_rec: Vec<Option<Tensor>> = (0..n).map(|_| None).collect();
5048            m.restore_ssm_states(empty_conv, empty_rec);
5049        }
5050
5051        // Turn 2: shared prefix + new suffix, partial prefill from prefix_len.
5052        let prompt2: [i64; 9] = [1, 5, 9, 2, 7, 3, 4, 8, 6];
5053        let s2 = serial_generate(&model_serial, &prompt2, GEN, prefix_len);
5054        let (p2, _, _) = spec_generate(
5055            &model_spec,
5056            &prompt2,
5057            GEN,
5058            prefix_len,
5059            DraftMode::Natural,
5060            None,
5061        );
5062        assert_eq!(
5063            p2, s2,
5064            "turn 2 (prefix hit): speculative diverged from serial"
5065        );
5066    }
5067
5068    /// Loading without `mtp.*` keys (or with an unconsumable head) must fall
5069    /// back to plain non-speculative decode — `has_mtp() == false`, no error.
5070    #[test]
5071    fn mtp_head_absent_falls_back_to_serial() {
5072        let mut w = tiny_weights();
5073        let model = Qwen3_5Model::from_weights(
5074            &mut w,
5075            tiny_config(),
5076            None,
5077            &Device::Cpu,
5078            Kind::Float,
5079            KVQuantType::None,
5080            true,
5081        )
5082        .unwrap();
5083        assert!(
5084            !model.has_mtp(),
5085            "model without mtp.* weights must not speculate"
5086        );
5087    }
5088
5089    /// Regression guard for the causal-mask offset in cached multi-token
5090    /// forwards (the speculative verify path): with `q_len > 1` and
5091    /// `start_pos > 0` the mask must be `tril(kv_len - q_len)`, not `tril(0)` —
5092    /// otherwise every cached row attends only to the first `i+1` keys and all
5093    /// of the prompt history is masked out. Compares LOGITS (not just argmax):
5094    /// the token-level spec==serial gates alone are insensitive to this on a
5095    /// tiny model because the residual stream dominates the attention delta.
5096    ///
5097    /// Uses an all-full-attention variant of the tiny model so the only
5098    /// divergence source is attention itself (the hybrid GDN stack adds
5099    /// ~1e-3 chunked-vs-recurrent kernel noise, too loose for this guard).
5100    #[test]
5101    fn cached_two_token_forward_matches_serial_decode_steps() {
5102        let _g = tch::no_grad_guard();
5103        let allattn_config = || {
5104            let mut cfg = tiny_config();
5105            cfg.layer_types = (0..cfg.num_hidden_layers)
5106                .map(|_| "full_attention".to_owned())
5107                .collect();
5108            cfg
5109        };
5110        let build = || {
5111            let mut w = tiny_weights_mtp();
5112            let cfg = allattn_config();
5113            // Replace GDN mixers with full-attention ones (deterministic pattern).
5114            let opt = (Kind::Float, Device::Cpu);
5115            let mut seed: i64 = 20_000;
5116            let mut pat = |dims: &[i64]| -> Tensor {
5117                let n: i64 = dims.iter().product();
5118                seed += 7;
5119                (Tensor::arange(n, opt) * 0.017 + seed as f64 * 0.013)
5120                    .sin()
5121                    .reshape(dims)
5122                    * 0.05
5123            };
5124            for i in 0..cfg.num_hidden_layers as usize {
5125                if (i + 1) % 4 == 0 {
5126                    continue; // already full-attention in tiny_weights
5127                }
5128                let p = format!("model.layers.{i}");
5129                w.retain(|k, _| !k.starts_with(&format!("{p}.linear_attn.")));
5130                let ap = format!("{p}.self_attn");
5131                w.insert(
5132                    format!("{ap}.q_proj.weight"),
5133                    pat(&[HEADS * HEAD_DIM * 2, HIDDEN]),
5134                );
5135                w.insert(
5136                    format!("{ap}.k_proj.weight"),
5137                    pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
5138                );
5139                w.insert(
5140                    format!("{ap}.v_proj.weight"),
5141                    pat(&[KV_HEADS * HEAD_DIM, HIDDEN]),
5142                );
5143                w.insert(
5144                    format!("{ap}.o_proj.weight"),
5145                    pat(&[HIDDEN, HEADS * HEAD_DIM]),
5146                );
5147                w.insert(format!("{ap}.q_norm.weight"), pat(&[HEAD_DIM]));
5148                w.insert(format!("{ap}.k_norm.weight"), pat(&[HEAD_DIM]));
5149            }
5150            Qwen3_5Model::from_weights(
5151                &mut w,
5152                allattn_config(),
5153                None,
5154                &Device::Cpu,
5155                Kind::Float,
5156                KVQuantType::None,
5157                true,
5158            )
5159            .unwrap()
5160        };
5161
5162        // Serial reference: prefill, then two one-token decode steps.
5163        let serial = build();
5164        let prompt_t = Tensor::from_slice(&PROMPT).reshape([1, PROMPT.len() as i64]);
5165        let _ = serial.forward_with_cache(&prompt_t, 0).unwrap();
5166        let pos = PROMPT.len();
5167        let step1 = Tensor::from_slice(&[11i64]).reshape([1, 1]);
5168        let ref1 = serial.forward_with_cache(&step1, pos).unwrap(); // [1, 1, V]
5169        let step2 = Tensor::from_slice(&[13i64]).reshape([1, 1]);
5170        let ref2 = serial.forward_with_cache(&step2, pos + 1).unwrap();
5171
5172        // Cached two-token forward (the verify shape): same model state, [11, 13] at pos.
5173        let cached = build();
5174        let _ = cached.forward_with_cache(&prompt_t, 0).unwrap();
5175        let pair = Tensor::from_slice(&[11i64, 13]).reshape([1, 2]);
5176        let (logits2, _) = cached.forward_with_cache_hidden(&pair, pos, false).unwrap();
5177
5178        for (row, ref_l) in [(0i64, &ref1), (1, &ref2)] {
5179            let got = logits2.select(1, row).reshape([-1i64]);
5180            let want = ref_l.reshape([-1i64]);
5181            let max_diff = (&got - &want).abs().max().double_value(&[]);
5182            assert!(
5183                got.allclose(&want, 1e-4, 1e-4, false),
5184                "cached 2-token forward row {row} diverged from the serial decode step \
5185                 (max_diff={max_diff}); the causal mask must keep kv_len - q_len history, \
5186                 not tril(0)"
5187            );
5188            assert_eq!(
5189                got.argmax(-1, false).int64_value(&[]),
5190                want.argmax(-1, false).int64_value(&[]),
5191                "row {row} argmax differs",
5192            );
5193        }
5194    }
5195}