Skip to main content

hyprstream_inference/runtime/architectures/
llama.rs

1//! Llama model implementation with support for Llama 1/2/3 and GQA
2
3use super::{ArchitectureConfig, ModelArchitecture, ModelOperations};
4// use super::lora_adapter::ArchitectureAwareLoRAAdapter; // Module removed
5use crate::runtime::device_pool::LayerDeviceMap;
6use crate::runtime::rope::RoPE;
7use crate::runtime::tensor_helpers::{
8    broadcast_add, broadcast_mul, dims3, dims4, scalar_tensor, square_tensor,
9};
10use anyhow::{anyhow, Result};
11use std::collections::HashMap;
12use std::path::Path;
13use tch::{nn, Device, Kind as DType, Tensor};
14
15// Thread-local cache of RoPE instances (one per layer/head-dim/theta config).
16//
17// Each entry holds sin/cos tables built on the compute device — i.e. real CUDA
18// tensors when the model runs on GPU. The cache is keyed so the tables are
19// reused across decode steps instead of recomputing every token.
20//
21// Teardown (#429): because this is thread-local, these device tensors are *not*
22// dropped when the engine struct drops — they survive until the owning thread
23// exits. The inference service thread is detached, so its TLS destructors can
24// fire during process-exit cleanup, *after* libtorch's atexit handler has torn
25// down the CUDA context — producing the `c10::Error: invalid device pointer`
26// SIGABRT. `clear_rope_cache` drops them explicitly on the service thread,
27// while the context is still alive, at shutdown.
28thread_local! {
29    static ROPE_CACHE: std::cell::RefCell<HashMap<(usize, i64, u32), RoPE>> =
30        std::cell::RefCell::new(HashMap::new());
31}
32
33/// Drop every cached RoPE instance on the *calling* thread.
34///
35/// Must be called on the same thread that ran the forward passes (the
36/// inference service thread) so it clears the entries that matter. Idempotent.
37/// See [`ROPE_CACHE`] for why this exists.
38pub(crate) fn clear_rope_cache() {
39    ROPE_CACHE.with(|cache| cache.borrow_mut().clear());
40}
41
42/// Linear projection layer with optional bias and optional FP8 block scale.
43///
44/// Weights may be stored as FP8 (E4M3 or E5M2) to save VRAM. When a `scale`
45/// tensor is present (shape `[out/128, in/128]`), `apply()` performs block-wise
46/// dequantization: cast FP8 → BF16 on GPU, expand scale to weight shape, multiply.
47/// ROCm 7.1+ dispatches the FP8 cast on-GPU without CPU fallback.
48pub(crate) struct LinearProjection {
49    pub(crate) weight: Tensor,
50    pub(crate) bias: Option<Tensor>,
51    /// Block-wise scale for FP8 weights: shape [out/block, in/block], BF16.
52    /// None for non-FP8 weights.
53    pub(crate) scale: Option<Tensor>,
54    /// `scale` repacked at load time for the `_scaled_mm_v2`
55    /// `BlockWise128x128` recipe: shape `[N/128, L4]` contiguous (f32), where
56    /// `L4 = round_up(K/128, 4)` — K-block-minor order with the K-block
57    /// dimension zero-padded to a multiple of 4 (cuBLASLt TMA requirement, cf.
58    /// `_pad_128x128_scales` in pytorch's test_scaled_matmul_cuda.py). The
59    /// call site passes `scale_v2.t()`, yielding `[L4, N/128]` with strides
60    /// `(1, L4)` as the kernel's TORCH_CHECK_VALUE preconditions demand.
61    /// Derived in `with_scale`; None when `scale` is None.
62    pub(crate) scale_v2: Option<Tensor>,
63    /// Rowwise-requantized copy of the FP8 weight for the v1 rowwise recipe —
64    /// the SM100/SM120 arm (see `apply_fp8_scaled_mm`; needs torch ≥ 2.11 on
65    /// SM120, issue #1524): `(weight [K, N] e4m3, scale_b [1, N] f32)` with
66    /// per-output-channel amax/448 scales. Derived at load in `with_scale`
67    /// ONLY when `HYPRSTREAM_FP8_GEMM` is on (costs +1x FP8 weight memory;
68    /// per-column scales are coarser than the checkpoint's 128x128 blocks —
69    /// measured envelope in `fp8_rowwise_requant_error_envelope`). None when
70    /// the flag is off or `scale` is None. Boxed: keeps `LinearProjection`
71    /// small enough for the `Qwen3_5Mlp` enum (clippy large_enum_variant).
72    pub(crate) rowwise: Option<Box<RowwiseRequant>>,
73    /// Per-output-channel E4M3FN multiplier [1,N], distinct from block scales.
74    pub(crate) channel_scale: Option<Tensor>,
75    /// Preserve original quantization boundaries through mixed projection fusion.
76    pub(crate) children: Option<Vec<LinearProjection>>,
77}
78
79/// Load-time rowwise representation used by the v1 scaled-mm recipe.
80///
81/// Keeping this as an owned value, rather than an anonymous tensor pair, makes
82/// its lifetime explicit at projection-fusion boundaries: consuming a source
83/// projection drops its derived rowwise copy before the fused copy is built.
84pub(crate) struct RowwiseRequant {
85    weight: Tensor,
86    scale: Tensor,
87}
88
89impl RowwiseRequant {
90    fn new(weight: Tensor, scale: Tensor) -> Self {
91        #[cfg(test)]
92        {
93            let live = ROWWISE_REQUANT_LIVE.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
94            ROWWISE_REQUANT_PEAK.fetch_max(live, std::sync::atomic::Ordering::SeqCst);
95        }
96        Self { weight, scale }
97    }
98}
99
100impl Drop for RowwiseRequant {
101    fn drop(&mut self) {
102        #[cfg(test)]
103        {
104            ROWWISE_REQUANT_LIVE.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
105        }
106    }
107}
108
109/// Test-only allocation witnesses for causal construction/fusion probes.
110/// The probes run in exact child processes, so these counters describe one
111/// construction sequence without interference from parallel tests.
112#[cfg(test)]
113static ROWWISE_REQUANT_LIVE: std::sync::atomic::AtomicUsize =
114    std::sync::atomic::AtomicUsize::new(0);
115#[cfg(test)]
116static ROWWISE_REQUANT_PEAK: std::sync::atomic::AtomicUsize =
117    std::sync::atomic::AtomicUsize::new(0);
118#[cfg(test)]
119pub(crate) fn reset_rowwise_requant_peak() {
120    ROWWISE_REQUANT_PEAK.store(
121        ROWWISE_REQUANT_LIVE.load(std::sync::atomic::Ordering::SeqCst),
122        std::sync::atomic::Ordering::SeqCst,
123    );
124}
125#[cfg(test)]
126pub(crate) fn rowwise_requant_live() -> usize {
127    ROWWISE_REQUANT_LIVE.load(std::sync::atomic::Ordering::SeqCst)
128}
129#[cfg(test)]
130pub(crate) fn rowwise_requant_peak() -> usize {
131    ROWWISE_REQUANT_PEAK.load(std::sync::atomic::Ordering::SeqCst)
132}
133
134/// Cached read of the FP8-GEMM gate ([`crate::config::RuntimeConfig::fp8_gemm`]).
135///
136/// The runtime reads the env default directly (same precedent as
137/// `RuntimeConfig.mmap`, which `model_factory` also reads from the env): the
138/// model-construction path has no `RuntimeConfig` plumbing today, so
139/// `LinearProjection::apply` consults this cached value. Static per process.
140fn fp8_gemm_enabled() -> bool {
141    static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
142    *ENABLED.get_or_init(crate::config::default_fp8_gemm)
143}
144
145/// Test-only path witness. The causal gate probes run in fresh subprocesses so
146/// the process-cached env gate cannot leak between scenarios. `1` means
147/// `apply()` returned a scaled-mm result; `2` means it took lazy dequant.
148#[cfg(test)]
149static FP8_APPLY_PATH: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
150
151/// Quantize activations to FP8 e4m3 with per-token 1x128 block scales.
152///
153/// Input: `[M, K]` float tensor (any float kind; K must be a multiple of 128).
154/// Returns `(q, scale)` where `q` is e4m3 `[M, K]` and `scale` is f32
155/// `[M, K/128]` **row-major** with `x ≈ q * scale` per 128-wide block. The
156/// `_scaled_mm_v2` call site must repack the scale to column-major
157/// (strides `(1, M)`) — the `BlockWise1x128` recipe requires it.
158/// Scales are amax-based (`amax / 448`); 448 is the max finite e4m3 value.
159/// Values are clamped to ±448 before the cast so out-of-range inputs saturate
160/// instead of producing inf/nan bytes. The input is made contiguous first:
161/// `to_kind` on an already-f32 tensor is a shallow clone, so a non-contiguous
162/// input would otherwise panic in `view`.
163fn quantize_activations_1x128(x: &Tensor) -> (Tensor, Tensor) {
164    const FP8_MAX: f64 = 448.0; // e4m3 max finite value
165    let x = x.contiguous();
166    let size = x.size();
167    let (m, k) = (size[0], size[1]);
168    let blocks = k / 128;
169    let x3 = x.to_kind(tch::Kind::Float).view([m, blocks, 128]);
170    let amax = x3.abs().amax(-1, false); // [m, blocks]
171    let scale = (amax / FP8_MAX).clamp_min(1e-12);
172    let q = (x3 / scale.unsqueeze(-1))
173        .clamp(-FP8_MAX, FP8_MAX)
174        .to_kind(tch::Kind::Float8e4m3fn)
175        .view([m, k]);
176    (q, scale)
177}
178
179/// Quantize activations to FP8 e4m3 with per-token (rowwise) scales for the
180/// v1 rowwise recipe. Input `[M, K]`; returns `(q [M, K] e4m3, scale [M, 1]
181/// f32 contiguous)` with `x ≈ q * scale` per row. Coarser than the 1x128
182/// block quantizer used by the v2 arm — one scale covers the whole token.
183fn quantize_activations_rowwise(x: &Tensor) -> (Tensor, Tensor) {
184    const FP8_MAX: f64 = 448.0; // e4m3 max finite value
185    let x = x.contiguous();
186    let xf = x.to_kind(tch::Kind::Float);
187    let scale = (xf.abs().amax(-1, false) / FP8_MAX)
188        .clamp_min(1e-12)
189        .unsqueeze(-1)
190        .contiguous(); // [M, 1]
191    let q = (xf / &scale)
192        .clamp(-FP8_MAX, FP8_MAX)
193        .to_kind(tch::Kind::Float8e4m3fn);
194    (q, scale)
195}
196
197/// Derive the v1-rowwise weight pair from the checkpoint's 128x128-block FP8
198/// weight: dequantize to f32 via the block scales (same broadcast as the lazy
199/// path), take the per-output-channel (column) amax over K, requantize to e4m3
200/// with `amax/448` scales. Returns `(weight [K, N] e4m3, scale_b [1, N] f32)`.
201///
202/// Precision trade vs 128x128 blocks: one scale now covers the whole K extent
203/// of an output channel, so channels with large dynamic range along K lose
204/// resolution — measured envelope in `fp8_rowwise_requant_error_envelope`.
205fn build_rowwise_requant(weight: &Tensor, scale: &Tensor) -> RowwiseRequant {
206    const FP8_MAX: f64 = 448.0;
207    let ws = weight.size(); // [K, N]
208    let ss = scale.size(); // [K/128, N/128]
209    let (br, bc) = (ws[0] / ss[0], ws[1] / ss[1]);
210    let w_f = (weight
211        .to_kind(tch::Kind::Float)
212        .view([ss[0], br, ss[1], bc])
213        * scale.to_kind(tch::Kind::Float).view([ss[0], 1, ss[1], 1]))
214    .reshape([ws[0], ws[1]]);
215    let s_b = (w_f.abs().amax(0, false) / FP8_MAX).clamp_min(1e-12); // [N]
216    let w_rq = (w_f / s_b.unsqueeze(0))
217        .clamp(-FP8_MAX, FP8_MAX)
218        .to_kind(tch::Kind::Float8e4m3fn)
219        .contiguous();
220    RowwiseRequant::new(w_rq, s_b.view([1, ws[1]]).contiguous())
221}
222
223/// `ScalingType::BlockWise1x128` / `BlockWise128x128` and `SwizzleType::NO_SWIZZLE`
224/// (ATen/BlasBackend.h) as the `IntList` values `_scaled_mm_v2` expects.
225const RECIPE_BLOCKWISE_1X128: i64 = 4;
226const RECIPE_BLOCKWISE_128X128: i64 = 5;
227const NO_SWIZZLE: i64 = 0;
228
229/// Which `_scaled_mm` recipe a latch entry refers to — failures are latched
230/// per (device, recipe) so a blockwise failure on SM120 doesn't suppress the
231/// rowwise arm on the same device, and neither affects other devices.
232#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
233enum ScaledMmRecipe {
234    /// v2: 1x128 activation × 128x128 weight blocks (Hopper SM90 only).
235    BlockwiseV2,
236    /// v1: per-token × per-output-channel scales (SM89+; SM120 needs
237    /// torch ≥ 2.11 — the earlier gate routes SM120 rowwise to cuBLASLt,
238    /// which rejects it, so this latches off there until #1524).
239    RowwiseV1,
240}
241
242/// Per-(CUDA-device, recipe) sticky kernel-failure latch. The deepseek
243/// (1x128 × 128x128) recipe hard-errors off NVIDIA Hopper (SM90 + cuBLASLt ≥
244/// 12.9: `_check_deepseek_support` in ScaledBlas.cpp), and tch-rs exposes no
245/// compute-capability query, so the first kernel error *for a given recipe on
246/// a given device* latches only that combination off: later calls skip that
247/// arm silently (one warning per device per recipe), while other recipes and
248/// other devices stay live. Per-device rather than process-global because a
249/// multi-GPU `LayerDeviceMap` can mix Hopper with another CUDA architecture —
250/// support is a per-device compute-capability property (PR #1523 review).
251/// Failed state only ever transitions false → true.
252#[derive(Default)]
253struct DeviceLatch(parking_lot::Mutex<std::collections::HashSet<(usize, ScaledMmRecipe)>>);
254
255impl DeviceLatch {
256    /// True if `recipe` has already failed on CUDA device `dev`.
257    fn is_failed(&self, dev: usize, recipe: ScaledMmRecipe) -> bool {
258        self.0.lock().contains(&(dev, recipe))
259    }
260
261    /// Mark `recipe` failed on CUDA device `dev`. Returns true iff this call
262    /// was the first failure for the combination — i.e. the caller should log
263    /// the warning (warn-once per device per recipe).
264    fn mark_failed(&self, dev: usize, recipe: ScaledMmRecipe) -> bool {
265        self.0.lock().insert((dev, recipe))
266    }
267}
268
269static SCALED_MM_FAILED: std::sync::OnceLock<DeviceLatch> = std::sync::OnceLock::new();
270
271fn scaled_mm_latch() -> &'static DeviceLatch {
272    SCALED_MM_FAILED.get_or_init(DeviceLatch::default)
273}
274
275/// Repack a `[K/128, N/128]` block scale into the `_scaled_mm_v2`
276/// `BlockWise128x128` layout: `[N/128, L4]` contiguous f32 with
277/// `L4 = round_up(K/128, 4)` (K-block dim zero-padded to a multiple of 4).
278fn pack_scale_b_v2(scale: &Tensor) -> Tensor {
279    let size = scale.size();
280    let kb = size[0];
281    let l4 = (kb + 3) / 4 * 4; // i64::div_ceil is unstable on this toolchain
282                               // [N/128, K/128] contiguous, then zero-pad the K-block dim to L4.
283    let t = scale.to_kind(tch::Kind::Float).transpose(0, 1).contiguous();
284    if l4 == kb {
285        t
286    } else {
287        t.constant_pad_nd([0, l4 - kb]) // right-pad last dim with zeros
288    }
289    .contiguous()
290}
291
292impl LinearProjection {
293    /// FP8 GEMM via torch `_scaled_mm`. Returns `None` when this projection
294    /// can't use any FP8 path: weight not e4m3, no block scale, not on CUDA
295    /// (the CPU `_scaled_mm` v1 kernel has no rowwise kernel — it is
296    /// per-tensor-scale only, which cannot express the checkpoint's scales),
297    /// K not a multiple of 128, or every recipe latched off on the weight's
298    /// device (see [`SCALED_MM_FAILED`]).
299    ///
300    /// Each recipe latches off per device on kernel error; lazy dequant
301    /// remains the final fallback. The gate and latch key on the *weight's*
302    /// CUDA device — the kernel runs where the weight lives (the model's
303    /// forward moves activations to the layer's device), which is correct
304    /// under a multi-GPU `LayerDeviceMap`.
305    ///
306    /// Recipe selection (verified against the current 2.11 release
307    /// ScaledBlas.cpp / RowwiseScaledMM.cu):
308    ///
309    ///  1. v2 blockwise: 1x128 activation × 128x128 weight blocks — Hopper
310    ///     (SM90) + cuBLASLt ≥ 12.9 ONLY (`_check_deepseek_support`
311    ///     hard-errors elsewhere, incl. SM120).
312    ///  2. v1 rowwise: per-token × per-output-channel scales — SM89+ via the
313    ///     CUTLASS `f8f8bf16_rowwise` kernel, EXCEPT on SM120 under shipped
314    ///     Before torch 2.11, `_scaled_rowwise_rowwise`'s CUTLASS gate was
315    ///     `dprops->major == 10` (exactly SM100), so SM120 (major 12) with
316    ///     cuBLAS ≥ 12.9 is routed to cuBLASLt rowwise instead — which has no
317    ///     FP8 rowwise kernel for SM120 and fails with
318    ///     `CUBLAS_STATUS_NOT_SUPPORTED` out of `cublasLtMatmulAlgoGetHeuristic`
319    ///     even with the required TN layout (column-major B). The arm therefore
320    ///     latches off on SM120 and lazy dequant serves; release/2.11
321    ///     widens the gate to `major >= 10`, so the SM120 CUTLASS kernel lights
322    ///     up with the libtorch upgrade tracked in #1524 — no code change
323    ///     needed here. Uses the load-time requantized `rowwise` weight pair.
324    pub(crate) fn apply_fp8_scaled_mm(&self, input: &Tensor) -> Option<Tensor> {
325        if self.weight.kind() != tch::Kind::Float8e4m3fn {
326            return None;
327        }
328        let Device::Cuda(dev_idx) = self.weight.device() else {
329            return None;
330        };
331        let in_dims = input.size();
332        let k = *in_dims.last()?;
333        if k % 128 != 0 {
334            return None;
335        }
336        let m: i64 = in_dims[..in_dims.len() - 1].iter().product();
337        let out = self
338            .try_blockwise_v2(dev_idx, input, m, k)
339            .or_else(|| self.try_rowwise_v1(dev_idx, input, m, k))?;
340        let mut out_dims = in_dims;
341        *out_dims.last_mut()? = self.weight.size()[1];
342        Some(out.reshape(out_dims))
343    }
344
345    /// v2 blockwise arm (1x128 activation × 128x128 weight blocks) — Hopper
346    /// (SM90) + cuBLASLt ≥ 12.9 only. Returns None if unavailable or latched.
347    ///
348    /// Layout: `weight` is stored `[in, out]` (K, N) after `take()`; the kernel
349    /// wants `mat2` column-major `[K, N]`, so we materialize one FP8 transpose
350    /// per call (spike overhead; a load-time orientation change is the fix if
351    /// the GPU numbers justify it). Scales: `scale_a` is repacked per call to
352    /// `[M, K/128]` with strides `(1, M)` (column-major, like pytorch's own
353    /// `x_scales.t().contiguous().t()`); `scale_b` uses the load-time-packed
354    /// `scale_v2` (`[N/128, L4]` contiguous, passed as `.t()` → `[L4, N/128]`
355    /// strides `(1, L4)`). The stored scale is already a *multiply* factor (HF
356    /// `weight_scale_inv` semantics, matching the lazy path), so no reciprocal
357    /// is needed.
358    fn try_blockwise_v2(&self, dev_idx: usize, input: &Tensor, m: i64, k: i64) -> Option<Tensor> {
359        let scale_v2 = self.scale_v2.as_ref()?;
360        if scaled_mm_latch().is_failed(dev_idx, ScaledMmRecipe::BlockwiseV2) {
361            return None;
362        }
363        let x2 = input.reshape([m, k]);
364        let (q, scale_a) = quantize_activations_1x128(&x2);
365        // Column-major mat2 [K, N]: contiguous [N, K] copy, transposed view.
366        let mat2 = self.weight.transpose(0, 1).contiguous().transpose(0, 1);
367        // BlockWise1x128 scale_a must be [M, K/128] with strides (1, M).
368        let scale_a = scale_a.transpose(0, 1).contiguous().transpose(0, 1);
369        // BlockWise128x128 scale_b must be [L4, N/128] with strides (1, L4).
370        let scale_b = scale_v2.transpose(0, 1);
371        let scale_a_refs = [&scale_a];
372        let scale_b_refs = [&scale_b];
373        let bias = self.bias.as_ref().map(|b| b.to_kind(tch::Kind::BFloat16));
374        q.f_internal_scaled_mm_v2(
375            &mat2,
376            &scale_a_refs,
377            [RECIPE_BLOCKWISE_1X128],
378            [NO_SWIZZLE],
379            &scale_b_refs,
380            [RECIPE_BLOCKWISE_128X128],
381            [NO_SWIZZLE],
382            bias.as_ref(),
383            tch::Kind::BFloat16,
384            [] as [i64; 0],
385            false,
386        )
387        .map_err(|e| {
388            if scaled_mm_latch().mark_failed(dev_idx, ScaledMmRecipe::BlockwiseV2) {
389                tracing::warn!(
390                    "FP8 _scaled_mm_v2 (1x128 x 128x128) failed on CUDA device {dev_idx} \
391                     (latched off for this device; trying rowwise/lazy fallback): {e}"
392                );
393            }
394            e
395        })
396        .ok()
397    }
398
399    /// v1 rowwise arm (per-token activation scale × per-output-channel weight
400    /// scale) — SM89+ via the CUTLASS `f8f8bf16_rowwise` kernel. On SM120 this
401    /// requires torch ≥ 2.11: the prior `_scaled_rowwise_rowwise` gate
402    /// (`dprops->major == 10`, ScaledBlas.cpp) sends SM120 to the cuBLASLt
403    /// rowwise path, which rejects the problem
404    /// (`CUBLAS_STATUS_NOT_SUPPORTED` from `cublasLtMatmulAlgoGetHeuristic`)
405    /// even with the correct TN layout; the first failure then latches this
406    /// recipe off and lazy dequant serves until the #1524 libtorch upgrade.
407    /// Uses the load-time-requantized `rowwise` weight pair; returns None if
408    /// it wasn't built (flag off at load) or the recipe is latched off on
409    /// this device.
410    ///
411    /// Layout (RowwiseScaledMM.cu `check_inputs`): A
412    /// `[M, K]` row-major, `scale_a` `[M, 1]` f32 contiguous, `scale_b`
413    /// `[1, N]` f32 contiguous, and `mat2` `[K, N]` COLUMN-major
414    /// (stride(0) == 1) — the requantized weight is stored row-major
415    /// `[K, N]`, so we materialize one FP8 transpose per call (same spike
416    /// overhead as the blockwise arm).
417    fn try_rowwise_v1(&self, dev_idx: usize, input: &Tensor, m: i64, k: i64) -> Option<Tensor> {
418        let rowwise = self.rowwise.as_deref()?;
419        if scaled_mm_latch().is_failed(dev_idx, ScaledMmRecipe::RowwiseV1) {
420            return None;
421        }
422        let x2 = input.reshape([m, k]);
423        let (q, scale_a) = quantize_activations_rowwise(&x2);
424        // Column-major mat2 [K, N]: contiguous [N, K] copy, transposed view.
425        let mat2 = rowwise.weight.transpose(0, 1).contiguous().transpose(0, 1);
426        let bias = self.bias.as_ref().map(|b| b.to_kind(tch::Kind::BFloat16));
427        q.f_internal_scaled_mm(
428            &mat2,
429            &scale_a,
430            &rowwise.scale,
431            bias.as_ref(),
432            None::<&Tensor>,
433            tch::Kind::BFloat16,
434            false,
435        )
436        .map_err(|e| {
437            if scaled_mm_latch().mark_failed(dev_idx, ScaledMmRecipe::RowwiseV1) {
438                tracing::warn!(
439                    "FP8 rowwise _scaled_mm failed on CUDA device {dev_idx} \
440                     (latched off for this device; using lazy dequant there): {e}"
441                );
442            }
443            e
444        })
445        .ok()
446    }
447}
448
449impl LinearProjection {
450    /// Create projection from weight only (no bias, no scale).
451    #[inline]
452    pub(crate) fn new(weight: Tensor) -> Self {
453        Self {
454            weight,
455            bias: None,
456            scale: None,
457            scale_v2: None,
458            rowwise: None,
459            channel_scale: None,
460            children: None,
461        }
462    }
463
464    /// Create projection with weight and bias (no scale).
465    #[inline]
466    pub(crate) fn with_bias(weight: Tensor, bias: Tensor) -> Self {
467        Self {
468            weight,
469            bias: Some(bias),
470            scale: None,
471            scale_v2: None,
472            rowwise: None,
473            channel_scale: None,
474            children: None,
475        }
476    }
477
478    /// Create FP8 projection with block-wise scale `[K/128, N/128]` (the
479    /// post-`take()` orientation). Also derives the load-time-packed
480    /// `_scaled_mm_v2` scale layout (`scale_v2`) so the FP8 GEMM arm pays no
481    /// per-call repack/padding cost, and — only when `HYPRSTREAM_FP8_GEMM` is
482    /// on for E4M3 weights — the rowwise-requantized weight pair (`rowwise`)
483    /// for the v1 recipe arm (SM89+; SM120 needs torch ≥ 2.11, see
484    /// `apply_fp8_scaled_mm`). E5M2 never has a compatible scaled-mm recipe,
485    /// so it must not pay for a persistent E4M3 copy or its FP32 construction
486    /// temporary.
487    #[inline]
488    pub(crate) fn with_scale(weight: Tensor, scale: Tensor) -> Self {
489        let scale_v2 = pack_scale_b_v2(&scale);
490        let rowwise = if fp8_gemm_enabled() && weight.kind() == tch::Kind::Float8e4m3fn {
491            Some(Box::new(build_rowwise_requant(&weight, &scale)))
492        } else {
493            None
494        };
495        Self {
496            weight,
497            bias: None,
498            scale: Some(scale),
499            scale_v2: Some(scale_v2),
500            rowwise,
501            channel_scale: None,
502            children: None,
503        }
504    }
505
506    /// Move all owned tensors to `device` (no-op per tensor already there).
507    /// Used by the 2b pipeline to place a layer on its mapped device (#314).
508    #[inline]
509    pub(crate) fn into_device(self, device: Device) -> Self {
510        Self {
511            channel_scale: self.channel_scale.map(|s| s.to_device(device)),
512            children: self
513                .children
514                .map(|ps| ps.into_iter().map(|p| p.into_device(device)).collect()),
515            weight: self.weight.to_device(device),
516            bias: self.bias.map(|b| b.to_device(device)),
517            scale: self.scale.map(|s| s.to_device(device)),
518            scale_v2: self.scale_v2.map(|s| s.to_device(device)),
519            rowwise: self.rowwise.map(|bs| {
520                Box::new(RowwiseRequant::new(
521                    bs.weight.to_device(device),
522                    bs.scale.to_device(device),
523                ))
524            }),
525        }
526    }
527
528    /// Apply projection to input: output = input @ weight + bias
529    ///
530    /// Input shape:  [*, in_features]
531    /// Weight shape: [in_features, out_features]  (pre-transposed at load time)
532    /// Output shape: [*, out_features]
533    ///
534    /// FP8 weights are dequantized lazily: `LinearProjection::take` captures the
535    /// companion `_scale_inv` tensor, and `apply()` expands the block scale and
536    /// multiplies on each forward pass (keeping VRAM at FP8 size).
537    #[inline]
538    pub(crate) fn apply(&self, input: &Tensor) -> Tensor {
539        if self.children.is_some() || self.channel_scale.is_some() {
540            return self
541                .try_apply(input)
542                .unwrap_or_else(|e| panic!("legacy infallible projection: {e}"));
543        }
544        // FP8 GEMM spike: config-gated (`RuntimeConfig::fp8_gemm` /
545        // HYPRSTREAM_FP8_GEMM), NVIDIA Hopper (SM90, cuBLASLt ≥ 12.9) only in
546        // practice. Falls through to lazy dequant when the path is unavailable
547        // or has latched off after a kernel error. `_scaled_mm` has no backward
548        // path for this recipe, so a gradient-carrying activation must retain
549        // the differentiable lazy matmul instead.
550        let fp8_out = if fp8_gemm_enabled() && !input.requires_grad() {
551            self.apply_fp8_scaled_mm(input)
552        } else {
553            None
554        };
555        if let Some(out) = fp8_out {
556            #[cfg(test)]
557            FP8_APPLY_PATH.store(1, std::sync::atomic::Ordering::SeqCst);
558            return out;
559        }
560        #[cfg(test)]
561        FP8_APPLY_PATH.store(2, std::sync::atomic::Ordering::SeqCst);
562        let w = match self.weight.kind() {
563            tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2 => {
564                // Cast FP8 → BF16 on GPU (ROCm 7.1+ dispatches on-device, no CPU fallback).
565                let w_bf16 = self.weight.to_kind(input.kind());
566                if let Some(scale) = &self.scale {
567                    // scale: [in/128, out/128] — apply block-wise via reshape+expand+reshape.
568                    // This avoids repeat_interleave (which fully materializes a copy) by using
569                    // a lazy expand view over the 128-element block strides.
570                    let ws = w_bf16.size(); // [in, out]
571                    let ss = scale.size(); // [in/128, out/128]
572                    let block_r = ws[0] / ss[0];
573                    let block_c = ws[1] / ss[1];
574                    // Block-wise dequantization via 4D broadcast multiply.
575                    // The scale is broadcast across block dimensions without materializing
576                    // a full [in, out] expanded copy — PyTorch's CUDA kernel reads from
577                    // the stride-zero view directly. The multiply output is contiguous,
578                    // so the final reshape back to 2D is zero-copy.
579                    let w_4d = w_bf16.view([ss[0], block_r, ss[1], block_c]);
580                    let s_4d = scale.to_kind(input.kind()).view([ss[0], 1, ss[1], 1]);
581                    (w_4d * s_4d).reshape([ws[0], ws[1]])
582                } else {
583                    w_bf16
584                }
585            }
586            _ => self.weight.shallow_clone(),
587        };
588        let output = input.matmul(&w);
589        match &self.bias {
590            Some(bias) => output + bias,
591            None => output,
592        }
593    }
594
595    /// Fallible projection path used throughout native Qwen/Clef inference.
596    /// Quantized execution errors propagate to the request boundary without
597    /// unwinding the serving thread. Legacy floating/block call sites retain
598    /// the infallible `apply` API until independently migrated.
599    pub(crate) fn try_apply(&self, input: &Tensor) -> Result<Tensor> {
600        if let Some(children) = &self.children {
601            let outputs = children
602                .iter()
603                .map(|p| p.try_apply(input))
604                .collect::<Result<Vec<_>>>()?;
605            let output = Tensor::cat(&outputs, -1);
606            return Ok(match &self.bias {
607                Some(bias) => output + bias,
608                None => output,
609            });
610        }
611        if let Some(scale) = &self.channel_scale {
612            let output = super::super::channel_fp8::channel_matmul(input, &self.weight, scale)
613                .map_err(|e| super::super::channel_fp8::Fp8ExecutionError(e.to_string()))?;
614            return Ok(match &self.bias {
615                Some(bias) => output + bias,
616                None => output,
617            });
618        }
619        Ok(self.apply(input))
620    }
621
622    /// Remove weight (and its companion `_scale_inv` if present) from `weights`,
623    /// transpose from `[out, in]` → `[in, out]`, and build a `LinearProjection`.
624    ///
625    /// FP8 weights are kept as FP8 in VRAM. The scale (if present) is stored for
626    /// use during `apply()`. This keeps memory at FP8 size (e.g. 7.4 GB for 35B),
627    /// which is necessary when the BF16 equivalent would exceed VRAM (e.g. 70 GB > 64 GB).
628    pub(crate) fn take(weights: &mut HashMap<String, Tensor>, key: &str) -> Result<Self> {
629        if let Some(scale) = weights.remove(&format!("{key}_scale")) {
630            let weight = weights
631                .remove(key)
632                .ok_or_else(|| anyhow!("Missing FP8 weight: {key}"))?;
633            super::super::channel_fp8::validate_channel_scale(&weight, &scale)?;
634            anyhow::ensure!(
635                !weights.contains_key(&format!("{key}_scale_inv")),
636                "ambiguous FP8 scale: {key}"
637            );
638            let mut p = Self::new(weight.transpose(0, 1));
639            p.channel_scale = Some(scale.transpose(0, 1));
640            return Ok(p);
641        }
642        let (weight, scale) = Self::take_weight_and_scale(weights, key)?;
643        Ok(match scale {
644            Some(s) => Self::with_scale(weight, s),
645            None => Self::new(weight),
646        })
647    }
648
649    /// Remove and orient checkpoint tensors without constructing projection
650    /// metadata. Batched MoE experts consume only these originals; deriving
651    /// packed scales or rowwise weights here would be discarded work.
652    pub(crate) fn take_weight_and_scale(
653        weights: &mut HashMap<String, Tensor>,
654        key: &str,
655    ) -> Result<(Tensor, Option<Tensor>)> {
656        anyhow::ensure!(
657            !weights.contains_key(&format!("{key}_scale")),
658            "channel FP8 cannot enter block-only expert loader: {key}"
659        );
660        let weight = weights
661            .remove(key)
662            .ok_or_else(|| anyhow!("Missing weight tensor: {}", key))?
663            .transpose(0, 1)
664            .contiguous();
665        // Scale stored as [out/128, in/128]; transpose to match [in, out] weight orientation.
666        let scale = weights
667            .remove(&format!("{key}_scale_inv"))
668            .map(|s| s.transpose(0, 1).contiguous());
669        if matches!(
670            weight.kind(),
671            tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2
672        ) {
673            anyhow::ensure!(scale.is_some(), "missing FP8 scale for {key}");
674            let ss = scale.as_ref().map(Tensor::size).unwrap_or_default();
675            let ws = weight.size();
676            anyhow::ensure!(
677                ss.len() == 2
678                    && ss.iter().all(|d| *d > 0)
679                    && ws.len() == 2
680                    && ws[0] % ss[0] == 0
681                    && ws[1] % ss[1] == 0,
682                "invalid FP8 block scale for {key}"
683            );
684        }
685        Ok((weight, scale))
686    }
687
688    /// Like `take`, but also attaches an optional bias tensor (not transposed).
689    /// Tries `bias_key` in `weights`; skips silently if absent.
690    pub(crate) fn take_with_optional_bias(
691        weights: &mut HashMap<String, Tensor>,
692        weight_key: &str,
693        bias_key: &str,
694    ) -> Result<Self> {
695        let mut proj = Self::take(weights, weight_key)?;
696        if let Some(bias) = weights.get(bias_key).map(tch::Tensor::shallow_clone) {
697            proj.bias = Some(bias);
698        }
699        Ok(proj)
700    }
701
702    /// Get output dimension
703    #[inline]
704    #[allow(dead_code)]
705    fn out_features(&self) -> i64 {
706        self.weight.size()[1]
707    }
708}
709
710unsafe impl Send for LinearProjection {}
711unsafe impl Sync for LinearProjection {}
712
713/// Calculate padded vocabulary size to multiple of 64 (for performance optimization)
714#[inline]
715fn calculate_padded_vocab_size(vocab_size: u32) -> u32 {
716    vocab_size.div_ceil(64) * 64
717}
718
719/// Llama model configuration
720#[derive(Debug, Clone)]
721pub struct LlamaConfig {
722    /// Llama version (1, 2, or 3)
723    pub version: u8,
724    /// Number of attention heads for queries
725    pub num_attention_heads: u32,
726    /// Number of key-value heads (for GQA in Llama 2/3)
727    pub num_key_value_heads: u32,
728    /// Hidden dimension size
729    pub hidden_size: u32,
730    /// Head dimension
731    pub head_dim: u32,
732    /// Intermediate size for FFN
733    pub intermediate_size: u32,
734    /// Maximum position embeddings
735    pub max_position_embeddings: u32,
736    /// RMSNorm epsilon
737    pub rms_norm_eps: f32,
738    /// Vocabulary size (may be padded)
739    pub vocab_size: u32,
740    /// Original vocabulary size (before padding)
741    pub original_vocab_size: u32,
742    /// Number of hidden layers
743    pub num_hidden_layers: u32,
744    /// RoPE theta
745    pub rope_theta: f32,
746    /// RoPE scaling (for Llama 3)
747    pub rope_scaling: Option<RopeScaling>,
748    /// Hidden activation function (silu for Llama, gelu_pytorch_tanh for Gemma3)
749    pub hidden_activation: String,
750    /// Query pre-attention scalar for QK-norm (Gemma3)
751    pub query_pre_attn_scalar: Option<f32>,
752    /// Use QK-norm (Gemma3)
753    pub use_qk_norm: bool,
754    /// Scale embeddings by sqrt(hidden_size) (Gemma3)
755    pub scale_embeddings: bool,
756    /// Layer types for sliding window attention (Gemma3)
757    pub layer_types: Vec<String>,
758    /// RoPE theta for local attention layers (Gemma3)
759    pub rope_local_base_freq: Option<f32>,
760    /// The *real* model architecture this LlamaModel stands in for.
761    ///
762    /// `LlamaModel` is reused as the runtime for several architecturally-llama
763    /// families (Llama, Qwen, Mistral). The tokenizer-config dispatch keys on
764    /// `ModelOperations::architecture()`, so a Qwen3 checkpoint loaded via
765    /// `create_qwen_model` must report `Qwen` here — otherwise the
766    /// `QwenTokenizerConfig` (which pads the tokenizer vocab up to the model's
767    /// embedding size) is never applied and the tokenizer/embedding vocab
768    /// mismatch silently causes a CUDA index-out-of-bounds on the embedding
769    /// lookup (#143). Defaults to `Llama`; `create_qwen_model` overrides it.
770    pub model_architecture: ModelArchitecture,
771}
772
773/// RoPE scaling configuration (Llama 3)
774#[derive(Debug, Clone)]
775pub struct RopeScaling {
776    pub scaling_type: String,
777    pub factor: f32,
778}
779
780impl Default for LlamaConfig {
781    fn default() -> Self {
782        // Llama 2 7B configuration
783        let vocab_size = 32000;
784        Self {
785            version: 2,
786            num_attention_heads: 32,
787            num_key_value_heads: 32, // No GQA in base 7B
788            hidden_size: 4096,
789            head_dim: 128,
790            intermediate_size: 11008,
791            max_position_embeddings: 4096,
792            rms_norm_eps: 1e-5,
793            vocab_size,
794            original_vocab_size: vocab_size, // Same as vocab_size initially
795            num_hidden_layers: 32,
796            rope_theta: 10000.0,
797            rope_scaling: None,
798            hidden_activation: "silu".to_owned(),
799            query_pre_attn_scalar: None,
800            use_qk_norm: false,
801            scale_embeddings: false,
802            layer_types: vec![],
803            rope_local_base_freq: None,
804            model_architecture: ModelArchitecture::Llama { version: 2 },
805        }
806    }
807}
808
809impl LlamaConfig {
810    /// Create config for Llama 3 8B
811    pub fn llama3_8b() -> Self {
812        let vocab_size = 128256;
813        Self {
814            version: 3,
815            num_attention_heads: 32,
816            num_key_value_heads: 8, // GQA with 8 KV heads
817            hidden_size: 4096,
818            head_dim: 128,
819            intermediate_size: 14336,
820            max_position_embeddings: 8192,
821            rms_norm_eps: 1e-5,
822            vocab_size,
823            original_vocab_size: vocab_size,
824            num_hidden_layers: 32,
825            rope_theta: 500000.0,
826            rope_scaling: Some(RopeScaling {
827                scaling_type: "linear".to_owned(),
828                factor: 8.0,
829            }),
830            hidden_activation: "silu".to_owned(),
831            query_pre_attn_scalar: None,
832            use_qk_norm: false,
833            scale_embeddings: false,
834            layer_types: vec![],
835            rope_local_base_freq: None,
836            model_architecture: ModelArchitecture::Llama { version: 3 },
837        }
838    }
839
840    /// Create config for Llama 3 70B
841    pub fn llama3_70b() -> Self {
842        let vocab_size = 128256;
843        Self {
844            version: 3,
845            num_attention_heads: 64,
846            num_key_value_heads: 8, // GQA with 8 KV heads
847            hidden_size: 8192,
848            head_dim: 128,
849            intermediate_size: 28672,
850            max_position_embeddings: 8192,
851            rms_norm_eps: 1e-5,
852            vocab_size,
853            original_vocab_size: vocab_size,
854            num_hidden_layers: 80,
855            rope_theta: 500000.0,
856            rope_scaling: Some(RopeScaling {
857                scaling_type: "linear".to_owned(),
858                factor: 8.0,
859            }),
860            hidden_activation: "silu".to_owned(),
861            query_pre_attn_scalar: None,
862            use_qk_norm: false,
863            scale_embeddings: false,
864            layer_types: vec![],
865            rope_local_base_freq: None,
866            model_architecture: ModelArchitecture::Llama { version: 3 },
867        }
868    }
869}
870
871impl ArchitectureConfig for LlamaConfig {
872    fn num_attention_heads(&self) -> usize {
873        self.num_attention_heads as usize
874    }
875
876    fn num_key_value_heads(&self) -> usize {
877        self.num_key_value_heads as usize
878    }
879
880    fn hidden_size(&self) -> usize {
881        self.hidden_size as usize
882    }
883
884    fn intermediate_size(&self) -> usize {
885        self.intermediate_size as usize
886    }
887
888    fn head_dim(&self) -> usize {
889        self.head_dim as usize
890    }
891
892    fn vocab_size(&self) -> usize {
893        self.vocab_size as usize
894    }
895
896    fn max_position_embeddings(&self) -> usize {
897        self.max_position_embeddings as usize
898    }
899
900    fn rope_theta(&self) -> Option<f32> {
901        Some(self.rope_theta)
902    }
903
904    fn rope_dim(&self) -> Option<usize> {
905        Some(self.head_dim as usize)
906    }
907
908    fn layer_norm_eps(&self) -> f32 {
909        self.rms_norm_eps
910    }
911
912    fn use_rms_norm(&self) -> bool {
913        true // All Llama versions use RMSNorm
914    }
915}
916
917/// Llama model implementation
918pub struct LlamaModel {
919    config: LlamaConfig,
920    #[allow(dead_code)]
921    device: Device,
922    #[allow(dead_code)]
923    dtype: DType,
924
925    /// Per-global-layer device assignment (#314, 2b pipeline). For the unsplit
926    /// fast path this is a single-device map of length `num_hidden_layers`
927    /// (`is_single_device() == true`); for a pipeline shard it is the *global*
928    /// map (still length `num_hidden_layers`) and `layer_offset` selects the
929    /// owned window. Drives both per-layer placement at construction and the lone
930    /// boundary `to_device` in `forward_layers`.
931    device_map: LayerDeviceMap,
932
933    /// Global index of `self.layers[0]`. `0` for a whole model; `a` for a shard
934    /// owning global layers `[a..a+self.layers.len())`. The single place a global
935    /// index is needed at runtime is mapping a local KV-cache slot / device
936    /// lookup back to its global layer.
937    layer_offset: usize,
938
939    // Model weights
940    embed_tokens: Option<Tensor>,
941    layers: Vec<LlamaLayer>,
942    norm: Option<RMSNorm>,
943    lm_head: Option<Tensor>,
944
945    /// Pre-transposed lm_head for tied weights (avoids transpose per forward pass)
946    /// This is set when lm_head is None but embed_tokens is Some
947    lm_head_transposed: Option<Tensor>,
948
949    // KV cache for efficient generation (thread-safe with parking_lot::Mutex)
950    kv_cache: Option<std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>>,
951
952    // VarStore for training (if model was created with training support)
953    vs: Option<nn::VarStore>,
954}
955
956// SAFETY: Tch tensors are thread-safe when used correctly
957// We ensure no mutable access without proper synchronization
958unsafe impl Send for LlamaModel {}
959unsafe impl Sync for LlamaModel {}
960
961/// Single Llama transformer layer
962struct LlamaLayer {
963    self_attn: LlamaAttention,
964    mlp: LlamaMLP,
965    input_layernorm: RMSNorm,
966    post_attention_layernorm: RMSNorm,
967}
968
969unsafe impl Send for LlamaLayer {}
970unsafe impl Sync for LlamaLayer {}
971
972impl LlamaLayer {
973    /// Move every weight in this layer to `device` (#314 pipeline placement).
974    /// A no-op per tensor already resident on `device`.
975    #[inline]
976    fn into_device(self, device: Device) -> Self {
977        Self {
978            self_attn: self.self_attn.into_device(device),
979            mlp: self.mlp.into_device(device),
980            input_layernorm: self.input_layernorm.into_device(device),
981            post_attention_layernorm: self.post_attention_layernorm.into_device(device),
982        }
983    }
984}
985
986/// Llama attention with optional GQA support
987struct LlamaAttention {
988    q_proj: LinearProjection,
989    k_proj: LinearProjection,
990    v_proj: LinearProjection,
991    o_proj: LinearProjection,
992    num_heads: usize,
993    num_kv_heads: usize,
994    head_dim: usize,
995    rope_theta: f32,
996    #[allow(dead_code)]
997    rope_scaling: Option<RopeScaling>,
998    // QK-norm weights (Gemma3)
999    q_norm: Option<Tensor>,
1000    k_norm: Option<Tensor>,
1001    #[allow(dead_code)]
1002    query_pre_attn_scalar: Option<f32>,
1003    // Sliding window attention (Gemma3)
1004    sliding_window: Option<usize>,
1005    layer_type: String, // "local" or "global" for Gemma3
1006    layer_idx: usize,
1007}
1008
1009unsafe impl Send for LlamaAttention {}
1010unsafe impl Sync for LlamaAttention {}
1011
1012impl LlamaAttention {
1013    /// Move all projection + QK-norm weights to `device` (#314 pipeline placement).
1014    #[inline]
1015    fn into_device(self, device: Device) -> Self {
1016        Self {
1017            q_proj: self.q_proj.into_device(device),
1018            k_proj: self.k_proj.into_device(device),
1019            v_proj: self.v_proj.into_device(device),
1020            o_proj: self.o_proj.into_device(device),
1021            q_norm: self.q_norm.map(|t| t.to_device(device)),
1022            k_norm: self.k_norm.map(|t| t.to_device(device)),
1023            ..self
1024        }
1025    }
1026
1027    /// Apply attention with optional GQA, KV caching, and delta injection
1028    fn forward(
1029        &self,
1030        hidden_states: &Tensor,
1031        position_ids: Option<&Tensor>,
1032        kv_cache: Option<&mut crate::runtime::kv_cache::LayerKVCache>,
1033        start_pos: usize,
1034        delta: Option<&crate::training::TenantDelta>,
1035    ) -> Result<Tensor> {
1036        let (batch_size, seq_len, hidden_size) = dims3(hidden_states)?;
1037
1038        tracing::trace!("LlamaAttention forward: batch_size={}, seq_len={}, hidden_size={}, start_pos={}, has_cache={}",
1039                      batch_size, seq_len, hidden_size, start_pos, kv_cache.is_some());
1040
1041        // Debug: Check shapes and dtypes before projection
1042        // tracing::debug!("Attention forward - hidden_states shape: {:?}, dtype: {:?}", hidden_states.size(), hidden_states.kind());
1043
1044        // Reshape hidden_states for matmul: [batch*seq, hidden_size]
1045        let hidden_states_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
1046        // tracing::debug!("Reshaped hidden_states_2d shape: {:?}, dtype: {:?}", hidden_states_2d.size(), hidden_states_2d.kind());
1047
1048        // Project to Q, K, V using the projection API (handles biases automatically)
1049        let mut q = self.q_proj.apply(&hidden_states_2d);
1050        let mut k = self.k_proj.apply(&hidden_states_2d);
1051        let mut v = self.v_proj.apply(&hidden_states_2d);
1052
1053        // Delta injection — add LoRA corrections to attention projections
1054        // During inference (no_grad), corrections are detached tensors (no gradient overhead).
1055        // During training, these participate in the computation graph.
1056        if let Some(delta) = delta {
1057            for (name, proj) in [("q_proj", &mut q), ("k_proj", &mut k), ("v_proj", &mut v)] {
1058                if delta.has_module(name, self.layer_idx) {
1059                    let correction = delta.forward_2d(&hidden_states_2d, name, self.layer_idx)?;
1060                    if start_pos == 0 && self.layer_idx == 0 {
1061                        tracing::info!(
1062                            "[TTT] Layer 0 {}: correction_norm={:.6}, proj_norm={:.4}, ratio={:.6}",
1063                            name,
1064                            correction.norm().double_value(&[]),
1065                            proj.norm().double_value(&[]),
1066                            correction.norm().double_value(&[])
1067                                / (proj.norm().double_value(&[]) + 1e-10)
1068                        );
1069                    }
1070                    // #139/#440: coerce fp32 delta to base dtype before adding (shared helper).
1071                    let correction_size = correction.size();
1072                    *proj = proj.f_add(&correction.to_kind(proj.kind()))
1073                        .map_err(|e| anyhow::anyhow!(
1074                            "Delta correction shape mismatch at layer {} {}: proj {:?} vs correction {:?}: {}",
1075                            self.layer_idx, name, proj.size(), correction_size, e
1076                        ))?;
1077                }
1078            }
1079        }
1080
1081        // tracing::debug!("After projection - Q shape: {:?}, K shape: {:?}, V shape: {:?}",
1082        //               q.size(), k.size(), v.size());
1083        // tracing::debug!("Expected reshape: [{}, {}, {}, {}] = {} elements, actual Q elements: {}",
1084        //               batch_size, seq_len, self.num_heads, self.head_dim,
1085        //               batch_size * seq_len * self.num_heads * self.head_dim,
1086        //               q.size().iter().product::<usize>());
1087
1088        // Determine actual K/V heads from tensor dimensions
1089        let k_elements = k.size().iter().product::<i64>();
1090        let _v_elements = v.size().iter().product::<i64>();
1091
1092        // For K and V, we need to figure out the actual number of heads
1093        // They might be different from what we detected in config
1094        let kv_heads = if k_elements
1095            == batch_size * seq_len * (self.num_kv_heads as i64) * (self.head_dim as i64)
1096        {
1097            self.num_kv_heads // Config is correct
1098        } else if k_elements % (batch_size * seq_len * (self.head_dim as i64)) == 0 {
1099            // Recalculate based on actual size
1100            (k_elements / (batch_size * seq_len * (self.head_dim as i64))) as usize
1101        } else if k_elements % (batch_size * seq_len) == 0 {
1102            // Try different head_dim
1103            let kv_dim = (k_elements / (batch_size * seq_len)) as usize;
1104            if kv_dim == 256 && self.head_dim == 128 {
1105                2 // 2 heads with 128 dim
1106            } else if kv_dim == 256 {
1107                8 // 8 heads with 32 dim
1108            } else {
1109                self.num_kv_heads // Fallback to config
1110            }
1111        } else {
1112            self.num_kv_heads // Fallback
1113        };
1114
1115        // tracing::debug!("Actual KV heads: {}, config KV heads: {}", kv_heads, self.num_kv_heads);
1116
1117        // Reshape for attention
1118        // Q: [batch, seq, num_heads, head_dim]
1119        let mut q = q.reshape([
1120            batch_size,
1121            seq_len,
1122            self.num_heads as i64,
1123            self.head_dim as i64,
1124        ]);
1125        // K, V: [batch, seq, num_kv_heads, head_dim] - use actual kv_heads
1126        let mut k = k.reshape([batch_size, seq_len, kv_heads as i64, self.head_dim as i64]);
1127        let v = v.reshape([batch_size, seq_len, kv_heads as i64, self.head_dim as i64]);
1128
1129        // Apply QK-norm if configured (Gemma3)
1130        if let Some(q_norm) = &self.q_norm {
1131            q = self.apply_qk_norm(&q, q_norm, self.num_heads)?;
1132        }
1133        if let Some(k_norm) = &self.k_norm {
1134            k = self.apply_qk_norm(&k, k_norm, kv_heads)?;
1135        }
1136
1137        // Apply RoPE if position_ids provided
1138        let (q, k) = if let Some(pos_ids) = position_ids {
1139            (self.apply_rope(&q, pos_ids)?, self.apply_rope(&k, pos_ids)?)
1140        } else {
1141            (q, k)
1142        };
1143
1144        // Handle KV caching
1145        let (k_for_attn, v_for_attn) = if let Some(cache) = kv_cache {
1146            // Update cache with new K and V
1147            cache.update(&k, &v, start_pos)?;
1148
1149            // Get full cached K and V for attention computation
1150            let (cached_k, cached_v) = cache.get()?;
1151            (cached_k, cached_v)
1152        } else {
1153            // No caching, use current K and V directly
1154            (k.shallow_clone(), v.shallow_clone())
1155        };
1156
1157        // Expand K, V for GQA if needed (use cached versions)
1158        let (k_expanded, v_expanded) = if kv_heads < self.num_heads {
1159            // tracing::debug!("Expanding KV from {} heads to {} heads", kv_heads, self.num_heads);
1160            (
1161                self.expand_kv_for_gqa_with_heads(&k_for_attn, kv_heads)?,
1162                self.expand_kv_for_gqa_with_heads(&v_for_attn, kv_heads)?,
1163            )
1164        } else {
1165            (k_for_attn, v_for_attn)
1166        };
1167
1168        // Memory-efficient fused attention. For long prefills it chunks over the
1169        // query axis so the [q_len, k_len] FP32 score matrix is never materialized
1170        // at full size — capping peak VRAM at O(CHUNK * k_len) instead of
1171        // O(q_len * k_len). An 8K-token prompt otherwise balloons the score/softmax
1172        // tensors past 20 GB and the CUDA caching allocator retains that peak,
1173        // OOMing the next request. Backend-agnostic (plain matmul/softmax): SDPA's
1174        // flash/efficient kernels panic on this ROCm/libtorch version and
1175        // math_attention is slower, so chunking is used instead.
1176        let attn_output = self.compute_attention(&q, &k_expanded, &v_expanded)?;
1177
1178        // Transpose back: [batch, heads, seq, dim] -> [batch, seq, heads, dim]
1179        // PERF: .contiguous() before reshape ensures optimal memory access
1180        let attn_output = attn_output.transpose(1, 2).contiguous();
1181
1182        // Reshape to combine heads: [batch, seq, heads*dim]
1183        let attn_output =
1184            attn_output.reshape([batch_size, seq_len, (self.num_heads * self.head_dim) as i64]);
1185
1186        // Reshape for output projection: [batch*seq, heads*dim]
1187        let attn_output_2d = attn_output.reshape([
1188            batch_size * seq_len,
1189            (self.num_heads * self.head_dim) as i64,
1190        ]);
1191
1192        // Apply output projection using the projection API
1193        let mut attn_output = self.o_proj.apply(&attn_output_2d);
1194
1195        // Delta injection for o_proj
1196        if let Some(delta) = delta {
1197            if delta.has_module("o_proj", self.layer_idx) {
1198                let correction = delta.forward_2d(&attn_output_2d, "o_proj", self.layer_idx)?;
1199                let kind = attn_output.kind();
1200                attn_output += correction.to_kind(kind);
1201            }
1202        }
1203
1204        // Reshape back to 3D: [batch, seq, hidden_size]
1205        let attn_output = attn_output.reshape([batch_size, seq_len, hidden_size]);
1206
1207        Ok(attn_output)
1208    }
1209
1210    /// Detect the actual number of KV heads from a projected K tensor.
1211    ///
1212    /// Mirrors the inline detection in [`LlamaAttention::forward`] so the batched
1213    /// path stays byte-consistent with the reference. Extracted to keep both paths
1214    /// in lock-step (DRY).
1215    fn detect_kv_heads(&self, k_elements: i64, rows: i64) -> usize {
1216        if k_elements == rows * (self.num_kv_heads as i64) * (self.head_dim as i64) {
1217            self.num_kv_heads
1218        } else if k_elements % (rows * (self.head_dim as i64)) == 0 {
1219            (k_elements / (rows * (self.head_dim as i64))) as usize
1220        } else if k_elements % rows == 0 {
1221            let kv_dim = (k_elements / rows) as usize;
1222            if kv_dim == 256 && self.head_dim == 128 {
1223                2
1224            } else if kv_dim == 256 {
1225                8
1226            } else {
1227                self.num_kv_heads
1228            }
1229        } else {
1230            self.num_kv_heads
1231        }
1232    }
1233
1234    /// Batched ragged self-attention for continuous decode (#329, Llama-only).
1235    ///
1236    /// Processes `B` sequences in one batched forward. Each row `r` has:
1237    /// - `hidden_states[r]`: `[q, H]` (all rows share `q`; v1 decode uses `q == 1`),
1238    /// - its own absolute `start_positions[r]` and per-row `LayerKVCache`.
1239    ///
1240    /// Q/K/V are projected on the whole `[B, q, H]` batch (one matmul; same-delta
1241    /// LoRA injection applies uniformly). RoPE is applied per row with that row's
1242    /// absolute positions (`apply_rope`'s 1D path is exact; a 2D batched index
1243    /// would mis-broadcast). KV: each row's new K/V is written to its own cache,
1244    /// then every row's dense history is read back and **right-padded** to the
1245    /// batch-max KV length, stacked to `[B, kv_max, heads, dim]`. The supplied
1246    /// additive `attn_mask` (`[B, 1, q, kv_max]`) masks both the causal structure
1247    /// and each row's padding, so a single masked matmul is numerically identical
1248    /// to running each row through [`LlamaAttention::forward`] serially.
1249    #[allow(clippy::too_many_arguments)]
1250    fn forward_batched(
1251        &self,
1252        hidden_states: &Tensor,
1253        position_ids: &[Tensor],
1254        kv_caches: &mut [&mut crate::runtime::kv_cache::LayerKVCache],
1255        start_positions: &[usize],
1256        attn_mask: &Tensor,
1257        delta: Option<&crate::training::TenantDelta>,
1258    ) -> Result<Tensor> {
1259        let (batch_size, seq_len, hidden_size) = dims3(hidden_states)?;
1260        debug_assert_eq!(batch_size as usize, kv_caches.len());
1261        debug_assert_eq!(batch_size as usize, position_ids.len());
1262        debug_assert_eq!(batch_size as usize, start_positions.len());
1263
1264        // Project Q/K/V on the full batch: [B*q, H] -> projections.
1265        let hidden_states_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
1266        let mut q = self.q_proj.apply(&hidden_states_2d);
1267        let mut k = self.k_proj.apply(&hidden_states_2d);
1268        let mut v = self.v_proj.apply(&hidden_states_2d);
1269
1270        // Same-delta LoRA injection (scheduler guarantees one delta per batch).
1271        if let Some(delta) = delta {
1272            for (name, proj) in [("q_proj", &mut q), ("k_proj", &mut k), ("v_proj", &mut v)] {
1273                if delta.has_module(name, self.layer_idx) {
1274                    let correction = delta.forward_2d(&hidden_states_2d, name, self.layer_idx)?;
1275                    let kind = proj.kind();
1276                    *proj = proj.f_add(&correction.to_kind(kind)).map_err(|e| {
1277                        anyhow::anyhow!(
1278                            "Delta correction shape mismatch at layer {} {}: {}",
1279                            self.layer_idx,
1280                            name,
1281                            e
1282                        )
1283                    })?;
1284                }
1285            }
1286        }
1287
1288        let rows = batch_size * seq_len;
1289        let kv_heads = self.detect_kv_heads(k.size().iter().product::<i64>(), rows);
1290
1291        let mut q = q.reshape([
1292            batch_size,
1293            seq_len,
1294            self.num_heads as i64,
1295            self.head_dim as i64,
1296        ]);
1297        let mut k = k.reshape([batch_size, seq_len, kv_heads as i64, self.head_dim as i64]);
1298        let v = v.reshape([batch_size, seq_len, kv_heads as i64, self.head_dim as i64]);
1299
1300        if let Some(q_norm) = &self.q_norm {
1301            q = self.apply_qk_norm(&q, q_norm, self.num_heads)?;
1302        }
1303        if let Some(k_norm) = &self.k_norm {
1304            k = self.apply_qk_norm(&k, k_norm, kv_heads)?;
1305        }
1306
1307        // Per-row RoPE with that row's absolute positions, then re-stack.
1308        let mut q_rows: Vec<Tensor> = Vec::with_capacity(batch_size as usize);
1309        let mut k_rows: Vec<Tensor> = Vec::with_capacity(batch_size as usize);
1310        for (r, pos) in position_ids.iter().enumerate() {
1311            let q_r = q.narrow(0, r as i64, 1); // [1, q, heads, dim]
1312            let k_r = k.narrow(0, r as i64, 1); // [1, q, kv_heads, dim]
1313            q_rows.push(self.apply_rope(&q_r, pos)?);
1314            k_rows.push(self.apply_rope(&k_r, pos)?);
1315        }
1316        let q = Tensor::cat(&q_rows, 0);
1317        let k = Tensor::cat(&k_rows, 0);
1318
1319        // Update each row's cache and collect its dense KV history.
1320        let mut row_keys: Vec<Tensor> = Vec::with_capacity(batch_size as usize);
1321        let mut row_values: Vec<Tensor> = Vec::with_capacity(batch_size as usize);
1322        let mut kv_lens: Vec<i64> = Vec::with_capacity(batch_size as usize);
1323        for r in 0..batch_size as usize {
1324            let k_r = k.narrow(0, r as i64, 1).contiguous();
1325            let v_r = v.narrow(0, r as i64, 1).contiguous();
1326            let cache = &mut *kv_caches[r];
1327            cache.update(&k_r, &v_r, start_positions[r])?;
1328            let (ck, cv) = cache.get()?; // [1, kv_r, kv_heads, dim]
1329            kv_lens.push(ck.size()[1]);
1330            row_keys.push(ck);
1331            row_values.push(cv);
1332        }
1333
1334        // Right-pad every row's KV to the batch-max length so they stack. Padding
1335        // positions are masked out by `attn_mask`, so their values are irrelevant.
1336        let kv_max = kv_lens.iter().copied().max().unwrap_or(0);
1337        let opt = (k.kind(), k.device());
1338        let pad_row = |t: &Tensor, len: i64| -> Tensor {
1339            if len == kv_max {
1340                t.shallow_clone()
1341            } else {
1342                // [1, pad_len, kv_heads, dim] zeros appended on the seq axis.
1343                let mut shape = t.size();
1344                shape[1] = kv_max - len;
1345                let pad = Tensor::zeros(&shape[..], opt);
1346                Tensor::cat(&[t.shallow_clone(), pad], 1)
1347            }
1348        };
1349        let keys_padded: Vec<Tensor> = row_keys
1350            .iter()
1351            .zip(&kv_lens)
1352            .map(|(t, &len)| pad_row(t, len))
1353            .collect();
1354        let values_padded: Vec<Tensor> = row_values
1355            .iter()
1356            .zip(&kv_lens)
1357            .map(|(t, &len)| pad_row(t, len))
1358            .collect();
1359        let k_stacked = Tensor::cat(&keys_padded, 0); // [B, kv_max, kv_heads, dim]
1360        let v_stacked = Tensor::cat(&values_padded, 0);
1361
1362        // GQA expansion on the stacked KV.
1363        let (k_expanded, v_expanded) = if kv_heads < self.num_heads {
1364            (
1365                self.expand_kv_for_gqa_with_heads(&k_stacked, kv_heads)?,
1366                self.expand_kv_for_gqa_with_heads(&v_stacked, kv_heads)?,
1367            )
1368        } else {
1369            (k_stacked, v_stacked)
1370        };
1371
1372        // One masked batched attention for all rows.
1373        let attn_output =
1374            self.compute_attention_masked(&q, &k_expanded, &v_expanded, Some(attn_mask))?;
1375
1376        let attn_output = attn_output.transpose(1, 2).contiguous();
1377        let attn_output =
1378            attn_output.reshape([batch_size, seq_len, (self.num_heads * self.head_dim) as i64]);
1379        let attn_output_2d = attn_output.reshape([
1380            batch_size * seq_len,
1381            (self.num_heads * self.head_dim) as i64,
1382        ]);
1383        let mut attn_output = self.o_proj.apply(&attn_output_2d);
1384
1385        if let Some(delta) = delta {
1386            if delta.has_module("o_proj", self.layer_idx) {
1387                let correction = delta.forward_2d(&attn_output_2d, "o_proj", self.layer_idx)?;
1388                let kind = attn_output.kind();
1389                attn_output += correction.to_kind(kind);
1390            }
1391        }
1392
1393        Ok(attn_output.reshape([batch_size, seq_len, hidden_size]))
1394    }
1395
1396    /// Expand KV tensors for GQA with explicit head count
1397    fn expand_kv_for_gqa_with_heads(&self, kv: &Tensor, actual_kv_heads: usize) -> Result<Tensor> {
1398        let (batch_size, seq_len, _detected_heads, head_dim) = dims4(kv)?;
1399        let repeat_factor = self.num_heads / actual_kv_heads;
1400
1401        if repeat_factor == 1 {
1402            return Ok(kv.shallow_clone());
1403        }
1404
1405        // tracing::debug!("GQA expansion: {} KV heads -> {} Q heads (repeat {}x)",
1406        //               actual_kv_heads, self.num_heads, repeat_factor);
1407
1408        // Expand by repeating KV heads
1409        Ok(kv
1410            .unsqueeze(3) // [batch, seq, num_kv_heads, 1, head_dim]
1411            .expand(
1412                [
1413                    batch_size,
1414                    seq_len,
1415                    actual_kv_heads as i64,
1416                    repeat_factor as i64,
1417                    head_dim,
1418                ],
1419                false,
1420            )
1421            .reshape([batch_size, seq_len, self.num_heads as i64, head_dim]))
1422    }
1423
1424    /// Apply Rotary Position Embeddings with optional scaling
1425    fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor> {
1426        ROPE_CACHE.with(|cache| {
1427            let mut cache = cache.borrow_mut();
1428
1429            // Create a unique key for this layer's RoPE configuration
1430            let key = (
1431                self.layer_idx,
1432                self.head_dim as i64,
1433                (self.rope_theta * 1000.0) as u32, // Convert to integer for hashing
1434            );
1435
1436            // Get or create RoPE instance
1437            if let std::collections::hash_map::Entry::Vacant(e) = cache.entry(key) {
1438                // Use appropriate base frequency for layer type
1439                let base = if self.layer_type == "local" {
1440                    // Local layers in Gemma3 use different base
1441                    10000.0
1442                } else {
1443                    self.rope_theta as f64
1444                };
1445
1446                tracing::debug!(
1447                    "Creating RoPE for layer {} with base={}, head_dim={}",
1448                    self.layer_idx,
1449                    base,
1450                    self.head_dim
1451                );
1452
1453                // Create RoPE with the same dtype as the input tensor
1454                let rope = RoPE::new_with_dtype(
1455                    self.head_dim as i64,
1456                    base,
1457                    8192, // max_seq_len - should come from config
1458                    tensor.device(),
1459                    tensor.kind(), // Use same dtype as input tensor
1460                )
1461                .map_err(|e| {
1462                    anyhow!("Failed to create RoPE for layer {}: {}", self.layer_idx, e)
1463                })?;
1464
1465                e.insert(rope);
1466            }
1467
1468            // SAFETY: We just inserted the key above, so it must exist
1469            let rope = cache.get_mut(&key).ok_or_else(|| {
1470                anyhow!(
1471                    "Internal error: RoPE cache entry missing for layer {}",
1472                    self.layer_idx
1473                )
1474            })?;
1475
1476            // Apply RoPE with the actual position_ids
1477            rope.forward(tensor, Some(position_ids))
1478        })
1479    }
1480
1481    /// Apply QK-norm (Gemma3)
1482    fn apply_qk_norm(
1483        &self,
1484        tensor: &Tensor,
1485        norm_weight: &Tensor,
1486        actual_heads: usize,
1487    ) -> Result<Tensor> {
1488        // QK-norm normalizes each head separately
1489        // tensor shape: [batch, seq, heads, dim]
1490        // norm_weight shape: [heads * dim] where heads could be Q heads or KV heads
1491        let (_batch_size, _seq_len, tensor_heads, head_dim) = dims4(tensor)?;
1492
1493        // Debug logging
1494        tracing::trace!(
1495            "apply_qk_norm: tensor heads={}, head_dim={}, actual_heads={}, norm_weight shape={:?}",
1496            tensor_heads,
1497            head_dim,
1498            actual_heads,
1499            norm_weight.size()
1500        );
1501
1502        // First, reshape the norm weights to match [1, 1, heads, dim]
1503        // The norm weights are typically stored as a flat vector [heads * dim]
1504        let norm_weight_reshaped = if norm_weight.size().len() == 1 {
1505            let norm_elements = norm_weight.size()[0];
1506            // For Gemma3, norm weights are per-head
1507            // If norm has 256 elements and we have 1 KV head with 256 dim, reshape to [1, 256]
1508            // If norm has 1024 elements and we have 4 Q heads with 256 dim, reshape to [4, 256]
1509            if norm_elements == (actual_heads * head_dim as usize) as i64 {
1510                // Standard case: reshape from [heads * dim] to [1, 1, heads, dim]
1511                norm_weight
1512                    .reshape([actual_heads as i64, head_dim])
1513                    .unsqueeze(0) // Add batch dimension
1514                    .unsqueeze(0) // Add seq dimension
1515            } else if norm_elements == head_dim {
1516                // Special case: norm is per-dimension only (for single head)
1517                norm_weight
1518                    .unsqueeze(0) // Add head dimension
1519                    .unsqueeze(0) // Add batch dimension
1520                    .unsqueeze(0) // Add seq dimension
1521            } else {
1522                return Err(anyhow!(
1523                    "QK-norm weight size {} doesn't match expected size for {} heads with dim {}",
1524                    norm_elements,
1525                    actual_heads,
1526                    head_dim
1527                ));
1528            }
1529        } else {
1530            norm_weight.shallow_clone()
1531        };
1532
1533        // Apply RMSNorm per head, preserving dtype
1534        let original_dtype = tensor.kind();
1535        let x2 = square_tensor(tensor)?;
1536        let mean = x2.mean_dim(&[-1i64][..], true, original_dtype);
1537        let eps = 1e-6;
1538        let rrms = (mean + eps).reciprocal().sqrt();
1539
1540        // Apply normalization and scale with norm weights
1541        broadcast_mul(&broadcast_mul(tensor, &rrms)?, &norm_weight_reshaped)
1542    }
1543
1544    /// Fused scaled dot-product attention with causal masking.
1545    ///
1546    /// Computes `softmax(Q·Kᵀ · scale + mask)·V` in FP32 for numerical stability
1547    /// (BF16 attention causes progressive precision loss in long sequences,
1548    /// matching HuggingFace/vLLM practice). Fusing the QK, softmax, and V steps
1549    /// lets the score matrix be produced and consumed one query-chunk at a time.
1550    ///
1551    /// For long prefills (`q_len > CHUNK`) the query axis is processed in chunks
1552    /// so the `[q_len, k_len]` FP32 score/softmax tensors are never materialized at
1553    /// full size — peak VRAM is O(CHUNK · k_len) rather than O(q_len · k_len).
1554    /// The decode path (`q_len == 1`) and short prompts take the single-shot
1555    /// branch, byte-for-byte equivalent to the previous implementation.
1556    ///
1557    /// `q`, `k`, `v` are `[batch, seq, heads, dim]` (K/V already GQA-expanded by
1558    /// the caller); the result is `[batch, heads, q_len, dim]`.
1559    fn compute_attention(&self, q: &Tensor, k: &Tensor, v: &Tensor) -> Result<Tensor> {
1560        self.compute_attention_masked(q, k, v, None)
1561    }
1562
1563    /// Scaled dot-product attention with an explicit additive attention mask.
1564    ///
1565    /// When `attn_mask` is `Some`, it is an additive mask broadcastable to the
1566    /// `[batch, heads, q_len, k_len]` score tensor (`0.0` for attend, a large
1567    /// negative for mask-out) and *replaces* the implicit `tril` causal mask.
1568    /// This is the entry the batched/ragged decode path uses to express, in one
1569    /// matmul, both per-row causality AND per-row KV padding (rows in a batch have
1570    /// different valid KV lengths). When `attn_mask` is `None` the behavior is
1571    /// byte-identical to the original inline-`tril` single-sequence path — kept as
1572    /// the equivalence reference.
1573    fn compute_attention_masked(
1574        &self,
1575        q: &Tensor,
1576        k: &Tensor,
1577        v: &Tensor,
1578        attn_mask: Option<&Tensor>,
1579    ) -> Result<Tensor> {
1580        /// Query-chunk size for long-prefill attention. 1024 keeps the per-chunk
1581        /// score matrix small while amortizing kernel-launch overhead.
1582        const CHUNK: i64 = 1024;
1583
1584        let scale = 1.0 / (self.head_dim as f32).sqrt();
1585        let original_kind = q.kind();
1586
1587        // [batch, seq, heads, dim] -> [batch, heads, seq, dim] (K transposed for matmul)
1588        // PERF: .contiguous() required for optimal batched matmul on ROCm/AMD
1589        let q = q.transpose(1, 2).contiguous();
1590        let k_t = k.transpose(1, 2).transpose(2, 3).contiguous(); // [batch, heads, dim, k_len]
1591        let v = v.transpose(1, 2).contiguous(); // [batch, heads, k_len, dim]
1592
1593        // Upcast to FP32 for precise score computation (matches prior behavior).
1594        let upcast = original_kind != tch::Kind::Float;
1595        let q = if upcast {
1596            q.to_kind(tch::Kind::Float)
1597        } else {
1598            q
1599        };
1600        let k_t = if upcast {
1601            k_t.to_kind(tch::Kind::Float)
1602        } else {
1603            k_t
1604        };
1605        let v = if upcast {
1606            v.to_kind(tch::Kind::Float)
1607        } else {
1608            v
1609        };
1610        // Match the score dtype (FP32) so broadcast_add doesn't upcast/clash.
1611        let attn_mask = attn_mask.map(|m| {
1612            if m.kind() != tch::Kind::Float {
1613                m.to_kind(tch::Kind::Float)
1614            } else {
1615                m.shallow_clone()
1616            }
1617        });
1618
1619        let q_len = q.size()[2];
1620        let k_len = k_t.size()[3];
1621        let device = q.device();
1622
1623        // An explicit mask must go through the single-shot path: it already encodes
1624        // causality + padding for the whole [q_len, k_len] grid, and chunking would
1625        // need to re-slice it per chunk. Decode batches have q_len == 1 anyway.
1626        let single_shot = attn_mask.is_some() || self.sliding_window.is_some() || q_len <= CHUNK;
1627
1628        let attn_output = if single_shot {
1629            // [batch, heads, q_len, k_len]
1630            let mut scores = q.matmul(&k_t) * (scale as f64);
1631
1632            if let Some(mask) = attn_mask.as_ref() {
1633                // Explicit additive mask supplied by the batched path; it fully
1634                // determines which (q, kv) pairs are valid, so skip the inline tril.
1635                scores = broadcast_add(&scores, mask)?;
1636            } else if q_len > 1 {
1637                // Causal mask only when processing multiple query tokens (prompt
1638                // phase); for q_len == 1 (decode) all past positions are valid.
1639                // Offset by the cached history length: query row i sits at
1640                // absolute position (k_len - q_len) + i, so it may attend keys
1641                // 0..=(k_len - q_len + i). tril(0) would mask out all history
1642                // whenever start_pos > 0 (e.g. partial prefill on a prefix hit).
1643                let mask =
1644                    Tensor::ones([q_len, k_len], (tch::Kind::Float, device)).tril(k_len - q_len);
1645                let mask = mask.unsqueeze(0).unsqueeze(0).expand_as(&scores);
1646                scores = scores.masked_fill(&mask.eq(0.0), -10000.0f64);
1647            }
1648
1649            // Sliding window mask if configured (Gemma3)
1650            if let Some(window_size) = self.sliding_window {
1651                if self.layer_type == "local" {
1652                    scores = self.apply_sliding_window_mask(&scores, window_size)?;
1653                }
1654            }
1655
1656            scores.softmax(-1, tch::Kind::Float).matmul(&v)
1657        } else {
1658            // Chunk over the query axis. Per-chunk causal mask
1659            // `tril(k_len - q_len + start)` is the exact restriction of the full
1660            // causal mask (history offset + row index) to rows [start, start+cur),
1661            // so masking is identical to the single-shot path.
1662            let mut outputs: Vec<Tensor> = Vec::new();
1663            let mut start = 0i64;
1664            while start < q_len {
1665                let cur = (q_len - start).min(CHUNK);
1666                let q_chunk = q.narrow(2, start, cur); // [batch, heads, cur, dim]
1667                let mut scores = q_chunk.matmul(&k_t) * (scale as f64); // [batch, heads, cur, k_len]
1668                let mask = Tensor::ones([cur, k_len], (tch::Kind::Float, device))
1669                    .tril(k_len - q_len + start);
1670                let mask = mask.unsqueeze(0).unsqueeze(0).expand_as(&scores);
1671                scores = scores.masked_fill(&mask.eq(0.0), -10000.0f64);
1672                outputs.push(scores.softmax(-1, tch::Kind::Float).matmul(&v)); // [batch, heads, cur, dim]
1673                start += cur;
1674            }
1675            Tensor::cat(&outputs, 2) // [batch, heads, q_len, dim]
1676        };
1677
1678        Ok(if upcast {
1679            attn_output.to_kind(original_kind)
1680        } else {
1681            attn_output
1682        })
1683    }
1684
1685    /// Apply sliding window mask for local attention layers
1686    /// Optimized to use GPU-native ops instead of O(n²) CPU loop
1687    fn apply_sliding_window_mask(&self, scores: &Tensor, window_size: usize) -> Result<Tensor> {
1688        let (batch_size, num_heads, seq_len, _) = dims4(scores)?;
1689        let device = scores.device();
1690        let dtype = scores.kind();
1691
1692        // Create sliding window mask using GPU-native tril/triu operations
1693        // Valid positions: j <= i (causal) AND i - j < window_size (in window)
1694
1695        // Causal mask: lower triangular (j <= i)
1696        let causal = Tensor::ones([seq_len, seq_len], (dtype, device)).tril(0);
1697
1698        // Window mask: upper triangular starting from -(window_size-1)
1699        // This keeps positions where j >= i - window_size + 1
1700        let window = Tensor::ones([seq_len, seq_len], (dtype, device)).triu(1 - window_size as i64);
1701
1702        // Valid = causal AND window (element-wise multiplication of 0/1 masks)
1703        let valid_mask = &causal * &window;
1704
1705        // Convert to additive mask: 0 where valid, -10000 where invalid
1706        // invalid = 1 - valid, then scale by -10000
1707        let additive_mask =
1708            (Tensor::ones([seq_len, seq_len], (dtype, device)) - valid_mask) * (-10000.0f64);
1709
1710        // Broadcast to [batch, heads, seq, seq]
1711        let mask = additive_mask
1712            .unsqueeze(0)
1713            .unsqueeze(0)
1714            .expand([batch_size, num_heads, seq_len, seq_len], false);
1715
1716        // Add mask to scores
1717        broadcast_add(scores, &mask)
1718    }
1719}
1720
1721/// Llama MLP/FFN layer
1722pub(crate) struct LlamaMLP {
1723    pub(crate) gate_proj: LinearProjection,
1724    pub(crate) up_proj: LinearProjection,
1725    pub(crate) down_proj: LinearProjection,
1726    pub(crate) activation: String, // Activation function name
1727    pub(crate) layer_idx: usize,   // Layer index for per-layer delta lookup
1728}
1729
1730unsafe impl Send for LlamaMLP {}
1731unsafe impl Sync for LlamaMLP {}
1732
1733impl LlamaMLP {
1734    /// Move all projection weights to `device` (#314 pipeline placement).
1735    /// `pub(crate)` so qwen3_5's dense-MLP stage placement can reuse it.
1736    #[inline]
1737    pub(crate) fn into_device(self, device: Device) -> Self {
1738        Self {
1739            gate_proj: self.gate_proj.into_device(device),
1740            up_proj: self.up_proj.into_device(device),
1741            down_proj: self.down_proj.into_device(device),
1742            activation: self.activation,
1743            layer_idx: self.layer_idx,
1744        }
1745    }
1746
1747    pub(crate) fn forward(
1748        &self,
1749        hidden_states: &Tensor,
1750        delta: Option<&crate::training::TenantDelta>,
1751    ) -> Result<Tensor> {
1752        // Get dimensions
1753        let original_shape = hidden_states.size();
1754        let (batch_size, seq_len, hidden_size) = if original_shape.len() == 3 {
1755            (original_shape[0], original_shape[1], original_shape[2])
1756        } else {
1757            // Already 2D
1758            return self.forward_2d(hidden_states, delta);
1759        };
1760
1761        // Reshape to 2D for matmul
1762        let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
1763
1764        // Apply SwiGLU: down(act(gate(x)) * up(x)) with delta corrections
1765        let mut gate_pre = self.gate_proj.try_apply(&hidden_2d)?;
1766        let mut up = self.up_proj.try_apply(&hidden_2d)?;
1767
1768        if let Some(delta) = delta {
1769            if delta.has_module("gate_proj", self.layer_idx) {
1770                let correction = delta.forward_2d(&hidden_2d, "gate_proj", self.layer_idx)?;
1771                let kind = gate_pre.kind();
1772                gate_pre += correction.to_kind(kind);
1773            }
1774            if delta.has_module("up_proj", self.layer_idx) {
1775                let correction = delta.forward_2d(&hidden_2d, "up_proj", self.layer_idx)?;
1776                let kind = up.kind();
1777                up += correction.to_kind(kind);
1778            }
1779        }
1780
1781        let gate = self.apply_activation(&gate_pre)?;
1782        let gated = &gate * &up;
1783
1784        let mut output = self.down_proj.try_apply(&gated)?;
1785        if let Some(delta) = delta {
1786            if delta.has_module("down_proj", self.layer_idx) {
1787                let correction = delta.forward_2d(&gated, "down_proj", self.layer_idx)?;
1788                let kind = output.kind();
1789                output += correction.to_kind(kind);
1790            }
1791        }
1792
1793        // Reshape back to 3D
1794        let out_size = output.size()[1];
1795        Ok(output.reshape([batch_size, seq_len, out_size]))
1796    }
1797
1798    fn forward_2d(
1799        &self,
1800        hidden_states: &Tensor,
1801        delta: Option<&crate::training::TenantDelta>,
1802    ) -> Result<Tensor> {
1803        // Apply SwiGLU: down(act(gate(x)) * up(x))
1804        let mut gate_pre = self.gate_proj.try_apply(hidden_states)?;
1805        let mut up = self.up_proj.try_apply(hidden_states)?;
1806
1807        if let Some(delta) = delta {
1808            if delta.has_module("gate_proj", self.layer_idx) {
1809                let correction = delta.forward_2d(hidden_states, "gate_proj", self.layer_idx)?;
1810                let kind = gate_pre.kind();
1811                gate_pre += correction.to_kind(kind);
1812            }
1813            if delta.has_module("up_proj", self.layer_idx) {
1814                let correction = delta.forward_2d(hidden_states, "up_proj", self.layer_idx)?;
1815                let kind = up.kind();
1816                up += correction.to_kind(kind);
1817            }
1818        }
1819
1820        let gate = self.apply_activation(&gate_pre)?;
1821        let gated = &gate * &up;
1822
1823        let mut output = self.down_proj.try_apply(&gated)?;
1824        if let Some(delta) = delta {
1825            if delta.has_module("down_proj", self.layer_idx) {
1826                let correction = delta.forward_2d(&gated, "down_proj", self.layer_idx)?;
1827                let kind = output.kind();
1828                output += correction.to_kind(kind);
1829            }
1830        }
1831
1832        Ok(output)
1833    }
1834
1835    /// Apply the configured activation function
1836    fn apply_activation(&self, x: &Tensor) -> Result<Tensor> {
1837        match self.activation.as_str() {
1838            "silu" => Ok(x.silu()),
1839            "gelu_pytorch_tanh" => self.gelu_pytorch_tanh(x),
1840            "gelu" => {
1841                // Standard GELU approximation
1842                // GELU(x) = x * Φ(x) where Φ is the CDF of standard normal
1843                // Using tanh approximation: 0.5 * x * (1 + tanh(sqrt(2/π) * (x + 0.044715 * x^3)))
1844                self.gelu_pytorch_tanh(x)
1845            }
1846            _ => {
1847                tracing::warn!(
1848                    "Unknown activation '{}', falling back to SiLU",
1849                    self.activation
1850                );
1851                Ok(x.silu())
1852            }
1853        }
1854    }
1855
1856    /// GELU activation with PyTorch tanh approximation (for Gemma3)
1857    /// GELU(x) = 0.5 * x * (1 + tanh(sqrt(2/π) * (x + 0.044715 * x^3)))
1858    fn gelu_pytorch_tanh(&self, x: &Tensor) -> Result<Tensor> {
1859        use std::f32::consts::PI;
1860
1861        // Constants for the approximation
1862        let sqrt_2_over_pi = (2.0_f32 / PI).sqrt();
1863        let coeff = 0.044715_f32;
1864
1865        // x^3
1866        let x_cubed = x.pow_tensor_scalar(3.0);
1867
1868        // x + 0.044715 * x^3
1869        let coeff_tensor = scalar_tensor(coeff, x.device(), x.kind());
1870        let inner = x + broadcast_mul(&x_cubed, &coeff_tensor)?;
1871
1872        // sqrt(2/π) * (x + 0.044715 * x^3)
1873        let sqrt_tensor = scalar_tensor(sqrt_2_over_pi, x.device(), x.kind());
1874        let scaled = broadcast_mul(&inner, &sqrt_tensor)?;
1875
1876        // tanh(sqrt(2/π) * (x + 0.044715 * x^3))
1877        let tanh_result = scaled.tanh();
1878
1879        // 1 + tanh(...)
1880        let one_tensor = scalar_tensor(1.0_f32, x.device(), x.kind());
1881        let one_plus_tanh = broadcast_add(&tanh_result, &one_tensor)?;
1882
1883        // 0.5 * x * (1 + tanh(...))
1884        let half_tensor = scalar_tensor(0.5_f32, x.device(), x.kind());
1885        broadcast_mul(&(x * &one_plus_tanh), &half_tensor)
1886    }
1887}
1888
1889/// RMSNorm implementation for Llama
1890pub(crate) struct RMSNorm {
1891    pub(crate) weight: Tensor,
1892    pub(crate) eps: f32,
1893}
1894
1895unsafe impl Send for RMSNorm {}
1896unsafe impl Sync for RMSNorm {}
1897
1898impl RMSNorm {
1899    /// Move the norm weight to `device` (#314 pipeline placement).
1900    #[inline]
1901    fn into_device(self, device: Device) -> Self {
1902        Self {
1903            weight: self.weight.to_device(device),
1904            eps: self.eps,
1905        }
1906    }
1907}
1908
1909impl RMSNorm {
1910    pub(crate) fn forward(&self, x: &Tensor) -> Result<Tensor> {
1911        // Compute RMS, preserving the original dtype
1912        let original_dtype = x.kind();
1913        let x2 = square_tensor(x)?;
1914        let mean = x2.mean_dim(&[-1i64][..], true, original_dtype); // Keep original dtype
1915        let rrms = (mean + self.eps as f64).reciprocal().sqrt();
1916
1917        // Apply normalization and scaling
1918        broadcast_mul(&broadcast_mul(x, &rrms)?, &self.weight)
1919    }
1920}
1921
1922impl LlamaModel {
1923    /// Create Llama model from SafeTensors
1924    pub fn from_safetensors(path: &Path, device: &Device, dtype: DType) -> Result<Self> {
1925        // Load configuration
1926        let config_path = path
1927            .parent()
1928            .ok_or_else(|| anyhow!("Invalid model path"))?
1929            .join("config.json");
1930
1931        let config = if config_path.exists() {
1932            let config_str = std::fs::read_to_string(&config_path)?;
1933            Self::parse_config(&config_str)?
1934        } else {
1935            LlamaConfig::default()
1936        };
1937
1938        // Load weights from SafeTensors (simplified)
1939        let layers = Vec::new();
1940
1941        // Whole-model single-device map (this simplified path has no layers yet;
1942        // size the map to the declared depth so the invariant holds).
1943        let device_map =
1944            LayerDeviceMap::single(*device, (config.num_hidden_layers as usize).max(1))?;
1945
1946        Ok(Self {
1947            config,
1948            device: *device,
1949            dtype,
1950            device_map,
1951            layer_offset: 0,
1952            embed_tokens: None,
1953            layers,
1954            norm: None,
1955            lm_head: None,
1956            lm_head_transposed: None,
1957            kv_cache: None, // Cache initialized on first use
1958            vs: None,       // No VarStore for safetensors loading
1959        })
1960    }
1961
1962    /// Create Llama model from pre-loaded weight tensors
1963    pub fn from_weights(
1964        weights: &mut HashMap<String, Tensor>,
1965        device: &Device,
1966        dtype: DType,
1967    ) -> Result<Self> {
1968        // Parse config from weights if possible, otherwise use defaults
1969        let config = Self::detect_config_from_weights(weights)?;
1970        Self::from_weights_with_config(
1971            weights,
1972            config,
1973            device,
1974            dtype,
1975            hyprstream_rpc_std::model_client::KVQuantType::None,
1976        )
1977    }
1978
1979    /// Create Llama model with explicit config (allows Qwen models to override)
1980    /// Build model from weights, taking mutable reference to free tensors as they're processed.
1981    /// This reduces peak memory by ~50% by removing original weight tensors after transposing.
1982    pub fn from_weights_with_config(
1983        weights: &mut HashMap<String, Tensor>,
1984        mut config: LlamaConfig,
1985        device: &Device,
1986        dtype: DType,
1987        kv_quant_type: hyprstream_rpc_std::model_client::KVQuantType,
1988    ) -> Result<Self> {
1989        tracing::info!(
1990            "[from_weights_with_config] Received config.max_position_embeddings = {}, kv_quant = {:?}",
1991            config.max_position_embeddings,
1992            kv_quant_type
1993        );
1994
1995        // Extract key tensors (with padding for models that need it).
1996        // #405: place top-level weights on `device` here, mirroring the
1997        // `build_layer`→`into_device` move applied to decoder layers. Without
1998        // this, constructing from CPU weights with device=Cuda(0) leaves
1999        // embed_tokens on CPU while layers move to CUDA — a mixed-device model.
2000        let embed_tokens = weights
2001            .get("model.embed_tokens.weight")
2002            .or_else(|| weights.get("embed_tokens.weight"))
2003            .map(|w| {
2004                let vocab_size = w.size()[0] as u32;
2005                let hidden_size = w.size()[1];
2006
2007                // Pad vocabulary size to multiple of 64 for models that need it
2008                let padded_vocab_size = calculate_padded_vocab_size(vocab_size);
2009
2010                let placed = if padded_vocab_size > vocab_size {
2011                    tracing::info!(
2012                        "Padding embedding vocab size from {} to {} (multiple of 64)",
2013                        vocab_size,
2014                        padded_vocab_size
2015                    );
2016
2017                    // Create padded tensor filled with zeros (embeddings can be zero-initialized)
2018                    let padded = Tensor::zeros(
2019                        [padded_vocab_size as i64, hidden_size],
2020                        (w.kind(), w.device()),
2021                    );
2022
2023                    // Copy original weights to the padded tensor
2024                    padded.narrow(0, 0, vocab_size as i64).copy_(w);
2025                    padded
2026                } else {
2027                    w.shallow_clone()
2028                };
2029                placed.to_device(*device)
2030            });
2031
2032        // Update config with padded vocab size if needed
2033        let original_vocab_size = config.vocab_size;
2034        let padded_vocab_size = calculate_padded_vocab_size(original_vocab_size);
2035
2036        // Store original vocab size before padding
2037        config.original_vocab_size = original_vocab_size;
2038
2039        if padded_vocab_size != original_vocab_size {
2040            tracing::info!(
2041                "Updating config vocab_size from {} to padded size {}",
2042                original_vocab_size,
2043                padded_vocab_size
2044            );
2045            config.vocab_size = padded_vocab_size;
2046        }
2047
2048        // Handle lm_head - Gemma models use weight tying (lm_head shares weights with embed_tokens)
2049        // Try to find explicit lm_head first, otherwise we'll use tied weights from embeddings
2050        let lm_head = weights
2051            .get("lm_head.weight")
2052            .or_else(|| weights.get("model.lm_head.weight"))
2053            .map(|w| {
2054                // LM head is stored as [vocab_size, hidden_size] in HuggingFace
2055                let vocab_size = w.size()[0] as u32;
2056                let hidden_size = w.size()[1];
2057
2058                // Pad vocabulary size to multiple of 64 (like SGLang does for Qwen)
2059                // This prevents sampling invalid token IDs
2060                let padded_vocab_size = calculate_padded_vocab_size(vocab_size);
2061
2062                let padded_w = if padded_vocab_size > vocab_size {
2063                    tracing::info!(
2064                        "Padding lm_head vocab size from {} to {} (multiple of 64)",
2065                        vocab_size,
2066                        padded_vocab_size
2067                    );
2068
2069                    // Create padded tensor filled with zeros
2070                    // NOTE: We can't use -1e10 here because this is the weight matrix, not logits
2071                    // The actual masking needs to happen after the matmul that produces logits
2072                    let padded = Tensor::zeros(
2073                        [padded_vocab_size as i64, hidden_size],
2074                        (w.kind(), w.device()),
2075                    );
2076
2077                    // Copy original weights to the padded tensor
2078                    padded.narrow(0, 0, vocab_size as i64).copy_(w);
2079                    padded
2080                } else {
2081                    w.shallow_clone()
2082                };
2083
2084                // We need [hidden_size, vocab_size] for matmul.
2085                // #405: place on `device` (see embed_tokens note above).
2086                padded_w.transpose(0, 1).contiguous().to_device(*device)
2087            });
2088
2089        // Pre-compute transposed lm_head for tied weights case
2090        // This avoids allocating a new tensor on every forward pass
2091        let lm_head_transposed = if lm_head.is_none() {
2092            embed_tokens.as_ref().map(|embed| {
2093                tracing::info!(
2094                    "Pre-computing lm_head transpose for tied weights (saves ~740MB per forward)"
2095                );
2096                embed.transpose(0, 1).contiguous()
2097            })
2098        } else {
2099            None
2100        };
2101
2102        // Extract final layer norm (#405: place on `device`).
2103        let norm = weights
2104            .get("model.norm.weight")
2105            .or_else(|| weights.get("norm.weight"))
2106            .map(|w| RMSNorm {
2107                weight: w.shallow_clone().to_device(*device),
2108                eps: config.rms_norm_eps,
2109            });
2110
2111        // Build transformer layers incrementally, freeing weight tensors as we go
2112        // This reduces peak memory by ~50% (7.7GB savings for Qwen3-4B)
2113        let mut layers = Vec::new();
2114        for layer_idx in 0..config.num_hidden_layers as usize {
2115            if let Some(layer) = Self::build_layer(layer_idx, weights, &config, device)? {
2116                layers.push(layer);
2117            }
2118        }
2119
2120        // Initialize KV cache if we have layers
2121        let kv_cache = if !layers.is_empty() {
2122            tracing::info!(
2123                "[LlamaModel] Creating KV cache: num_layers={}, max_seq_len={}, quant={:?}",
2124                layers.len(),
2125                config.max_position_embeddings,
2126                kv_quant_type
2127            );
2128            Some(std::sync::Arc::new(parking_lot::Mutex::new(
2129                crate::runtime::kv_cache::KVCacheManager::new(
2130                    layers.len(),
2131                    config.max_position_embeddings as usize,
2132                    kv_quant_type,
2133                ),
2134            )))
2135        } else {
2136            None
2137        };
2138
2139        // Unsplit fast path: every layer on the one device. This keeps the
2140        // whole-model forward byte-identical — `forward_layers` over this
2141        // single-device map performs zero cross-device copies (#314).
2142        let device_map = LayerDeviceMap::single(*device, layers.len().max(1))?;
2143
2144        Ok(Self {
2145            config,
2146            device: *device,
2147            dtype,
2148            device_map,
2149            layer_offset: 0,
2150            embed_tokens,
2151            layers,
2152            norm,
2153            lm_head,
2154            lm_head_transposed,
2155            kv_cache,
2156            vs: None, // No VarStore for weight loading - weights are stored directly
2157        })
2158    }
2159
2160    /// Build a **single pipeline stage** of a Llama model (#314, 2b layer-split).
2161    ///
2162    /// Loads only global decoder layers `[layer_range.start..layer_range.end)`
2163    /// onto their mapped devices (`devices.device_for(g)`), and gates the
2164    /// non-layer weights by stage position so middle stages carry none (M-LOAD
2165    /// seam #1):
2166    /// - `is_first` (`layer_range.start == 0`)  → keep `embed_tokens`.
2167    /// - `is_last`  (`layer_range.end == devices.len()`) → keep `norm` + `lm_head`
2168    ///   (or the tied transpose).
2169    ///
2170    /// `weights` must already contain (at least) this stage's tensors. Production
2171    /// stage requests use `ModelFactory::create_stage`, whose authoritative
2172    /// HF-index loader supplies only the requested range and role tensors.
2173    /// Tensors not on the target device are moved with a single `.to()` at
2174    /// construction — the *only* placement cost; the forward path then does zero
2175    /// intra-stage copies.
2176    ///
2177    /// State sizing follows the **owned** layer count, not the global count
2178    /// (M-LOAD seam #2): the KV-cache manager and `self.layers` are both sized to
2179    /// `layer_range.len()`, and the forward loop indexes them locally.
2180    #[allow(clippy::too_many_arguments)]
2181    pub fn stage_from_weights_with_config(
2182        weights: &mut HashMap<String, Tensor>,
2183        mut config: LlamaConfig,
2184        devices: &LayerDeviceMap,
2185        layer_range: std::ops::Range<usize>,
2186        dtype: DType,
2187        kv_quant_type: hyprstream_rpc_std::model_client::KVQuantType,
2188    ) -> Result<Self> {
2189        let num_global = config.num_hidden_layers as usize;
2190        if devices.len() != num_global {
2191            return Err(anyhow!(
2192                "stage_from_weights: device map covers {} layers but config has {} \
2193                 (num_hidden_layers)",
2194                devices.len(),
2195                num_global
2196            ));
2197        }
2198        if layer_range.end > num_global || layer_range.start >= layer_range.end {
2199            return Err(anyhow!(
2200                "stage_from_weights: invalid layer range {:?} for {} layers",
2201                layer_range,
2202                num_global
2203            ));
2204        }
2205
2206        let is_first = layer_range.start == 0;
2207        let is_last = layer_range.end == num_global;
2208        let layer_offset = layer_range.start;
2209        // Stage device = device of this stage's first owned layer (its inputs and
2210        // KV cache live here). `embed_tokens`, when present, lives on the first
2211        // stage's first device, which is exactly this when is_first.
2212        let stage_device = devices.device_for(layer_range.start);
2213
2214        // --- Non-layer weights, gated by stage position (M-LOAD seam #1) ---
2215        let original_vocab_size = config.vocab_size;
2216        let padded_vocab_size = calculate_padded_vocab_size(original_vocab_size);
2217        config.original_vocab_size = original_vocab_size;
2218        if padded_vocab_size != original_vocab_size {
2219            config.vocab_size = padded_vocab_size;
2220        }
2221
2222        let embed_tokens = if is_first {
2223            let embed = weights
2224                .get("model.embed_tokens.weight")
2225                .or_else(|| weights.get("embed_tokens.weight"))
2226                .map(|w| Self::pad_embedding_to(w, padded_vocab_size, stage_device))
2227                .ok_or_else(|| {
2228                    anyhow!("stage_from_weights: first stage requires model.embed_tokens.weight")
2229                })?;
2230            Some(embed)
2231        } else {
2232            None
2233        };
2234
2235        let (lm_head, norm) = if is_last {
2236            let lm_head = weights
2237                .get("lm_head.weight")
2238                .or_else(|| weights.get("model.lm_head.weight"))
2239                .map(|w| {
2240                    Self::pad_embedding_to(w, padded_vocab_size, stage_device)
2241                        .transpose(0, 1)
2242                        .contiguous()
2243                });
2244            let norm = weights
2245                .get("model.norm.weight")
2246                .or_else(|| weights.get("norm.weight"))
2247                .map(|w| RMSNorm {
2248                    weight: w.shallow_clone().to_device(stage_device),
2249                    eps: config.rms_norm_eps,
2250                });
2251            if norm.is_none() {
2252                return Err(anyhow!(
2253                    "stage_from_weights: last stage requires model.norm.weight"
2254                ));
2255            }
2256            (lm_head, norm)
2257        } else {
2258            (None, None)
2259        };
2260
2261        // Tied lm_head: only meaningful on the last stage, and only if it also
2262        // holds embed_tokens (single-stage model). A middle/last stage that does
2263        // not own the embedding cannot tie — it must ship an explicit lm_head.
2264        let lm_head_transposed = if is_last && lm_head.is_none() {
2265            embed_tokens
2266                .as_ref()
2267                .map(|embed| embed.transpose(0, 1).contiguous())
2268        } else {
2269            None
2270        };
2271        if is_last && lm_head.is_none() && lm_head_transposed.is_none() {
2272            return Err(anyhow!(
2273                "stage_from_weights: last stage requires lm_head.weight (no tied embedding present)"
2274            ));
2275        }
2276
2277        // --- Owned decoder layers, each on its mapped device (per-layer .to) ---
2278        let mut layers = Vec::with_capacity(layer_range.len());
2279        for g in layer_range.clone() {
2280            let target = devices.device_for(g);
2281            match Self::build_layer(g, weights, &config, &target)? {
2282                Some(layer) => layers.push(layer.into_device(target)),
2283                None => {
2284                    return Err(anyhow!(
2285                        "stage_from_weights: expected weights for global layer {g} but none found"
2286                    ))
2287                }
2288            }
2289        }
2290
2291        // KV cache sized to the OWNED layer count (M-LOAD seam #2); indexed
2292        // locally in the forward loop. Lives on the stage device.
2293        let kv_cache = if layers.is_empty() {
2294            None
2295        } else {
2296            Some(std::sync::Arc::new(parking_lot::Mutex::new(
2297                crate::runtime::kv_cache::KVCacheManager::new(
2298                    layers.len(),
2299                    config.max_position_embeddings as usize,
2300                    kv_quant_type,
2301                ),
2302            )))
2303        };
2304
2305        let device_map = devices.clone();
2306
2307        Ok(Self {
2308            config,
2309            device: stage_device,
2310            dtype,
2311            device_map,
2312            layer_offset,
2313            embed_tokens,
2314            layers,
2315            norm,
2316            lm_head,
2317            lm_head_transposed,
2318            kv_cache,
2319            vs: None,
2320        })
2321    }
2322
2323    /// Pad an embedding/lm_head weight `[vocab, hidden]` up to `padded_vocab` (a
2324    /// no-op shallow clone when already large enough) and place it on `device`.
2325    fn pad_embedding_to(w: &Tensor, padded_vocab: u32, device: Device) -> Tensor {
2326        let vocab_size = w.size()[0] as u32;
2327        let hidden_size = w.size()[1];
2328        let placed = if padded_vocab > vocab_size {
2329            let padded = Tensor::zeros([padded_vocab as i64, hidden_size], (w.kind(), w.device()));
2330            padded.narrow(0, 0, vocab_size as i64).copy_(w);
2331            padded
2332        } else {
2333            w.shallow_clone()
2334        };
2335        placed.to_device(device)
2336    }
2337
2338    /// Detect configuration from weight tensor shapes
2339    pub fn detect_config_from_weights(weights: &HashMap<String, Tensor>) -> Result<LlamaConfig> {
2340        // Try to infer config from tensor shapes
2341        let mut config = LlamaConfig::default();
2342        tracing::info!(
2343            "Starting detect_config_from_weights, default rope_theta={}",
2344            config.rope_theta
2345        );
2346
2347        // Get vocab size from embedding
2348        if let Some(embed) = weights
2349            .get("model.embed_tokens.weight")
2350            .or_else(|| weights.get("embed_tokens.weight"))
2351        {
2352            let shape = embed.size();
2353            if shape.len() >= 2 {
2354                config.vocab_size = shape[0] as u32;
2355                config.original_vocab_size = shape[0] as u32; // Initially same
2356                config.hidden_size = shape[1] as u32;
2357
2358                // Detect Gemma3 by vocab size (262144) and set specific parameters
2359                if config.vocab_size == 262144 {
2360                    config.hidden_activation = "gelu_pytorch_tanh".to_owned();
2361                    config.use_qk_norm = true;
2362                    config.scale_embeddings = true;
2363                    config.query_pre_attn_scalar = Some(256.0); // Common value for Gemma3
2364                    config.rope_local_base_freq = Some(10000.0);
2365                    config.rope_theta = 1000000.0; // Global attention theta for Gemma3
2366                    tracing::info!("Detected Gemma3 model (vocab_size=262144), configuring with:");
2367                    tracing::info!("  - GELU PyTorch tanh activation");
2368                    tracing::info!("  - QK-norm enabled");
2369                    tracing::info!("  - Embedding scaling enabled");
2370                    tracing::info!("  - RoPE theta: global=1000000, local=10000");
2371                }
2372            }
2373        }
2374
2375        // Count layers
2376        let layer_count = weights
2377            .keys()
2378            .filter(|k| k.contains("layers.") && k.contains(".self_attn.q_proj"))
2379            .count();
2380        if layer_count > 0 {
2381            config.num_hidden_layers = layer_count as u32;
2382        }
2383
2384        // Get attention heads from q_proj and k_proj shapes
2385        if let Some(q_proj) = weights.get("model.layers.0.self_attn.q_proj.weight") {
2386            let q_shape = q_proj.size();
2387            if q_shape.len() >= 2 {
2388                // shape[0] is output dim (num_heads * head_dim)
2389                // shape[1] is hidden_size
2390                config.hidden_size = q_shape[1] as u32;
2391                let q_proj_out_dim = q_shape[0] as usize;
2392
2393                // Also check k_proj to detect GQA (Grouped Query Attention)
2394                let k_proj_out_dim =
2395                    if let Some(k_proj) = weights.get("model.layers.0.self_attn.k_proj.weight") {
2396                        k_proj.size()[0] as usize
2397                    } else {
2398                        q_proj_out_dim // Assume same as Q if K not found
2399                    };
2400
2401                tracing::debug!(
2402                    "Q projection output: {}, K projection output: {}",
2403                    q_proj_out_dim,
2404                    k_proj_out_dim
2405                );
2406
2407                // Try common head_dim values to find the right configuration
2408                // Gemma3 uses head_dim=256, so check that first for Gemma models
2409                let possible_head_dims = if config.vocab_size == 262144 {
2410                    // Gemma3 detected - prioritize 256 head dim
2411                    vec![256, 128, 64, 32]
2412                } else if q_proj_out_dim <= 512 {
2413                    vec![32, 64, 48, 40, 128] // Smaller models often use smaller head_dim
2414                } else {
2415                    vec![128, 64, 80, 96, 160, 256]
2416                };
2417
2418                let mut found_config = false;
2419
2420                for &head_dim in &possible_head_dims {
2421                    if q_proj_out_dim.is_multiple_of(head_dim) && k_proj_out_dim % head_dim == 0 {
2422                        config.num_attention_heads = (q_proj_out_dim / head_dim) as u32;
2423                        config.num_key_value_heads = (k_proj_out_dim / head_dim) as u32;
2424                        config.head_dim = head_dim as u32;
2425                        found_config = true;
2426                        tracing::info!("Detected attention config: Q={} heads, KV={} heads, dim={}, total Q={}", 
2427                                     config.num_attention_heads, config.num_key_value_heads,
2428                                     config.head_dim, q_proj_out_dim);
2429                        break;
2430                    }
2431                }
2432
2433                if !found_config {
2434                    // Fallback: try to guess based on common patterns
2435                    // Many small models use head_dim=32 or 64
2436                    if q_proj_out_dim == 256 && k_proj_out_dim == 256 {
2437                        // Likely 8 heads with dim 32
2438                        config.num_attention_heads = 8;
2439                        config.num_key_value_heads = 8;
2440                        config.head_dim = 32;
2441                        tracing::info!("Using common small model config: 8 heads x 32 dim");
2442                    } else {
2443                        // Last resort defaults
2444                        config.num_attention_heads = 8;
2445                        config.num_key_value_heads = 8;
2446                        config.head_dim = 64;
2447                        tracing::warn!("Could not detect attention config from shapes Q:{:?} K:{:?}, using defaults", 
2448                                     q_shape, k_proj_out_dim);
2449                    }
2450                }
2451            }
2452        }
2453
2454        tracing::info!(
2455            "Finished detect_config_from_weights, final rope_theta={}, vocab_size={}",
2456            config.rope_theta,
2457            config.vocab_size
2458        );
2459        Ok(config)
2460    }
2461
2462    /// Build a single transformer layer from weights
2463    /// Takes mutable reference to remove processed weights, reducing peak memory
2464    fn build_layer(
2465        layer_idx: usize,
2466        weights: &mut HashMap<String, Tensor>,
2467        config: &LlamaConfig,
2468        _device: &Device,
2469    ) -> Result<Option<LlamaLayer>> {
2470        let prefix = format!("model.layers.{layer_idx}");
2471
2472        // Check if this layer exists (handle both separate and combined projections)
2473        let has_separate_qkv = weights.contains_key(&format!("{prefix}.self_attn.q_proj.weight"));
2474        let has_combined_qkv = weights.contains_key(&format!("{prefix}.self_attn.c_attn.weight"));
2475
2476        if !has_separate_qkv && !has_combined_qkv {
2477            return Ok(None);
2478        }
2479
2480        // Build attention
2481        let (q_proj, k_proj, v_proj) = if has_separate_qkv {
2482            // Standard separate Q, K, V projections (Llama/Qwen style)
2483
2484            let q_proj = LinearProjection::take_with_optional_bias(
2485                weights,
2486                &format!("{prefix}.self_attn.q_proj.weight"),
2487                &format!("{prefix}.self_attn.q_proj.bias"),
2488            )?;
2489            let k_proj = LinearProjection::take_with_optional_bias(
2490                weights,
2491                &format!("{prefix}.self_attn.k_proj.weight"),
2492                &format!("{prefix}.self_attn.k_proj.bias"),
2493            )?;
2494            let v_proj = LinearProjection::take_with_optional_bias(
2495                weights,
2496                &format!("{prefix}.self_attn.v_proj.weight"),
2497                &format!("{prefix}.self_attn.v_proj.bias"),
2498            )?;
2499
2500            (q_proj, k_proj, v_proj)
2501        } else {
2502            // Combined QKV projection (some Qwen models use c_attn)
2503            let c_attn_weight = weights
2504                .remove(&format!("{prefix}.self_attn.c_attn.weight"))
2505                .ok_or_else(|| anyhow!("Missing c_attn weight"))?
2506                .transpose(0, 1)
2507                .contiguous();
2508
2509            // c_attn has shape [hidden_size, 3 * projection_size]
2510            let total_proj_size = c_attn_weight.size()[1];
2511            let proj_size = total_proj_size / 3;
2512
2513            let q_weight = c_attn_weight.narrow(1, 0, proj_size);
2514            let k_weight = c_attn_weight.narrow(1, proj_size, proj_size);
2515            let v_weight = c_attn_weight.narrow(1, proj_size * 2, proj_size);
2516
2517            let (q_proj, k_proj, v_proj) =
2518                if let Some(bias) = weights.get(&format!("{prefix}.self_attn.c_attn.bias")) {
2519                    tracing::debug!(
2520                        "Layer {}: Loading combined QKV bias (Qwen-style)",
2521                        layer_idx
2522                    );
2523                    let q_bias = bias.narrow(0, 0, proj_size);
2524                    let k_bias = bias.narrow(0, proj_size, proj_size);
2525                    let v_bias = bias.narrow(0, proj_size * 2, proj_size);
2526                    (
2527                        LinearProjection::with_bias(q_weight, q_bias),
2528                        LinearProjection::with_bias(k_weight, k_bias),
2529                        LinearProjection::with_bias(v_weight, v_bias),
2530                    )
2531                } else {
2532                    (
2533                        LinearProjection::new(q_weight),
2534                        LinearProjection::new(k_weight),
2535                        LinearProjection::new(v_weight),
2536                    )
2537                };
2538
2539            (q_proj, k_proj, v_proj)
2540        };
2541
2542        // Check for QK-norm weights (Gemma3)
2543        let q_norm = weights
2544            .get(&format!("{prefix}.self_attn.q_norm.weight"))
2545            .map(tch::Tensor::shallow_clone);
2546        let k_norm = weights
2547            .get(&format!("{prefix}.self_attn.k_norm.weight"))
2548            .map(tch::Tensor::shallow_clone);
2549
2550        if q_norm.is_some() || k_norm.is_some() {
2551            tracing::debug!("Layer {} has QK-norm weights", layer_idx);
2552        }
2553
2554        // Determine layer type for Gemma3 sliding window attention
2555        let layer_type = if !config.layer_types.is_empty() && layer_idx < config.layer_types.len() {
2556            config.layer_types[layer_idx].clone()
2557        } else if config.layer_types.is_empty() && config.use_qk_norm {
2558            // Gemma3 pattern: every 6th layer is global, others are local
2559            if (layer_idx + 1).is_multiple_of(6) {
2560                "global".to_owned()
2561            } else {
2562                "local".to_owned()
2563            }
2564        } else {
2565            "global".to_owned() // Default to global attention
2566        };
2567
2568        // Determine sliding window size for Gemma3
2569        let sliding_window = if config.use_qk_norm && layer_type == "local" {
2570            Some(512) // Gemma3 uses 512 token sliding window
2571        } else {
2572            None
2573        };
2574
2575        // Use different RoPE theta for local vs global layers if configured
2576        let rope_theta = if layer_type == "local" {
2577            config.rope_local_base_freq.unwrap_or(config.rope_theta)
2578        } else {
2579            config.rope_theta
2580        };
2581
2582        // Load output projection (typically no bias, but check anyway)
2583        // Try o_proj first, then c_proj (some Qwen models use c_proj for output)
2584        let o_proj = if weights.contains_key(&format!("{prefix}.self_attn.o_proj.weight")) {
2585            LinearProjection::take_with_optional_bias(
2586                weights,
2587                &format!("{prefix}.self_attn.o_proj.weight"),
2588                &format!("{prefix}.self_attn.o_proj.bias"),
2589            )?
2590        } else {
2591            LinearProjection::take_with_optional_bias(
2592                weights,
2593                &format!("{prefix}.self_attn.c_proj.weight"),
2594                &format!("{prefix}.self_attn.c_proj.bias"),
2595            )?
2596        };
2597
2598        let self_attn = LlamaAttention {
2599            num_heads: config.num_attention_heads as usize,
2600            num_kv_heads: config.num_key_value_heads as usize,
2601            head_dim: config.head_dim as usize,
2602            q_proj,
2603            k_proj,
2604            v_proj,
2605            o_proj,
2606            rope_theta,
2607            rope_scaling: config.rope_scaling.clone(),
2608            q_norm,
2609            k_norm,
2610            query_pre_attn_scalar: config.query_pre_attn_scalar,
2611            sliding_window,
2612            layer_type,
2613            layer_idx,
2614        };
2615
2616        // Build MLP with optional biases
2617        let gate_proj = LinearProjection::take_with_optional_bias(
2618            weights,
2619            &format!("{prefix}.mlp.gate_proj.weight"),
2620            &format!("{prefix}.mlp.gate_proj.bias"),
2621        )?;
2622        let up_proj = LinearProjection::take_with_optional_bias(
2623            weights,
2624            &format!("{prefix}.mlp.up_proj.weight"),
2625            &format!("{prefix}.mlp.up_proj.bias"),
2626        )?;
2627        let down_proj = LinearProjection::take_with_optional_bias(
2628            weights,
2629            &format!("{prefix}.mlp.down_proj.weight"),
2630            &format!("{prefix}.mlp.down_proj.bias"),
2631        )?;
2632
2633        let mlp = LlamaMLP {
2634            gate_proj,
2635            up_proj,
2636            down_proj,
2637            activation: config.hidden_activation.clone(),
2638            layer_idx,
2639        };
2640
2641        // Build layer norms
2642        let input_layernorm = RMSNorm {
2643            weight: weights
2644                .get(&format!("{prefix}.input_layernorm.weight"))
2645                .ok_or_else(|| anyhow!("Missing input_layernorm weight"))?
2646                .shallow_clone(),
2647            eps: config.rms_norm_eps,
2648        };
2649
2650        let post_attention_layernorm = RMSNorm {
2651            weight: weights
2652                .get(&format!("{prefix}.post_attention_layernorm.weight"))
2653                .ok_or_else(|| anyhow!("Missing post_attention_layernorm weight"))?
2654                .shallow_clone(),
2655            eps: config.rms_norm_eps,
2656        };
2657
2658        Ok(Some(LlamaLayer {
2659            self_attn,
2660            mlp,
2661            input_layernorm,
2662            post_attention_layernorm,
2663        }))
2664    }
2665
2666    /// Parse configuration from JSON
2667    pub fn parse_config(json_str: &str) -> Result<LlamaConfig> {
2668        let json: serde_json::Value = serde_json::from_str(json_str)?;
2669
2670        // Mandatory architectural dimensions — never silently defaulted (mirrors
2671        // #315's no-magic-number hardening in model_config.rs). A wrong layer or
2672        // head count silently truncates / mis-shapes a pipeline split (#314).
2673        let require_u64 = |field: &str| -> Result<u64> {
2674            json[field].as_u64().ok_or_else(|| {
2675                anyhow!(
2676                    "config.json is missing required field `{field}` \
2677                     (or it is not a non-negative integer)"
2678                )
2679            })
2680        };
2681        let num_attention_heads = require_u64("num_attention_heads")? as u32;
2682        let hidden_size = require_u64("hidden_size")? as u32;
2683        let num_hidden_layers = require_u64("num_hidden_layers")? as u32;
2684        if num_attention_heads == 0 || hidden_size == 0 || num_hidden_layers == 0 {
2685            return Err(anyhow!(
2686                "config.json has a zero architectural dimension \
2687                 (num_attention_heads={num_attention_heads}, hidden_size={hidden_size}, \
2688                 num_hidden_layers={num_hidden_layers})"
2689            ));
2690        }
2691
2692        // Detect version from config
2693        let version = if json.get("rope_scaling").is_some() {
2694            3
2695        } else if json["num_key_value_heads"] != json["num_attention_heads"] {
2696            2
2697        } else {
2698            1
2699        };
2700
2701        let mut config = LlamaConfig {
2702            version,
2703            num_attention_heads,
2704            // num_key_value_heads legitimately defaults to num_attention_heads
2705            // (MHA) when absent — an HF-spec optional field, not a magic number.
2706            num_key_value_heads: json
2707                .get("num_key_value_heads")
2708                .and_then(serde_json::Value::as_u64)
2709                .map(|v| v as u32)
2710                .unwrap_or(num_attention_heads),
2711            hidden_size,
2712            head_dim: json
2713                .get("head_dim")
2714                .and_then(serde_json::Value::as_u64)
2715                .map(|v| v as u32)
2716                .unwrap_or_else(|| {
2717                    // head_dim is optional in HF configs; derive it from the now-
2718                    // mandatory hidden_size / num_attention_heads.
2719                    let heads = num_attention_heads;
2720                    let hidden = hidden_size;
2721                    // Common head_dim values are 64, 128, 256
2722                    if hidden.is_multiple_of(heads * 256) {
2723                        256
2724                    } else if hidden.is_multiple_of(heads * 128) {
2725                        128
2726                    } else if hidden.is_multiple_of(heads * 64) {
2727                        64
2728                    } else {
2729                        128 // Default fallback
2730                    }
2731                }),
2732            intermediate_size: json["intermediate_size"].as_u64().unwrap_or(11008) as u32,
2733            max_position_embeddings: json["max_position_embeddings"].as_u64().unwrap_or(4096)
2734                as u32,
2735            rms_norm_eps: json["rms_norm_eps"].as_f64().unwrap_or(1e-5) as f32,
2736            vocab_size: json["vocab_size"].as_u64().unwrap_or(32000) as u32,
2737            original_vocab_size: json["vocab_size"].as_u64().unwrap_or(32000) as u32, // Initially same
2738            num_hidden_layers,
2739            rope_theta: json
2740                .get("rope_theta")
2741                .and_then(serde_json::Value::as_f64)
2742                .unwrap_or(10000.0) as f32,
2743            rope_scaling: None,
2744            hidden_activation: json
2745                .get("hidden_activation")
2746                .and_then(|v| v.as_str())
2747                .unwrap_or("silu")
2748                .to_owned(),
2749            query_pre_attn_scalar: json
2750                .get("query_pre_attn_scalar")
2751                .and_then(serde_json::Value::as_f64)
2752                .map(|v| v as f32),
2753            use_qk_norm: false, // Will be set based on vocab size or explicit config
2754            scale_embeddings: false, // Will be set based on vocab size or explicit config
2755            layer_types: vec![],
2756            rope_local_base_freq: None,
2757            // Best-effort architecture hint from model_type so tokenizer-config
2758            // dispatch sees Qwen when this path is used directly. The factory's
2759            // authoritative `create_*_model` path overwrites this via the unified
2760            // ModelConfig (#143).
2761            model_architecture: json
2762                .get("model_type")
2763                .and_then(|v| v.as_str())
2764                .map(|mt| {
2765                    let mt = mt.to_lowercase();
2766                    match mt.as_str() {
2767                        "qwen" | "qwen2" | "qwen3" => ModelArchitecture::Qwen {
2768                            version: if mt.starts_with("qwen3") {
2769                                3
2770                            } else if mt.starts_with("qwen2") {
2771                                2
2772                            } else {
2773                                1
2774                            },
2775                            is_moe: false,
2776                            context_length: json["max_position_embeddings"].as_u64().unwrap_or(4096)
2777                                as usize,
2778                        },
2779                        "mistral" => ModelArchitecture::Mistral,
2780                        "gemma" => ModelArchitecture::Gemma,
2781                        _ => ModelArchitecture::Llama { version: 2 },
2782                    }
2783                })
2784                .unwrap_or(ModelArchitecture::Llama { version: 2 }),
2785        };
2786
2787        // Parse rope scaling if present
2788        if let Some(rope_scaling) = json.get("rope_scaling") {
2789            config.rope_scaling = Some(RopeScaling {
2790                scaling_type: rope_scaling["type"].as_str().unwrap_or("linear").to_owned(),
2791                factor: rope_scaling["factor"].as_f64().unwrap_or(8.0) as f32,
2792            });
2793        }
2794
2795        // Check for Gemma3 specific configurations
2796        if config.vocab_size == 262144 {
2797            config.hidden_activation = "gelu_pytorch_tanh".to_owned();
2798            config.use_qk_norm = true;
2799            config.scale_embeddings = true;
2800            if config.query_pre_attn_scalar.is_none() {
2801                config.query_pre_attn_scalar = Some(256.0);
2802            }
2803            config.rope_local_base_freq = Some(10000.0);
2804            config.rope_theta = 1000000.0; // Global attention theta
2805            tracing::info!("Detected Gemma3 model from config.json, applying Gemma3 settings");
2806        }
2807
2808        tracing::info!(
2809            "Parsed Llama config from JSON: rope_theta={}, vocab_size={}",
2810            config.rope_theta,
2811            config.vocab_size
2812        );
2813        Ok(config)
2814    }
2815
2816    /// Core forward pass with optional delta injection (internal helper)
2817    ///
2818    /// Shared implementation for `forward_with_cache` (delta=None) and
2819    /// `forward_with_cache_and_delta` (delta=Some). Delta corrections are
2820    /// injected in each attention layer's Q/V projections before KV cache update.
2821    fn forward_with_cache_inner(
2822        &self,
2823        input: &Tensor,
2824        start_pos: usize,
2825        delta: Option<&crate::training::TenantDelta>,
2826    ) -> Result<Tensor> {
2827        tracing::trace!("LlamaModel forward_with_cache_inner: input shape={:?}, start_pos={}, has_delta={}, config: hidden_size={}, num_layers={}",
2828                     input.size(), start_pos, delta.is_some(), self.config.hidden_size, self.layers.len());
2829
2830        // Input should be token IDs with shape [batch_size, seq_len]
2831        let mut hidden_states = if let Some(embed) = &self.embed_tokens {
2832            // Convert token IDs to embeddings
2833            let input_shape = input.size();
2834
2835            let batch_size = input_shape[0];
2836            let seq_len = if input_shape.len() > 1 {
2837                input_shape[1]
2838            } else {
2839                1
2840            };
2841
2842            // Flatten input for embedding lookup (embedding expects 1D tensor)
2843            let flat_input = input.flatten(0, -1);
2844
2845            // Perform embedding lookup using index_select
2846            let embeddings = embed.index_select(0, &flat_input);
2847
2848            // Get the actual hidden size from the embedding result
2849            let emb_dims = embeddings.size();
2850            let hidden_size = emb_dims[emb_dims.len() - 1];
2851
2852            // Reshape back to [batch_size, seq_len, hidden_size]
2853            let mut embeddings = embeddings.reshape([batch_size, seq_len, hidden_size]);
2854
2855            // Scale embeddings by sqrt(hidden_size) for Gemma3
2856            if self.config.scale_embeddings {
2857                let scale = (hidden_size as f32).sqrt();
2858                let scale_tensor = Tensor::from_slice(&[scale])
2859                    .to_kind(embeddings.kind())
2860                    .to_device(embeddings.device());
2861                embeddings = broadcast_mul(&embeddings, &scale_tensor)?;
2862            }
2863
2864            embeddings
2865        } else {
2866            // If no embedding layer, assume input is already embedded
2867            input.shallow_clone()
2868        };
2869
2870        // Apply transformer layers
2871        tracing::trace!("Total layers to process: {}", self.layers.len());
2872
2873        // Generate position_ids based on start_pos (for proper KV cache usage)
2874        let seq_len = hidden_states.size()[1];
2875        let position_ids = if start_pos == 0 {
2876            Tensor::arange(seq_len, (tch::Kind::Int64, hidden_states.device()))
2877        } else {
2878            Tensor::arange_start(
2879                start_pos as i64,
2880                (start_pos + seq_len as usize) as i64,
2881                (tch::Kind::Int64, hidden_states.device()),
2882            )
2883        };
2884        let mut position_ids = position_ids;
2885
2886        // PERF: Lock KV cache ONCE before the layer loop (not 28 times inside!)
2887        let cache_guard = self.kv_cache.as_ref().map(|cache_ref| cache_ref.lock());
2888
2889        for (idx, layer) in self.layers.iter().enumerate() {
2890            // Production cached forwards own the full mapped layer range, so
2891            // carry both device-bound inputs across each real layer boundary.
2892            // This mirrors `forward_layers_inner`; the unsplit fast path never
2893            // enters this branch.
2894            let global_idx = self.layer_offset + idx;
2895            let target = self.device_map.device_for(global_idx);
2896            if hidden_states.device() != target {
2897                tracing::debug!(
2898                    layer = global_idx,
2899                    from = ?hidden_states.device(),
2900                    to = ?target,
2901                    "Llama cached forward crossing device boundary"
2902                );
2903                hidden_states = hidden_states.to_device(target);
2904                position_ids = position_ids.to_device(target);
2905            }
2906
2907            let residual = hidden_states.shallow_clone();
2908
2909            // Self-attention block with optional KV cache and delta injection
2910            hidden_states = layer.input_layernorm.forward(&hidden_states)?;
2911
2912            // Handle KV cache per layer with proper start_pos
2913            let attn_output = if let Some(ref cache_manager) = cache_guard {
2914                if let Some(result) = cache_manager.with_layer_cache(idx, |layer_cache| {
2915                    tracing::trace!(
2916                        "Layer {}: Using KV cache, cache_pos={}",
2917                        idx,
2918                        layer_cache.seq_pos
2919                    );
2920                    layer.self_attn.forward(
2921                        &hidden_states,
2922                        Some(&position_ids),
2923                        Some(layer_cache),
2924                        start_pos,
2925                        delta,
2926                    )
2927                }) {
2928                    result?
2929                } else {
2930                    layer.self_attn.forward(
2931                        &hidden_states,
2932                        Some(&position_ids),
2933                        None,
2934                        start_pos,
2935                        delta,
2936                    )?
2937                }
2938            } else {
2939                layer.self_attn.forward(
2940                    &hidden_states,
2941                    Some(&position_ids),
2942                    None,
2943                    start_pos,
2944                    delta,
2945                )?
2946            };
2947            hidden_states = residual + attn_output;
2948
2949            // FFN block with delta injection for MLP projections
2950            let residual = hidden_states.shallow_clone();
2951            hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
2952            let ffn_output = layer.mlp.forward(&hidden_states, delta)?;
2953            hidden_states = residual + ffn_output;
2954        }
2955
2956        // Non-layer weights live on the engine's primary device. A split model
2957        // finishes its decoder stack on the final layer's device, so return the
2958        // activation once before final norm / lm_head. This is also a no-op for
2959        // the single-device path.
2960        if hidden_states.device() != self.device {
2961            tracing::debug!(
2962                from = ?hidden_states.device(),
2963                to = ?self.device,
2964                "Llama cached forward returning activation to output device"
2965            );
2966            hidden_states = hidden_states.to_device(self.device);
2967        }
2968
2969        // Final layer norm
2970        if let Some(norm) = &self.norm {
2971            hidden_states = norm.forward(&hidden_states)?;
2972        }
2973
2974        // LM head
2975        if let Some(lm_head) = &self.lm_head {
2976            hidden_states = hidden_states.matmul(lm_head);
2977        } else if let Some(output_proj) = &self.lm_head_transposed {
2978            let hs_shape = hidden_states.size();
2979            if hs_shape.len() == 3 {
2980                let (batch_size, seq_len, hidden_size) = (hs_shape[0], hs_shape[1], hs_shape[2]);
2981                let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
2982                let logits_2d = hidden_2d.matmul(output_proj);
2983                hidden_states = logits_2d.reshape([batch_size, seq_len, output_proj.size()[1]]);
2984            } else {
2985                hidden_states = hidden_states.matmul(output_proj);
2986            }
2987        } else {
2988            tracing::warn!("No LM head or embedding weights found - returning hidden states!");
2989        }
2990
2991        // Mask padded vocabulary entries if vocab was padded
2992        let original_vocab_size = self.config.original_vocab_size;
2993        let padded_vocab_size = self.config.vocab_size;
2994        if padded_vocab_size > original_vocab_size && original_vocab_size > 0 {
2995            let logits_shape = hidden_states.size();
2996            let actual_vocab_size = logits_shape[logits_shape.len() - 1] as u32;
2997
2998            if actual_vocab_size == padded_vocab_size {
2999                let mask_start = original_vocab_size as i64;
3000                let mask_count = (padded_vocab_size - original_vocab_size) as i64;
3001
3002                if mask_count > 0 {
3003                    let mask_values = Tensor::full(
3004                        [mask_count],
3005                        -1e10_f64,
3006                        (hidden_states.kind(), hidden_states.device()),
3007                    );
3008
3009                    if logits_shape.len() == 3 {
3010                        let batch_size = logits_shape[0];
3011                        let seq_len = logits_shape[1];
3012                        for b in 0..batch_size {
3013                            for s in 0..seq_len {
3014                                let slice = hidden_states.select(0, b).select(0, s);
3015                                slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3016                            }
3017                        }
3018                    } else if logits_shape.len() == 2 {
3019                        let rows = logits_shape[0];
3020                        for i in 0..rows {
3021                            let slice = hidden_states.select(0, i);
3022                            slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3023                        }
3024                    } else {
3025                        hidden_states
3026                            .narrow(0, mask_start, mask_count)
3027                            .copy_(&mask_values);
3028                    }
3029                }
3030            }
3031        }
3032
3033        Ok(hidden_states)
3034    }
3035
3036    /// Mask padded vocabulary columns to `-1e10` in-place on a logits tensor.
3037    ///
3038    /// Shared by the single-sequence and batched forward paths so both produce
3039    /// identical logits over the padded columns. No-op when the vocab was not
3040    /// padded. `logits` may be `[B, seq, vocab]` or `[rows, vocab]`.
3041    fn mask_padded_vocab(&self, logits: &Tensor) {
3042        let original_vocab_size = self.config.original_vocab_size;
3043        let padded_vocab_size = self.config.vocab_size;
3044        if padded_vocab_size <= original_vocab_size || original_vocab_size == 0 {
3045            return;
3046        }
3047        let logits_shape = logits.size();
3048        let actual_vocab_size = logits_shape[logits_shape.len() - 1] as u32;
3049        if actual_vocab_size != padded_vocab_size {
3050            return;
3051        }
3052        let mask_start = original_vocab_size as i64;
3053        let mask_count = (padded_vocab_size - original_vocab_size) as i64;
3054        if mask_count <= 0 {
3055            return;
3056        }
3057        let mask_values = Tensor::full([mask_count], -1e10_f64, (logits.kind(), logits.device()));
3058        match logits_shape.len() {
3059            3 => {
3060                for b in 0..logits_shape[0] {
3061                    for s in 0..logits_shape[1] {
3062                        let slice = logits.select(0, b).select(0, s);
3063                        slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3064                    }
3065                }
3066            }
3067            2 => {
3068                for i in 0..logits_shape[0] {
3069                    let slice = logits.select(0, i);
3070                    slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3071                }
3072            }
3073            _ => {
3074                logits.narrow(0, mask_start, mask_count).copy_(&mask_values);
3075            }
3076        }
3077    }
3078
3079    /// Batched ragged forward for continuous decode (#329, Llama-only).
3080    ///
3081    /// `sequences[r]` = `(new_token_ids, start_pos, per-sequence KVCacheManager)`.
3082    /// All rows share the same query length `q = new_token_ids.len()` (v1 decode
3083    /// drives `q == 1`); each row brings its own absolute `start_pos` and isolated
3084    /// per-sequence KV cache (tenant isolation). `delta` is the single tenant
3085    /// delta shared by every row in the batch (the scheduler groups by delta).
3086    ///
3087    /// Returns stacked logits `[B, q, vocab]`. The result for row `r` is bit-for-
3088    /// bit equivalent (within fp tolerance) to `forward_with_cache_and_delta` run
3089    /// on that sequence alone — this is the #329 PR-1 correctness gate.
3090    ///
3091    /// Inherent impl behind the [`ModelOperations::forward_batched`] override.
3092    fn forward_batched_impl(
3093        &self,
3094        sequences: &mut [(
3095            Vec<i64>,
3096            usize,
3097            std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
3098        )],
3099        delta: Option<&crate::training::TenantDelta>,
3100    ) -> Result<Tensor> {
3101        let batch_size = sequences.len();
3102        if batch_size == 0 {
3103            return Err(anyhow!("forward_batched: empty batch"));
3104        }
3105        let q_len = sequences[0].0.len();
3106        if q_len == 0 {
3107            return Err(anyhow!("forward_batched: empty sequence"));
3108        }
3109        if sequences.iter().any(|(toks, ..)| toks.len() != q_len) {
3110            return Err(anyhow!(
3111                "forward_batched: ragged query lengths are not supported in v1 (all rows must share q_len)"
3112            ));
3113        }
3114        let device = self.device;
3115
3116        // Build the [B, q] input id tensor and embed it as one batch.
3117        let flat_ids: Vec<i64> = sequences
3118            .iter()
3119            .flat_map(|(t, ..)| t.iter().copied())
3120            .collect();
3121        let input_ids = Tensor::from_slice(&flat_ids)
3122            .reshape([batch_size as i64, q_len as i64])
3123            .to_device(device);
3124        let mut hidden_states = self.embed_tokens(&input_ids)?;
3125
3126        // Per-row absolute position ids [q] for RoPE, and post-update KV lengths.
3127        let start_positions: Vec<usize> = sequences.iter().map(|&(_, sp, _)| sp).collect();
3128        let position_ids: Vec<Tensor> = start_positions
3129            .iter()
3130            .map(|&sp| {
3131                Tensor::arange_start(sp as i64, (sp + q_len) as i64, (tch::Kind::Int64, device))
3132            })
3133            .collect();
3134        // After this step each row holds start_pos + q_len cached tokens.
3135        let kv_lens: Vec<i64> = start_positions
3136            .iter()
3137            .map(|&sp| (sp + q_len) as i64)
3138            .collect();
3139        let kv_max = kv_lens.iter().copied().max().unwrap_or(0);
3140
3141        // Additive attention mask [B, 1, q, kv_max]: row r, query i (absolute pos
3142        // sp_r + i) attends to kv col j iff j <= sp_r + i AND j < kv_len_r. Padding
3143        // columns (j >= kv_len_r) and future columns are set to a large negative.
3144        let neg = -1.0e9_f64;
3145        let mask = Tensor::zeros(
3146            [batch_size as i64, 1, q_len as i64, kv_max],
3147            (tch::Kind::Float, device),
3148        );
3149        {
3150            // Column index row vector [kv_max].
3151            let cols = Tensor::arange(kv_max, (tch::Kind::Int64, device));
3152            for r in 0..batch_size {
3153                let sp = start_positions[r] as i64;
3154                let kv_len = kv_lens[r];
3155                for i in 0..q_len as i64 {
3156                    let allowed_upto = sp + i; // inclusive causal bound
3157                                               // valid = (cols <= allowed_upto) & (cols < kv_len)
3158                    let valid = cols.le(allowed_upto).logical_and(&cols.lt(kv_len));
3159                    let row_mask = valid.logical_not().to_kind(tch::Kind::Float) * neg; // 0 where valid, neg where invalid
3160                    mask.select(0, r as i64)
3161                        .select(0, 0)
3162                        .select(0, i)
3163                        .copy_(&row_mask);
3164                }
3165            }
3166        }
3167
3168        // Lock all per-sequence cache managers for the whole layer loop (one lock
3169        // each, mirroring the single-sequence path's lock-once strategy).
3170        let guards: Vec<_> = sequences.iter().map(|(_, _, c)| c.lock()).collect();
3171
3172        let mut position_refs: Vec<Tensor> = position_ids; // already per-row [q]
3173        let mut mask = mask;
3174
3175        for (idx, layer) in self.layers.iter().enumerate() {
3176            // Continuous batching traverses the same mapped layer range as the
3177            // single-sequence cached path. Move every attention input exactly
3178            // once at a real device change.
3179            let global_idx = self.layer_offset + idx;
3180            let target = self.device_map.device_for(global_idx);
3181            if hidden_states.device() != target {
3182                tracing::debug!(
3183                    layer = global_idx,
3184                    from = ?hidden_states.device(),
3185                    to = ?target,
3186                    "Llama batched forward crossing device boundary"
3187                );
3188                hidden_states = hidden_states.to_device(target);
3189                for positions in &mut position_refs {
3190                    *positions = positions.to_device(target);
3191                }
3192                mask = mask.to_device(target);
3193            }
3194
3195            let residual = hidden_states.shallow_clone();
3196            hidden_states = layer.input_layernorm.forward(&hidden_states)?;
3197
3198            // Collect this layer's per-row LayerKVCache mutable refs. DashMap's
3199            // get_mut borrows the manager, so we briefly take ownership of each
3200            // layer cache via with_layer_cache-equivalent direct access.
3201            let attn_output = {
3202                let mut layer_caches: Vec<
3203                    dashmap::mapref::one::RefMut<usize, crate::runtime::kv_cache::LayerKVCache>,
3204                > = Vec::with_capacity(batch_size);
3205                for g in guards.iter() {
3206                    let cell = g
3207                        .layer_cache_ref(idx)
3208                        .ok_or_else(|| anyhow!("forward_batched: missing layer cache {idx}"))?;
3209                    layer_caches.push(cell);
3210                }
3211                let mut cache_refs: Vec<&mut crate::runtime::kv_cache::LayerKVCache> =
3212                    layer_caches.iter_mut().map(|c| &mut **c).collect();
3213                layer.self_attn.forward_batched(
3214                    &hidden_states,
3215                    &position_refs,
3216                    &mut cache_refs,
3217                    &start_positions,
3218                    &mask,
3219                    delta,
3220                )?
3221            };
3222            hidden_states = residual + attn_output;
3223
3224            let residual = hidden_states.shallow_clone();
3225            hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
3226            let ffn_output = layer.mlp.forward(&hidden_states, delta)?;
3227            hidden_states = residual + ffn_output;
3228        }
3229        drop(guards);
3230
3231        if hidden_states.device() != self.device {
3232            tracing::debug!(
3233                from = ?hidden_states.device(),
3234                to = ?self.device,
3235                "Llama batched forward returning activation to output device"
3236            );
3237            hidden_states = hidden_states.to_device(self.device);
3238        }
3239
3240        hidden_states = self.apply_final_norm(&hidden_states)?;
3241        let logits = self.lm_head(&hidden_states)?;
3242        self.mask_padded_vocab(&logits);
3243        Ok(logits)
3244    }
3245
3246    /// Run global decoder layers `[range.start..range.end)` — the 2b pipeline
3247    /// layer-range runner (#314). See the trait docs for the stage contract.
3248    ///
3249    /// Layers are remapped to local `self.layers` indices via `layer_offset`; the
3250    /// per-layer KV cache is indexed locally (sized to the owned count). The lone
3251    /// cross-device copy is `hidden.to_device(next)`, inserted only when the next
3252    /// layer's mapped device differs from where `hidden` currently lives — never
3253    /// within a stage, never when source == dest (the unsplit map is a no-op).
3254    fn forward_layers_inner(
3255        &self,
3256        hidden: &Tensor,
3257        range: std::ops::Range<usize>,
3258        start_pos: usize,
3259        delta: Option<&crate::training::TenantDelta>,
3260    ) -> Result<Tensor> {
3261        let num_global = self.config.num_hidden_layers as usize;
3262        if range.end > num_global || range.start >= range.end {
3263            return Err(anyhow!(
3264                "forward_layers: invalid global range {range:?} for {num_global} layers"
3265            ));
3266        }
3267        // The owned window must contain the requested range.
3268        let owned = self.layer_offset..self.layer_offset + self.layers.len();
3269        if range.start < owned.start || range.end > owned.end {
3270            return Err(anyhow!(
3271                "forward_layers: range {range:?} not within this stage's owned layers {owned:?}"
3272            ));
3273        }
3274
3275        let mut hidden_states = hidden.shallow_clone();
3276
3277        // position_ids recomputed from start_pos + seq (never carried across a
3278        // boundary). Built on the *input* device (the first owned layer's), so a
3279        // moved first layer still matches at SDPA.
3280        let seq_len = hidden_states.size()[1];
3281        let position_ids = if start_pos == 0 {
3282            Tensor::arange(seq_len, (tch::Kind::Int64, hidden_states.device()))
3283        } else {
3284            Tensor::arange_start(
3285                start_pos as i64,
3286                (start_pos + seq_len as usize) as i64,
3287                (tch::Kind::Int64, hidden_states.device()),
3288            )
3289        };
3290        // position_ids must follow `hidden` across device boundaries; track it.
3291        let mut position_ids = position_ids;
3292
3293        let cache_guard = self.kv_cache.as_ref().map(|cache_ref| cache_ref.lock());
3294
3295        for g in range {
3296            let local_idx = g - self.layer_offset;
3297            let layer = &self.layers[local_idx];
3298
3299            // The single cross-device transfer: only when this layer's device
3300            // differs from where `hidden` currently is (zero-copy otherwise).
3301            let target = self.device_map.device_for(g);
3302            if hidden_states.device() != target {
3303                hidden_states = hidden_states.to_device(target);
3304                position_ids = position_ids.to_device(target);
3305            }
3306
3307            let residual = hidden_states.shallow_clone();
3308            hidden_states = layer.input_layernorm.forward(&hidden_states)?;
3309
3310            let attn_output = if let Some(ref cache_manager) = cache_guard {
3311                if let Some(result) = cache_manager.with_layer_cache(local_idx, |layer_cache| {
3312                    layer.self_attn.forward(
3313                        &hidden_states,
3314                        Some(&position_ids),
3315                        Some(layer_cache),
3316                        start_pos,
3317                        delta,
3318                    )
3319                }) {
3320                    result?
3321                } else {
3322                    layer.self_attn.forward(
3323                        &hidden_states,
3324                        Some(&position_ids),
3325                        None,
3326                        start_pos,
3327                        delta,
3328                    )?
3329                }
3330            } else {
3331                layer.self_attn.forward(
3332                    &hidden_states,
3333                    Some(&position_ids),
3334                    None,
3335                    start_pos,
3336                    delta,
3337                )?
3338            };
3339            hidden_states = residual + attn_output;
3340
3341            let residual = hidden_states.shallow_clone();
3342            hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
3343            let ffn_output = layer.mlp.forward(&hidden_states, delta)?;
3344            hidden_states = residual + ffn_output;
3345        }
3346
3347        Ok(hidden_states)
3348    }
3349
3350    /// Training-path sibling of [`Self::forward_layers_inner`] — the cross-device
3351    /// autograd primitive for TTT-on-split (#316). See the trait
3352    /// `forward_layers_train` docs for the contract.
3353    ///
3354    /// Identical layer loop to the inference runner with three deliberate
3355    /// differences: **no KV cache** (full causal attention, computed fresh from
3356    /// the whole context), **`start_pos = 0`** (`position_ids = 0..seq`), and the
3357    /// autograd graph is left intact (no `no_grad` guard — the caller wraps this
3358    /// in `with_grad`). The lone cross-device `hidden.to_device(next)` carries the
3359    /// autograd graph across the boundary, so `loss.backward()` materializes grads
3360    /// on each parameter's own device.
3361    fn forward_layers_train_inner(
3362        &self,
3363        hidden: &Tensor,
3364        range: std::ops::Range<usize>,
3365        delta: Option<&crate::training::TenantDelta>,
3366    ) -> Result<Tensor> {
3367        let num_global = self.config.num_hidden_layers as usize;
3368        if range.end > num_global || range.start >= range.end {
3369            return Err(anyhow!(
3370                "forward_layers_train: invalid global range {range:?} for {num_global} layers"
3371            ));
3372        }
3373        let owned = self.layer_offset..self.layer_offset + self.layers.len();
3374        if range.start < owned.start || range.end > owned.end {
3375            return Err(anyhow!(
3376                "forward_layers_train: range {range:?} not within this stage's owned layers {owned:?}"
3377            ));
3378        }
3379
3380        let mut hidden_states = hidden.shallow_clone();
3381
3382        // Training path: start_pos is always 0 → position_ids = 0..seq, built on
3383        // the input device and moved alongside `hidden` at each device boundary.
3384        let seq_len = hidden_states.size()[1];
3385        let mut position_ids = Tensor::arange(seq_len, (tch::Kind::Int64, hidden_states.device()));
3386
3387        // No KV cache in the training path: full causal attention is recomputed
3388        // over the entire context every step (compute_attention applies the tril
3389        // mask when no cache is present).
3390        for g in range {
3391            let local_idx = g - self.layer_offset;
3392            let layer = &self.layers[local_idx];
3393
3394            // The single cross-device transfer (autograd-transparent): only when
3395            // this layer's mapped device differs from where `hidden` currently is.
3396            let target = self.device_map.device_for(g);
3397            if hidden_states.device() != target {
3398                hidden_states = hidden_states.to_device(target);
3399                position_ids = position_ids.to_device(target);
3400            }
3401
3402            let residual = hidden_states.shallow_clone();
3403            hidden_states = layer.input_layernorm.forward(&hidden_states)?;
3404            let attn_output = layer.self_attn.forward(
3405                &hidden_states,
3406                Some(&position_ids),
3407                None, // no KV cache (training)
3408                0,    // start_pos = 0 (training)
3409                delta,
3410            )?;
3411            hidden_states = residual + attn_output;
3412
3413            let residual = hidden_states.shallow_clone();
3414            hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
3415            let ffn_output = layer.mlp.forward(&hidden_states, delta)?;
3416            hidden_states = residual + ffn_output;
3417        }
3418
3419        Ok(hidden_states)
3420    }
3421}
3422
3423impl ModelOperations for LlamaModel {
3424    fn architecture(&self) -> ModelArchitecture {
3425        // Report the *real* family (Qwen/Mistral/...), not the Llama stand-in,
3426        // so architecture-keyed dispatch (notably tokenizer-config padding for
3427        // Qwen, #143) targets the correct implementation. Falls back to Llama
3428        // with the parsed version when no override was set.
3429        match &self.config.model_architecture {
3430            ModelArchitecture::Llama { .. } | ModelArchitecture::Custom(_) => {
3431                ModelArchitecture::Llama {
3432                    version: self.config.version,
3433                }
3434            }
3435            other => other.clone(),
3436        }
3437    }
3438
3439    fn config(&self) -> &dyn ArchitectureConfig {
3440        &self.config
3441    }
3442
3443    fn forward(&self, input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor> {
3444        // Default forward just calls forward_with_cache with start_pos = 0
3445        self.forward_with_cache(input, 0)
3446    }
3447
3448    fn var_store(&self) -> Option<&nn::VarStore> {
3449        self.vs.as_ref()
3450    }
3451
3452    fn var_store_mut(&mut self) -> Option<&mut nn::VarStore> {
3453        self.vs.as_mut()
3454    }
3455
3456    fn forward_with_cache(&self, input: &Tensor, start_pos: usize) -> Result<Tensor> {
3457        self.forward_with_cache_inner(input, start_pos, None)
3458    }
3459
3460    fn forward_with_cache_and_delta(
3461        &self,
3462        input: &Tensor,
3463        start_pos: usize,
3464        delta: Option<&crate::training::TenantDelta>,
3465    ) -> Result<Tensor> {
3466        self.forward_with_cache_inner(input, start_pos, delta)
3467    }
3468
3469    fn forward_batched(
3470        &self,
3471        sequences: &mut [(
3472            Vec<i64>,
3473            usize,
3474            std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
3475        )],
3476        delta: Option<&crate::training::TenantDelta>,
3477    ) -> Result<Tensor> {
3478        self.forward_batched_impl(sequences, delta)
3479    }
3480
3481    fn embed_tokens(&self, input_ids: &Tensor) -> Result<Tensor> {
3482        if let Some(embed) = &self.embed_tokens {
3483            // Get input shape
3484            let input_shape = input_ids.size();
3485            let batch_size = input_shape[0];
3486            let seq_len = if input_shape.len() > 1 {
3487                input_shape[1]
3488            } else {
3489                1
3490            };
3491
3492            // Flatten input for embedding lookup
3493            let flat_input = input_ids.flatten(0, -1);
3494
3495            // Perform embedding lookup
3496            let embeddings = embed.index_select(0, &flat_input);
3497
3498            // Get hidden size from embedding result
3499            let emb_dims = embeddings.size();
3500            let hidden_size = emb_dims[emb_dims.len() - 1];
3501
3502            // Reshape to [batch_size, seq_len, hidden_size]
3503            let mut embeddings = embeddings.reshape([batch_size, seq_len, hidden_size]);
3504
3505            // Scale embeddings if needed (Gemma3)
3506            if self.config.scale_embeddings {
3507                let scale = (hidden_size as f32).sqrt();
3508                let scale_tensor = Tensor::from_slice(&[scale])
3509                    .to_kind(embeddings.kind())
3510                    .to_device(embeddings.device());
3511                embeddings = broadcast_mul(&embeddings, &scale_tensor)?;
3512            }
3513
3514            Ok(embeddings)
3515        } else {
3516            Err(anyhow!("No embedding layer available in model"))
3517        }
3518    }
3519
3520    fn forward_from_embeddings(&self, embeddings: &Tensor, start_pos: usize) -> Result<Tensor> {
3521        tracing::trace!(
3522            "LlamaModel forward_from_embeddings: embeddings shape={:?}, start_pos={}",
3523            embeddings.size(),
3524            start_pos
3525        );
3526
3527        // Start with pre-computed embeddings instead of embedding token IDs
3528        let mut hidden_states = embeddings.shallow_clone();
3529
3530        // Apply transformer layers (same as forward_with_cache)
3531        let seq_len = hidden_states.size()[1];
3532        let position_ids = if start_pos == 0 {
3533            Tensor::arange(seq_len, (tch::Kind::Int64, hidden_states.device()))
3534        } else {
3535            Tensor::arange_start(
3536                start_pos as i64,
3537                (start_pos + seq_len as usize) as i64,
3538                (tch::Kind::Int64, hidden_states.device()),
3539            )
3540        };
3541
3542        for (idx, layer) in self.layers.iter().enumerate() {
3543            let residual = hidden_states.shallow_clone();
3544
3545            // Self-attention block with optional KV cache
3546            hidden_states = layer.input_layernorm.forward(&hidden_states)?;
3547
3548            // Handle KV cache per layer with proper start_pos
3549            let attn_output = if let Some(cache_ref) = self.kv_cache.as_ref() {
3550                let cache_manager = cache_ref.lock();
3551                if let Some(result) = cache_manager.with_layer_cache(idx, |layer_cache| {
3552                    layer.self_attn.forward(
3553                        &hidden_states,
3554                        Some(&position_ids),
3555                        Some(layer_cache),
3556                        start_pos,
3557                        None,
3558                    )
3559                }) {
3560                    result?
3561                } else {
3562                    layer.self_attn.forward(
3563                        &hidden_states,
3564                        Some(&position_ids),
3565                        None,
3566                        start_pos,
3567                        None,
3568                    )?
3569                }
3570            } else {
3571                layer.self_attn.forward(
3572                    &hidden_states,
3573                    Some(&position_ids),
3574                    None,
3575                    start_pos,
3576                    None,
3577                )?
3578            };
3579            hidden_states = residual + attn_output;
3580
3581            // FFN block
3582            let residual = hidden_states.shallow_clone();
3583            hidden_states = layer.post_attention_layernorm.forward(&hidden_states)?;
3584            let ffn_output = layer.mlp.forward(&hidden_states, None)?;
3585            hidden_states = residual + ffn_output;
3586        }
3587
3588        // Final layer norm
3589        if let Some(norm) = &self.norm {
3590            hidden_states = norm.forward(&hidden_states)?;
3591        }
3592
3593        // LM head (same logic as forward_with_cache)
3594        if let Some(lm_head) = &self.lm_head {
3595            hidden_states = hidden_states.matmul(lm_head);
3596        } else if let Some(output_proj) = &self.lm_head_transposed {
3597            // Use pre-computed transposed embedding (tied weights)
3598            let hs_shape = hidden_states.size();
3599            if hs_shape.len() == 3 {
3600                let (batch_size, seq_len, hidden_size) = (hs_shape[0], hs_shape[1], hs_shape[2]);
3601                let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
3602                let logits_2d = hidden_2d.matmul(output_proj);
3603                hidden_states = logits_2d.reshape([batch_size, seq_len, -1]);
3604            } else {
3605                hidden_states = hidden_states.matmul(output_proj);
3606            }
3607        }
3608
3609        // Mask padded tokens (same as forward_with_cache)
3610        let padded_vocab_size = self.config.vocab_size;
3611        let original_vocab_size = self.config.original_vocab_size;
3612        if padded_vocab_size > original_vocab_size && original_vocab_size > 0 {
3613            let logits_shape = hidden_states.size();
3614            let actual_vocab_size = logits_shape[logits_shape.len() - 1] as u32;
3615
3616            if actual_vocab_size == padded_vocab_size {
3617                let mask_start = original_vocab_size as i64;
3618                let mask_count = (padded_vocab_size - original_vocab_size) as i64;
3619
3620                if mask_count > 0 {
3621                    let mask_values = Tensor::full(
3622                        [mask_count],
3623                        -1e10_f64,
3624                        (hidden_states.kind(), hidden_states.device()),
3625                    );
3626
3627                    if logits_shape.len() == 3 {
3628                        let batch_size = logits_shape[0];
3629                        let seq_len = logits_shape[1];
3630                        for b in 0..batch_size {
3631                            for s in 0..seq_len {
3632                                let slice = hidden_states.select(0, b).select(0, s);
3633                                slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3634                            }
3635                        }
3636                    } else if logits_shape.len() == 2 {
3637                        let rows = logits_shape[0];
3638                        for i in 0..rows {
3639                            let slice = hidden_states.select(0, i);
3640                            slice.narrow(0, mask_start, mask_count).copy_(&mask_values);
3641                        }
3642                    } else {
3643                        hidden_states
3644                            .narrow(0, mask_start, mask_count)
3645                            .copy_(&mask_values);
3646                    }
3647                }
3648            }
3649        }
3650
3651        Ok(hidden_states)
3652    }
3653
3654    fn reshape_for_attention(&self, tensor: &Tensor, is_key_value: bool) -> Result<Tensor> {
3655        let (batch_size, seq_len, _hidden_size) = dims3(tensor)?;
3656
3657        if is_key_value {
3658            // For K,V: reshape to [batch, seq, num_kv_heads, head_dim]
3659            Ok(tensor.reshape([
3660                batch_size,
3661                seq_len,
3662                self.config.num_key_value_heads as i64,
3663                self.config.head_dim as i64,
3664            ]))
3665        } else {
3666            // For Q: reshape to [batch, seq, num_attention_heads, head_dim]
3667            Ok(tensor.reshape([
3668                batch_size,
3669                seq_len,
3670                self.config.num_attention_heads as i64,
3671                self.config.head_dim as i64,
3672            ]))
3673        }
3674    }
3675
3676    fn decode_layer(
3677        &self,
3678        layer_idx: usize,
3679        hidden_states: &Tensor,
3680        _attention_mask: Option<&Tensor>,
3681        position_ids: Option<&Tensor>,
3682        _past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
3683    ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
3684        let layer = self.layers.get(layer_idx).ok_or_else(|| {
3685            anyhow!(
3686                "Layer index {} out of range (model has {} layers)",
3687                layer_idx,
3688                self.layers.len()
3689            )
3690        })?;
3691
3692        let residual = hidden_states.shallow_clone();
3693
3694        // Self-attention block
3695        let normed = layer.input_layernorm.forward(hidden_states)?;
3696
3697        // Handle KV cache per layer
3698        let attn_output = if let Some(cache_ref) = self.kv_cache.as_ref() {
3699            let cache_manager = cache_ref.lock();
3700            if let Some(result) = cache_manager.with_layer_cache(layer_idx, |layer_cache| {
3701                layer.self_attn.forward(
3702                    &normed,
3703                    position_ids,
3704                    Some(layer_cache),
3705                    0, // start_pos handled by cache
3706                    None,
3707                )
3708            }) {
3709                result?
3710            } else {
3711                layer
3712                    .self_attn
3713                    .forward(&normed, position_ids, None, 0, None)?
3714            }
3715        } else {
3716            layer
3717                .self_attn
3718                .forward(&normed, position_ids, None, 0, None)?
3719        };
3720
3721        let hidden_states = residual + attn_output;
3722
3723        // FFN block
3724        let residual = hidden_states.shallow_clone();
3725        let normed = layer.post_attention_layernorm.forward(&hidden_states)?;
3726        let ffn_output = layer.mlp.forward(&normed, None)?;
3727        let hidden_states = residual + ffn_output;
3728
3729        // No separate KV return — cache is managed internally by LlamaAttention
3730        Ok((hidden_states, None))
3731    }
3732
3733    fn decode_layer_with_delta(
3734        &self,
3735        layer_idx: usize,
3736        hidden_states: &Tensor,
3737        _attention_mask: Option<&Tensor>,
3738        position_ids: Option<&Tensor>,
3739        _past_kv: Option<&crate::runtime::kv_cache::LayerKVCache>,
3740        delta: &crate::training::TenantDelta,
3741    ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
3742        let layer = self.layers.get(layer_idx).ok_or_else(|| {
3743            anyhow!(
3744                "Layer index {} out of range (model has {} layers)",
3745                layer_idx,
3746                self.layers.len()
3747            )
3748        })?;
3749
3750        let residual = hidden_states.shallow_clone();
3751
3752        // Self-attention block with delta injection (no KV cache on training path)
3753        let normed = layer.input_layernorm.forward(hidden_states)?;
3754        let attn_output = layer
3755            .self_attn
3756            .forward(&normed, position_ids, None, 0, Some(delta))?;
3757        let hidden_states = residual + attn_output;
3758
3759        // FFN block with delta injection for MLP projections
3760        let residual = hidden_states.shallow_clone();
3761        let normed = layer.post_attention_layernorm.forward(&hidden_states)?;
3762        let ffn_output = layer.mlp.forward(&normed, Some(delta))?;
3763        let hidden_states = residual + ffn_output;
3764
3765        Ok((hidden_states, None))
3766    }
3767
3768    fn forward_layers(
3769        &self,
3770        hidden: &Tensor,
3771        range: std::ops::Range<usize>,
3772        start_pos: usize,
3773        delta: Option<&crate::training::TenantDelta>,
3774    ) -> Result<Tensor> {
3775        self.forward_layers_inner(hidden, range, start_pos, delta)
3776    }
3777
3778    fn forward_layers_train(
3779        &self,
3780        hidden: &Tensor,
3781        range: std::ops::Range<usize>,
3782        delta: Option<&crate::training::TenantDelta>,
3783    ) -> Result<Tensor> {
3784        self.forward_layers_train_inner(hidden, range, delta)
3785    }
3786
3787    fn apply_final_norm(&self, hidden_states: &Tensor) -> Result<Tensor> {
3788        if let Some(norm) = &self.norm {
3789            norm.forward(hidden_states)
3790        } else {
3791            Ok(hidden_states.shallow_clone())
3792        }
3793    }
3794
3795    fn lm_head(&self, hidden_states: &Tensor) -> Result<Tensor> {
3796        if let Some(lm_head) = &self.lm_head {
3797            // Cast weight UP to match hidden_states dtype instead of casting hidden_states DOWN.
3798            // During training, hidden_states may be f32 for gradient precision — casting it
3799            // down to bf16 destroys gradients. Casting the (frozen) weight up is a no-op
3800            // during inference where both are already bf16.
3801            let lm_head = lm_head.to_kind(hidden_states.kind());
3802            Ok(hidden_states
3803                .f_matmul(&lm_head)
3804                .map_err(|e| anyhow!("lm_head matmul failed: {}", e))?)
3805        } else if let Some(output_proj) = &self.lm_head_transposed {
3806            let output_proj = output_proj.to_kind(hidden_states.kind());
3807            let hs_shape = hidden_states.size();
3808            if hs_shape.len() == 3 {
3809                let (batch_size, seq_len, hidden_size) = (hs_shape[0], hs_shape[1], hs_shape[2]);
3810                let hidden_2d = hidden_states.reshape([batch_size * seq_len, hidden_size]);
3811                let logits_2d = hidden_2d
3812                    .f_matmul(&output_proj)
3813                    .map_err(|e| anyhow!("lm_head transposed matmul failed: {}", e))?;
3814                Ok(logits_2d.reshape([batch_size, seq_len, -1]))
3815            } else {
3816                Ok(hidden_states
3817                    .f_matmul(&output_proj)
3818                    .map_err(|e| anyhow!("lm_head transposed matmul failed: {}", e))?)
3819            }
3820        } else {
3821            Err(anyhow!("No LM head or embedding weights found"))
3822        }
3823    }
3824
3825    fn num_layers(&self) -> usize {
3826        self.config.num_hidden_layers as usize
3827    }
3828
3829    fn apply_rope(&self, _tensor: &Tensor, _position_ids: &Tensor) -> Result<Tensor> {
3830        // RoPE is applied in the attention layers, not at the model level
3831        // This method should not be called
3832        Err(anyhow!(
3833            "RoPE should be applied in attention layers, not at model level"
3834        ))
3835    }
3836
3837    fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
3838        // Llama uses RMSNorm, preserving dtype
3839        let original_dtype = tensor.kind();
3840        let x2 = square_tensor(tensor)?;
3841        let mean = x2.mean_dim(&[-1i64][..], true, original_dtype);
3842        let rrms = (mean + self.config.rms_norm_eps as f64).reciprocal().sqrt();
3843        broadcast_mul(tensor, &rrms)
3844    }
3845
3846    fn get_attention_mask(&self, seq_len: usize, past_kv_len: usize) -> Result<Tensor> {
3847        // Create causal attention mask using model dtype
3848        let total_len = seq_len + past_kv_len;
3849        let mask = Tensor::ones(
3850            [seq_len as i64, total_len as i64],
3851            (self.dtype, self.device),
3852        );
3853
3854        // Apply causal masking (simplified)
3855        Ok(mask)
3856    }
3857
3858    fn as_any(&self) -> &dyn std::any::Any {
3859        self
3860    }
3861
3862    // ============================================================================
3863    // KV Cache Management (trait implementation)
3864    // ============================================================================
3865
3866    fn clear_kv_cache(&self) {
3867        if let Some(cache_ref) = self.kv_cache.as_ref() {
3868            let mut cache_manager = cache_ref.lock();
3869            cache_manager.clear_all();
3870        }
3871    }
3872
3873    fn kv_cache_memory_usage(&self) -> usize {
3874        self.kv_cache
3875            .as_ref()
3876            .map(|cache_ref| cache_ref.lock().memory_usage())
3877            .unwrap_or(0)
3878    }
3879
3880    fn set_kv_cache(
3881        &mut self,
3882        cache: std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
3883    ) {
3884        self.kv_cache = Some(cache);
3885    }
3886
3887    fn get_kv_cache(
3888        &self,
3889    ) -> Option<std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>> {
3890        self.kv_cache.clone()
3891    }
3892
3893    fn take_kv_cache(
3894        &mut self,
3895    ) -> Option<std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>> {
3896        self.kv_cache.take()
3897    }
3898}
3899
3900// KV cache methods are now defined in the ModelOperations trait implementation above
3901
3902#[cfg(test)]
3903#[allow(clippy::unwrap_used, clippy::expect_used)]
3904mod pipeline_tests {
3905    use super::*;
3906    use crate::runtime::device_pool::LayerDeviceMap;
3907    use hyprstream_rpc_std::model_client::KVQuantType;
3908
3909    const HIDDEN: i64 = 16;
3910    const HEADS: i64 = 2;
3911    const HEAD_DIM: i64 = 8; // HEADS * HEAD_DIM == HIDDEN
3912    const INTER: i64 = 32;
3913    const VOCAB: i64 = 48; // multiple of 64? no → padded to 64; both paths pad identically
3914    const LAYERS: usize = 4;
3915
3916    fn tiny_config() -> LlamaConfig {
3917        LlamaConfig {
3918            version: 2,
3919            num_attention_heads: HEADS as u32,
3920            num_key_value_heads: HEADS as u32,
3921            hidden_size: HIDDEN as u32,
3922            head_dim: HEAD_DIM as u32,
3923            intermediate_size: INTER as u32,
3924            max_position_embeddings: 64,
3925            rms_norm_eps: 1e-5,
3926            vocab_size: VOCAB as u32,
3927            original_vocab_size: VOCAB as u32,
3928            num_hidden_layers: LAYERS as u32,
3929            rope_theta: 10000.0,
3930            rope_scaling: None,
3931            hidden_activation: "silu".to_owned(),
3932            query_pre_attn_scalar: None,
3933            use_qk_norm: false,
3934            scale_embeddings: false,
3935            layer_types: vec![],
3936            rope_local_base_freq: None,
3937            model_architecture: ModelArchitecture::Llama { version: 2 },
3938        }
3939    }
3940
3941    /// Deterministic small weight set on CPU/f32 so the two construction paths are
3942    /// numerically comparable. HF stores projections as `[out, in]`.
3943    ///
3944    /// Uses a fixed, RNG-free pattern (a bounded sin of the flat index) rather
3945    /// than `Tensor::randn`: tests run in parallel and `manual_seed` is global, so
3946    /// two `randn`-based builds could interleave and diverge. This makes every
3947    /// call byte-identical regardless of scheduling.
3948    fn tiny_weights() -> HashMap<String, Tensor> {
3949        let opt = (DType::Float, Device::Cpu);
3950        let mut w = HashMap::new();
3951        let randn = |dims: &[i64]| {
3952            let n: i64 = dims.iter().product();
3953            // Deterministic small values in roughly [-0.05, 0.05].
3954            (Tensor::arange(n, opt) * 0.017).sin().reshape(dims) * 0.05
3955        };
3956
3957        w.insert(
3958            "model.embed_tokens.weight".to_owned(),
3959            randn(&[VOCAB, HIDDEN]),
3960        );
3961        w.insert("model.norm.weight".to_owned(), Tensor::ones([HIDDEN], opt));
3962        w.insert("lm_head.weight".to_owned(), randn(&[VOCAB, HIDDEN]));
3963
3964        for i in 0..LAYERS {
3965            let p = format!("model.layers.{i}");
3966            // attention projections: q/k/v are [heads*head_dim, hidden]; o is [hidden, heads*head_dim]
3967            w.insert(
3968                format!("{p}.self_attn.q_proj.weight"),
3969                randn(&[HEADS * HEAD_DIM, HIDDEN]),
3970            );
3971            w.insert(
3972                format!("{p}.self_attn.k_proj.weight"),
3973                randn(&[HEADS * HEAD_DIM, HIDDEN]),
3974            );
3975            w.insert(
3976                format!("{p}.self_attn.v_proj.weight"),
3977                randn(&[HEADS * HEAD_DIM, HIDDEN]),
3978            );
3979            w.insert(
3980                format!("{p}.self_attn.o_proj.weight"),
3981                randn(&[HIDDEN, HEADS * HEAD_DIM]),
3982            );
3983            // mlp
3984            w.insert(format!("{p}.mlp.gate_proj.weight"), randn(&[INTER, HIDDEN]));
3985            w.insert(format!("{p}.mlp.up_proj.weight"), randn(&[INTER, HIDDEN]));
3986            w.insert(format!("{p}.mlp.down_proj.weight"), randn(&[HIDDEN, INTER]));
3987            // norms
3988            w.insert(
3989                format!("{p}.input_layernorm.weight"),
3990                Tensor::ones([HIDDEN], opt),
3991            );
3992            w.insert(
3993                format!("{p}.post_attention_layernorm.weight"),
3994                Tensor::ones([HIDDEN], opt),
3995            );
3996        }
3997        w
3998    }
3999
4000    fn whole_model() -> LlamaModel {
4001        let mut w = tiny_weights();
4002        LlamaModel::from_weights_with_config(
4003            &mut w,
4004            tiny_config(),
4005            &Device::Cpu,
4006            DType::Float,
4007            KVQuantType::None,
4008        )
4009        .unwrap()
4010    }
4011
4012    /// Orchestrate the pipeline path on a model the way an engine would:
4013    /// embed → forward_layers(0..N) → final norm → lm_head.
4014    fn orchestrated_logits(m: &LlamaModel, input: &Tensor) -> Tensor {
4015        let emb = m.embed_tokens(input).unwrap();
4016        let h = m.forward_layers(&emb, 0..m.num_layers(), 0, None).unwrap();
4017        let h = m.apply_final_norm(&h).unwrap();
4018        m.lm_head(&h).unwrap()
4019    }
4020
4021    #[test]
4022    fn forward_layers_full_range_matches_whole_model_forward() {
4023        // An all-CPU LayerDeviceMap means forward_layers(0..N) must equal the
4024        // whole-model single-device forward (the byte-identity equivalence test).
4025        let input = Tensor::from_slice(&[1i64, 5, 9, 2]).reshape([1, 4]);
4026
4027        // Reference: whole-model forward (embed→loop→norm→lm_head w/ vocab mask).
4028        let reference = whole_model();
4029        let ref_logits = reference.forward_with_cache(&input, 0).unwrap();
4030
4031        // Pipeline path on a fresh model (independent KV), full range over a
4032        // single-device map. Compare on the ORIGINAL vocab columns (the whole-
4033        // model path additionally masks padded columns; lm_head() does not).
4034        let piped = whole_model();
4035        let pipe_logits = orchestrated_logits(&piped, &input);
4036
4037        let orig = reference.config.original_vocab_size as i64;
4038        let ref_c = ref_logits.narrow(2, 0, orig);
4039        let pipe_c = pipe_logits.narrow(2, 0, orig);
4040        let max_diff = (&ref_c - &pipe_c).abs().max().double_value(&[]);
4041        assert!(
4042            ref_c.allclose(&pipe_c, 1e-4, 1e-4, false),
4043            "forward_layers(0..N) over a single-device map must equal whole-model forward \
4044             (max_diff={max_diff}, ref_shape={:?})",
4045            ref_c.size()
4046        );
4047        // Sanity: the device map for a whole model is single-device (zero-copy).
4048        assert!(piped.device_map.is_single_device());
4049        assert_eq!(piped.layer_offset, 0);
4050    }
4051
4052    #[test]
4053    fn staged_split_equals_whole_model() {
4054        // Build TWO stages over an all-CPU 2-way split and run them as a pipeline:
4055        // stage0: embed → forward_layers(0..b); stage1: forward_layers(b..N) →
4056        // norm → lm_head. Result must equal the whole-model forward. This
4057        // exercises the global↔local remap, is_first/is_last gating, and the
4058        // (zero, since same-device) boundary copy.
4059        let input = Tensor::from_slice(&[3i64, 7, 1, 4]).reshape([1, 4]);
4060
4061        let reference = whole_model();
4062        let ref_logits = reference.forward_with_cache(&input, 0).unwrap();
4063
4064        // Single CPU device but split the layer RANGE across two stages.
4065        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4066        let split = LAYERS / 2;
4067
4068        let mut w0 = tiny_weights();
4069        let stage0 = LlamaModel::stage_from_weights_with_config(
4070            &mut w0,
4071            tiny_config(),
4072            &map,
4073            0..split,
4074            DType::Float,
4075            KVQuantType::None,
4076        )
4077        .unwrap();
4078        assert_eq!(stage0.layer_offset, 0);
4079        assert!(stage0.embed_tokens.is_some(), "first stage keeps embed");
4080        assert!(
4081            stage0.norm.is_none() && stage0.lm_head.is_none(),
4082            "first stage has no head"
4083        );
4084
4085        let mut w1 = tiny_weights();
4086        let stage1 = LlamaModel::stage_from_weights_with_config(
4087            &mut w1,
4088            tiny_config(),
4089            &map,
4090            split..LAYERS,
4091            DType::Float,
4092            KVQuantType::None,
4093        )
4094        .unwrap();
4095        assert_eq!(stage1.layer_offset, split);
4096        assert!(stage1.embed_tokens.is_none(), "last stage has no embed");
4097        assert!(stage1.norm.is_some(), "last stage keeps final norm");
4098
4099        // Drive the pipeline.
4100        let emb = stage0.embed_tokens(&input).unwrap();
4101        let h0 = stage0.forward_layers(&emb, 0..split, 0, None).unwrap();
4102        let h1 = stage1.forward_layers(&h0, split..LAYERS, 0, None).unwrap();
4103        let h1 = stage1.apply_final_norm(&h1).unwrap();
4104        let logits = stage1.lm_head(&h1).unwrap();
4105
4106        let orig = reference.config.original_vocab_size as i64;
4107        let ref_c = ref_logits.narrow(2, 0, orig);
4108        let pipe_c = logits.narrow(2, 0, orig);
4109        // 1e-4: the lm_head() trait casts the weight and uses f_matmul; the whole-
4110        // model path uses plain matmul — float reassociation, not a logic diff.
4111        let max_diff = (&ref_c - &pipe_c).abs().max().double_value(&[]);
4112        assert!(
4113            ref_c.allclose(&pipe_c, 1e-4, 1e-4, false),
4114            "two-stage split (same device) must equal whole-model forward (max_diff={max_diff})"
4115        );
4116        // The last stage built an explicit lm_head (not the tied transpose).
4117        assert!(stage1.lm_head_transposed.is_none());
4118    }
4119
4120    #[test]
4121    fn forward_layers_rejects_out_of_window_range() {
4122        // A stage that owns [2..4) must reject a range outside its window.
4123        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4124        let mut w = tiny_weights();
4125        let stage = LlamaModel::stage_from_weights_with_config(
4126            &mut w,
4127            tiny_config(),
4128            &map,
4129            2..LAYERS,
4130            DType::Float,
4131            KVQuantType::None,
4132        )
4133        .unwrap();
4134        let emb = Tensor::randn([1, 3, HIDDEN], (DType::Float, Device::Cpu));
4135        assert!(
4136            stage.forward_layers(&emb, 0..2, 0, None).is_err(),
4137            "range below window"
4138        );
4139        assert!(
4140            stage.forward_layers(&emb, 2..LAYERS, 0, None).is_ok(),
4141            "owned range ok"
4142        );
4143    }
4144
4145    #[test]
4146    fn boundary_copy_skipped_when_same_device() {
4147        // With a single-device map, no layer is a boundary, so forward_layers does
4148        // zero `to_device` calls. We can only assert the predicate directly here
4149        // (tch gives no copy counter), which the device-map test also covers.
4150        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4151        for g in 1..LAYERS {
4152            assert!(!map.is_boundary(Device::Cpu, g));
4153        }
4154    }
4155
4156    #[test]
4157    fn stage_loader_rejects_missing_required_weights() {
4158        // Last stage with no norm weight must error (M-LOAD seam #1 gating).
4159        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4160        let mut w = tiny_weights();
4161        w.remove("model.norm.weight");
4162        let result = LlamaModel::stage_from_weights_with_config(
4163            &mut w,
4164            tiny_config(),
4165            &map,
4166            0..LAYERS,
4167            DType::Float,
4168            KVQuantType::None,
4169        );
4170        match result {
4171            Ok(_) => panic!("last stage without norm weight must error"),
4172            Err(e) => assert!(e.to_string().contains("norm"), "got: {e}"),
4173        }
4174    }
4175
4176    #[test]
4177    fn parse_config_requires_core_dims() {
4178        // #315 follow-up: mandatory architectural dims, no silent magic numbers.
4179        let ok = r#"{"num_attention_heads":2,"hidden_size":16,"num_hidden_layers":4}"#;
4180        assert!(LlamaModel::parse_config(ok).is_ok());
4181        let missing = r#"{"hidden_size":16,"num_hidden_layers":4}"#;
4182        let err = LlamaModel::parse_config(missing).unwrap_err();
4183        assert!(
4184            err.to_string().contains("num_attention_heads"),
4185            "got: {err}"
4186        );
4187    }
4188
4189    // ========================================================================
4190    // #316 — TTT-on-split cross-device autograd equivalence (CPU-verifiable).
4191    // ========================================================================
4192
4193    use crate::training::tenant_delta::{TenantDelta, TenantDeltaConfig};
4194
4195    /// Build a per-layer q_proj/v_proj LoRA delta on CPU with a deterministic,
4196    /// RNG-free A **and** non-zero B so that gradients w.r.t. BOTH A and B are
4197    /// non-trivial (B is zero-initialized by default → dL/dA would be zero).
4198    fn tiny_delta() -> TenantDelta {
4199        let mut dims = std::collections::HashMap::new();
4200        dims.insert("q_proj".to_owned(), (HIDDEN as usize, HIDDEN as usize));
4201        dims.insert("v_proj".to_owned(), (HIDDEN as usize, HIDDEN as usize));
4202        let cfg = TenantDeltaConfig {
4203            rank: 2,
4204            alpha: 2.0,
4205            target_modules: vec!["q_proj".to_owned(), "v_proj".to_owned()],
4206            ..Default::default()
4207        };
4208        let delta = TenantDelta::new(&cfg, &dims, Device::Cpu, LAYERS).unwrap();
4209        // Seed deterministic, non-zero A and B in-place (no_grad: weight init, not
4210        // graph). Seed by a STABLE per-key offset — HashMap iteration order is not
4211        // deterministic, so two independent `tiny_delta()` builds (whole vs split)
4212        // must agree key-for-key, not enumeration-index-for-index.
4213        let key_offset = |key: &str| -> i64 { key.bytes().map(|b| b as i64).sum::<i64>() % 17 };
4214        let _g = tch::no_grad_guard();
4215        for (k, a) in delta.lora_a.iter() {
4216            let n: i64 = a.size().iter().product();
4217            let vals = (Tensor::arange(n, (DType::Float, Device::Cpu)) + key_offset(k)).sin() * 0.1;
4218            // copy_ needs &mut; shallow_clone shares storage so the delta param is mutated.
4219            a.shallow_clone().copy_(&vals.reshape(a.size()));
4220        }
4221        for (k, b) in delta.lora_b.iter() {
4222            let n: i64 = b.size().iter().product();
4223            let vals =
4224                (Tensor::arange(n, (DType::Float, Device::Cpu)) + key_offset(k) + 7).cos() * 0.1;
4225            b.shallow_clone().copy_(&vals.reshape(b.size()));
4226        }
4227        delta
4228    }
4229
4230    /// Sum of squared L2 grad norms over all delta params, plus a per-key map.
4231    fn grad_snapshot(delta: &TenantDelta) -> std::collections::HashMap<String, f64> {
4232        let mut out = std::collections::HashMap::new();
4233        for (k, a) in &delta.lora_a {
4234            assert!(a.grad().defined(), "A grad undefined for {k}");
4235            out.insert(format!("A:{k}"), a.grad().norm().double_value(&[]));
4236        }
4237        for (k, b) in &delta.lora_b {
4238            assert!(b.grad().defined(), "B grad undefined for {k}");
4239            out.insert(format!("B:{k}"), b.grad().norm().double_value(&[]));
4240        }
4241        out
4242    }
4243
4244    /// The core correctness guardrail (#316): a TTT/training step (forward_layers_train
4245    /// → NTP loss → backward) over a two-stage CPU split must produce the SAME delta
4246    /// gradients as the whole-model training forward, within ~1e-4. A wrong
4247    /// cross-device autograd hookup or mis-placed grad would diverge.
4248    #[test]
4249    fn ttt_split_autograd_matches_whole_model_grads() {
4250        use crate::training::pipeline::{compute_ntp_loss_split, TrainStage};
4251
4252        let input = Tensor::from_slice(&[3i64, 7, 1, 4, 9, 2]).reshape([1, 6]);
4253
4254        // --- (a) whole-model training forward over a single-device map ---
4255        let whole = whole_model();
4256        let whole_delta = tiny_delta();
4257        let whole_stage = [TrainStage {
4258            model: &whole,
4259            range: 0..LAYERS,
4260        }];
4261        let loss_whole = compute_ntp_loss_split(&whole_stage, &input, Some(&whole_delta)).unwrap();
4262        loss_whole.backward();
4263        let whole_grads = grad_snapshot(&whole_delta);
4264        whole_delta.zero_grad();
4265
4266        // --- (b) two-stage split over an all-CPU map (exercises the boundary
4267        // to_device autograd transparency + global↔local remap) ---
4268        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4269        let split = LAYERS / 2;
4270        let mut w0 = tiny_weights();
4271        let stage0 = LlamaModel::stage_from_weights_with_config(
4272            &mut w0,
4273            tiny_config(),
4274            &map,
4275            0..split,
4276            DType::Float,
4277            KVQuantType::None,
4278        )
4279        .unwrap();
4280        let mut w1 = tiny_weights();
4281        let stage1 = LlamaModel::stage_from_weights_with_config(
4282            &mut w1,
4283            tiny_config(),
4284            &map,
4285            split..LAYERS,
4286            DType::Float,
4287            KVQuantType::None,
4288        )
4289        .unwrap();
4290        let split_delta = tiny_delta();
4291        let stages = [
4292            TrainStage {
4293                model: &stage0,
4294                range: 0..split,
4295            },
4296            TrainStage {
4297                model: &stage1,
4298                range: split..LAYERS,
4299            },
4300        ];
4301        let loss_split = compute_ntp_loss_split(&stages, &input, Some(&split_delta)).unwrap();
4302        loss_split.backward();
4303        let split_grads = grad_snapshot(&split_delta);
4304
4305        // Losses must match (forward equivalence).
4306        let lw = loss_whole.double_value(&[]);
4307        let ls = loss_split.double_value(&[]);
4308        assert!(
4309            (lw - ls).abs() < 1e-4,
4310            "loss diverged: whole={lw} split={ls}"
4311        );
4312
4313        // Per-key gradient norms must match within tolerance (backward equivalence).
4314        assert_eq!(whole_grads.len(), split_grads.len());
4315        for (k, &gw) in &whole_grads {
4316            let gs = *split_grads
4317                .get(k)
4318                .unwrap_or_else(|| panic!("missing grad {k}"));
4319            assert!(
4320                (gw - gs).abs() <= 1e-4 + 1e-4 * gw.abs(),
4321                "grad norm diverged for {k}: whole={gw} split={gs}"
4322            );
4323            // A non-trivial gradient must actually flow (guards against a silently
4324            // detached path that would make both sides spuriously equal at 0).
4325            assert!(
4326                gw > 0.0,
4327                "grad for {k} is zero — autograd path not exercised"
4328            );
4329        }
4330    }
4331
4332    // ========================================================================
4333    // #329 PR-1 — batched ragged decode == serial decode (CPU merge gate).
4334    // ========================================================================
4335
4336    use crate::runtime::kv_cache::KVCacheManager;
4337    use parking_lot::Mutex;
4338    use std::sync::Arc;
4339
4340    fn fresh_cache() -> Arc<Mutex<KVCacheManager>> {
4341        Arc::new(Mutex::new(KVCacheManager::new(
4342            LAYERS,
4343            64,
4344            KVQuantType::None,
4345        )))
4346    }
4347
4348    /// Decode logits (last position) for one sequence run through the trusted
4349    /// batch=1 path: prefill the prompt, then one decode step for `next_tok`.
4350    fn serial_decode_logits(prompt: &[i64], next_tok: i64) -> Tensor {
4351        let model = whole_model();
4352        let cache = Arc::new(Mutex::new(KVCacheManager::new(
4353            LAYERS,
4354            64,
4355            KVQuantType::None,
4356        )));
4357        // Inject a fresh isolated cache so the batch=1 path actually caches.
4358        let model = {
4359            let mut m = model;
4360            <LlamaModel as ModelOperations>::set_kv_cache(&mut m, cache);
4361            m
4362        };
4363        // Prefill prompt at pos 0.
4364        let prompt_t = Tensor::from_slice(prompt).reshape([1, prompt.len() as i64]);
4365        let _ = model.forward_with_cache(&prompt_t, 0).unwrap();
4366        // Decode one token at pos = prompt.len().
4367        let dec = Tensor::from_slice(&[next_tok]).reshape([1, 1]);
4368        let logits = model.forward_with_cache(&dec, prompt.len()).unwrap();
4369        // [1, 1, vocab] -> [vocab]
4370        logits.select(0, 0).select(0, 0)
4371    }
4372
4373    #[test]
4374    fn batched_ragged_decode_matches_serial() {
4375        // Three sequences with DIFFERENT prompt lengths (ragged KV histories) +
4376        // one decode token each. The batched decode step (B=3, q=1, ragged kv)
4377        // must match each sequence's serially-run decode logits.
4378        let prompts: [&[i64]; 3] = [&[1, 5, 9, 2], &[7, 3], &[4, 8, 6, 1, 0]];
4379        let next: [i64; 3] = [11, 13, 6];
4380
4381        // Reference (serial, trusted batch=1 path).
4382        let ref_logits: Vec<Tensor> = prompts
4383            .iter()
4384            .zip(&next)
4385            .map(|(p, &t)| serial_decode_logits(p, t))
4386            .collect();
4387
4388        // Batched path: one model, per-sequence isolated caches.
4389        let model = whole_model();
4390        let mut seqs: Vec<(Vec<i64>, usize, Arc<Mutex<KVCacheManager>>)> = prompts
4391            .iter()
4392            .map(|_| (Vec::new(), 0usize, fresh_cache()))
4393            .collect();
4394
4395        // Prefill each sequence's cache individually (batch=1 calls to the batched
4396        // path — ragged q across rows is not mixed in v1).
4397        for (i, p) in prompts.iter().enumerate() {
4398            let mut one = vec![(p.to_vec(), 0usize, seqs[i].2.clone())];
4399            let _ = model.forward_batched(&mut one, None).unwrap();
4400        }
4401
4402        // One batched decode step over the ragged histories.
4403        for (i, &t) in next.iter().enumerate() {
4404            seqs[i].0 = vec![t];
4405            seqs[i].1 = prompts[i].len();
4406        }
4407        let batched = model.forward_batched(&mut seqs, None).unwrap(); // [3, 1, vocab]
4408
4409        let orig = model.config.original_vocab_size as i64;
4410        for (i, ref_l) in ref_logits.iter().enumerate() {
4411            let b_l = batched.select(0, i as i64).select(0, 0); // [vocab]
4412            let r_c = ref_l.narrow(0, 0, orig);
4413            let b_c = b_l.narrow(0, 0, orig);
4414            let max_diff = (&r_c - &b_c).abs().max().double_value(&[]);
4415            assert!(
4416                r_c.allclose(&b_c, 1e-4, 1e-4, false),
4417                "batched decode row {i} diverged from serial (max_diff={max_diff})"
4418            );
4419            // Argmax (sampled token) must agree — the user-visible invariant.
4420            assert_eq!(
4421                r_c.argmax(0, false).int64_value(&[]),
4422                b_c.argmax(0, false).int64_value(&[]),
4423                "row {i} argmax token differs",
4424            );
4425        }
4426    }
4427
4428    #[test]
4429    fn batched_single_row_matches_serial_prefill() {
4430        // A batch of ONE through forward_batched must equal the batch=1 reference
4431        // (guards the padded-dense KV + explicit-mask path against the inline tril).
4432        let prompt: [i64; 4] = [1, 5, 9, 2];
4433        let next = 11i64;
4434        let ref_l = serial_decode_logits(&prompt, next);
4435
4436        let model = whole_model();
4437        let cache = fresh_cache();
4438        let mut prefill = vec![(prompt.to_vec(), 0usize, cache.clone())];
4439        let _ = model.forward_batched(&mut prefill, None).unwrap();
4440        let mut dec = vec![(vec![next], prompt.len(), cache)];
4441        let logits = model.forward_batched(&mut dec, None).unwrap();
4442        let b_l = logits.select(0, 0).select(0, 0);
4443
4444        let orig = model.config.original_vocab_size as i64;
4445        let r_c = ref_l.narrow(0, 0, orig);
4446        let b_c = b_l.narrow(0, 0, orig);
4447        let max_diff = (&r_c - &b_c).abs().max().double_value(&[]);
4448        assert!(
4449            r_c.allclose(&b_c, 1e-4, 1e-4, false),
4450            "single-row batched decode diverged from serial (max_diff={max_diff})"
4451        );
4452    }
4453
4454    /// Regression guard for the causal-mask offset in cached multi-token
4455    /// forwards: with `q_len > 1` and `start_pos > 0` the mask must be
4456    /// `tril(k_len - q_len)`, not `tril(0)` — otherwise every cached row attends
4457    /// only to the first `i+1` keys and all prompt history is masked out. This
4458    /// is exactly the serial partial-prefill shape taken on a session
4459    /// prefix-cache hit with a >=2-token suffix. Compares LOGITS (not just
4460    /// argmax) at fp tolerance.
4461    #[test]
4462    fn cached_two_token_forward_matches_serial_decode_steps() {
4463        let prompt: [i64; 4] = [1, 5, 9, 2];
4464
4465        // Serial reference: prefill, then two one-token decode steps.
4466        let serial = whole_model();
4467        let prompt_t = Tensor::from_slice(&prompt).reshape([1, 4]);
4468        let _ = serial.forward_with_cache(&prompt_t, 0).unwrap();
4469        let ref1 = serial
4470            .forward_with_cache(&Tensor::from_slice(&[11i64]).reshape([1, 1]), 4)
4471            .unwrap();
4472        let ref2 = serial
4473            .forward_with_cache(&Tensor::from_slice(&[13i64]).reshape([1, 1]), 5)
4474            .unwrap();
4475
4476        // Cached two-token forward at start_pos=4 (partial-prefill shape).
4477        let cached = whole_model();
4478        let _ = cached.forward_with_cache(&prompt_t, 0).unwrap();
4479        let logits2 = cached
4480            .forward_with_cache(&Tensor::from_slice(&[11i64, 13]).reshape([1, 2]), 4)
4481            .unwrap();
4482
4483        let orig = serial.config.original_vocab_size as i64;
4484        for (row, ref_l) in [(0i64, &ref1), (1, &ref2)] {
4485            let got = logits2.select(1, row).select(0, 0).narrow(0, 0, orig);
4486            let want = ref_l.select(0, 0).select(0, 0).narrow(0, 0, orig);
4487            let max_diff = (&got - &want).abs().max().double_value(&[]);
4488            assert!(
4489                got.allclose(&want, 1e-4, 1e-4, false),
4490                "cached 2-token forward row {row} diverged from the serial decode step \
4491                 (max_diff={max_diff}); the causal mask must keep k_len - q_len history, \
4492                 not tril(0)"
4493            );
4494            assert_eq!(
4495                got.argmax(-1, false).int64_value(&[]),
4496                want.argmax(-1, false).int64_value(&[]),
4497                "row {row} argmax differs",
4498            );
4499        }
4500    }
4501
4502    /// `forward_layers_train` must reject ranges outside the stage's owned window
4503    /// (same guardrail as the inference runner).
4504    #[test]
4505    fn forward_layers_train_rejects_out_of_window_range() {
4506        let map = LayerDeviceMap::single(Device::Cpu, LAYERS).unwrap();
4507        let mut w = tiny_weights();
4508        let stage = LlamaModel::stage_from_weights_with_config(
4509            &mut w,
4510            tiny_config(),
4511            &map,
4512            2..LAYERS,
4513            DType::Float,
4514            KVQuantType::None,
4515        )
4516        .unwrap();
4517        let emb = Tensor::randn([1, 3, HIDDEN], (DType::Float, Device::Cpu));
4518        assert!(
4519            stage.forward_layers_train(&emb, 0..2, None).is_err(),
4520            "range below window"
4521        );
4522        assert!(
4523            stage.forward_layers_train(&emb, 2..LAYERS, None).is_ok(),
4524            "owned range ok"
4525        );
4526    }
4527}
4528
4529// ============================================================================
4530// FP8 _scaled_mm spike — CPU capability contract + numerics + microbenchmark.
4531// ============================================================================
4532#[cfg(test)]
4533#[allow(
4534    clippy::unwrap_used,
4535    clippy::expect_used,
4536    clippy::print_stdout,
4537    clippy::print_stderr
4538)]
4539mod fp8_gemm_tests {
4540    use super::*;
4541    use std::process::Command;
4542    use tch::Kind;
4543
4544    const FP8: Kind = Kind::Float8e4m3fn;
4545    const OPT: (Kind, Device) = (Kind::Float, Device::Cpu);
4546
4547    /// Deterministic small float tensor, RNG-free so parallel tests can't
4548    /// interleave `manual_seed`.
4549    fn det(dims: &[i64], seed: i64) -> Tensor {
4550        let n: i64 = dims.iter().product();
4551        ((Tensor::arange(n, OPT) + seed).sin() * 2.0).reshape(dims)
4552    }
4553
4554    fn max_rel_err(a: &Tensor, b: &Tensor) -> f64 {
4555        let a = a.to_kind(Kind::Double);
4556        let b = b.to_kind(Kind::Double);
4557        let num = (&a - &b).abs();
4558        let den = b.abs().clamp_min(1e-6);
4559        (num / den).max().double_value(&[])
4560    }
4561
4562    /// ‖a−b‖ / ‖b‖ (Frobenius) — stable under near-zero elements, unlike the
4563    /// per-element ratio.
4564    fn rel_frob_err(a: &Tensor, b: &Tensor) -> f64 {
4565        let a = a.to_kind(Kind::Double);
4566        let b = b.to_kind(Kind::Double);
4567        let num = (&a - &b).norm().double_value(&[]);
4568        let den = b.norm().double_value(&[]).max(1e-12);
4569        num / den
4570    }
4571
4572    fn mean_cuda_us(iterations: u32, mut op: impl FnMut()) -> f64 {
4573        tch::Cuda::synchronize(0);
4574        let started = std::time::Instant::now();
4575        for _ in 0..iterations {
4576            op();
4577            tch::Cuda::synchronize(0);
4578        }
4579        started.elapsed().as_micros() as f64 / f64::from(iterations)
4580    }
4581
4582    /// Compute capability of a CUDA-visible NVIDIA device. `tch` presents
4583    /// ROCm through CUDA-compatible APIs, so availability does not prove that
4584    /// `nvidia-smi` exists. Return `None` to skip NVIDIA-only expectations.
4585    fn nvidia_cuda_compute_capability(device: usize) -> Option<(u32, u32)> {
4586        if tch::utils::has_hip() {
4587            eprintln!("skipping NVIDIA FP8 capability test on ROCm/HIP backend");
4588            return None;
4589        }
4590        let gpu = std::env::var("CUDA_VISIBLE_DEVICES")
4591            .ok()
4592            .and_then(|visible| {
4593                visible
4594                    .split(',')
4595                    .nth(device)
4596                    .map(str::trim)
4597                    .map(str::to_owned)
4598            })
4599            .filter(|id| !id.is_empty())
4600            .unwrap_or_else(|| device.to_string());
4601        let output = match Command::new("nvidia-smi")
4602            .arg(format!("--id={gpu}"))
4603            .arg("--query-gpu=compute_cap")
4604            .arg("--format=csv,noheader,nounits")
4605            .output()
4606        {
4607            Ok(output) if output.status.success() => output,
4608            Ok(output) => {
4609                eprintln!("skipping NVIDIA FP8 capability test: nvidia-smi failed for CUDA device {device}: {}", String::from_utf8_lossy(&output.stderr).trim());
4610                return None;
4611            }
4612            Err(error) => {
4613                eprintln!("skipping NVIDIA FP8 capability test: cannot run nvidia-smi: {error}");
4614                return None;
4615            }
4616        };
4617        let capability = match String::from_utf8(output.stdout) {
4618            Ok(capability) => capability,
4619            Err(error) => {
4620                eprintln!(
4621                    "skipping NVIDIA FP8 capability test: non-UTF-8 capability output: {error}"
4622                );
4623                return None;
4624            }
4625        };
4626        let capability = capability.trim();
4627        let Some((major, minor)) = capability.split_once('.') else {
4628            eprintln!(
4629                "skipping NVIDIA FP8 capability test: malformed compute capability {capability:?}"
4630            );
4631            return None;
4632        };
4633        match (major.parse(), minor.parse()) {
4634            (Ok(major), Ok(minor)) => Some((major, minor)),
4635            _ => {
4636                eprintln!("skipping NVIDIA FP8 capability test: non-numeric compute capability {capability:?}");
4637                None
4638            }
4639        }
4640    }
4641
4642    /// Deterministic pseudo-random in [-1, 1]: frac(sin(i·12.9898)·43758.5453),
4643    /// computed in f64 for argument precision. RNG-free like `det()`, but
4644    /// without `sin(flat_index)`'s catastrophic dot-product cancellation
4645    /// (which inflates relative error metrics ~20x — a data artifact).
4646    fn det_rand(dims: &[i64], seed: i64) -> Tensor {
4647        let n: i64 = dims.iter().product();
4648        let idx = Tensor::arange(n, (Kind::Double, Device::Cpu)) + seed;
4649        let h = (idx * 12.9898).sin() * 43758.5453;
4650        ((&h - h.floor()) * 2.0 - 1.0)
4651            .to_kind(Kind::Float)
4652            .reshape(dims)
4653    }
4654
4655    /// Asserted CPU capability contract for the shipped libtorch `_scaled_mm` with e4m3
4656    /// (measured by this spike; update if a future libtorch widens it):
4657    ///  - v1 CPU: per-TENSOR scales only ("_scaled_mm only supports per-tensor
4658    ///    scaling for CPU backend", ATen/native/Blas.cpp). Row/col/blockwise
4659    ///    scale layouts are rejected.
4660    ///  - v2 CPU: not registered at all — dispatch covers CUDA/XPU only
4661    ///    (native_functions.yaml; no CPU kernel). The 1x128/128x128 recipe is
4662    ///    further restricted to NVIDIA Hopper (SM90 + cuBLASLt ≥ 12.9) by
4663    ///    `_check_deepseek_support`.
4664    #[test]
4665    fn scaled_mm_cpu_capability_contract() {
4666        let (m, k, n) = (8, 128, 64);
4667        let a = det(&[m, k], 1).to_kind(FP8);
4668        let b = det(&[k, n], 2).to_kind(FP8);
4669        let one = Tensor::ones([1, 1], OPT);
4670        // v1 tensorwise: works, BF16 out.
4671        let out = a
4672            .f_internal_scaled_mm(
4673                &b,
4674                &one,
4675                &one,
4676                None::<&Tensor>,
4677                None::<&Tensor>,
4678                Kind::BFloat16,
4679                false,
4680            )
4681            .expect("v1 tensorwise e4m3 must work on CPU");
4682        assert_eq!(out.size(), [m, n]);
4683        assert_eq!(out.kind(), Kind::BFloat16);
4684        // Sanity: scale == 1 so out == dequant(a) @ dequant(b) in bf16 tolerance.
4685        let expect = a.to_kind(Kind::Float).matmul(&b.to_kind(Kind::Float));
4686        assert!(
4687            max_rel_err(&out, &expect) < 1e-2,
4688            "v1 tensorwise result diverged from plain matmul of dequantized inputs"
4689        );
4690        // v1 rowwise: rejected on CPU.
4691        let sa = Tensor::ones([m, 1], OPT);
4692        let sb = Tensor::ones([1, n], OPT);
4693        let err = a
4694            .f_internal_scaled_mm(
4695                &b,
4696                &sa,
4697                &sb,
4698                None::<&Tensor>,
4699                None::<&Tensor>,
4700                Kind::BFloat16,
4701                false,
4702            )
4703            .expect_err("v1 rowwise must be rejected on CPU");
4704        assert!(
4705            err.to_string().contains("per-tensor scaling"),
4706            "unexpected: {err}"
4707        );
4708        // v2: not registered for CPU at all.
4709        let sa_refs = [&sa];
4710        let sb_refs = [&sb];
4711        let err = a
4712            .f_internal_scaled_mm_v2(
4713                &b,
4714                &sa_refs,
4715                [RECIPE_BLOCKWISE_1X128],
4716                [NO_SWIZZLE],
4717                &sb_refs,
4718                [RECIPE_BLOCKWISE_128X128],
4719                [NO_SWIZZLE],
4720                None::<&Tensor>,
4721                Kind::BFloat16,
4722                [] as [i64; 0],
4723                false,
4724            )
4725            .expect_err("v2 must be unavailable on CPU");
4726        assert!(
4727            err.to_string().contains("_scaled_mm_v2"),
4728            "unexpected: {err}"
4729        );
4730    }
4731
4732    /// The 1x128 activation quantizer: roundtrip q*scale ≈ x. This is the
4733    /// device-independent half of the FP8 path — the CUDA kernel consumes
4734    /// exactly these (q, scale) tensors, so their error bounds the end-to-end
4735    /// error regardless of backend.
4736    #[test]
4737    fn quantize_activations_1x128_roundtrip() {
4738        let x = det(&[7, 256], 3);
4739        let (q, scale) = quantize_activations_1x128(&x);
4740        assert_eq!(q.kind(), FP8);
4741        assert_eq!(q.size(), x.size());
4742        assert_eq!(scale.kind(), Kind::Float);
4743        assert_eq!(scale.size(), [7, 2]);
4744        // Dequantize and compare.
4745        let xq = (q.to_kind(Kind::Float).view([7, 2, 128]) * scale.unsqueeze(-1)).view([7, 256]);
4746        let rel = max_rel_err(&xq, &x);
4747        // e4m3 has 3 mantissa bits → per-element rel err ≤ 2^-4 = 6.25% for
4748        // NORMAL outputs (round-to-nearest can tie exactly at the bound, hence
4749        // `<=`). Subnormal-quantized outputs can exceed 2^-4 in relative
4750        // terms, but amax normalization keeps block magnitudes ≳ 448·2^-9,
4751        // well above the e4m3 subnormal floor — so the bound holds here.
4752        assert!(
4753            rel <= 0.0625,
4754            "roundtrip max rel err {rel} exceeds e4m3 ulp bound"
4755        );
4756        assert!(q.to_kind(Kind::Float).abs().max().double_value(&[]) <= 448.0);
4757    }
4758
4759    /// Simulate the full scaled_mm math (activation 1x128 quant + weight
4760    /// 128x128 block dequant) in plain f32 ops — exactly what the CUDA v2
4761    /// kernel computes — and bound the error vs (a) the lazy-dequant BF16
4762    /// baseline and (b) a full-precision reference. The kernel accumulates in
4763    /// f32, so real-GPU error should be at or below this envelope.
4764    ///
4765    /// Uses hash-based pseudo-random data, NOT `det()`: `sin(flat_index)`
4766    /// rows/columns produce dot products with catastrophic cancellation
4767    /// (signal O(1), error at the sqrt(K)·term scale), which inflated the
4768    /// measured rel err to ~0.4 — an artifact of the data, not the scheme.
4769    #[test]
4770    fn fp8_blockwise_matmul_error_envelope() {
4771        let (m, k, n) = (8, 256, 256);
4772        let w = det_rand(&[k, n], 4) * 0.05; // weight-scale values
4773        let x = det_rand(&[m, k], 5);
4774
4775        // Quantize weight per 128x128 block (what the checkpoint stores).
4776        let w4 = w.view([k / 128, 128, n / 128, 128]);
4777        let ws = w4.abs().amax(-1, false).amax(1, false).clamp_min(1e-12) / 448.0; // [k/128, n/128]
4778        let wq = (w4 / ws.view([k / 128, 1, n / 128, 1]))
4779            .clamp(-448.0, 448.0)
4780            .to_kind(FP8);
4781        let w_deq = (wq.to_kind(Kind::Float) * ws.view([k / 128, 1, n / 128, 1])).view([k, n]);
4782
4783        // Simulated scaled_mm: quantize activations, dequantize both, matmul.
4784        let (q, sa) = quantize_activations_1x128(&x);
4785        let x_deq =
4786            (q.to_kind(Kind::Float).view([m, k / 128, 128]) * sa.unsqueeze(-1)).view([m, k]);
4787        let out_sim = x_deq.matmul(&w_deq);
4788
4789        // (a) vs lazy-dequant baseline: BF16 cast of w_deq then matmul.
4790        let out_lazy = x
4791            .to_kind(Kind::BFloat16)
4792            .matmul(&w_deq.to_kind(Kind::BFloat16))
4793            .to_kind(Kind::Float);
4794        let rel_vs_lazy = rel_frob_err(&out_sim, &out_lazy);
4795        // (b) vs full-precision reference.
4796        let out_ref = x.matmul(&w);
4797        let rel_vs_ref = rel_frob_err(&out_sim, &out_ref);
4798        // Error is dominated by the e4m3 mantissa (≤ 6.25% per element,
4799        // ~2.3% RMS measured above, partially averaged over K=256
4800        // accumulation). The observed envelope is printed for the spike
4801        // report; assert conservative bounds.
4802        assert!(
4803            rel_vs_ref < 0.10,
4804            "sim vs full-precision rel err {rel_vs_ref}"
4805        );
4806        assert!(
4807            rel_vs_lazy < 0.10,
4808            "sim vs lazy-dequant rel err {rel_vs_lazy}"
4809        );
4810        println!(
4811            "error envelope (frob rel): x_deq={:.4} w_deq={:.4} sim_vs_lazy={:.4} sim_vs_ref={:.4}",
4812            rel_frob_err(&x_deq, &x),
4813            rel_frob_err(&w_deq, &w),
4814            rel_vs_lazy,
4815            rel_vs_ref
4816        );
4817    }
4818
4819    /// Error envelope for the v1 rowwise arm (per-token activation scale ×
4820    /// per-output-channel weight scale — the SM120/Blackwell path), simulating
4821    /// exactly the math the rowwise kernel computes. Multiple deterministic
4822    /// seeds and within-column dynamic ranges exercise both ordinary and
4823    /// difficult-but-valid FP8 inputs. The tight 0.12 bound is deliberately
4824    /// sensitive: it leaves architecture/rounding margin while rejecting a
4825    /// several-fold regression from the observed 0.0251 real-SM120 and
4826    /// 0.0253–0.0363 simulation errors. The per-case and worst-case assertions
4827    /// are the negative-control evidence: broken scale orientation or a lost
4828    /// requantization path must exceed this envelope rather than being hidden
4829    /// by the former 0.30 bound.
4830    #[test]
4831    fn fp8_rowwise_requant_error_envelope() {
4832        let (m, k, n) = (8, 256, 256);
4833        let mut worst_lazy = 0.0f64;
4834        let mut worst_ref = 0.0f64;
4835        for &(seed, spread) in &[(4i64, 1.0f64), (17, 8.0), (101, 64.0)] {
4836            // The second half of K has a smaller magnitude within each output
4837            // channel. This is a representative dynamic range for the
4838            // rowwise requantization's coarser per-column scale.
4839            let row_factors = Tensor::ones([k, 1], OPT);
4840            let small = Tensor::full([k / 2, 1], 1.0 / spread, OPT);
4841            row_factors.narrow(0, k / 2, k / 2).copy_(&small);
4842            let w = det_rand(&[k, n], seed) * 0.05 * row_factors;
4843            let x = det_rand(&[m, k], seed + 1);
4844
4845            // Quantize weight per 128x128 block (what the checkpoint stores).
4846            let w4 = w.view([k / 128, 128, n / 128, 128]);
4847            let ws = w4.abs().amax(-1, false).amax(1, false).clamp_min(1e-12) / 448.0;
4848            let wq = (w4 / ws.view([k / 128, 1, n / 128, 1]))
4849                .clamp(-448.0, 448.0)
4850                .to_kind(FP8);
4851            let w_deq_block =
4852                (wq.to_kind(Kind::Float) * ws.view([k / 128, 1, n / 128, 1])).view([k, n]);
4853
4854            // Load-time rowwise requant from the FP8 checkpoint tensors.
4855            let rowwise = build_rowwise_requant(&wq.view([k, n]), &ws);
4856            assert_eq!(rowwise.scale.size(), [1, n]);
4857
4858            // Simulated v1 rowwise scaled_mm.
4859            let (q, s_a) = quantize_activations_rowwise(&x);
4860            assert_eq!(s_a.size(), [m, 1]);
4861            let x_deq = q.to_kind(Kind::Float) * s_a;
4862            let w_deq = rowwise.weight.to_kind(Kind::Float) * &rowwise.scale;
4863            let out_sim = x_deq.matmul(&w_deq);
4864
4865            let out_lazy = x
4866                .to_kind(Kind::BFloat16)
4867                .matmul(&w_deq_block.to_kind(Kind::BFloat16))
4868                .to_kind(Kind::Float);
4869            let rel_vs_lazy = rel_frob_err(&out_sim, &out_lazy);
4870            let out_ref = x.matmul(&w);
4871            let rel_vs_ref = rel_frob_err(&out_sim, &out_ref);
4872            worst_lazy = worst_lazy.max(rel_vs_lazy);
4873            worst_ref = worst_ref.max(rel_vs_ref);
4874            assert!(
4875                rel_vs_lazy < 0.12,
4876                "rowwise seed={seed} spread={spread} vs lazy err {rel_vs_lazy}"
4877            );
4878            assert!(
4879                rel_vs_ref < 0.12,
4880                "rowwise seed={seed} spread={spread} vs reference err {rel_vs_ref}"
4881            );
4882            println!(
4883                "rowwise envelope seed={seed} spread={spread}: sim_vs_lazy={rel_vs_lazy:.4} sim_vs_ref={rel_vs_ref:.4}"
4884            );
4885        }
4886        println!("rowwise envelope worst: sim_vs_lazy={worst_lazy:.4} sim_vs_ref={worst_ref:.4}");
4887    }
4888
4889    /// Load-time `scale_b` packing: `[K/128, N/128]` → `[N/128, L4]`
4890    /// contiguous with the K-block dim zero-padded to a multiple of 4
4891    /// (cuBLASLt TMA). CPU-verifiable layout contract.
4892    #[test]
4893    fn fp8_scale_v2_packed_layout() {
4894        // K=256 → K/128=2 blocks → L4=4 (padding exercised); N=384 → 3 blocks.
4895        let scale = det(&[2, 3], 9);
4896        let packed = pack_scale_b_v2(&scale);
4897        assert_eq!(packed.size(), [3, 4]);
4898        assert_eq!(packed.kind(), Kind::Float);
4899        assert!(packed.is_contiguous());
4900        // K-block-minor order: packed[j, i] == scale[i, j] for real blocks.
4901        let expect = scale.transpose(0, 1);
4902        assert!(packed.narrow(1, 0, 2).allclose(&expect, 0.0, 0.0, false));
4903        // Padding region is zeros (never read by the kernel; TMA allocation only).
4904        assert_eq!(packed.narrow(1, 2, 2).abs().max().double_value(&[]), 0.0);
4905        // Passed as .t() at the call site: [L4, N/128] with strides (1, L4).
4906        let passed = packed.transpose(0, 1);
4907        assert_eq!(passed.size(), [4, 3]);
4908        assert_eq!(passed.stride(), [1, 4]);
4909        // Exact-multiple case (K/128=4) must not pad.
4910        let packed4 = pack_scale_b_v2(&det(&[4, 2], 10));
4911        assert_eq!(packed4.size(), [2, 4]);
4912        // with_scale derives scale_v2 automatically.
4913        let proj = LinearProjection::with_scale(det(&[256, 384], 11).to_kind(FP8), det(&[2, 3], 9));
4914        assert_eq!(proj.scale_v2.as_ref().unwrap().size(), [3, 4]);
4915    }
4916
4917    /// The failure latch is per-(CUDA-device, recipe) (PR #1523 review +
4918    /// SM120 rowwise arm): a failure of one recipe on one device must not
4919    /// disable the other recipe or other devices, and each combination warns
4920    /// exactly once. CPU-testable without GPUs since the latch is keyed on
4921    /// the (device, recipe) pair alone.
4922    #[test]
4923    fn device_latch_isolation() {
4924        use ScaledMmRecipe::{BlockwiseV2, RowwiseV1};
4925        let latch = DeviceLatch::default();
4926        assert!(!latch.is_failed(0, BlockwiseV2) && !latch.is_failed(1, BlockwiseV2));
4927        // First failure on (0, BlockwiseV2) reports (caller warns); repeats don't.
4928        assert!(latch.mark_failed(0, BlockwiseV2));
4929        assert!(!latch.mark_failed(0, BlockwiseV2));
4930        // Device 0 blockwise latched; device 1 and the rowwise recipe unaffected.
4931        assert!(latch.is_failed(0, BlockwiseV2));
4932        assert!(!latch.is_failed(1, BlockwiseV2));
4933        assert!(!latch.is_failed(0, RowwiseV1));
4934        // Rowwise on the same device fails and reports independently.
4935        assert!(latch.mark_failed(0, RowwiseV1));
4936        assert!(latch.is_failed(0, RowwiseV1));
4937        // Device 1 failing later still reports.
4938        assert!(latch.mark_failed(1, BlockwiseV2));
4939        assert!(latch.is_failed(1, BlockwiseV2));
4940    }
4941
4942    /// Sticky-latch / warn-once contract for a FAILING recipe. On the SM120
4943    /// validation host this is the blockwise v2 arm: it is Hopper-only, so it
4944    /// must latch before the rowwise arm is attempted.
4945    /// Drives a `DeviceLatch` through exactly the guard sequence the
4946    /// `try_blockwise_v2`/`try_rowwise_v1` arms use — `is_failed` check before
4947    /// touching the kernel, `mark_failed` on error with the warning emitted
4948    /// only on first failure — with a stub kernel that always errors. Asserts
4949    /// the kernel is attempted exactly once, the warning decision fires
4950    /// exactly once, and every later call short-circuits to the lazy fallback
4951    /// without re-attempting or re-warning. CPU-testable, while the real CUDA
4952    /// test below proves the blockwise-to-rowwise fallback on SM120.
4953    #[test]
4954    fn failing_recipe_latches_off_without_warning_spam() {
4955        let latch = DeviceLatch::default();
4956        let attempts = std::cell::Cell::new(0);
4957        let warnings = std::cell::Cell::new(0);
4958        // Mirrors the try_* arm bodies: None when latched off or when the
4959        // (here always-failing) kernel errors; the caller then falls back to
4960        // lazy dequant.
4961        let arm = |latch: &DeviceLatch| -> Option<()> {
4962            if latch.is_failed(0, ScaledMmRecipe::BlockwiseV2) {
4963                return None; // latched: straight to lazy, no kernel, no warning
4964            }
4965            attempts.set(attempts.get() + 1);
4966            let res: Result<(), &str> = Err("SM120 does not support blockwise v2");
4967            res.inspect_err(|_| {
4968                if latch.mark_failed(0, ScaledMmRecipe::BlockwiseV2) {
4969                    warnings.set(warnings.get() + 1); // the tracing::warn! site
4970                }
4971            })
4972            .ok()
4973        };
4974        for _ in 0..8 {
4975            assert!(arm(&latch).is_none());
4976        }
4977        assert_eq!(
4978            attempts.get(),
4979            1,
4980            "kernel must only be attempted before the latch trips"
4981        );
4982        assert_eq!(warnings.get(), 1, "warn-once per (device, recipe)");
4983        assert!(latch.is_failed(0, ScaledMmRecipe::BlockwiseV2));
4984        // The other recipe on the same device is unaffected.
4985        assert!(!latch.is_failed(0, ScaledMmRecipe::RowwiseV1));
4986    }
4987
4988    /// On CPU the gated path must decline (no CUDA) and `apply` must fall back
4989    /// to the lazy-dequant result unchanged.
4990    #[test]
4991    fn fp8_scaled_mm_declines_on_cpu() {
4992        let w = det(&[128, 256], 6).clamp(-2.0, 2.0).to_kind(FP8); // [in, out]
4993        let scale = Tensor::ones([1, 2], OPT); // [in/128, out/128]
4994        let proj = LinearProjection::with_scale(w, scale);
4995        let x = det(&[3, 128], 7).to_kind(Kind::BFloat16);
4996        assert!(
4997            proj.apply_fp8_scaled_mm(&x).is_none(),
4998            "CPU must decline the scaled_mm path"
4999        );
5000        // Lazy fallback still computes.
5001        let out = proj.apply(&x);
5002        assert_eq!(out.size(), [3, 256]);
5003    }
5004
5005    /// Probe body for [`fp8_gate_causal_subprocesses`]. It intentionally runs
5006    /// in a fresh process: `fp8_gemm_enabled()` is process-cached, so toggling
5007    /// env vars in one test process would make gate tests order-dependent.
5008    #[test]
5009    fn fp8_gate_probe() {
5010        let Ok(mode) = std::env::var("HYPRSTREAM_FP8_GATE_PROBE") else {
5011            return;
5012        };
5013        let gate_on = std::env::var("HYPRSTREAM_FP8_GEMM").as_deref() == Ok("1");
5014        let dequant_on = std::env::var("HYPRSTREAM_FP8_DEQUANT_LOAD").as_deref() == Ok("1");
5015        assert_eq!(
5016            fp8_gemm_enabled(),
5017            gate_on,
5018            "FP8 gate env was not read independently"
5019        );
5020        assert_eq!(
5021            crate::config::default_fp8_dequant_load(),
5022            dequant_on,
5023            "load-time dequant gate changed independently of its own env"
5024        );
5025
5026        let w = det(&[256, 256], 31).clamp(-2.0, 2.0).to_kind(FP8);
5027        let p = LinearProjection::with_scale(w, Tensor::ones([2, 2], OPT));
5028        let x_cpu = det(&[2, 256], 32).to_kind(Kind::BFloat16);
5029        let _ = p.apply(&x_cpu);
5030        if mode == "cpu" {
5031            // Negative control: with no CUDA device involved, production apply
5032            // must stay on lazy dequant even when the FP8 gate is enabled.
5033            assert_eq!(
5034                FP8_APPLY_PATH.load(std::sync::atomic::Ordering::SeqCst),
5035                2,
5036                "CPU probe unexpectedly selected a scaled-mm path"
5037            );
5038            println!("causal gate probe: CPU lazy fallback observed");
5039            return;
5040        }
5041
5042        assert_eq!(mode, "sm120", "unknown FP8 gate probe mode");
5043        assert!(
5044            gate_on && !dequant_on,
5045            "SM120 probe must be FP8-on/dequant-off"
5046        );
5047        assert!(tch::Cuda::is_available(), "SM120 probe requires CUDA");
5048        let dev = Device::Cuda(0);
5049        let p = p.into_device(dev);
5050        let x_train = x_cpu.to_device(dev).set_requires_grad(true);
5051        let out_train = p.apply(&x_train);
5052        assert_eq!(
5053            FP8_APPLY_PATH.load(std::sync::atomic::Ordering::SeqCst),
5054            2,
5055            "autograd-capable input unexpectedly selected scaled-mm"
5056        );
5057        out_train.sum(Kind::Float).backward();
5058        assert!(
5059            x_train.grad().defined(),
5060            "lazy matmul did not retain a backward path"
5061        );
5062
5063        let x = x_cpu.to_device(dev);
5064        let out = p.apply(&x);
5065        assert_eq!(out.size(), [2, 256]);
5066        assert_eq!(
5067            FP8_APPLY_PATH.load(std::sync::atomic::Ordering::SeqCst),
5068            1,
5069            "production apply did not return the gated scaled-mm result"
5070        );
5071        assert!(
5072            scaled_mm_latch().is_failed(0, ScaledMmRecipe::BlockwiseV2),
5073            "SM120 blockwise recipe did not latch off"
5074        );
5075        assert!(
5076            !scaled_mm_latch().is_failed(0, ScaledMmRecipe::RowwiseV1),
5077            "SM120 rowwise recipe latched despite libtorch 2.11"
5078        );
5079        println!(
5080            "causal gate probe: FP8 on selected rowwise after blockwise latch; dequant gate off"
5081        );
5082    }
5083
5084    #[test]
5085    fn fp8_e5m2_no_requant_probe() {
5086        if std::env::var_os("HYPRSTREAM_FP8_E5M2_PROBE").is_none() {
5087            return;
5088        }
5089        assert!(
5090            fp8_gemm_enabled(),
5091            "E5M2 allocation probe requires FP8 gate on"
5092        );
5093        reset_rowwise_requant_peak();
5094        let _projection = LinearProjection::with_scale(
5095            det(&[256, 256], 41).to_kind(Kind::Float8e5m2),
5096            Tensor::ones([2, 2], OPT),
5097        );
5098        assert_eq!(
5099            rowwise_requant_live(),
5100            0,
5101            "unsupported E5M2 construction retained a rowwise allocation"
5102        );
5103        assert_eq!(
5104            rowwise_requant_peak(),
5105            0,
5106            "unsupported E5M2 construction invoked rowwise requantization"
5107        );
5108        println!("causal allocation probe: E5M2 did not construct rowwise requantization");
5109    }
5110
5111    /// Causal gate test with negative controls. Each child asserts the
5112    /// production `apply()` path, metadata construction, and independent
5113    /// dequant default; a broken gate, fallback, or env coupling fails here.
5114    /// The GPU child additionally proves real SM120 rowwise selection without
5115    /// manually injecting `rowwise` metadata.
5116    #[test]
5117    fn fp8_gate_causal_subprocesses() {
5118        fn run_probe(name: &str, fp8: &str, dequant: &str, probe: &str, marker: &str) {
5119            let exe = std::env::current_exe().expect("test executable path");
5120            let output = Command::new(exe)
5121                .arg("--exact")
5122                .arg(format!(
5123                    "runtime::architectures::llama::fp8_gemm_tests::{name}"
5124                ))
5125                .arg("--nocapture")
5126                .env("HYPRSTREAM_FP8_GEMM", fp8)
5127                .env("HYPRSTREAM_FP8_DEQUANT_LOAD", dequant)
5128                .env("HYPRSTREAM_FP8_GATE_PROBE", probe)
5129                .output()
5130                .expect("spawn FP8 gate probe");
5131            let transcript = format!(
5132                "{}{}",
5133                String::from_utf8_lossy(&output.stdout),
5134                String::from_utf8_lossy(&output.stderr)
5135            );
5136            assert!(
5137                output.status.success(),
5138                "FP8 gate probe {probe} failed: {}\n{transcript}",
5139                output.status
5140            );
5141            assert!(
5142                transcript.contains("1 passed"),
5143                "FP8 gate probe {probe} did not execute exactly one test:\n{transcript}"
5144            );
5145            assert!(
5146                transcript.contains(marker),
5147                "FP8 gate probe {probe} missed expected marker {marker:?}:\n{transcript}"
5148            );
5149        }
5150
5151        let cpu_marker = "causal gate probe: CPU lazy fallback observed";
5152        run_probe("fp8_gate_probe", "0", "0", "cpu", cpu_marker);
5153        run_probe("fp8_gate_probe", "1", "0", "cpu", cpu_marker);
5154        run_probe("fp8_gate_probe", "0", "1", "cpu", cpu_marker);
5155        let exe = std::env::current_exe().expect("test executable path");
5156        let output = Command::new(&exe)
5157            .arg("--exact")
5158            .arg("runtime::architectures::llama::fp8_gemm_tests::fp8_e5m2_no_requant_probe")
5159            .arg("--nocapture")
5160            .env("HYPRSTREAM_FP8_GEMM", "1")
5161            .env("HYPRSTREAM_FP8_E5M2_PROBE", "1")
5162            .output()
5163            .expect("spawn E5M2 allocation probe");
5164        let transcript = format!(
5165            "{}{}",
5166            String::from_utf8_lossy(&output.stdout),
5167            String::from_utf8_lossy(&output.stderr)
5168        );
5169        assert!(
5170            output.status.success(),
5171            "E5M2 allocation probe failed: {}\n{transcript}",
5172            output.status
5173        );
5174        assert!(
5175            transcript.contains("1 passed")
5176                && transcript.contains(
5177                    "causal allocation probe: E5M2 did not construct rowwise requantization"
5178                ),
5179            "E5M2 allocation probe did not execute its expected child:\n{transcript}"
5180        );
5181
5182        if tch::Cuda::is_available() && matches!(nvidia_cuda_compute_capability(0), Some((12, _))) {
5183            run_probe("fp8_gate_probe", "1", "0", "sm120", "causal gate probe: FP8 on selected rowwise after blockwise latch; dequant gate off");
5184        }
5185
5186        // Fused Qwen projections run in their own exact child so their
5187        // `with_scale` reconstruction is checked with FP8 GEMM enabled from
5188        // process start, rather than inheriting this test's cached env state.
5189        let output = Command::new(exe)
5190            .arg("--exact")
5191            .arg("runtime::architectures::qwen3_5::pipeline_tests::fp8_fused_projection_metadata_probe")
5192            .arg("--nocapture")
5193            .env("HYPRSTREAM_FP8_GEMM", "1")
5194            .env("HYPRSTREAM_FP8_DEQUANT_LOAD", "0")
5195            .env("HYPRSTREAM_QWEN_FP8_PROBE", "1")
5196            .output()
5197            .expect("spawn Qwen FP8 metadata probe");
5198        let transcript = format!(
5199            "{}{}",
5200            String::from_utf8_lossy(&output.stdout),
5201            String::from_utf8_lossy(&output.stderr)
5202        );
5203        assert!(
5204            output.status.success(),
5205            "Qwen fused metadata probe failed: {}\n{transcript}",
5206            output.status
5207        );
5208        assert!(transcript.contains("1 passed") && transcript.contains("Qwen fused FP8 allocation probe: source rowwise copies dropped before fused allocation"),
5209            "Qwen fused allocation probe did not execute its expected child:\n{transcript}");
5210    }
5211
5212    /// Real-kernel test on CUDA: rowwise must match the lazy-dequant baseline.
5213    /// Not ignored — self-skips without CUDA. On SM120, blockwise v2 is
5214    /// expected to reject and latch exactly once; libtorch 2.11's widened
5215    /// CUTLASS gate must then execute rowwise rather than latching it to lazy
5216    /// fallback. K=256 → K/128=2 blocks, so the L4=4 zero-padding in
5217    /// `pack_scale_b_v2` is exercised.
5218    #[test]
5219    fn fp8_scaled_mm_kernel_cuda() {
5220        if !tch::Cuda::is_available() {
5221            eprintln!("skipping: no CUDA device");
5222            return;
5223        }
5224        let dev = Device::Cuda(0);
5225        let (m, k, n) = (8, 256, 256);
5226        let w = det_rand(&[k, n], 4) * 0.05;
5227        let w4 = w.view([k / 128, 128, n / 128, 128]);
5228        let ws = w4.abs().amax(-1, false).amax(1, false).clamp_min(1e-12) / 448.0;
5229        let wq = (w4 / ws.view([k / 128, 1, n / 128, 1]))
5230            .clamp(-448.0, 448.0)
5231            .to_kind(FP8)
5232            .view([k, n]);
5233        // Flag is off in test env, so with_scale doesn't build the rowwise
5234        // pair — build it explicitly (exactly what the gated load path does).
5235        let mut proj = LinearProjection::with_scale(wq, ws.view([k / 128, n / 128]));
5236        proj.rowwise = Some(Box::new(build_rowwise_requant(
5237            &proj.weight,
5238            proj.scale.as_ref().unwrap(),
5239        )));
5240        // Load-time pack: [N/128, L4] with L4 = round_up(2, 4) = 4.
5241        assert_eq!(proj.scale_v2.as_ref().unwrap().size(), [n / 128, 4]);
5242        let proj = proj.into_device(dev);
5243        let x = det_rand(&[m, k], 5).to_kind(Kind::BFloat16).to_device(dev);
5244
5245        let out_lazy = proj.apply(&x); // flag off in tests → lazy dequant
5246        let Some((cc_major, cc_minor)) = nvidia_cuda_compute_capability(0) else {
5247            return;
5248        };
5249        println!("CUDA device 0 compute capability: {cc_major}.{cc_minor}");
5250
5251        if cc_major == 9 {
5252            // Hopper's supported path is blockwise v2. Its success must not
5253            // be mistaken for the Blackwell fallback/latch contract below.
5254            let out = proj
5255                .try_blockwise_v2(0, &x, m, k)
5256                .expect("SM90/Hopper must execute the blockwise v2 FP8 kernel");
5257            assert!(
5258                !scaled_mm_latch().is_failed(0, ScaledMmRecipe::BlockwiseV2),
5259                "blockwise v2 latched despite a successful SM90 kernel call"
5260            );
5261            let rel = rel_frob_err(&out, &out_lazy);
5262            println!("blockwise v2 (SM90) vs lazy-dequant frob rel err: {rel:.4}");
5263            assert!(rel < 0.30, "blockwise v2 diverged on SM90: {rel}");
5264            return;
5265        }
5266        if cc_major < 10 {
5267            // SM89 and older have a different PyTorch FP8 support matrix. The
5268            // exact SM90 and SM100+ contracts are tested above/below; do not
5269            // impose either one on another CUDA architecture.
5270            eprintln!(
5271                "skipping architecture-specific FP8 recipe assertion on SM{cc_major}{cc_minor}"
5272            );
5273            return;
5274        }
5275
5276        // SM100+ uses the widened 2.11 CUTLASS rowwise gate. Blockwise v2 is
5277        // Hopper-only, so its first failure is expected to latch while the
5278        // rowwise kernel must remain live.
5279        assert!(
5280            proj.try_blockwise_v2(0, &x, m, k).is_none(),
5281            "blockwise v2 must reject on SM100+ before rowwise runs"
5282        );
5283        assert!(
5284            scaled_mm_latch().is_failed(0, ScaledMmRecipe::BlockwiseV2),
5285            "blockwise v2 neither ran nor latched off on this CUDA device"
5286        );
5287        // A second call must short-circuit: the kernel and warning are not retried.
5288        assert!(proj.try_blockwise_v2(0, &x, m, k).is_none());
5289
5290        let out = proj.try_rowwise_v1(0, &x, m, k).expect(
5291            "libtorch 2.11 must execute the SM100+ rowwise kernel instead of latching to lazy fallback",
5292        );
5293        assert!(
5294            !scaled_mm_latch().is_failed(0, ScaledMmRecipe::RowwiseV1),
5295            "rowwise v1 latched despite a successful SM100+ kernel call"
5296        );
5297        let rel = rel_frob_err(&out, &out_lazy);
5298        println!("rowwise v1 (SM{cc_major}{cc_minor}) vs lazy-dequant frob rel err: {rel:.4}");
5299        // The deterministic multi-seed envelope test above observes <=0.0373;
5300        // retain headroom while rejecting a several-fold regression here.
5301        assert!(rel < 0.12, "rowwise v1 diverged: {rel}");
5302
5303        // Benchmark a realistic projection after the correctness check. `apply`
5304        // is deliberately lazy here because HYPRSTREAM_FP8_GEMM is off by default.
5305        let (bench_m, bench_k, bench_n) = (1, 5120, 5120);
5306        let bench_w = det_rand(&[bench_k, bench_n], 14) * 0.05;
5307        let bench_w4 = bench_w.view([bench_k / 128, 128, bench_n / 128, 128]);
5308        let bench_scale = bench_w4
5309            .abs()
5310            .amax(-1, false)
5311            .amax(1, false)
5312            .clamp_min(1e-12)
5313            / 448.0;
5314        let bench_wq = (bench_w4 / bench_scale.view([bench_k / 128, 1, bench_n / 128, 1]))
5315            .clamp(-448.0, 448.0)
5316            .to_kind(FP8)
5317            .view([bench_k, bench_n]);
5318        let mut bench_proj = LinearProjection::with_scale(bench_wq, bench_scale);
5319        bench_proj.rowwise = Some(Box::new(build_rowwise_requant(
5320            &bench_proj.weight,
5321            bench_proj.scale.as_ref().unwrap(),
5322        )));
5323        let bench_proj = bench_proj.into_device(dev);
5324        let bench_x = det_rand(&[bench_m, bench_k], 15)
5325            .to_kind(Kind::BFloat16)
5326            .to_device(dev);
5327        let iters = 20;
5328        let lazy_us = mean_cuda_us(iters, || {
5329            let _ = std::hint::black_box(bench_proj.apply(&bench_x));
5330        });
5331        let rowwise_us = mean_cuda_us(iters, || {
5332            let _ = std::hint::black_box(
5333                bench_proj
5334                    .try_rowwise_v1(0, &bench_x, bench_m, bench_k)
5335                    .expect("rowwise benchmark kernel must run"),
5336            );
5337        });
5338        println!(
5339            "SM120 FP8 projection [{bench_m},{bench_k}]x[{bench_k},{bench_n}]: \
5340             lazy_dequant={lazy_us:.0}us ({:.2} iter/s), rowwise={rowwise_us:.0}us ({:.2} iter/s), ratio={:.3}x",
5341            1_000_000.0 / lazy_us,
5342            1_000_000.0 / rowwise_us,
5343            lazy_us / rowwise_us,
5344        );
5345    }
5346
5347    /// Microbenchmark: lazy-dequant matmul vs the scaled_mm path composition
5348    /// at a realistic layer shape. On CPU the v2 kernel is unavailable, so
5349    /// this times (a) lazy apply, (b) activation quantization alone (the added
5350    /// per-call overhead), (c) v1 tensorwise scaled_mm as a GEMM proxy. Real
5351    /// GPU latency numbers must be taken on the CUDA box (run --ignored).
5352    #[test]
5353    #[ignore = "microbenchmark — run explicitly; GPU numbers need the CUDA box"]
5354    fn bench_fp8_gemm_vs_lazy_dequant() {
5355        let (m, k, n) = (1, 5120, 5120);
5356        let w = Tensor::zeros([k, n], OPT).clamp(-1.0, 1.0).to_kind(FP8);
5357        let scale = Tensor::ones([k / 128, n / 128], OPT);
5358        let proj = LinearProjection::with_scale(w, scale);
5359        let x = det(&[m, k], 8).to_kind(Kind::BFloat16);
5360        let iters = 20;
5361
5362        let t0 = std::time::Instant::now();
5363        for _ in 0..iters {
5364            let _ = proj.apply(&x);
5365        }
5366        let lazy_us = t0.elapsed().as_micros() as f64 / iters as f64;
5367
5368        let x2 = x.reshape([m, k]);
5369        let t0 = std::time::Instant::now();
5370        for _ in 0..iters {
5371            let _ = quantize_activations_1x128(&x2);
5372        }
5373        let quant_us = t0.elapsed().as_micros() as f64 / iters as f64;
5374
5375        // v1 tensorwise scaled_mm (the only CPU kernel) as a GEMM-time proxy.
5376        let one = Tensor::ones([1, 1], OPT);
5377        let (q, _) = quantize_activations_1x128(&x2);
5378        let t0 = std::time::Instant::now();
5379        for _ in 0..iters {
5380            let _ = q.internal_scaled_mm(
5381                &proj.weight,
5382                &one,
5383                &one,
5384                None::<&Tensor>,
5385                None::<&Tensor>,
5386                Kind::BFloat16,
5387                false,
5388            );
5389        }
5390        let smm_us = t0.elapsed().as_micros() as f64 / iters as f64;
5391
5392        println!(
5393            "shape [{m},{k}]x[{k},{n}] CPU: lazy_dequant={lazy_us:.0}us \
5394             act_quant={quant_us:.0}us v1_tensorwise_scaled_mm={smm_us:.0}us"
5395        );
5396    }
5397}