Skip to main content

hyprstream_inference/runtime/
model_factory.rs

1//! Model factory for creating models with unified configuration management
2//!
3//! This replaces the chaotic multiple paths for model creation with a single,
4//! clean factory pattern.
5
6use anyhow::{anyhow, bail, Context, Result};
7use serde::Deserialize;
8use std::collections::{BTreeMap, BTreeSet, HashMap};
9use std::ops::Range;
10use std::path::{Path, PathBuf};
11use tch::{Device, Kind as DType, Tensor};
12use tracing::{debug, info, instrument, warn};
13
14use super::architectures::{gemma::GemmaModel, llama::LlamaModel, ModelOperations};
15use super::device_pool::DevicePool;
16use super::model_config::{ModelArchitecture, ModelConfig};
17use super::torch_utils::{estimate_tensor_size_mb, safe_to_device};
18use crate::worktree_ext::WorktreeClientExt;
19use hyprstream_rpc_std::model_client::KVQuantType;
20use hyprstream_rpc_std::registry_client::WorktreeClient;
21
22/// Strict-loader opt-in (#315): when set truthy, multi-shard models that lack a
23/// `model.safetensors.index.json` manifest are rejected instead of silently
24/// reconstructing the shard set from a filename glob (which is fragile across
25/// model families and can silently drop/duplicate shards).
26const STRICT_LOADER_ENV: &str = "HYPRSTREAM_STRICT_LOADER";
27
28/// Follow the factory's authoritative config architecture detection, including
29/// accepted case variants and architectures-only metadata. Validating bytes on
30/// an existing route does not add MoE or vision inference support. Other
31/// architectures and default/unknown loading retain their existing policy.
32fn native_finite_policy(model_path: &Path) -> bool {
33    ModelConfig::detect_architecture(model_path) == ModelArchitecture::Qwen3_5
34}
35
36/// Scan raw little-endian values with constant auxiliary memory, before the
37/// corresponding tensor blob/copy/cast. This is per admitted tensor, not an
38/// all-shard preflight before ANY allocation. A failure aborts the whole load
39/// before a model is returned for inference. Packed FP8 remains packed.
40fn validate_finite_payload(name: &str, dtype: safetensors::Dtype, data: &[u8]) -> Result<()> {
41    use safetensors::Dtype;
42    let invalid = match dtype {
43        Dtype::BF16 => data
44            .chunks_exact(2)
45            .position(|b| u16::from_le_bytes([b[0], b[1]]) & 0x7f80 == 0x7f80),
46        Dtype::F16 => data
47            .chunks_exact(2)
48            .position(|b| u16::from_le_bytes([b[0], b[1]]) & 0x7c00 == 0x7c00),
49        Dtype::F32 => data.chunks_exact(4).position(|b| {
50            u32::from_le_bytes([b[0], b[1], b[2], b[3]]) & 0x7f80_0000 == 0x7f80_0000
51        }),
52        Dtype::F8_E4M3 => data.iter().position(|b| b & 0x7f == 0x7f),
53        Dtype::F8_E5M2 => data.iter().position(|b| b & 0x7c == 0x7c),
54        // Unsupported materialization dtypes still fail at the existing loader
55        // boundary below; this policy does not admit another tensor format.
56        _ => None,
57    };
58    if let Some(index) = invalid {
59        bail!("nonfinite native Qwen3.5 checkpoint tensor `{name}` ({dtype:?}) at element {index}");
60    }
61    Ok(())
62}
63
64fn strict_loader_enabled() -> bool {
65    std::env::var(STRICT_LOADER_ENV)
66        .map(|v| {
67            matches!(
68                v.trim().to_lowercase().as_str(),
69                "1" | "true" | "yes" | "on"
70            )
71        })
72        .unwrap_or(false)
73}
74
75// Match safetensors' header bound. Only metadata is read during the MTP
76// preflight; checkpoint tensor payloads are left to the existing loader.
77const MTP_HEADER_LIMIT: usize = 100_000_000;
78
79fn mtp_runtime_supported(requested: bool, kv_quant_type: KVQuantType) -> bool {
80    // Match TextStream admission: quantized KV cannot rewind a rejected draft.
81    // Exclude its unusable head before metadata preflight or tensor allocation.
82    requested && kv_quant_type == KVQuantType::None
83}
84
85fn mtp_header_len(prefix: &[u8]) -> Result<usize> {
86    let encoded: [u8; 8] = prefix
87        .try_into()
88        .context("truncated safetensors header length")?;
89    let len = usize::try_from(u64::from_le_bytes(encoded))?;
90    anyhow::ensure!(
91        len <= MTP_HEADER_LIMIT,
92        "safetensors header exceeds metadata limit"
93    );
94    Ok(len)
95}
96
97fn mtp_config_supported(config: &serde_json::Value) -> bool {
98    // Qwen checkpoints may store these fields at the root or in text_config.
99    // A declared unsupported layout must win over absent/default fields.
100    [config, &config["text_config"]].into_iter().all(|cfg| {
101        !cfg["model_type"]
102            .as_str()
103            .is_some_and(|name| name.contains("moe"))
104            && cfg["num_experts"].as_u64().unwrap_or(0) == 0
105            && cfg
106                .get("mtp_num_hidden_layers")
107                .is_none_or(|n| n.as_u64() == Some(1))
108    })
109}
110
111fn mtp_header_supported(header: &[u8]) -> Result<bool> {
112    let entries: serde_json::Map<String, serde_json::Value> = serde_json::from_slice(header)?;
113    Ok(entries.keys().all(|name| {
114        if !name.starts_with("mtp.") {
115            return true;
116        }
117        !name.contains(".experts.")
118            && !name.contains(".shared_expert.")
119            && name
120                .strip_prefix("mtp.layers.")
121                .is_none_or(|layer| layer.starts_with("0."))
122    }))
123}
124
125/// Materialize every FP8 weight in `weights` as BF16 once, applying the
126/// companion block-wise `<name>_scale_inv` scales during the conversion, and
127/// drop the scale tensors. Gated by `HYPRSTREAM_FP8_DEQUANT_LOAD` /
128/// [`crate::config::RuntimeConfig::fp8_dequant_load`].
129///
130/// This runs on the raw checkpoint tensor map (all FP8 weights are 2D there,
131/// stored `[out, in]` with scales `[out/128, in/128]`), so a single generic
132/// pass covers every FP8 site downstream — llama-family `LinearProjection`,
133/// Qwen3.5 fused QKV/gate-up projections, MoE expert stacks, and lm_head all
134/// observe plain BF16 weights with no scales, and their lazy per-matmul
135/// dequant branches never trigger. Returns the number of tensors converted.
136pub(crate) fn dequantize_fp8_weights_at_load(weights: &mut HashMap<String, Tensor>) -> usize {
137    #[inline]
138    fn is_fp8(kind: tch::Kind) -> bool {
139        matches!(kind, tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2)
140    }
141
142    let fp8_keys: Vec<String> = weights
143        .iter()
144        .filter(|(name, tensor)| is_fp8(tensor.kind()) && !name.ends_with("_scale_inv"))
145        .map(|(name, _)| name.clone())
146        .collect();
147    let mut converted = 0;
148
149    for key in fp8_keys {
150        if weights.contains_key(&format!("{key}_scale")) {
151            continue;
152        }
153        let Some(weight) = weights.remove(&key) else {
154            continue;
155        };
156        let scale_key = format!("{key}_scale_inv");
157        let scale = weights.remove(&scale_key);
158        if let Some(s) = &scale {
159            let ws = weight.size();
160            let ss = s.size();
161            if !(ws.len() == 2 && ss.len() == 2 && ws[0] % ss[0] == 0 && ws[1] % ss[1] == 0) {
162                // Non-conforming scale shape: leave the weight FP8 and the scale
163                // in the map untouched. Downstream lazy dequant keeps ownership
164                // of the pair and fails loudly if the shapes really are unusable,
165                // instead of silently running on raw FP8-code magnitudes.
166                warn!(
167                    "FP8 weight '{}' scale shape {:?} does not block-divide weight shape {:?}; \
168                     leaving FP8 + scale untouched for the lazy dequant path",
169                    key, ss, ws
170                );
171                weights.insert(scale_key, s.shallow_clone());
172                weights.insert(key, weight);
173                continue;
174            }
175        }
176        // Block-wise dequantization via 4D broadcast multiply (same math as the
177        // lazy path in `LinearProjection::apply`): scale [r/128, c/128] is
178        // broadcast across each 128-element block. Elementwise per block, so
179        // the stored [out, in] orientation needs no special handling.
180        //
181        // Peak-memory discipline (P2 review): the scale is applied IN PLACE on
182        // the BF16 cast buffer — no second BF16 allocation — and the FP8
183        // source is dropped before the result is retained, so converting one
184        // tensor peaks at FP8 + 1x BF16 instead of the old FP8 + 2x BF16
185        // (out-of-place multiply held all three). With HashMap iteration order
186        // arbitrary, that spike could land on top of the already-converted
187        // bulk of the model and OOM a load whose final BF16 size would fit.
188        let w_bf16 = weight.to_kind(tch::Kind::BFloat16);
189        if let Some(scale) = scale {
190            let ws = w_bf16.size();
191            let ss = scale.size();
192            let block_r = ws[0] / ss[0];
193            let block_c = ws[1] / ss[1];
194            let s_4d = scale
195                .to_kind(tch::Kind::BFloat16)
196                .view([ss[0], 1, ss[1], 1]);
197            // In-place broadcast multiply on a block view of the contiguous
198            // BF16 buffer; the buffer already IS the 2D result (the 4D view
199            // is zero-copy). On the (practically unreachable) kernel error,
200            // restore the untouched FP8 weight + scale for the lazy path
201            // rather than retaining an unscaled cast.
202            if let Err(e) = w_bf16.view([ss[0], block_r, ss[1], block_c]).f_mul_(&s_4d) {
203                warn!(
204                    "FP8 weight '{key}' in-place scale multiply failed ({e}); \
205                     leaving FP8 + scale untouched for the lazy dequant path"
206                );
207                drop(w_bf16);
208                weights.insert(scale_key, scale);
209                weights.insert(key, weight);
210                continue;
211            }
212            drop(scale);
213        }
214        drop(weight);
215        weights.insert(key, w_bf16);
216        converted += 1;
217    }
218    converted
219}
220
221/// Run load-time FP8 dequantization when requested — but never for a
222/// multi-device pipeline: every checkpoint tensor is loaded onto the pool's
223/// *primary* device first, so materializing the whole model as BF16 there
224/// (before `stage_from_weights_with_config` spreads layers across the pool)
225/// would peak at full-BF16 on one device and could OOM a model whose BF16
226/// weights only fit across the pool. In that case the flag is ignored with a
227/// warning and the FP8 lazy-dequant path is kept.
228fn maybe_dequantize_fp8_weights_at_load(
229    weights: &mut HashMap<String, Tensor>,
230    fp8_dequant_load: bool,
231    device_pool: Option<&DevicePool>,
232) {
233    if !fp8_dequant_load {
234        return;
235    }
236    if device_pool.is_some_and(|p| !p.is_single()) {
237        warn!(
238            "FP8 dequant-at-load ignored for multi-device pipeline: all weights land on the \
239             primary device before layer distribution, so full-model BF16 materialization \
240             there could OOM; keeping FP8 lazy per-matmul dequant"
241        );
242        return;
243    }
244    let converted = dequantize_fp8_weights_at_load(weights);
245    if converted > 0 {
246        info!(
247            "Dequantized {} FP8 weight tensor(s) to BF16 at load",
248            converted
249        );
250    }
251}
252
253/// Glob-fallback shard discovery, shared by the sync/async loader paths.
254///
255/// Loud by design (#315): a multi-shard model reached via glob (no index.json) is
256/// a silent-degrade path — it warns, and under [`STRICT_LOADER_ENV`] it is a hard
257/// error. A single globbed shard is unambiguous and passes quietly.
258fn glob_shard_files(model_path: &Path) -> Result<Vec<std::path::PathBuf>> {
259    let mut shard_files = Vec::new();
260    for entry in std::fs::read_dir(model_path)? {
261        let entry = entry?;
262        if let Some(name) = entry.file_name().to_str() {
263            if (name.starts_with("model-") || name.starts_with("model.safetensors-"))
264                && name.ends_with(".safetensors")
265            {
266                shard_files.push(entry.path());
267            }
268        }
269    }
270    shard_files.sort();
271
272    if shard_files.len() > 1 {
273        if strict_loader_enabled() {
274            return Err(anyhow!(
275                "{} multi-shard safetensors files found in {} but no \
276                 model.safetensors.index.json manifest; refusing to reconstruct the \
277                 shard set from a filename glob under {STRICT_LOADER_ENV}. \
278                 Provide the index.json manifest.",
279                shard_files.len(),
280                model_path.display()
281            ));
282        }
283        warn!(
284            "⚠️ No model.safetensors.index.json in {}; reconstructing {} shards from a \
285             filename glob (fragile). Set {STRICT_LOADER_ENV}=1 to require the manifest.",
286            model_path.display(),
287            shard_files.len()
288        );
289    }
290    Ok(shard_files)
291}
292
293/// Factory for creating models with proper configuration management
294pub struct ModelFactory;
295
296/// Authoritative request for one contiguous Llama-family pipeline stage.
297///
298/// Stage position is derived from the range and the checkpoint's authoritative
299/// `num_hidden_layers`: a range starting at zero owns the embedding, while a
300/// range ending at `num_hidden_layers` owns the final norm and LM head. Keeping
301/// those roles implicit prevents a caller from requesting an inconsistent
302/// range/role combination.
303#[derive(Clone, Debug, Eq, PartialEq)]
304pub struct ModelStageRequest {
305    pub layer_range: Range<usize>,
306}
307
308#[derive(Debug, Deserialize)]
309struct SafetensorsIndex {
310    weight_map: BTreeMap<String, String>,
311}
312
313/// Resolved content of a weight shard after transparent LFS/XET pointer
314/// resolution — the exact bytes the loader builds tensors from.
315///
316/// Large ordinary shards are returned as an open [`File`] so the KV-compat
317/// base-weight digest can *stream* them (the on-disk bytes ARE the content —
318/// already smudged, or never a pointer). A resolved pointer, or any small file,
319/// is returned owned. This is the single shared resolution surface used by both
320/// the loader ([`ModelFactory::load_file_with_pointer_detection`]) and the
321/// KV-compat digest ([`ModelFactory::resolve_weight_for_digest`]) so the two
322/// never disagree about what bytes a shard resolves to (#1277).
323pub(crate) enum ResolvedWeight {
324    /// Resolved bytes held in memory (a resolved LFS/XET pointer, or a small
325    /// ordinary file).
326    Owned(Vec<u8>),
327    /// An open ordinary file ≥1 KiB; read or stream it directly.
328    File(std::fs::File),
329}
330
331impl ResolvedWeight {
332    /// Borrow resolved content without copying owned payloads. Reset file
333    /// position for every bounded metadata pass; keep the same handle alive.
334    pub(crate) fn with_reader<T>(
335        &mut self,
336        read: impl FnOnce(&mut dyn std::io::Read) -> Result<T>,
337    ) -> Result<T> {
338        match self {
339            Self::File(file) => {
340                use std::io::Seek as _;
341                file.rewind()?;
342                read(file)
343            }
344            Self::Owned(bytes) => read(&mut std::io::Cursor::new(bytes.as_slice())),
345        }
346    }
347}
348
349impl ModelFactory {
350    /// Detect the dtype of a model by examining its tensors
351    pub async fn detect_model_dtype(model_path: &Path) -> Result<DType> {
352        // Check for single file first
353        let single_file = model_path.join("model.safetensors");
354        let file_to_check = if single_file.exists() {
355            single_file
356        } else {
357            // Look for first shard file
358            let shard_files = Self::find_shard_files(model_path)?;
359            if shard_files.is_empty() {
360                return Err(anyhow!(
361                    "No model weights found in {}",
362                    model_path.display()
363                ));
364            }
365            shard_files[0].clone()
366        };
367
368        // Load just the metadata to check dtype
369        let file_content = std::fs::read(&file_to_check)?;
370        let tensors = safetensors::SafeTensors::deserialize(&file_content)?;
371
372        // Check the first few tensors to determine predominant dtype.
373        // FP8 models store quantized weights as FP8 but use BF16 for compute
374        // (activations, KV cache, non-quantized modules). We detect FP8 separately
375        // and return BF16 as the compute dtype.
376        let mut f16_count = 0;
377        let mut bf16_count = 0;
378        let mut f32_count = 0;
379        let mut fp8_count = 0;
380
381        for (_, tensor) in tensors.tensors().into_iter().take(10) {
382            match tensor.dtype() {
383                safetensors::Dtype::F16 => f16_count += 1,
384                safetensors::Dtype::BF16 => bf16_count += 1,
385                safetensors::Dtype::F8_E4M3 | safetensors::Dtype::F8_E5M2 => fp8_count += 1,
386                safetensors::Dtype::F32 => f32_count += 1,
387                _ => {}
388            }
389        }
390
391        // FP8 models: weights are FP8 but compute dtype is BF16
392        if fp8_count > 0 && fp8_count >= bf16_count && fp8_count >= f16_count {
393            info!("Detected FP8 quantized model (storage: FP8, compute dtype: BF16)");
394            return Ok(tch::Kind::BFloat16);
395        }
396
397        // Return the most common dtype
398        if f16_count > bf16_count && f16_count > f32_count {
399            info!("Detected F16 model");
400            Ok(tch::Kind::Half)
401        } else if bf16_count >= f16_count && bf16_count >= f32_count {
402            info!("Detected BF16 model");
403            Ok(tch::Kind::BFloat16)
404        } else if f32_count > 0 {
405            info!("Detected F32 model");
406            Ok(tch::Kind::Float)
407        } else {
408            info!("Could not detect model dtype, defaulting to BF16");
409            Ok(tch::Kind::BFloat16)
410        }
411    }
412
413    /// Create a model from a directory containing weights and optionally config.json
414    /// This is the ONLY way models should be created to ensure consistency
415    ///
416    /// # Multi-device pipeline (#314 wiring, #310 epic)
417    ///
418    /// Pass `device_pool = Some(pool)` with a pool of >1 device to build the model
419    /// as a single multi-device pipeline stage: decoder layers are spread across
420    /// the pool's devices via [`LayerDeviceMap::even_split`], and the model's
421    /// `forward_layers` performs the lone cross-device copy at each stage boundary.
422    /// `device` must be the pool's primary (first) device — non-layer weights
423    /// (embeddings, final norm, lm_head) live there. When `device_pool` is `None`
424    /// or holds a single device, construction is unchanged (whole model on
425    /// `device`). At runtime this depends on #405 (from_weights device-placement
426    /// fix) to actually place per-layer tensors on non-primary devices.
427    /// `speculative_decoding` is the resolved runtime policy (including TOML).
428    /// When false, KV is quantized, or config/checkpoint metadata declares an
429    /// unsupported MTP layout, draft weights skip tensor/device allocation.
430    #[instrument(name = "model_factory.create", skip(device, dtype, device_pool), fields(model_path = %model_path.display()))]
431    pub async fn create(
432        model_path: &Path,
433        device: &Device,
434        dtype: DType,
435        max_context: Option<usize>,
436        kv_quant_type: KVQuantType,
437        device_pool: Option<&DevicePool>,
438        fp8_dequant_load: bool,
439        speculative_decoding: bool,
440    ) -> Result<Box<dyn ModelOperations>> {
441        // Validate the declared compute profile before allocating shard payloads.
442        let channel_profile = super::channel_fp8::ChannelFp8Profile::read(model_path)?;
443        if channel_profile.is_some() {
444            anyhow::ensure!(
445                *device == Device::Cpu,
446                "FP8_DYNAMIC is CPU-only pending CPU acceptance"
447            );
448            anyhow::ensure!(!fp8_dequant_load, "FP8_DYNAMIC must retain activation QDQ; eager weight-only decoding is not equivalent");
449        }
450        info!("Loading model: {}", model_path.display());
451        if let Some(mc) = max_context {
452            info!("Using max_context override: {} tokens", mc);
453        }
454        if kv_quant_type != KVQuantType::None {
455            info!("Using KV cache quantization: {:?}", kv_quant_type);
456        }
457        if fp8_dequant_load {
458            info!(
459                "FP8 dequant-at-load enabled: FP8 weights will be materialized as BF16 during load"
460            );
461        }
462        if let Some(pool) = device_pool {
463            if !pool.is_single() {
464                info!(
465                    "🔀 Multi-device pipeline: spreading layers across {} device(s) {:?} \
466                     (primary {:?}); depends on #405 for correct per-layer placement",
467                    pool.len(),
468                    pool.devices(),
469                    pool.primary(),
470                );
471            }
472        }
473
474        // Check if we have sharded files that need incremental loading
475        let shard_files = Self::find_shard_files(model_path)?;
476
477        // Decide eligibility across the whole checkpoint before the first
478        // shard can allocate a draft tensor. A later shard may reveal layer 1.
479        let mtp_shards = if shard_files.len() <= 1 && model_path.join("model.safetensors").exists()
480        {
481            vec![model_path.join("model.safetensors")]
482        } else {
483            shard_files.clone()
484        };
485        // Snapshot finite admission at the authoritative model root before
486        // any asynchronous pointer resolution, just like other whole loads.
487        let channel_native_finite = channel_profile.is_some() && native_finite_policy(model_path);
488        let mut channel_inputs = if let Some(profile) = &channel_profile {
489            Some(
490                Self::prepare_channel_inputs(profile, &mtp_shards, |path| async move {
491                    Self::resolve_weight_for_digest(&path).await
492                })
493                .await?,
494            )
495        } else {
496            None
497        };
498        let requested = mtp_runtime_supported(speculative_decoding, kv_quant_type);
499        let speculative_decoding = if let Some(inputs) = channel_inputs.as_mut() {
500            Self::mtp_loading_policy_resolved(model_path, inputs, requested).await?
501        } else {
502            Self::mtp_loading_policy(model_path, &mtp_shards, requested).await?
503        };
504
505        if let Some(inputs) = channel_inputs {
506            let mut weights = HashMap::new();
507            for (path, resolved) in inputs {
508                Self::load_resolved_safetensors_file_inner(
509                    &path,
510                    resolved,
511                    None,
512                    &mut weights,
513                    device,
514                    dtype,
515                    speculative_decoding,
516                    channel_native_finite,
517                    true,
518                )
519                .await?;
520            }
521            let config = ModelConfig::load(model_path, &weights)?;
522            return Self::create_model_from_config(
523                config,
524                weights,
525                device,
526                dtype,
527                max_context,
528                kv_quant_type,
529                model_path,
530                device_pool,
531                fp8_dequant_load,
532                speculative_decoding,
533            );
534        }
535
536        if !shard_files.is_empty() && shard_files.len() > 1 {
537            // Use incremental loading for large sharded models
538            info!(
539                "📦 Using incremental loading for {} shards",
540                shard_files.len()
541            );
542            Self::create_incremental(
543                model_path,
544                device,
545                dtype,
546                shard_files,
547                max_context,
548                kv_quant_type,
549                device_pool,
550                fp8_dequant_load,
551                speculative_decoding,
552            )
553            .await
554        } else {
555            // Standard loading for single files or small models
556            let weights =
557                Self::load_weights(model_path, device, dtype, speculative_decoding).await?;
558            let config = ModelConfig::load(model_path, &weights)?;
559            let model = Self::create_model_from_config(
560                config,
561                weights,
562                device,
563                dtype,
564                max_context,
565                kv_quant_type,
566                model_path,
567                device_pool,
568                fp8_dequant_load,
569                speculative_decoding,
570            )?;
571            info!("✅ ModelFactory: Model created successfully");
572            Ok(model)
573        }
574    }
575
576    /// Create one Llama-family pipeline stage from an authoritative HF index.
577    ///
578    /// Unlike [`Self::create`], stage mode never falls back to a single-file or
579    /// glob-discovered whole-model load. It requires
580    /// `model.safetensors.index.json`, resolves the exact tensor names for the
581    /// requested global layer range and its first/last role, opens only shards
582    /// containing those names, and constructs no tensor outside that set.
583    ///
584    /// Llama, Qwen, and Mistral checkpoints use the production-wired
585    /// `LlamaModel` stage path. Qwen3.5 remains deliberately out of scope.
586    #[instrument(
587        name = "model_factory.create_stage",
588        skip(device, dtype, device_pool),
589        fields(
590            model_path = %model_path.display(),
591            layer_start = request.layer_range.start,
592            layer_end = request.layer_range.end,
593        )
594    )]
595    pub async fn create_stage(
596        model_path: &Path,
597        device: &Device,
598        dtype: DType,
599        max_context: Option<usize>,
600        kv_quant_type: KVQuantType,
601        device_pool: Option<&DevicePool>,
602        request: ModelStageRequest,
603        fp8_dequant_load: bool,
604    ) -> Result<Box<dyn ModelOperations>> {
605        let config_path = model_path.join("config.json");
606        if !config_path.is_file() {
607            bail!(
608                "stage loading requires authoritative config {}",
609                config_path.display()
610            );
611        }
612        let empty_weights = HashMap::new();
613        let config = ModelConfig::load(model_path, &empty_weights)?;
614        if !matches!(
615            &config.architecture,
616            ModelArchitecture::Llama | ModelArchitecture::Qwen | ModelArchitecture::Mistral
617        ) {
618            bail!(
619                "stage loading is supported only for Llama-family checkpoints; got {:?}",
620                config.architecture
621            );
622        }
623
624        Self::validate_stage_range(&request.layer_range, config.num_hidden_layers)?;
625        let plan = Self::stage_weight_plan(
626            model_path,
627            config.num_hidden_layers,
628            request.layer_range.clone(),
629        )?;
630        let metadata = Self::stage_tensor_metadata(&plan).await?;
631        Self::validate_stage_tensor_schema(&config, &request.layer_range, &metadata)?;
632        let mut weights = Self::load_weights_for_stage_plan(
633            plan,
634            request.layer_range.clone(),
635            device,
636            dtype,
637            native_finite_policy(model_path),
638        )
639        .await?;
640        maybe_dequantize_fp8_weights_at_load(&mut weights, fp8_dequant_load, device_pool);
641
642        Self::create_llama_model(
643            config,
644            weights,
645            device,
646            dtype,
647            max_context,
648            kv_quant_type,
649            device_pool,
650            Some(request.layer_range),
651        )
652    }
653
654    /// Find all shard files in a model directory.
655    ///
656    /// Prefers `model.safetensors.index.json` (authoritative HuggingFace shard manifest)
657    /// over filename glob patterns, which are fragile across model families.
658    ///
659    /// This is THE loader's shard selector — also used by the KV-compat
660    /// base-weight digest (#1277) so the fingerprint covers exactly the
661    /// loader-selected set, never extra/unreferenced `.safetensors` files.
662    pub(crate) fn find_shard_files(model_path: &Path) -> Result<Vec<std::path::PathBuf>> {
663        // 1. Use index file if present (most reliable)
664        let index_path = model_path.join("model.safetensors.index.json");
665        if index_path.exists() {
666            return Self::shard_files_from_index(model_path, &index_path);
667        }
668
669        // 2. Single unsharded file
670        let single_file = model_path.join("model.safetensors");
671        if single_file.exists() {
672            return Ok(vec![single_file]);
673        }
674
675        // 3. Fallback: glob for known shard naming patterns (loud / strict-gated)
676        glob_shard_files(model_path)
677    }
678
679    /// Shard selection for the KV-compat base-weight digest (#1277).
680    ///
681    /// Fingerprint authority is stricter than loader permissiveness: the
682    /// loader may warn-and-continue on a manifest-less multi-shard glob
683    /// (unless [`STRICT_LOADER_ENV`]), but the digest must NEVER mint an
684    /// authoritative identity over that ambiguous set — a glob can silently
685    /// drop/duplicate shards, so two snapshots could share a fingerprint
686    /// while loading different weights. Such a set is rejected here
687    /// regardless of `HYPRSTREAM_STRICT_LOADER`, and the caller declines KV
688    /// reuse. A single globbed shard remains unambiguous and passes.
689    pub(crate) fn find_shard_files_for_digest(
690        model_path: &Path,
691    ) -> Result<Vec<std::path::PathBuf>> {
692        let shards = Self::find_shard_files(model_path)?;
693        if shards.len() > 1 && !model_path.join("model.safetensors.index.json").exists() {
694            return Err(anyhow!(
695                "{} loader-selected .safetensors shards in {} but no \
696                 model.safetensors.index.json manifest; the glob-selected set is \
697                 ambiguous, so no authoritative base-weight digest can be minted \
698                 (KV reuse is declined). Provide the index.json manifest.",
699                shards.len(),
700                model_path.display()
701            ));
702        }
703        Ok(shards)
704    }
705
706    /// Parse shard filenames from `model.safetensors.index.json`.
707    /// Returns unique shard paths in sorted order (guaranteed by the index).
708    fn shard_files_from_index(
709        model_path: &Path,
710        index_path: &Path,
711    ) -> Result<Vec<std::path::PathBuf>> {
712        let content = std::fs::read_to_string(index_path)?;
713        let index: serde_json::Value = serde_json::from_str(&content)?;
714        let weight_map = index["weight_map"]
715            .as_object()
716            .ok_or_else(|| anyhow!("model.safetensors.index.json missing weight_map"))?;
717
718        let mut seen = std::collections::BTreeSet::new();
719        for filename in weight_map.values() {
720            if let Some(s) = filename.as_str() {
721                seen.insert(s.to_owned());
722            }
723        }
724
725        let mut paths: Vec<_> = seen
726            .into_iter()
727            .map(|name| model_path.join(&name))
728            .collect();
729        paths.sort();
730
731        info!(
732            "📋 Index file lists {} shard(s): {:?}",
733            paths.len(),
734            paths
735                .iter()
736                .map(|p| p.file_name().unwrap_or_default())
737                .collect::<Vec<_>>()
738        );
739        Ok(paths)
740    }
741
742    fn validate_stage_range(layer_range: &Range<usize>, num_layers: usize) -> Result<()> {
743        if layer_range.start >= layer_range.end || layer_range.end > num_layers {
744            bail!(
745                "invalid stage layer range {:?} for checkpoint with {} layers",
746                layer_range,
747                num_layers
748            );
749        }
750        Ok(())
751    }
752
753    fn llama_layer_index(name: &str) -> Option<usize> {
754        let rest = name.strip_prefix("model.layers.")?;
755        let (index, suffix) = rest.split_once('.')?;
756        if suffix.is_empty() {
757            return None;
758        }
759        index.parse().ok()
760    }
761
762    fn resolve_required_alias(
763        weight_map: &BTreeMap<String, String>,
764        aliases: &[&str],
765        description: &str,
766    ) -> Result<String> {
767        aliases
768            .iter()
769            .find(|name| weight_map.contains_key(**name))
770            .map(|name| (*name).to_owned())
771            .ok_or_else(|| {
772                anyhow!(
773                    "stage weight_map is missing required {description}; expected one of {}",
774                    aliases.join(", ")
775                )
776            })
777    }
778
779    /// Add the block-wise `<name>_scale_inv` companion of a top-level stage
780    /// tensor when the manifest has one. Layer-prefixed FP8 scales are
781    /// selected automatically by the layer-index filter, but top-level tensors
782    /// (embedding, lm_head) enter the plan by exact name — without this the
783    /// stage loads a scale-less FP8 tensor and `dequantize_fp8_weights_at_load`
784    /// casts raw FP8 codes straight to BF16 (#1519 review). No-op for
785    /// unquantized checkpoints, which have no such key.
786    fn insert_companion_scale(
787        weight_map: &BTreeMap<String, String>,
788        required: &mut BTreeSet<String>,
789        name: &str,
790    ) {
791        for suffix in ["_scale_inv", "_scale"] {
792            let scale = format!("{name}{suffix}");
793            if weight_map.contains_key(&scale) {
794                required.insert(scale);
795            }
796        }
797    }
798
799    /// Resolve one stage to exact manifest tensor names grouped by shard.
800    ///
801    /// The returned map is the complete I/O plan: callers must not open any
802    /// other shard or enumerate any other tensor while constructing weights.
803    fn stage_weight_plan(
804        model_path: &Path,
805        num_layers: usize,
806        layer_range: Range<usize>,
807    ) -> Result<BTreeMap<PathBuf, BTreeSet<String>>> {
808        Self::validate_stage_range(&layer_range, num_layers)?;
809
810        let index_path = model_path.join("model.safetensors.index.json");
811        let content = std::fs::read_to_string(&index_path).with_context(|| {
812            format!(
813                "stage loading requires authoritative shard index {}",
814                index_path.display()
815            )
816        })?;
817        let index: SafetensorsIndex = serde_json::from_str(&content).with_context(|| {
818            format!(
819                "stage loading requires a valid `weight_map` object in {}",
820                index_path.display()
821            )
822        })?;
823        if index.weight_map.is_empty() {
824            bail!(
825                "stage loading requires a non-empty `weight_map` in {}",
826                index_path.display()
827            );
828        }
829        if let Some((name, _)) = index
830            .weight_map
831            .iter()
832            .find(|(name, shard)| name.is_empty() || shard.is_empty())
833        {
834            bail!(
835                "stage weight_map in {} contains an empty tensor name or shard for `{name}`",
836                index_path.display()
837            );
838        }
839
840        let manifest_layers: BTreeSet<_> = index
841            .weight_map
842            .keys()
843            .filter_map(|name| Self::llama_layer_index(name))
844            .collect();
845        let expected_layers: BTreeSet<_> = (0..num_layers).collect();
846        if manifest_layers != expected_layers {
847            bail!(
848                "stage weight_map decoder layers {:?} do not exactly match expected range 0..{}",
849                manifest_layers,
850                num_layers
851            );
852        }
853
854        let mut required = BTreeSet::new();
855        for layer in layer_range.clone() {
856            let layer_names: Vec<_> = index
857                .weight_map
858                .keys()
859                .filter(|name| Self::llama_layer_index(name) == Some(layer))
860                .cloned()
861                .collect();
862            if layer_names.is_empty() {
863                bail!("stage weight_map has no tensors for required decoder layer {layer}");
864            }
865            required.extend(layer_names);
866        }
867
868        let is_first = layer_range.start == 0;
869        let is_last = layer_range.end == num_layers;
870        if is_first {
871            let embed = Self::resolve_required_alias(
872                &index.weight_map,
873                &["model.embed_tokens.weight", "embed_tokens.weight"],
874                "embedding tensor",
875            )?;
876            Self::insert_companion_scale(&index.weight_map, &mut required, &embed);
877            required.insert(embed);
878        }
879        if is_last {
880            required.insert(Self::resolve_required_alias(
881                &index.weight_map,
882                &["model.norm.weight", "norm.weight"],
883                "final norm tensor",
884            )?);
885
886            if let Some(lm_head) = ["lm_head.weight", "model.lm_head.weight"]
887                .into_iter()
888                .find(|name| index.weight_map.contains_key(*name))
889            {
890                Self::insert_companion_scale(&index.weight_map, &mut required, lm_head);
891                required.insert(lm_head.to_owned());
892            } else if !is_first {
893                bail!("a non-first final stage requires an explicit lm_head tensor in weight_map");
894            }
895        }
896
897        let mut plan: BTreeMap<PathBuf, BTreeSet<String>> = BTreeMap::new();
898        for name in required {
899            let shard = index.weight_map.get(&name).ok_or_else(|| {
900                anyhow!("required stage tensor `{name}` disappeared from weight_map")
901            })?;
902            plan.entry(model_path.join(shard)).or_default().insert(name);
903        }
904
905        Ok(plan)
906    }
907
908    /// Read only selected safetensors headers before allocating any `tch` tensor.
909    ///
910    /// This preflight deliberately reuses the authoritative I/O plan. It opens no
911    /// shard outside that plan and calls neither `TensorView::data()` nor a
912    /// device-placement API, so a schema mismatch fails before model memory is
913    /// materialized.
914    async fn stage_tensor_metadata(
915        plan: &BTreeMap<PathBuf, BTreeSet<String>>,
916    ) -> Result<BTreeMap<String, Vec<usize>>> {
917        use memmap2::Mmap;
918
919        let mut metadata = BTreeMap::new();
920        for (shard, names) in plan {
921            match Self::resolve_weight_for_digest(shard).await? {
922                ResolvedWeight::File(file) => {
923                    let mmap = unsafe { Mmap::map(&file) }.with_context(|| {
924                        format!("failed to mmap required stage shard {}", shard.display())
925                    })?;
926                    let tensors =
927                        safetensors::SafeTensors::deserialize(&mmap).with_context(|| {
928                            format!("failed to parse required stage shard {}", shard.display())
929                        })?;
930                    Self::record_stage_tensor_metadata(tensors, names, shard, &mut metadata)?;
931                }
932                ResolvedWeight::Owned(bytes) => {
933                    let tensors =
934                        safetensors::SafeTensors::deserialize(&bytes).with_context(|| {
935                            format!("failed to parse required stage shard {}", shard.display())
936                        })?;
937                    Self::record_stage_tensor_metadata(tensors, names, shard, &mut metadata)?;
938                }
939            }
940        }
941        Ok(metadata)
942    }
943
944    fn record_stage_tensor_metadata(
945        tensors: safetensors::SafeTensors<'_>,
946        names: &BTreeSet<String>,
947        shard: &Path,
948        metadata: &mut BTreeMap<String, Vec<usize>>,
949    ) -> Result<()> {
950        for name in names {
951            let view = tensors.tensor(name).with_context(|| {
952                format!(
953                    "authoritative weight_map assigns `{name}` to {}, but the tensor is absent",
954                    shard.display()
955                )
956            })?;
957            if metadata
958                .insert(name.clone(), view.shape().to_vec())
959                .is_some()
960            {
961                bail!("authoritative stage plan contains duplicate tensor `{name}`");
962            }
963        }
964        Ok(())
965    }
966
967    fn stage_metadata_alias<'a>(
968        metadata: &'a BTreeMap<String, Vec<usize>>,
969        aliases: &[&str],
970        description: &str,
971    ) -> Result<(String, &'a [usize])> {
972        aliases
973            .iter()
974            .find_map(|name| {
975                metadata
976                    .get(*name)
977                    .map(|shape| ((*name).to_owned(), shape.as_slice()))
978            })
979            .ok_or_else(|| {
980                anyhow!(
981                    "selected stage metadata is missing required {description}; expected one of {}",
982                    aliases.join(", ")
983                )
984            })
985    }
986
987    fn require_stage_shape(
988        metadata: &BTreeMap<String, Vec<usize>>,
989        name: &str,
990        expected: &[usize],
991    ) -> Result<()> {
992        let actual = metadata.get(name).ok_or_else(|| {
993            anyhow!("selected stage metadata is missing required tensor `{name}`")
994        })?;
995        if actual.as_slice() != expected {
996            bail!(
997                "stage tensor `{name}` shape mismatch: config requires {:?}, checkpoint has {:?}",
998                expected,
999                actual
1000            );
1001        }
1002        Ok(())
1003    }
1004
1005    fn require_optional_stage_shape(
1006        metadata: &BTreeMap<String, Vec<usize>>,
1007        name: &str,
1008        expected: &[usize],
1009    ) -> Result<()> {
1010        if metadata.contains_key(name) {
1011            Self::require_stage_shape(metadata, name, expected)?;
1012        }
1013        Ok(())
1014    }
1015
1016    /// Fail closed when authoritative config dimensions disagree with selected
1017    /// tensor headers. This is stricter than the legacy whole-model loader,
1018    /// which historically reconciles embedding dimensions after loading: stage
1019    /// mode must prove the checkpoint/config pair is coherent before allocating
1020    /// a requested subset.
1021    fn validate_stage_tensor_schema(
1022        config: &ModelConfig,
1023        layer_range: &Range<usize>,
1024        metadata: &BTreeMap<String, Vec<usize>>,
1025    ) -> Result<()> {
1026        Self::validate_stage_range(layer_range, config.num_hidden_layers)?;
1027
1028        let hidden = config.hidden_size;
1029        let intermediate = config.intermediate_size;
1030        let query_width = config
1031            .num_attention_heads
1032            .checked_mul(config.head_dim)
1033            .ok_or_else(|| anyhow!("query projection width overflows usize"))?;
1034        let kv_width = config
1035            .num_key_value_heads
1036            .checked_mul(config.head_dim)
1037            .ok_or_else(|| anyhow!("key/value projection width overflows usize"))?;
1038        if hidden == 0 || intermediate == 0 || query_width == 0 || kv_width == 0 {
1039            bail!(
1040                "stage config contains an invalid zero dimension: hidden={hidden}, \
1041                 intermediate={intermediate}, query_width={query_width}, kv_width={kv_width}"
1042            );
1043        }
1044
1045        let is_first = layer_range.start == 0;
1046        let is_last = layer_range.end == config.num_hidden_layers;
1047        if is_first {
1048            let (name, shape) = Self::stage_metadata_alias(
1049                metadata,
1050                &["model.embed_tokens.weight", "embed_tokens.weight"],
1051                "embedding tensor",
1052            )?;
1053            let expected = [config.vocab_size, hidden];
1054            if shape != expected {
1055                bail!(
1056                    "stage tensor `{name}` shape mismatch: config requires {:?}, checkpoint has {:?}",
1057                    expected,
1058                    shape
1059                );
1060            }
1061        }
1062
1063        if is_last {
1064            let (norm_name, norm_shape) = Self::stage_metadata_alias(
1065                metadata,
1066                &["model.norm.weight", "norm.weight"],
1067                "final norm tensor",
1068            )?;
1069            let expected_norm = [hidden];
1070            if norm_shape != expected_norm {
1071                bail!(
1072                    "stage tensor `{norm_name}` shape mismatch: config requires {:?}, checkpoint has {:?}",
1073                    expected_norm,
1074                    norm_shape
1075                );
1076            }
1077
1078            if let Some((head_name, head_shape)) = ["lm_head.weight", "model.lm_head.weight"]
1079                .into_iter()
1080                .find_map(|name| metadata.get(name).map(|shape| (name, shape.as_slice())))
1081            {
1082                let expected_head = [config.vocab_size, hidden];
1083                if head_shape != expected_head {
1084                    bail!(
1085                        "stage tensor `{head_name}` shape mismatch: config requires {:?}, \
1086                         checkpoint has {:?}",
1087                        expected_head,
1088                        head_shape
1089                    );
1090                }
1091            } else if !is_first {
1092                bail!("a non-first final stage requires an explicit lm_head tensor");
1093            }
1094            // A complete-range stage without an explicit head ties the already
1095            // validated embedding, so its [vocab, hidden] contract is identical.
1096        }
1097
1098        for layer in layer_range.clone() {
1099            let prefix = format!("model.layers.{layer}");
1100            let q_proj = format!("{prefix}.self_attn.q_proj.weight");
1101            let c_attn = format!("{prefix}.self_attn.c_attn.weight");
1102            if metadata.contains_key(&q_proj) {
1103                let k_proj = format!("{prefix}.self_attn.k_proj.weight");
1104                let v_proj = format!("{prefix}.self_attn.v_proj.weight");
1105                Self::require_stage_shape(metadata, &q_proj, &[query_width, hidden])?;
1106                Self::require_stage_shape(metadata, &k_proj, &[kv_width, hidden])?;
1107                Self::require_stage_shape(metadata, &v_proj, &[kv_width, hidden])?;
1108                Self::require_optional_stage_shape(
1109                    metadata,
1110                    &format!("{prefix}.self_attn.q_proj.bias"),
1111                    &[query_width],
1112                )?;
1113                Self::require_optional_stage_shape(
1114                    metadata,
1115                    &format!("{prefix}.self_attn.k_proj.bias"),
1116                    &[kv_width],
1117                )?;
1118                Self::require_optional_stage_shape(
1119                    metadata,
1120                    &format!("{prefix}.self_attn.v_proj.bias"),
1121                    &[kv_width],
1122                )?;
1123            } else if metadata.contains_key(&c_attn) {
1124                if config.num_key_value_heads != config.num_attention_heads {
1125                    bail!(
1126                        "stage layer {layer} uses combined c_attn with grouped-query config \
1127                         (attention heads {}, key/value heads {}), which the dense Llama \
1128                         constructor cannot represent",
1129                        config.num_attention_heads,
1130                        config.num_key_value_heads
1131                    );
1132                }
1133                let combined_width = query_width
1134                    .checked_mul(3)
1135                    .ok_or_else(|| anyhow!("combined attention width overflows usize"))?;
1136                Self::require_stage_shape(metadata, &c_attn, &[combined_width, hidden])?;
1137                Self::require_optional_stage_shape(
1138                    metadata,
1139                    &format!("{prefix}.self_attn.c_attn.bias"),
1140                    &[combined_width],
1141                )?;
1142            } else {
1143                bail!(
1144                    "selected stage metadata is missing required attention projection for layer \
1145                     {layer}; expected `{q_proj}` or `{c_attn}`"
1146                );
1147            }
1148
1149            let o_proj = format!("{prefix}.self_attn.o_proj.weight");
1150            let c_proj = format!("{prefix}.self_attn.c_proj.weight");
1151            let output_name = if metadata.contains_key(&o_proj) {
1152                o_proj.as_str()
1153            } else {
1154                c_proj.as_str()
1155            };
1156            Self::require_stage_shape(metadata, output_name, &[hidden, query_width])?;
1157            let output_bias = output_name
1158                .strip_suffix(".weight")
1159                .map(|prefix| format!("{prefix}.bias"))
1160                .unwrap_or_else(|| format!("{output_name}.bias"));
1161            Self::require_optional_stage_shape(metadata, &output_bias, &[hidden])?;
1162
1163            for (suffix, expected) in [
1164                ("mlp.gate_proj.weight", [intermediate, hidden]),
1165                ("mlp.up_proj.weight", [intermediate, hidden]),
1166                ("mlp.down_proj.weight", [hidden, intermediate]),
1167            ] {
1168                Self::require_stage_shape(metadata, &format!("{prefix}.{suffix}"), &expected)?;
1169            }
1170            for (suffix, expected) in [
1171                ("mlp.gate_proj.bias", [intermediate]),
1172                ("mlp.up_proj.bias", [intermediate]),
1173                ("mlp.down_proj.bias", [hidden]),
1174            ] {
1175                Self::require_optional_stage_shape(
1176                    metadata,
1177                    &format!("{prefix}.{suffix}"),
1178                    &expected,
1179                )?;
1180            }
1181            Self::require_stage_shape(
1182                metadata,
1183                &format!("{prefix}.input_layernorm.weight"),
1184                &[hidden],
1185            )?;
1186            Self::require_stage_shape(
1187                metadata,
1188                &format!("{prefix}.post_attention_layernorm.weight"),
1189                &[hidden],
1190            )?;
1191
1192            let q_norm = format!("{prefix}.self_attn.q_norm.weight");
1193            let k_norm = format!("{prefix}.self_attn.k_norm.weight");
1194            if config.use_qk_norm
1195                || metadata.contains_key(&q_norm)
1196                || metadata.contains_key(&k_norm)
1197            {
1198                Self::require_stage_shape(metadata, &q_norm, &[config.head_dim])?;
1199                Self::require_stage_shape(metadata, &k_norm, &[config.head_dim])?;
1200            }
1201        }
1202
1203        Ok(())
1204    }
1205
1206    #[cfg(test)]
1207    async fn load_weights_for_stage(
1208        model_path: &Path,
1209        num_layers: usize,
1210        layer_range: Range<usize>,
1211        device: &Device,
1212        dtype: DType,
1213    ) -> Result<HashMap<String, Tensor>> {
1214        let plan = Self::stage_weight_plan(model_path, num_layers, layer_range.clone())?;
1215        Self::load_weights_for_stage_plan(
1216            plan,
1217            layer_range,
1218            device,
1219            dtype,
1220            native_finite_policy(model_path),
1221        )
1222        .await
1223    }
1224
1225    async fn load_weights_for_stage_plan(
1226        plan: BTreeMap<PathBuf, BTreeSet<String>>,
1227        layer_range: Range<usize>,
1228        device: &Device,
1229        dtype: DType,
1230        native_finite: bool,
1231    ) -> Result<HashMap<String, Tensor>> {
1232        let planned_tensors: usize = plan.values().map(BTreeSet::len).sum();
1233        info!(
1234            "Loading stage {:?}: {} exact tensor(s) from {} shard(s)",
1235            layer_range,
1236            planned_tensors,
1237            plan.len()
1238        );
1239
1240        let mut weights = HashMap::with_capacity(planned_tensors);
1241        for (shard, names) in plan {
1242            Self::load_safetensors_file_selected(
1243                &shard,
1244                &names,
1245                &mut weights,
1246                device,
1247                dtype,
1248                native_finite,
1249            )
1250            .await?;
1251        }
1252
1253        if weights.len() != planned_tensors {
1254            bail!(
1255                "stage loader planned {planned_tensors} tensors but materialized {}",
1256                weights.len()
1257            );
1258        }
1259        let loaded_bytes: usize = weights
1260            .values()
1261            .map(|tensor| tensor.numel() * tensor.kind().elt_size_in_bytes())
1262            .sum();
1263        info!(
1264            "Loaded authoritative stage {:?}: {} tensor(s), {} byte(s)",
1265            layer_range,
1266            weights.len(),
1267            loaded_bytes
1268        );
1269        Ok(weights)
1270    }
1271
1272    async fn mtp_config_policy(model_path: &Path) -> Result<bool> {
1273        match tokio::fs::read(model_path.join("config.json")).await {
1274            Ok(bytes) => Ok(mtp_config_supported(&serde_json::from_slice(&bytes)?)),
1275            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(true),
1276            Err(e) => Err(e.into()),
1277        }
1278    }
1279
1280    /// Preflight all shard headers before materializing any MTP tensor. The
1281    /// same resolved bytes/LFS handling as normal loading are used locally.
1282    /// Channel preflight and materialization share exactly one resolution per
1283    /// shard. File-backed staging remains mmap-backed. Each pointer-owned
1284    /// buffer is verified/spilled to anonymous private disk backing before the
1285    /// next resolution, never retained as a whole-checkpoint heap or resolved
1286    /// a second time. The same file serves preflight and tensor loading.
1287    async fn prepare_channel_inputs<F, Fut>(
1288        profile: &super::channel_fp8::ChannelFp8Profile,
1289        shards: &[PathBuf],
1290        mut resolve: F,
1291    ) -> Result<Vec<(PathBuf, ResolvedWeight)>>
1292    where
1293        F: FnMut(PathBuf) -> Fut,
1294        Fut: std::future::Future<Output = Result<ResolvedWeight>>,
1295    {
1296        Self::prepare_channel_inputs_spooling(profile, shards, &mut resolve, |bytes| async move {
1297            tokio::task::spawn_blocking(move || super::resolved_spool::spool(bytes))
1298                .await
1299                .context("resolved FP8 spool task failed")?
1300        })
1301        .await
1302    }
1303
1304    async fn prepare_channel_inputs_spooling<F, Fut, S, SFut>(
1305        profile: &super::channel_fp8::ChannelFp8Profile,
1306        shards: &[PathBuf],
1307        mut resolve: F,
1308        mut spool: S,
1309    ) -> Result<Vec<(PathBuf, ResolvedWeight)>>
1310    where
1311        F: FnMut(PathBuf) -> Fut,
1312        Fut: std::future::Future<Output = Result<ResolvedWeight>>,
1313        S: FnMut(Vec<u8>) -> SFut,
1314        SFut: std::future::Future<Output = Result<std::fs::File>>,
1315    {
1316        let mut inputs = Vec::with_capacity(shards.len());
1317        for path in shards {
1318            let resolved = match resolve(path.clone()).await? {
1319                ResolvedWeight::Owned(bytes) => ResolvedWeight::File(spool(bytes).await?),
1320                file @ ResolvedWeight::File(_) => file,
1321            };
1322            inputs.push((path.clone(), resolved));
1323        }
1324        profile.preflight_resolved(&mut inputs)?;
1325        Ok(inputs)
1326    }
1327
1328    async fn mtp_loading_policy_resolved(
1329        model_path: &Path,
1330        inputs: &mut [(PathBuf, ResolvedWeight)],
1331        requested: bool,
1332    ) -> Result<bool> {
1333        if !requested || !Self::mtp_config_policy(model_path).await? {
1334            return Ok(false);
1335        }
1336        for (_, input) in inputs {
1337            let supported = input.with_reader(|reader| {
1338                let mut prefix = [0; 8];
1339                reader.read_exact(&mut prefix)?;
1340                let mut header = vec![0; mtp_header_len(&prefix)?];
1341                reader.read_exact(&mut header)?;
1342                mtp_header_supported(&header)
1343            })?;
1344            if !supported {
1345                return Ok(false);
1346            }
1347        }
1348        Ok(true)
1349    }
1350
1351    async fn mtp_loading_policy(
1352        model_path: &Path,
1353        shards: &[PathBuf],
1354        requested: bool,
1355    ) -> Result<bool> {
1356        if !requested {
1357            return Ok(false);
1358        }
1359        if !Self::mtp_config_policy(model_path).await? {
1360            return Ok(false);
1361        }
1362        for shard in shards {
1363            let resolved = Self::resolve_weight_for_digest(shard).await?;
1364            let supported = tokio::task::spawn_blocking(move || -> Result<bool> {
1365                use std::io::Read;
1366                let mut reader: Box<dyn Read> = match resolved {
1367                    ResolvedWeight::File(file) => Box::new(file),
1368                    ResolvedWeight::Owned(bytes) => Box::new(std::io::Cursor::new(bytes)),
1369                };
1370                let mut prefix = [0; 8];
1371                reader.read_exact(&mut prefix)?;
1372                let mut header = vec![0; mtp_header_len(&prefix)?];
1373                reader.read_exact(&mut header)?;
1374                mtp_header_supported(&header)
1375            })
1376            .await??;
1377            if !supported {
1378                return Ok(false);
1379            }
1380        }
1381        Ok(true)
1382    }
1383
1384    async fn mtp_loading_policy_fs(
1385        model_path: &Path,
1386        fs: &WorktreeClient,
1387        shards: &[String],
1388        requested: bool,
1389    ) -> Result<bool> {
1390        if !requested {
1391            return Ok(false);
1392        }
1393        // Config loading already uses model_path on this entry point. Keep
1394        // that same config authority while reading weight metadata via FsOps.
1395        if !Self::mtp_config_policy(model_path).await? {
1396            return Ok(false);
1397        }
1398        for shard in shards {
1399            let prefix = fs.read_file_prefix(shard, 8).await?;
1400            let len = mtp_header_len(&prefix)?;
1401            let bytes = fs.read_file_prefix(shard, 8 + len).await?;
1402            anyhow::ensure!(bytes.len() == 8 + len, "truncated safetensors metadata");
1403            if !mtp_header_supported(&bytes[8..])? {
1404                return Ok(false);
1405            }
1406        }
1407        Ok(true)
1408    }
1409
1410    /// Create model using incremental loading for large sharded models
1411    #[instrument(name = "model_factory.create_incremental", skip(device, dtype, shard_files, device_pool), fields(shard_count = shard_files.len()))]
1412    async fn create_incremental(
1413        model_path: &Path,
1414        device: &Device,
1415        dtype: DType,
1416        shard_files: Vec<std::path::PathBuf>,
1417        max_context: Option<usize>,
1418        kv_quant_type: KVQuantType,
1419        device_pool: Option<&DevicePool>,
1420        fp8_dequant_load: bool,
1421        speculative_decoding: bool,
1422    ) -> Result<Box<dyn ModelOperations>> {
1423        // For now, we still need to load all weights, but we do it more efficiently
1424        // by processing shards sequentially and immediately transferring to GPU
1425        info!("Loading {} weight shards", shard_files.len());
1426
1427        let native_finite = native_finite_policy(model_path);
1428        let mut all_weights = HashMap::new();
1429
1430        for (idx, shard_file) in shard_files.iter().enumerate() {
1431            info!("Loading shard {}/{}", idx + 1, shard_files.len());
1432
1433            // Load shard weights directly to GPU to minimize CPU memory usage
1434            Self::load_safetensors_file(
1435                shard_file,
1436                &mut all_weights,
1437                device,
1438                dtype,
1439                speculative_decoding,
1440                native_finite,
1441            )
1442            .await?;
1443
1444            // Note: In a true streaming implementation, we would:
1445            // 1. Load layer weights
1446            // 2. Create that layer on GPU
1447            // 3. Free CPU memory before loading next layer
1448            // But this requires refactoring model architectures
1449        }
1450
1451        // Load config and create model
1452        let config = ModelConfig::load(model_path, &all_weights)?;
1453        let model = Self::create_model_from_config(
1454            config,
1455            all_weights,
1456            device,
1457            dtype,
1458            max_context,
1459            kv_quant_type,
1460            model_path,
1461            device_pool,
1462            fp8_dequant_load,
1463            speculative_decoding,
1464        )?;
1465
1466        info!("Model loaded");
1467        Ok(model)
1468    }
1469
1470    /// Load weights from safetensors files
1471    async fn load_weights(
1472        model_path: &Path,
1473        device: &Device,
1474        dtype: DType,
1475        speculative_decoding: bool,
1476    ) -> Result<HashMap<String, Tensor>> {
1477        let native_finite = native_finite_policy(model_path);
1478        let mut all_weights = HashMap::new();
1479
1480        let single_file = model_path.join("model.safetensors");
1481        if single_file.exists() {
1482            info!("Loading model.safetensors");
1483            Self::load_safetensors_file(
1484                &single_file,
1485                &mut all_weights,
1486                device,
1487                dtype,
1488                speculative_decoding,
1489                native_finite,
1490            )
1491            .await?;
1492            return Ok(all_weights);
1493        }
1494
1495        // Look for sharded safetensors files — prefer index file, then glob patterns
1496        let model_path_buf = model_path.to_path_buf();
1497        let mut shard_files =
1498            tokio::task::spawn_blocking(move || -> Result<Vec<std::path::PathBuf>> {
1499                // Use index if present
1500                let index_path = model_path_buf.join("model.safetensors.index.json");
1501                if index_path.exists() {
1502                    return Self::shard_files_from_index(&model_path_buf, &index_path);
1503                }
1504                // Fallback glob (loud / strict-gated)
1505                glob_shard_files(&model_path_buf)
1506            })
1507            .await??;
1508
1509        if !shard_files.is_empty() {
1510            shard_files.sort();
1511            info!("Loading {} weight shards", shard_files.len());
1512            for shard_file in shard_files {
1513                Self::load_safetensors_file(
1514                    &shard_file,
1515                    &mut all_weights,
1516                    device,
1517                    dtype,
1518                    speculative_decoding,
1519                    native_finite,
1520                )
1521                .await?;
1522            }
1523            return Ok(all_weights);
1524        }
1525
1526        Err(anyhow!(
1527            "No safetensors files found in {}",
1528            model_path.display()
1529        ))
1530    }
1531
1532    /// Load a single safetensors file
1533    #[instrument(name = "model_factory.load_safetensor_file", skip(weights, device, dtype), fields(file = %path.display()))]
1534    async fn load_safetensors_file(
1535        path: &Path,
1536        weights: &mut HashMap<String, Tensor>,
1537        device: &Device,
1538        dtype: DType,
1539        speculative_decoding: bool,
1540        native_finite: bool,
1541    ) -> Result<()> {
1542        Self::load_safetensors_file_inner(
1543            path,
1544            None,
1545            weights,
1546            device,
1547            dtype,
1548            speculative_decoding,
1549            native_finite,
1550        )
1551        .await
1552    }
1553
1554    /// Load only `selected` tensor names from one safetensors shard.
1555    ///
1556    /// Selection happens before `TensorView::data()` and before any `tch::Tensor`
1557    /// construction, so an unselected tensor is never materialized even when it
1558    /// shares a physical shard with selected tensors.
1559    async fn load_safetensors_file_selected(
1560        path: &Path,
1561        selected: &BTreeSet<String>,
1562        weights: &mut HashMap<String, Tensor>,
1563        device: &Device,
1564        dtype: DType,
1565        native_finite: bool,
1566    ) -> Result<()> {
1567        Self::load_safetensors_file_inner(
1568            path,
1569            Some(selected),
1570            weights,
1571            device,
1572            dtype,
1573            false,
1574            native_finite,
1575        )
1576        .await
1577    }
1578
1579    async fn load_safetensors_file_inner(
1580        path: &Path,
1581        selected: Option<&BTreeSet<String>>,
1582        weights: &mut HashMap<String, Tensor>,
1583        device: &Device,
1584        dtype: DType,
1585        speculative_decoding: bool,
1586        native_finite: bool,
1587    ) -> Result<()> {
1588        let resolved = Self::resolve_weight_for_digest(path).await?;
1589        Self::load_resolved_safetensors_file_inner(
1590            path,
1591            resolved,
1592            selected,
1593            weights,
1594            device,
1595            dtype,
1596            speculative_decoding,
1597            native_finite,
1598            false,
1599        )
1600        .await
1601    }
1602
1603    async fn load_resolved_safetensors_file_inner(
1604        path: &Path,
1605        resolved: ResolvedWeight,
1606        selected: Option<&BTreeSet<String>>,
1607        weights: &mut HashMap<String, Tensor>,
1608        device: &Device,
1609        dtype: DType,
1610        speculative_decoding: bool,
1611        native_finite: bool,
1612        force_mmap: bool,
1613    ) -> Result<()> {
1614        // Stage selection always uses mmap: only selected tensor pages are copied
1615        // into owned CPU/device storage, so unrelated bytes in a shared shard do
1616        // not become a whole-shard heap allocation. Whole-model loading retains
1617        // its existing environment-controlled behavior.
1618        //
1619        // #315 note: for whole-model mode this env var is independent of
1620        // `RuntimeConfig.mmap`, which today gates nothing here.
1621        // FP8_DYNAMIC is a bounded-memory CPU path: never add a full-shard
1622        // anonymous read buffer on top of the resident packed weights.
1623        let channel_mmap = force_mmap
1624            || path
1625                .parent()
1626                .map(super::channel_fp8::ChannelFp8Profile::read)
1627                .transpose()?
1628                .flatten()
1629                .is_some();
1630        let use_mmap = selected.is_some()
1631            || channel_mmap
1632            || std::env::var("HYPRSTREAM_USE_MMAP")
1633                .map(|v| v.to_lowercase() == "true" || v == "1")
1634                .unwrap_or(false);
1635        debug!(
1636            "safetensors load path: {}",
1637            if use_mmap { "mmap" } else { "read" }
1638        );
1639
1640        // Load file data in a blocking task to avoid blocking the async runtime
1641        let path_buf = path.to_path_buf();
1642
1643        // Resolve before deciding whether mmap is safe. A stage selects only a
1644        // few tensors, but a pointer shard must be resolved before safetensors
1645        // sees it; mmap is allowed only for an already-resolved ordinary file.
1646        match resolved {
1647            ResolvedWeight::File(file) if use_mmap => {
1648                use memmap2::Mmap;
1649
1650                let mmap = unsafe { Mmap::map(&file)? };
1651                let tensors = safetensors::SafeTensors::deserialize(&mmap)?;
1652                Self::create_tensors_from_safetensors_selected(
1653                    tensors,
1654                    selected,
1655                    weights,
1656                    device,
1657                    dtype,
1658                    speculative_decoding,
1659                    native_finite,
1660                )
1661            }
1662            ResolvedWeight::File(_) => {
1663                let tensor_data = Self::load_file_with_pointer_detection(&path_buf).await?;
1664                let tensors = safetensors::SafeTensors::deserialize(&tensor_data)?;
1665                Self::create_tensors_from_safetensors_selected(
1666                    tensors,
1667                    selected,
1668                    weights,
1669                    device,
1670                    dtype,
1671                    speculative_decoding,
1672                    native_finite,
1673                )
1674            }
1675            ResolvedWeight::Owned(tensor_data) => {
1676                let tensors = safetensors::SafeTensors::deserialize(&tensor_data)?;
1677                Self::create_tensors_from_safetensors_selected(
1678                    tensors,
1679                    selected,
1680                    weights,
1681                    device,
1682                    dtype,
1683                    speculative_decoding,
1684                    native_finite,
1685                )
1686            }
1687        }
1688    }
1689
1690    /// Load file with automatic LFS/XET pointer detection
1691    ///
1692    /// Fast path for already-smudged files, fallback for un-smudged pointers.
1693    /// Resolution is shared with the KV-compat base-weight digest via
1694    /// [`Self::try_resolve_lfs_pointer`] so both see the identical bytes (#1277).
1695    async fn load_file_with_pointer_detection(path: &Path) -> Result<Vec<u8>> {
1696        let metadata = tokio::fs::metadata(path).await?;
1697
1698        // Large files cannot be LFS pointers (which are < 1KB)
1699        if metadata.len() >= 1024 {
1700            return tokio::fs::read(path).await.map_err(Into::into);
1701        }
1702
1703        let data = tokio::fs::read(path).await?;
1704        if let Some(resolved) = Self::try_resolve_lfs_pointer(path, &data).await? {
1705            return Ok(resolved);
1706        }
1707        Ok(data)
1708    }
1709
1710    /// If `data` is an un-smudged LFS/XET pointer, resolve it to the actual
1711    /// object content via the git2db LFS fallback. Returns `Ok(None)` when
1712    /// `data` is ordinary content (no pointer header, or too large to be one).
1713    /// Any resolution failure propagates as an error — **fail-closed**: the
1714    /// caller (loader *and* the KV-compat digest) never silently hashes or
1715    /// deserializes a pointer stub in place of the resolved bytes.
1716    ///
1717    /// Single source of truth for pointer resolution: extracted from the old
1718    /// inline block in `load_file_with_pointer_detection`.
1719    async fn try_resolve_lfs_pointer(path: &Path, data: &[u8]) -> Result<Option<Vec<u8>>> {
1720        // Large buffers cannot be LFS pointers (which are < 1 KiB).
1721        if data.len() >= 1024 {
1722            return Ok(None);
1723        }
1724        let text = match std::str::from_utf8(data) {
1725            Ok(t) => t,
1726            Err(_) => return Ok(None),
1727        };
1728        if !text.starts_with("version https://git-lfs")
1729            && !text.starts_with("version https://hawser")
1730        {
1731            return Ok(None);
1732        }
1733        #[cfg(feature = "xet")]
1734        {
1735            debug!(
1736                "Un-smudged LFS pointer, resolving via git2db::lfs fallback: {}",
1737                path.display()
1738            );
1739            let config = git2db::XetConfig::default();
1740            let storage = git2db::LfsStorage::new(&config)
1741                .await
1742                .map_err(|e| anyhow!("Failed to create LfsStorage: {}", e))?;
1743            let bytes = storage
1744                .load_file(path)
1745                .await
1746                .map_err(|e| anyhow!("Failed to load LFS file: {}", e))?;
1747            Ok(Some(bytes))
1748        }
1749        #[cfg(not(feature = "xet"))]
1750        {
1751            let _ = path;
1752            anyhow::bail!(
1753                "Un-smudged LFS pointer at {} but XET feature disabled. \
1754                 Enable with --features xet or ensure files are smudged during checkout.",
1755                path.display()
1756            )
1757        }
1758    }
1759
1760    /// Resolve a weight shard for the KV-compat base-weight digest (#1277).
1761    ///
1762    /// Returns the shard's resolved content: a large ordinary shard as an open
1763    /// [`File`] (to stream), and a resolved pointer or any small file as owned
1764    /// bytes. This walks the **same** resolution path as the loader
1765    /// ([`Self::load_file_with_pointer_detection`] → [`Self::try_resolve_lfs_pointer`]),
1766    /// so the digest hashes the actual tensor bytes — never a pointer stub — and
1767    /// fails closed (returns `Err`) on any metadata/read/resolve error.
1768    pub(crate) async fn resolve_weight_for_digest(path: &Path) -> Result<ResolvedWeight> {
1769        let metadata = tokio::fs::metadata(path).await?;
1770        // A large file cannot be an LFS pointer (pointers are < 1 KiB): its
1771        // on-disk bytes are the resolved content — stream it, don't buffer it.
1772        if metadata.len() >= 1024 {
1773            let file = std::fs::File::open(path)?;
1774            return Ok(ResolvedWeight::File(file));
1775        }
1776        let data = tokio::fs::read(path).await?;
1777        if let Some(resolved) = Self::try_resolve_lfs_pointer(path, &data).await? {
1778            return Ok(ResolvedWeight::Owned(resolved));
1779        }
1780        Ok(ResolvedWeight::Owned(data))
1781    }
1782
1783    /// Create tensors from deserialized safetensors
1784    #[cfg(test)]
1785    fn create_tensors_from_safetensors(
1786        tensors: safetensors::SafeTensors,
1787        weights: &mut HashMap<String, Tensor>,
1788        device: &Device,
1789        dtype: DType,
1790        speculative_decoding: bool,
1791    ) -> Result<()> {
1792        Self::create_tensors_from_safetensors_selected(
1793            tensors,
1794            None,
1795            weights,
1796            device,
1797            dtype,
1798            speculative_decoding,
1799            false,
1800        )
1801    }
1802
1803    fn create_tensors_from_safetensors_selected(
1804        tensors: safetensors::SafeTensors,
1805        selected: Option<&BTreeSet<String>>,
1806        weights: &mut HashMap<String, Tensor>,
1807        device: &Device,
1808        dtype: DType,
1809        speculative_decoding: bool,
1810        native_finite: bool,
1811    ) -> Result<()> {
1812        let mut total_size_mb = 0.0;
1813        let tensors_list = if let Some(selected) = selected {
1814            selected
1815                .iter()
1816                .map(|name| {
1817                    tensors
1818                        .tensor(name)
1819                        .map(|view| (name.clone(), view))
1820                        .with_context(|| {
1821                            format!(
1822                                "authoritative weight_map assigns `{name}` to this shard, \
1823                                 but the tensor is absent"
1824                            )
1825                        })
1826                })
1827                .collect::<Result<Vec<_>>>()?
1828        } else {
1829            tensors.tensors()
1830        };
1831        let tensor_count = tensors_list.len();
1832
1833        info!("Loading {} tensors to device {:?}", tensor_count, device);
1834
1835        for (idx, (name, tensor_view)) in tensors_list.into_iter().enumerate() {
1836            // Runtime policy reaches every file/FsOps path. Skip the draft head
1837            // before reading tensor bytes or constructing CPU/device tensors,
1838            // including when MTP and decoder weights share the same shard.
1839            if !speculative_decoding && name.starts_with("mtp.") {
1840                continue;
1841            }
1842            let shape: Vec<i64> = tensor_view.shape().iter().map(|&x| x as i64).collect();
1843            let data = tensor_view.data();
1844            if native_finite {
1845                // All admitted payloads are scanned, including scales and any
1846                // vision bytes already loaded by this architecture. This is byte
1847                // validation, not a claim of vision inference support. Selection
1848                // and disabled-MTP exclusion have already happened above.
1849                validate_finite_payload(&name, tensor_view.dtype(), data)?;
1850            }
1851
1852            // Map safetensors dtype to tch::Kind for size estimation
1853            let tensor_kind = match tensor_view.dtype() {
1854                safetensors::Dtype::BF16 => tch::Kind::BFloat16,
1855                safetensors::Dtype::F16 => tch::Kind::Half,
1856                safetensors::Dtype::F64 => tch::Kind::Double,
1857                safetensors::Dtype::F8_E4M3 => tch::Kind::Float8e4m3fn,
1858                safetensors::Dtype::F8_E5M2 => tch::Kind::Float8e5m2,
1859                // F32 and other types default to Float
1860                _ => tch::Kind::Float,
1861            };
1862
1863            // Scales retain their stored precision, independent of compute dtype.
1864            let dtype = if name.ends_with(".weight_scale") || name.ends_with(".weight_scale_inv") {
1865                tensor_kind
1866            } else {
1867                dtype
1868            };
1869
1870            // Calculate tensor size for progress reporting
1871            let tensor_size_mb = estimate_tensor_size_mb(&shape, tensor_kind);
1872            total_size_mb += tensor_size_mb;
1873
1874            // Support both F16 and BF16 models
1875            let tensor = match tensor_view.dtype() {
1876                safetensors::Dtype::BF16 => {
1877                    // Verify target dtype matches
1878                    if !matches!(
1879                        dtype,
1880                        tch::Kind::BFloat16 | tch::Kind::Half | tch::Kind::Float
1881                    ) {
1882                        return Err(anyhow!("Model dtype BF16 but target dtype is {:?}", dtype));
1883                    }
1884
1885                    // Create tensor from borrowed data, then make an owned copy
1886                    // IMPORTANT: from_blob only borrows the data pointer, so we must
1887                    // copy the tensor to own the data before the source buffer is freed
1888                    let borrowed_tensor = unsafe {
1889                        Tensor::from_blob(
1890                            data.as_ptr(),
1891                            &shape,
1892                            &[],
1893                            tch::Kind::BFloat16,
1894                            Device::Cpu,
1895                        )
1896                    };
1897
1898                    // Make an owned copy to prevent use-after-free
1899                    let cpu_tensor = borrowed_tensor.copy();
1900
1901                    // Convert dtype if needed
1902                    let cpu_tensor = if dtype != tch::Kind::BFloat16 {
1903                        cpu_tensor.to_kind(dtype)
1904                    } else {
1905                        cpu_tensor
1906                    };
1907
1908                    // Transfer to target device if needed (with OOM handling)
1909                    if *device != Device::Cpu {
1910                        let gpu_tensor = cpu_tensor.to_device(*device);
1911                        drop(cpu_tensor); // Explicitly free CPU memory
1912                        gpu_tensor
1913                    } else {
1914                        cpu_tensor
1915                    }
1916                }
1917                safetensors::Dtype::F16 => {
1918                    // Verify target dtype matches
1919                    if !matches!(
1920                        dtype,
1921                        tch::Kind::BFloat16 | tch::Kind::Half | tch::Kind::Float
1922                    ) {
1923                        return Err(anyhow!("Model dtype F16 but target dtype is {:?}", dtype));
1924                    }
1925
1926                    // Create tensor from borrowed data
1927                    let borrowed_tensor = unsafe {
1928                        Tensor::from_blob(data.as_ptr(), &shape, &[], tch::Kind::Half, Device::Cpu)
1929                    };
1930
1931                    // Make an owned copy to prevent use-after-free
1932                    let cpu_tensor = borrowed_tensor.copy();
1933
1934                    // Convert dtype if needed
1935                    let cpu_tensor = if dtype != tch::Kind::Half {
1936                        cpu_tensor.to_kind(dtype)
1937                    } else {
1938                        cpu_tensor
1939                    };
1940
1941                    // Transfer to target device if needed (with OOM handling)
1942                    if *device != Device::Cpu {
1943                        let gpu_tensor = cpu_tensor.to_device(*device);
1944                        drop(cpu_tensor); // Explicitly free CPU memory
1945                        gpu_tensor
1946                    } else {
1947                        cpu_tensor
1948                    }
1949                }
1950                safetensors::Dtype::F32 => {
1951                    // Support F32 as well for completeness
1952                    let borrowed_tensor = unsafe {
1953                        Tensor::from_blob(data.as_ptr(), &shape, &[], tch::Kind::Float, Device::Cpu)
1954                    };
1955
1956                    // Make an owned copy
1957                    let cpu_tensor = borrowed_tensor.copy();
1958
1959                    // Convert to target dtype
1960                    let cpu_tensor = match dtype {
1961                        tch::Kind::Half => cpu_tensor.to_kind(tch::Kind::Half),
1962                        tch::Kind::BFloat16 => cpu_tensor.to_kind(tch::Kind::BFloat16),
1963                        tch::Kind::Float => cpu_tensor,
1964                        _ => return Err(anyhow!("Cannot convert F32 to target dtype {:?}", dtype)),
1965                    };
1966
1967                    // Transfer to target device if needed (with OOM handling)
1968                    if *device != Device::Cpu {
1969                        // Log progress for large models
1970                        if idx % 100 == 0 || tensor_size_mb > 10.0 {
1971                            info!(
1972                                "Transferring tensor {}/{} to GPU: {} ({:.2} MB, total: {:.2} MB)",
1973                                idx + 1,
1974                                tensor_count,
1975                                name,
1976                                tensor_size_mb,
1977                                total_size_mb
1978                            );
1979                        }
1980
1981                        // Use safe wrapper to catch OOM panics
1982                        match safe_to_device(&cpu_tensor, *device) {
1983                            Ok(gpu_tensor) => {
1984                                drop(cpu_tensor); // Explicitly free CPU memory
1985                                gpu_tensor
1986                            }
1987                            Err(e) => {
1988                                drop(cpu_tensor); // CRITICAL: Free CPU memory before returning error
1989                                return Err(anyhow!(
1990                                    "GPU OOM loading tensor '{}': {} | Progress: {}/{} ({:.1} MB) | Try: smaller model, reduce max_position_embeddings, or free GPU memory",
1991                                    name, e, idx + 1, tensor_count, total_size_mb
1992                                ));
1993                            }
1994                        }
1995                    } else {
1996                        cpu_tensor
1997                    }
1998                }
1999                // FP8 formats: keep as FP8 in VRAM. ROCm 7.1+ dispatches FP8→BF16 on-GPU.
2000                // Block-wise scale tensors (weight_scale_inv) are loaded alongside and stored
2001                // in LinearProjection.scale for correct dequantization at matmul time.
2002                safetensors::Dtype::F8_E4M3 => {
2003                    let borrowed_tensor = unsafe {
2004                        Tensor::from_blob(
2005                            data.as_ptr(),
2006                            &shape,
2007                            &[],
2008                            tch::Kind::Float8e4m3fn,
2009                            Device::Cpu,
2010                        )
2011                    };
2012                    let cpu_tensor = borrowed_tensor.copy();
2013                    if *device != Device::Cpu {
2014                        cpu_tensor.to_device(*device)
2015                    } else {
2016                        cpu_tensor
2017                    }
2018                }
2019                safetensors::Dtype::F8_E5M2 => {
2020                    let borrowed_tensor = unsafe {
2021                        Tensor::from_blob(
2022                            data.as_ptr(),
2023                            &shape,
2024                            &[],
2025                            tch::Kind::Float8e5m2,
2026                            Device::Cpu,
2027                        )
2028                    };
2029                    let cpu_tensor = borrowed_tensor.copy();
2030                    if *device != Device::Cpu {
2031                        cpu_tensor.to_device(*device)
2032                    } else {
2033                        cpu_tensor
2034                    }
2035                }
2036                dtype => {
2037                    return Err(anyhow!(
2038                        "Tensor '{}' has unsupported dtype {:?}. Supported: F16, BF16, F32, F8_E4M3, F8_E5M2",
2039                        name, dtype
2040                    ));
2041                }
2042            };
2043
2044            weights.insert(name.clone(), tensor);
2045        }
2046
2047        info!(
2048            "✅ Successfully loaded {} tensors ({:.2} MB total) to device {:?}",
2049            tensor_count, total_size_mb, device
2050        );
2051
2052        Ok(())
2053    }
2054
2055    /// Create model instance from configuration
2056    fn create_model_from_config(
2057        config: ModelConfig,
2058        mut weights: HashMap<String, Tensor>,
2059        device: &Device,
2060        dtype: DType,
2061        max_context: Option<usize>,
2062        kv_quant_type: KVQuantType,
2063        model_path: &Path,
2064        device_pool: Option<&DevicePool>,
2065        fp8_dequant_load: bool,
2066        speculative_decoding: bool,
2067    ) -> Result<Box<dyn ModelOperations>> {
2068        if let Some(profile) = super::channel_fp8::ChannelFp8Profile::read(model_path)? {
2069            anyhow::ensure!(
2070                !fp8_dequant_load && *device == Device::Cpu,
2071                "unsupported FP8_DYNAMIC execution policy"
2072            );
2073            profile.validate(&weights)?;
2074        } else {
2075            anyhow::ensure!(
2076                !weights.keys().any(|k| k.ends_with(".weight_scale")),
2077                "channel FP8 weights require authoritative compressed-tensors config"
2078            );
2079        }
2080        // Run TTN analysis: Tier 1 (embedded) → Tier 2 (cached) → Tier 3 (weight entropy SVD).
2081        // Weights are available here, enabling Tier 3 for unknown models.
2082        // Result cached to .analysis/layer_profile.json; non-fatal if it fails.
2083        // Runs in sync/blocking context (called from spawn_blocking), safe for SVD.
2084        if let Err(e) =
2085            crate::runtime::ttn_profile::get_layer_profile(model_path, &config, Some(&weights))
2086        {
2087            tracing::warn!("TTN profile analysis failed (non-fatal): {e}");
2088        }
2089
2090        // FP8 dequant-once-at-load: materialize every FP8 weight as BF16 with
2091        // its block scale applied, before any architecture constructor consumes
2092        // the map. Downstream FP8 lazy-dequant branches then never trigger —
2093        // the per-matmul dequant disappears from the hot path.
2094        maybe_dequantize_fp8_weights_at_load(&mut weights, fp8_dequant_load, device_pool);
2095
2096        // Multi-device is only wired for Llama-family architectures today (the
2097        // only family with a `stage_from_weights_with_config` that honors a
2098        // `LayerDeviceMap`). Other architectures log and fall back to the
2099        // single-device path on the primary device.
2100        let wants_multi = device_pool.is_some_and(|p| !p.is_single());
2101
2102        match config.architecture {
2103            ModelArchitecture::Llama => {
2104                info!("Creating Llama model");
2105                Self::create_llama_model(
2106                    config,
2107                    weights,
2108                    device,
2109                    dtype,
2110                    max_context,
2111                    kv_quant_type,
2112                    device_pool,
2113                    None,
2114                )
2115            }
2116            ModelArchitecture::Qwen => {
2117                info!("Creating Qwen model");
2118                Self::create_qwen_model(
2119                    config,
2120                    weights,
2121                    device,
2122                    dtype,
2123                    max_context,
2124                    kv_quant_type,
2125                    device_pool,
2126                )
2127            }
2128            ModelArchitecture::Gemma => {
2129                info!("Creating Gemma model");
2130                Self::create_gemma_model(config, weights, device, dtype, max_context, kv_quant_type)
2131            }
2132            ModelArchitecture::Mistral => {
2133                info!("Creating Mistral model");
2134                // For now, Mistral uses Llama architecture
2135                Self::create_llama_model(
2136                    config,
2137                    weights,
2138                    device,
2139                    dtype,
2140                    max_context,
2141                    kv_quant_type,
2142                    device_pool,
2143                    None,
2144                )
2145            }
2146            ModelArchitecture::Janus => {
2147                info!("Creating Janus multimodal model");
2148                Self::create_janus_model(config, weights, device, dtype, max_context, kv_quant_type)
2149            }
2150            ModelArchitecture::Qwen3_5 => {
2151                info!("Creating Qwen3.5 hybrid SSM/attention model");
2152                if wants_multi {
2153                    warn!(
2154                        "Multi-device requested for Qwen3.5, which does not yet implement the \
2155                         layer-split path; building single-device on {:?}",
2156                        device
2157                    );
2158                }
2159                Self::create_qwen3_5_model(
2160                    config,
2161                    weights,
2162                    device,
2163                    dtype,
2164                    max_context,
2165                    kv_quant_type,
2166                    speculative_decoding,
2167                )
2168            }
2169            ModelArchitecture::Unknown(arch) => Err(anyhow!("Unknown architecture: {}", arch)),
2170        }
2171    }
2172
2173    fn create_llama_model(
2174        config: ModelConfig,
2175        mut weights: HashMap<String, Tensor>,
2176        device: &Device,
2177        dtype: DType,
2178        max_context: Option<usize>,
2179        kv_quant_type: KVQuantType,
2180        device_pool: Option<&DevicePool>,
2181        stage_range: Option<Range<usize>>,
2182    ) -> Result<Box<dyn ModelOperations>> {
2183        use super::architectures::llama::LlamaConfig;
2184        use super::device_pool::LayerDeviceMap;
2185
2186        // Apply max_context override if specified
2187        let effective_max_pos = max_context.unwrap_or(config.max_position_embeddings);
2188        if max_context.is_some() {
2189            info!(
2190                "Overriding max_position_embeddings: {} -> {}",
2191                config.max_position_embeddings, effective_max_pos
2192            );
2193        }
2194
2195        // Convert unified config to LlamaConfig
2196        let llama_config = LlamaConfig {
2197            version: config.version as u8,
2198            num_attention_heads: config.num_attention_heads as u32,
2199            num_key_value_heads: config.num_key_value_heads as u32,
2200            hidden_size: config.hidden_size as u32,
2201            head_dim: config.head_dim as u32,
2202            intermediate_size: config.intermediate_size as u32,
2203            max_position_embeddings: effective_max_pos as u32,
2204            rms_norm_eps: config.rms_norm_eps,
2205            vocab_size: config.vocab_size as u32,
2206            original_vocab_size: config.vocab_size as u32, // Will be updated if padding is applied
2207            num_hidden_layers: config.num_hidden_layers as u32,
2208            rope_theta: config.rope_theta,
2209            rope_scaling: None,
2210            hidden_activation: config.hidden_activation,
2211            query_pre_attn_scalar: config.query_pre_attn_scalar,
2212            use_qk_norm: config.use_qk_norm,
2213            scale_embeddings: config.scale_embeddings,
2214            layer_types: vec![],
2215            rope_local_base_freq: None,
2216            // Preserve the *real* architecture so tokenizer-config dispatch (and
2217            // any other architecture-keyed behavior) sees Qwen/Mistral rather
2218            // than the Llama stand-in. Without this, a Qwen3 checkpoint loaded
2219            // via create_qwen_model→create_llama_model reports `Llama`, the
2220            // QwenTokenizerConfig (which pads the tokenizer vocab to the model's
2221            // embedding size) is never applied, and the tokenizer/embedding vocab
2222            // mismatch causes a CUDA index-out-of-bounds on the embedding lookup (#143).
2223            model_architecture: match config.architecture {
2224                super::model_config::ModelArchitecture::Qwen => {
2225                    super::architectures::ModelArchitecture::Qwen {
2226                        version: config.version as u8,
2227                        is_moe: config.is_moe,
2228                        context_length: config.max_position_embeddings,
2229                    }
2230                }
2231                super::model_config::ModelArchitecture::Mistral => {
2232                    super::architectures::ModelArchitecture::Mistral
2233                }
2234                super::model_config::ModelArchitecture::Gemma => {
2235                    super::architectures::ModelArchitecture::Gemma
2236                }
2237                // Llama, Janus, Qwen3_5 (not routed through LlamaModel), and unknowns
2238                // keep the Llama identity with the parsed version.
2239                _ => super::architectures::ModelArchitecture::Llama {
2240                    version: config.version as u8,
2241                },
2242            },
2243        };
2244
2245        info!(
2246            "[create_llama_model] Passing llama_config.max_position_embeddings = {} to from_weights_with_config",
2247            llama_config.max_position_embeddings
2248        );
2249
2250        // Multi-device pipeline (#314 wiring): when a multi-device `DevicePool`
2251        // is present, spread all decoder layers `[0..num_hidden_layers)` across
2252        // the pool via an even (parameter-balanced) split and build the model as
2253        // a single stage owning the full range with that device map. The
2254        // architecture's `forward_layers` then performs the lone cross-device
2255        // copy at each stage boundary. Single-device pools and `None` take the
2256        // original whole-model path unchanged.
2257        let num_layers = llama_config.num_hidden_layers as usize;
2258        let model = if let Some(layer_range) = stage_range {
2259            let device_map = if let Some(pool) = device_pool.filter(|p| !p.is_single()) {
2260                LayerDeviceMap::even_split(pool, num_layers)?
2261            } else {
2262                LayerDeviceMap::single(*device, num_layers)?
2263            };
2264            info!(
2265                "Building authoritative Llama stage {:?} of {} global layer(s)",
2266                layer_range, num_layers
2267            );
2268            LlamaModel::stage_from_weights_with_config(
2269                &mut weights,
2270                llama_config,
2271                &device_map,
2272                layer_range,
2273                dtype,
2274                kv_quant_type,
2275            )?
2276        } else if let Some(pool) = device_pool.filter(|p| !p.is_single()) {
2277            let device_map = LayerDeviceMap::even_split(pool, num_layers)?;
2278            info!(
2279                "🔀 Building Llama model as a multi-device pipeline: {} layer(s) across \
2280                 {} device(s) ({} stage boundary copies per forward)",
2281                num_layers,
2282                pool.len(),
2283                pool.len() - 1,
2284            );
2285            LlamaModel::stage_from_weights_with_config(
2286                &mut weights,
2287                llama_config,
2288                &device_map,
2289                0..num_layers,
2290                dtype,
2291                kv_quant_type,
2292            )?
2293        } else {
2294            // Pass mutable reference to allow incremental tensor freeing during construction
2295            LlamaModel::from_weights_with_config(
2296                &mut weights,
2297                llama_config,
2298                device,
2299                dtype,
2300                kv_quant_type,
2301            )?
2302        };
2303
2304        Ok(Box::new(model))
2305    }
2306
2307    fn create_qwen_model(
2308        config: ModelConfig,
2309        weights: HashMap<String, Tensor>,
2310        device: &Device,
2311        dtype: DType,
2312        max_context: Option<usize>,
2313        kv_quant_type: KVQuantType,
2314        device_pool: Option<&DevicePool>,
2315    ) -> Result<Box<dyn ModelOperations>> {
2316        // Qwen uses Llama architecture with specific configuration
2317        // The key difference is in the config values, not the architecture
2318        info!("   Using Llama architecture with Qwen configuration");
2319        info!("   rope_theta: {} (from config)", config.rope_theta);
2320        Self::create_llama_model(
2321            config,
2322            weights,
2323            device,
2324            dtype,
2325            max_context,
2326            kv_quant_type,
2327            device_pool,
2328            None,
2329        )
2330    }
2331
2332    fn create_gemma_model(
2333        _config: ModelConfig,
2334        weights: HashMap<String, Tensor>,
2335        device: &Device,
2336        dtype: DType,
2337        _max_context: Option<usize>,
2338        _kv_quant_type: KVQuantType,
2339    ) -> Result<Box<dyn ModelOperations>> {
2340        // Gemma has its own implementation
2341        // TODO: Pass max_context and kv_quant_type to GemmaModel when it supports them
2342        Ok(Box::new(GemmaModel::from_weights(&weights, device, dtype)?))
2343    }
2344
2345    fn create_janus_model(
2346        config: ModelConfig,
2347        weights: HashMap<String, Tensor>,
2348        device: &Device,
2349        dtype: DType,
2350        max_context: Option<usize>,
2351        _kv_quant_type: KVQuantType,
2352    ) -> Result<Box<dyn ModelOperations>> {
2353        use super::architectures::janus::{
2354            JanusConfig, JanusModel, ProjectorConfig, VisionEncoderConfig,
2355        };
2356        use super::architectures::VisionEncoderType;
2357
2358        // Apply max_context override if specified
2359        let effective_max_pos = max_context.unwrap_or(config.max_position_embeddings);
2360
2361        // For now, create a simplified Janus config
2362        // In practice, this would be derived from the model's config.json
2363        let janus_config = JanusConfig {
2364            // Use Llama config for the language model
2365            language_config: Box::new(super::architectures::llama::LlamaConfig {
2366                version: 3,
2367                num_attention_heads: config.num_attention_heads as u32,
2368                num_key_value_heads: config.num_key_value_heads as u32,
2369                hidden_size: config.hidden_size as u32,
2370                head_dim: config.head_dim as u32,
2371                intermediate_size: config.intermediate_size as u32,
2372                max_position_embeddings: effective_max_pos as u32,
2373                rms_norm_eps: config.rms_norm_eps,
2374                vocab_size: config.vocab_size as u32,
2375                original_vocab_size: config.vocab_size as u32,
2376                num_hidden_layers: config.num_hidden_layers as u32,
2377                rope_theta: config.rope_theta,
2378                rope_scaling: None, // TODO: Convert from config.rope_scaling
2379                hidden_activation: config.hidden_activation.clone(),
2380                query_pre_attn_scalar: config.query_pre_attn_scalar,
2381                use_qk_norm: config.use_qk_norm,
2382                scale_embeddings: config.scale_embeddings,
2383                layer_types: vec!["global".to_owned(); config.num_hidden_layers],
2384                rope_local_base_freq: None,
2385                model_architecture: super::architectures::ModelArchitecture::Llama { version: 3 },
2386            }),
2387            vision_config: VisionEncoderConfig {
2388                encoder_type: VisionEncoderType::SigLIP {
2389                    hidden_size: 1024, // From config: vision_config.hidden_size
2390                    image_size: 384,
2391                    patch_size: 16,
2392                    num_layers: 24,
2393                },
2394                hidden_size: 1024,
2395                image_size: 384,
2396                patch_size: 16,
2397                num_layers: 24,
2398                num_patches: (384 / 16) * (384 / 16), // 576 patches
2399                num_attention_heads: Some(16), // From config: vision_config.num_attention_heads
2400                intermediate_size: Some(4096), // From config: vision_config.intermediate_size
2401            },
2402            aligner_config: ProjectorConfig {
2403                input_dim: 1024, // Vision hidden size (matches vision_config.hidden_size)
2404                output_dim: config.hidden_size, // Language model hidden size
2405                hidden_dim: Some(config.hidden_size), // 2-layer MLP
2406            },
2407            generation_config: None, // No image generation for now
2408            device: *device,
2409            dtype,
2410        };
2411
2412        Ok(Box::new(JanusModel::from_weights(
2413            weights,
2414            janus_config,
2415            *device,
2416            dtype,
2417        )?))
2418    }
2419
2420    fn create_qwen3_5_model(
2421        config: ModelConfig,
2422        mut weights: HashMap<String, Tensor>,
2423        device: &Device,
2424        dtype: DType,
2425        max_context: Option<usize>,
2426        kv_quant_type: KVQuantType,
2427        speculative_decoding: bool,
2428    ) -> Result<Box<dyn ModelOperations>> {
2429        use super::architectures::qwen3_5::{Qwen3_5Model, Qwen3_5TextConfig};
2430        use super::architectures::qwen3_5_vision::Qwen3_5VisionConfig;
2431
2432        let effective_max_pos = max_context.unwrap_or(config.max_position_embeddings);
2433
2434        let text_cfg = Qwen3_5TextConfig::from_model_config(&config, effective_max_pos);
2435
2436        // Build vision config if the checkpoint has vision weights
2437        let vision_cfg = if config.has_vision {
2438            // vision_config is stored in the ModelConfig's raw JSON during loading
2439            // Use the fields already parsed into ModelConfig
2440            Some(Qwen3_5VisionConfig {
2441                depth: 27,
2442                hidden_size: 1152,
2443                intermediate_size: 4304,
2444                num_heads: 16,
2445                head_dim: 72,
2446                patch_size: 16,
2447                temporal_patch_size: 2,
2448                spatial_merge_size: 2,
2449                out_hidden_size: config.vision_out_hidden_size,
2450                rms_norm_eps: 1e-6,
2451            })
2452        } else if weights.keys().any(|k| k.starts_with("visual.")) {
2453            // Fallback: detect from weights even if config didn't set has_vision
2454            Some(Qwen3_5VisionConfig {
2455                depth: 27,
2456                hidden_size: 1152,
2457                intermediate_size: 4304,
2458                num_heads: 16,
2459                head_dim: 72,
2460                patch_size: 16,
2461                temporal_patch_size: 2,
2462                spatial_merge_size: 2,
2463                out_hidden_size: if config.vision_out_hidden_size > 0 {
2464                    config.vision_out_hidden_size
2465                } else {
2466                    3584
2467                },
2468                rms_norm_eps: 1e-6,
2469            })
2470        } else {
2471            None
2472        };
2473
2474        Ok(Box::new(Qwen3_5Model::from_weights(
2475            &mut weights,
2476            text_cfg,
2477            vision_cfg,
2478            device,
2479            dtype,
2480            kv_quant_type,
2481            speculative_decoding,
2482        )?))
2483    }
2484
2485    // =========================================================================
2486    // FsOps-aware methods (worktree-scoped, path-contained access)
2487    // =========================================================================
2488
2489    /// Create a model using FsOps for weight loading.
2490    ///
2491    /// Uses FsOps::read_file() instead of direct filesystem access.
2492    /// The `model_path` is still needed for ModelConfig and architecture detection
2493    /// (which parse config.json), but weight data is read through FsOps.
2494    /// Explicit policy and whole-checkpoint layout preflight filter MTP tensors
2495    /// before materialization here too; policy is not re-read from the environment.
2496    #[instrument(name = "model_factory.create_with_fs", skip(device, dtype, fs, device_pool), fields(model_path = %model_path.display()))]
2497    pub async fn create_with_fs(
2498        model_path: &Path,
2499        device: &Device,
2500        dtype: DType,
2501        max_context: Option<usize>,
2502        kv_quant_type: KVQuantType,
2503        fs: &WorktreeClient,
2504        device_pool: Option<&DevicePool>,
2505        fp8_dequant_load: bool,
2506        speculative_decoding: bool,
2507    ) -> Result<Box<dyn ModelOperations>> {
2508        info!("Loading model via FsOps: {}", model_path.display());
2509
2510        let shard_names = Self::find_shard_names_fs(fs).await?;
2511
2512        let speculative_decoding = Self::mtp_loading_policy_fs(
2513            model_path,
2514            fs,
2515            &shard_names,
2516            mtp_runtime_supported(speculative_decoding, kv_quant_type),
2517        )
2518        .await?;
2519
2520        if shard_names.len() > 1 {
2521            info!("Loading {} weight shards via FsOps", shard_names.len());
2522        }
2523
2524        let weights = Self::load_weights_fs(
2525            fs,
2526            &shard_names,
2527            device,
2528            dtype,
2529            speculative_decoding,
2530            native_finite_policy(model_path),
2531        )
2532        .await?;
2533        let config = ModelConfig::load(model_path, &weights)?;
2534        let model = Self::create_model_from_config(
2535            config,
2536            weights,
2537            device,
2538            dtype,
2539            max_context,
2540            kv_quant_type,
2541            model_path,
2542            device_pool,
2543            fp8_dequant_load,
2544            speculative_decoding,
2545        )?;
2546        info!("Model created successfully via FsOps");
2547        Ok(model)
2548    }
2549
2550    /// Detect model dtype using FsOps for file reading.
2551    pub async fn detect_model_dtype_fs(fs: &WorktreeClient) -> Result<DType> {
2552        let shard_names = Self::find_shard_names_fs(fs).await?;
2553        if shard_names.is_empty() {
2554            return Err(anyhow!("No model weights found"));
2555        }
2556
2557        let file_content = fs.read_file_chunked(&shard_names[0]).await?;
2558        let tensors = safetensors::SafeTensors::deserialize(&file_content)?;
2559
2560        let mut f16_count = 0;
2561        let mut bf16_count = 0;
2562        let mut f32_count = 0;
2563
2564        for (_, tensor) in tensors.tensors().into_iter().take(10) {
2565            match tensor.dtype() {
2566                safetensors::Dtype::F16 => f16_count += 1,
2567                safetensors::Dtype::BF16
2568                | safetensors::Dtype::F8_E4M3
2569                | safetensors::Dtype::F8_E5M2 => bf16_count += 1,
2570                safetensors::Dtype::F32 => f32_count += 1,
2571                _ => {}
2572            }
2573        }
2574
2575        if f16_count > bf16_count && f16_count > f32_count {
2576            info!("Detected F16 model (via FsOps)");
2577            Ok(tch::Kind::Half)
2578        } else if bf16_count >= f16_count && bf16_count >= f32_count {
2579            info!("Detected BF16 model (via FsOps)");
2580            Ok(tch::Kind::BFloat16)
2581        } else if f32_count > 0 {
2582            info!("Detected F32 model (via FsOps)");
2583            Ok(tch::Kind::Float)
2584        } else {
2585            info!("Could not detect model dtype via FsOps, defaulting to BF16");
2586            Ok(tch::Kind::BFloat16)
2587        }
2588    }
2589
2590    /// Find shard file names via FsOps (returns relative paths).
2591    async fn find_shard_names_fs(fs: &WorktreeClient) -> Result<Vec<String>> {
2592        // Check for single file first
2593        if fs
2594            .stat_path("model.safetensors")
2595            .await
2596            .map(|s| s.exists)
2597            .unwrap_or(false)
2598        {
2599            return Ok(vec!["model.safetensors".to_owned()]);
2600        }
2601
2602        // Look for sharded files
2603        let entries = fs.list_dir_path(".").await?;
2604        let mut shard_names: Vec<String> = entries
2605            .into_iter()
2606            .filter(|e| e.name.starts_with("model-") && e.name.ends_with(".safetensors"))
2607            .map(|e| e.name)
2608            .collect();
2609
2610        shard_names.sort();
2611        Ok(shard_names)
2612    }
2613
2614    /// Load weights from safetensors files via FsOps.
2615    async fn load_weights_fs(
2616        fs: &WorktreeClient,
2617        shard_names: &[String],
2618        device: &Device,
2619        dtype: DType,
2620        speculative_decoding: bool,
2621        native_finite: bool,
2622    ) -> Result<HashMap<String, Tensor>> {
2623        let mut all_weights = HashMap::new();
2624
2625        if shard_names.is_empty() {
2626            return Err(anyhow!("No safetensors files found"));
2627        }
2628
2629        for (idx, name) in shard_names.iter().enumerate() {
2630            if shard_names.len() > 1 {
2631                info!(
2632                    "Loading shard {}/{} via FsOps: {}",
2633                    idx + 1,
2634                    shard_names.len(),
2635                    name
2636                );
2637            }
2638
2639            let data = fs.read_file_chunked(name).await?;
2640            let tensors = safetensors::SafeTensors::deserialize(&data)?;
2641            Self::create_tensors_from_safetensors_selected(
2642                tensors,
2643                None,
2644                &mut all_weights,
2645                device,
2646                dtype,
2647                speculative_decoding,
2648                native_finite,
2649            )?;
2650        }
2651
2652        Ok(all_weights)
2653    }
2654}
2655
2656#[cfg(test)]
2657#[allow(clippy::expect_used, clippy::unwrap_used)]
2658mod stage_subset_tests {
2659    use super::*;
2660    use safetensors::tensor::{serialize, Dtype, TensorView};
2661
2662    struct FixtureTensor {
2663        name: String,
2664        dtype: Dtype,
2665        shape: Vec<usize>,
2666        data: Vec<u8>,
2667    }
2668
2669    impl FixtureTensor {
2670        fn f32(name: impl Into<String>, shape: &[usize], seed: f32) -> Self {
2671            let elements = shape.iter().product();
2672            let data = (0..elements)
2673                .flat_map(|index| (seed + index as f32 * 0.01).to_le_bytes())
2674                .collect();
2675            Self {
2676                name: name.into(),
2677                dtype: Dtype::F32,
2678                shape: shape.to_vec(),
2679                data,
2680            }
2681        }
2682
2683        fn i64(name: impl Into<String>) -> Self {
2684            Self::i64_shape(name, &[1])
2685        }
2686
2687        fn i64_shape(name: impl Into<String>, shape: &[usize]) -> Self {
2688            let elements = shape.iter().product();
2689            let data = (0..elements)
2690                .flat_map(|index| (index as i64 + 7).to_le_bytes())
2691                .collect();
2692            Self {
2693                name: name.into(),
2694                dtype: Dtype::I64,
2695                shape: shape.to_vec(),
2696                data,
2697            }
2698        }
2699    }
2700
2701    fn write_shard(dir: &Path, name: &str, tensors: &[FixtureTensor]) {
2702        let views: Vec<_> = tensors
2703            .iter()
2704            .map(|tensor| {
2705                (
2706                    tensor.name.clone(),
2707                    TensorView::new(tensor.dtype, tensor.shape.clone(), &tensor.data).unwrap(),
2708                )
2709            })
2710            .collect();
2711        let bytes = serialize(views, &None).unwrap();
2712        std::fs::write(dir.join(name), bytes).unwrap();
2713    }
2714
2715    fn write_index(dir: &Path, entries: &[(&str, &str)]) {
2716        let weight_map: BTreeMap<_, _> = entries
2717            .iter()
2718            .map(|(tensor, shard)| ((*tensor).to_owned(), (*shard).to_owned()))
2719            .collect();
2720        let index = serde_json::json!({ "weight_map": weight_map });
2721        std::fs::write(
2722            dir.join("model.safetensors.index.json"),
2723            serde_json::to_vec(&index).unwrap(),
2724        )
2725        .unwrap();
2726    }
2727
2728    fn finite_fixture(name: &str, dtype: Dtype, data: Vec<u8>) -> FixtureTensor {
2729        let elements = data.len() / dtype.size();
2730        FixtureTensor {
2731            name: name.to_owned(),
2732            dtype,
2733            shape: vec![elements],
2734            data,
2735        }
2736    }
2737
2738    fn finite_config(dir: &Path, config: serde_json::Value) {
2739        std::fs::write(
2740            dir.join("config.json"),
2741            serde_json::to_vec(&config).unwrap(),
2742        )
2743        .unwrap();
2744    }
2745
2746    fn channel_profile_config() -> serde_json::Value {
2747        serde_json::json!({"model_type":"qwen3_5_text","quantization_config":{
2748            "quant_method":"compressed-tensors", "version":"0.19.0", "format":"float-quantized", "quantization_status":"compressed", "ignore":["lm_head"],
2749            "config_groups":{"g":{"format":"float-quantized", "targets":["Linear"],
2750                "weights":{"num_bits":8,"type":"float","strategy":"channel","symmetric":true,"dynamic":false},
2751                "input_activations":{"num_bits":8,"type":"float","strategy":"token","symmetric":true,"dynamic":true}}}}})
2752    }
2753
2754    #[tokio::test]
2755    async fn channel_prepared_factory_single_and_sharded_cpu_forward_match() {
2756        let mut results = Vec::new();
2757        for sharded in [false, true] {
2758            let dir = tempfile::tempdir().unwrap();
2759            let mut tensors = tiny_dense_tensors(true);
2760            let mut scales = Vec::new();
2761            for matrix in tensors.iter_mut().filter(|tensor| {
2762                tensor.shape.len() == 2 && tensor.name.starts_with("model.layers.")
2763            }) {
2764                matrix.dtype = Dtype::F8_E4M3;
2765                matrix.data = vec![0x38; matrix.shape.iter().product()];
2766                scales.push(FixtureTensor::f32(
2767                    format!("{}_scale", matrix.name),
2768                    &[matrix.shape[0], 1],
2769                    0.25,
2770                ));
2771            }
2772            tensors.extend(scales);
2773            write_tiny_dense_checkpoint(dir.path(), &tensors);
2774            let mut config: serde_json::Value =
2775                serde_json::from_slice(&std::fs::read(dir.path().join("config.json")).unwrap())
2776                    .unwrap();
2777            config["quantization_config"] = channel_profile_config()["quantization_config"].clone();
2778            finite_config(dir.path(), config);
2779            if sharded {
2780                let (matrices, scales): (Vec<_>, Vec<_>) = tensors
2781                    .into_iter()
2782                    .partition(|tensor| !tensor.name.ends_with("_scale"));
2783                write_shard(dir.path(), "a.safetensors", &matrices);
2784                write_shard(dir.path(), "b.safetensors", &scales);
2785                let index: Vec<_> = matrices
2786                    .iter()
2787                    .map(|tensor| (tensor.name.as_str(), "a.safetensors"))
2788                    .chain(
2789                        scales
2790                            .iter()
2791                            .map(|tensor| (tensor.name.as_str(), "b.safetensors")),
2792                    )
2793                    .collect();
2794                write_index(dir.path(), &index);
2795            }
2796            let model = ModelFactory::create(
2797                dir.path(),
2798                &Device::Cpu,
2799                DType::Float,
2800                Some(16),
2801                KVQuantType::None,
2802                None,
2803                false,
2804                false,
2805            )
2806            .await
2807            .unwrap();
2808            let result = model
2809                .forward(&Tensor::from_slice(&[1i64, 2]).reshape([1, 2]), None)
2810                .unwrap();
2811            assert_eq!(result.kind(), DType::Float);
2812            assert!(result.isfinite().all().int64_value(&[]) != 0);
2813            results.push(result);
2814        }
2815        assert!(results[0].allclose(&results[1], 1e-6, 1e-6, false));
2816    }
2817
2818    #[cfg(target_os = "linux")]
2819    fn channel_private_spool_fixture_dir() -> tempfile::TempDir {
2820        use std::os::unix::fs::PermissionsExt as _;
2821        tempfile::Builder::new()
2822            .permissions(std::fs::Permissions::from_mode(0o700))
2823            .tempdir_in(std::env::current_dir().unwrap())
2824            .unwrap()
2825    }
2826
2827    #[cfg(target_os = "linux")]
2828    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2829    async fn channel_prepared_cancel_during_spool_does_not_resolve_next_and_releases_fd() {
2830        use parking_lot::{Condvar, Mutex as TestMutex};
2831        use std::os::fd::AsRawFd as _;
2832        use std::os::unix::fs::MetadataExt as _;
2833        use std::sync::{
2834            atomic::{AtomicUsize, Ordering},
2835            Arc,
2836        };
2837        let dir = channel_private_spool_fixture_dir();
2838        write_shard(
2839            dir.path(),
2840            "w.safetensors",
2841            &[FixtureTensor {
2842                name: "p.weight".into(),
2843                dtype: Dtype::F8_E4M3,
2844                shape: vec![2, 3],
2845                data: vec![0x38; 6],
2846            }],
2847        );
2848        let mut payload = std::fs::read(dir.path().join("w.safetensors")).unwrap();
2849        let profile =
2850            super::super::channel_fp8::ChannelFp8Profile::parse(&channel_profile_config())
2851                .unwrap()
2852                .unwrap();
2853        let resolve_count = Arc::new(AtomicUsize::new(0));
2854        let started = Arc::new(tokio::sync::Notify::new());
2855        let release = Arc::new((TestMutex::new(false), Condvar::new()));
2856        let fd_witness = Arc::new(TestMutex::new(None));
2857        let task = {
2858            let count = resolve_count.clone();
2859            let started = started.clone();
2860            let release = release.clone();
2861            let witness = fd_witness.clone();
2862            let parent = dir.path().to_path_buf();
2863            tokio::spawn(async move {
2864                ModelFactory::prepare_channel_inputs_spooling(
2865                    &profile,
2866                    &[PathBuf::from("w"), PathBuf::from("later")],
2867                    move |_| {
2868                        assert_eq!(
2869                            count.fetch_add(1, Ordering::SeqCst),
2870                            0,
2871                            "cancelled preparation resolved another shard"
2872                        );
2873                        std::future::ready(Ok(ResolvedWeight::Owned(std::mem::take(&mut payload))))
2874                    },
2875                    move |bytes| {
2876                        let parent = parent.clone();
2877                        let started = started.clone();
2878                        let release = release.clone();
2879                        let witness = witness.clone();
2880                        async move {
2881                            tokio::task::spawn_blocking(move || {
2882                                let file = super::super::resolved_spool::spool_in(bytes, &parent)?;
2883                                *witness.lock() = Some((file.as_raw_fd(), file.metadata()?.ino()));
2884                                started.notify_one();
2885                                let (lock, cv) = &*release;
2886                                let mut guard = lock.lock();
2887                                cv.wait_while(&mut guard, |ready| !*ready);
2888                                drop(guard);
2889                                Ok(file)
2890                            })
2891                            .await?
2892                        }
2893                    },
2894                )
2895                .await
2896            })
2897        };
2898        tokio::time::timeout(std::time::Duration::from_secs(5), started.notified())
2899            .await
2900            .unwrap();
2901        task.abort();
2902        assert!(task.await.err().unwrap().is_cancelled());
2903        assert_eq!(resolve_count.load(Ordering::SeqCst), 1);
2904        {
2905            let (lock, cv) = &*release;
2906            *lock.lock() = true;
2907            cv.notify_one();
2908        }
2909        let (fd, ino) = fd_witness.lock().unwrap();
2910        tokio::time::timeout(std::time::Duration::from_secs(5), async {
2911            while std::fs::metadata(format!("/proc/self/fd/{fd}"))
2912                .map(|m| m.ino() == ino)
2913                .unwrap_or(false)
2914            {
2915                tokio::task::yield_now().await;
2916            }
2917        })
2918        .await
2919        .unwrap();
2920        assert_eq!(resolve_count.load(Ordering::SeqCst), 1);
2921    }
2922
2923    #[cfg(target_os = "linux")]
2924    #[tokio::test]
2925    async fn channel_prepared_multiple_owned_spool_before_next_resolution() {
2926        use std::os::fd::AsRawFd as _;
2927        use std::os::unix::fs::MetadataExt as _;
2928        use std::sync::atomic::{AtomicUsize, Ordering};
2929        let dir = channel_private_spool_fixture_dir();
2930        let config = channel_profile_config();
2931        finite_config(dir.path(), config.clone());
2932        let profile = super::super::channel_fp8::ChannelFp8Profile::parse(&config)
2933            .unwrap()
2934            .unwrap();
2935        for (name, tensor) in [("a.safetensors", "p.weight"), ("b.safetensors", "q.weight")] {
2936            write_shard(
2937                dir.path(),
2938                name,
2939                &[FixtureTensor {
2940                    name: tensor.into(),
2941                    dtype: Dtype::F8_E4M3,
2942                    shape: vec![2, 3],
2943                    data: vec![0x38; 6],
2944                }],
2945            );
2946        }
2947        write_shard(
2948            dir.path(),
2949            "c.safetensors",
2950            &[
2951                FixtureTensor::f32("p.weight_scale", &[2, 1], 0.5),
2952                FixtureTensor::f32("q.weight_scale", &[2, 1], 0.5),
2953            ],
2954        );
2955        let paths: Vec<_> = ["a.safetensors", "b.safetensors", "c.safetensors"]
2956            .iter()
2957            .map(|n| dir.path().join(n))
2958            .collect();
2959        let mut payloads: Vec<_> = paths.iter().map(|p| std::fs::read(p).unwrap()).collect();
2960        for path in &paths {
2961            std::fs::write(path, b"version https://git-lfs.github.com/spec/v1\n").unwrap();
2962        }
2963        let resolved = AtomicUsize::new(0);
2964        let spooled = AtomicUsize::new(0);
2965        let mut fds = Vec::new();
2966        let inputs = ModelFactory::prepare_channel_inputs_spooling(
2967            &profile,
2968            &paths,
2969            |_| {
2970                let index = resolved.fetch_add(1, Ordering::SeqCst);
2971                // Fails causally if the factory resolves all Owned payloads before
2972                // converting/releasing the earlier buffer.
2973                assert_eq!(spooled.load(Ordering::SeqCst), index);
2974                std::future::ready(Ok(ResolvedWeight::Owned(std::mem::take(
2975                    &mut payloads[index],
2976                ))))
2977            },
2978            |bytes| {
2979                let file = super::super::resolved_spool::spool_in(bytes, dir.path()).unwrap();
2980                fds.push((file.as_raw_fd(), file.metadata().unwrap().ino()));
2981                spooled.fetch_add(1, Ordering::SeqCst);
2982                std::future::ready(Ok(file))
2983            },
2984        )
2985        .await
2986        .unwrap();
2987        assert_eq!(resolved.load(Ordering::SeqCst), 3);
2988        assert!(inputs
2989            .iter()
2990            .all(|(_, r)| matches!(r, ResolvedWeight::File(_))));
2991        for path in &paths {
2992            std::fs::remove_file(path).unwrap();
2993        }
2994        let mut weights = HashMap::new();
2995        for (path, input) in inputs {
2996            ModelFactory::load_resolved_safetensors_file_inner(
2997                &path,
2998                input,
2999                None,
3000                &mut weights,
3001                &Device::Cpu,
3002                DType::Float,
3003                false,
3004                true,
3005                true,
3006            )
3007            .await
3008            .unwrap();
3009        }
3010        let p =
3011            crate::runtime::architectures::llama::LinearProjection::take(&mut weights, "p.weight")
3012                .unwrap();
3013        let output = p
3014            .try_apply(&Tensor::ones([1, 3], (DType::Float, Device::Cpu)))
3015            .unwrap();
3016        assert!((output.double_value(&[0, 0]) - 1.5).abs() < 1e-6);
3017        assert!((output.double_value(&[0, 1]) - 1.53).abs() < 1e-6);
3018        for (fd, ino) in fds {
3019            assert!(
3020                std::fs::metadata(format!("/proc/self/fd/{fd}"))
3021                    .map(|m| m.ino() != ino)
3022                    .unwrap_or(true),
3023                "retained spool FD leaked after load"
3024            );
3025        }
3026    }
3027
3028    #[cfg(target_os = "linux")]
3029    #[tokio::test]
3030    async fn channel_prepared_owned_preflight_failure_releases_all_spools() {
3031        use std::os::fd::AsRawFd as _;
3032        use std::os::unix::fs::MetadataExt as _;
3033        let dir = channel_private_spool_fixture_dir();
3034        let profile =
3035            super::super::channel_fp8::ChannelFp8Profile::parse(&channel_profile_config())
3036                .unwrap()
3037                .unwrap();
3038        write_shard(
3039            dir.path(),
3040            "a.safetensors",
3041            &[FixtureTensor {
3042                name: "p.weight".into(),
3043                dtype: Dtype::F8_E4M3,
3044                shape: vec![2, 3],
3045                data: vec![0x38; 6],
3046            }],
3047        );
3048        let path = dir.path().join("a.safetensors");
3049        let mut payload = std::fs::read(&path).unwrap();
3050        let mut fds = Vec::new();
3051        let result = ModelFactory::prepare_channel_inputs_spooling(
3052            &profile,
3053            &[path],
3054            |_| std::future::ready(Ok(ResolvedWeight::Owned(std::mem::take(&mut payload)))),
3055            |bytes| {
3056                let file = super::super::resolved_spool::spool_in(bytes, dir.path()).unwrap();
3057                fds.push((file.as_raw_fd(), file.metadata().unwrap().ino()));
3058                std::future::ready(Ok(file))
3059            },
3060        )
3061        .await;
3062        let error = result.err().unwrap();
3063        assert!(error.to_string().contains("scale"), "{error}");
3064        for (fd, ino) in fds {
3065            assert!(std::fs::metadata(format!("/proc/self/fd/{fd}"))
3066                .map(|m| m.ino() != ino)
3067                .unwrap_or(true));
3068        }
3069    }
3070
3071    #[cfg(target_os = "linux")]
3072    #[tokio::test]
3073    async fn channel_prepared_file_inputs_never_require_spool_cache() {
3074        let dir = tempfile::tempdir().unwrap();
3075        let profile =
3076            super::super::channel_fp8::ChannelFp8Profile::parse(&channel_profile_config())
3077                .unwrap()
3078                .unwrap();
3079        write_shard(
3080            dir.path(),
3081            "a.safetensors",
3082            &[
3083                FixtureTensor {
3084                    name: "p.weight".into(),
3085                    dtype: Dtype::F8_E4M3,
3086                    shape: vec![2, 3],
3087                    data: vec![0x38; 6],
3088                },
3089                FixtureTensor::f32("p.weight_scale", &[2, 1], 0.5),
3090            ],
3091        );
3092        let path = dir.path().join("a.safetensors");
3093        let inputs = ModelFactory::prepare_channel_inputs_spooling(
3094            &profile,
3095            &[path],
3096            |path| std::future::ready(Ok(ResolvedWeight::File(std::fs::File::open(path).unwrap()))),
3097            |_| std::future::ready(Err(anyhow!("cache unavailable"))),
3098        )
3099        .await
3100        .unwrap();
3101        assert_eq!(inputs.len(), 1);
3102    }
3103
3104    #[tokio::test]
3105    async fn channel_prepared_pointer_and_file_resolve_once_through_preflight_and_load() {
3106        use std::sync::atomic::{AtomicUsize, Ordering};
3107        let dir = tempfile::tempdir().unwrap();
3108        let config = channel_profile_config();
3109        finite_config(dir.path(), config.clone());
3110        let profile = super::super::channel_fp8::ChannelFp8Profile::parse(&config)
3111            .unwrap()
3112            .unwrap();
3113        let weight = FixtureTensor {
3114            name: "p.weight".into(),
3115            dtype: Dtype::F8_E4M3,
3116            shape: vec![2, 3],
3117            data: vec![0x38; 6],
3118        };
3119        write_shard(dir.path(), "w.safetensors", &[weight]);
3120        let weight_path = dir.path().join("w.safetensors");
3121        let resolved_bytes = std::fs::read(&weight_path).unwrap();
3122        // Actual path contains a pointer-shaped stub. Metadata MUST consume
3123        // supplied resolved bytes, not parse this prefix as a tensor header.
3124        std::fs::write(
3125            &weight_path,
3126            b"version https://git-lfs.github.com/spec/v1\noid sha256:fixture\n",
3127        )
3128        .unwrap();
3129        write_shard(
3130            dir.path(),
3131            "s.safetensors",
3132            &[FixtureTensor::f32("p.weight_scale", &[2, 1], 0.5)],
3133        );
3134        let scale_path = dir.path().join("s.safetensors");
3135        let count = AtomicUsize::new(0);
3136        let mut inputs = ModelFactory::prepare_channel_inputs(
3137            &profile,
3138            &[weight_path.clone(), scale_path.clone()],
3139            |path| {
3140                count.fetch_add(1, Ordering::SeqCst);
3141                std::future::ready(Ok(if path == weight_path {
3142                    ResolvedWeight::Owned(resolved_bytes.clone())
3143                } else {
3144                    ResolvedWeight::File(std::fs::File::open(path).unwrap())
3145                }))
3146            },
3147        )
3148        .await
3149        .unwrap();
3150        assert_eq!(count.load(Ordering::SeqCst), 2);
3151        // Another bounded header pass borrows retained ownership and rewinds
3152        // file readers; no payload clone/resolver call belongs to this pass.
3153        profile.preflight_resolved(&mut inputs).unwrap();
3154        std::fs::remove_file(&weight_path).unwrap();
3155        std::fs::remove_file(&scale_path).unwrap();
3156        let mut weights = HashMap::new();
3157        for (path, resolved) in inputs {
3158            ModelFactory::load_resolved_safetensors_file_inner(
3159                &path,
3160                resolved,
3161                None,
3162                &mut weights,
3163                &Device::Cpu,
3164                DType::Float,
3165                false,
3166                true,
3167                true,
3168            )
3169            .await
3170            .unwrap();
3171        }
3172        assert_eq!(
3173            count.load(Ordering::SeqCst),
3174            2,
3175            "factory preflight+load must resolve once per shard"
3176        );
3177        assert_eq!(weights["p.weight"].kind(), DType::Float8e4m3fn);
3178        assert_eq!(weights["p.weight_scale"].size(), [2, 1]);
3179        let projection =
3180            crate::runtime::architectures::llama::LinearProjection::take(&mut weights, "p.weight")
3181                .unwrap();
3182        let output = projection
3183            .try_apply(&Tensor::ones([1, 3], (DType::Float, Device::Cpu)))
3184            .unwrap();
3185        assert!((output.double_value(&[0, 0]) - 1.5).abs() < 1e-6);
3186        assert!((output.double_value(&[0, 1]) - 1.53).abs() < 1e-6);
3187    }
3188
3189    #[tokio::test]
3190    async fn channel_prepared_header_and_resolver_failures_are_closed_before_tensors() {
3191        let profile =
3192            super::super::channel_fp8::ChannelFp8Profile::parse(&channel_profile_config())
3193                .unwrap()
3194                .unwrap();
3195        let path = PathBuf::from("unresolved-pointer.safetensors");
3196        let error =
3197            ModelFactory::prepare_channel_inputs(&profile, std::slice::from_ref(&path), |_| {
3198                std::future::ready(Err(anyhow!("malformed captured LFS pointer")))
3199            })
3200            .await
3201            .err()
3202            .unwrap();
3203        assert!(error.to_string().contains("malformed captured LFS pointer"));
3204        for data in [
3205            vec![],
3206            (16 * 1024 * 1024u64 + 1).to_le_bytes().to_vec(),
3207            b"version https://git-lfs.github.com/spec/v1".to_vec(),
3208            10u64.to_le_bytes().to_vec(),
3209        ] {
3210            assert!(ModelFactory::prepare_channel_inputs(
3211                &profile,
3212                std::slice::from_ref(&path),
3213                |_| std::future::ready(Ok(ResolvedWeight::Owned(data.clone())))
3214            )
3215            .await
3216            .is_err());
3217        }
3218    }
3219
3220    #[tokio::test]
3221    async fn native_finite_path_rejects_nonfinite_float_payloads() {
3222        let dir = tempfile::tempdir().unwrap();
3223        finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5"}));
3224        let path = dir.path().join("weights.safetensors");
3225        let cases = [
3226            (Dtype::BF16, vec![0x7f80u32, 0xff80, 0x7fc1, 0xffc1]),
3227            (Dtype::F16, vec![0x7c00, 0xfc00, 0x7e01, 0xfe01]),
3228            (
3229                Dtype::F32,
3230                vec![0x7f80_0000, 0xff80_0000, 0x7fc0_0001, 0xffc0_0001],
3231            ),
3232        ];
3233        for (dtype, patterns) in cases {
3234            for (index, bits) in patterns.into_iter().enumerate() {
3235                let name = [
3236                    "model.embed_tokens.weight",
3237                    "model.norm.weight",
3238                    "model.layers.0.linear_attn.A_log",
3239                    "model.layers.0.self_attn.q_proj.weight_scale",
3240                ][index];
3241                // A finite element before the poison verifies the reported
3242                // element offset; no request tensor reaches a forward pass.
3243                let data = if dtype == Dtype::F32 {
3244                    [0u32.to_le_bytes(), bits.to_le_bytes()].concat()
3245                } else {
3246                    [0u16.to_le_bytes(), (bits as u16).to_le_bytes()].concat()
3247                };
3248                write_shard(
3249                    dir.path(),
3250                    "weights.safetensors",
3251                    &[finite_fixture(name, dtype, data)],
3252                );
3253                let mut weights = HashMap::new();
3254                let error = ModelFactory::load_safetensors_file(
3255                    &path,
3256                    &mut weights,
3257                    &Device::Cpu,
3258                    DType::Float,
3259                    false,
3260                    native_finite_policy(dir.path()),
3261                )
3262                .await
3263                .unwrap_err()
3264                .to_string();
3265                assert!(error.contains("nonfinite native Qwen3.5"), "{error}");
3266                assert!(
3267                    error.contains(name) && error.contains("element 1"),
3268                    "{error}"
3269                );
3270                assert!(weights.is_empty(), "offending tensor was inserted");
3271            }
3272        }
3273    }
3274
3275    #[tokio::test]
3276    async fn native_finite_floating_boundaries_preserve_stored_values() {
3277        let dir = tempfile::tempdir().unwrap();
3278        finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5"}));
3279        let path = dir.path().join("weights.safetensors");
3280        let cases = [
3281            (
3282                Dtype::BF16,
3283                DType::BFloat16,
3284                [0u16, 0x8000, 1, 0x8001, 0x7f7f, 0xff7f]
3285                    .into_iter()
3286                    .flat_map(u16::to_le_bytes)
3287                    .collect::<Vec<_>>(),
3288            ),
3289            (
3290                Dtype::F16,
3291                DType::Half,
3292                [0u16, 0x8000, 1, 0x8001, 0x7bff, 0xfbff]
3293                    .into_iter()
3294                    .flat_map(u16::to_le_bytes)
3295                    .collect(),
3296            ),
3297            (
3298                Dtype::F32,
3299                DType::Float,
3300                [0u32, 0x8000_0000, 1, 0x8000_0001, 0x7f7f_ffff, 0xff7f_ffff]
3301                    .into_iter()
3302                    .flat_map(u32::to_le_bytes)
3303                    .collect(),
3304            ),
3305        ];
3306        for (dtype, compute, data) in cases {
3307            write_shard(
3308                dir.path(),
3309                "weights.safetensors",
3310                &[finite_fixture("model.norm.weight", dtype, data.clone())],
3311            );
3312            let mut weights = HashMap::new();
3313            ModelFactory::load_safetensors_file(
3314                &path,
3315                &mut weights,
3316                &Device::Cpu,
3317                compute,
3318                false,
3319                native_finite_policy(dir.path()),
3320            )
3321            .await
3322            .unwrap();
3323            let tensor = &weights["model.norm.weight"];
3324            assert_eq!(tensor.kind(), compute);
3325            let mut actual = vec![0u8; data.len()];
3326            tensor.copy_data_u8(&mut actual, tensor.numel());
3327            assert_eq!(actual, data, "finite boundary bytes changed for{dtype:?}");
3328        }
3329    }
3330
3331    #[tokio::test]
3332    async fn native_finite_fp8_path_preserves_finite_codes_rejects_nan() {
3333        let dir = tempfile::tempdir().unwrap();
3334        finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5_text"}));
3335        let path = dir.path().join("weights.safetensors");
3336        let finite_codes = vec![0x00, 0x80, 0x01, 0x81, 0x7e, 0xfe];
3337        write_shard(
3338            dir.path(),
3339            "weights.safetensors",
3340            &[finite_fixture(
3341                "p.weight",
3342                Dtype::F8_E4M3,
3343                finite_codes.clone(),
3344            )],
3345        );
3346        let mut weights = HashMap::new();
3347        ModelFactory::load_safetensors_file(
3348            &path,
3349            &mut weights,
3350            &Device::Cpu,
3351            DType::Float,
3352            false,
3353            native_finite_policy(dir.path()),
3354        )
3355        .await
3356        .unwrap();
3357        let tensor = &weights["p.weight"];
3358        assert_eq!(tensor.kind(), DType::Float8e4m3fn);
3359        let mut actual = vec![0u8; finite_codes.len()];
3360        tensor.copy_data_u8(&mut actual, tensor.numel());
3361        assert_eq!(actual, finite_codes); // includes signed zero and subnormals
3362        let floating = tensor.to_kind(DType::Float);
3363        assert_eq!(floating.double_value(&[4]), 448.0);
3364        assert_eq!(floating.double_value(&[5]), -448.0);
3365        for code in [0x7f, 0xff] {
3366            write_shard(
3367                dir.path(),
3368                "weights.safetensors",
3369                &[finite_fixture("p.weight", Dtype::F8_E4M3, vec![code])],
3370            );
3371            let mut rejected = HashMap::new();
3372            let error = ModelFactory::load_safetensors_file(
3373                &path,
3374                &mut rejected,
3375                &Device::Cpu,
3376                DType::Float,
3377                false,
3378                native_finite_policy(dir.path()),
3379            )
3380            .await
3381            .unwrap_err()
3382            .to_string();
3383            assert!(error.contains("nonfinite") && error.contains("p.weight"));
3384            assert!(rejected.is_empty());
3385        }
3386        // Finite validation does not reinterpret or admit E5M2 as the separate
3387        // compressed FP8_DYNAMIC profile; this is the existing loader format.
3388        for code in [0x7c, 0xfc, 0x7d, 0xfd] {
3389            assert!(validate_finite_payload("p.weight", Dtype::F8_E5M2, &[code]).is_err());
3390        }
3391        validate_finite_payload(
3392            "p.weight",
3393            Dtype::F8_E5M2,
3394            &[0x00, 0x80, 1, 0x81, 0x7b, 0xfb],
3395        )
3396        .unwrap();
3397    }
3398
3399    #[tokio::test]
3400    async fn native_finite_selection_and_mtp_exclusion_precede_scan() {
3401        let dir = tempfile::tempdir().unwrap();
3402        finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5"}));
3403        let path = dir.path().join("weights.safetensors");
3404        let poison = || {
3405            finite_fixture(
3406                "model.embed_tokens.weight",
3407                Dtype::F32,
3408                f32::NAN.to_le_bytes().to_vec(),
3409            )
3410        };
3411        let mtp_poison = || {
3412            finite_fixture(
3413                "mtp.norm.weight",
3414                Dtype::F32,
3415                f32::INFINITY.to_le_bytes().to_vec(),
3416            )
3417        };
3418        write_shard(
3419            dir.path(),
3420            "weights.safetensors",
3421            &[
3422                FixtureTensor::f32("model.norm.weight", &[1], 1.0),
3423                poison(),
3424                mtp_poison(),
3425            ],
3426        );
3427        let selected =
3428            BTreeSet::from(["model.norm.weight".to_owned(), "mtp.norm.weight".to_owned()]);
3429        let mut weights = HashMap::new();
3430        // Actual selected path forces mmap; neither unselected text nor an
3431        // explicitly selected but disabled MTP tensor is scanned/materialized.
3432        ModelFactory::load_safetensors_file_selected(
3433            &path,
3434            &selected,
3435            &mut weights,
3436            &Device::Cpu,
3437            DType::Float,
3438            native_finite_policy(dir.path()),
3439        )
3440        .await
3441        .unwrap();
3442        assert_eq!(weights.len(), 1);
3443        assert!(weights.contains_key("model.norm.weight"));
3444        let mut rejected = HashMap::new();
3445        let selected = BTreeSet::from(["model.embed_tokens.weight".to_owned()]);
3446        assert!(ModelFactory::load_safetensors_file_selected(
3447            &path,
3448            &selected,
3449            &mut rejected,
3450            &Device::Cpu,
3451            DType::Float,
3452            native_finite_policy(dir.path())
3453        )
3454        .await
3455        .unwrap_err()
3456        .to_string()
3457        .contains("model.embed_tokens.weight"));
3458        assert!(rejected.is_empty());
3459        write_shard(
3460            dir.path(),
3461            "weights.safetensors",
3462            &[
3463                FixtureTensor::f32("model.norm.weight", &[1], 1.0),
3464                mtp_poison(),
3465            ],
3466        );
3467        for enabled in [false, true] {
3468            let mut weights = HashMap::new();
3469            let result = ModelFactory::load_safetensors_file(
3470                &path,
3471                &mut weights,
3472                &Device::Cpu,
3473                DType::Float,
3474                enabled,
3475                native_finite_policy(dir.path()),
3476            )
3477            .await;
3478            if enabled {
3479                assert!(result.unwrap_err().to_string().contains("mtp.norm.weight"));
3480            } else {
3481                result.unwrap();
3482                assert_eq!(weights.len(), 1);
3483            }
3484        }
3485    }
3486
3487    #[tokio::test]
3488    async fn native_finite_gate_preserves_other_architecture_and_default_loading() {
3489        let dir = tempfile::tempdir().unwrap();
3490        let path = dir.path().join("weights.safetensors");
3491        write_shard(
3492            dir.path(),
3493            "weights.safetensors",
3494            &[finite_fixture(
3495                "model.norm.weight",
3496                Dtype::F32,
3497                f32::NAN.to_le_bytes().to_vec(),
3498            )],
3499        );
3500        for (config, strict) in [
3501            (None, false),
3502            (Some(serde_json::json!({"model_type":"llama"})), false),
3503            (
3504                Some(
3505                    serde_json::json!({"model_type":"llama","text_config":{"model_type":"qwen3_5_text"}}),
3506                ),
3507                false,
3508            ),
3509            (
3510                Some(serde_json::json!({"text_config":{"model_type":"qwen3_5_text"}})),
3511                false,
3512            ),
3513            (Some(serde_json::json!({"model_type":"QWEN3_5"})), true),
3514            (Some(serde_json::json!({"model_type":"qwen3_5_moe"})), true),
3515            (
3516                Some(serde_json::json!({"architectures":["Qwen3_5ForConditionalGeneration"]})),
3517                true,
3518            ),
3519            (
3520                Some(serde_json::json!({"architectures":["LlamaForCausalLM"]})),
3521                false,
3522            ),
3523            (
3524                Some(
3525                    serde_json::json!({"model_type":"llama","architectures":["Qwen3_5ForConditionalGeneration"]}),
3526                ),
3527                false,
3528            ),
3529        ] {
3530            if let Some(config) = config {
3531                finite_config(dir.path(), config);
3532            }
3533            assert_eq!(native_finite_policy(dir.path()), strict);
3534            let mut weights = HashMap::new();
3535            let result = ModelFactory::load_safetensors_file(
3536                &path,
3537                &mut weights,
3538                &Device::Cpu,
3539                DType::Float,
3540                false,
3541                native_finite_policy(dir.path()),
3542            )
3543            .await;
3544            if strict {
3545                assert!(result.unwrap_err().to_string().contains("nonfinite"));
3546            } else {
3547                result.unwrap();
3548                assert_eq!(
3549                    weights["model.norm.weight"]
3550                        .isfinite()
3551                        .all()
3552                        .int64_value(&[]),
3553                    0
3554                );
3555            }
3556        }
3557    }
3558
3559    #[tokio::test]
3560    async fn native_finite_nested_shard_retains_model_root_policy() {
3561        let dir = tempfile::tempdir().unwrap();
3562        finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5"}));
3563        let nested = dir.path().join("weights");
3564        std::fs::create_dir(&nested).unwrap();
3565        // A shard-local config cannot override the authoritative model root.
3566        finite_config(&nested, serde_json::json!({"model_type":"llama"}));
3567        write_shard(
3568            &nested,
3569            "shard.safetensors",
3570            &[finite_fixture(
3571                "model.norm.weight",
3572                Dtype::BF16,
3573                0x7fc1u16.to_le_bytes().to_vec(),
3574            )],
3575        );
3576        write_index(
3577            dir.path(),
3578            &[("model.norm.weight", "weights/shard.safetensors")],
3579        );
3580        let error = ModelFactory::load_weights(dir.path(), &Device::Cpu, DType::Float, false)
3581            .await
3582            .unwrap_err()
3583            .to_string();
3584        assert!(error.contains("nonfinite") && error.contains("model.norm.weight"));
3585    }
3586
3587    #[test]
3588    fn native_finite_fsops_materialization_seam_rejects_nonfinite() {
3589        let dir = tempfile::tempdir().unwrap();
3590        finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5"}));
3591        write_shard(
3592            dir.path(),
3593            "weights.safetensors",
3594            &[finite_fixture(
3595                "model.layers.0.linear_attn.dt_bias",
3596                Dtype::F32,
3597                f32::NEG_INFINITY.to_le_bytes().to_vec(),
3598            )],
3599        );
3600        let bytes = std::fs::read(dir.path().join("weights.safetensors")).unwrap();
3601        let mut weights = HashMap::new();
3602        // Same policy and materializer called after FsOps::read_file_chunked;
3603        // byte acquisition is local here, no fake RPC integration claim.
3604        let error = ModelFactory::create_tensors_from_safetensors_selected(
3605            safetensors::SafeTensors::deserialize(&bytes).unwrap(),
3606            None,
3607            &mut weights,
3608            &Device::Cpu,
3609            DType::Float,
3610            false,
3611            native_finite_policy(dir.path()),
3612        )
3613        .unwrap_err()
3614        .to_string();
3615        assert!(error.contains("nonfinite") && error.contains("dt_bias"));
3616        assert!(weights.is_empty());
3617    }
3618
3619    fn loaded_bytes(weights: &HashMap<String, Tensor>) -> usize {
3620        weights
3621            .values()
3622            .map(|tensor| tensor.numel() * tensor.kind().elt_size_in_bytes())
3623            .sum()
3624    }
3625
3626    fn tiny_dense_tensors(explicit_head: bool) -> Vec<FixtureTensor> {
3627        let mut tensors = vec![
3628            FixtureTensor::f32("model.embed_tokens.weight", &[8, 4], 0.01),
3629            FixtureTensor::f32("model.norm.weight", &[4], 1.0),
3630        ];
3631        if explicit_head {
3632            tensors.push(FixtureTensor::f32("lm_head.weight", &[8, 4], 0.02));
3633        }
3634
3635        let prefix = "model.layers.0";
3636        for (suffix, shape, seed) in [
3637            ("self_attn.q_proj.weight", vec![4, 4], 0.03),
3638            ("self_attn.k_proj.weight", vec![4, 4], 0.04),
3639            ("self_attn.v_proj.weight", vec![4, 4], 0.05),
3640            ("self_attn.o_proj.weight", vec![4, 4], 0.06),
3641            ("mlp.gate_proj.weight", vec![8, 4], 0.07),
3642            ("mlp.up_proj.weight", vec![8, 4], 0.08),
3643            ("mlp.down_proj.weight", vec![4, 8], 0.09),
3644            ("input_layernorm.weight", vec![4], 1.0),
3645            ("post_attention_layernorm.weight", vec![4], 1.0),
3646        ] {
3647            tensors.push(FixtureTensor::f32(
3648                format!("{prefix}.{suffix}"),
3649                &shape,
3650                seed,
3651            ));
3652        }
3653        tensors
3654    }
3655
3656    #[tokio::test]
3657    async fn mtp_disabled_factory_skips_head_before_materialization() {
3658        for sharded in [false, true] {
3659            let dir = tempfile::tempdir().unwrap();
3660            let mut tensors = tiny_dense_tensors(true);
3661            tensors.push(FixtureTensor::i64("mtp.unmaterializable"));
3662            write_tiny_dense_checkpoint(dir.path(), &tensors);
3663            if sharded {
3664                let base = tiny_dense_tensors(true);
3665                write_shard(dir.path(), "model-00001-of-00001.safetensors", &base);
3666                write_shard(
3667                    dir.path(),
3668                    "mtp.safetensors",
3669                    &[FixtureTensor::i64("mtp.unmaterializable")],
3670                );
3671                let mut entries: Vec<_> = base
3672                    .iter()
3673                    .map(|tensor| (tensor.name.as_str(), "model-00001-of-00001.safetensors"))
3674                    .collect();
3675                entries.push(("mtp.unmaterializable", "mtp.safetensors"));
3676                write_index(dir.path(), &entries);
3677            }
3678            for enabled in [false, true] {
3679                let configured = crate::config::RuntimeConfig {
3680                    use_gpu: false,
3681                    speculative_decoding: enabled,
3682                    ..Default::default()
3683                };
3684                let config: crate::config::RuntimeConfig =
3685                    toml::from_str(&toml::to_string(&configured).unwrap()).unwrap();
3686                let result = ModelFactory::create(
3687                    dir.path(),
3688                    &Device::Cpu,
3689                    DType::Float,
3690                    Some(16),
3691                    KVQuantType::None,
3692                    None,
3693                    false,
3694                    config.speculative_decoding,
3695                )
3696                .await;
3697                if enabled {
3698                    let error = result.err().expect("enabled MTP reaches unsupported dtype");
3699                    assert!(error.to_string().contains("mtp.unmaterializable"));
3700                    assert!(error.to_string().contains("unsupported dtype"));
3701                } else {
3702                    assert!(
3703                        result.is_ok(),
3704                        "disabled MTP was materialized: {:?}",
3705                        result.err()
3706                    );
3707                }
3708            }
3709        }
3710    }
3711
3712    #[test]
3713    fn mtp_fsops_materializer_obeys_loading_policy() {
3714        let dir = tempfile::tempdir().unwrap();
3715        write_shard(
3716            dir.path(),
3717            "weights.safetensors",
3718            &[
3719                FixtureTensor::f32("model.norm.weight", &[4], 1.0),
3720                FixtureTensor::f32("mtp.norm.weight", &[4], 1.0),
3721            ],
3722        );
3723        let bytes = std::fs::read(dir.path().join("weights.safetensors")).unwrap();
3724        for enabled in [false, true] {
3725            let mut weights = HashMap::new();
3726            // Same materializer used after FsOps::read_file_chunked; the remote
3727            // read changes byte acquisition, not pre-allocation selection.
3728            ModelFactory::create_tensors_from_safetensors(
3729                safetensors::SafeTensors::deserialize(&bytes).unwrap(),
3730                &mut weights,
3731                &Device::Cpu,
3732                DType::Float,
3733                enabled,
3734            )
3735            .unwrap();
3736            assert!(weights.contains_key("model.norm.weight"));
3737            assert_eq!(weights.contains_key("mtp.norm.weight"), enabled);
3738        }
3739    }
3740
3741    #[tokio::test]
3742    async fn mtp_unsupported_config_skips_before_materialization() {
3743        for config in [
3744            serde_json::json!({"model_type": "qwen3_5_moe"}),
3745            serde_json::json!({"model_type": "qwen3_5", "num_experts": 16}),
3746            serde_json::json!({"model_type": "qwen3_5", "text_config": {"num_experts": 16}}),
3747            serde_json::json!({"model_type": "qwen3_5", "mtp_num_hidden_layers": 2}),
3748            serde_json::json!({"model_type": "qwen3_5", "text_config": {"mtp_num_hidden_layers": 2}}),
3749        ] {
3750            let dir = tempfile::tempdir().unwrap();
3751            std::fs::write(dir.path().join("config.json"), config.to_string()).unwrap();
3752            write_shard(
3753                dir.path(),
3754                "model.safetensors",
3755                &[
3756                    FixtureTensor::f32("model.norm.weight", &[4], 1.0),
3757                    FixtureTensor::i64("mtp.unmaterializable"),
3758                ],
3759            );
3760            let shards = vec![dir.path().join("model.safetensors")];
3761            let allowed = ModelFactory::mtp_loading_policy(dir.path(), &shards, true)
3762                .await
3763                .unwrap();
3764            assert!(!allowed, "unsupported config admitted: {config}");
3765            // This is the actual weight-loading phase, before model construction
3766            // (a tiny fixture need not allocate a complete MoE backbone).
3767            let weights =
3768                ModelFactory::load_weights(dir.path(), &Device::Cpu, DType::Float, allowed)
3769                    .await
3770                    .unwrap();
3771            assert_eq!(weights.len(), 1);
3772            assert!(weights.contains_key("model.norm.weight"));
3773        }
3774    }
3775
3776    async fn check_mtp_multilayer_factory(sharded: bool) {
3777        for layer in ["0", "1", "7"] {
3778            let dir = tempfile::tempdir().unwrap();
3779            let mut first = tiny_dense_tensors(true);
3780            first.push(FixtureTensor::i64("mtp.unmaterializable"));
3781            write_tiny_dense_checkpoint(dir.path(), &first);
3782            let marker = FixtureTensor::f32(format!("mtp.layers.{layer}.norm.weight"), &[4], 1.0);
3783            if sharded {
3784                // The poison is in the first shard, while the unsupported
3785                // layer is only discoverable in the last shard's header.
3786                write_shard(dir.path(), "model-00001-of-00002.safetensors", &first);
3787                write_shard(dir.path(), "model-00002-of-00002.safetensors", &[marker]);
3788                let mut entries: Vec<_> = first
3789                    .iter()
3790                    .map(|t| (t.name.as_str(), "model-00001-of-00002.safetensors"))
3791                    .collect();
3792                let marker_name = format!("mtp.layers.{layer}.norm.weight");
3793                entries.push((&marker_name, "model-00002-of-00002.safetensors"));
3794                write_index(dir.path(), &entries);
3795            } else {
3796                first.push(marker);
3797                write_shard(dir.path(), "model.safetensors", &first);
3798            }
3799            let result = ModelFactory::create(
3800                dir.path(),
3801                &Device::Cpu,
3802                DType::Float,
3803                Some(16),
3804                KVQuantType::None,
3805                None,
3806                false,
3807                true,
3808            )
3809            .await;
3810            if layer == "0" {
3811                let error = result
3812                    .err()
3813                    .expect("supported layout must still load enabled head");
3814                assert!(error.to_string().contains("mtp.unmaterializable"));
3815                assert!(error.to_string().contains("unsupported dtype"));
3816            } else {
3817                assert!(
3818                    result.is_ok(),
3819                    "unsupported head was materialized: {:?}",
3820                    result.err()
3821                );
3822            }
3823        }
3824    }
3825
3826    #[tokio::test]
3827    async fn mtp_multilayer_single_file_skips_before_materialization() {
3828        check_mtp_multilayer_factory(false).await;
3829    }
3830
3831    #[tokio::test]
3832    async fn mtp_multilayer_sharded_skips_before_materialization() {
3833        check_mtp_multilayer_factory(true).await;
3834    }
3835
3836    async fn check_mtp_quantized_factory(sharded: bool) {
3837        let dir = tempfile::tempdir().unwrap();
3838        let mut base = tiny_dense_tensors(true);
3839        write_tiny_dense_checkpoint(dir.path(), &base);
3840        if sharded {
3841            write_shard(
3842                dir.path(),
3843                "model-00002-of-00002.safetensors",
3844                &[FixtureTensor::i64("mtp.unmaterializable")],
3845            );
3846            let mut entries: Vec<_> = base
3847                .iter()
3848                .map(|t| (t.name.as_str(), "model-00001-of-00001.safetensors"))
3849                .collect();
3850            entries.push(("mtp.unmaterializable", "model-00002-of-00002.safetensors"));
3851            write_index(dir.path(), &entries);
3852        } else {
3853            base.push(FixtureTensor::i64("mtp.unmaterializable"));
3854            write_shard(dir.path(), "model.safetensors", &base);
3855        }
3856        for requested in [false, true] {
3857            for kv in [
3858                KVQuantType::None,
3859                KVQuantType::Int8,
3860                KVQuantType::Nf4,
3861                KVQuantType::Fp4,
3862            ] {
3863                let result = ModelFactory::create(
3864                    dir.path(),
3865                    &Device::Cpu,
3866                    DType::Float,
3867                    Some(16),
3868                    kv,
3869                    None,
3870                    false,
3871                    requested,
3872                )
3873                .await;
3874                if requested && kv == KVQuantType::None {
3875                    let error = result
3876                        .err()
3877                        .expect("unquantized enabled head must still load");
3878                    assert!(error.to_string().contains("mtp.unmaterializable"));
3879                    assert!(error.to_string().contains("unsupported dtype"));
3880                } else {
3881                    assert!(
3882                        result.is_ok(),
3883                        "unusable MTP reached allocation: requested={requested}, kv={kv:?}, error={:?}",
3884                        result.err()
3885                    );
3886                }
3887            }
3888        }
3889    }
3890
3891    #[tokio::test]
3892    async fn mtp_quantized_single_file_skips_before_materialization() {
3893        check_mtp_quantized_factory(false).await;
3894    }
3895
3896    #[tokio::test]
3897    async fn mtp_quantized_sharded_skips_before_materialization() {
3898        check_mtp_quantized_factory(true).await;
3899    }
3900
3901    #[test]
3902    fn mtp_quantized_fsops_policy_skips_before_materialization() {
3903        let dir = tempfile::tempdir().unwrap();
3904        write_shard(
3905            dir.path(),
3906            "weights.safetensors",
3907            &[
3908                FixtureTensor::f32("model.norm.weight", &[4], 1.0),
3909                FixtureTensor::i64("mtp.unmaterializable"),
3910            ],
3911        );
3912        let bytes = std::fs::read(dir.path().join("weights.safetensors")).unwrap();
3913        for requested in [false, true] {
3914            for kv in [
3915                KVQuantType::None,
3916                KVQuantType::Int8,
3917                KVQuantType::Nf4,
3918                KVQuantType::Fp4,
3919            ] {
3920                let mut weights = HashMap::new();
3921                // FsOps uses the same effective-runtime gate before its layout
3922                // preflight; exercise the admitted flag at its materializer.
3923                let result = ModelFactory::create_tensors_from_safetensors(
3924                    safetensors::SafeTensors::deserialize(&bytes).unwrap(),
3925                    &mut weights,
3926                    &Device::Cpu,
3927                    DType::Float,
3928                    mtp_runtime_supported(requested, kv),
3929                );
3930                if requested && kv == KVQuantType::None {
3931                    assert!(result
3932                        .unwrap_err()
3933                        .to_string()
3934                        .contains("mtp.unmaterializable"));
3935                } else {
3936                    result.unwrap();
3937                    assert_eq!(weights.len(), 1);
3938                    assert!(weights.contains_key("model.norm.weight"));
3939                }
3940            }
3941        }
3942    }
3943
3944    #[test]
3945    fn mtp_fsops_header_policy_filters_unsupported_before_materialization() {
3946        let dir = tempfile::tempdir().unwrap();
3947        for marker in [
3948            "mtp.layers.0.norm.weight",
3949            "mtp.layers.1.norm.weight",
3950            "mtp.layers.0.mlp.experts.0.weight",
3951        ] {
3952            write_shard(
3953                dir.path(),
3954                "weights.safetensors",
3955                &[
3956                    FixtureTensor::f32("model.norm.weight", &[4], 1.0),
3957                    FixtureTensor::f32(marker, &[4], 1.0),
3958                    FixtureTensor::i64("mtp.unmaterializable"),
3959                ],
3960            );
3961            let bytes = std::fs::read(dir.path().join("weights.safetensors")).unwrap();
3962            // Exactly the bounded prefix/header parser and materializer used
3963            // by the FsOps entry point, without an RPC service in this test.
3964            let len = mtp_header_len(&bytes[..8]).unwrap();
3965            let allowed = mtp_header_supported(&bytes[8..8 + len]).unwrap();
3966            let mut weights = HashMap::new();
3967            let result = ModelFactory::create_tensors_from_safetensors(
3968                safetensors::SafeTensors::deserialize(&bytes).unwrap(),
3969                &mut weights,
3970                &Device::Cpu,
3971                DType::Float,
3972                allowed,
3973            );
3974            if marker == "mtp.layers.0.norm.weight" {
3975                assert!(result
3976                    .unwrap_err()
3977                    .to_string()
3978                    .contains("mtp.unmaterializable"));
3979            } else {
3980                result.unwrap();
3981                assert_eq!(weights.len(), 1);
3982                assert!(weights.contains_key("model.norm.weight"));
3983            }
3984        }
3985        assert!(mtp_header_len(&[0; 7]).is_err());
3986        assert!(mtp_header_len(&u64::MAX.to_le_bytes()).is_err());
3987    }
3988
3989    fn write_tiny_dense_checkpoint(dir: &Path, tensors: &[FixtureTensor]) {
3990        const SHARD: &str = "model-00001-of-00001.safetensors";
3991        write_shard(dir, SHARD, tensors);
3992        let entries: Vec<_> = tensors
3993            .iter()
3994            .map(|tensor| (tensor.name.as_str(), SHARD))
3995            .collect();
3996        write_index(dir, &entries);
3997
3998        let config = serde_json::json!({
3999            "architectures": ["LlamaForCausalLM"],
4000            "model_type": "llama",
4001            "hidden_size": 4,
4002            "intermediate_size": 8,
4003            "num_hidden_layers": 1,
4004            "num_attention_heads": 2,
4005            "num_key_value_heads": 2,
4006            "head_dim": 2,
4007            "vocab_size": 8,
4008            "max_position_embeddings": 16,
4009            "rms_norm_eps": 0.00001
4010        });
4011        std::fs::write(
4012            dir.join("config.json"),
4013            serde_json::to_vec(&config).unwrap(),
4014        )
4015        .unwrap();
4016    }
4017
4018    fn assert_pointer_resolution_error(error: anyhow::Error) {
4019        let message = error.to_string();
4020        let normalized = message.to_lowercase();
4021        assert!(
4022            normalized.contains("pointer")
4023                || normalized.contains("lfs")
4024                || normalized.contains("xet"),
4025            "stage path did not report pointer resolution: {message}"
4026        );
4027        assert!(
4028            !normalized.contains("failed to parse required stage shard")
4029                && !normalized.contains("invalid safetensors"),
4030            "stage path reached safetensors parsing before pointer resolution: {message}"
4031        );
4032    }
4033
4034    #[tokio::test]
4035    async fn stage_schema_validates_all_dense_roles() {
4036        let dir = tempfile::tempdir().unwrap();
4037        write_tiny_dense_checkpoint(dir.path(), &tiny_dense_tensors(true));
4038
4039        let config = ModelConfig::load(dir.path(), &HashMap::new()).unwrap();
4040        let plan = ModelFactory::stage_weight_plan(dir.path(), 1, 0..1).unwrap();
4041        let metadata = ModelFactory::stage_tensor_metadata(&plan).await.unwrap();
4042        ModelFactory::validate_stage_tensor_schema(&config, &(0..1), &metadata).unwrap();
4043
4044        for (name, bad_shape) in [
4045            ("model.embed_tokens.weight", vec![7, 4]),
4046            ("model.embed_tokens.weight", vec![8, 5]),
4047            ("lm_head.weight", vec![7, 4]),
4048            ("model.norm.weight", vec![5]),
4049            ("model.layers.0.self_attn.q_proj.weight", vec![3, 4]),
4050            ("model.layers.0.mlp.down_proj.weight", vec![4, 7]),
4051            ("model.layers.0.input_layernorm.weight", vec![5]),
4052        ] {
4053            let mut mismatched = metadata.clone();
4054            mismatched.insert(name.to_owned(), bad_shape);
4055            let error = ModelFactory::validate_stage_tensor_schema(&config, &(0..1), &mismatched)
4056                .unwrap_err();
4057            assert!(
4058                error.to_string().contains(name) && error.to_string().contains("shape mismatch"),
4059                "unexpected schema error for {name}: {error}"
4060            );
4061        }
4062
4063        let mut missing_layer_weight = metadata.clone();
4064        missing_layer_weight.remove("model.layers.0.self_attn.k_proj.weight");
4065        let missing_error =
4066            ModelFactory::validate_stage_tensor_schema(&config, &(0..1), &missing_layer_weight)
4067                .unwrap_err();
4068        assert!(
4069            missing_error
4070                .to_string()
4071                .contains("model.layers.0.self_attn.k_proj.weight"),
4072            "unexpected missing-weight error: {missing_error}"
4073        );
4074
4075        // With no explicit head, a complete-range stage ties the already
4076        // validated [vocab, hidden] embedding.
4077        let mut tied = metadata;
4078        tied.remove("lm_head.weight");
4079        ModelFactory::validate_stage_tensor_schema(&config, &(0..1), &tied).unwrap();
4080    }
4081
4082    #[tokio::test]
4083    async fn stage_schema_mismatch_fails_before_tensor_materialization() {
4084        let dir = tempfile::tempdir().unwrap();
4085        let mut tensors = tiny_dense_tensors(false);
4086        *tensors
4087            .iter_mut()
4088            .find(|tensor| tensor.name == "model.embed_tokens.weight")
4089            .unwrap() = FixtureTensor::f32("model.embed_tokens.weight", &[7, 4], 0.01);
4090        *tensors
4091            .iter_mut()
4092            .find(|tensor| tensor.name == "model.layers.0.self_attn.q_proj.weight")
4093            .unwrap() = FixtureTensor::i64_shape("model.layers.0.self_attn.q_proj.weight", &[4, 4]);
4094        write_tiny_dense_checkpoint(dir.path(), &tensors);
4095
4096        let error = ModelFactory::create_stage(
4097            dir.path(),
4098            &Device::Cpu,
4099            DType::Float,
4100            None,
4101            KVQuantType::None,
4102            None,
4103            ModelStageRequest { layer_range: 0..1 },
4104            false,
4105        )
4106        .await
4107        .err()
4108        .expect("config/tensor mismatch must fail");
4109        assert!(
4110            error.to_string().contains("model.embed_tokens.weight")
4111                && error.to_string().contains("shape mismatch"),
4112            "config mismatch must fail before the selected unsupported-dtype sentinel is \
4113             materialized: {error}"
4114        );
4115        assert!(
4116            !error.to_string().contains("unsupported dtype"),
4117            "selected tensors were materialized before schema validation: {error}"
4118        );
4119    }
4120
4121    #[tokio::test]
4122    async fn stage_schema_complete_range_matches_whole_model() {
4123        let dir = tempfile::tempdir().unwrap();
4124        write_tiny_dense_checkpoint(dir.path(), &tiny_dense_tensors(false));
4125
4126        let whole = ModelFactory::create(
4127            dir.path(),
4128            &Device::Cpu,
4129            DType::Float,
4130            None,
4131            KVQuantType::None,
4132            None,
4133            false,
4134            false,
4135        )
4136        .await
4137        .unwrap();
4138        let stage = ModelFactory::create_stage(
4139            dir.path(),
4140            &Device::Cpu,
4141            DType::Float,
4142            None,
4143            KVQuantType::None,
4144            None,
4145            ModelStageRequest { layer_range: 0..1 },
4146            false,
4147        )
4148        .await
4149        .unwrap();
4150
4151        let input = Tensor::from_slice(&[0_i64, 1, 2]).reshape([1, 3]);
4152        let whole_logits = whole.forward(&input, None).unwrap();
4153        let stage_logits = stage.forward(&input, None).unwrap();
4154        assert_eq!(whole_logits.size(), stage_logits.size());
4155        assert!(
4156            whole_logits.to_device(Device::Cpu).allclose(
4157                &stage_logits.to_device(Device::Cpu),
4158                1e-5,
4159                1e-5,
4160                false
4161            ),
4162            "valid complete-range stage forward diverged from ModelFactory::create"
4163        );
4164    }
4165
4166    #[test]
4167    fn stage_weight_plan_includes_top_level_fp8_companion_scales() {
4168        // FP8 checkpoints carry block-wise `<name>_scale_inv` companions for
4169        // quantized tensors. Layer-prefixed scales enter the stage plan via the
4170        // layer-index filter; top-level embedding/lm_head scales must be added
4171        // explicitly, or the stage loads a scale-less FP8 tensor and eager
4172        // dequant casts raw FP8 codes to BF16 (#1519 Codex review).
4173        let dir = tempfile::tempdir().unwrap();
4174        let shard = "model-00001-of-00001.safetensors";
4175        write_index(
4176            dir.path(),
4177            &[
4178                ("model.embed_tokens.weight", shard),
4179                ("model.embed_tokens.weight_scale_inv", shard),
4180                ("model.norm.weight", shard),
4181                ("lm_head.weight", shard),
4182                ("lm_head.weight_scale_inv", shard),
4183                ("model.layers.0.self_attn.q_proj.weight", shard),
4184                ("model.layers.0.self_attn.q_proj.weight_scale_inv", shard),
4185            ],
4186        );
4187
4188        let plan = ModelFactory::stage_weight_plan(dir.path(), 1, 0..1).unwrap();
4189        let names: Vec<&str> = plan.values().flatten().map(String::as_str).collect();
4190        for expected in [
4191            "model.embed_tokens.weight",
4192            "model.embed_tokens.weight_scale_inv",
4193            "model.norm.weight",
4194            "lm_head.weight",
4195            "lm_head.weight_scale_inv",
4196            "model.layers.0.self_attn.q_proj.weight",
4197            "model.layers.0.self_attn.q_proj.weight_scale_inv",
4198        ] {
4199            assert!(names.contains(&expected), "stage plan missing {expected}");
4200        }
4201        // No phantom scale entries: the plan contains exactly these tensors.
4202        assert_eq!(names.len(), 7, "unexpected extra tensors in stage plan");
4203    }
4204
4205    #[tokio::test]
4206    async fn stage_pointer_shard_fails_before_mmap_in_preflight_and_selected_load() {
4207        const SHARD: &str = "model-00001-of-00001.safetensors";
4208
4209        let dir = tempfile::tempdir().unwrap();
4210        write_tiny_dense_checkpoint(dir.path(), &tiny_dense_tensors(true));
4211        std::fs::write(
4212            dir.path().join(SHARD),
4213            b"version https://git-lfs.github.com/spec/v1\n\
4214oid sha256:0000000000000000000000000000000000000000000000000000000000000000\n\
4215size 4096\n",
4216        )
4217        .unwrap();
4218
4219        let plan = ModelFactory::stage_weight_plan(dir.path(), 1, 0..1).unwrap();
4220        let preflight_error = ModelFactory::stage_tensor_metadata(&plan)
4221            .await
4222            .unwrap_err();
4223        assert_pointer_resolution_error(preflight_error);
4224
4225        let selected_error = ModelFactory::load_weights_for_stage_plan(
4226            plan,
4227            0..1,
4228            &Device::Cpu,
4229            DType::Float,
4230            native_finite_policy(dir.path()),
4231        )
4232        .await
4233        .unwrap_err();
4234        assert_pointer_resolution_error(selected_error);
4235    }
4236
4237    #[tokio::test]
4238    async fn out_of_range_sentinel_in_needed_shard_is_never_materialized() {
4239        let dir = tempfile::tempdir().unwrap();
4240        let shard = "model-00001-of-00001.safetensors";
4241        write_shard(
4242            dir.path(),
4243            shard,
4244            &[
4245                FixtureTensor::f32("model.embed_tokens.weight", &[2, 2], 0.1),
4246                FixtureTensor::f32("model.layers.0.marker.weight", &[2, 2], 0.2),
4247                FixtureTensor::i64("model.layers.1.sentinel.weight"),
4248            ],
4249        );
4250        write_index(
4251            dir.path(),
4252            &[
4253                ("model.embed_tokens.weight", shard),
4254                ("model.layers.0.marker.weight", shard),
4255                ("model.layers.1.sentinel.weight", shard),
4256            ],
4257        );
4258
4259        let stage =
4260            ModelFactory::load_weights_for_stage(dir.path(), 2, 0..1, &Device::Cpu, DType::Float)
4261                .await
4262                .unwrap();
4263        assert_eq!(stage.len(), 2);
4264        assert!(stage.contains_key("model.embed_tokens.weight"));
4265        assert!(stage.contains_key("model.layers.0.marker.weight"));
4266        assert!(!stage.contains_key("model.layers.1.sentinel.weight"));
4267
4268        let full_error = ModelFactory::load_weights(dir.path(), &Device::Cpu, DType::Float, false)
4269            .await
4270            .unwrap_err();
4271        assert!(
4272            full_error.to_string().contains("unsupported dtype"),
4273            "the sentinel must prove that whole-shard tensor materialization fails: {full_error}"
4274        );
4275    }
4276
4277    #[tokio::test]
4278    async fn stage_create_uses_real_range_and_opens_only_needed_shards() {
4279        const NEEDED: &str = "model-00001-of-00002.safetensors";
4280        const UNNEEDED: &str = "model-00002-of-00002.safetensors";
4281
4282        let dir = tempfile::tempdir().unwrap();
4283        let mut tensors = vec![FixtureTensor::f32(
4284            "model.embed_tokens.weight",
4285            &[8, 4],
4286            0.1,
4287        )];
4288        let prefix = "model.layers.0";
4289        for (suffix, shape) in [
4290            ("self_attn.q_proj.weight", vec![4, 4]),
4291            ("self_attn.k_proj.weight", vec![4, 4]),
4292            ("self_attn.v_proj.weight", vec![4, 4]),
4293            ("self_attn.o_proj.weight", vec![4, 4]),
4294            ("mlp.gate_proj.weight", vec![8, 4]),
4295            ("mlp.up_proj.weight", vec![8, 4]),
4296            ("mlp.down_proj.weight", vec![4, 8]),
4297            ("input_layernorm.weight", vec![4]),
4298            ("post_attention_layernorm.weight", vec![4]),
4299        ] {
4300            tensors.push(FixtureTensor::f32(
4301                format!("{prefix}.{suffix}"),
4302                &shape,
4303                0.2,
4304            ));
4305        }
4306        write_shard(dir.path(), NEEDED, &tensors);
4307
4308        let mut entries: Vec<(String, String)> = tensors
4309            .iter()
4310            .map(|tensor| (tensor.name.clone(), NEEDED.to_owned()))
4311            .collect();
4312        entries.push((
4313            "model.layers.1.self_attn.q_proj.weight".to_owned(),
4314            UNNEEDED.to_owned(),
4315        ));
4316        let borrowed_entries: Vec<_> = entries
4317            .iter()
4318            .map(|(tensor, shard)| (tensor.as_str(), shard.as_str()))
4319            .collect();
4320        write_index(dir.path(), &borrowed_entries);
4321
4322        let config = serde_json::json!({
4323            "architectures": ["LlamaForCausalLM"],
4324            "model_type": "llama",
4325            "hidden_size": 4,
4326            "intermediate_size": 8,
4327            "num_hidden_layers": 2,
4328            "num_attention_heads": 2,
4329            "num_key_value_heads": 2,
4330            "head_dim": 2,
4331            "vocab_size": 8,
4332            "max_position_embeddings": 16,
4333            "rms_norm_eps": 0.00001
4334        });
4335        std::fs::write(
4336            dir.path().join("config.json"),
4337            serde_json::to_vec(&config).unwrap(),
4338        )
4339        .unwrap();
4340
4341        // UNNEEDED intentionally does not exist. Successful construction proves
4342        // the real [0,1) request did not open it.
4343        let model = ModelFactory::create_stage(
4344            dir.path(),
4345            &Device::Cpu,
4346            DType::Float,
4347            None,
4348            KVQuantType::None,
4349            None,
4350            ModelStageRequest { layer_range: 0..1 },
4351            false,
4352        )
4353        .await
4354        .unwrap();
4355        let input = Tensor::from_slice(&[0_i64, 1]).reshape([1, 2]);
4356        let embedded = model.embed_tokens(&input).unwrap();
4357        assert_eq!(embedded.size(), [1, 2, 4]);
4358    }
4359
4360    #[tokio::test]
4361    async fn partial_stage_loads_fewer_tensors_and_bytes_than_whole_model() {
4362        let dir = tempfile::tempdir().unwrap();
4363        let first = "model-00001-of-00002.safetensors";
4364        let last = "model-00002-of-00002.safetensors";
4365        write_shard(
4366            dir.path(),
4367            first,
4368            &[
4369                FixtureTensor::f32("model.embed_tokens.weight", &[8, 4], 0.1),
4370                FixtureTensor::f32("model.layers.0.marker.weight", &[4, 4], 0.2),
4371            ],
4372        );
4373        write_shard(
4374            dir.path(),
4375            last,
4376            &[
4377                FixtureTensor::f32("model.layers.1.marker.weight", &[16, 16], 0.3),
4378                FixtureTensor::f32("model.norm.weight", &[4], 0.4),
4379                FixtureTensor::f32("lm_head.weight", &[8, 4], 0.5),
4380            ],
4381        );
4382        write_index(
4383            dir.path(),
4384            &[
4385                ("model.embed_tokens.weight", first),
4386                ("model.layers.0.marker.weight", first),
4387                ("model.layers.1.marker.weight", last),
4388                ("model.norm.weight", last),
4389                ("lm_head.weight", last),
4390            ],
4391        );
4392
4393        let partial =
4394            ModelFactory::load_weights_for_stage(dir.path(), 2, 1..2, &Device::Cpu, DType::Float)
4395                .await
4396                .unwrap();
4397        let whole = ModelFactory::load_weights(dir.path(), &Device::Cpu, DType::Float, false)
4398            .await
4399            .unwrap();
4400
4401        let partial_names: BTreeSet<_> = partial.keys().map(String::as_str).collect();
4402        assert_eq!(
4403            partial_names,
4404            BTreeSet::from([
4405                "lm_head.weight",
4406                "model.layers.1.marker.weight",
4407                "model.norm.weight",
4408            ])
4409        );
4410        assert!(partial.len() < whole.len());
4411        assert!(loaded_bytes(&partial) < loaded_bytes(&whole));
4412    }
4413
4414    #[tokio::test]
4415    async fn stage_mode_fails_closed_without_a_valid_index() {
4416        let dir = tempfile::tempdir().unwrap();
4417        write_shard(
4418            dir.path(),
4419            "model.safetensors",
4420            &[FixtureTensor::f32(
4421                "model.embed_tokens.weight",
4422                &[2, 2],
4423                0.1,
4424            )],
4425        );
4426
4427        let missing =
4428            ModelFactory::load_weights_for_stage(dir.path(), 1, 0..1, &Device::Cpu, DType::Float)
4429                .await
4430                .unwrap_err();
4431        assert!(
4432            missing
4433                .to_string()
4434                .contains("requires authoritative shard index"),
4435            "got: {missing}"
4436        );
4437
4438        std::fs::write(
4439            dir.path().join("model.safetensors.index.json"),
4440            br#"{"weight_map":[]}"#,
4441        )
4442        .unwrap();
4443        let malformed =
4444            ModelFactory::load_weights_for_stage(dir.path(), 1, 0..1, &Device::Cpu, DType::Float)
4445                .await
4446                .unwrap_err();
4447        assert!(
4448            malformed.to_string().contains("valid `weight_map` object"),
4449            "got: {malformed}"
4450        );
4451    }
4452}
4453
4454#[cfg(test)]
4455#[allow(clippy::expect_used, clippy::unwrap_used)]
4456mod fp8_dequant_tests {
4457    use super::*;
4458    use crate::runtime::architectures::llama::LinearProjection;
4459    use safetensors::tensor::{serialize, Dtype, TensorView};
4460
4461    /// FP8 E4M3 byte for an exactly-representable power of two:
4462    /// value = (-1)^sign * 2^log2 (S EEEE MMM, exponent bias 7, mantissa 0).
4463    fn f8e4m3_pow2(negative: bool, log2: i32) -> u8 {
4464        let sign = if negative { 0x80u8 } else { 0 };
4465        sign | (((log2 + 7) as u8) << 3)
4466    }
4467
4468    /// Build an FP8 E4M3 weight tensor with a deterministic power-of-two
4469    /// pattern (exactly representable in both E4M3 and BF16).
4470    fn fp8_weight(rows: i64, cols: i64) -> Tensor {
4471        let vals: Vec<f32> = (0..(rows * cols))
4472            .map(|i| {
4473                let mag = [0.5f32, 1.0, 2.0][(i % 3) as usize];
4474                if i % 2 == 1 {
4475                    -mag
4476                } else {
4477                    mag
4478                }
4479            })
4480            .collect();
4481        Tensor::from_slice(&vals)
4482            .reshape([rows, cols])
4483            .to_kind(DType::Float8e4m3fn)
4484    }
4485
4486    /// Deterministic FP8 weight + companion BF16 block scale (single block).
4487    fn fp8_weight_with_scale(rows: i64, cols: i64, scale: f32) -> (Tensor, Tensor) {
4488        let weight = fp8_weight(rows, cols);
4489        let scale = Tensor::from_slice(&[scale])
4490            .reshape([1, 1])
4491            .to_kind(DType::BFloat16);
4492        (weight, scale)
4493    }
4494
4495    #[test]
4496    fn dequant_at_load_materializes_bf16_and_drops_scales() {
4497        let (weight, scale) = fp8_weight_with_scale(4, 4, 0.5);
4498        let untouched = Tensor::from_slice(&[1.0f32, 2.0, 3.0, 4.0]).to_kind(DType::BFloat16);
4499        let mut weights = HashMap::from([
4500            ("w.weight".to_owned(), weight),
4501            ("w.weight_scale_inv".to_owned(), scale),
4502            ("norm.weight".to_owned(), untouched),
4503        ]);
4504
4505        let converted = dequantize_fp8_weights_at_load(&mut weights);
4506
4507        assert_eq!(converted, 1);
4508        let w = weights.get("w.weight").expect("weight kept");
4509        assert_eq!(
4510            w.kind(),
4511            DType::BFloat16,
4512            "FP8 weight must be materialized as BF16"
4513        );
4514        assert!(
4515            !weights.contains_key("w.weight_scale_inv"),
4516            "block scale must be dropped after load-time dequant"
4517        );
4518        assert_eq!(
4519            weights.get("norm.weight").expect("norm kept").kind(),
4520            DType::BFloat16,
4521            "non-FP8 tensors untouched"
4522        );
4523
4524        // Values must equal the eager block dequantization: pattern is
4525        // ±{0.5, 1, 2} with scale 0.5 → ±{0.25, 0.5, 1} (exact in BF16).
4526        let expected: Vec<f32> = (0..16)
4527            .map(|i| {
4528                let mag = [0.25f32, 0.5, 1.0][(i % 3) as usize];
4529                if i % 2 == 1 {
4530                    -mag
4531                } else {
4532                    mag
4533                }
4534            })
4535            .collect();
4536        let expected = Tensor::from_slice(&expected).reshape([4, 4]);
4537        let got = w.to_kind(DType::Float);
4538        assert!(
4539            got.allclose(&expected, 1e-6, 1e-6, false),
4540            "load-time dequant values diverged from eager block dequantization"
4541        );
4542    }
4543
4544    #[test]
4545    fn dequant_at_load_forward_matches_lazy_path() {
4546        // Same stored checkpoint tensor map fed through the production
4547        // `LinearProjection::take` load path twice: once left FP8 (lazy
4548        // per-matmul dequant), once dequantized at load.
4549        let mut lazy_weights = HashMap::new();
4550        let (w, s) = fp8_weight_with_scale(4, 8, 0.5);
4551        lazy_weights.insert("p.weight".to_owned(), w);
4552        lazy_weights.insert("p.weight_scale_inv".to_owned(), s);
4553        let lazy_proj = LinearProjection::take(&mut lazy_weights, "p.weight").unwrap();
4554        assert_eq!(lazy_proj.weight.kind(), DType::Float8e4m3fn);
4555        assert!(lazy_proj.scale.is_some());
4556
4557        let mut eager_weights = HashMap::new();
4558        let (w, s) = fp8_weight_with_scale(4, 8, 0.5);
4559        eager_weights.insert("p.weight".to_owned(), w);
4560        eager_weights.insert("p.weight_scale_inv".to_owned(), s);
4561        assert_eq!(dequantize_fp8_weights_at_load(&mut eager_weights), 1);
4562        let eager_proj = LinearProjection::take(&mut eager_weights, "p.weight").unwrap();
4563        assert_eq!(eager_proj.weight.kind(), DType::BFloat16);
4564        assert!(eager_proj.scale.is_none(), "scale must be consumed at load");
4565
4566        let input_vals: Vec<f32> = (0..8).map(|i| 0.125 * (i as f32 + 1.0)).collect();
4567        let input = Tensor::from_slice(&input_vals)
4568            .reshape([1, 8])
4569            .to_kind(DType::BFloat16);
4570        let y_lazy = lazy_proj.apply(&input).to_kind(DType::Float);
4571        let y_eager = eager_proj.apply(&input).to_kind(DType::Float);
4572        assert_eq!(y_lazy.size(), y_eager.size());
4573        assert!(
4574            y_lazy.allclose(&y_eager, 1e-3, 1e-3, false),
4575            "load-time dequant forward diverged from lazy per-matmul dequant"
4576        );
4577    }
4578
4579    #[test]
4580    fn cpu_fp32_loader_preserves_bf16_values_and_channel_fp8_scales() {
4581        let floating = RawTensor::bf16("model.norm.weight", &[2], 0.125);
4582        let packed = RawTensor::f8("p.weight", &[2, 3]);
4583        let scale = RawTensor::bf16("p.weight_scale", &[2, 1], 0.25);
4584        let fixtures = [floating, packed, scale];
4585        let views: Vec<_> = fixtures
4586            .iter()
4587            .map(|t| {
4588                (
4589                    t.name.clone(),
4590                    TensorView::new(t.dtype, t.shape.clone(), &t.data).unwrap(),
4591                )
4592            })
4593            .collect();
4594        let bytes = serialize(views, &None).unwrap();
4595        let tensors = safetensors::SafeTensors::deserialize(&bytes).unwrap();
4596        let mut weights = HashMap::new();
4597        ModelFactory::create_tensors_from_safetensors(
4598            tensors,
4599            &mut weights,
4600            &Device::Cpu,
4601            DType::Float,
4602            false,
4603        )
4604        .unwrap();
4605        assert_eq!(weights["model.norm.weight"].kind(), DType::Float);
4606        assert_eq!(weights["model.norm.weight"].double_value(&[0]), 0.125);
4607        assert_eq!(weights["p.weight"].kind(), DType::Float8e4m3fn);
4608        assert_eq!(weights["p.weight_scale"].kind(), DType::BFloat16);
4609        assert_eq!(
4610            dequantize_fp8_weights_at_load(&mut weights),
4611            0,
4612            "eager block-only decoder must not erase dynamic activation QDQ"
4613        );
4614        assert!(weights.contains_key("p.weight_scale"));
4615        let projection = LinearProjection::take(&mut weights, "p.weight").unwrap();
4616        let result = projection.apply(&Tensor::ones([1, 3], (DType::Float, Device::Cpu)));
4617        assert_eq!(result.kind(), DType::Float);
4618        assert!(result.isfinite().all().int64_value(&[]) != 0);
4619    }
4620
4621    #[test]
4622    fn dequant_at_load_noop_without_fp8() {
4623        let mut weights = HashMap::from([(
4624            "w.weight".to_owned(),
4625            Tensor::from_slice(&[1.0f32, 2.0]).to_kind(DType::BFloat16),
4626        )]);
4627        assert_eq!(dequantize_fp8_weights_at_load(&mut weights), 0);
4628        assert_eq!(weights["w.weight"].kind(), DType::BFloat16);
4629    }
4630
4631    #[test]
4632    fn dequant_at_load_leaves_nonconforming_scale_untouched() {
4633        // Scale shape [3, 1] does not block-divide weight [4, 4]: the pair must
4634        // be left FP8 + scale for the lazy path (which fails loudly), not cast
4635        // to BF16 without scales, and must not count as converted.
4636        let (weight, _) = fp8_weight_with_scale(4, 4, 0.5);
4637        let bad_scale = Tensor::from_slice(&[0.5f32, 0.5, 0.5])
4638            .reshape([3, 1])
4639            .to_kind(DType::BFloat16);
4640        let mut weights = HashMap::from([
4641            ("w.weight".to_owned(), weight),
4642            ("w.weight_scale_inv".to_owned(), bad_scale),
4643        ]);
4644
4645        assert_eq!(dequantize_fp8_weights_at_load(&mut weights), 0);
4646        assert_eq!(
4647            weights["w.weight"].kind(),
4648            DType::Float8e4m3fn,
4649            "non-conforming weight must stay FP8 for the lazy dequant path"
4650        );
4651        assert!(
4652            weights.contains_key("w.weight_scale_inv"),
4653            "non-conforming scale must stay in the map"
4654        );
4655    }
4656
4657    /// Byte-level fixture for the end-to-end checkpoint test.
4658    struct RawTensor {
4659        name: String,
4660        dtype: Dtype,
4661        shape: Vec<usize>,
4662        data: Vec<u8>,
4663    }
4664
4665    impl RawTensor {
4666        fn bf16(name: impl Into<String>, shape: &[usize], fill: f32) -> Self {
4667            let elements = shape.iter().product();
4668            let bits = ((fill.to_bits() >> 16) as u16).to_le_bytes();
4669            Self {
4670                name: name.into(),
4671                dtype: Dtype::BF16,
4672                shape: shape.to_vec(),
4673                data: (0..elements).flat_map(|_| bits).collect(),
4674            }
4675        }
4676
4677        fn f8(name: impl Into<String>, shape: &[usize]) -> Self {
4678            let elements = shape.iter().product();
4679            Self {
4680                name: name.into(),
4681                dtype: Dtype::F8_E4M3,
4682                shape: shape.to_vec(),
4683                data: (0..elements)
4684                    .map(|i| f8e4m3_pow2(i % 2 == 1, ((i % 3) as i32) - 1))
4685                    .collect(),
4686            }
4687        }
4688
4689        fn f8_scale(name: impl Into<String>, scale: f32) -> Self {
4690            Self::bf16(name, &[1, 1], scale)
4691        }
4692    }
4693
4694    /// Write a tiny Llama checkpoint with every projection weight stored as
4695    /// FP8 E4M3 + a 1x1 BF16 block scale (norms/embeddings stay BF16, as in
4696    /// real FP8 checkpoints).
4697    fn write_tiny_fp8_checkpoint(dir: &Path) {
4698        const SHARD: &str = "model-00001-of-00001.safetensors";
4699        let mut tensors = vec![
4700            RawTensor::bf16("model.embed_tokens.weight", &[8, 4], 0.02),
4701            RawTensor::bf16("model.norm.weight", &[4], 1.0),
4702        ];
4703        let prefix = "model.layers.0";
4704        for (suffix, shape) in [
4705            ("self_attn.q_proj.weight", vec![4, 4]),
4706            ("self_attn.k_proj.weight", vec![4, 4]),
4707            ("self_attn.v_proj.weight", vec![4, 4]),
4708            ("self_attn.o_proj.weight", vec![4, 4]),
4709            ("mlp.gate_proj.weight", vec![8, 4]),
4710            ("mlp.up_proj.weight", vec![8, 4]),
4711            ("mlp.down_proj.weight", vec![4, 8]),
4712        ] {
4713            tensors.push(RawTensor::f8(format!("{prefix}.{suffix}"), &shape));
4714            tensors.push(RawTensor::f8_scale(
4715                format!("{prefix}.{suffix}_scale_inv"),
4716                0.5,
4717            ));
4718        }
4719        tensors.push(RawTensor::bf16(
4720            format!("{prefix}.input_layernorm.weight"),
4721            &[4],
4722            1.0,
4723        ));
4724        tensors.push(RawTensor::bf16(
4725            format!("{prefix}.post_attention_layernorm.weight"),
4726            &[4],
4727            1.0,
4728        ));
4729
4730        let views: Vec<_> = tensors
4731            .iter()
4732            .map(|tensor| {
4733                (
4734                    tensor.name.clone(),
4735                    TensorView::new(tensor.dtype, tensor.shape.clone(), &tensor.data).unwrap(),
4736                )
4737            })
4738            .collect();
4739        let bytes = serialize(views, &None).unwrap();
4740        std::fs::write(dir.join(SHARD), bytes).unwrap();
4741
4742        let weight_map: BTreeMap<_, _> = tensors
4743            .iter()
4744            .map(|tensor| (tensor.name.clone(), SHARD.to_owned()))
4745            .collect();
4746        let index = serde_json::json!({ "weight_map": weight_map });
4747        std::fs::write(
4748            dir.join("model.safetensors.index.json"),
4749            serde_json::to_vec(&index).unwrap(),
4750        )
4751        .unwrap();
4752
4753        let config = serde_json::json!({
4754            "architectures": ["LlamaForCausalLM"],
4755            "model_type": "llama",
4756            "hidden_size": 4,
4757            "intermediate_size": 8,
4758            "num_hidden_layers": 1,
4759            "num_attention_heads": 2,
4760            "num_key_value_heads": 2,
4761            "head_dim": 2,
4762            "vocab_size": 8,
4763            "max_position_embeddings": 16,
4764            "rms_norm_eps": 0.00001
4765        });
4766        std::fs::write(
4767            dir.join("config.json"),
4768            serde_json::to_vec(&config).unwrap(),
4769        )
4770        .unwrap();
4771    }
4772
4773    /// Resource-admitted, external-checkpoint CPU gate. Never downloads models.
4774    /// Run only as the exact ignored test in its own process (four CPU threads).
4775    #[tokio::test]
4776    #[ignore = "requires explicit real-model path, receipt path and parent memory admission"]
4777    async fn real_clef_fp8_cpu_backbone_smoke() {
4778        let model_path = PathBuf::from(
4779            std::env::var("HYPRSTREAM_CLEF_FP8_MODEL").expect("explicit model path required"),
4780        );
4781        let receipt_path = PathBuf::from(
4782            std::env::var("HYPRSTREAM_CLEF_BACKBONE_RECEIPT")
4783                .expect("explicit smoke receipt path required"),
4784        );
4785        assert!(model_path.is_dir());
4786        let memory =
4787            std::fs::read_to_string("/proc/meminfo").expect("Linux memory admission required");
4788        let available_kib = memory
4789            .lines()
4790            .find(|line| line.starts_with("MemAvailable:"))
4791            .and_then(|line| line.split_whitespace().nth(1))
4792            .and_then(|value| value.parse::<u64>().ok())
4793            .expect("MemAvailable required");
4794        assert!(
4795            available_kib >= 21 * 1024 * 1024,
4796            "CPU smoke requires21GiB MemAvailable at actual test start; got{}KiB",
4797            available_kib
4798        );
4799        tch::set_num_threads(4);
4800        tch::set_num_interop_threads(1);
4801        let tokenizer =
4802            tokenizers::Tokenizer::from_file(model_path.join("tokenizer.json")).unwrap();
4803        let encode = |text: &str| {
4804            let encoded = tokenizer.encode(text, false).unwrap();
4805            let ids: Vec<i64> = encoded.get_ids().iter().map(|id| *id as i64).collect();
4806            assert!(!ids.is_empty() && ids.len() <= 32);
4807            ids
4808        };
4809        let ids_a = encode("The quick brown fox jumps over a lazy dog.");
4810        let ids_b = encode("A different independent request about ocean waves.");
4811        let a = Tensor::from_slice(&ids_a).reshape([1, ids_a.len() as i64]);
4812        let b = Tensor::from_slice(&ids_b).reshape([1, ids_b.len() as i64]);
4813        let started = std::time::Instant::now();
4814        let model = ModelFactory::create(
4815            &model_path,
4816            &Device::Cpu,
4817            DType::Float,
4818            Some(64),
4819            KVQuantType::None,
4820            None,
4821            false,
4822            false,
4823        )
4824        .await
4825        .unwrap();
4826        let load_ms = started.elapsed().as_millis();
4827        assert!(matches!(
4828            model.architecture(),
4829            crate::runtime::architectures::ModelArchitecture::Qwen3_5
4830        ));
4831        let _no_grad = tch::no_grad_guard();
4832        let forward_start = std::time::Instant::now();
4833        let hidden = model.decision_hidden_states(&a).unwrap();
4834        let first_forward_ms = forward_start.elapsed().as_millis();
4835        assert_eq!(hidden.size(), [ids_a.len() as i64, 4096]);
4836        assert_eq!(hidden.kind(), DType::Float);
4837        assert!(hidden.isfinite().all().int64_value(&[]) != 0);
4838        let rows = model
4839            .decision_output_embedding_rows(&ids_a[..3.min(ids_a.len())])
4840            .unwrap();
4841        assert_eq!(rows.size(), [3.min(ids_a.len()) as i64, 4096]);
4842        assert_eq!(rows.kind(), hidden.kind());
4843        assert!(rows.isfinite().all().int64_value(&[]) != 0);
4844        // Exercise another fresh request, then contaminate persistent generation
4845        // state without performing any vocabulary projection.
4846        let other = model.decision_hidden_states(&b).unwrap();
4847        assert!(other.isfinite().all().int64_value(&[]) != 0);
4848        let emb = model.embed_tokens(&b).unwrap();
4849        let _ = model
4850            .forward_layers(&emb, 0..model.num_layers(), 0, None)
4851            .unwrap();
4852        let repeated = model.decision_hidden_states(&a).unwrap();
4853        let repeated_max_abs = (&hidden - &repeated).abs().max().double_value(&[]);
4854        assert!(
4855            hidden.allclose(&repeated, 1e-6, 1e-6, false),
4856            "cross-request state contamination"
4857        );
4858        let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
4859        let receipt = serde_json::json!({
4860            "gate":"native-clef-backbone-cpu-smoke", "result":"PASS",
4861            "model_path":model_path,"candidate_sha":std::env::var("HYPRSTREAM_CANDIDATE_SHA").ok(),
4862            "device":"cpu","compute_dtype":"float32","cpu_threads":4,"context_limit":64,"admission_available_kib":available_kib,
4863            "input_a_tokens":ids_a,"input_b_tokens":ids_b,"load_ms":load_ms,
4864            "first_forward_ms":first_forward_ms,"elapsed_ms":started.elapsed().as_millis(),
4865            "hidden_shape":hidden.size(),"lm_rows_shape":rows.size(),"repeated_max_abs":repeated_max_abs,
4866            "process_memory":status.lines().filter(|line| line.starts_with("VmRSS:") || line.starts_with("VmHWM:")).collect::<Vec<_>>(),
4867            "acceptance_scope":"native-backbone-smoke-only",
4868            "not_proven":["joint-head parity","trusted-reference parity","authenticated serving","GPU eligibility"]
4869        });
4870        std::fs::write(receipt_path, serde_json::to_vec_pretty(&receipt).unwrap()).unwrap();
4871    }
4872
4873    #[tokio::test]
4874    async fn fp8_dequant_load_checkpoint_forward_matches_lazy_path() {
4875        let dir = tempfile::tempdir().unwrap();
4876        write_tiny_fp8_checkpoint(dir.path());
4877
4878        let eager = ModelFactory::create(
4879            dir.path(),
4880            &Device::Cpu,
4881            DType::BFloat16,
4882            None,
4883            KVQuantType::None,
4884            None,
4885            true,
4886            false,
4887        )
4888        .await
4889        .unwrap();
4890        let lazy = ModelFactory::create(
4891            dir.path(),
4892            &Device::Cpu,
4893            DType::BFloat16,
4894            None,
4895            KVQuantType::None,
4896            None,
4897            false,
4898            false,
4899        )
4900        .await
4901        .unwrap();
4902
4903        let input = Tensor::from_slice(&[0_i64, 1, 2]).reshape([1, 3]);
4904        let eager_logits = eager.forward(&input, None).unwrap().to_kind(DType::Float);
4905        let lazy_logits = lazy.forward(&input, None).unwrap().to_kind(DType::Float);
4906        assert_eq!(eager_logits.size(), lazy_logits.size());
4907        assert!(
4908            eager_logits.allclose(&lazy_logits, 1e-3, 1e-3, false),
4909            "checkpoint loaded with fp8_dequant_load diverged from lazy-dequant load"
4910        );
4911    }
4912}