1use anyhow::{anyhow, bail, Context, Result};
7use serde::Deserialize;
8use std::collections::{BTreeMap, BTreeSet, HashMap};
9use std::ops::Range;
10use std::path::{Path, PathBuf};
11use tch::{Device, Kind as DType, Tensor};
12use tracing::{debug, info, instrument, warn};
13
14use super::architectures::{gemma::GemmaModel, llama::LlamaModel, ModelOperations};
15use super::device_pool::DevicePool;
16use super::model_config::{ModelArchitecture, ModelConfig};
17use super::torch_utils::{estimate_tensor_size_mb, safe_to_device};
18use crate::worktree_ext::WorktreeClientExt;
19use hyprstream_rpc_std::model_client::KVQuantType;
20use hyprstream_rpc_std::registry_client::WorktreeClient;
21
22const STRICT_LOADER_ENV: &str = "HYPRSTREAM_STRICT_LOADER";
27
28fn native_finite_policy(model_path: &Path) -> bool {
33 ModelConfig::detect_architecture(model_path) == ModelArchitecture::Qwen3_5
34}
35
36fn validate_finite_payload(name: &str, dtype: safetensors::Dtype, data: &[u8]) -> Result<()> {
41 use safetensors::Dtype;
42 let invalid = match dtype {
43 Dtype::BF16 => data
44 .chunks_exact(2)
45 .position(|b| u16::from_le_bytes([b[0], b[1]]) & 0x7f80 == 0x7f80),
46 Dtype::F16 => data
47 .chunks_exact(2)
48 .position(|b| u16::from_le_bytes([b[0], b[1]]) & 0x7c00 == 0x7c00),
49 Dtype::F32 => data.chunks_exact(4).position(|b| {
50 u32::from_le_bytes([b[0], b[1], b[2], b[3]]) & 0x7f80_0000 == 0x7f80_0000
51 }),
52 Dtype::F8_E4M3 => data.iter().position(|b| b & 0x7f == 0x7f),
53 Dtype::F8_E5M2 => data.iter().position(|b| b & 0x7c == 0x7c),
54 _ => None,
57 };
58 if let Some(index) = invalid {
59 bail!("nonfinite native Qwen3.5 checkpoint tensor `{name}` ({dtype:?}) at element {index}");
60 }
61 Ok(())
62}
63
64fn strict_loader_enabled() -> bool {
65 std::env::var(STRICT_LOADER_ENV)
66 .map(|v| {
67 matches!(
68 v.trim().to_lowercase().as_str(),
69 "1" | "true" | "yes" | "on"
70 )
71 })
72 .unwrap_or(false)
73}
74
75const MTP_HEADER_LIMIT: usize = 100_000_000;
78
79fn mtp_runtime_supported(requested: bool, kv_quant_type: KVQuantType) -> bool {
80 requested && kv_quant_type == KVQuantType::None
83}
84
85fn mtp_header_len(prefix: &[u8]) -> Result<usize> {
86 let encoded: [u8; 8] = prefix
87 .try_into()
88 .context("truncated safetensors header length")?;
89 let len = usize::try_from(u64::from_le_bytes(encoded))?;
90 anyhow::ensure!(
91 len <= MTP_HEADER_LIMIT,
92 "safetensors header exceeds metadata limit"
93 );
94 Ok(len)
95}
96
97fn mtp_config_supported(config: &serde_json::Value) -> bool {
98 [config, &config["text_config"]].into_iter().all(|cfg| {
101 !cfg["model_type"]
102 .as_str()
103 .is_some_and(|name| name.contains("moe"))
104 && cfg["num_experts"].as_u64().unwrap_or(0) == 0
105 && cfg
106 .get("mtp_num_hidden_layers")
107 .is_none_or(|n| n.as_u64() == Some(1))
108 })
109}
110
111fn mtp_header_supported(header: &[u8]) -> Result<bool> {
112 let entries: serde_json::Map<String, serde_json::Value> = serde_json::from_slice(header)?;
113 Ok(entries.keys().all(|name| {
114 if !name.starts_with("mtp.") {
115 return true;
116 }
117 !name.contains(".experts.")
118 && !name.contains(".shared_expert.")
119 && name
120 .strip_prefix("mtp.layers.")
121 .is_none_or(|layer| layer.starts_with("0."))
122 }))
123}
124
125pub(crate) fn dequantize_fp8_weights_at_load(weights: &mut HashMap<String, Tensor>) -> usize {
137 #[inline]
138 fn is_fp8(kind: tch::Kind) -> bool {
139 matches!(kind, tch::Kind::Float8e4m3fn | tch::Kind::Float8e5m2)
140 }
141
142 let fp8_keys: Vec<String> = weights
143 .iter()
144 .filter(|(name, tensor)| is_fp8(tensor.kind()) && !name.ends_with("_scale_inv"))
145 .map(|(name, _)| name.clone())
146 .collect();
147 let mut converted = 0;
148
149 for key in fp8_keys {
150 if weights.contains_key(&format!("{key}_scale")) {
151 continue;
152 }
153 let Some(weight) = weights.remove(&key) else {
154 continue;
155 };
156 let scale_key = format!("{key}_scale_inv");
157 let scale = weights.remove(&scale_key);
158 if let Some(s) = &scale {
159 let ws = weight.size();
160 let ss = s.size();
161 if !(ws.len() == 2 && ss.len() == 2 && ws[0] % ss[0] == 0 && ws[1] % ss[1] == 0) {
162 warn!(
167 "FP8 weight '{}' scale shape {:?} does not block-divide weight shape {:?}; \
168 leaving FP8 + scale untouched for the lazy dequant path",
169 key, ss, ws
170 );
171 weights.insert(scale_key, s.shallow_clone());
172 weights.insert(key, weight);
173 continue;
174 }
175 }
176 let w_bf16 = weight.to_kind(tch::Kind::BFloat16);
189 if let Some(scale) = scale {
190 let ws = w_bf16.size();
191 let ss = scale.size();
192 let block_r = ws[0] / ss[0];
193 let block_c = ws[1] / ss[1];
194 let s_4d = scale
195 .to_kind(tch::Kind::BFloat16)
196 .view([ss[0], 1, ss[1], 1]);
197 if let Err(e) = w_bf16.view([ss[0], block_r, ss[1], block_c]).f_mul_(&s_4d) {
203 warn!(
204 "FP8 weight '{key}' in-place scale multiply failed ({e}); \
205 leaving FP8 + scale untouched for the lazy dequant path"
206 );
207 drop(w_bf16);
208 weights.insert(scale_key, scale);
209 weights.insert(key, weight);
210 continue;
211 }
212 drop(scale);
213 }
214 drop(weight);
215 weights.insert(key, w_bf16);
216 converted += 1;
217 }
218 converted
219}
220
221fn maybe_dequantize_fp8_weights_at_load(
229 weights: &mut HashMap<String, Tensor>,
230 fp8_dequant_load: bool,
231 device_pool: Option<&DevicePool>,
232) {
233 if !fp8_dequant_load {
234 return;
235 }
236 if device_pool.is_some_and(|p| !p.is_single()) {
237 warn!(
238 "FP8 dequant-at-load ignored for multi-device pipeline: all weights land on the \
239 primary device before layer distribution, so full-model BF16 materialization \
240 there could OOM; keeping FP8 lazy per-matmul dequant"
241 );
242 return;
243 }
244 let converted = dequantize_fp8_weights_at_load(weights);
245 if converted > 0 {
246 info!(
247 "Dequantized {} FP8 weight tensor(s) to BF16 at load",
248 converted
249 );
250 }
251}
252
253fn glob_shard_files(model_path: &Path) -> Result<Vec<std::path::PathBuf>> {
259 let mut shard_files = Vec::new();
260 for entry in std::fs::read_dir(model_path)? {
261 let entry = entry?;
262 if let Some(name) = entry.file_name().to_str() {
263 if (name.starts_with("model-") || name.starts_with("model.safetensors-"))
264 && name.ends_with(".safetensors")
265 {
266 shard_files.push(entry.path());
267 }
268 }
269 }
270 shard_files.sort();
271
272 if shard_files.len() > 1 {
273 if strict_loader_enabled() {
274 return Err(anyhow!(
275 "{} multi-shard safetensors files found in {} but no \
276 model.safetensors.index.json manifest; refusing to reconstruct the \
277 shard set from a filename glob under {STRICT_LOADER_ENV}. \
278 Provide the index.json manifest.",
279 shard_files.len(),
280 model_path.display()
281 ));
282 }
283 warn!(
284 "⚠️ No model.safetensors.index.json in {}; reconstructing {} shards from a \
285 filename glob (fragile). Set {STRICT_LOADER_ENV}=1 to require the manifest.",
286 model_path.display(),
287 shard_files.len()
288 );
289 }
290 Ok(shard_files)
291}
292
293pub struct ModelFactory;
295
296#[derive(Clone, Debug, Eq, PartialEq)]
304pub struct ModelStageRequest {
305 pub layer_range: Range<usize>,
306}
307
308#[derive(Debug, Deserialize)]
309struct SafetensorsIndex {
310 weight_map: BTreeMap<String, String>,
311}
312
313pub(crate) enum ResolvedWeight {
324 Owned(Vec<u8>),
327 File(std::fs::File),
329}
330
331impl ResolvedWeight {
332 pub(crate) fn with_reader<T>(
335 &mut self,
336 read: impl FnOnce(&mut dyn std::io::Read) -> Result<T>,
337 ) -> Result<T> {
338 match self {
339 Self::File(file) => {
340 use std::io::Seek as _;
341 file.rewind()?;
342 read(file)
343 }
344 Self::Owned(bytes) => read(&mut std::io::Cursor::new(bytes.as_slice())),
345 }
346 }
347}
348
349impl ModelFactory {
350 pub async fn detect_model_dtype(model_path: &Path) -> Result<DType> {
352 let single_file = model_path.join("model.safetensors");
354 let file_to_check = if single_file.exists() {
355 single_file
356 } else {
357 let shard_files = Self::find_shard_files(model_path)?;
359 if shard_files.is_empty() {
360 return Err(anyhow!(
361 "No model weights found in {}",
362 model_path.display()
363 ));
364 }
365 shard_files[0].clone()
366 };
367
368 let file_content = std::fs::read(&file_to_check)?;
370 let tensors = safetensors::SafeTensors::deserialize(&file_content)?;
371
372 let mut f16_count = 0;
377 let mut bf16_count = 0;
378 let mut f32_count = 0;
379 let mut fp8_count = 0;
380
381 for (_, tensor) in tensors.tensors().into_iter().take(10) {
382 match tensor.dtype() {
383 safetensors::Dtype::F16 => f16_count += 1,
384 safetensors::Dtype::BF16 => bf16_count += 1,
385 safetensors::Dtype::F8_E4M3 | safetensors::Dtype::F8_E5M2 => fp8_count += 1,
386 safetensors::Dtype::F32 => f32_count += 1,
387 _ => {}
388 }
389 }
390
391 if fp8_count > 0 && fp8_count >= bf16_count && fp8_count >= f16_count {
393 info!("Detected FP8 quantized model (storage: FP8, compute dtype: BF16)");
394 return Ok(tch::Kind::BFloat16);
395 }
396
397 if f16_count > bf16_count && f16_count > f32_count {
399 info!("Detected F16 model");
400 Ok(tch::Kind::Half)
401 } else if bf16_count >= f16_count && bf16_count >= f32_count {
402 info!("Detected BF16 model");
403 Ok(tch::Kind::BFloat16)
404 } else if f32_count > 0 {
405 info!("Detected F32 model");
406 Ok(tch::Kind::Float)
407 } else {
408 info!("Could not detect model dtype, defaulting to BF16");
409 Ok(tch::Kind::BFloat16)
410 }
411 }
412
413 #[instrument(name = "model_factory.create", skip(device, dtype, device_pool), fields(model_path = %model_path.display()))]
431 pub async fn create(
432 model_path: &Path,
433 device: &Device,
434 dtype: DType,
435 max_context: Option<usize>,
436 kv_quant_type: KVQuantType,
437 device_pool: Option<&DevicePool>,
438 fp8_dequant_load: bool,
439 speculative_decoding: bool,
440 ) -> Result<Box<dyn ModelOperations>> {
441 let channel_profile = super::channel_fp8::ChannelFp8Profile::read(model_path)?;
443 if channel_profile.is_some() {
444 anyhow::ensure!(
445 *device == Device::Cpu,
446 "FP8_DYNAMIC is CPU-only pending CPU acceptance"
447 );
448 anyhow::ensure!(!fp8_dequant_load, "FP8_DYNAMIC must retain activation QDQ; eager weight-only decoding is not equivalent");
449 }
450 info!("Loading model: {}", model_path.display());
451 if let Some(mc) = max_context {
452 info!("Using max_context override: {} tokens", mc);
453 }
454 if kv_quant_type != KVQuantType::None {
455 info!("Using KV cache quantization: {:?}", kv_quant_type);
456 }
457 if fp8_dequant_load {
458 info!(
459 "FP8 dequant-at-load enabled: FP8 weights will be materialized as BF16 during load"
460 );
461 }
462 if let Some(pool) = device_pool {
463 if !pool.is_single() {
464 info!(
465 "🔀 Multi-device pipeline: spreading layers across {} device(s) {:?} \
466 (primary {:?}); depends on #405 for correct per-layer placement",
467 pool.len(),
468 pool.devices(),
469 pool.primary(),
470 );
471 }
472 }
473
474 let shard_files = Self::find_shard_files(model_path)?;
476
477 let mtp_shards = if shard_files.len() <= 1 && model_path.join("model.safetensors").exists()
480 {
481 vec![model_path.join("model.safetensors")]
482 } else {
483 shard_files.clone()
484 };
485 let channel_native_finite = channel_profile.is_some() && native_finite_policy(model_path);
488 let mut channel_inputs = if let Some(profile) = &channel_profile {
489 Some(
490 Self::prepare_channel_inputs(profile, &mtp_shards, |path| async move {
491 Self::resolve_weight_for_digest(&path).await
492 })
493 .await?,
494 )
495 } else {
496 None
497 };
498 let requested = mtp_runtime_supported(speculative_decoding, kv_quant_type);
499 let speculative_decoding = if let Some(inputs) = channel_inputs.as_mut() {
500 Self::mtp_loading_policy_resolved(model_path, inputs, requested).await?
501 } else {
502 Self::mtp_loading_policy(model_path, &mtp_shards, requested).await?
503 };
504
505 if let Some(inputs) = channel_inputs {
506 let mut weights = HashMap::new();
507 for (path, resolved) in inputs {
508 Self::load_resolved_safetensors_file_inner(
509 &path,
510 resolved,
511 None,
512 &mut weights,
513 device,
514 dtype,
515 speculative_decoding,
516 channel_native_finite,
517 true,
518 )
519 .await?;
520 }
521 let config = ModelConfig::load(model_path, &weights)?;
522 return Self::create_model_from_config(
523 config,
524 weights,
525 device,
526 dtype,
527 max_context,
528 kv_quant_type,
529 model_path,
530 device_pool,
531 fp8_dequant_load,
532 speculative_decoding,
533 );
534 }
535
536 if !shard_files.is_empty() && shard_files.len() > 1 {
537 info!(
539 "📦 Using incremental loading for {} shards",
540 shard_files.len()
541 );
542 Self::create_incremental(
543 model_path,
544 device,
545 dtype,
546 shard_files,
547 max_context,
548 kv_quant_type,
549 device_pool,
550 fp8_dequant_load,
551 speculative_decoding,
552 )
553 .await
554 } else {
555 let weights =
557 Self::load_weights(model_path, device, dtype, speculative_decoding).await?;
558 let config = ModelConfig::load(model_path, &weights)?;
559 let model = Self::create_model_from_config(
560 config,
561 weights,
562 device,
563 dtype,
564 max_context,
565 kv_quant_type,
566 model_path,
567 device_pool,
568 fp8_dequant_load,
569 speculative_decoding,
570 )?;
571 info!("✅ ModelFactory: Model created successfully");
572 Ok(model)
573 }
574 }
575
576 #[instrument(
587 name = "model_factory.create_stage",
588 skip(device, dtype, device_pool),
589 fields(
590 model_path = %model_path.display(),
591 layer_start = request.layer_range.start,
592 layer_end = request.layer_range.end,
593 )
594 )]
595 pub async fn create_stage(
596 model_path: &Path,
597 device: &Device,
598 dtype: DType,
599 max_context: Option<usize>,
600 kv_quant_type: KVQuantType,
601 device_pool: Option<&DevicePool>,
602 request: ModelStageRequest,
603 fp8_dequant_load: bool,
604 ) -> Result<Box<dyn ModelOperations>> {
605 let config_path = model_path.join("config.json");
606 if !config_path.is_file() {
607 bail!(
608 "stage loading requires authoritative config {}",
609 config_path.display()
610 );
611 }
612 let empty_weights = HashMap::new();
613 let config = ModelConfig::load(model_path, &empty_weights)?;
614 if !matches!(
615 &config.architecture,
616 ModelArchitecture::Llama | ModelArchitecture::Qwen | ModelArchitecture::Mistral
617 ) {
618 bail!(
619 "stage loading is supported only for Llama-family checkpoints; got {:?}",
620 config.architecture
621 );
622 }
623
624 Self::validate_stage_range(&request.layer_range, config.num_hidden_layers)?;
625 let plan = Self::stage_weight_plan(
626 model_path,
627 config.num_hidden_layers,
628 request.layer_range.clone(),
629 )?;
630 let metadata = Self::stage_tensor_metadata(&plan).await?;
631 Self::validate_stage_tensor_schema(&config, &request.layer_range, &metadata)?;
632 let mut weights = Self::load_weights_for_stage_plan(
633 plan,
634 request.layer_range.clone(),
635 device,
636 dtype,
637 native_finite_policy(model_path),
638 )
639 .await?;
640 maybe_dequantize_fp8_weights_at_load(&mut weights, fp8_dequant_load, device_pool);
641
642 Self::create_llama_model(
643 config,
644 weights,
645 device,
646 dtype,
647 max_context,
648 kv_quant_type,
649 device_pool,
650 Some(request.layer_range),
651 )
652 }
653
654 pub(crate) fn find_shard_files(model_path: &Path) -> Result<Vec<std::path::PathBuf>> {
663 let index_path = model_path.join("model.safetensors.index.json");
665 if index_path.exists() {
666 return Self::shard_files_from_index(model_path, &index_path);
667 }
668
669 let single_file = model_path.join("model.safetensors");
671 if single_file.exists() {
672 return Ok(vec![single_file]);
673 }
674
675 glob_shard_files(model_path)
677 }
678
679 pub(crate) fn find_shard_files_for_digest(
690 model_path: &Path,
691 ) -> Result<Vec<std::path::PathBuf>> {
692 let shards = Self::find_shard_files(model_path)?;
693 if shards.len() > 1 && !model_path.join("model.safetensors.index.json").exists() {
694 return Err(anyhow!(
695 "{} loader-selected .safetensors shards in {} but no \
696 model.safetensors.index.json manifest; the glob-selected set is \
697 ambiguous, so no authoritative base-weight digest can be minted \
698 (KV reuse is declined). Provide the index.json manifest.",
699 shards.len(),
700 model_path.display()
701 ));
702 }
703 Ok(shards)
704 }
705
706 fn shard_files_from_index(
709 model_path: &Path,
710 index_path: &Path,
711 ) -> Result<Vec<std::path::PathBuf>> {
712 let content = std::fs::read_to_string(index_path)?;
713 let index: serde_json::Value = serde_json::from_str(&content)?;
714 let weight_map = index["weight_map"]
715 .as_object()
716 .ok_or_else(|| anyhow!("model.safetensors.index.json missing weight_map"))?;
717
718 let mut seen = std::collections::BTreeSet::new();
719 for filename in weight_map.values() {
720 if let Some(s) = filename.as_str() {
721 seen.insert(s.to_owned());
722 }
723 }
724
725 let mut paths: Vec<_> = seen
726 .into_iter()
727 .map(|name| model_path.join(&name))
728 .collect();
729 paths.sort();
730
731 info!(
732 "📋 Index file lists {} shard(s): {:?}",
733 paths.len(),
734 paths
735 .iter()
736 .map(|p| p.file_name().unwrap_or_default())
737 .collect::<Vec<_>>()
738 );
739 Ok(paths)
740 }
741
742 fn validate_stage_range(layer_range: &Range<usize>, num_layers: usize) -> Result<()> {
743 if layer_range.start >= layer_range.end || layer_range.end > num_layers {
744 bail!(
745 "invalid stage layer range {:?} for checkpoint with {} layers",
746 layer_range,
747 num_layers
748 );
749 }
750 Ok(())
751 }
752
753 fn llama_layer_index(name: &str) -> Option<usize> {
754 let rest = name.strip_prefix("model.layers.")?;
755 let (index, suffix) = rest.split_once('.')?;
756 if suffix.is_empty() {
757 return None;
758 }
759 index.parse().ok()
760 }
761
762 fn resolve_required_alias(
763 weight_map: &BTreeMap<String, String>,
764 aliases: &[&str],
765 description: &str,
766 ) -> Result<String> {
767 aliases
768 .iter()
769 .find(|name| weight_map.contains_key(**name))
770 .map(|name| (*name).to_owned())
771 .ok_or_else(|| {
772 anyhow!(
773 "stage weight_map is missing required {description}; expected one of {}",
774 aliases.join(", ")
775 )
776 })
777 }
778
779 fn insert_companion_scale(
787 weight_map: &BTreeMap<String, String>,
788 required: &mut BTreeSet<String>,
789 name: &str,
790 ) {
791 for suffix in ["_scale_inv", "_scale"] {
792 let scale = format!("{name}{suffix}");
793 if weight_map.contains_key(&scale) {
794 required.insert(scale);
795 }
796 }
797 }
798
799 fn stage_weight_plan(
804 model_path: &Path,
805 num_layers: usize,
806 layer_range: Range<usize>,
807 ) -> Result<BTreeMap<PathBuf, BTreeSet<String>>> {
808 Self::validate_stage_range(&layer_range, num_layers)?;
809
810 let index_path = model_path.join("model.safetensors.index.json");
811 let content = std::fs::read_to_string(&index_path).with_context(|| {
812 format!(
813 "stage loading requires authoritative shard index {}",
814 index_path.display()
815 )
816 })?;
817 let index: SafetensorsIndex = serde_json::from_str(&content).with_context(|| {
818 format!(
819 "stage loading requires a valid `weight_map` object in {}",
820 index_path.display()
821 )
822 })?;
823 if index.weight_map.is_empty() {
824 bail!(
825 "stage loading requires a non-empty `weight_map` in {}",
826 index_path.display()
827 );
828 }
829 if let Some((name, _)) = index
830 .weight_map
831 .iter()
832 .find(|(name, shard)| name.is_empty() || shard.is_empty())
833 {
834 bail!(
835 "stage weight_map in {} contains an empty tensor name or shard for `{name}`",
836 index_path.display()
837 );
838 }
839
840 let manifest_layers: BTreeSet<_> = index
841 .weight_map
842 .keys()
843 .filter_map(|name| Self::llama_layer_index(name))
844 .collect();
845 let expected_layers: BTreeSet<_> = (0..num_layers).collect();
846 if manifest_layers != expected_layers {
847 bail!(
848 "stage weight_map decoder layers {:?} do not exactly match expected range 0..{}",
849 manifest_layers,
850 num_layers
851 );
852 }
853
854 let mut required = BTreeSet::new();
855 for layer in layer_range.clone() {
856 let layer_names: Vec<_> = index
857 .weight_map
858 .keys()
859 .filter(|name| Self::llama_layer_index(name) == Some(layer))
860 .cloned()
861 .collect();
862 if layer_names.is_empty() {
863 bail!("stage weight_map has no tensors for required decoder layer {layer}");
864 }
865 required.extend(layer_names);
866 }
867
868 let is_first = layer_range.start == 0;
869 let is_last = layer_range.end == num_layers;
870 if is_first {
871 let embed = Self::resolve_required_alias(
872 &index.weight_map,
873 &["model.embed_tokens.weight", "embed_tokens.weight"],
874 "embedding tensor",
875 )?;
876 Self::insert_companion_scale(&index.weight_map, &mut required, &embed);
877 required.insert(embed);
878 }
879 if is_last {
880 required.insert(Self::resolve_required_alias(
881 &index.weight_map,
882 &["model.norm.weight", "norm.weight"],
883 "final norm tensor",
884 )?);
885
886 if let Some(lm_head) = ["lm_head.weight", "model.lm_head.weight"]
887 .into_iter()
888 .find(|name| index.weight_map.contains_key(*name))
889 {
890 Self::insert_companion_scale(&index.weight_map, &mut required, lm_head);
891 required.insert(lm_head.to_owned());
892 } else if !is_first {
893 bail!("a non-first final stage requires an explicit lm_head tensor in weight_map");
894 }
895 }
896
897 let mut plan: BTreeMap<PathBuf, BTreeSet<String>> = BTreeMap::new();
898 for name in required {
899 let shard = index.weight_map.get(&name).ok_or_else(|| {
900 anyhow!("required stage tensor `{name}` disappeared from weight_map")
901 })?;
902 plan.entry(model_path.join(shard)).or_default().insert(name);
903 }
904
905 Ok(plan)
906 }
907
908 async fn stage_tensor_metadata(
915 plan: &BTreeMap<PathBuf, BTreeSet<String>>,
916 ) -> Result<BTreeMap<String, Vec<usize>>> {
917 use memmap2::Mmap;
918
919 let mut metadata = BTreeMap::new();
920 for (shard, names) in plan {
921 match Self::resolve_weight_for_digest(shard).await? {
922 ResolvedWeight::File(file) => {
923 let mmap = unsafe { Mmap::map(&file) }.with_context(|| {
924 format!("failed to mmap required stage shard {}", shard.display())
925 })?;
926 let tensors =
927 safetensors::SafeTensors::deserialize(&mmap).with_context(|| {
928 format!("failed to parse required stage shard {}", shard.display())
929 })?;
930 Self::record_stage_tensor_metadata(tensors, names, shard, &mut metadata)?;
931 }
932 ResolvedWeight::Owned(bytes) => {
933 let tensors =
934 safetensors::SafeTensors::deserialize(&bytes).with_context(|| {
935 format!("failed to parse required stage shard {}", shard.display())
936 })?;
937 Self::record_stage_tensor_metadata(tensors, names, shard, &mut metadata)?;
938 }
939 }
940 }
941 Ok(metadata)
942 }
943
944 fn record_stage_tensor_metadata(
945 tensors: safetensors::SafeTensors<'_>,
946 names: &BTreeSet<String>,
947 shard: &Path,
948 metadata: &mut BTreeMap<String, Vec<usize>>,
949 ) -> Result<()> {
950 for name in names {
951 let view = tensors.tensor(name).with_context(|| {
952 format!(
953 "authoritative weight_map assigns `{name}` to {}, but the tensor is absent",
954 shard.display()
955 )
956 })?;
957 if metadata
958 .insert(name.clone(), view.shape().to_vec())
959 .is_some()
960 {
961 bail!("authoritative stage plan contains duplicate tensor `{name}`");
962 }
963 }
964 Ok(())
965 }
966
967 fn stage_metadata_alias<'a>(
968 metadata: &'a BTreeMap<String, Vec<usize>>,
969 aliases: &[&str],
970 description: &str,
971 ) -> Result<(String, &'a [usize])> {
972 aliases
973 .iter()
974 .find_map(|name| {
975 metadata
976 .get(*name)
977 .map(|shape| ((*name).to_owned(), shape.as_slice()))
978 })
979 .ok_or_else(|| {
980 anyhow!(
981 "selected stage metadata is missing required {description}; expected one of {}",
982 aliases.join(", ")
983 )
984 })
985 }
986
987 fn require_stage_shape(
988 metadata: &BTreeMap<String, Vec<usize>>,
989 name: &str,
990 expected: &[usize],
991 ) -> Result<()> {
992 let actual = metadata.get(name).ok_or_else(|| {
993 anyhow!("selected stage metadata is missing required tensor `{name}`")
994 })?;
995 if actual.as_slice() != expected {
996 bail!(
997 "stage tensor `{name}` shape mismatch: config requires {:?}, checkpoint has {:?}",
998 expected,
999 actual
1000 );
1001 }
1002 Ok(())
1003 }
1004
1005 fn require_optional_stage_shape(
1006 metadata: &BTreeMap<String, Vec<usize>>,
1007 name: &str,
1008 expected: &[usize],
1009 ) -> Result<()> {
1010 if metadata.contains_key(name) {
1011 Self::require_stage_shape(metadata, name, expected)?;
1012 }
1013 Ok(())
1014 }
1015
1016 fn validate_stage_tensor_schema(
1022 config: &ModelConfig,
1023 layer_range: &Range<usize>,
1024 metadata: &BTreeMap<String, Vec<usize>>,
1025 ) -> Result<()> {
1026 Self::validate_stage_range(layer_range, config.num_hidden_layers)?;
1027
1028 let hidden = config.hidden_size;
1029 let intermediate = config.intermediate_size;
1030 let query_width = config
1031 .num_attention_heads
1032 .checked_mul(config.head_dim)
1033 .ok_or_else(|| anyhow!("query projection width overflows usize"))?;
1034 let kv_width = config
1035 .num_key_value_heads
1036 .checked_mul(config.head_dim)
1037 .ok_or_else(|| anyhow!("key/value projection width overflows usize"))?;
1038 if hidden == 0 || intermediate == 0 || query_width == 0 || kv_width == 0 {
1039 bail!(
1040 "stage config contains an invalid zero dimension: hidden={hidden}, \
1041 intermediate={intermediate}, query_width={query_width}, kv_width={kv_width}"
1042 );
1043 }
1044
1045 let is_first = layer_range.start == 0;
1046 let is_last = layer_range.end == config.num_hidden_layers;
1047 if is_first {
1048 let (name, shape) = Self::stage_metadata_alias(
1049 metadata,
1050 &["model.embed_tokens.weight", "embed_tokens.weight"],
1051 "embedding tensor",
1052 )?;
1053 let expected = [config.vocab_size, hidden];
1054 if shape != expected {
1055 bail!(
1056 "stage tensor `{name}` shape mismatch: config requires {:?}, checkpoint has {:?}",
1057 expected,
1058 shape
1059 );
1060 }
1061 }
1062
1063 if is_last {
1064 let (norm_name, norm_shape) = Self::stage_metadata_alias(
1065 metadata,
1066 &["model.norm.weight", "norm.weight"],
1067 "final norm tensor",
1068 )?;
1069 let expected_norm = [hidden];
1070 if norm_shape != expected_norm {
1071 bail!(
1072 "stage tensor `{norm_name}` shape mismatch: config requires {:?}, checkpoint has {:?}",
1073 expected_norm,
1074 norm_shape
1075 );
1076 }
1077
1078 if let Some((head_name, head_shape)) = ["lm_head.weight", "model.lm_head.weight"]
1079 .into_iter()
1080 .find_map(|name| metadata.get(name).map(|shape| (name, shape.as_slice())))
1081 {
1082 let expected_head = [config.vocab_size, hidden];
1083 if head_shape != expected_head {
1084 bail!(
1085 "stage tensor `{head_name}` shape mismatch: config requires {:?}, \
1086 checkpoint has {:?}",
1087 expected_head,
1088 head_shape
1089 );
1090 }
1091 } else if !is_first {
1092 bail!("a non-first final stage requires an explicit lm_head tensor");
1093 }
1094 }
1097
1098 for layer in layer_range.clone() {
1099 let prefix = format!("model.layers.{layer}");
1100 let q_proj = format!("{prefix}.self_attn.q_proj.weight");
1101 let c_attn = format!("{prefix}.self_attn.c_attn.weight");
1102 if metadata.contains_key(&q_proj) {
1103 let k_proj = format!("{prefix}.self_attn.k_proj.weight");
1104 let v_proj = format!("{prefix}.self_attn.v_proj.weight");
1105 Self::require_stage_shape(metadata, &q_proj, &[query_width, hidden])?;
1106 Self::require_stage_shape(metadata, &k_proj, &[kv_width, hidden])?;
1107 Self::require_stage_shape(metadata, &v_proj, &[kv_width, hidden])?;
1108 Self::require_optional_stage_shape(
1109 metadata,
1110 &format!("{prefix}.self_attn.q_proj.bias"),
1111 &[query_width],
1112 )?;
1113 Self::require_optional_stage_shape(
1114 metadata,
1115 &format!("{prefix}.self_attn.k_proj.bias"),
1116 &[kv_width],
1117 )?;
1118 Self::require_optional_stage_shape(
1119 metadata,
1120 &format!("{prefix}.self_attn.v_proj.bias"),
1121 &[kv_width],
1122 )?;
1123 } else if metadata.contains_key(&c_attn) {
1124 if config.num_key_value_heads != config.num_attention_heads {
1125 bail!(
1126 "stage layer {layer} uses combined c_attn with grouped-query config \
1127 (attention heads {}, key/value heads {}), which the dense Llama \
1128 constructor cannot represent",
1129 config.num_attention_heads,
1130 config.num_key_value_heads
1131 );
1132 }
1133 let combined_width = query_width
1134 .checked_mul(3)
1135 .ok_or_else(|| anyhow!("combined attention width overflows usize"))?;
1136 Self::require_stage_shape(metadata, &c_attn, &[combined_width, hidden])?;
1137 Self::require_optional_stage_shape(
1138 metadata,
1139 &format!("{prefix}.self_attn.c_attn.bias"),
1140 &[combined_width],
1141 )?;
1142 } else {
1143 bail!(
1144 "selected stage metadata is missing required attention projection for layer \
1145 {layer}; expected `{q_proj}` or `{c_attn}`"
1146 );
1147 }
1148
1149 let o_proj = format!("{prefix}.self_attn.o_proj.weight");
1150 let c_proj = format!("{prefix}.self_attn.c_proj.weight");
1151 let output_name = if metadata.contains_key(&o_proj) {
1152 o_proj.as_str()
1153 } else {
1154 c_proj.as_str()
1155 };
1156 Self::require_stage_shape(metadata, output_name, &[hidden, query_width])?;
1157 let output_bias = output_name
1158 .strip_suffix(".weight")
1159 .map(|prefix| format!("{prefix}.bias"))
1160 .unwrap_or_else(|| format!("{output_name}.bias"));
1161 Self::require_optional_stage_shape(metadata, &output_bias, &[hidden])?;
1162
1163 for (suffix, expected) in [
1164 ("mlp.gate_proj.weight", [intermediate, hidden]),
1165 ("mlp.up_proj.weight", [intermediate, hidden]),
1166 ("mlp.down_proj.weight", [hidden, intermediate]),
1167 ] {
1168 Self::require_stage_shape(metadata, &format!("{prefix}.{suffix}"), &expected)?;
1169 }
1170 for (suffix, expected) in [
1171 ("mlp.gate_proj.bias", [intermediate]),
1172 ("mlp.up_proj.bias", [intermediate]),
1173 ("mlp.down_proj.bias", [hidden]),
1174 ] {
1175 Self::require_optional_stage_shape(
1176 metadata,
1177 &format!("{prefix}.{suffix}"),
1178 &expected,
1179 )?;
1180 }
1181 Self::require_stage_shape(
1182 metadata,
1183 &format!("{prefix}.input_layernorm.weight"),
1184 &[hidden],
1185 )?;
1186 Self::require_stage_shape(
1187 metadata,
1188 &format!("{prefix}.post_attention_layernorm.weight"),
1189 &[hidden],
1190 )?;
1191
1192 let q_norm = format!("{prefix}.self_attn.q_norm.weight");
1193 let k_norm = format!("{prefix}.self_attn.k_norm.weight");
1194 if config.use_qk_norm
1195 || metadata.contains_key(&q_norm)
1196 || metadata.contains_key(&k_norm)
1197 {
1198 Self::require_stage_shape(metadata, &q_norm, &[config.head_dim])?;
1199 Self::require_stage_shape(metadata, &k_norm, &[config.head_dim])?;
1200 }
1201 }
1202
1203 Ok(())
1204 }
1205
1206 #[cfg(test)]
1207 async fn load_weights_for_stage(
1208 model_path: &Path,
1209 num_layers: usize,
1210 layer_range: Range<usize>,
1211 device: &Device,
1212 dtype: DType,
1213 ) -> Result<HashMap<String, Tensor>> {
1214 let plan = Self::stage_weight_plan(model_path, num_layers, layer_range.clone())?;
1215 Self::load_weights_for_stage_plan(
1216 plan,
1217 layer_range,
1218 device,
1219 dtype,
1220 native_finite_policy(model_path),
1221 )
1222 .await
1223 }
1224
1225 async fn load_weights_for_stage_plan(
1226 plan: BTreeMap<PathBuf, BTreeSet<String>>,
1227 layer_range: Range<usize>,
1228 device: &Device,
1229 dtype: DType,
1230 native_finite: bool,
1231 ) -> Result<HashMap<String, Tensor>> {
1232 let planned_tensors: usize = plan.values().map(BTreeSet::len).sum();
1233 info!(
1234 "Loading stage {:?}: {} exact tensor(s) from {} shard(s)",
1235 layer_range,
1236 planned_tensors,
1237 plan.len()
1238 );
1239
1240 let mut weights = HashMap::with_capacity(planned_tensors);
1241 for (shard, names) in plan {
1242 Self::load_safetensors_file_selected(
1243 &shard,
1244 &names,
1245 &mut weights,
1246 device,
1247 dtype,
1248 native_finite,
1249 )
1250 .await?;
1251 }
1252
1253 if weights.len() != planned_tensors {
1254 bail!(
1255 "stage loader planned {planned_tensors} tensors but materialized {}",
1256 weights.len()
1257 );
1258 }
1259 let loaded_bytes: usize = weights
1260 .values()
1261 .map(|tensor| tensor.numel() * tensor.kind().elt_size_in_bytes())
1262 .sum();
1263 info!(
1264 "Loaded authoritative stage {:?}: {} tensor(s), {} byte(s)",
1265 layer_range,
1266 weights.len(),
1267 loaded_bytes
1268 );
1269 Ok(weights)
1270 }
1271
1272 async fn mtp_config_policy(model_path: &Path) -> Result<bool> {
1273 match tokio::fs::read(model_path.join("config.json")).await {
1274 Ok(bytes) => Ok(mtp_config_supported(&serde_json::from_slice(&bytes)?)),
1275 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(true),
1276 Err(e) => Err(e.into()),
1277 }
1278 }
1279
1280 async fn prepare_channel_inputs<F, Fut>(
1288 profile: &super::channel_fp8::ChannelFp8Profile,
1289 shards: &[PathBuf],
1290 mut resolve: F,
1291 ) -> Result<Vec<(PathBuf, ResolvedWeight)>>
1292 where
1293 F: FnMut(PathBuf) -> Fut,
1294 Fut: std::future::Future<Output = Result<ResolvedWeight>>,
1295 {
1296 Self::prepare_channel_inputs_spooling(profile, shards, &mut resolve, |bytes| async move {
1297 tokio::task::spawn_blocking(move || super::resolved_spool::spool(bytes))
1298 .await
1299 .context("resolved FP8 spool task failed")?
1300 })
1301 .await
1302 }
1303
1304 async fn prepare_channel_inputs_spooling<F, Fut, S, SFut>(
1305 profile: &super::channel_fp8::ChannelFp8Profile,
1306 shards: &[PathBuf],
1307 mut resolve: F,
1308 mut spool: S,
1309 ) -> Result<Vec<(PathBuf, ResolvedWeight)>>
1310 where
1311 F: FnMut(PathBuf) -> Fut,
1312 Fut: std::future::Future<Output = Result<ResolvedWeight>>,
1313 S: FnMut(Vec<u8>) -> SFut,
1314 SFut: std::future::Future<Output = Result<std::fs::File>>,
1315 {
1316 let mut inputs = Vec::with_capacity(shards.len());
1317 for path in shards {
1318 let resolved = match resolve(path.clone()).await? {
1319 ResolvedWeight::Owned(bytes) => ResolvedWeight::File(spool(bytes).await?),
1320 file @ ResolvedWeight::File(_) => file,
1321 };
1322 inputs.push((path.clone(), resolved));
1323 }
1324 profile.preflight_resolved(&mut inputs)?;
1325 Ok(inputs)
1326 }
1327
1328 async fn mtp_loading_policy_resolved(
1329 model_path: &Path,
1330 inputs: &mut [(PathBuf, ResolvedWeight)],
1331 requested: bool,
1332 ) -> Result<bool> {
1333 if !requested || !Self::mtp_config_policy(model_path).await? {
1334 return Ok(false);
1335 }
1336 for (_, input) in inputs {
1337 let supported = input.with_reader(|reader| {
1338 let mut prefix = [0; 8];
1339 reader.read_exact(&mut prefix)?;
1340 let mut header = vec![0; mtp_header_len(&prefix)?];
1341 reader.read_exact(&mut header)?;
1342 mtp_header_supported(&header)
1343 })?;
1344 if !supported {
1345 return Ok(false);
1346 }
1347 }
1348 Ok(true)
1349 }
1350
1351 async fn mtp_loading_policy(
1352 model_path: &Path,
1353 shards: &[PathBuf],
1354 requested: bool,
1355 ) -> Result<bool> {
1356 if !requested {
1357 return Ok(false);
1358 }
1359 if !Self::mtp_config_policy(model_path).await? {
1360 return Ok(false);
1361 }
1362 for shard in shards {
1363 let resolved = Self::resolve_weight_for_digest(shard).await?;
1364 let supported = tokio::task::spawn_blocking(move || -> Result<bool> {
1365 use std::io::Read;
1366 let mut reader: Box<dyn Read> = match resolved {
1367 ResolvedWeight::File(file) => Box::new(file),
1368 ResolvedWeight::Owned(bytes) => Box::new(std::io::Cursor::new(bytes)),
1369 };
1370 let mut prefix = [0; 8];
1371 reader.read_exact(&mut prefix)?;
1372 let mut header = vec![0; mtp_header_len(&prefix)?];
1373 reader.read_exact(&mut header)?;
1374 mtp_header_supported(&header)
1375 })
1376 .await??;
1377 if !supported {
1378 return Ok(false);
1379 }
1380 }
1381 Ok(true)
1382 }
1383
1384 async fn mtp_loading_policy_fs(
1385 model_path: &Path,
1386 fs: &WorktreeClient,
1387 shards: &[String],
1388 requested: bool,
1389 ) -> Result<bool> {
1390 if !requested {
1391 return Ok(false);
1392 }
1393 if !Self::mtp_config_policy(model_path).await? {
1396 return Ok(false);
1397 }
1398 for shard in shards {
1399 let prefix = fs.read_file_prefix(shard, 8).await?;
1400 let len = mtp_header_len(&prefix)?;
1401 let bytes = fs.read_file_prefix(shard, 8 + len).await?;
1402 anyhow::ensure!(bytes.len() == 8 + len, "truncated safetensors metadata");
1403 if !mtp_header_supported(&bytes[8..])? {
1404 return Ok(false);
1405 }
1406 }
1407 Ok(true)
1408 }
1409
1410 #[instrument(name = "model_factory.create_incremental", skip(device, dtype, shard_files, device_pool), fields(shard_count = shard_files.len()))]
1412 async fn create_incremental(
1413 model_path: &Path,
1414 device: &Device,
1415 dtype: DType,
1416 shard_files: Vec<std::path::PathBuf>,
1417 max_context: Option<usize>,
1418 kv_quant_type: KVQuantType,
1419 device_pool: Option<&DevicePool>,
1420 fp8_dequant_load: bool,
1421 speculative_decoding: bool,
1422 ) -> Result<Box<dyn ModelOperations>> {
1423 info!("Loading {} weight shards", shard_files.len());
1426
1427 let native_finite = native_finite_policy(model_path);
1428 let mut all_weights = HashMap::new();
1429
1430 for (idx, shard_file) in shard_files.iter().enumerate() {
1431 info!("Loading shard {}/{}", idx + 1, shard_files.len());
1432
1433 Self::load_safetensors_file(
1435 shard_file,
1436 &mut all_weights,
1437 device,
1438 dtype,
1439 speculative_decoding,
1440 native_finite,
1441 )
1442 .await?;
1443
1444 }
1450
1451 let config = ModelConfig::load(model_path, &all_weights)?;
1453 let model = Self::create_model_from_config(
1454 config,
1455 all_weights,
1456 device,
1457 dtype,
1458 max_context,
1459 kv_quant_type,
1460 model_path,
1461 device_pool,
1462 fp8_dequant_load,
1463 speculative_decoding,
1464 )?;
1465
1466 info!("Model loaded");
1467 Ok(model)
1468 }
1469
1470 async fn load_weights(
1472 model_path: &Path,
1473 device: &Device,
1474 dtype: DType,
1475 speculative_decoding: bool,
1476 ) -> Result<HashMap<String, Tensor>> {
1477 let native_finite = native_finite_policy(model_path);
1478 let mut all_weights = HashMap::new();
1479
1480 let single_file = model_path.join("model.safetensors");
1481 if single_file.exists() {
1482 info!("Loading model.safetensors");
1483 Self::load_safetensors_file(
1484 &single_file,
1485 &mut all_weights,
1486 device,
1487 dtype,
1488 speculative_decoding,
1489 native_finite,
1490 )
1491 .await?;
1492 return Ok(all_weights);
1493 }
1494
1495 let model_path_buf = model_path.to_path_buf();
1497 let mut shard_files =
1498 tokio::task::spawn_blocking(move || -> Result<Vec<std::path::PathBuf>> {
1499 let index_path = model_path_buf.join("model.safetensors.index.json");
1501 if index_path.exists() {
1502 return Self::shard_files_from_index(&model_path_buf, &index_path);
1503 }
1504 glob_shard_files(&model_path_buf)
1506 })
1507 .await??;
1508
1509 if !shard_files.is_empty() {
1510 shard_files.sort();
1511 info!("Loading {} weight shards", shard_files.len());
1512 for shard_file in shard_files {
1513 Self::load_safetensors_file(
1514 &shard_file,
1515 &mut all_weights,
1516 device,
1517 dtype,
1518 speculative_decoding,
1519 native_finite,
1520 )
1521 .await?;
1522 }
1523 return Ok(all_weights);
1524 }
1525
1526 Err(anyhow!(
1527 "No safetensors files found in {}",
1528 model_path.display()
1529 ))
1530 }
1531
1532 #[instrument(name = "model_factory.load_safetensor_file", skip(weights, device, dtype), fields(file = %path.display()))]
1534 async fn load_safetensors_file(
1535 path: &Path,
1536 weights: &mut HashMap<String, Tensor>,
1537 device: &Device,
1538 dtype: DType,
1539 speculative_decoding: bool,
1540 native_finite: bool,
1541 ) -> Result<()> {
1542 Self::load_safetensors_file_inner(
1543 path,
1544 None,
1545 weights,
1546 device,
1547 dtype,
1548 speculative_decoding,
1549 native_finite,
1550 )
1551 .await
1552 }
1553
1554 async fn load_safetensors_file_selected(
1560 path: &Path,
1561 selected: &BTreeSet<String>,
1562 weights: &mut HashMap<String, Tensor>,
1563 device: &Device,
1564 dtype: DType,
1565 native_finite: bool,
1566 ) -> Result<()> {
1567 Self::load_safetensors_file_inner(
1568 path,
1569 Some(selected),
1570 weights,
1571 device,
1572 dtype,
1573 false,
1574 native_finite,
1575 )
1576 .await
1577 }
1578
1579 async fn load_safetensors_file_inner(
1580 path: &Path,
1581 selected: Option<&BTreeSet<String>>,
1582 weights: &mut HashMap<String, Tensor>,
1583 device: &Device,
1584 dtype: DType,
1585 speculative_decoding: bool,
1586 native_finite: bool,
1587 ) -> Result<()> {
1588 let resolved = Self::resolve_weight_for_digest(path).await?;
1589 Self::load_resolved_safetensors_file_inner(
1590 path,
1591 resolved,
1592 selected,
1593 weights,
1594 device,
1595 dtype,
1596 speculative_decoding,
1597 native_finite,
1598 false,
1599 )
1600 .await
1601 }
1602
1603 async fn load_resolved_safetensors_file_inner(
1604 path: &Path,
1605 resolved: ResolvedWeight,
1606 selected: Option<&BTreeSet<String>>,
1607 weights: &mut HashMap<String, Tensor>,
1608 device: &Device,
1609 dtype: DType,
1610 speculative_decoding: bool,
1611 native_finite: bool,
1612 force_mmap: bool,
1613 ) -> Result<()> {
1614 let channel_mmap = force_mmap
1624 || path
1625 .parent()
1626 .map(super::channel_fp8::ChannelFp8Profile::read)
1627 .transpose()?
1628 .flatten()
1629 .is_some();
1630 let use_mmap = selected.is_some()
1631 || channel_mmap
1632 || std::env::var("HYPRSTREAM_USE_MMAP")
1633 .map(|v| v.to_lowercase() == "true" || v == "1")
1634 .unwrap_or(false);
1635 debug!(
1636 "safetensors load path: {}",
1637 if use_mmap { "mmap" } else { "read" }
1638 );
1639
1640 let path_buf = path.to_path_buf();
1642
1643 match resolved {
1647 ResolvedWeight::File(file) if use_mmap => {
1648 use memmap2::Mmap;
1649
1650 let mmap = unsafe { Mmap::map(&file)? };
1651 let tensors = safetensors::SafeTensors::deserialize(&mmap)?;
1652 Self::create_tensors_from_safetensors_selected(
1653 tensors,
1654 selected,
1655 weights,
1656 device,
1657 dtype,
1658 speculative_decoding,
1659 native_finite,
1660 )
1661 }
1662 ResolvedWeight::File(_) => {
1663 let tensor_data = Self::load_file_with_pointer_detection(&path_buf).await?;
1664 let tensors = safetensors::SafeTensors::deserialize(&tensor_data)?;
1665 Self::create_tensors_from_safetensors_selected(
1666 tensors,
1667 selected,
1668 weights,
1669 device,
1670 dtype,
1671 speculative_decoding,
1672 native_finite,
1673 )
1674 }
1675 ResolvedWeight::Owned(tensor_data) => {
1676 let tensors = safetensors::SafeTensors::deserialize(&tensor_data)?;
1677 Self::create_tensors_from_safetensors_selected(
1678 tensors,
1679 selected,
1680 weights,
1681 device,
1682 dtype,
1683 speculative_decoding,
1684 native_finite,
1685 )
1686 }
1687 }
1688 }
1689
1690 async fn load_file_with_pointer_detection(path: &Path) -> Result<Vec<u8>> {
1696 let metadata = tokio::fs::metadata(path).await?;
1697
1698 if metadata.len() >= 1024 {
1700 return tokio::fs::read(path).await.map_err(Into::into);
1701 }
1702
1703 let data = tokio::fs::read(path).await?;
1704 if let Some(resolved) = Self::try_resolve_lfs_pointer(path, &data).await? {
1705 return Ok(resolved);
1706 }
1707 Ok(data)
1708 }
1709
1710 async fn try_resolve_lfs_pointer(path: &Path, data: &[u8]) -> Result<Option<Vec<u8>>> {
1720 if data.len() >= 1024 {
1722 return Ok(None);
1723 }
1724 let text = match std::str::from_utf8(data) {
1725 Ok(t) => t,
1726 Err(_) => return Ok(None),
1727 };
1728 if !text.starts_with("version https://git-lfs")
1729 && !text.starts_with("version https://hawser")
1730 {
1731 return Ok(None);
1732 }
1733 #[cfg(feature = "xet")]
1734 {
1735 debug!(
1736 "Un-smudged LFS pointer, resolving via git2db::lfs fallback: {}",
1737 path.display()
1738 );
1739 let config = git2db::XetConfig::default();
1740 let storage = git2db::LfsStorage::new(&config)
1741 .await
1742 .map_err(|e| anyhow!("Failed to create LfsStorage: {}", e))?;
1743 let bytes = storage
1744 .load_file(path)
1745 .await
1746 .map_err(|e| anyhow!("Failed to load LFS file: {}", e))?;
1747 Ok(Some(bytes))
1748 }
1749 #[cfg(not(feature = "xet"))]
1750 {
1751 let _ = path;
1752 anyhow::bail!(
1753 "Un-smudged LFS pointer at {} but XET feature disabled. \
1754 Enable with --features xet or ensure files are smudged during checkout.",
1755 path.display()
1756 )
1757 }
1758 }
1759
1760 pub(crate) async fn resolve_weight_for_digest(path: &Path) -> Result<ResolvedWeight> {
1769 let metadata = tokio::fs::metadata(path).await?;
1770 if metadata.len() >= 1024 {
1773 let file = std::fs::File::open(path)?;
1774 return Ok(ResolvedWeight::File(file));
1775 }
1776 let data = tokio::fs::read(path).await?;
1777 if let Some(resolved) = Self::try_resolve_lfs_pointer(path, &data).await? {
1778 return Ok(ResolvedWeight::Owned(resolved));
1779 }
1780 Ok(ResolvedWeight::Owned(data))
1781 }
1782
1783 #[cfg(test)]
1785 fn create_tensors_from_safetensors(
1786 tensors: safetensors::SafeTensors,
1787 weights: &mut HashMap<String, Tensor>,
1788 device: &Device,
1789 dtype: DType,
1790 speculative_decoding: bool,
1791 ) -> Result<()> {
1792 Self::create_tensors_from_safetensors_selected(
1793 tensors,
1794 None,
1795 weights,
1796 device,
1797 dtype,
1798 speculative_decoding,
1799 false,
1800 )
1801 }
1802
1803 fn create_tensors_from_safetensors_selected(
1804 tensors: safetensors::SafeTensors,
1805 selected: Option<&BTreeSet<String>>,
1806 weights: &mut HashMap<String, Tensor>,
1807 device: &Device,
1808 dtype: DType,
1809 speculative_decoding: bool,
1810 native_finite: bool,
1811 ) -> Result<()> {
1812 let mut total_size_mb = 0.0;
1813 let tensors_list = if let Some(selected) = selected {
1814 selected
1815 .iter()
1816 .map(|name| {
1817 tensors
1818 .tensor(name)
1819 .map(|view| (name.clone(), view))
1820 .with_context(|| {
1821 format!(
1822 "authoritative weight_map assigns `{name}` to this shard, \
1823 but the tensor is absent"
1824 )
1825 })
1826 })
1827 .collect::<Result<Vec<_>>>()?
1828 } else {
1829 tensors.tensors()
1830 };
1831 let tensor_count = tensors_list.len();
1832
1833 info!("Loading {} tensors to device {:?}", tensor_count, device);
1834
1835 for (idx, (name, tensor_view)) in tensors_list.into_iter().enumerate() {
1836 if !speculative_decoding && name.starts_with("mtp.") {
1840 continue;
1841 }
1842 let shape: Vec<i64> = tensor_view.shape().iter().map(|&x| x as i64).collect();
1843 let data = tensor_view.data();
1844 if native_finite {
1845 validate_finite_payload(&name, tensor_view.dtype(), data)?;
1850 }
1851
1852 let tensor_kind = match tensor_view.dtype() {
1854 safetensors::Dtype::BF16 => tch::Kind::BFloat16,
1855 safetensors::Dtype::F16 => tch::Kind::Half,
1856 safetensors::Dtype::F64 => tch::Kind::Double,
1857 safetensors::Dtype::F8_E4M3 => tch::Kind::Float8e4m3fn,
1858 safetensors::Dtype::F8_E5M2 => tch::Kind::Float8e5m2,
1859 _ => tch::Kind::Float,
1861 };
1862
1863 let dtype = if name.ends_with(".weight_scale") || name.ends_with(".weight_scale_inv") {
1865 tensor_kind
1866 } else {
1867 dtype
1868 };
1869
1870 let tensor_size_mb = estimate_tensor_size_mb(&shape, tensor_kind);
1872 total_size_mb += tensor_size_mb;
1873
1874 let tensor = match tensor_view.dtype() {
1876 safetensors::Dtype::BF16 => {
1877 if !matches!(
1879 dtype,
1880 tch::Kind::BFloat16 | tch::Kind::Half | tch::Kind::Float
1881 ) {
1882 return Err(anyhow!("Model dtype BF16 but target dtype is {:?}", dtype));
1883 }
1884
1885 let borrowed_tensor = unsafe {
1889 Tensor::from_blob(
1890 data.as_ptr(),
1891 &shape,
1892 &[],
1893 tch::Kind::BFloat16,
1894 Device::Cpu,
1895 )
1896 };
1897
1898 let cpu_tensor = borrowed_tensor.copy();
1900
1901 let cpu_tensor = if dtype != tch::Kind::BFloat16 {
1903 cpu_tensor.to_kind(dtype)
1904 } else {
1905 cpu_tensor
1906 };
1907
1908 if *device != Device::Cpu {
1910 let gpu_tensor = cpu_tensor.to_device(*device);
1911 drop(cpu_tensor); gpu_tensor
1913 } else {
1914 cpu_tensor
1915 }
1916 }
1917 safetensors::Dtype::F16 => {
1918 if !matches!(
1920 dtype,
1921 tch::Kind::BFloat16 | tch::Kind::Half | tch::Kind::Float
1922 ) {
1923 return Err(anyhow!("Model dtype F16 but target dtype is {:?}", dtype));
1924 }
1925
1926 let borrowed_tensor = unsafe {
1928 Tensor::from_blob(data.as_ptr(), &shape, &[], tch::Kind::Half, Device::Cpu)
1929 };
1930
1931 let cpu_tensor = borrowed_tensor.copy();
1933
1934 let cpu_tensor = if dtype != tch::Kind::Half {
1936 cpu_tensor.to_kind(dtype)
1937 } else {
1938 cpu_tensor
1939 };
1940
1941 if *device != Device::Cpu {
1943 let gpu_tensor = cpu_tensor.to_device(*device);
1944 drop(cpu_tensor); gpu_tensor
1946 } else {
1947 cpu_tensor
1948 }
1949 }
1950 safetensors::Dtype::F32 => {
1951 let borrowed_tensor = unsafe {
1953 Tensor::from_blob(data.as_ptr(), &shape, &[], tch::Kind::Float, Device::Cpu)
1954 };
1955
1956 let cpu_tensor = borrowed_tensor.copy();
1958
1959 let cpu_tensor = match dtype {
1961 tch::Kind::Half => cpu_tensor.to_kind(tch::Kind::Half),
1962 tch::Kind::BFloat16 => cpu_tensor.to_kind(tch::Kind::BFloat16),
1963 tch::Kind::Float => cpu_tensor,
1964 _ => return Err(anyhow!("Cannot convert F32 to target dtype {:?}", dtype)),
1965 };
1966
1967 if *device != Device::Cpu {
1969 if idx % 100 == 0 || tensor_size_mb > 10.0 {
1971 info!(
1972 "Transferring tensor {}/{} to GPU: {} ({:.2} MB, total: {:.2} MB)",
1973 idx + 1,
1974 tensor_count,
1975 name,
1976 tensor_size_mb,
1977 total_size_mb
1978 );
1979 }
1980
1981 match safe_to_device(&cpu_tensor, *device) {
1983 Ok(gpu_tensor) => {
1984 drop(cpu_tensor); gpu_tensor
1986 }
1987 Err(e) => {
1988 drop(cpu_tensor); return Err(anyhow!(
1990 "GPU OOM loading tensor '{}': {} | Progress: {}/{} ({:.1} MB) | Try: smaller model, reduce max_position_embeddings, or free GPU memory",
1991 name, e, idx + 1, tensor_count, total_size_mb
1992 ));
1993 }
1994 }
1995 } else {
1996 cpu_tensor
1997 }
1998 }
1999 safetensors::Dtype::F8_E4M3 => {
2003 let borrowed_tensor = unsafe {
2004 Tensor::from_blob(
2005 data.as_ptr(),
2006 &shape,
2007 &[],
2008 tch::Kind::Float8e4m3fn,
2009 Device::Cpu,
2010 )
2011 };
2012 let cpu_tensor = borrowed_tensor.copy();
2013 if *device != Device::Cpu {
2014 cpu_tensor.to_device(*device)
2015 } else {
2016 cpu_tensor
2017 }
2018 }
2019 safetensors::Dtype::F8_E5M2 => {
2020 let borrowed_tensor = unsafe {
2021 Tensor::from_blob(
2022 data.as_ptr(),
2023 &shape,
2024 &[],
2025 tch::Kind::Float8e5m2,
2026 Device::Cpu,
2027 )
2028 };
2029 let cpu_tensor = borrowed_tensor.copy();
2030 if *device != Device::Cpu {
2031 cpu_tensor.to_device(*device)
2032 } else {
2033 cpu_tensor
2034 }
2035 }
2036 dtype => {
2037 return Err(anyhow!(
2038 "Tensor '{}' has unsupported dtype {:?}. Supported: F16, BF16, F32, F8_E4M3, F8_E5M2",
2039 name, dtype
2040 ));
2041 }
2042 };
2043
2044 weights.insert(name.clone(), tensor);
2045 }
2046
2047 info!(
2048 "✅ Successfully loaded {} tensors ({:.2} MB total) to device {:?}",
2049 tensor_count, total_size_mb, device
2050 );
2051
2052 Ok(())
2053 }
2054
2055 fn create_model_from_config(
2057 config: ModelConfig,
2058 mut weights: HashMap<String, Tensor>,
2059 device: &Device,
2060 dtype: DType,
2061 max_context: Option<usize>,
2062 kv_quant_type: KVQuantType,
2063 model_path: &Path,
2064 device_pool: Option<&DevicePool>,
2065 fp8_dequant_load: bool,
2066 speculative_decoding: bool,
2067 ) -> Result<Box<dyn ModelOperations>> {
2068 if let Some(profile) = super::channel_fp8::ChannelFp8Profile::read(model_path)? {
2069 anyhow::ensure!(
2070 !fp8_dequant_load && *device == Device::Cpu,
2071 "unsupported FP8_DYNAMIC execution policy"
2072 );
2073 profile.validate(&weights)?;
2074 } else {
2075 anyhow::ensure!(
2076 !weights.keys().any(|k| k.ends_with(".weight_scale")),
2077 "channel FP8 weights require authoritative compressed-tensors config"
2078 );
2079 }
2080 if let Err(e) =
2085 crate::runtime::ttn_profile::get_layer_profile(model_path, &config, Some(&weights))
2086 {
2087 tracing::warn!("TTN profile analysis failed (non-fatal): {e}");
2088 }
2089
2090 maybe_dequantize_fp8_weights_at_load(&mut weights, fp8_dequant_load, device_pool);
2095
2096 let wants_multi = device_pool.is_some_and(|p| !p.is_single());
2101
2102 match config.architecture {
2103 ModelArchitecture::Llama => {
2104 info!("Creating Llama model");
2105 Self::create_llama_model(
2106 config,
2107 weights,
2108 device,
2109 dtype,
2110 max_context,
2111 kv_quant_type,
2112 device_pool,
2113 None,
2114 )
2115 }
2116 ModelArchitecture::Qwen => {
2117 info!("Creating Qwen model");
2118 Self::create_qwen_model(
2119 config,
2120 weights,
2121 device,
2122 dtype,
2123 max_context,
2124 kv_quant_type,
2125 device_pool,
2126 )
2127 }
2128 ModelArchitecture::Gemma => {
2129 info!("Creating Gemma model");
2130 Self::create_gemma_model(config, weights, device, dtype, max_context, kv_quant_type)
2131 }
2132 ModelArchitecture::Mistral => {
2133 info!("Creating Mistral model");
2134 Self::create_llama_model(
2136 config,
2137 weights,
2138 device,
2139 dtype,
2140 max_context,
2141 kv_quant_type,
2142 device_pool,
2143 None,
2144 )
2145 }
2146 ModelArchitecture::Janus => {
2147 info!("Creating Janus multimodal model");
2148 Self::create_janus_model(config, weights, device, dtype, max_context, kv_quant_type)
2149 }
2150 ModelArchitecture::Qwen3_5 => {
2151 info!("Creating Qwen3.5 hybrid SSM/attention model");
2152 if wants_multi {
2153 warn!(
2154 "Multi-device requested for Qwen3.5, which does not yet implement the \
2155 layer-split path; building single-device on {:?}",
2156 device
2157 );
2158 }
2159 Self::create_qwen3_5_model(
2160 config,
2161 weights,
2162 device,
2163 dtype,
2164 max_context,
2165 kv_quant_type,
2166 speculative_decoding,
2167 )
2168 }
2169 ModelArchitecture::Unknown(arch) => Err(anyhow!("Unknown architecture: {}", arch)),
2170 }
2171 }
2172
2173 fn create_llama_model(
2174 config: ModelConfig,
2175 mut weights: HashMap<String, Tensor>,
2176 device: &Device,
2177 dtype: DType,
2178 max_context: Option<usize>,
2179 kv_quant_type: KVQuantType,
2180 device_pool: Option<&DevicePool>,
2181 stage_range: Option<Range<usize>>,
2182 ) -> Result<Box<dyn ModelOperations>> {
2183 use super::architectures::llama::LlamaConfig;
2184 use super::device_pool::LayerDeviceMap;
2185
2186 let effective_max_pos = max_context.unwrap_or(config.max_position_embeddings);
2188 if max_context.is_some() {
2189 info!(
2190 "Overriding max_position_embeddings: {} -> {}",
2191 config.max_position_embeddings, effective_max_pos
2192 );
2193 }
2194
2195 let llama_config = LlamaConfig {
2197 version: config.version as u8,
2198 num_attention_heads: config.num_attention_heads as u32,
2199 num_key_value_heads: config.num_key_value_heads as u32,
2200 hidden_size: config.hidden_size as u32,
2201 head_dim: config.head_dim as u32,
2202 intermediate_size: config.intermediate_size as u32,
2203 max_position_embeddings: effective_max_pos as u32,
2204 rms_norm_eps: config.rms_norm_eps,
2205 vocab_size: config.vocab_size as u32,
2206 original_vocab_size: config.vocab_size as u32, num_hidden_layers: config.num_hidden_layers as u32,
2208 rope_theta: config.rope_theta,
2209 rope_scaling: None,
2210 hidden_activation: config.hidden_activation,
2211 query_pre_attn_scalar: config.query_pre_attn_scalar,
2212 use_qk_norm: config.use_qk_norm,
2213 scale_embeddings: config.scale_embeddings,
2214 layer_types: vec![],
2215 rope_local_base_freq: None,
2216 model_architecture: match config.architecture {
2224 super::model_config::ModelArchitecture::Qwen => {
2225 super::architectures::ModelArchitecture::Qwen {
2226 version: config.version as u8,
2227 is_moe: config.is_moe,
2228 context_length: config.max_position_embeddings,
2229 }
2230 }
2231 super::model_config::ModelArchitecture::Mistral => {
2232 super::architectures::ModelArchitecture::Mistral
2233 }
2234 super::model_config::ModelArchitecture::Gemma => {
2235 super::architectures::ModelArchitecture::Gemma
2236 }
2237 _ => super::architectures::ModelArchitecture::Llama {
2240 version: config.version as u8,
2241 },
2242 },
2243 };
2244
2245 info!(
2246 "[create_llama_model] Passing llama_config.max_position_embeddings = {} to from_weights_with_config",
2247 llama_config.max_position_embeddings
2248 );
2249
2250 let num_layers = llama_config.num_hidden_layers as usize;
2258 let model = if let Some(layer_range) = stage_range {
2259 let device_map = if let Some(pool) = device_pool.filter(|p| !p.is_single()) {
2260 LayerDeviceMap::even_split(pool, num_layers)?
2261 } else {
2262 LayerDeviceMap::single(*device, num_layers)?
2263 };
2264 info!(
2265 "Building authoritative Llama stage {:?} of {} global layer(s)",
2266 layer_range, num_layers
2267 );
2268 LlamaModel::stage_from_weights_with_config(
2269 &mut weights,
2270 llama_config,
2271 &device_map,
2272 layer_range,
2273 dtype,
2274 kv_quant_type,
2275 )?
2276 } else if let Some(pool) = device_pool.filter(|p| !p.is_single()) {
2277 let device_map = LayerDeviceMap::even_split(pool, num_layers)?;
2278 info!(
2279 "🔀 Building Llama model as a multi-device pipeline: {} layer(s) across \
2280 {} device(s) ({} stage boundary copies per forward)",
2281 num_layers,
2282 pool.len(),
2283 pool.len() - 1,
2284 );
2285 LlamaModel::stage_from_weights_with_config(
2286 &mut weights,
2287 llama_config,
2288 &device_map,
2289 0..num_layers,
2290 dtype,
2291 kv_quant_type,
2292 )?
2293 } else {
2294 LlamaModel::from_weights_with_config(
2296 &mut weights,
2297 llama_config,
2298 device,
2299 dtype,
2300 kv_quant_type,
2301 )?
2302 };
2303
2304 Ok(Box::new(model))
2305 }
2306
2307 fn create_qwen_model(
2308 config: ModelConfig,
2309 weights: HashMap<String, Tensor>,
2310 device: &Device,
2311 dtype: DType,
2312 max_context: Option<usize>,
2313 kv_quant_type: KVQuantType,
2314 device_pool: Option<&DevicePool>,
2315 ) -> Result<Box<dyn ModelOperations>> {
2316 info!(" Using Llama architecture with Qwen configuration");
2319 info!(" rope_theta: {} (from config)", config.rope_theta);
2320 Self::create_llama_model(
2321 config,
2322 weights,
2323 device,
2324 dtype,
2325 max_context,
2326 kv_quant_type,
2327 device_pool,
2328 None,
2329 )
2330 }
2331
2332 fn create_gemma_model(
2333 _config: ModelConfig,
2334 weights: HashMap<String, Tensor>,
2335 device: &Device,
2336 dtype: DType,
2337 _max_context: Option<usize>,
2338 _kv_quant_type: KVQuantType,
2339 ) -> Result<Box<dyn ModelOperations>> {
2340 Ok(Box::new(GemmaModel::from_weights(&weights, device, dtype)?))
2343 }
2344
2345 fn create_janus_model(
2346 config: ModelConfig,
2347 weights: HashMap<String, Tensor>,
2348 device: &Device,
2349 dtype: DType,
2350 max_context: Option<usize>,
2351 _kv_quant_type: KVQuantType,
2352 ) -> Result<Box<dyn ModelOperations>> {
2353 use super::architectures::janus::{
2354 JanusConfig, JanusModel, ProjectorConfig, VisionEncoderConfig,
2355 };
2356 use super::architectures::VisionEncoderType;
2357
2358 let effective_max_pos = max_context.unwrap_or(config.max_position_embeddings);
2360
2361 let janus_config = JanusConfig {
2364 language_config: Box::new(super::architectures::llama::LlamaConfig {
2366 version: 3,
2367 num_attention_heads: config.num_attention_heads as u32,
2368 num_key_value_heads: config.num_key_value_heads as u32,
2369 hidden_size: config.hidden_size as u32,
2370 head_dim: config.head_dim as u32,
2371 intermediate_size: config.intermediate_size as u32,
2372 max_position_embeddings: effective_max_pos as u32,
2373 rms_norm_eps: config.rms_norm_eps,
2374 vocab_size: config.vocab_size as u32,
2375 original_vocab_size: config.vocab_size as u32,
2376 num_hidden_layers: config.num_hidden_layers as u32,
2377 rope_theta: config.rope_theta,
2378 rope_scaling: None, hidden_activation: config.hidden_activation.clone(),
2380 query_pre_attn_scalar: config.query_pre_attn_scalar,
2381 use_qk_norm: config.use_qk_norm,
2382 scale_embeddings: config.scale_embeddings,
2383 layer_types: vec!["global".to_owned(); config.num_hidden_layers],
2384 rope_local_base_freq: None,
2385 model_architecture: super::architectures::ModelArchitecture::Llama { version: 3 },
2386 }),
2387 vision_config: VisionEncoderConfig {
2388 encoder_type: VisionEncoderType::SigLIP {
2389 hidden_size: 1024, image_size: 384,
2391 patch_size: 16,
2392 num_layers: 24,
2393 },
2394 hidden_size: 1024,
2395 image_size: 384,
2396 patch_size: 16,
2397 num_layers: 24,
2398 num_patches: (384 / 16) * (384 / 16), num_attention_heads: Some(16), intermediate_size: Some(4096), },
2402 aligner_config: ProjectorConfig {
2403 input_dim: 1024, output_dim: config.hidden_size, hidden_dim: Some(config.hidden_size), },
2407 generation_config: None, device: *device,
2409 dtype,
2410 };
2411
2412 Ok(Box::new(JanusModel::from_weights(
2413 weights,
2414 janus_config,
2415 *device,
2416 dtype,
2417 )?))
2418 }
2419
2420 fn create_qwen3_5_model(
2421 config: ModelConfig,
2422 mut weights: HashMap<String, Tensor>,
2423 device: &Device,
2424 dtype: DType,
2425 max_context: Option<usize>,
2426 kv_quant_type: KVQuantType,
2427 speculative_decoding: bool,
2428 ) -> Result<Box<dyn ModelOperations>> {
2429 use super::architectures::qwen3_5::{Qwen3_5Model, Qwen3_5TextConfig};
2430 use super::architectures::qwen3_5_vision::Qwen3_5VisionConfig;
2431
2432 let effective_max_pos = max_context.unwrap_or(config.max_position_embeddings);
2433
2434 let text_cfg = Qwen3_5TextConfig::from_model_config(&config, effective_max_pos);
2435
2436 let vision_cfg = if config.has_vision {
2438 Some(Qwen3_5VisionConfig {
2441 depth: 27,
2442 hidden_size: 1152,
2443 intermediate_size: 4304,
2444 num_heads: 16,
2445 head_dim: 72,
2446 patch_size: 16,
2447 temporal_patch_size: 2,
2448 spatial_merge_size: 2,
2449 out_hidden_size: config.vision_out_hidden_size,
2450 rms_norm_eps: 1e-6,
2451 })
2452 } else if weights.keys().any(|k| k.starts_with("visual.")) {
2453 Some(Qwen3_5VisionConfig {
2455 depth: 27,
2456 hidden_size: 1152,
2457 intermediate_size: 4304,
2458 num_heads: 16,
2459 head_dim: 72,
2460 patch_size: 16,
2461 temporal_patch_size: 2,
2462 spatial_merge_size: 2,
2463 out_hidden_size: if config.vision_out_hidden_size > 0 {
2464 config.vision_out_hidden_size
2465 } else {
2466 3584
2467 },
2468 rms_norm_eps: 1e-6,
2469 })
2470 } else {
2471 None
2472 };
2473
2474 Ok(Box::new(Qwen3_5Model::from_weights(
2475 &mut weights,
2476 text_cfg,
2477 vision_cfg,
2478 device,
2479 dtype,
2480 kv_quant_type,
2481 speculative_decoding,
2482 )?))
2483 }
2484
2485 #[instrument(name = "model_factory.create_with_fs", skip(device, dtype, fs, device_pool), fields(model_path = %model_path.display()))]
2497 pub async fn create_with_fs(
2498 model_path: &Path,
2499 device: &Device,
2500 dtype: DType,
2501 max_context: Option<usize>,
2502 kv_quant_type: KVQuantType,
2503 fs: &WorktreeClient,
2504 device_pool: Option<&DevicePool>,
2505 fp8_dequant_load: bool,
2506 speculative_decoding: bool,
2507 ) -> Result<Box<dyn ModelOperations>> {
2508 info!("Loading model via FsOps: {}", model_path.display());
2509
2510 let shard_names = Self::find_shard_names_fs(fs).await?;
2511
2512 let speculative_decoding = Self::mtp_loading_policy_fs(
2513 model_path,
2514 fs,
2515 &shard_names,
2516 mtp_runtime_supported(speculative_decoding, kv_quant_type),
2517 )
2518 .await?;
2519
2520 if shard_names.len() > 1 {
2521 info!("Loading {} weight shards via FsOps", shard_names.len());
2522 }
2523
2524 let weights = Self::load_weights_fs(
2525 fs,
2526 &shard_names,
2527 device,
2528 dtype,
2529 speculative_decoding,
2530 native_finite_policy(model_path),
2531 )
2532 .await?;
2533 let config = ModelConfig::load(model_path, &weights)?;
2534 let model = Self::create_model_from_config(
2535 config,
2536 weights,
2537 device,
2538 dtype,
2539 max_context,
2540 kv_quant_type,
2541 model_path,
2542 device_pool,
2543 fp8_dequant_load,
2544 speculative_decoding,
2545 )?;
2546 info!("Model created successfully via FsOps");
2547 Ok(model)
2548 }
2549
2550 pub async fn detect_model_dtype_fs(fs: &WorktreeClient) -> Result<DType> {
2552 let shard_names = Self::find_shard_names_fs(fs).await?;
2553 if shard_names.is_empty() {
2554 return Err(anyhow!("No model weights found"));
2555 }
2556
2557 let file_content = fs.read_file_chunked(&shard_names[0]).await?;
2558 let tensors = safetensors::SafeTensors::deserialize(&file_content)?;
2559
2560 let mut f16_count = 0;
2561 let mut bf16_count = 0;
2562 let mut f32_count = 0;
2563
2564 for (_, tensor) in tensors.tensors().into_iter().take(10) {
2565 match tensor.dtype() {
2566 safetensors::Dtype::F16 => f16_count += 1,
2567 safetensors::Dtype::BF16
2568 | safetensors::Dtype::F8_E4M3
2569 | safetensors::Dtype::F8_E5M2 => bf16_count += 1,
2570 safetensors::Dtype::F32 => f32_count += 1,
2571 _ => {}
2572 }
2573 }
2574
2575 if f16_count > bf16_count && f16_count > f32_count {
2576 info!("Detected F16 model (via FsOps)");
2577 Ok(tch::Kind::Half)
2578 } else if bf16_count >= f16_count && bf16_count >= f32_count {
2579 info!("Detected BF16 model (via FsOps)");
2580 Ok(tch::Kind::BFloat16)
2581 } else if f32_count > 0 {
2582 info!("Detected F32 model (via FsOps)");
2583 Ok(tch::Kind::Float)
2584 } else {
2585 info!("Could not detect model dtype via FsOps, defaulting to BF16");
2586 Ok(tch::Kind::BFloat16)
2587 }
2588 }
2589
2590 async fn find_shard_names_fs(fs: &WorktreeClient) -> Result<Vec<String>> {
2592 if fs
2594 .stat_path("model.safetensors")
2595 .await
2596 .map(|s| s.exists)
2597 .unwrap_or(false)
2598 {
2599 return Ok(vec!["model.safetensors".to_owned()]);
2600 }
2601
2602 let entries = fs.list_dir_path(".").await?;
2604 let mut shard_names: Vec<String> = entries
2605 .into_iter()
2606 .filter(|e| e.name.starts_with("model-") && e.name.ends_with(".safetensors"))
2607 .map(|e| e.name)
2608 .collect();
2609
2610 shard_names.sort();
2611 Ok(shard_names)
2612 }
2613
2614 async fn load_weights_fs(
2616 fs: &WorktreeClient,
2617 shard_names: &[String],
2618 device: &Device,
2619 dtype: DType,
2620 speculative_decoding: bool,
2621 native_finite: bool,
2622 ) -> Result<HashMap<String, Tensor>> {
2623 let mut all_weights = HashMap::new();
2624
2625 if shard_names.is_empty() {
2626 return Err(anyhow!("No safetensors files found"));
2627 }
2628
2629 for (idx, name) in shard_names.iter().enumerate() {
2630 if shard_names.len() > 1 {
2631 info!(
2632 "Loading shard {}/{} via FsOps: {}",
2633 idx + 1,
2634 shard_names.len(),
2635 name
2636 );
2637 }
2638
2639 let data = fs.read_file_chunked(name).await?;
2640 let tensors = safetensors::SafeTensors::deserialize(&data)?;
2641 Self::create_tensors_from_safetensors_selected(
2642 tensors,
2643 None,
2644 &mut all_weights,
2645 device,
2646 dtype,
2647 speculative_decoding,
2648 native_finite,
2649 )?;
2650 }
2651
2652 Ok(all_weights)
2653 }
2654}
2655
2656#[cfg(test)]
2657#[allow(clippy::expect_used, clippy::unwrap_used)]
2658mod stage_subset_tests {
2659 use super::*;
2660 use safetensors::tensor::{serialize, Dtype, TensorView};
2661
2662 struct FixtureTensor {
2663 name: String,
2664 dtype: Dtype,
2665 shape: Vec<usize>,
2666 data: Vec<u8>,
2667 }
2668
2669 impl FixtureTensor {
2670 fn f32(name: impl Into<String>, shape: &[usize], seed: f32) -> Self {
2671 let elements = shape.iter().product();
2672 let data = (0..elements)
2673 .flat_map(|index| (seed + index as f32 * 0.01).to_le_bytes())
2674 .collect();
2675 Self {
2676 name: name.into(),
2677 dtype: Dtype::F32,
2678 shape: shape.to_vec(),
2679 data,
2680 }
2681 }
2682
2683 fn i64(name: impl Into<String>) -> Self {
2684 Self::i64_shape(name, &[1])
2685 }
2686
2687 fn i64_shape(name: impl Into<String>, shape: &[usize]) -> Self {
2688 let elements = shape.iter().product();
2689 let data = (0..elements)
2690 .flat_map(|index| (index as i64 + 7).to_le_bytes())
2691 .collect();
2692 Self {
2693 name: name.into(),
2694 dtype: Dtype::I64,
2695 shape: shape.to_vec(),
2696 data,
2697 }
2698 }
2699 }
2700
2701 fn write_shard(dir: &Path, name: &str, tensors: &[FixtureTensor]) {
2702 let views: Vec<_> = tensors
2703 .iter()
2704 .map(|tensor| {
2705 (
2706 tensor.name.clone(),
2707 TensorView::new(tensor.dtype, tensor.shape.clone(), &tensor.data).unwrap(),
2708 )
2709 })
2710 .collect();
2711 let bytes = serialize(views, &None).unwrap();
2712 std::fs::write(dir.join(name), bytes).unwrap();
2713 }
2714
2715 fn write_index(dir: &Path, entries: &[(&str, &str)]) {
2716 let weight_map: BTreeMap<_, _> = entries
2717 .iter()
2718 .map(|(tensor, shard)| ((*tensor).to_owned(), (*shard).to_owned()))
2719 .collect();
2720 let index = serde_json::json!({ "weight_map": weight_map });
2721 std::fs::write(
2722 dir.join("model.safetensors.index.json"),
2723 serde_json::to_vec(&index).unwrap(),
2724 )
2725 .unwrap();
2726 }
2727
2728 fn finite_fixture(name: &str, dtype: Dtype, data: Vec<u8>) -> FixtureTensor {
2729 let elements = data.len() / dtype.size();
2730 FixtureTensor {
2731 name: name.to_owned(),
2732 dtype,
2733 shape: vec![elements],
2734 data,
2735 }
2736 }
2737
2738 fn finite_config(dir: &Path, config: serde_json::Value) {
2739 std::fs::write(
2740 dir.join("config.json"),
2741 serde_json::to_vec(&config).unwrap(),
2742 )
2743 .unwrap();
2744 }
2745
2746 fn channel_profile_config() -> serde_json::Value {
2747 serde_json::json!({"model_type":"qwen3_5_text","quantization_config":{
2748 "quant_method":"compressed-tensors", "version":"0.19.0", "format":"float-quantized", "quantization_status":"compressed", "ignore":["lm_head"],
2749 "config_groups":{"g":{"format":"float-quantized", "targets":["Linear"],
2750 "weights":{"num_bits":8,"type":"float","strategy":"channel","symmetric":true,"dynamic":false},
2751 "input_activations":{"num_bits":8,"type":"float","strategy":"token","symmetric":true,"dynamic":true}}}}})
2752 }
2753
2754 #[tokio::test]
2755 async fn channel_prepared_factory_single_and_sharded_cpu_forward_match() {
2756 let mut results = Vec::new();
2757 for sharded in [false, true] {
2758 let dir = tempfile::tempdir().unwrap();
2759 let mut tensors = tiny_dense_tensors(true);
2760 let mut scales = Vec::new();
2761 for matrix in tensors.iter_mut().filter(|tensor| {
2762 tensor.shape.len() == 2 && tensor.name.starts_with("model.layers.")
2763 }) {
2764 matrix.dtype = Dtype::F8_E4M3;
2765 matrix.data = vec![0x38; matrix.shape.iter().product()];
2766 scales.push(FixtureTensor::f32(
2767 format!("{}_scale", matrix.name),
2768 &[matrix.shape[0], 1],
2769 0.25,
2770 ));
2771 }
2772 tensors.extend(scales);
2773 write_tiny_dense_checkpoint(dir.path(), &tensors);
2774 let mut config: serde_json::Value =
2775 serde_json::from_slice(&std::fs::read(dir.path().join("config.json")).unwrap())
2776 .unwrap();
2777 config["quantization_config"] = channel_profile_config()["quantization_config"].clone();
2778 finite_config(dir.path(), config);
2779 if sharded {
2780 let (matrices, scales): (Vec<_>, Vec<_>) = tensors
2781 .into_iter()
2782 .partition(|tensor| !tensor.name.ends_with("_scale"));
2783 write_shard(dir.path(), "a.safetensors", &matrices);
2784 write_shard(dir.path(), "b.safetensors", &scales);
2785 let index: Vec<_> = matrices
2786 .iter()
2787 .map(|tensor| (tensor.name.as_str(), "a.safetensors"))
2788 .chain(
2789 scales
2790 .iter()
2791 .map(|tensor| (tensor.name.as_str(), "b.safetensors")),
2792 )
2793 .collect();
2794 write_index(dir.path(), &index);
2795 }
2796 let model = ModelFactory::create(
2797 dir.path(),
2798 &Device::Cpu,
2799 DType::Float,
2800 Some(16),
2801 KVQuantType::None,
2802 None,
2803 false,
2804 false,
2805 )
2806 .await
2807 .unwrap();
2808 let result = model
2809 .forward(&Tensor::from_slice(&[1i64, 2]).reshape([1, 2]), None)
2810 .unwrap();
2811 assert_eq!(result.kind(), DType::Float);
2812 assert!(result.isfinite().all().int64_value(&[]) != 0);
2813 results.push(result);
2814 }
2815 assert!(results[0].allclose(&results[1], 1e-6, 1e-6, false));
2816 }
2817
2818 #[cfg(target_os = "linux")]
2819 fn channel_private_spool_fixture_dir() -> tempfile::TempDir {
2820 use std::os::unix::fs::PermissionsExt as _;
2821 tempfile::Builder::new()
2822 .permissions(std::fs::Permissions::from_mode(0o700))
2823 .tempdir_in(std::env::current_dir().unwrap())
2824 .unwrap()
2825 }
2826
2827 #[cfg(target_os = "linux")]
2828 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2829 async fn channel_prepared_cancel_during_spool_does_not_resolve_next_and_releases_fd() {
2830 use parking_lot::{Condvar, Mutex as TestMutex};
2831 use std::os::fd::AsRawFd as _;
2832 use std::os::unix::fs::MetadataExt as _;
2833 use std::sync::{
2834 atomic::{AtomicUsize, Ordering},
2835 Arc,
2836 };
2837 let dir = channel_private_spool_fixture_dir();
2838 write_shard(
2839 dir.path(),
2840 "w.safetensors",
2841 &[FixtureTensor {
2842 name: "p.weight".into(),
2843 dtype: Dtype::F8_E4M3,
2844 shape: vec![2, 3],
2845 data: vec![0x38; 6],
2846 }],
2847 );
2848 let mut payload = std::fs::read(dir.path().join("w.safetensors")).unwrap();
2849 let profile =
2850 super::super::channel_fp8::ChannelFp8Profile::parse(&channel_profile_config())
2851 .unwrap()
2852 .unwrap();
2853 let resolve_count = Arc::new(AtomicUsize::new(0));
2854 let started = Arc::new(tokio::sync::Notify::new());
2855 let release = Arc::new((TestMutex::new(false), Condvar::new()));
2856 let fd_witness = Arc::new(TestMutex::new(None));
2857 let task = {
2858 let count = resolve_count.clone();
2859 let started = started.clone();
2860 let release = release.clone();
2861 let witness = fd_witness.clone();
2862 let parent = dir.path().to_path_buf();
2863 tokio::spawn(async move {
2864 ModelFactory::prepare_channel_inputs_spooling(
2865 &profile,
2866 &[PathBuf::from("w"), PathBuf::from("later")],
2867 move |_| {
2868 assert_eq!(
2869 count.fetch_add(1, Ordering::SeqCst),
2870 0,
2871 "cancelled preparation resolved another shard"
2872 );
2873 std::future::ready(Ok(ResolvedWeight::Owned(std::mem::take(&mut payload))))
2874 },
2875 move |bytes| {
2876 let parent = parent.clone();
2877 let started = started.clone();
2878 let release = release.clone();
2879 let witness = witness.clone();
2880 async move {
2881 tokio::task::spawn_blocking(move || {
2882 let file = super::super::resolved_spool::spool_in(bytes, &parent)?;
2883 *witness.lock() = Some((file.as_raw_fd(), file.metadata()?.ino()));
2884 started.notify_one();
2885 let (lock, cv) = &*release;
2886 let mut guard = lock.lock();
2887 cv.wait_while(&mut guard, |ready| !*ready);
2888 drop(guard);
2889 Ok(file)
2890 })
2891 .await?
2892 }
2893 },
2894 )
2895 .await
2896 })
2897 };
2898 tokio::time::timeout(std::time::Duration::from_secs(5), started.notified())
2899 .await
2900 .unwrap();
2901 task.abort();
2902 assert!(task.await.err().unwrap().is_cancelled());
2903 assert_eq!(resolve_count.load(Ordering::SeqCst), 1);
2904 {
2905 let (lock, cv) = &*release;
2906 *lock.lock() = true;
2907 cv.notify_one();
2908 }
2909 let (fd, ino) = fd_witness.lock().unwrap();
2910 tokio::time::timeout(std::time::Duration::from_secs(5), async {
2911 while std::fs::metadata(format!("/proc/self/fd/{fd}"))
2912 .map(|m| m.ino() == ino)
2913 .unwrap_or(false)
2914 {
2915 tokio::task::yield_now().await;
2916 }
2917 })
2918 .await
2919 .unwrap();
2920 assert_eq!(resolve_count.load(Ordering::SeqCst), 1);
2921 }
2922
2923 #[cfg(target_os = "linux")]
2924 #[tokio::test]
2925 async fn channel_prepared_multiple_owned_spool_before_next_resolution() {
2926 use std::os::fd::AsRawFd as _;
2927 use std::os::unix::fs::MetadataExt as _;
2928 use std::sync::atomic::{AtomicUsize, Ordering};
2929 let dir = channel_private_spool_fixture_dir();
2930 let config = channel_profile_config();
2931 finite_config(dir.path(), config.clone());
2932 let profile = super::super::channel_fp8::ChannelFp8Profile::parse(&config)
2933 .unwrap()
2934 .unwrap();
2935 for (name, tensor) in [("a.safetensors", "p.weight"), ("b.safetensors", "q.weight")] {
2936 write_shard(
2937 dir.path(),
2938 name,
2939 &[FixtureTensor {
2940 name: tensor.into(),
2941 dtype: Dtype::F8_E4M3,
2942 shape: vec![2, 3],
2943 data: vec![0x38; 6],
2944 }],
2945 );
2946 }
2947 write_shard(
2948 dir.path(),
2949 "c.safetensors",
2950 &[
2951 FixtureTensor::f32("p.weight_scale", &[2, 1], 0.5),
2952 FixtureTensor::f32("q.weight_scale", &[2, 1], 0.5),
2953 ],
2954 );
2955 let paths: Vec<_> = ["a.safetensors", "b.safetensors", "c.safetensors"]
2956 .iter()
2957 .map(|n| dir.path().join(n))
2958 .collect();
2959 let mut payloads: Vec<_> = paths.iter().map(|p| std::fs::read(p).unwrap()).collect();
2960 for path in &paths {
2961 std::fs::write(path, b"version https://git-lfs.github.com/spec/v1\n").unwrap();
2962 }
2963 let resolved = AtomicUsize::new(0);
2964 let spooled = AtomicUsize::new(0);
2965 let mut fds = Vec::new();
2966 let inputs = ModelFactory::prepare_channel_inputs_spooling(
2967 &profile,
2968 &paths,
2969 |_| {
2970 let index = resolved.fetch_add(1, Ordering::SeqCst);
2971 assert_eq!(spooled.load(Ordering::SeqCst), index);
2974 std::future::ready(Ok(ResolvedWeight::Owned(std::mem::take(
2975 &mut payloads[index],
2976 ))))
2977 },
2978 |bytes| {
2979 let file = super::super::resolved_spool::spool_in(bytes, dir.path()).unwrap();
2980 fds.push((file.as_raw_fd(), file.metadata().unwrap().ino()));
2981 spooled.fetch_add(1, Ordering::SeqCst);
2982 std::future::ready(Ok(file))
2983 },
2984 )
2985 .await
2986 .unwrap();
2987 assert_eq!(resolved.load(Ordering::SeqCst), 3);
2988 assert!(inputs
2989 .iter()
2990 .all(|(_, r)| matches!(r, ResolvedWeight::File(_))));
2991 for path in &paths {
2992 std::fs::remove_file(path).unwrap();
2993 }
2994 let mut weights = HashMap::new();
2995 for (path, input) in inputs {
2996 ModelFactory::load_resolved_safetensors_file_inner(
2997 &path,
2998 input,
2999 None,
3000 &mut weights,
3001 &Device::Cpu,
3002 DType::Float,
3003 false,
3004 true,
3005 true,
3006 )
3007 .await
3008 .unwrap();
3009 }
3010 let p =
3011 crate::runtime::architectures::llama::LinearProjection::take(&mut weights, "p.weight")
3012 .unwrap();
3013 let output = p
3014 .try_apply(&Tensor::ones([1, 3], (DType::Float, Device::Cpu)))
3015 .unwrap();
3016 assert!((output.double_value(&[0, 0]) - 1.5).abs() < 1e-6);
3017 assert!((output.double_value(&[0, 1]) - 1.53).abs() < 1e-6);
3018 for (fd, ino) in fds {
3019 assert!(
3020 std::fs::metadata(format!("/proc/self/fd/{fd}"))
3021 .map(|m| m.ino() != ino)
3022 .unwrap_or(true),
3023 "retained spool FD leaked after load"
3024 );
3025 }
3026 }
3027
3028 #[cfg(target_os = "linux")]
3029 #[tokio::test]
3030 async fn channel_prepared_owned_preflight_failure_releases_all_spools() {
3031 use std::os::fd::AsRawFd as _;
3032 use std::os::unix::fs::MetadataExt as _;
3033 let dir = channel_private_spool_fixture_dir();
3034 let profile =
3035 super::super::channel_fp8::ChannelFp8Profile::parse(&channel_profile_config())
3036 .unwrap()
3037 .unwrap();
3038 write_shard(
3039 dir.path(),
3040 "a.safetensors",
3041 &[FixtureTensor {
3042 name: "p.weight".into(),
3043 dtype: Dtype::F8_E4M3,
3044 shape: vec![2, 3],
3045 data: vec![0x38; 6],
3046 }],
3047 );
3048 let path = dir.path().join("a.safetensors");
3049 let mut payload = std::fs::read(&path).unwrap();
3050 let mut fds = Vec::new();
3051 let result = ModelFactory::prepare_channel_inputs_spooling(
3052 &profile,
3053 &[path],
3054 |_| std::future::ready(Ok(ResolvedWeight::Owned(std::mem::take(&mut payload)))),
3055 |bytes| {
3056 let file = super::super::resolved_spool::spool_in(bytes, dir.path()).unwrap();
3057 fds.push((file.as_raw_fd(), file.metadata().unwrap().ino()));
3058 std::future::ready(Ok(file))
3059 },
3060 )
3061 .await;
3062 let error = result.err().unwrap();
3063 assert!(error.to_string().contains("scale"), "{error}");
3064 for (fd, ino) in fds {
3065 assert!(std::fs::metadata(format!("/proc/self/fd/{fd}"))
3066 .map(|m| m.ino() != ino)
3067 .unwrap_or(true));
3068 }
3069 }
3070
3071 #[cfg(target_os = "linux")]
3072 #[tokio::test]
3073 async fn channel_prepared_file_inputs_never_require_spool_cache() {
3074 let dir = tempfile::tempdir().unwrap();
3075 let profile =
3076 super::super::channel_fp8::ChannelFp8Profile::parse(&channel_profile_config())
3077 .unwrap()
3078 .unwrap();
3079 write_shard(
3080 dir.path(),
3081 "a.safetensors",
3082 &[
3083 FixtureTensor {
3084 name: "p.weight".into(),
3085 dtype: Dtype::F8_E4M3,
3086 shape: vec![2, 3],
3087 data: vec![0x38; 6],
3088 },
3089 FixtureTensor::f32("p.weight_scale", &[2, 1], 0.5),
3090 ],
3091 );
3092 let path = dir.path().join("a.safetensors");
3093 let inputs = ModelFactory::prepare_channel_inputs_spooling(
3094 &profile,
3095 &[path],
3096 |path| std::future::ready(Ok(ResolvedWeight::File(std::fs::File::open(path).unwrap()))),
3097 |_| std::future::ready(Err(anyhow!("cache unavailable"))),
3098 )
3099 .await
3100 .unwrap();
3101 assert_eq!(inputs.len(), 1);
3102 }
3103
3104 #[tokio::test]
3105 async fn channel_prepared_pointer_and_file_resolve_once_through_preflight_and_load() {
3106 use std::sync::atomic::{AtomicUsize, Ordering};
3107 let dir = tempfile::tempdir().unwrap();
3108 let config = channel_profile_config();
3109 finite_config(dir.path(), config.clone());
3110 let profile = super::super::channel_fp8::ChannelFp8Profile::parse(&config)
3111 .unwrap()
3112 .unwrap();
3113 let weight = FixtureTensor {
3114 name: "p.weight".into(),
3115 dtype: Dtype::F8_E4M3,
3116 shape: vec![2, 3],
3117 data: vec![0x38; 6],
3118 };
3119 write_shard(dir.path(), "w.safetensors", &[weight]);
3120 let weight_path = dir.path().join("w.safetensors");
3121 let resolved_bytes = std::fs::read(&weight_path).unwrap();
3122 std::fs::write(
3125 &weight_path,
3126 b"version https://git-lfs.github.com/spec/v1\noid sha256:fixture\n",
3127 )
3128 .unwrap();
3129 write_shard(
3130 dir.path(),
3131 "s.safetensors",
3132 &[FixtureTensor::f32("p.weight_scale", &[2, 1], 0.5)],
3133 );
3134 let scale_path = dir.path().join("s.safetensors");
3135 let count = AtomicUsize::new(0);
3136 let mut inputs = ModelFactory::prepare_channel_inputs(
3137 &profile,
3138 &[weight_path.clone(), scale_path.clone()],
3139 |path| {
3140 count.fetch_add(1, Ordering::SeqCst);
3141 std::future::ready(Ok(if path == weight_path {
3142 ResolvedWeight::Owned(resolved_bytes.clone())
3143 } else {
3144 ResolvedWeight::File(std::fs::File::open(path).unwrap())
3145 }))
3146 },
3147 )
3148 .await
3149 .unwrap();
3150 assert_eq!(count.load(Ordering::SeqCst), 2);
3151 profile.preflight_resolved(&mut inputs).unwrap();
3154 std::fs::remove_file(&weight_path).unwrap();
3155 std::fs::remove_file(&scale_path).unwrap();
3156 let mut weights = HashMap::new();
3157 for (path, resolved) in inputs {
3158 ModelFactory::load_resolved_safetensors_file_inner(
3159 &path,
3160 resolved,
3161 None,
3162 &mut weights,
3163 &Device::Cpu,
3164 DType::Float,
3165 false,
3166 true,
3167 true,
3168 )
3169 .await
3170 .unwrap();
3171 }
3172 assert_eq!(
3173 count.load(Ordering::SeqCst),
3174 2,
3175 "factory preflight+load must resolve once per shard"
3176 );
3177 assert_eq!(weights["p.weight"].kind(), DType::Float8e4m3fn);
3178 assert_eq!(weights["p.weight_scale"].size(), [2, 1]);
3179 let projection =
3180 crate::runtime::architectures::llama::LinearProjection::take(&mut weights, "p.weight")
3181 .unwrap();
3182 let output = projection
3183 .try_apply(&Tensor::ones([1, 3], (DType::Float, Device::Cpu)))
3184 .unwrap();
3185 assert!((output.double_value(&[0, 0]) - 1.5).abs() < 1e-6);
3186 assert!((output.double_value(&[0, 1]) - 1.53).abs() < 1e-6);
3187 }
3188
3189 #[tokio::test]
3190 async fn channel_prepared_header_and_resolver_failures_are_closed_before_tensors() {
3191 let profile =
3192 super::super::channel_fp8::ChannelFp8Profile::parse(&channel_profile_config())
3193 .unwrap()
3194 .unwrap();
3195 let path = PathBuf::from("unresolved-pointer.safetensors");
3196 let error =
3197 ModelFactory::prepare_channel_inputs(&profile, std::slice::from_ref(&path), |_| {
3198 std::future::ready(Err(anyhow!("malformed captured LFS pointer")))
3199 })
3200 .await
3201 .err()
3202 .unwrap();
3203 assert!(error.to_string().contains("malformed captured LFS pointer"));
3204 for data in [
3205 vec![],
3206 (16 * 1024 * 1024u64 + 1).to_le_bytes().to_vec(),
3207 b"version https://git-lfs.github.com/spec/v1".to_vec(),
3208 10u64.to_le_bytes().to_vec(),
3209 ] {
3210 assert!(ModelFactory::prepare_channel_inputs(
3211 &profile,
3212 std::slice::from_ref(&path),
3213 |_| std::future::ready(Ok(ResolvedWeight::Owned(data.clone())))
3214 )
3215 .await
3216 .is_err());
3217 }
3218 }
3219
3220 #[tokio::test]
3221 async fn native_finite_path_rejects_nonfinite_float_payloads() {
3222 let dir = tempfile::tempdir().unwrap();
3223 finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5"}));
3224 let path = dir.path().join("weights.safetensors");
3225 let cases = [
3226 (Dtype::BF16, vec![0x7f80u32, 0xff80, 0x7fc1, 0xffc1]),
3227 (Dtype::F16, vec![0x7c00, 0xfc00, 0x7e01, 0xfe01]),
3228 (
3229 Dtype::F32,
3230 vec![0x7f80_0000, 0xff80_0000, 0x7fc0_0001, 0xffc0_0001],
3231 ),
3232 ];
3233 for (dtype, patterns) in cases {
3234 for (index, bits) in patterns.into_iter().enumerate() {
3235 let name = [
3236 "model.embed_tokens.weight",
3237 "model.norm.weight",
3238 "model.layers.0.linear_attn.A_log",
3239 "model.layers.0.self_attn.q_proj.weight_scale",
3240 ][index];
3241 let data = if dtype == Dtype::F32 {
3244 [0u32.to_le_bytes(), bits.to_le_bytes()].concat()
3245 } else {
3246 [0u16.to_le_bytes(), (bits as u16).to_le_bytes()].concat()
3247 };
3248 write_shard(
3249 dir.path(),
3250 "weights.safetensors",
3251 &[finite_fixture(name, dtype, data)],
3252 );
3253 let mut weights = HashMap::new();
3254 let error = ModelFactory::load_safetensors_file(
3255 &path,
3256 &mut weights,
3257 &Device::Cpu,
3258 DType::Float,
3259 false,
3260 native_finite_policy(dir.path()),
3261 )
3262 .await
3263 .unwrap_err()
3264 .to_string();
3265 assert!(error.contains("nonfinite native Qwen3.5"), "{error}");
3266 assert!(
3267 error.contains(name) && error.contains("element 1"),
3268 "{error}"
3269 );
3270 assert!(weights.is_empty(), "offending tensor was inserted");
3271 }
3272 }
3273 }
3274
3275 #[tokio::test]
3276 async fn native_finite_floating_boundaries_preserve_stored_values() {
3277 let dir = tempfile::tempdir().unwrap();
3278 finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5"}));
3279 let path = dir.path().join("weights.safetensors");
3280 let cases = [
3281 (
3282 Dtype::BF16,
3283 DType::BFloat16,
3284 [0u16, 0x8000, 1, 0x8001, 0x7f7f, 0xff7f]
3285 .into_iter()
3286 .flat_map(u16::to_le_bytes)
3287 .collect::<Vec<_>>(),
3288 ),
3289 (
3290 Dtype::F16,
3291 DType::Half,
3292 [0u16, 0x8000, 1, 0x8001, 0x7bff, 0xfbff]
3293 .into_iter()
3294 .flat_map(u16::to_le_bytes)
3295 .collect(),
3296 ),
3297 (
3298 Dtype::F32,
3299 DType::Float,
3300 [0u32, 0x8000_0000, 1, 0x8000_0001, 0x7f7f_ffff, 0xff7f_ffff]
3301 .into_iter()
3302 .flat_map(u32::to_le_bytes)
3303 .collect(),
3304 ),
3305 ];
3306 for (dtype, compute, data) in cases {
3307 write_shard(
3308 dir.path(),
3309 "weights.safetensors",
3310 &[finite_fixture("model.norm.weight", dtype, data.clone())],
3311 );
3312 let mut weights = HashMap::new();
3313 ModelFactory::load_safetensors_file(
3314 &path,
3315 &mut weights,
3316 &Device::Cpu,
3317 compute,
3318 false,
3319 native_finite_policy(dir.path()),
3320 )
3321 .await
3322 .unwrap();
3323 let tensor = &weights["model.norm.weight"];
3324 assert_eq!(tensor.kind(), compute);
3325 let mut actual = vec![0u8; data.len()];
3326 tensor.copy_data_u8(&mut actual, tensor.numel());
3327 assert_eq!(actual, data, "finite boundary bytes changed for{dtype:?}");
3328 }
3329 }
3330
3331 #[tokio::test]
3332 async fn native_finite_fp8_path_preserves_finite_codes_rejects_nan() {
3333 let dir = tempfile::tempdir().unwrap();
3334 finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5_text"}));
3335 let path = dir.path().join("weights.safetensors");
3336 let finite_codes = vec![0x00, 0x80, 0x01, 0x81, 0x7e, 0xfe];
3337 write_shard(
3338 dir.path(),
3339 "weights.safetensors",
3340 &[finite_fixture(
3341 "p.weight",
3342 Dtype::F8_E4M3,
3343 finite_codes.clone(),
3344 )],
3345 );
3346 let mut weights = HashMap::new();
3347 ModelFactory::load_safetensors_file(
3348 &path,
3349 &mut weights,
3350 &Device::Cpu,
3351 DType::Float,
3352 false,
3353 native_finite_policy(dir.path()),
3354 )
3355 .await
3356 .unwrap();
3357 let tensor = &weights["p.weight"];
3358 assert_eq!(tensor.kind(), DType::Float8e4m3fn);
3359 let mut actual = vec![0u8; finite_codes.len()];
3360 tensor.copy_data_u8(&mut actual, tensor.numel());
3361 assert_eq!(actual, finite_codes); let floating = tensor.to_kind(DType::Float);
3363 assert_eq!(floating.double_value(&[4]), 448.0);
3364 assert_eq!(floating.double_value(&[5]), -448.0);
3365 for code in [0x7f, 0xff] {
3366 write_shard(
3367 dir.path(),
3368 "weights.safetensors",
3369 &[finite_fixture("p.weight", Dtype::F8_E4M3, vec![code])],
3370 );
3371 let mut rejected = HashMap::new();
3372 let error = ModelFactory::load_safetensors_file(
3373 &path,
3374 &mut rejected,
3375 &Device::Cpu,
3376 DType::Float,
3377 false,
3378 native_finite_policy(dir.path()),
3379 )
3380 .await
3381 .unwrap_err()
3382 .to_string();
3383 assert!(error.contains("nonfinite") && error.contains("p.weight"));
3384 assert!(rejected.is_empty());
3385 }
3386 for code in [0x7c, 0xfc, 0x7d, 0xfd] {
3389 assert!(validate_finite_payload("p.weight", Dtype::F8_E5M2, &[code]).is_err());
3390 }
3391 validate_finite_payload(
3392 "p.weight",
3393 Dtype::F8_E5M2,
3394 &[0x00, 0x80, 1, 0x81, 0x7b, 0xfb],
3395 )
3396 .unwrap();
3397 }
3398
3399 #[tokio::test]
3400 async fn native_finite_selection_and_mtp_exclusion_precede_scan() {
3401 let dir = tempfile::tempdir().unwrap();
3402 finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5"}));
3403 let path = dir.path().join("weights.safetensors");
3404 let poison = || {
3405 finite_fixture(
3406 "model.embed_tokens.weight",
3407 Dtype::F32,
3408 f32::NAN.to_le_bytes().to_vec(),
3409 )
3410 };
3411 let mtp_poison = || {
3412 finite_fixture(
3413 "mtp.norm.weight",
3414 Dtype::F32,
3415 f32::INFINITY.to_le_bytes().to_vec(),
3416 )
3417 };
3418 write_shard(
3419 dir.path(),
3420 "weights.safetensors",
3421 &[
3422 FixtureTensor::f32("model.norm.weight", &[1], 1.0),
3423 poison(),
3424 mtp_poison(),
3425 ],
3426 );
3427 let selected =
3428 BTreeSet::from(["model.norm.weight".to_owned(), "mtp.norm.weight".to_owned()]);
3429 let mut weights = HashMap::new();
3430 ModelFactory::load_safetensors_file_selected(
3433 &path,
3434 &selected,
3435 &mut weights,
3436 &Device::Cpu,
3437 DType::Float,
3438 native_finite_policy(dir.path()),
3439 )
3440 .await
3441 .unwrap();
3442 assert_eq!(weights.len(), 1);
3443 assert!(weights.contains_key("model.norm.weight"));
3444 let mut rejected = HashMap::new();
3445 let selected = BTreeSet::from(["model.embed_tokens.weight".to_owned()]);
3446 assert!(ModelFactory::load_safetensors_file_selected(
3447 &path,
3448 &selected,
3449 &mut rejected,
3450 &Device::Cpu,
3451 DType::Float,
3452 native_finite_policy(dir.path())
3453 )
3454 .await
3455 .unwrap_err()
3456 .to_string()
3457 .contains("model.embed_tokens.weight"));
3458 assert!(rejected.is_empty());
3459 write_shard(
3460 dir.path(),
3461 "weights.safetensors",
3462 &[
3463 FixtureTensor::f32("model.norm.weight", &[1], 1.0),
3464 mtp_poison(),
3465 ],
3466 );
3467 for enabled in [false, true] {
3468 let mut weights = HashMap::new();
3469 let result = ModelFactory::load_safetensors_file(
3470 &path,
3471 &mut weights,
3472 &Device::Cpu,
3473 DType::Float,
3474 enabled,
3475 native_finite_policy(dir.path()),
3476 )
3477 .await;
3478 if enabled {
3479 assert!(result.unwrap_err().to_string().contains("mtp.norm.weight"));
3480 } else {
3481 result.unwrap();
3482 assert_eq!(weights.len(), 1);
3483 }
3484 }
3485 }
3486
3487 #[tokio::test]
3488 async fn native_finite_gate_preserves_other_architecture_and_default_loading() {
3489 let dir = tempfile::tempdir().unwrap();
3490 let path = dir.path().join("weights.safetensors");
3491 write_shard(
3492 dir.path(),
3493 "weights.safetensors",
3494 &[finite_fixture(
3495 "model.norm.weight",
3496 Dtype::F32,
3497 f32::NAN.to_le_bytes().to_vec(),
3498 )],
3499 );
3500 for (config, strict) in [
3501 (None, false),
3502 (Some(serde_json::json!({"model_type":"llama"})), false),
3503 (
3504 Some(
3505 serde_json::json!({"model_type":"llama","text_config":{"model_type":"qwen3_5_text"}}),
3506 ),
3507 false,
3508 ),
3509 (
3510 Some(serde_json::json!({"text_config":{"model_type":"qwen3_5_text"}})),
3511 false,
3512 ),
3513 (Some(serde_json::json!({"model_type":"QWEN3_5"})), true),
3514 (Some(serde_json::json!({"model_type":"qwen3_5_moe"})), true),
3515 (
3516 Some(serde_json::json!({"architectures":["Qwen3_5ForConditionalGeneration"]})),
3517 true,
3518 ),
3519 (
3520 Some(serde_json::json!({"architectures":["LlamaForCausalLM"]})),
3521 false,
3522 ),
3523 (
3524 Some(
3525 serde_json::json!({"model_type":"llama","architectures":["Qwen3_5ForConditionalGeneration"]}),
3526 ),
3527 false,
3528 ),
3529 ] {
3530 if let Some(config) = config {
3531 finite_config(dir.path(), config);
3532 }
3533 assert_eq!(native_finite_policy(dir.path()), strict);
3534 let mut weights = HashMap::new();
3535 let result = ModelFactory::load_safetensors_file(
3536 &path,
3537 &mut weights,
3538 &Device::Cpu,
3539 DType::Float,
3540 false,
3541 native_finite_policy(dir.path()),
3542 )
3543 .await;
3544 if strict {
3545 assert!(result.unwrap_err().to_string().contains("nonfinite"));
3546 } else {
3547 result.unwrap();
3548 assert_eq!(
3549 weights["model.norm.weight"]
3550 .isfinite()
3551 .all()
3552 .int64_value(&[]),
3553 0
3554 );
3555 }
3556 }
3557 }
3558
3559 #[tokio::test]
3560 async fn native_finite_nested_shard_retains_model_root_policy() {
3561 let dir = tempfile::tempdir().unwrap();
3562 finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5"}));
3563 let nested = dir.path().join("weights");
3564 std::fs::create_dir(&nested).unwrap();
3565 finite_config(&nested, serde_json::json!({"model_type":"llama"}));
3567 write_shard(
3568 &nested,
3569 "shard.safetensors",
3570 &[finite_fixture(
3571 "model.norm.weight",
3572 Dtype::BF16,
3573 0x7fc1u16.to_le_bytes().to_vec(),
3574 )],
3575 );
3576 write_index(
3577 dir.path(),
3578 &[("model.norm.weight", "weights/shard.safetensors")],
3579 );
3580 let error = ModelFactory::load_weights(dir.path(), &Device::Cpu, DType::Float, false)
3581 .await
3582 .unwrap_err()
3583 .to_string();
3584 assert!(error.contains("nonfinite") && error.contains("model.norm.weight"));
3585 }
3586
3587 #[test]
3588 fn native_finite_fsops_materialization_seam_rejects_nonfinite() {
3589 let dir = tempfile::tempdir().unwrap();
3590 finite_config(dir.path(), serde_json::json!({"model_type":"qwen3_5"}));
3591 write_shard(
3592 dir.path(),
3593 "weights.safetensors",
3594 &[finite_fixture(
3595 "model.layers.0.linear_attn.dt_bias",
3596 Dtype::F32,
3597 f32::NEG_INFINITY.to_le_bytes().to_vec(),
3598 )],
3599 );
3600 let bytes = std::fs::read(dir.path().join("weights.safetensors")).unwrap();
3601 let mut weights = HashMap::new();
3602 let error = ModelFactory::create_tensors_from_safetensors_selected(
3605 safetensors::SafeTensors::deserialize(&bytes).unwrap(),
3606 None,
3607 &mut weights,
3608 &Device::Cpu,
3609 DType::Float,
3610 false,
3611 native_finite_policy(dir.path()),
3612 )
3613 .unwrap_err()
3614 .to_string();
3615 assert!(error.contains("nonfinite") && error.contains("dt_bias"));
3616 assert!(weights.is_empty());
3617 }
3618
3619 fn loaded_bytes(weights: &HashMap<String, Tensor>) -> usize {
3620 weights
3621 .values()
3622 .map(|tensor| tensor.numel() * tensor.kind().elt_size_in_bytes())
3623 .sum()
3624 }
3625
3626 fn tiny_dense_tensors(explicit_head: bool) -> Vec<FixtureTensor> {
3627 let mut tensors = vec![
3628 FixtureTensor::f32("model.embed_tokens.weight", &[8, 4], 0.01),
3629 FixtureTensor::f32("model.norm.weight", &[4], 1.0),
3630 ];
3631 if explicit_head {
3632 tensors.push(FixtureTensor::f32("lm_head.weight", &[8, 4], 0.02));
3633 }
3634
3635 let prefix = "model.layers.0";
3636 for (suffix, shape, seed) in [
3637 ("self_attn.q_proj.weight", vec![4, 4], 0.03),
3638 ("self_attn.k_proj.weight", vec![4, 4], 0.04),
3639 ("self_attn.v_proj.weight", vec![4, 4], 0.05),
3640 ("self_attn.o_proj.weight", vec![4, 4], 0.06),
3641 ("mlp.gate_proj.weight", vec![8, 4], 0.07),
3642 ("mlp.up_proj.weight", vec![8, 4], 0.08),
3643 ("mlp.down_proj.weight", vec![4, 8], 0.09),
3644 ("input_layernorm.weight", vec![4], 1.0),
3645 ("post_attention_layernorm.weight", vec![4], 1.0),
3646 ] {
3647 tensors.push(FixtureTensor::f32(
3648 format!("{prefix}.{suffix}"),
3649 &shape,
3650 seed,
3651 ));
3652 }
3653 tensors
3654 }
3655
3656 #[tokio::test]
3657 async fn mtp_disabled_factory_skips_head_before_materialization() {
3658 for sharded in [false, true] {
3659 let dir = tempfile::tempdir().unwrap();
3660 let mut tensors = tiny_dense_tensors(true);
3661 tensors.push(FixtureTensor::i64("mtp.unmaterializable"));
3662 write_tiny_dense_checkpoint(dir.path(), &tensors);
3663 if sharded {
3664 let base = tiny_dense_tensors(true);
3665 write_shard(dir.path(), "model-00001-of-00001.safetensors", &base);
3666 write_shard(
3667 dir.path(),
3668 "mtp.safetensors",
3669 &[FixtureTensor::i64("mtp.unmaterializable")],
3670 );
3671 let mut entries: Vec<_> = base
3672 .iter()
3673 .map(|tensor| (tensor.name.as_str(), "model-00001-of-00001.safetensors"))
3674 .collect();
3675 entries.push(("mtp.unmaterializable", "mtp.safetensors"));
3676 write_index(dir.path(), &entries);
3677 }
3678 for enabled in [false, true] {
3679 let configured = crate::config::RuntimeConfig {
3680 use_gpu: false,
3681 speculative_decoding: enabled,
3682 ..Default::default()
3683 };
3684 let config: crate::config::RuntimeConfig =
3685 toml::from_str(&toml::to_string(&configured).unwrap()).unwrap();
3686 let result = ModelFactory::create(
3687 dir.path(),
3688 &Device::Cpu,
3689 DType::Float,
3690 Some(16),
3691 KVQuantType::None,
3692 None,
3693 false,
3694 config.speculative_decoding,
3695 )
3696 .await;
3697 if enabled {
3698 let error = result.err().expect("enabled MTP reaches unsupported dtype");
3699 assert!(error.to_string().contains("mtp.unmaterializable"));
3700 assert!(error.to_string().contains("unsupported dtype"));
3701 } else {
3702 assert!(
3703 result.is_ok(),
3704 "disabled MTP was materialized: {:?}",
3705 result.err()
3706 );
3707 }
3708 }
3709 }
3710 }
3711
3712 #[test]
3713 fn mtp_fsops_materializer_obeys_loading_policy() {
3714 let dir = tempfile::tempdir().unwrap();
3715 write_shard(
3716 dir.path(),
3717 "weights.safetensors",
3718 &[
3719 FixtureTensor::f32("model.norm.weight", &[4], 1.0),
3720 FixtureTensor::f32("mtp.norm.weight", &[4], 1.0),
3721 ],
3722 );
3723 let bytes = std::fs::read(dir.path().join("weights.safetensors")).unwrap();
3724 for enabled in [false, true] {
3725 let mut weights = HashMap::new();
3726 ModelFactory::create_tensors_from_safetensors(
3729 safetensors::SafeTensors::deserialize(&bytes).unwrap(),
3730 &mut weights,
3731 &Device::Cpu,
3732 DType::Float,
3733 enabled,
3734 )
3735 .unwrap();
3736 assert!(weights.contains_key("model.norm.weight"));
3737 assert_eq!(weights.contains_key("mtp.norm.weight"), enabled);
3738 }
3739 }
3740
3741 #[tokio::test]
3742 async fn mtp_unsupported_config_skips_before_materialization() {
3743 for config in [
3744 serde_json::json!({"model_type": "qwen3_5_moe"}),
3745 serde_json::json!({"model_type": "qwen3_5", "num_experts": 16}),
3746 serde_json::json!({"model_type": "qwen3_5", "text_config": {"num_experts": 16}}),
3747 serde_json::json!({"model_type": "qwen3_5", "mtp_num_hidden_layers": 2}),
3748 serde_json::json!({"model_type": "qwen3_5", "text_config": {"mtp_num_hidden_layers": 2}}),
3749 ] {
3750 let dir = tempfile::tempdir().unwrap();
3751 std::fs::write(dir.path().join("config.json"), config.to_string()).unwrap();
3752 write_shard(
3753 dir.path(),
3754 "model.safetensors",
3755 &[
3756 FixtureTensor::f32("model.norm.weight", &[4], 1.0),
3757 FixtureTensor::i64("mtp.unmaterializable"),
3758 ],
3759 );
3760 let shards = vec![dir.path().join("model.safetensors")];
3761 let allowed = ModelFactory::mtp_loading_policy(dir.path(), &shards, true)
3762 .await
3763 .unwrap();
3764 assert!(!allowed, "unsupported config admitted: {config}");
3765 let weights =
3768 ModelFactory::load_weights(dir.path(), &Device::Cpu, DType::Float, allowed)
3769 .await
3770 .unwrap();
3771 assert_eq!(weights.len(), 1);
3772 assert!(weights.contains_key("model.norm.weight"));
3773 }
3774 }
3775
3776 async fn check_mtp_multilayer_factory(sharded: bool) {
3777 for layer in ["0", "1", "7"] {
3778 let dir = tempfile::tempdir().unwrap();
3779 let mut first = tiny_dense_tensors(true);
3780 first.push(FixtureTensor::i64("mtp.unmaterializable"));
3781 write_tiny_dense_checkpoint(dir.path(), &first);
3782 let marker = FixtureTensor::f32(format!("mtp.layers.{layer}.norm.weight"), &[4], 1.0);
3783 if sharded {
3784 write_shard(dir.path(), "model-00001-of-00002.safetensors", &first);
3787 write_shard(dir.path(), "model-00002-of-00002.safetensors", &[marker]);
3788 let mut entries: Vec<_> = first
3789 .iter()
3790 .map(|t| (t.name.as_str(), "model-00001-of-00002.safetensors"))
3791 .collect();
3792 let marker_name = format!("mtp.layers.{layer}.norm.weight");
3793 entries.push((&marker_name, "model-00002-of-00002.safetensors"));
3794 write_index(dir.path(), &entries);
3795 } else {
3796 first.push(marker);
3797 write_shard(dir.path(), "model.safetensors", &first);
3798 }
3799 let result = ModelFactory::create(
3800 dir.path(),
3801 &Device::Cpu,
3802 DType::Float,
3803 Some(16),
3804 KVQuantType::None,
3805 None,
3806 false,
3807 true,
3808 )
3809 .await;
3810 if layer == "0" {
3811 let error = result
3812 .err()
3813 .expect("supported layout must still load enabled head");
3814 assert!(error.to_string().contains("mtp.unmaterializable"));
3815 assert!(error.to_string().contains("unsupported dtype"));
3816 } else {
3817 assert!(
3818 result.is_ok(),
3819 "unsupported head was materialized: {:?}",
3820 result.err()
3821 );
3822 }
3823 }
3824 }
3825
3826 #[tokio::test]
3827 async fn mtp_multilayer_single_file_skips_before_materialization() {
3828 check_mtp_multilayer_factory(false).await;
3829 }
3830
3831 #[tokio::test]
3832 async fn mtp_multilayer_sharded_skips_before_materialization() {
3833 check_mtp_multilayer_factory(true).await;
3834 }
3835
3836 async fn check_mtp_quantized_factory(sharded: bool) {
3837 let dir = tempfile::tempdir().unwrap();
3838 let mut base = tiny_dense_tensors(true);
3839 write_tiny_dense_checkpoint(dir.path(), &base);
3840 if sharded {
3841 write_shard(
3842 dir.path(),
3843 "model-00002-of-00002.safetensors",
3844 &[FixtureTensor::i64("mtp.unmaterializable")],
3845 );
3846 let mut entries: Vec<_> = base
3847 .iter()
3848 .map(|t| (t.name.as_str(), "model-00001-of-00001.safetensors"))
3849 .collect();
3850 entries.push(("mtp.unmaterializable", "model-00002-of-00002.safetensors"));
3851 write_index(dir.path(), &entries);
3852 } else {
3853 base.push(FixtureTensor::i64("mtp.unmaterializable"));
3854 write_shard(dir.path(), "model.safetensors", &base);
3855 }
3856 for requested in [false, true] {
3857 for kv in [
3858 KVQuantType::None,
3859 KVQuantType::Int8,
3860 KVQuantType::Nf4,
3861 KVQuantType::Fp4,
3862 ] {
3863 let result = ModelFactory::create(
3864 dir.path(),
3865 &Device::Cpu,
3866 DType::Float,
3867 Some(16),
3868 kv,
3869 None,
3870 false,
3871 requested,
3872 )
3873 .await;
3874 if requested && kv == KVQuantType::None {
3875 let error = result
3876 .err()
3877 .expect("unquantized enabled head must still load");
3878 assert!(error.to_string().contains("mtp.unmaterializable"));
3879 assert!(error.to_string().contains("unsupported dtype"));
3880 } else {
3881 assert!(
3882 result.is_ok(),
3883 "unusable MTP reached allocation: requested={requested}, kv={kv:?}, error={:?}",
3884 result.err()
3885 );
3886 }
3887 }
3888 }
3889 }
3890
3891 #[tokio::test]
3892 async fn mtp_quantized_single_file_skips_before_materialization() {
3893 check_mtp_quantized_factory(false).await;
3894 }
3895
3896 #[tokio::test]
3897 async fn mtp_quantized_sharded_skips_before_materialization() {
3898 check_mtp_quantized_factory(true).await;
3899 }
3900
3901 #[test]
3902 fn mtp_quantized_fsops_policy_skips_before_materialization() {
3903 let dir = tempfile::tempdir().unwrap();
3904 write_shard(
3905 dir.path(),
3906 "weights.safetensors",
3907 &[
3908 FixtureTensor::f32("model.norm.weight", &[4], 1.0),
3909 FixtureTensor::i64("mtp.unmaterializable"),
3910 ],
3911 );
3912 let bytes = std::fs::read(dir.path().join("weights.safetensors")).unwrap();
3913 for requested in [false, true] {
3914 for kv in [
3915 KVQuantType::None,
3916 KVQuantType::Int8,
3917 KVQuantType::Nf4,
3918 KVQuantType::Fp4,
3919 ] {
3920 let mut weights = HashMap::new();
3921 let result = ModelFactory::create_tensors_from_safetensors(
3924 safetensors::SafeTensors::deserialize(&bytes).unwrap(),
3925 &mut weights,
3926 &Device::Cpu,
3927 DType::Float,
3928 mtp_runtime_supported(requested, kv),
3929 );
3930 if requested && kv == KVQuantType::None {
3931 assert!(result
3932 .unwrap_err()
3933 .to_string()
3934 .contains("mtp.unmaterializable"));
3935 } else {
3936 result.unwrap();
3937 assert_eq!(weights.len(), 1);
3938 assert!(weights.contains_key("model.norm.weight"));
3939 }
3940 }
3941 }
3942 }
3943
3944 #[test]
3945 fn mtp_fsops_header_policy_filters_unsupported_before_materialization() {
3946 let dir = tempfile::tempdir().unwrap();
3947 for marker in [
3948 "mtp.layers.0.norm.weight",
3949 "mtp.layers.1.norm.weight",
3950 "mtp.layers.0.mlp.experts.0.weight",
3951 ] {
3952 write_shard(
3953 dir.path(),
3954 "weights.safetensors",
3955 &[
3956 FixtureTensor::f32("model.norm.weight", &[4], 1.0),
3957 FixtureTensor::f32(marker, &[4], 1.0),
3958 FixtureTensor::i64("mtp.unmaterializable"),
3959 ],
3960 );
3961 let bytes = std::fs::read(dir.path().join("weights.safetensors")).unwrap();
3962 let len = mtp_header_len(&bytes[..8]).unwrap();
3965 let allowed = mtp_header_supported(&bytes[8..8 + len]).unwrap();
3966 let mut weights = HashMap::new();
3967 let result = ModelFactory::create_tensors_from_safetensors(
3968 safetensors::SafeTensors::deserialize(&bytes).unwrap(),
3969 &mut weights,
3970 &Device::Cpu,
3971 DType::Float,
3972 allowed,
3973 );
3974 if marker == "mtp.layers.0.norm.weight" {
3975 assert!(result
3976 .unwrap_err()
3977 .to_string()
3978 .contains("mtp.unmaterializable"));
3979 } else {
3980 result.unwrap();
3981 assert_eq!(weights.len(), 1);
3982 assert!(weights.contains_key("model.norm.weight"));
3983 }
3984 }
3985 assert!(mtp_header_len(&[0; 7]).is_err());
3986 assert!(mtp_header_len(&u64::MAX.to_le_bytes()).is_err());
3987 }
3988
3989 fn write_tiny_dense_checkpoint(dir: &Path, tensors: &[FixtureTensor]) {
3990 const SHARD: &str = "model-00001-of-00001.safetensors";
3991 write_shard(dir, SHARD, tensors);
3992 let entries: Vec<_> = tensors
3993 .iter()
3994 .map(|tensor| (tensor.name.as_str(), SHARD))
3995 .collect();
3996 write_index(dir, &entries);
3997
3998 let config = serde_json::json!({
3999 "architectures": ["LlamaForCausalLM"],
4000 "model_type": "llama",
4001 "hidden_size": 4,
4002 "intermediate_size": 8,
4003 "num_hidden_layers": 1,
4004 "num_attention_heads": 2,
4005 "num_key_value_heads": 2,
4006 "head_dim": 2,
4007 "vocab_size": 8,
4008 "max_position_embeddings": 16,
4009 "rms_norm_eps": 0.00001
4010 });
4011 std::fs::write(
4012 dir.join("config.json"),
4013 serde_json::to_vec(&config).unwrap(),
4014 )
4015 .unwrap();
4016 }
4017
4018 fn assert_pointer_resolution_error(error: anyhow::Error) {
4019 let message = error.to_string();
4020 let normalized = message.to_lowercase();
4021 assert!(
4022 normalized.contains("pointer")
4023 || normalized.contains("lfs")
4024 || normalized.contains("xet"),
4025 "stage path did not report pointer resolution: {message}"
4026 );
4027 assert!(
4028 !normalized.contains("failed to parse required stage shard")
4029 && !normalized.contains("invalid safetensors"),
4030 "stage path reached safetensors parsing before pointer resolution: {message}"
4031 );
4032 }
4033
4034 #[tokio::test]
4035 async fn stage_schema_validates_all_dense_roles() {
4036 let dir = tempfile::tempdir().unwrap();
4037 write_tiny_dense_checkpoint(dir.path(), &tiny_dense_tensors(true));
4038
4039 let config = ModelConfig::load(dir.path(), &HashMap::new()).unwrap();
4040 let plan = ModelFactory::stage_weight_plan(dir.path(), 1, 0..1).unwrap();
4041 let metadata = ModelFactory::stage_tensor_metadata(&plan).await.unwrap();
4042 ModelFactory::validate_stage_tensor_schema(&config, &(0..1), &metadata).unwrap();
4043
4044 for (name, bad_shape) in [
4045 ("model.embed_tokens.weight", vec![7, 4]),
4046 ("model.embed_tokens.weight", vec![8, 5]),
4047 ("lm_head.weight", vec![7, 4]),
4048 ("model.norm.weight", vec![5]),
4049 ("model.layers.0.self_attn.q_proj.weight", vec![3, 4]),
4050 ("model.layers.0.mlp.down_proj.weight", vec![4, 7]),
4051 ("model.layers.0.input_layernorm.weight", vec![5]),
4052 ] {
4053 let mut mismatched = metadata.clone();
4054 mismatched.insert(name.to_owned(), bad_shape);
4055 let error = ModelFactory::validate_stage_tensor_schema(&config, &(0..1), &mismatched)
4056 .unwrap_err();
4057 assert!(
4058 error.to_string().contains(name) && error.to_string().contains("shape mismatch"),
4059 "unexpected schema error for {name}: {error}"
4060 );
4061 }
4062
4063 let mut missing_layer_weight = metadata.clone();
4064 missing_layer_weight.remove("model.layers.0.self_attn.k_proj.weight");
4065 let missing_error =
4066 ModelFactory::validate_stage_tensor_schema(&config, &(0..1), &missing_layer_weight)
4067 .unwrap_err();
4068 assert!(
4069 missing_error
4070 .to_string()
4071 .contains("model.layers.0.self_attn.k_proj.weight"),
4072 "unexpected missing-weight error: {missing_error}"
4073 );
4074
4075 let mut tied = metadata;
4078 tied.remove("lm_head.weight");
4079 ModelFactory::validate_stage_tensor_schema(&config, &(0..1), &tied).unwrap();
4080 }
4081
4082 #[tokio::test]
4083 async fn stage_schema_mismatch_fails_before_tensor_materialization() {
4084 let dir = tempfile::tempdir().unwrap();
4085 let mut tensors = tiny_dense_tensors(false);
4086 *tensors
4087 .iter_mut()
4088 .find(|tensor| tensor.name == "model.embed_tokens.weight")
4089 .unwrap() = FixtureTensor::f32("model.embed_tokens.weight", &[7, 4], 0.01);
4090 *tensors
4091 .iter_mut()
4092 .find(|tensor| tensor.name == "model.layers.0.self_attn.q_proj.weight")
4093 .unwrap() = FixtureTensor::i64_shape("model.layers.0.self_attn.q_proj.weight", &[4, 4]);
4094 write_tiny_dense_checkpoint(dir.path(), &tensors);
4095
4096 let error = ModelFactory::create_stage(
4097 dir.path(),
4098 &Device::Cpu,
4099 DType::Float,
4100 None,
4101 KVQuantType::None,
4102 None,
4103 ModelStageRequest { layer_range: 0..1 },
4104 false,
4105 )
4106 .await
4107 .err()
4108 .expect("config/tensor mismatch must fail");
4109 assert!(
4110 error.to_string().contains("model.embed_tokens.weight")
4111 && error.to_string().contains("shape mismatch"),
4112 "config mismatch must fail before the selected unsupported-dtype sentinel is \
4113 materialized: {error}"
4114 );
4115 assert!(
4116 !error.to_string().contains("unsupported dtype"),
4117 "selected tensors were materialized before schema validation: {error}"
4118 );
4119 }
4120
4121 #[tokio::test]
4122 async fn stage_schema_complete_range_matches_whole_model() {
4123 let dir = tempfile::tempdir().unwrap();
4124 write_tiny_dense_checkpoint(dir.path(), &tiny_dense_tensors(false));
4125
4126 let whole = ModelFactory::create(
4127 dir.path(),
4128 &Device::Cpu,
4129 DType::Float,
4130 None,
4131 KVQuantType::None,
4132 None,
4133 false,
4134 false,
4135 )
4136 .await
4137 .unwrap();
4138 let stage = ModelFactory::create_stage(
4139 dir.path(),
4140 &Device::Cpu,
4141 DType::Float,
4142 None,
4143 KVQuantType::None,
4144 None,
4145 ModelStageRequest { layer_range: 0..1 },
4146 false,
4147 )
4148 .await
4149 .unwrap();
4150
4151 let input = Tensor::from_slice(&[0_i64, 1, 2]).reshape([1, 3]);
4152 let whole_logits = whole.forward(&input, None).unwrap();
4153 let stage_logits = stage.forward(&input, None).unwrap();
4154 assert_eq!(whole_logits.size(), stage_logits.size());
4155 assert!(
4156 whole_logits.to_device(Device::Cpu).allclose(
4157 &stage_logits.to_device(Device::Cpu),
4158 1e-5,
4159 1e-5,
4160 false
4161 ),
4162 "valid complete-range stage forward diverged from ModelFactory::create"
4163 );
4164 }
4165
4166 #[test]
4167 fn stage_weight_plan_includes_top_level_fp8_companion_scales() {
4168 let dir = tempfile::tempdir().unwrap();
4174 let shard = "model-00001-of-00001.safetensors";
4175 write_index(
4176 dir.path(),
4177 &[
4178 ("model.embed_tokens.weight", shard),
4179 ("model.embed_tokens.weight_scale_inv", shard),
4180 ("model.norm.weight", shard),
4181 ("lm_head.weight", shard),
4182 ("lm_head.weight_scale_inv", shard),
4183 ("model.layers.0.self_attn.q_proj.weight", shard),
4184 ("model.layers.0.self_attn.q_proj.weight_scale_inv", shard),
4185 ],
4186 );
4187
4188 let plan = ModelFactory::stage_weight_plan(dir.path(), 1, 0..1).unwrap();
4189 let names: Vec<&str> = plan.values().flatten().map(String::as_str).collect();
4190 for expected in [
4191 "model.embed_tokens.weight",
4192 "model.embed_tokens.weight_scale_inv",
4193 "model.norm.weight",
4194 "lm_head.weight",
4195 "lm_head.weight_scale_inv",
4196 "model.layers.0.self_attn.q_proj.weight",
4197 "model.layers.0.self_attn.q_proj.weight_scale_inv",
4198 ] {
4199 assert!(names.contains(&expected), "stage plan missing {expected}");
4200 }
4201 assert_eq!(names.len(), 7, "unexpected extra tensors in stage plan");
4203 }
4204
4205 #[tokio::test]
4206 async fn stage_pointer_shard_fails_before_mmap_in_preflight_and_selected_load() {
4207 const SHARD: &str = "model-00001-of-00001.safetensors";
4208
4209 let dir = tempfile::tempdir().unwrap();
4210 write_tiny_dense_checkpoint(dir.path(), &tiny_dense_tensors(true));
4211 std::fs::write(
4212 dir.path().join(SHARD),
4213 b"version https://git-lfs.github.com/spec/v1\n\
4214oid sha256:0000000000000000000000000000000000000000000000000000000000000000\n\
4215size 4096\n",
4216 )
4217 .unwrap();
4218
4219 let plan = ModelFactory::stage_weight_plan(dir.path(), 1, 0..1).unwrap();
4220 let preflight_error = ModelFactory::stage_tensor_metadata(&plan)
4221 .await
4222 .unwrap_err();
4223 assert_pointer_resolution_error(preflight_error);
4224
4225 let selected_error = ModelFactory::load_weights_for_stage_plan(
4226 plan,
4227 0..1,
4228 &Device::Cpu,
4229 DType::Float,
4230 native_finite_policy(dir.path()),
4231 )
4232 .await
4233 .unwrap_err();
4234 assert_pointer_resolution_error(selected_error);
4235 }
4236
4237 #[tokio::test]
4238 async fn out_of_range_sentinel_in_needed_shard_is_never_materialized() {
4239 let dir = tempfile::tempdir().unwrap();
4240 let shard = "model-00001-of-00001.safetensors";
4241 write_shard(
4242 dir.path(),
4243 shard,
4244 &[
4245 FixtureTensor::f32("model.embed_tokens.weight", &[2, 2], 0.1),
4246 FixtureTensor::f32("model.layers.0.marker.weight", &[2, 2], 0.2),
4247 FixtureTensor::i64("model.layers.1.sentinel.weight"),
4248 ],
4249 );
4250 write_index(
4251 dir.path(),
4252 &[
4253 ("model.embed_tokens.weight", shard),
4254 ("model.layers.0.marker.weight", shard),
4255 ("model.layers.1.sentinel.weight", shard),
4256 ],
4257 );
4258
4259 let stage =
4260 ModelFactory::load_weights_for_stage(dir.path(), 2, 0..1, &Device::Cpu, DType::Float)
4261 .await
4262 .unwrap();
4263 assert_eq!(stage.len(), 2);
4264 assert!(stage.contains_key("model.embed_tokens.weight"));
4265 assert!(stage.contains_key("model.layers.0.marker.weight"));
4266 assert!(!stage.contains_key("model.layers.1.sentinel.weight"));
4267
4268 let full_error = ModelFactory::load_weights(dir.path(), &Device::Cpu, DType::Float, false)
4269 .await
4270 .unwrap_err();
4271 assert!(
4272 full_error.to_string().contains("unsupported dtype"),
4273 "the sentinel must prove that whole-shard tensor materialization fails: {full_error}"
4274 );
4275 }
4276
4277 #[tokio::test]
4278 async fn stage_create_uses_real_range_and_opens_only_needed_shards() {
4279 const NEEDED: &str = "model-00001-of-00002.safetensors";
4280 const UNNEEDED: &str = "model-00002-of-00002.safetensors";
4281
4282 let dir = tempfile::tempdir().unwrap();
4283 let mut tensors = vec![FixtureTensor::f32(
4284 "model.embed_tokens.weight",
4285 &[8, 4],
4286 0.1,
4287 )];
4288 let prefix = "model.layers.0";
4289 for (suffix, shape) in [
4290 ("self_attn.q_proj.weight", vec![4, 4]),
4291 ("self_attn.k_proj.weight", vec![4, 4]),
4292 ("self_attn.v_proj.weight", vec![4, 4]),
4293 ("self_attn.o_proj.weight", vec![4, 4]),
4294 ("mlp.gate_proj.weight", vec![8, 4]),
4295 ("mlp.up_proj.weight", vec![8, 4]),
4296 ("mlp.down_proj.weight", vec![4, 8]),
4297 ("input_layernorm.weight", vec![4]),
4298 ("post_attention_layernorm.weight", vec![4]),
4299 ] {
4300 tensors.push(FixtureTensor::f32(
4301 format!("{prefix}.{suffix}"),
4302 &shape,
4303 0.2,
4304 ));
4305 }
4306 write_shard(dir.path(), NEEDED, &tensors);
4307
4308 let mut entries: Vec<(String, String)> = tensors
4309 .iter()
4310 .map(|tensor| (tensor.name.clone(), NEEDED.to_owned()))
4311 .collect();
4312 entries.push((
4313 "model.layers.1.self_attn.q_proj.weight".to_owned(),
4314 UNNEEDED.to_owned(),
4315 ));
4316 let borrowed_entries: Vec<_> = entries
4317 .iter()
4318 .map(|(tensor, shard)| (tensor.as_str(), shard.as_str()))
4319 .collect();
4320 write_index(dir.path(), &borrowed_entries);
4321
4322 let config = serde_json::json!({
4323 "architectures": ["LlamaForCausalLM"],
4324 "model_type": "llama",
4325 "hidden_size": 4,
4326 "intermediate_size": 8,
4327 "num_hidden_layers": 2,
4328 "num_attention_heads": 2,
4329 "num_key_value_heads": 2,
4330 "head_dim": 2,
4331 "vocab_size": 8,
4332 "max_position_embeddings": 16,
4333 "rms_norm_eps": 0.00001
4334 });
4335 std::fs::write(
4336 dir.path().join("config.json"),
4337 serde_json::to_vec(&config).unwrap(),
4338 )
4339 .unwrap();
4340
4341 let model = ModelFactory::create_stage(
4344 dir.path(),
4345 &Device::Cpu,
4346 DType::Float,
4347 None,
4348 KVQuantType::None,
4349 None,
4350 ModelStageRequest { layer_range: 0..1 },
4351 false,
4352 )
4353 .await
4354 .unwrap();
4355 let input = Tensor::from_slice(&[0_i64, 1]).reshape([1, 2]);
4356 let embedded = model.embed_tokens(&input).unwrap();
4357 assert_eq!(embedded.size(), [1, 2, 4]);
4358 }
4359
4360 #[tokio::test]
4361 async fn partial_stage_loads_fewer_tensors_and_bytes_than_whole_model() {
4362 let dir = tempfile::tempdir().unwrap();
4363 let first = "model-00001-of-00002.safetensors";
4364 let last = "model-00002-of-00002.safetensors";
4365 write_shard(
4366 dir.path(),
4367 first,
4368 &[
4369 FixtureTensor::f32("model.embed_tokens.weight", &[8, 4], 0.1),
4370 FixtureTensor::f32("model.layers.0.marker.weight", &[4, 4], 0.2),
4371 ],
4372 );
4373 write_shard(
4374 dir.path(),
4375 last,
4376 &[
4377 FixtureTensor::f32("model.layers.1.marker.weight", &[16, 16], 0.3),
4378 FixtureTensor::f32("model.norm.weight", &[4], 0.4),
4379 FixtureTensor::f32("lm_head.weight", &[8, 4], 0.5),
4380 ],
4381 );
4382 write_index(
4383 dir.path(),
4384 &[
4385 ("model.embed_tokens.weight", first),
4386 ("model.layers.0.marker.weight", first),
4387 ("model.layers.1.marker.weight", last),
4388 ("model.norm.weight", last),
4389 ("lm_head.weight", last),
4390 ],
4391 );
4392
4393 let partial =
4394 ModelFactory::load_weights_for_stage(dir.path(), 2, 1..2, &Device::Cpu, DType::Float)
4395 .await
4396 .unwrap();
4397 let whole = ModelFactory::load_weights(dir.path(), &Device::Cpu, DType::Float, false)
4398 .await
4399 .unwrap();
4400
4401 let partial_names: BTreeSet<_> = partial.keys().map(String::as_str).collect();
4402 assert_eq!(
4403 partial_names,
4404 BTreeSet::from([
4405 "lm_head.weight",
4406 "model.layers.1.marker.weight",
4407 "model.norm.weight",
4408 ])
4409 );
4410 assert!(partial.len() < whole.len());
4411 assert!(loaded_bytes(&partial) < loaded_bytes(&whole));
4412 }
4413
4414 #[tokio::test]
4415 async fn stage_mode_fails_closed_without_a_valid_index() {
4416 let dir = tempfile::tempdir().unwrap();
4417 write_shard(
4418 dir.path(),
4419 "model.safetensors",
4420 &[FixtureTensor::f32(
4421 "model.embed_tokens.weight",
4422 &[2, 2],
4423 0.1,
4424 )],
4425 );
4426
4427 let missing =
4428 ModelFactory::load_weights_for_stage(dir.path(), 1, 0..1, &Device::Cpu, DType::Float)
4429 .await
4430 .unwrap_err();
4431 assert!(
4432 missing
4433 .to_string()
4434 .contains("requires authoritative shard index"),
4435 "got: {missing}"
4436 );
4437
4438 std::fs::write(
4439 dir.path().join("model.safetensors.index.json"),
4440 br#"{"weight_map":[]}"#,
4441 )
4442 .unwrap();
4443 let malformed =
4444 ModelFactory::load_weights_for_stage(dir.path(), 1, 0..1, &Device::Cpu, DType::Float)
4445 .await
4446 .unwrap_err();
4447 assert!(
4448 malformed.to_string().contains("valid `weight_map` object"),
4449 "got: {malformed}"
4450 );
4451 }
4452}
4453
4454#[cfg(test)]
4455#[allow(clippy::expect_used, clippy::unwrap_used)]
4456mod fp8_dequant_tests {
4457 use super::*;
4458 use crate::runtime::architectures::llama::LinearProjection;
4459 use safetensors::tensor::{serialize, Dtype, TensorView};
4460
4461 fn f8e4m3_pow2(negative: bool, log2: i32) -> u8 {
4464 let sign = if negative { 0x80u8 } else { 0 };
4465 sign | (((log2 + 7) as u8) << 3)
4466 }
4467
4468 fn fp8_weight(rows: i64, cols: i64) -> Tensor {
4471 let vals: Vec<f32> = (0..(rows * cols))
4472 .map(|i| {
4473 let mag = [0.5f32, 1.0, 2.0][(i % 3) as usize];
4474 if i % 2 == 1 {
4475 -mag
4476 } else {
4477 mag
4478 }
4479 })
4480 .collect();
4481 Tensor::from_slice(&vals)
4482 .reshape([rows, cols])
4483 .to_kind(DType::Float8e4m3fn)
4484 }
4485
4486 fn fp8_weight_with_scale(rows: i64, cols: i64, scale: f32) -> (Tensor, Tensor) {
4488 let weight = fp8_weight(rows, cols);
4489 let scale = Tensor::from_slice(&[scale])
4490 .reshape([1, 1])
4491 .to_kind(DType::BFloat16);
4492 (weight, scale)
4493 }
4494
4495 #[test]
4496 fn dequant_at_load_materializes_bf16_and_drops_scales() {
4497 let (weight, scale) = fp8_weight_with_scale(4, 4, 0.5);
4498 let untouched = Tensor::from_slice(&[1.0f32, 2.0, 3.0, 4.0]).to_kind(DType::BFloat16);
4499 let mut weights = HashMap::from([
4500 ("w.weight".to_owned(), weight),
4501 ("w.weight_scale_inv".to_owned(), scale),
4502 ("norm.weight".to_owned(), untouched),
4503 ]);
4504
4505 let converted = dequantize_fp8_weights_at_load(&mut weights);
4506
4507 assert_eq!(converted, 1);
4508 let w = weights.get("w.weight").expect("weight kept");
4509 assert_eq!(
4510 w.kind(),
4511 DType::BFloat16,
4512 "FP8 weight must be materialized as BF16"
4513 );
4514 assert!(
4515 !weights.contains_key("w.weight_scale_inv"),
4516 "block scale must be dropped after load-time dequant"
4517 );
4518 assert_eq!(
4519 weights.get("norm.weight").expect("norm kept").kind(),
4520 DType::BFloat16,
4521 "non-FP8 tensors untouched"
4522 );
4523
4524 let expected: Vec<f32> = (0..16)
4527 .map(|i| {
4528 let mag = [0.25f32, 0.5, 1.0][(i % 3) as usize];
4529 if i % 2 == 1 {
4530 -mag
4531 } else {
4532 mag
4533 }
4534 })
4535 .collect();
4536 let expected = Tensor::from_slice(&expected).reshape([4, 4]);
4537 let got = w.to_kind(DType::Float);
4538 assert!(
4539 got.allclose(&expected, 1e-6, 1e-6, false),
4540 "load-time dequant values diverged from eager block dequantization"
4541 );
4542 }
4543
4544 #[test]
4545 fn dequant_at_load_forward_matches_lazy_path() {
4546 let mut lazy_weights = HashMap::new();
4550 let (w, s) = fp8_weight_with_scale(4, 8, 0.5);
4551 lazy_weights.insert("p.weight".to_owned(), w);
4552 lazy_weights.insert("p.weight_scale_inv".to_owned(), s);
4553 let lazy_proj = LinearProjection::take(&mut lazy_weights, "p.weight").unwrap();
4554 assert_eq!(lazy_proj.weight.kind(), DType::Float8e4m3fn);
4555 assert!(lazy_proj.scale.is_some());
4556
4557 let mut eager_weights = HashMap::new();
4558 let (w, s) = fp8_weight_with_scale(4, 8, 0.5);
4559 eager_weights.insert("p.weight".to_owned(), w);
4560 eager_weights.insert("p.weight_scale_inv".to_owned(), s);
4561 assert_eq!(dequantize_fp8_weights_at_load(&mut eager_weights), 1);
4562 let eager_proj = LinearProjection::take(&mut eager_weights, "p.weight").unwrap();
4563 assert_eq!(eager_proj.weight.kind(), DType::BFloat16);
4564 assert!(eager_proj.scale.is_none(), "scale must be consumed at load");
4565
4566 let input_vals: Vec<f32> = (0..8).map(|i| 0.125 * (i as f32 + 1.0)).collect();
4567 let input = Tensor::from_slice(&input_vals)
4568 .reshape([1, 8])
4569 .to_kind(DType::BFloat16);
4570 let y_lazy = lazy_proj.apply(&input).to_kind(DType::Float);
4571 let y_eager = eager_proj.apply(&input).to_kind(DType::Float);
4572 assert_eq!(y_lazy.size(), y_eager.size());
4573 assert!(
4574 y_lazy.allclose(&y_eager, 1e-3, 1e-3, false),
4575 "load-time dequant forward diverged from lazy per-matmul dequant"
4576 );
4577 }
4578
4579 #[test]
4580 fn cpu_fp32_loader_preserves_bf16_values_and_channel_fp8_scales() {
4581 let floating = RawTensor::bf16("model.norm.weight", &[2], 0.125);
4582 let packed = RawTensor::f8("p.weight", &[2, 3]);
4583 let scale = RawTensor::bf16("p.weight_scale", &[2, 1], 0.25);
4584 let fixtures = [floating, packed, scale];
4585 let views: Vec<_> = fixtures
4586 .iter()
4587 .map(|t| {
4588 (
4589 t.name.clone(),
4590 TensorView::new(t.dtype, t.shape.clone(), &t.data).unwrap(),
4591 )
4592 })
4593 .collect();
4594 let bytes = serialize(views, &None).unwrap();
4595 let tensors = safetensors::SafeTensors::deserialize(&bytes).unwrap();
4596 let mut weights = HashMap::new();
4597 ModelFactory::create_tensors_from_safetensors(
4598 tensors,
4599 &mut weights,
4600 &Device::Cpu,
4601 DType::Float,
4602 false,
4603 )
4604 .unwrap();
4605 assert_eq!(weights["model.norm.weight"].kind(), DType::Float);
4606 assert_eq!(weights["model.norm.weight"].double_value(&[0]), 0.125);
4607 assert_eq!(weights["p.weight"].kind(), DType::Float8e4m3fn);
4608 assert_eq!(weights["p.weight_scale"].kind(), DType::BFloat16);
4609 assert_eq!(
4610 dequantize_fp8_weights_at_load(&mut weights),
4611 0,
4612 "eager block-only decoder must not erase dynamic activation QDQ"
4613 );
4614 assert!(weights.contains_key("p.weight_scale"));
4615 let projection = LinearProjection::take(&mut weights, "p.weight").unwrap();
4616 let result = projection.apply(&Tensor::ones([1, 3], (DType::Float, Device::Cpu)));
4617 assert_eq!(result.kind(), DType::Float);
4618 assert!(result.isfinite().all().int64_value(&[]) != 0);
4619 }
4620
4621 #[test]
4622 fn dequant_at_load_noop_without_fp8() {
4623 let mut weights = HashMap::from([(
4624 "w.weight".to_owned(),
4625 Tensor::from_slice(&[1.0f32, 2.0]).to_kind(DType::BFloat16),
4626 )]);
4627 assert_eq!(dequantize_fp8_weights_at_load(&mut weights), 0);
4628 assert_eq!(weights["w.weight"].kind(), DType::BFloat16);
4629 }
4630
4631 #[test]
4632 fn dequant_at_load_leaves_nonconforming_scale_untouched() {
4633 let (weight, _) = fp8_weight_with_scale(4, 4, 0.5);
4637 let bad_scale = Tensor::from_slice(&[0.5f32, 0.5, 0.5])
4638 .reshape([3, 1])
4639 .to_kind(DType::BFloat16);
4640 let mut weights = HashMap::from([
4641 ("w.weight".to_owned(), weight),
4642 ("w.weight_scale_inv".to_owned(), bad_scale),
4643 ]);
4644
4645 assert_eq!(dequantize_fp8_weights_at_load(&mut weights), 0);
4646 assert_eq!(
4647 weights["w.weight"].kind(),
4648 DType::Float8e4m3fn,
4649 "non-conforming weight must stay FP8 for the lazy dequant path"
4650 );
4651 assert!(
4652 weights.contains_key("w.weight_scale_inv"),
4653 "non-conforming scale must stay in the map"
4654 );
4655 }
4656
4657 struct RawTensor {
4659 name: String,
4660 dtype: Dtype,
4661 shape: Vec<usize>,
4662 data: Vec<u8>,
4663 }
4664
4665 impl RawTensor {
4666 fn bf16(name: impl Into<String>, shape: &[usize], fill: f32) -> Self {
4667 let elements = shape.iter().product();
4668 let bits = ((fill.to_bits() >> 16) as u16).to_le_bytes();
4669 Self {
4670 name: name.into(),
4671 dtype: Dtype::BF16,
4672 shape: shape.to_vec(),
4673 data: (0..elements).flat_map(|_| bits).collect(),
4674 }
4675 }
4676
4677 fn f8(name: impl Into<String>, shape: &[usize]) -> Self {
4678 let elements = shape.iter().product();
4679 Self {
4680 name: name.into(),
4681 dtype: Dtype::F8_E4M3,
4682 shape: shape.to_vec(),
4683 data: (0..elements)
4684 .map(|i| f8e4m3_pow2(i % 2 == 1, ((i % 3) as i32) - 1))
4685 .collect(),
4686 }
4687 }
4688
4689 fn f8_scale(name: impl Into<String>, scale: f32) -> Self {
4690 Self::bf16(name, &[1, 1], scale)
4691 }
4692 }
4693
4694 fn write_tiny_fp8_checkpoint(dir: &Path) {
4698 const SHARD: &str = "model-00001-of-00001.safetensors";
4699 let mut tensors = vec![
4700 RawTensor::bf16("model.embed_tokens.weight", &[8, 4], 0.02),
4701 RawTensor::bf16("model.norm.weight", &[4], 1.0),
4702 ];
4703 let prefix = "model.layers.0";
4704 for (suffix, shape) in [
4705 ("self_attn.q_proj.weight", vec![4, 4]),
4706 ("self_attn.k_proj.weight", vec![4, 4]),
4707 ("self_attn.v_proj.weight", vec![4, 4]),
4708 ("self_attn.o_proj.weight", vec![4, 4]),
4709 ("mlp.gate_proj.weight", vec![8, 4]),
4710 ("mlp.up_proj.weight", vec![8, 4]),
4711 ("mlp.down_proj.weight", vec![4, 8]),
4712 ] {
4713 tensors.push(RawTensor::f8(format!("{prefix}.{suffix}"), &shape));
4714 tensors.push(RawTensor::f8_scale(
4715 format!("{prefix}.{suffix}_scale_inv"),
4716 0.5,
4717 ));
4718 }
4719 tensors.push(RawTensor::bf16(
4720 format!("{prefix}.input_layernorm.weight"),
4721 &[4],
4722 1.0,
4723 ));
4724 tensors.push(RawTensor::bf16(
4725 format!("{prefix}.post_attention_layernorm.weight"),
4726 &[4],
4727 1.0,
4728 ));
4729
4730 let views: Vec<_> = tensors
4731 .iter()
4732 .map(|tensor| {
4733 (
4734 tensor.name.clone(),
4735 TensorView::new(tensor.dtype, tensor.shape.clone(), &tensor.data).unwrap(),
4736 )
4737 })
4738 .collect();
4739 let bytes = serialize(views, &None).unwrap();
4740 std::fs::write(dir.join(SHARD), bytes).unwrap();
4741
4742 let weight_map: BTreeMap<_, _> = tensors
4743 .iter()
4744 .map(|tensor| (tensor.name.clone(), SHARD.to_owned()))
4745 .collect();
4746 let index = serde_json::json!({ "weight_map": weight_map });
4747 std::fs::write(
4748 dir.join("model.safetensors.index.json"),
4749 serde_json::to_vec(&index).unwrap(),
4750 )
4751 .unwrap();
4752
4753 let config = serde_json::json!({
4754 "architectures": ["LlamaForCausalLM"],
4755 "model_type": "llama",
4756 "hidden_size": 4,
4757 "intermediate_size": 8,
4758 "num_hidden_layers": 1,
4759 "num_attention_heads": 2,
4760 "num_key_value_heads": 2,
4761 "head_dim": 2,
4762 "vocab_size": 8,
4763 "max_position_embeddings": 16,
4764 "rms_norm_eps": 0.00001
4765 });
4766 std::fs::write(
4767 dir.join("config.json"),
4768 serde_json::to_vec(&config).unwrap(),
4769 )
4770 .unwrap();
4771 }
4772
4773 #[tokio::test]
4776 #[ignore = "requires explicit real-model path, receipt path and parent memory admission"]
4777 async fn real_clef_fp8_cpu_backbone_smoke() {
4778 let model_path = PathBuf::from(
4779 std::env::var("HYPRSTREAM_CLEF_FP8_MODEL").expect("explicit model path required"),
4780 );
4781 let receipt_path = PathBuf::from(
4782 std::env::var("HYPRSTREAM_CLEF_BACKBONE_RECEIPT")
4783 .expect("explicit smoke receipt path required"),
4784 );
4785 assert!(model_path.is_dir());
4786 let memory =
4787 std::fs::read_to_string("/proc/meminfo").expect("Linux memory admission required");
4788 let available_kib = memory
4789 .lines()
4790 .find(|line| line.starts_with("MemAvailable:"))
4791 .and_then(|line| line.split_whitespace().nth(1))
4792 .and_then(|value| value.parse::<u64>().ok())
4793 .expect("MemAvailable required");
4794 assert!(
4795 available_kib >= 21 * 1024 * 1024,
4796 "CPU smoke requires21GiB MemAvailable at actual test start; got{}KiB",
4797 available_kib
4798 );
4799 tch::set_num_threads(4);
4800 tch::set_num_interop_threads(1);
4801 let tokenizer =
4802 tokenizers::Tokenizer::from_file(model_path.join("tokenizer.json")).unwrap();
4803 let encode = |text: &str| {
4804 let encoded = tokenizer.encode(text, false).unwrap();
4805 let ids: Vec<i64> = encoded.get_ids().iter().map(|id| *id as i64).collect();
4806 assert!(!ids.is_empty() && ids.len() <= 32);
4807 ids
4808 };
4809 let ids_a = encode("The quick brown fox jumps over a lazy dog.");
4810 let ids_b = encode("A different independent request about ocean waves.");
4811 let a = Tensor::from_slice(&ids_a).reshape([1, ids_a.len() as i64]);
4812 let b = Tensor::from_slice(&ids_b).reshape([1, ids_b.len() as i64]);
4813 let started = std::time::Instant::now();
4814 let model = ModelFactory::create(
4815 &model_path,
4816 &Device::Cpu,
4817 DType::Float,
4818 Some(64),
4819 KVQuantType::None,
4820 None,
4821 false,
4822 false,
4823 )
4824 .await
4825 .unwrap();
4826 let load_ms = started.elapsed().as_millis();
4827 assert!(matches!(
4828 model.architecture(),
4829 crate::runtime::architectures::ModelArchitecture::Qwen3_5
4830 ));
4831 let _no_grad = tch::no_grad_guard();
4832 let forward_start = std::time::Instant::now();
4833 let hidden = model.decision_hidden_states(&a).unwrap();
4834 let first_forward_ms = forward_start.elapsed().as_millis();
4835 assert_eq!(hidden.size(), [ids_a.len() as i64, 4096]);
4836 assert_eq!(hidden.kind(), DType::Float);
4837 assert!(hidden.isfinite().all().int64_value(&[]) != 0);
4838 let rows = model
4839 .decision_output_embedding_rows(&ids_a[..3.min(ids_a.len())])
4840 .unwrap();
4841 assert_eq!(rows.size(), [3.min(ids_a.len()) as i64, 4096]);
4842 assert_eq!(rows.kind(), hidden.kind());
4843 assert!(rows.isfinite().all().int64_value(&[]) != 0);
4844 let other = model.decision_hidden_states(&b).unwrap();
4847 assert!(other.isfinite().all().int64_value(&[]) != 0);
4848 let emb = model.embed_tokens(&b).unwrap();
4849 let _ = model
4850 .forward_layers(&emb, 0..model.num_layers(), 0, None)
4851 .unwrap();
4852 let repeated = model.decision_hidden_states(&a).unwrap();
4853 let repeated_max_abs = (&hidden - &repeated).abs().max().double_value(&[]);
4854 assert!(
4855 hidden.allclose(&repeated, 1e-6, 1e-6, false),
4856 "cross-request state contamination"
4857 );
4858 let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
4859 let receipt = serde_json::json!({
4860 "gate":"native-clef-backbone-cpu-smoke", "result":"PASS",
4861 "model_path":model_path,"candidate_sha":std::env::var("HYPRSTREAM_CANDIDATE_SHA").ok(),
4862 "device":"cpu","compute_dtype":"float32","cpu_threads":4,"context_limit":64,"admission_available_kib":available_kib,
4863 "input_a_tokens":ids_a,"input_b_tokens":ids_b,"load_ms":load_ms,
4864 "first_forward_ms":first_forward_ms,"elapsed_ms":started.elapsed().as_millis(),
4865 "hidden_shape":hidden.size(),"lm_rows_shape":rows.size(),"repeated_max_abs":repeated_max_abs,
4866 "process_memory":status.lines().filter(|line| line.starts_with("VmRSS:") || line.starts_with("VmHWM:")).collect::<Vec<_>>(),
4867 "acceptance_scope":"native-backbone-smoke-only",
4868 "not_proven":["joint-head parity","trusted-reference parity","authenticated serving","GPU eligibility"]
4869 });
4870 std::fs::write(receipt_path, serde_json::to_vec_pretty(&receipt).unwrap()).unwrap();
4871 }
4872
4873 #[tokio::test]
4874 async fn fp8_dequant_load_checkpoint_forward_matches_lazy_path() {
4875 let dir = tempfile::tempdir().unwrap();
4876 write_tiny_fp8_checkpoint(dir.path());
4877
4878 let eager = ModelFactory::create(
4879 dir.path(),
4880 &Device::Cpu,
4881 DType::BFloat16,
4882 None,
4883 KVQuantType::None,
4884 None,
4885 true,
4886 false,
4887 )
4888 .await
4889 .unwrap();
4890 let lazy = ModelFactory::create(
4891 dir.path(),
4892 &Device::Cpu,
4893 DType::BFloat16,
4894 None,
4895 KVQuantType::None,
4896 None,
4897 false,
4898 false,
4899 )
4900 .await
4901 .unwrap();
4902
4903 let input = Tensor::from_slice(&[0_i64, 1, 2]).reshape([1, 3]);
4904 let eager_logits = eager.forward(&input, None).unwrap().to_kind(DType::Float);
4905 let lazy_logits = lazy.forward(&input, None).unwrap().to_kind(DType::Float);
4906 assert_eq!(eager_logits.size(), lazy_logits.size());
4907 assert!(
4908 eager_logits.allclose(&lazy_logits, 1e-3, 1e-3, false),
4909 "checkpoint loaded with fp8_dequant_load diverged from lazy-dequant load"
4910 );
4911 }
4912}