Skip to main content

hyprstream_inference/runtime/
model_config.rs

1//! Unified model configuration management
2//!
3//! This module provides a single source of truth for model configurations,
4//! resolving the current chaos of multiple override points.
5
6use anyhow::{bail, Context, Result};
7use regex::Regex;
8use serde::{Deserialize, Serialize};
9use std::collections::HashMap;
10use std::path::Path;
11use std::sync::LazyLock;
12use tch::Tensor;
13use tracing::info;
14
15/// Regex for extracting numeric version from model type strings
16/// SAFETY: These patterns are compile-time constants that are guaranteed valid
17static VERSION_REGEX: LazyLock<Option<Regex>> = LazyLock::new(|| Regex::new(r"(\d+)").ok());
18
19/// Regex for extracting layer indices from weight key names
20static LAYER_REGEX: LazyLock<Option<Regex>> = LazyLock::new(|| Regex::new(r"layers\.(\d+)").ok());
21
22/// Regex for extracting **decoder** layer indices from a shard manifest's
23/// `weight_map` keys (#314 follow-up to #315).
24///
25/// Anchored to the language-model decoder namespace — `model.layers.<i>.` or
26/// `language_model.model.layers.<i>.` — so a multimodal checkpoint's *vision
27/// tower* (e.g. `vision_model.encoder.layers.<j>.` /
28/// `visual.blocks.<j>.`) cannot inflate the decoder layer count and corrupt a
29/// pipeline split. The trailing `.` ensures we match a true layer-index segment
30/// (`model.layers.12.` ) and not an unrelated key that merely contains the
31/// substring. Correct for llama/qwen3_5, whose decoder weights are
32/// `model.layers.<i>.…`.
33static DECODER_LAYER_REGEX: LazyLock<Option<Regex>> =
34    LazyLock::new(|| Regex::new(r"(?:^|\.)(?:language_model\.)?model\.layers\.(\d+)\.").ok());
35
36/// Unified model configuration that combines all sources
37#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct ModelConfig {
39    // Architecture identification
40    pub architecture: ModelArchitecture,
41    pub model_type: String,
42    pub version: u32,
43
44    // Core transformer parameters
45    pub hidden_size: usize,
46    pub num_hidden_layers: usize,
47    pub num_attention_heads: usize,
48    pub num_key_value_heads: usize,
49    pub head_dim: usize,
50    pub intermediate_size: usize,
51
52    // Vocabulary and embeddings
53    pub vocab_size: usize,
54    pub max_position_embeddings: usize,
55    pub rope_theta: f32,
56    pub rope_scaling: Option<RopeScaling>,
57
58    // Local-RoPE / sliding-window attention (Gemma3 local-attention layers).
59    // A non-`None` sliding window restricts which tokens a local layer attends to;
60    // `rope_local_base_freq` is the RoPE theta used in those local layers. Both
61    // change local-layer K/V values, so they are authoritative for KV reuse
62    // (#1277). `None` for architectures without local attention.
63    pub sliding_window: Option<usize>,
64    pub rope_local_base_freq: Option<f32>,
65
66    // Normalization
67    pub rms_norm_eps: f32,
68    pub layer_norm_eps: Option<f32>,
69
70    // Activation
71    pub hidden_activation: String,
72
73    // Special configurations
74    pub use_qk_norm: bool,
75    pub scale_embeddings: bool,
76    pub query_pre_attn_scalar: Option<f32>,
77
78    // Precision and optimization
79    pub dtype: String,
80    pub use_flash_attention: bool,
81    pub use_kv_cache: bool,
82
83    // Qwen3.5 hybrid SSM/attention fields
84    pub partial_rotary_factor: Option<f32>,
85    pub layer_types: Vec<String>,
86    pub linear_conv_kernel_dim: usize,
87    pub linear_key_head_dim: usize,
88    pub linear_value_head_dim: usize,
89    pub linear_num_key_heads: usize,
90    pub linear_num_value_heads: usize,
91
92    // Qwen3.5 MoE fields
93    pub is_moe: bool,
94    pub num_experts: usize,
95    pub num_experts_per_tok: usize,
96    pub moe_intermediate_size: usize,
97    pub shared_expert_intermediate_size: usize,
98
99    // Vision fields (Qwen3.5 multimodal)
100    pub has_vision: bool,
101    pub vision_out_hidden_size: usize,
102}
103
104#[derive(Debug, Clone, Serialize, Deserialize)]
105pub struct RopeScaling {
106    pub rope_type: String,
107    pub factor: f32,
108}
109
110#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
111pub enum ModelArchitecture {
112    Llama,
113    Qwen,
114    Gemma,
115    Mistral,
116    Janus,
117    /// Qwen3.5 hybrid GatedDeltaNet/full-attention model (dense and MoE variants)
118    Qwen3_5,
119    Unknown(String),
120}
121
122/// Configuration source for different model architectures
123#[derive(Debug, Clone)]
124pub enum ConfigSource {
125    /// Flat config (Llama, Qwen, Gemma, etc.)
126    Flat(Box<ModelConfig>),
127
128    /// Nested config (Janus, future multimodal models)
129    Nested(Box<NestedModelConfig>),
130}
131
132/// Nested configuration for multimodal models like Janus
133#[derive(Debug, Clone, Serialize, Deserialize)]
134pub struct NestedModelConfig {
135    /// Top-level architecture
136    pub architecture: ModelArchitecture,
137
138    /// Component configs (raw JSON for flexibility)
139    pub language_config: Option<serde_json::Value>,
140    pub vision_config: Option<serde_json::Value>,
141    pub aligner_config: Option<serde_json::Value>,
142
143    // Optional generation components
144    pub gen_aligner_config: Option<serde_json::Value>,
145    pub gen_vision_config: Option<serde_json::Value>,
146    pub gen_head_config: Option<serde_json::Value>,
147
148    /// Top-level metadata (not in sub-configs)
149    pub num_hidden_layers: Option<usize>,
150    pub image_token_id: Option<u32>,
151    pub pad_token_id: Option<u32>,
152    pub begin_image_token_id: Option<u32>,
153    pub torch_dtype: Option<String>,
154}
155
156/// Opt-in escape hatch: when set to a truthy value (`1`/`true`), a missing
157/// `config.json` falls back to the fragile weight-name-scan heuristic instead of
158/// hard-failing. The model's own `config.json` is authoritative; this exists only
159/// for legacy/dev checkpoints that ship raw weights with no metadata.
160const ALLOW_WEIGHT_DETECT_ENV: &str = "HYPRSTREAM_ALLOW_WEIGHT_DETECT";
161
162/// Returns true when an env var is set to a recognized truthy value.
163fn env_flag(name: &str) -> bool {
164    std::env::var(name)
165        .map(|v| {
166            matches!(
167                v.trim().to_lowercase().as_str(),
168                "1" | "true" | "yes" | "on"
169            )
170        })
171        .unwrap_or(false)
172}
173
174/// Extract a mandatory unsigned integer field from a config JSON object, failing
175/// fast with a clear error (including the config path) when it is absent or not a
176/// non-negative integer. Used for the architectural dimensions that must never be
177/// silently defaulted (#315).
178fn require_u64(source: &serde_json::Value, field: &str, config_path: &Path) -> Result<u64> {
179    source[field].as_u64().with_context(|| {
180        format!(
181            "config.json ({path}) is missing required field `{field}` (or it is not a non-negative integer)",
182            path = config_path.display()
183        )
184    })
185}
186
187impl ModelConfig {
188    /// Load configuration, treating the model's own `config.json` as authoritative.
189    ///
190    /// No-fragile-fallbacks (#315): `config.json` is **required** by default. If it
191    /// is absent, loading is a hard error — a silently-guessed config can corrupt a
192    /// pipeline split (e.g. a wrong layer count). The legacy weight-name-scan
193    /// heuristic remains available only behind the explicit
194    /// `HYPRSTREAM_ALLOW_WEIGHT_DETECT=1` opt-in.
195    ///
196    /// When `model.safetensors.index.json` is present, `num_hidden_layers` is
197    /// cross-checked against the shard manifest's `weight_map`; a mismatch is a
198    /// hard error rather than a silent corruption.
199    pub fn load(model_path: &Path, weights: &HashMap<String, Tensor>) -> Result<Self> {
200        // Step 1: config.json is authoritative and required.
201        let config_path = model_path.join("config.json");
202        let mut config = if config_path.exists() {
203            info!("Loading model configuration");
204            Self::from_json_file(&config_path)?
205        } else if env_flag(ALLOW_WEIGHT_DETECT_ENV) {
206            info!(
207                "⚠️ No config.json in {}; {ALLOW_WEIGHT_DETECT_ENV} is set, detecting from weights (fragile)",
208                model_path.display()
209            );
210            Self::detect_from_weights(weights)?
211        } else {
212            bail!(
213                "config.json is required but was not found in {}. \
214                 A model's config.json is authoritative for its architecture; refusing to guess. \
215                 Set {ALLOW_WEIGHT_DETECT_ENV}=1 to opt into the legacy weight-name heuristic.",
216                model_path.display()
217            );
218        };
219
220        // Step 1b: cross-check num_hidden_layers against the shard manifest.
221        config.validate_against_index(model_path)?;
222
223        // Step 2: Validate against weights
224        config.validate_with_weights(weights)?;
225
226        // Step 3: Log final configuration
227        info!("✅ Final model configuration:");
228        info!("   Architecture: {:?}", config.architecture);
229        info!("   Hidden size: {}", config.hidden_size);
230        info!("   Layers: {}", config.num_hidden_layers);
231        info!("   Attention heads: {}", config.num_attention_heads);
232        info!("   KV heads: {}", config.num_key_value_heads);
233        info!("   Vocab size: {}", config.vocab_size);
234        info!("   RoPE theta: {}", config.rope_theta);
235        info!("   Max position: {}", config.max_position_embeddings);
236        info!("   RMSNorm eps: {}", config.rms_norm_eps);
237
238        Ok(config)
239    }
240
241    /// Load from config.json
242    fn from_json_file(path: &Path) -> Result<Self> {
243        let content = std::fs::read_to_string(path)?;
244        let json: serde_json::Value = serde_json::from_str(&content)?;
245
246        // Detect architecture from model_type or architectures field
247        let architecture = Self::detect_architecture_from_json(&json);
248
249        // For Janus models, extract language_config and use that as the source.
250        // For Qwen3.5 models, extract text_config.
251        let nested_json_opt = if architecture == ModelArchitecture::Janus {
252            info!("Detected Janus multimodal model - loading from language_config");
253
254            let lang_config = json["language_config"].as_object().ok_or_else(|| {
255                anyhow::anyhow!("Janus config missing required 'language_config'")
256            })?;
257
258            let lang_json = serde_json::Value::Object(lang_config.clone());
259            info!(
260                "Language config hidden_size: {}",
261                lang_json["hidden_size"].as_u64().unwrap_or(0)
262            );
263            Some(lang_json)
264        } else if architecture == ModelArchitecture::Qwen3_5 {
265            info!("Detected Qwen3.5 model - loading from text_config");
266            if let Some(text_cfg) = json["text_config"].as_object() {
267                let text_json = serde_json::Value::Object(text_cfg.clone());
268                info!(
269                    "text_config hidden_size: {}",
270                    text_json["hidden_size"].as_u64().unwrap_or(0)
271                );
272                Some(text_json)
273            } else {
274                None // flat config (text-only checkpoint)
275            }
276        } else {
277            None
278        };
279
280        // Choose config source based on architecture
281        let config_source = nested_json_opt.as_ref().unwrap_or(&json);
282
283        // No-fragile-fallbacks (#315): the core architectural dimensions are
284        // *mandatory* when config.json is present. A wrong layer count or hidden
285        // size silently corrupts inference (and a pipeline split), so a
286        // missing/unparseable value is a hard error rather than a magic default.
287        // `num_hidden_layers` falls back to the top-level config for nested
288        // (Janus/Qwen3.5) layouts that carry it outside the sub-config.
289        let num_hidden_layers = config_source["num_hidden_layers"]
290            .as_u64()
291            .or_else(|| json["num_hidden_layers"].as_u64())
292            .map(|v| v as usize)
293            .with_context(|| {
294                format!(
295                    "config.json ({path}) is missing required field `num_hidden_layers`",
296                    path = path.display()
297                )
298            })?;
299        let hidden_size = require_u64(config_source, "hidden_size", path)? as usize;
300        let num_attention_heads = require_u64(config_source, "num_attention_heads", path)? as usize;
301        if num_attention_heads == 0 || num_hidden_layers == 0 || hidden_size == 0 {
302            bail!(
303                "config.json ({}) has invalid zero dimension(s): \
304                 hidden_size={hidden_size}, num_hidden_layers={num_hidden_layers}, \
305                 num_attention_heads={num_attention_heads}",
306                path.display()
307            );
308        }
309
310        // Extract all configuration values from the appropriate source
311        let config = Self {
312            architecture: architecture.clone(),
313            model_type: json["model_type"].as_str().unwrap_or("unknown").to_owned(),
314            version: Self::detect_version(&json, &architecture),
315
316            hidden_size,
317            num_hidden_layers,
318            num_attention_heads,
319            // KV heads legitimately default to attention heads (MHA, not GQA).
320            num_key_value_heads: config_source["num_key_value_heads"]
321                .as_u64()
322                .map(|v| v as usize)
323                .unwrap_or(num_attention_heads),
324            // head_dim is derivable from hidden_size / heads when omitted.
325            head_dim: config_source["head_dim"]
326                .as_u64()
327                .map(|v| v as usize)
328                .unwrap_or(hidden_size / num_attention_heads),
329            intermediate_size: config_source["intermediate_size"].as_u64().unwrap_or(11008)
330                as usize,
331
332            vocab_size: config_source["vocab_size"].as_u64().unwrap_or(32000) as usize,
333            max_position_embeddings: config_source["max_position_embeddings"]
334                .as_u64()
335                .unwrap_or(4096) as usize,
336            rope_theta: config_source["rope_parameters"]["rope_theta"]
337                .as_f64()
338                .or_else(|| config_source["rope_theta"].as_f64())
339                .unwrap_or(10_000.0) as f32,
340            rope_scaling: Self::parse_rope_scaling(config_source),
341            // Local-RoPE / sliding-window (Gemma3); absent (None) for non-local
342            // architectures — present only when config.json carries them.
343            sliding_window: config_source["sliding_window"].as_u64().map(|v| v as usize),
344            rope_local_base_freq: config_source["rope_local_base_freq"]
345                .as_f64()
346                .map(|v| v as f32),
347
348            rms_norm_eps: config_source["rms_norm_eps"].as_f64().unwrap_or(1e-5) as f32,
349            layer_norm_eps: config_source["layer_norm_eps"].as_f64().map(|v| v as f32),
350
351            hidden_activation: config_source["hidden_activation"]
352                .as_str()
353                .unwrap_or("silu")
354                .to_owned(),
355
356            use_qk_norm: config_source["use_qk_norm"].as_bool().unwrap_or(false),
357            scale_embeddings: config_source["scale_embeddings"].as_bool().unwrap_or(false),
358            query_pre_attn_scalar: config_source["query_pre_attn_scalar"]
359                .as_f64()
360                .map(|v| v as f32),
361
362            dtype: "bfloat16".to_owned(),
363            use_flash_attention: true,
364            use_kv_cache: true,
365
366            // Qwen3.5 hybrid SSM/attention fields
367            partial_rotary_factor: if architecture == ModelArchitecture::Qwen3_5 {
368                Some(
369                    config_source["rope_parameters"]["partial_rotary_factor"]
370                        .as_f64()
371                        .or_else(|| config_source["partial_rotary_factor"].as_f64())
372                        .unwrap_or(0.25) as f32,
373                )
374            } else {
375                None
376            },
377            layer_types: if architecture == ModelArchitecture::Qwen3_5 {
378                let num_layers = num_hidden_layers;
379                config_source["layer_types"]
380                    .as_array()
381                    .map(|a| {
382                        a.iter()
383                            .filter_map(|v| v.as_str().map(str::to_owned))
384                            .collect()
385                    })
386                    .unwrap_or_else(|| {
387                        let interval = config_source["full_attention_interval"]
388                            .as_u64()
389                            .unwrap_or(4) as usize;
390                        (0..num_layers)
391                            .map(|i| {
392                                if (i + 1) % interval == 0 {
393                                    "full_attention".to_owned()
394                                } else {
395                                    "linear_attention".to_owned()
396                                }
397                            })
398                            .collect()
399                    })
400            } else {
401                vec![]
402            },
403            linear_conv_kernel_dim: config_source["linear_conv_kernel_dim"]
404                .as_u64()
405                .unwrap_or(0) as usize,
406            linear_key_head_dim: config_source["linear_key_head_dim"].as_u64().unwrap_or(0)
407                as usize,
408            linear_value_head_dim: config_source["linear_value_head_dim"].as_u64().unwrap_or(0)
409                as usize,
410            linear_num_key_heads: config_source["linear_num_key_heads"].as_u64().unwrap_or(0)
411                as usize,
412            linear_num_value_heads: config_source["linear_num_value_heads"]
413                .as_u64()
414                .unwrap_or(0) as usize,
415
416            // Qwen3.5 MoE fields
417            is_moe: json["model_type"]
418                .as_str()
419                .map(|s| s.contains("moe"))
420                .unwrap_or(false),
421            num_experts: config_source["num_experts"].as_u64().unwrap_or(0) as usize,
422            num_experts_per_tok: config_source["num_experts_per_tok"].as_u64().unwrap_or(0)
423                as usize,
424            moe_intermediate_size: config_source["moe_intermediate_size"].as_u64().unwrap_or(0)
425                as usize,
426            shared_expert_intermediate_size: config_source["shared_expert_intermediate_size"]
427                .as_u64()
428                .unwrap_or(0) as usize,
429
430            // Vision fields
431            has_vision: json["vision_config"].is_object(),
432            vision_out_hidden_size: json["vision_config"]["out_hidden_size"]
433                .as_u64()
434                .unwrap_or(3584) as usize,
435        };
436
437        Ok(config)
438    }
439
440    /// Detect configuration from weights only (fallback)
441    #[allow(clippy::field_reassign_with_default)]
442    fn detect_from_weights(weights: &HashMap<String, Tensor>) -> Result<Self> {
443        // Start with defaults
444        let mut config = Self::default();
445
446        // Detect architecture from weight names
447        config.architecture = Self::detect_architecture_from_weights(weights);
448
449        // Extract dimensions from embeddings
450        if let Some(embed) = weights
451            .get("model.embed_tokens.weight")
452            .or_else(|| weights.get("embed_tokens.weight"))
453        {
454            let shape = embed.size();
455            config.vocab_size = shape[0] as usize;
456            config.hidden_size = shape[1] as usize;
457        }
458
459        // Count layers
460        config.num_hidden_layers = Self::count_layers(weights);
461
462        // Detect attention configuration from projections
463        Self::detect_attention_config(weights, &mut config)?;
464
465        // Apply architecture-specific defaults
466        config.apply_architecture_defaults();
467
468        Ok(config)
469    }
470
471    /// Detect the model architecture from a model directory's `config.json`.
472    ///
473    /// Returns `ModelArchitecture::Unknown` if the file is missing or unparseable.
474    /// This is a lightweight operation (reads only config.json, not weights).
475    pub fn detect_architecture(model_path: &Path) -> ModelArchitecture {
476        let config_path = model_path.join("config.json");
477        let content = match std::fs::read_to_string(&config_path) {
478            Ok(c) => c,
479            Err(_) => return ModelArchitecture::Unknown("unknown".to_owned()),
480        };
481        let json: serde_json::Value = match serde_json::from_str(&content) {
482            Ok(v) => v,
483            Err(_) => return ModelArchitecture::Unknown("unknown".to_owned()),
484        };
485        Self::detect_architecture_from_json(&json)
486    }
487
488    fn detect_architecture_from_json(json: &serde_json::Value) -> ModelArchitecture {
489        // Check model_type field
490        if let Some(model_type) = json["model_type"].as_str() {
491            return match model_type.to_lowercase().as_str() {
492                "janus" => ModelArchitecture::Janus,
493                "llama" => ModelArchitecture::Llama,
494                "qwen3_5" | "qwen3_5_text" | "qwen3_5_moe" => ModelArchitecture::Qwen3_5,
495                "qwen" | "qwen2" | "qwen3" => ModelArchitecture::Qwen,
496                "gemma" => ModelArchitecture::Gemma,
497                "mistral" => ModelArchitecture::Mistral,
498                _ => ModelArchitecture::Unknown(model_type.to_owned()),
499            };
500        }
501
502        // Check architectures field
503        if let Some(architectures) = json["architectures"].as_array() {
504            if let Some(first) = architectures.first() {
505                if let Some(arch_str) = first.as_str() {
506                    return match arch_str.to_lowercase().as_str() {
507                        s if s.contains("janus") => ModelArchitecture::Janus,
508                        s if s.contains("llama") => ModelArchitecture::Llama,
509                        s if s.contains("qwen3_5") || s.contains("Qwen3_5") => {
510                            ModelArchitecture::Qwen3_5
511                        }
512                        s if s.contains("qwen") => ModelArchitecture::Qwen,
513                        s if s.contains("gemma") => ModelArchitecture::Gemma,
514                        s if s.contains("mistral") => ModelArchitecture::Mistral,
515                        _ => ModelArchitecture::Unknown(arch_str.to_owned()),
516                    };
517                }
518            }
519        }
520
521        ModelArchitecture::Unknown("unknown".to_owned())
522    }
523
524    fn detect_architecture_from_weights(weights: &HashMap<String, Tensor>) -> ModelArchitecture {
525        // Check for architecture-specific weight patterns
526
527        // Check for Janus multimodal components first
528        let has_vision_model = weights
529            .keys()
530            .any(|k| k.starts_with("vision_model.") || k.starts_with("vision_encoder."));
531        let has_aligner = weights
532            .keys()
533            .any(|k| k.starts_with("aligner.") || k.starts_with("vision_aligner."));
534        let has_language_model = weights.keys().any(|k| k.starts_with("language_model."));
535
536        if has_vision_model || has_aligner || has_language_model {
537            info!("Detected Janus multimodal model from weight patterns");
538            info!("  has_vision_model: {}", has_vision_model);
539            info!("  has_aligner: {}", has_aligner);
540            info!("  has_language_model: {}", has_language_model);
541            return ModelArchitecture::Janus;
542        }
543
544        // Qwen3.5 has linear_attn layers (GatedDeltaNet)
545        if weights.keys().any(|k| k.contains("linear_attn.")) {
546            return ModelArchitecture::Qwen3_5;
547        }
548
549        for key in weights.keys() {
550            if key.contains("q_norm") || key.contains("k_norm") {
551                return ModelArchitecture::Qwen;
552            }
553            if key.contains("gated_gemm") {
554                return ModelArchitecture::Gemma;
555            }
556        }
557
558        // Default to Llama as most common
559        ModelArchitecture::Llama
560    }
561
562    fn detect_version(json: &serde_json::Value, architecture: &ModelArchitecture) -> u32 {
563        // Try to extract version from model_type field (e.g., "qwen2", "llama3")
564        if let Some(model_type) = json["model_type"].as_str() {
565            let lower = model_type.to_lowercase();
566
567            // Look for explicit version numbers in model_type
568            if lower.contains("qwen3") || lower.contains("qwen-3") {
569                return 3;
570            } else if lower.contains("qwen2") || lower.contains("qwen-2") {
571                return 2;
572            } else if lower.contains("llama3") || lower.contains("llama-3") {
573                return 3;
574            } else if lower.contains("llama2") || lower.contains("llama-2") {
575                return 2;
576            }
577
578            // Try generic number extraction as fallback
579            if let Some(ref regex) = *VERSION_REGEX {
580                if let Some(captures) = regex.captures(model_type) {
581                    if let Ok(version) = captures[1].parse::<u32>() {
582                        return version;
583                    }
584                }
585            }
586        }
587
588        // Try architectures field (e.g., ["Qwen3ForCausalLM"])
589        if let Some(architectures) = json["architectures"].as_array() {
590            if let Some(first_arch) = architectures.first().and_then(|v| v.as_str()) {
591                let lower = first_arch.to_lowercase();
592
593                if lower.contains("qwen3") {
594                    return 3;
595                } else if lower.contains("qwen2") {
596                    return 2;
597                } else if lower.contains("llama3") {
598                    return 3;
599                } else if lower.contains("llama2") {
600                    return 2;
601                }
602            }
603        }
604
605        // Architecture-specific defaults (conservative)
606        // Note: DO NOT use rope_theta or other config values to detect version
607        // as these are not reliable indicators
608        match architecture {
609            // Default to version 2 for Qwen/Llama if unknown
610            ModelArchitecture::Qwen | ModelArchitecture::Llama => 2,
611            _ => 1,
612        }
613    }
614
615    fn parse_rope_scaling(json: &serde_json::Value) -> Option<RopeScaling> {
616        json["rope_scaling"].as_object().map(|obj| RopeScaling {
617            rope_type: obj["type"].as_str().unwrap_or("linear").to_owned(),
618            factor: obj["factor"].as_f64().unwrap_or(1.0) as f32,
619        })
620    }
621
622    fn count_layers(weights: &HashMap<String, Tensor>) -> usize {
623        let mut max_layer = 0;
624        if let Some(ref regex) = *LAYER_REGEX {
625            for key in weights.keys() {
626                if let Some(captures) = regex.captures(key) {
627                    if let Ok(layer_idx) = captures[1].parse::<usize>() {
628                        max_layer = max_layer.max(layer_idx + 1);
629                    }
630                }
631            }
632        }
633        max_layer
634    }
635
636    fn detect_attention_config(weights: &HashMap<String, Tensor>, config: &mut Self) -> Result<()> {
637        // Find Q and K projection shapes
638        let q_proj = weights
639            .keys()
640            .find(|k| k.contains("q_proj.weight"))
641            .and_then(|k| weights.get(k));
642        let k_proj = weights
643            .keys()
644            .find(|k| k.contains("k_proj.weight"))
645            .and_then(|k| weights.get(k));
646
647        if let (Some(q), Some(k)) = (q_proj, k_proj) {
648            let q_out = q.size()[0] as usize;
649            let k_out = k.size()[0] as usize;
650
651            // Try different head dimensions
652            for head_dim in &[256, 128, 64, 32] {
653                if q_out.is_multiple_of(*head_dim) && k_out.is_multiple_of(*head_dim) {
654                    config.num_attention_heads = q_out / head_dim;
655                    config.num_key_value_heads = k_out / head_dim;
656                    config.head_dim = *head_dim;
657                    break;
658                }
659            }
660        }
661
662        Ok(())
663    }
664
665    fn apply_architecture_defaults(&mut self) {
666        match &self.architecture {
667            ModelArchitecture::Qwen => {
668                // Qwen3 specific defaults
669                if self.vocab_size == 151936 {
670                    self.rope_theta = 5_000_000.0; // Qwen3-4B uses 5M
671                    self.version = 3;
672                }
673            }
674            ModelArchitecture::Qwen3_5 => {
675                // Qwen3.5 defaults — nested config parsing handles most fields
676                if self.rope_theta == 10_000.0 {
677                    self.rope_theta = 10_000_000.0; // Qwen3.5 uses 10M
678                }
679            }
680            ModelArchitecture::Gemma if self.vocab_size == 262144 => {
681                self.rope_theta = 1_000_000.0;
682                self.use_qk_norm = true;
683                self.scale_embeddings = true;
684                self.query_pre_attn_scalar = Some(256.0);
685            }
686            _ => {}
687        }
688    }
689
690    /// Cross-check `num_hidden_layers` against the shard manifest's `weight_map`.
691    ///
692    /// When `model.safetensors.index.json` is present, the number of distinct
693    /// `model.layers.<i>.` (or `*.layers.<i>.` for nested layouts) indices listed
694    /// in its `weight_map` must equal the config's `num_hidden_layers`. A mismatch
695    /// means the config and the actual checkpoint disagree about depth, which would
696    /// silently truncate or over-allocate a pipeline split — so it is a hard error.
697    ///
698    /// If the index file is absent this is a no-op: per #315 the index is required
699    /// only where the *loader* consumes it (multi-shard models, validated in
700    /// `model_factory`); a single-file model legitimately has no manifest.
701    fn validate_against_index(&self, model_path: &Path) -> Result<()> {
702        let index_path = model_path.join("model.safetensors.index.json");
703        if !index_path.exists() {
704            return Ok(());
705        }
706
707        let content = std::fs::read_to_string(&index_path)
708            .with_context(|| format!("failed to read shard manifest {}", index_path.display()))?;
709        let index: serde_json::Value = serde_json::from_str(&content)
710            .with_context(|| format!("failed to parse shard manifest {}", index_path.display()))?;
711        let weight_map = index["weight_map"].as_object().ok_or_else(|| {
712            anyhow::anyhow!(
713                "shard manifest {} is missing required `weight_map` object",
714                index_path.display()
715            )
716        })?;
717
718        // Count only DECODER layers (`model.layers.<i>.`), not a multimodal
719        // vision tower's own `.layers.N` — those would inflate the count and
720        // corrupt a pipeline split (#314 follow-up to #315).
721        let mut layer_indices = std::collections::HashSet::new();
722        if let Some(ref regex) = *DECODER_LAYER_REGEX {
723            for key in weight_map.keys() {
724                if let Some(captures) = regex.captures(key) {
725                    if let Ok(idx) = captures[1].parse::<usize>() {
726                        layer_indices.insert(idx);
727                    }
728                }
729            }
730        }
731
732        // Only enforce when the manifest actually encodes per-layer weights.
733        // (Some manifests for non-transformer components may carry none.)
734        if layer_indices.is_empty() {
735            return Ok(());
736        }
737
738        let manifest_layers = layer_indices.len();
739        if manifest_layers != self.num_hidden_layers {
740            bail!(
741                "num_hidden_layers mismatch: config.json declares {} but \
742                 {} lists {} distinct transformer layers in its weight_map. \
743                 Refusing to load with a layer count that disagrees with the checkpoint.",
744                self.num_hidden_layers,
745                index_path.display(),
746                manifest_layers
747            );
748        }
749
750        info!(
751            "✅ num_hidden_layers ({}) matches shard manifest weight_map",
752            self.num_hidden_layers
753        );
754        Ok(())
755    }
756
757    /// Validate configuration against actual weights
758    fn validate_with_weights(&mut self, weights: &HashMap<String, Tensor>) -> Result<()> {
759        // Check if detected values match weights
760        if let Some(embed) = weights.get("model.embed_tokens.weight") {
761            let actual_vocab = embed.size()[0] as usize;
762            let actual_hidden = embed.size()[1] as usize;
763
764            if self.vocab_size != actual_vocab {
765                info!(
766                    "⚠️ Config vocab_size {} doesn't match weights {}, using weights",
767                    self.vocab_size, actual_vocab
768                );
769                self.vocab_size = actual_vocab;
770            }
771
772            if self.hidden_size != actual_hidden {
773                info!(
774                    "⚠️ Config hidden_size {} doesn't match weights {}, using weights",
775                    self.hidden_size, actual_hidden
776                );
777                self.hidden_size = actual_hidden;
778            }
779        }
780
781        Ok(())
782    }
783}
784
785impl Default for ModelConfig {
786    fn default() -> Self {
787        Self {
788            architecture: ModelArchitecture::Llama,
789            model_type: "llama".to_owned(),
790            version: 2,
791            hidden_size: 4096,
792            num_hidden_layers: 32,
793            num_attention_heads: 32,
794            num_key_value_heads: 32,
795            head_dim: 128,
796            intermediate_size: 11008,
797            vocab_size: 32000,
798            max_position_embeddings: 4096,
799            rope_theta: 10_000.0,
800            rope_scaling: None,
801            sliding_window: None,
802            rope_local_base_freq: None,
803            rms_norm_eps: 1e-5,
804            layer_norm_eps: None,
805            hidden_activation: "silu".to_owned(),
806            use_qk_norm: false,
807            scale_embeddings: false,
808            query_pre_attn_scalar: None,
809            dtype: "bfloat16".to_owned(),
810            use_flash_attention: true,
811            use_kv_cache: true,
812
813            partial_rotary_factor: None,
814            layer_types: vec![],
815            linear_conv_kernel_dim: 0,
816            linear_key_head_dim: 0,
817            linear_value_head_dim: 0,
818            linear_num_key_heads: 0,
819            linear_num_value_heads: 0,
820
821            is_moe: false,
822            num_experts: 0,
823            num_experts_per_tok: 0,
824            moe_intermediate_size: 0,
825            shared_expert_intermediate_size: 0,
826
827            has_vision: false,
828            vision_out_hidden_size: 0,
829        }
830    }
831}
832
833// =============================================================================
834// Training Configuration Load/Save (Phase D)
835// =============================================================================
836
837impl ModelConfig {
838    /// Load hyprstream training config from the model's config.json
839    ///
840    /// Extracts the `hyprstream_training` section if present, otherwise returns None.
841    pub fn load_training_config(
842        model_path: &Path,
843    ) -> Option<crate::config::HyprstreamTrainingConfig> {
844        let config_path = model_path.join("config.json");
845        if !config_path.exists() {
846            return None;
847        }
848
849        let content = std::fs::read_to_string(&config_path).ok()?;
850        let json: serde_json::Value = serde_json::from_str(&content).ok()?;
851
852        // Extract hyprstream_training section if present
853        json.get("hyprstream_training")
854            .and_then(|v| serde_json::from_value(v.clone()).ok())
855    }
856
857    /// Save hyprstream training config to the model's config.json
858    ///
859    /// Preserves all other fields in config.json, only updates/inserts `hyprstream_training`.
860    pub fn save_training_config(
861        model_path: &Path,
862        training_config: &crate::config::HyprstreamTrainingConfig,
863    ) -> Result<()> {
864        let config_path = model_path.join("config.json");
865
866        // Read existing config
867        let content = std::fs::read_to_string(&config_path)?;
868        let mut json: serde_json::Value = serde_json::from_str(&content)?;
869
870        // Update or insert hyprstream_training section
871        json["hyprstream_training"] = serde_json::to_value(training_config)?;
872
873        // Write back with pretty formatting
874        let output = serde_json::to_string_pretty(&json)?;
875        std::fs::write(&config_path, output)?;
876
877        info!(
878            "✅ Training config saved to {}: mode={:?}, target_adapter={:?}",
879            config_path.display(),
880            training_config.mode,
881            training_config.target_adapter
882        );
883
884        Ok(())
885    }
886}
887
888#[cfg(test)]
889#[allow(clippy::unwrap_used, clippy::expect_used)]
890mod tests {
891    use super::*;
892    use std::collections::HashMap;
893    use tch::Tensor;
894
895    fn empty_weights() -> HashMap<String, Tensor> {
896        HashMap::new()
897    }
898
899    fn write(dir: &Path, name: &str, contents: &str) {
900        std::fs::write(dir.join(name), contents).unwrap();
901    }
902
903    /// A minimal-but-valid Llama-style config.json.
904    fn valid_config_json(num_layers: usize) -> String {
905        format!(
906            r#"{{
907                "model_type": "llama",
908                "hidden_size": 4096,
909                "num_hidden_layers": {num_layers},
910                "num_attention_heads": 32,
911                "num_key_value_heads": 8,
912                "intermediate_size": 11008,
913                "vocab_size": 32000,
914                "max_position_embeddings": 4096,
915                "rms_norm_eps": 1e-5
916            }}"#
917        )
918    }
919
920    /// Happy path: a model that ships a proper config.json loads with the
921    /// declared dimensions (no magic-number substitution).
922    #[test]
923    fn valid_config_loads_with_declared_dims() {
924        let dir = tempfile::tempdir().unwrap();
925        write(dir.path(), "config.json", &valid_config_json(40));
926
927        let cfg = ModelConfig::load(dir.path(), &empty_weights()).expect("valid config must load");
928        assert_eq!(cfg.num_hidden_layers, 40);
929        assert_eq!(cfg.hidden_size, 4096);
930        assert_eq!(cfg.num_attention_heads, 32);
931        // head_dim derived from hidden_size / heads.
932        assert_eq!(cfg.head_dim, 128);
933        // KV heads honored (GQA), not silently set to attention-head count.
934        assert_eq!(cfg.num_key_value_heads, 8);
935    }
936
937    /// #315: a missing config.json is a hard error by default (no silent guess).
938    #[test]
939    fn missing_config_is_hard_error() {
940        let dir = tempfile::tempdir().unwrap();
941        let err = ModelConfig::load(dir.path(), &empty_weights())
942            .expect_err("missing config.json must fail");
943        assert!(
944            err.to_string().contains("config.json is required"),
945            "unexpected error: {err}"
946        );
947    }
948
949    /// #315: the silent magic-number default for num_hidden_layers is gone — a
950    /// config.json present but missing the field is a hard error (not 32).
951    #[test]
952    fn missing_num_hidden_layers_is_hard_error() {
953        let dir = tempfile::tempdir().unwrap();
954        write(
955            dir.path(),
956            "config.json",
957            r#"{"model_type":"llama","hidden_size":4096,"num_attention_heads":32}"#,
958        );
959        let err = ModelConfig::load(dir.path(), &empty_weights())
960            .expect_err("missing num_hidden_layers must fail");
961        assert!(
962            err.to_string().contains("num_hidden_layers"),
963            "unexpected error: {err}"
964        );
965    }
966
967    /// #315: the silent magic-number default for hidden_size is gone.
968    #[test]
969    fn missing_hidden_size_is_hard_error() {
970        let dir = tempfile::tempdir().unwrap();
971        write(
972            dir.path(),
973            "config.json",
974            r#"{"model_type":"llama","num_hidden_layers":32,"num_attention_heads":32}"#,
975        );
976        let err = ModelConfig::load(dir.path(), &empty_weights())
977            .expect_err("missing hidden_size must fail");
978        assert!(
979            err.to_string().contains("hidden_size"),
980            "unexpected error: {err}"
981        );
982    }
983
984    /// #315: num_hidden_layers must match the shard manifest's weight_map.
985    #[test]
986    fn index_layer_count_mismatch_is_hard_error() {
987        let dir = tempfile::tempdir().unwrap();
988        // config says 4 layers...
989        write(dir.path(), "config.json", &valid_config_json(4));
990        // ...but the manifest only has weights for layers 0,1,2 (3 layers).
991        write(
992            dir.path(),
993            "model.safetensors.index.json",
994            r#"{"weight_map":{
995                "model.layers.0.self_attn.q_proj.weight":"model-00001-of-00002.safetensors",
996                "model.layers.1.self_attn.q_proj.weight":"model-00001-of-00002.safetensors",
997                "model.layers.2.self_attn.q_proj.weight":"model-00002-of-00002.safetensors",
998                "model.embed_tokens.weight":"model-00001-of-00002.safetensors"
999            }}"#,
1000        );
1001        let err = ModelConfig::load(dir.path(), &empty_weights())
1002            .expect_err("layer-count mismatch must fail");
1003        assert!(
1004            err.to_string().contains("num_hidden_layers mismatch"),
1005            "unexpected error: {err}"
1006        );
1007    }
1008
1009    /// Happy path: config and manifest agree on layer count → loads cleanly.
1010    #[test]
1011    fn index_layer_count_match_loads() {
1012        let dir = tempfile::tempdir().unwrap();
1013        write(dir.path(), "config.json", &valid_config_json(3));
1014        write(
1015            dir.path(),
1016            "model.safetensors.index.json",
1017            r#"{"weight_map":{
1018                "model.layers.0.self_attn.q_proj.weight":"model-00001-of-00002.safetensors",
1019                "model.layers.1.self_attn.q_proj.weight":"model-00001-of-00002.safetensors",
1020                "model.layers.2.self_attn.q_proj.weight":"model-00002-of-00002.safetensors"
1021            }}"#,
1022        );
1023        let cfg = ModelConfig::load(dir.path(), &empty_weights())
1024            .expect("matching layer counts must load");
1025        assert_eq!(cfg.num_hidden_layers, 3);
1026    }
1027
1028    /// #314 follow-up: a multimodal vision tower's own `.layers.N` must NOT
1029    /// inflate the decoder layer count when cross-checking against the manifest.
1030    /// Decoder has 3 layers; the vision tower has 24 — only the decoder counts.
1031    #[test]
1032    fn vision_tower_layers_do_not_inflate_decoder_count() {
1033        let dir = tempfile::tempdir().unwrap();
1034        write(dir.path(), "config.json", &valid_config_json(3));
1035        write(
1036            dir.path(),
1037            "model.safetensors.index.json",
1038            r#"{"weight_map":{
1039                "model.layers.0.self_attn.q_proj.weight":"a.safetensors",
1040                "model.layers.1.self_attn.q_proj.weight":"a.safetensors",
1041                "model.layers.2.self_attn.q_proj.weight":"a.safetensors",
1042                "vision_model.encoder.layers.0.self_attn.q_proj.weight":"v.safetensors",
1043                "vision_model.encoder.layers.10.self_attn.q_proj.weight":"v.safetensors",
1044                "vision_model.encoder.layers.23.self_attn.q_proj.weight":"v.safetensors",
1045                "visual.blocks.5.attn.qkv.weight":"v.safetensors"
1046            }}"#,
1047        );
1048        let cfg = ModelConfig::load(dir.path(), &empty_weights())
1049            .expect("decoder layer count (3) must match despite a 24-layer vision tower");
1050        assert_eq!(cfg.num_hidden_layers, 3);
1051    }
1052
1053    /// The decoder regex also accepts the `language_model.model.layers.<i>.`
1054    /// nesting used by some multimodal checkpoints.
1055    #[test]
1056    fn nested_language_model_layers_are_counted() {
1057        let dir = tempfile::tempdir().unwrap();
1058        write(dir.path(), "config.json", &valid_config_json(2));
1059        write(
1060            dir.path(),
1061            "model.safetensors.index.json",
1062            r#"{"weight_map":{
1063                "language_model.model.layers.0.self_attn.q_proj.weight":"a.safetensors",
1064                "language_model.model.layers.1.self_attn.q_proj.weight":"a.safetensors",
1065                "vision_model.encoder.layers.0.self_attn.q_proj.weight":"v.safetensors"
1066            }}"#,
1067        );
1068        let cfg = ModelConfig::load(dir.path(), &empty_weights())
1069            .expect("nested language_model decoder layers must be counted");
1070        assert_eq!(cfg.num_hidden_layers, 2);
1071    }
1072
1073    /// A zero architectural dimension is rejected (avoids div-by-zero / corruption).
1074    #[test]
1075    fn zero_dimension_is_hard_error() {
1076        let dir = tempfile::tempdir().unwrap();
1077        write(
1078            dir.path(),
1079            "config.json",
1080            r#"{"model_type":"llama","hidden_size":4096,"num_hidden_layers":0,"num_attention_heads":32}"#,
1081        );
1082        let err =
1083            ModelConfig::load(dir.path(), &empty_weights()).expect_err("zero dimension must fail");
1084        assert!(
1085            err.to_string().contains("zero dimension"),
1086            "unexpected error: {err}"
1087        );
1088    }
1089}