Skip to main content

hyprstream_inference/
config.rs

1//! Inference-facing configuration types.
2//!
3//! Carved from the main crate's `config` module (Wave B, `hyprstream-inference`
4//! extraction): the runtime/model/generation/training configuration surface the
5//! inference engine and training pipeline own. The main crate re-exports these
6//! from `crate::config` so existing paths keep resolving.
7//!
8//! AGPL-3.0-only. This crate is part of hyprstream's combined AGPL distribution.
9
10use serde::{Deserialize, Serialize};
11use std::path::PathBuf;
12
13use crate::runtime::generation_metrics::GenerationQualityMetrics;
14
15/// Model loading and identification
16#[derive(Debug, Clone, Serialize, Deserialize)]
17pub struct ModelConfig {
18    /// Path to model file
19    pub path: PathBuf,
20    /// Model identifier (e.g., "qwen2-1.5b")
21    pub name: String,
22    /// Architecture type ("llama", "qwen", etc.)
23    pub architecture: String,
24    /// Expected parameter count
25    pub parameters: Option<u64>,
26    /// QUIC/WebTransport port for model service. None = no QUIC, Some(0) = ephemeral, Some(N) = explicit.
27    #[serde(default)]
28    pub quic_port: Option<u16>,
29}
30
31impl Default for ModelConfig {
32    fn default() -> Self {
33        Self {
34            path: PathBuf::new(),
35            name: String::new(),
36            architecture: String::new(),
37            parameters: None,
38            quic_port: None,
39        }
40    }
41}
42
43/// Runtime execution configuration
44#[derive(Debug, Clone, Serialize, Deserialize)]
45pub struct RuntimeConfig {
46    /// Context window size
47    pub context_length: usize,
48    /// Maximum context length override for KV cache allocation.
49    /// None = use model's max_position_embeddings (can be very large, e.g., 40K tokens)
50    /// Some(n) = cap KV cache at n tokens (significantly reduces GPU memory)
51    pub max_context: Option<u32>,
52    /// KV cache quantization type (None, INT8, NF4, FP4).
53    /// Reduces GPU memory by 50-75% at slight quality cost.
54    #[serde(default)]
55    pub kv_quant_type: hyprstream_rpc_std::model_client::KVQuantType,
56    /// Batch processing size
57    pub batch_size: usize,
58    /// CPU threads (None = auto-detect)
59    pub cpu_threads: Option<usize>,
60    /// Use GPU acceleration
61    pub use_gpu: bool,
62    /// GPU device ID (None = auto-detect, typically device 0).
63    ///
64    /// Legacy single-GPU selector. For multi-GPU, prefer [`Self::devices`];
65    /// this field remains the back-compat fallback when `devices` is empty.
66    pub gpu_device_id: Option<usize>,
67    /// Explicit set of GPU device indices for multi-GPU (#313, epic #310).
68    ///
69    /// Empty = unset → fall back to the single [`Self::gpu_device_id`] /
70    /// `HYPRSTREAM_GPU_DEVICE` (existing single-GPU behavior is unchanged).
71    /// Parsed from `HYPRSTREAM_GPU_DEVICES` (comma-separated, e.g. `0,1`).
72    /// Resolution + validation lives in [`Self::resolve_device_indices`] and is
73    /// consumed by `runtime::DevicePool`.
74    #[serde(default)]
75    pub devices: Vec<usize>,
76    /// Fail fast when a *requested* GPU is unavailable instead of silently
77    /// downgrading to CPU (#315, epic #310).
78    ///
79    /// A process told to run on GPU 3 that silently lands on CPU tanks a pipeline
80    /// split, so strictness is the safe default for the multi-GPU path. This only
81    /// affects the case where a GPU was *explicitly* requested (`use_gpu` with an
82    /// explicit `gpu_device_id`/`devices`); pure auto-detect (`use_gpu` with no
83    /// device requested) still falls back to CPU so the legacy single-GPU
84    /// "use a GPU if there is one" behavior is unchanged.
85    /// Defaults to `true`; override with `HYPRSTREAM_STRICT_DEVICE=0`.
86    #[serde(default = "default_strict_device")]
87    pub strict_device: bool,
88    /// GPU layers to offload (None = auto)
89    pub gpu_layers: Option<usize>,
90    /// Use memory mapping for model files
91    pub mmap: bool,
92    /// KV cache size in MB
93    pub kv_cache_size_mb: usize,
94    /// Case-insensitive compute policy: auto, BF16/bfloat16, FP16/float16,
95    /// FP32/float32, or FP8. FP8 describes weight-path selection: compute stays
96    /// BF16 except channel-FP8 CPU activation-QDQ/matmuls, which use Float32.
97    /// Checkpoint quantization remains independently validated from its tensors.
98    pub precision_mode: Option<String>,
99    // NEW: Concurrency and timeout settings
100    pub max_concurrent_loads: usize,
101    pub max_concurrent_generations: usize,
102    pub default_generation_timeout_ms: u64,
103    pub default_model_load_timeout_ms: u64,
104
105    /// Continuous / in-flight batching (#329, epic #310). **Default: off.**
106    ///
107    /// When enabled, the inference scheduler groups concurrent decode steps of
108    /// same-tenant-delta sequences into a single batched forward (Llama only).
109    /// When off, each stream runs the unchanged batch=1 decode path. Override
110    /// with `HYPRSTREAM_CONTINUOUS_BATCH` (truthy = on). Off by default while the
111    /// scheduler wiring lands incrementally — the batched kernel is correctness-
112    /// gated by `batched_ragged_decode_matches_serial`.
113    #[serde(default = "default_continuous_batching")]
114    pub continuous_batching: bool,
115    /// Max sequences fused into one batched decode step when
116    /// [`Self::continuous_batching`] is on (spike default 16). Tunable via
117    /// `HYPRSTREAM_CONTINUOUS_BATCH_MAX`.
118    #[serde(default = "default_continuous_batch_max")]
119    pub continuous_batch_max: usize,
120    /// FP8 GEMM via torch `_scaled_mm` for FP8 (e4m3) weight projections.
121    /// **Default: off.**
122    ///
123    /// When enabled, `LinearProjection::apply` computes FP8-weight matmuls
124    /// with `at::_scaled_mm` instead of the lazy BF16 dequant-then-matmul,
125    /// picking the first recipe the device supports (torch 2.11, verified
126    /// against release/2.11 ScaledBlas.cpp / RowwiseScaledMM.cu):
127    ///
128    ///  1. v2 blockwise (1x128 activation × 128x128 weight blocks, the
129    ///     checkpoint's native scales): **NVIDIA Hopper (SM90) + cuBLASLt ≥
130    ///     12.9 only** — `_check_deepseek_support` hard-errors on SM120,
131    ///     SM100, SM89, and there is no CPU kernel.
132    ///  2. v1 rowwise (per-token × per-output-channel scales): **SM90+,
133    ///     including SM120/Blackwell** — cuBLASLt rowwise at cuBLAS ≥ 12.9,
134    ///     otherwise the CUTLASS `f8f8bf16_rowwise` SM120 kernel. Requires a
135    ///     load-time requantization of the weight to per-output-channel
136    ///     scales (coarser than 128x128 blocks; +1x FP8 weight memory while
137    ///     the flag is on).
138    ///
139    /// Each recipe latches off per device after its first kernel error and the
140    /// runtime falls back to lazy dequant, so enabling this on unsupported
141    /// hardware costs one warning per device per recipe, not a failure.
142    ///
143    /// Precedence note: like `mmap`, this field is currently **env-only in
144    /// effect** — model construction has no `RuntimeConfig` plumbing, so the
145    /// runtime reads `HYPRSTREAM_FP8_GEMM` directly (cached per process);
146    /// setting the field programmatically does not reach the model path.
147    #[serde(default = "default_fp8_gemm")]
148    pub fp8_gemm: bool,
149
150    /// Self-speculative decoding via the Qwen3.5 MTP draft head. **Default: off.**
151    ///
152    /// When enabled (and the loaded model is a dense Qwen3.5 checkpoint with an
153    /// MTP head), each decode step drafts 1 token with the MTP head and verifies
154    /// it in the next main-model forward: a greedy exact-match accept emits 2
155    /// tokens, a reject emits the verifier's token and rewinds KV/SSM state.
156    /// v1 restrictions: batch=1 streams only, greedy sampling only
157    /// (temperature ≤ 0.01), no tenant LoRA delta, no quantized KV cache
158    /// (`truncate_to` is a full clear for quantized storage, which would silently
159    /// break the reject rewind), dense (non-MoE) MTP blocks only. Override with
160    /// `HYPRSTREAM_SPECULATIVE_DECODE` (truthy = on). Correctness is gated by
161    /// `mtp_decode_matches_serial`.
162    ///
163    /// # Performance envelope (measured)
164    ///
165    /// **Experimental — measured regression, keep default-off.** GPU validation
166    /// (RTX 5090 / SM120, Qwen3.5-4B BF16) found the path functionally correct
167    /// but 0.237x serial throughput (48.84% draft acceptance). The naive cost
168    /// model — accept = 1 forward per 2 tokens, reject = 2 forwards per 1
169    /// token, breakeven at α ≈ 0.5 acceptance with speedup ≈ (1+α)/(2−α) —
170    /// holds only BEFORE structural per-round overheads:
171    ///
172    /// - a full GDN conv/rec SSM-state deep copy (`snapshot_ssm_states`) every
173    ///   round — fixed cost per round, suspected dominant (profile this first
174    ///   if this is ever optimized);
175    /// - the reject-path single-token re-forward that re-syncs SSM state;
176    /// - the per-round MTP draft forward itself.
177    ///
178    /// At 4B with k=1 these overheads dominate the verify savings; break-even
179    /// needs the verification cost amortized (larger backbone) or a cheaper
180    /// draft path. Enable only per workload after measuring acceptance via the
181    /// `inference_speculative_tokens_total` counter (kind=accepted|rejected);
182    /// if a workload shows <~60–70% acceptance this flag is a pessimization.
183    #[serde(default = "default_speculative_decoding")]
184    pub speculative_decoding: bool,
185    /// Materialize FP8 weights as BF16 once at load time (applying the
186    /// block-wise `_scale_inv` scales during load) and drop the FP8+scale
187    /// tensors, instead of dequantizing inside every matmul on the hot path.
188    /// Trades ~2x weight VRAM for zero per-matmul dequant work. Tunable via
189    /// `HYPRSTREAM_FP8_DEQUANT_LOAD` (truthy = on). Off by default — off keeps
190    /// the FP8-in-VRAM behavior, which is required when the BF16 equivalent
191    /// would exceed VRAM. Ignored (with a warning) for multi-device pipelines:
192    /// weights are loaded onto the pool's primary device before layers are
193    /// distributed, so full-model BF16 materialization there could OOM.
194    #[serde(default = "default_fp8_dequant_load")]
195    pub fp8_dequant_load: bool,
196}
197
198/// Default for [`RuntimeConfig::continuous_batching`]: off unless
199/// `HYPRSTREAM_CONTINUOUS_BATCH` is set truthy (#329). Off is the safe default —
200/// the batch=1 path is the verified reference.
201fn default_continuous_batching() -> bool {
202    std::env::var("HYPRSTREAM_CONTINUOUS_BATCH")
203        .map(|v| {
204            matches!(
205                v.trim().to_lowercase().as_str(),
206                "1" | "true" | "yes" | "on"
207            )
208        })
209        .unwrap_or(false)
210}
211
212/// Default for [`RuntimeConfig::continuous_batch_max`] (#329): 16 (spike rec),
213/// overridable via `HYPRSTREAM_CONTINUOUS_BATCH_MAX`.
214fn default_continuous_batch_max() -> usize {
215    std::env::var("HYPRSTREAM_CONTINUOUS_BATCH_MAX")
216        .ok()
217        .and_then(|s| s.trim().parse::<usize>().ok())
218        .filter(|&n| n > 0)
219        .unwrap_or(16)
220}
221
222/// Default for [`RuntimeConfig::fp8_gemm`]: off unless `HYPRSTREAM_FP8_GEMM`
223/// is set truthy. Off is the safe default — the lazy BF16 dequant matmul is
224/// the verified reference, and the `_scaled_mm_v2` blockwise path requires
225/// NVIDIA Hopper (SM90) + cuBLASLt ≥ 12.9 (no CPU/ROCm kernel in torch 2.11).
226pub(crate) fn default_fp8_gemm() -> bool {
227    std::env::var("HYPRSTREAM_FP8_GEMM")
228        .map(|v| {
229            matches!(
230                v.trim().to_lowercase().as_str(),
231                "1" | "true" | "yes" | "on"
232            )
233        })
234        .unwrap_or(false)
235}
236
237/// Default for [`RuntimeConfig::speculative_decoding`]: off unless
238/// `HYPRSTREAM_SPECULATIVE_DECODE` is set truthy. Off is the safe default — the
239/// serial batch=1 path is the verified reference; speculation is a v1 perf
240/// experiment (greedy-only, batch=1-only, dense Qwen3.5 MTP only).
241fn default_speculative_decoding() -> bool {
242    std::env::var("HYPRSTREAM_SPECULATIVE_DECODE")
243        .map(|v| {
244            matches!(
245                v.trim().to_lowercase().as_str(),
246                "1" | "true" | "yes" | "on"
247            )
248        })
249        .unwrap_or(false)
250}
251
252/// Default for [`RuntimeConfig::fp8_dequant_load`]: off unless
253/// `HYPRSTREAM_FP8_DEQUANT_LOAD` is set truthy. Off is the safe default — it
254/// keeps FP8 weights at FP8 size in VRAM with lazy per-matmul dequantization.
255pub(crate) fn default_fp8_dequant_load() -> bool {
256    std::env::var("HYPRSTREAM_FP8_DEQUANT_LOAD")
257        .map(|v| {
258            matches!(
259                v.trim().to_lowercase().as_str(),
260                "1" | "true" | "yes" | "on"
261            )
262        })
263        .unwrap_or(false)
264}
265
266/// Default for [`RuntimeConfig::strict_device`]: strict (fail-fast) unless
267/// `HYPRSTREAM_STRICT_DEVICE` is set to a falsy value. Strictness is the safe
268/// default for the multi-GPU path (#315).
269fn default_strict_device() -> bool {
270    std::env::var("HYPRSTREAM_STRICT_DEVICE")
271        .map(|v| {
272            !matches!(
273                v.trim().to_lowercase().as_str(),
274                "0" | "false" | "no" | "off"
275            )
276        })
277        .unwrap_or(true)
278}
279
280impl Default for RuntimeConfig {
281    fn default() -> Self {
282        // Check environment variables for runtime configuration
283        // Precedence: CLI args > env vars (read here) > hardcoded defaults.
284        // Environment variables set initial defaults; CLI args may override them later.
285        let gpu_device_id = std::env::var("HYPRSTREAM_GPU_DEVICE")
286            .ok()
287            .and_then(|s| s.parse::<usize>().ok());
288
289        // `devices` is populated leniently here (Default cannot return errors);
290        // the authoritative strict parse + validation lives in
291        // `RuntimeConfig::resolve_device_indices`, which re-reads the env var and
292        // turns parse errors into hard errors.
293        let devices = std::env::var("HYPRSTREAM_GPU_DEVICES")
294            .ok()
295            .and_then(|s| Self::parse_device_list(&s).ok())
296            .unwrap_or_default();
297
298        let max_context = std::env::var("HYPRSTREAM_MAX_CONTEXT")
299            .ok()
300            .and_then(|s| s.parse::<u32>().ok());
301
302        let kv_quant_type = std::env::var("HYPRSTREAM_KV_QUANT")
303            .ok()
304            .and_then(|s| match s.to_lowercase().as_str() {
305                "int8" => Some(hyprstream_rpc_std::model_client::KVQuantType::Int8),
306                "nf4" => Some(hyprstream_rpc_std::model_client::KVQuantType::Nf4),
307                "fp4" => Some(hyprstream_rpc_std::model_client::KVQuantType::Fp4),
308                "none" | "" => Some(hyprstream_rpc_std::model_client::KVQuantType::None),
309                _ => None,
310            })
311            .unwrap_or(hyprstream_rpc_std::model_client::KVQuantType::None);
312
313        Self {
314            context_length: 4096,
315            max_context,
316            kv_quant_type,
317            batch_size: 512,
318            cpu_threads: None,
319            use_gpu: true,
320            gpu_device_id, // From env or None (auto-detect device 0)
321            devices,       // From HYPRSTREAM_GPU_DEVICES or empty (→ fall back to gpu_device_id)
322            strict_device: default_strict_device(),
323            gpu_layers: None,
324            mmap: true,
325            kv_cache_size_mb: 2048,
326            precision_mode: Some("auto".to_owned()),
327            max_concurrent_loads: 2,
328            max_concurrent_generations: 10,
329            default_generation_timeout_ms: 120000, // 2 minutes
330            default_model_load_timeout_ms: 300000, // 5 minutes
331            continuous_batching: default_continuous_batching(),
332            continuous_batch_max: default_continuous_batch_max(),
333            fp8_gemm: default_fp8_gemm(),
334            speculative_decoding: default_speculative_decoding(),
335            fp8_dequant_load: default_fp8_dequant_load(),
336        }
337    }
338}
339
340impl RuntimeConfig {
341    /// Parse a comma-separated GPU device list (e.g. `"0,1"`), strictly.
342    ///
343    /// Whitespace around entries is trimmed. Any non-numeric entry, or a
344    /// trailing/empty field (e.g. `"0,"` or `"0,,1"`), is a hard error — there
345    /// is no silent default. Returns the parsed indices (possibly with
346    /// duplicates; dedup/validation is the caller's job).
347    fn parse_device_list(raw: &str) -> anyhow::Result<Vec<usize>> {
348        raw.split(',')
349            .map(|part| {
350                let trimmed = part.trim();
351                trimmed.parse::<usize>().map_err(|e| {
352                    anyhow::anyhow!(
353                        "invalid GPU device index {trimmed:?} in HYPRSTREAM_GPU_DEVICES={raw:?}: {e}"
354                    )
355                })
356            })
357            .collect()
358    }
359
360    /// Resolve the explicitly-requested *multi-GPU* device set, fail-fast.
361    ///
362    /// This is the seam the multi-GPU foundation uses to decide whether to engage
363    /// the new `DevicePool` path. It considers **only** the explicit multi-GPU
364    /// inputs and deliberately excludes the legacy single [`Self::gpu_device_id`]
365    /// so that existing single-GPU behavior is left entirely on its old code
366    /// path (#313 introduces the pool without changing single-GPU runtime
367    /// behavior). Precedence:
368    ///
369    /// 1. `HYPRSTREAM_GPU_DEVICES` env var (re-parsed here strictly, so a
370    ///    malformed value is a hard error rather than a silent fallback).
371    /// 2. The [`Self::devices`] field (e.g. from a config file / CLI).
372    ///
373    /// Returns `Ok(None)` when no explicit multi-GPU set was requested,
374    /// `Ok(Some(indices))` (non-empty, duplicate-free) otherwise, and `Err` on
375    /// parse errors or duplicate indices.
376    pub fn resolve_explicit_multi_device_indices(&self) -> anyhow::Result<Option<Vec<usize>>> {
377        // (1) env var wins and is parsed strictly here. An absent or
378        //     empty/whitespace-only value is treated as "unset" so resolution
379        //     falls through to (2) the struct field (config file / CLI).
380        let env_raw = std::env::var("HYPRSTREAM_GPU_DEVICES").ok();
381        let env_trimmed = env_raw.as_deref().map(str::trim).filter(|s| !s.is_empty());
382        let indices = match env_trimmed {
383            Some(raw) => Self::parse_device_list(raw)?,
384            None => self.devices.clone(),
385        };
386
387        Self::validate_index_set(indices)
388    }
389
390    /// Resolve the GPU device indices to use, fail-fast, including the legacy
391    /// single-GPU fallback.
392    ///
393    /// This is the full-precedence resolver consumed by
394    /// `runtime::DevicePool::from_config`. It is
395    /// [`Self::resolve_explicit_multi_device_indices`] plus a final fallback to
396    /// the legacy single [`Self::gpu_device_id`] (mapped to a one-element set),
397    /// preserving back-compat for callers that build a pool directly from config.
398    ///
399    /// `Ok(None)` means nothing was requested (caller should auto-detect).
400    pub fn resolve_device_indices(&self) -> anyhow::Result<Option<Vec<usize>>> {
401        if let Some(indices) = self.resolve_explicit_multi_device_indices()? {
402            return Ok(Some(indices));
403        }
404        // Legacy single-GPU selector as the final fallback.
405        match self.gpu_device_id {
406            Some(id) => Self::validate_index_set(vec![id]),
407            None => Ok(None),
408        }
409    }
410
411    /// Shared post-processing for a resolved index list: empty → `None`, reject
412    /// duplicates, otherwise `Some(indices)`.
413    fn validate_index_set(indices: Vec<usize>) -> anyhow::Result<Option<Vec<usize>>> {
414        if indices.is_empty() {
415            return Ok(None);
416        }
417        let mut seen = std::collections::HashSet::with_capacity(indices.len());
418        for &idx in &indices {
419            if !seen.insert(idx) {
420                return Err(anyhow::anyhow!(
421                    "duplicate GPU device index {idx} in requested set {indices:?}"
422                ));
423            }
424        }
425        Ok(Some(indices))
426    }
427}
428
429/// Text generation parameters
430#[derive(Debug, Clone, Serialize, Deserialize)]
431pub struct GenerationConfig {
432    /// Maximum tokens to generate
433    pub max_tokens: usize,
434    /// Sampling temperature (0.0-2.0)
435    pub temperature: f32,
436    /// Nucleus sampling threshold
437    pub top_p: f32,
438    /// Top-k sampling limit
439    pub top_k: Option<usize>,
440    /// Repetition penalty
441    pub repeat_penalty: f32,
442    /// Stop sequences
443    pub stop_tokens: Vec<String>,
444    /// Random seed for reproducible generation
445    pub seed: Option<u32>,
446    /// Enable streaming output
447    pub stream: bool,
448}
449
450impl Default for GenerationConfig {
451    fn default() -> Self {
452        Self {
453            max_tokens: 2048,
454            temperature: 0.7,
455            top_p: 0.9,
456            top_k: Some(40),
457            repeat_penalty: 1.1,
458            stop_tokens: vec!["</s>".to_owned(), "<|endoftext|>".to_owned()],
459            seed: None,
460            stream: false,
461        }
462    }
463}
464
465/// Generation result with metadata
466#[derive(Debug, Clone, Serialize, Deserialize)]
467pub struct GenerationResult {
468    pub text: String,
469    pub tokens_generated: usize,
470    pub finish_reason: FinishReason,
471    pub generation_time_ms: u64,
472    pub tokens_per_second: f32,
473    /// Quality metrics for self-supervised training
474    #[serde(skip_serializing_if = "Option::is_none")]
475    pub quality_metrics: Option<GenerationQualityMetrics>,
476
477    // Prefill metrics (processing the prompt)
478    #[serde(default)]
479    pub prefill_tokens: usize,
480    #[serde(default)]
481    pub prefill_time_ms: u64,
482    #[serde(default)]
483    pub prefill_tokens_per_sec: f32,
484
485    // Inference metrics (generating new tokens, excluding prefill)
486    #[serde(default)]
487    pub inference_tokens: usize,
488    #[serde(default)]
489    pub inference_time_ms: u64,
490    #[serde(default)]
491    pub inference_tokens_per_sec: f32,
492
493    /// Online training (TTT) adaptation metrics
494    #[serde(skip_serializing_if = "Option::is_none")]
495    pub ttt_metrics: Option<TTTMetrics>,
496}
497
498/// TTT adaptation metrics (mirrors training::ttt::TTTResult)
499///
500/// Exposed as "Online Training" metrics in user-facing APIs.
501#[derive(Debug, Clone, Serialize, Deserialize)]
502pub struct TTTMetrics {
503    pub avg_loss: f32,
504    pub loss_improvement: f32,
505    pub steps_performed: usize,
506    pub adaptation_time_ms: u64,
507    pub skipped: bool,
508    #[serde(skip_serializing_if = "Option::is_none")]
509    pub skip_reason: Option<String>,
510
511    // Advanced metrics (expert recommendation)
512    pub avg_grad_norm: f32,
513    pub max_grad_norm: f32,
514    pub gradient_clipped: bool,
515    pub tokens_used: usize,
516    pub tokens_provided: usize,
517    pub was_truncated: bool,
518
519    // Tenant-aware TTT metrics
520    /// Initial perplexity before adaptation
521    #[serde(default)]
522    pub initial_perplexity: f32,
523    /// Final perplexity after adaptation
524    #[serde(default)]
525    pub final_perplexity: f32,
526    /// Server's recommendation: true = commit, false = rollback
527    #[serde(default)]
528    pub recommendation: bool,
529    /// Number of steps determined by perplexity gating
530    #[serde(default)]
531    pub gated_steps: usize,
532    /// Whether adaptation is pending client commit/rollback
533    #[serde(default)]
534    pub pending: bool,
535}
536
537impl From<crate::training::ttt::TTTResult> for TTTMetrics {
538    fn from(r: crate::training::ttt::TTTResult) -> Self {
539        Self {
540            avg_loss: r.avg_loss,
541            loss_improvement: r.loss_improvement,
542            steps_performed: r.steps_performed,
543            adaptation_time_ms: r.adaptation_time_ms,
544            skipped: r.skipped,
545            skip_reason: r.skip_reason,
546            avg_grad_norm: r.avg_grad_norm,
547            max_grad_norm: r.max_grad_norm,
548            gradient_clipped: r.gradient_clipped,
549            tokens_used: r.tokens_used,
550            tokens_provided: r.tokens_provided,
551            was_truncated: r.was_truncated,
552            initial_perplexity: r.initial_perplexity,
553            final_perplexity: r.final_perplexity,
554            recommendation: r.recommendation,
555            gated_steps: r.gated_steps,
556            pending: r.pending,
557        }
558    }
559}
560
561/// Why generation stopped
562#[derive(Debug, Clone, Serialize, Deserialize)]
563pub enum FinishReason {
564    MaxTokens,
565    StopToken(String),
566    EndOfSequence,
567    Error(String),
568    Stop,
569}
570
571// =============================================================================
572// Training Mode Configuration (Phase D)
573// =============================================================================
574
575/// Model-level training mode configuration (embedded in config.json under "hyprstream_training")
576///
577/// This allows inference to automatically adapt models when enabled.
578/// The training mode is set via `hyprstream training set test_time_training`.
579#[derive(Debug, Clone, Serialize, Deserialize)]
580pub struct HyprstreamTrainingConfig {
581    /// Training mode: disabled, test_time_training, supervised
582    #[serde(default)]
583    pub mode: TrainingMode,
584
585    /// Target adapter to train (e.g., "01_coding")
586    pub target_adapter: Option<String>,
587
588    /// Learning rate for training
589    #[serde(default = "default_training_learning_rate")]
590    pub learning_rate: f64,
591
592    /// Batch size for training (used by supervised mode)
593    #[serde(default = "default_training_batch_size")]
594    pub batch_size: usize,
595
596    /// Training steps per cycle (used by supervised mode)
597    #[serde(default = "default_training_steps_per_cycle")]
598    pub steps_per_cycle: usize,
599
600    /// Minimum quality score to keep examples (0.0-1.0)
601    #[serde(default = "default_training_min_quality")]
602    pub min_quality_threshold: f32,
603
604    /// Enable training on base model weights (vs LoRA only)
605    #[serde(default)]
606    pub train_base_model: bool,
607
608    /// TTT-specific configuration (for TestTimeTraining mode)
609    #[serde(default)]
610    pub ttt: TTTTrainingConfig,
611
612    /// LoRA rank for TTT delta (default: 8)
613    #[serde(default = "default_lora_rank")]
614    pub lora_rank: usize,
615
616    /// LoRA alpha scaling factor (default: None, which means alpha = rank)
617    #[serde(default)]
618    pub lora_alpha: Option<f32>,
619
620    /// Target modules for LoRA adaptation (default: ["q_proj", "v_proj"])
621    #[serde(default = "default_target_modules")]
622    pub target_modules: Vec<String>,
623}
624
625/// TTT-specific configuration
626#[derive(Debug, Clone, Serialize, Deserialize)]
627pub struct TTTTrainingConfig {
628    /// Learning rate for TTT adaptation (higher than fine-tuning)
629    #[serde(default = "default_ttt_learning_rate")]
630    pub learning_rate: f64,
631
632    /// Number of gradient steps per input
633    #[serde(default = "default_ttt_gradient_steps")]
634    pub gradient_steps: u32,
635
636    /// Maximum gradient norm for clipping
637    #[serde(default = "default_ttt_max_grad_norm")]
638    pub max_grad_norm: f64,
639
640    /// Minimum input length (tokens) to trigger TTT
641    #[serde(default = "default_ttt_min_input_length")]
642    pub min_input_length: u32,
643
644    /// Maximum input length to process for TTT
645    #[serde(default = "default_ttt_max_context")]
646    pub max_ttt_context: u32,
647
648    /// Rank oracle configuration (optional — omit to disable runtime rank adaptation)
649    #[serde(default)]
650    pub rank_oracle: Option<crate::training::RankOracleConfig>,
651
652    /// Per-layer gradient gating (optional — enabled by default)
653    #[serde(default)]
654    pub gradient_gating: Option<crate::training::GradientGatingConfig>,
655}
656
657fn default_ttt_learning_rate() -> f64 {
658    3e-4
659}
660fn default_ttt_gradient_steps() -> u32 {
661    3
662}
663fn default_ttt_max_grad_norm() -> f64 {
664    1.0
665}
666fn default_ttt_min_input_length() -> u32 {
667    32
668}
669fn default_ttt_max_context() -> u32 {
670    512
671}
672
673impl Default for TTTTrainingConfig {
674    fn default() -> Self {
675        Self {
676            learning_rate: default_ttt_learning_rate(),
677            gradient_steps: default_ttt_gradient_steps(),
678            max_grad_norm: default_ttt_max_grad_norm(),
679            min_input_length: default_ttt_min_input_length(),
680            max_ttt_context: default_ttt_max_context(),
681            rank_oracle: None,
682            gradient_gating: None,
683        }
684    }
685}
686
687impl HyprstreamTrainingConfig {
688    /// Check if training is enabled (mode != Disabled)
689    pub fn is_enabled(&self) -> bool {
690        self.mode != TrainingMode::Disabled
691    }
692}
693
694impl Default for HyprstreamTrainingConfig {
695    fn default() -> Self {
696        Self {
697            mode: TrainingMode::default(),
698            target_adapter: None,
699            learning_rate: default_training_learning_rate(),
700            batch_size: default_training_batch_size(),
701            steps_per_cycle: default_training_steps_per_cycle(),
702            min_quality_threshold: default_training_min_quality(),
703            train_base_model: false,
704            ttt: TTTTrainingConfig::default(),
705            lora_rank: default_lora_rank(),
706            lora_alpha: None,
707            target_modules: default_target_modules(),
708        }
709    }
710}
711
712/// Training mode configuration
713#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
714#[serde(rename_all = "snake_case")]
715pub enum TrainingMode {
716    /// Training disabled (default)
717    #[default]
718    Disabled,
719    /// Test-Time Training: adapts to input context before generation
720    /// Research-valid approach based on TTT-E2E
721    TestTimeTraining,
722    /// Supervised training with explicit training data
723    Supervised,
724}
725
726// Default functions for HyprstreamTrainingConfig
727pub fn default_lora_rank() -> usize {
728    8
729}
730pub fn default_target_modules() -> Vec<String> {
731    vec!["q_proj".to_owned(), "v_proj".to_owned()]
732}
733fn default_training_learning_rate() -> f64 {
734    1e-5
735}
736fn default_training_batch_size() -> usize {
737    4
738}
739fn default_training_steps_per_cycle() -> usize {
740    10
741}
742fn default_training_min_quality() -> f32 {
743    0.3
744}
745
746/// Multi-GPU device resolution tests (#313) for the moved `RuntimeConfig`.
747#[cfg(test)]
748mod runtime_config_tests {
749    #![allow(clippy::unwrap_used)]
750
751    use super::*;
752
753    /// RAII guard to set/unset a process env var for the duration of a test
754    /// and restore the previous value. Local copy of the helper that stays
755    /// with the main crate's config tests; test scaffolding only.
756    struct EnvVarGuard {
757        key: String,
758        prev: Option<String>,
759    }
760    impl EnvVarGuard {
761        fn set(key: &str, val: &str) -> Self {
762            let prev = std::env::var(key).ok();
763            std::env::set_var(key, val);
764            Self {
765                key: key.to_owned(),
766                prev,
767            }
768        }
769        fn unset(key: &str) -> Self {
770            let prev = std::env::var(key).ok();
771            std::env::remove_var(key);
772            Self {
773                key: key.to_owned(),
774                prev,
775            }
776        }
777    }
778    impl Drop for EnvVarGuard {
779        fn drop(&mut self) {
780            match &self.prev {
781                Some(v) => std::env::set_var(&self.key, v),
782                None => std::env::remove_var(&self.key),
783            }
784        }
785    }
786
787    #[test]
788    #[allow(clippy::unwrap_used)]
789    fn parse_device_list_basic() {
790        assert_eq!(RuntimeConfig::parse_device_list("0,1").unwrap(), vec![0, 1]);
791
792        // Whitespace around entries is tolerated.
793
794        assert_eq!(
795            RuntimeConfig::parse_device_list(" 0 , 2 ").unwrap(),
796            vec![0, 2]
797        );
798
799        assert_eq!(RuntimeConfig::parse_device_list("3").unwrap(), vec![3]);
800    }
801
802    #[test]
803    fn parse_device_list_rejects_garbage() {
804        // Non-numeric, empty fields, and negatives are hard errors (no silent default).
805
806        assert!(RuntimeConfig::parse_device_list("0,foo").is_err());
807
808        assert!(RuntimeConfig::parse_device_list("0,").is_err());
809
810        assert!(RuntimeConfig::parse_device_list("0,,1").is_err());
811
812        assert!(RuntimeConfig::parse_device_list("-1").is_err());
813    }
814
815    #[test]
816    #[allow(clippy::unwrap_used)]
817    fn validate_index_set_dedup_and_empty() {
818        // Empty → None (auto-detect).
819
820        assert_eq!(RuntimeConfig::validate_index_set(vec![]).unwrap(), None);
821
822        // Duplicates → error.
823
824        assert!(RuntimeConfig::validate_index_set(vec![0, 0]).is_err());
825
826        // Distinct → Some, order preserved.
827
828        assert_eq!(
829            RuntimeConfig::validate_index_set(vec![2, 0, 1]).unwrap(),
830            Some(vec![2, 0, 1])
831        );
832    }
833
834    /// Serializes the two tests that mutate the shared `HYPRSTREAM_GPU_DEVICES`
835    /// process env var so they don't race under the parallel test runner.
836    static GPU_DEVICES_ENV_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
837
838    #[test]
839    #[allow(clippy::unwrap_used)]
840    fn resolve_uses_devices_field_and_legacy_fallback() {
841        let _serial = GPU_DEVICES_ENV_LOCK.lock();
842
843        // Guard against the env var leaking from the ambient environment so the
844
845        // struct-field/legacy precedence is exercised deterministically.
846
847        let _guard = EnvVarGuard::unset("HYPRSTREAM_GPU_DEVICES");
848
849        // Explicit multi-device field wins.
850
851        let mut cfg = RuntimeConfig::default();
852
853        cfg.devices = vec![0, 1];
854
855        cfg.gpu_device_id = Some(7);
856
857        assert_eq!(
858            cfg.resolve_explicit_multi_device_indices().unwrap(),
859            Some(vec![0, 1])
860        );
861
862        assert_eq!(cfg.resolve_device_indices().unwrap(), Some(vec![0, 1]));
863
864        // No explicit multi-device set → explicit resolver is None, but the full
865
866        // resolver falls back to the legacy single gpu_device_id.
867
868        let mut legacy = RuntimeConfig::default();
869
870        legacy.devices = vec![];
871
872        legacy.gpu_device_id = Some(3);
873
874        assert_eq!(
875            legacy.resolve_explicit_multi_device_indices().unwrap(),
876            None
877        );
878
879        assert_eq!(legacy.resolve_device_indices().unwrap(), Some(vec![3]));
880
881        // Nothing requested anywhere → None (auto-detect path preserved).
882
883        let mut none = RuntimeConfig::default();
884
885        none.devices = vec![];
886
887        none.gpu_device_id = None;
888
889        assert_eq!(none.resolve_device_indices().unwrap(), None);
890    }
891
892    #[test]
893    #[allow(clippy::unwrap_used)]
894    fn resolve_env_var_overrides_and_is_strict() {
895        let _serial = GPU_DEVICES_ENV_LOCK.lock();
896
897        let mut cfg = RuntimeConfig::default();
898
899        cfg.devices = vec![5];
900
901        cfg.gpu_device_id = Some(9);
902
903        {
904            let _g = EnvVarGuard::set("HYPRSTREAM_GPU_DEVICES", "0,1,2");
905
906            assert_eq!(
907                cfg.resolve_explicit_multi_device_indices().unwrap(),
908                Some(vec![0, 1, 2]),
909                "env var must override the devices field"
910            );
911        }
912
913        {
914            // Malformed env var is a hard error (not a silent fallback to field).
915
916            let _g = EnvVarGuard::set("HYPRSTREAM_GPU_DEVICES", "0,nope");
917
918            assert!(cfg.resolve_explicit_multi_device_indices().is_err());
919        }
920
921        {
922            // Duplicate in env var → error.
923
924            let _g = EnvVarGuard::set("HYPRSTREAM_GPU_DEVICES", "1,1");
925
926            assert!(cfg.resolve_explicit_multi_device_indices().is_err());
927        }
928
929        {
930            // Explicitly-empty env var is treated as unset (falls back to field).
931
932            let _g = EnvVarGuard::set("HYPRSTREAM_GPU_DEVICES", "  ");
933
934            assert_eq!(
935                cfg.resolve_explicit_multi_device_indices().unwrap(),
936                Some(vec![5])
937            );
938        }
939    }
940
941    /// Serializes tests mutating the shared `HYPRSTREAM_STRICT_DEVICE` env var.
942    static STRICT_DEVICE_ENV_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
943
944    /// #315: strict_device defaults to fail-fast, and can be opted out via env.
945    #[test]
946    fn strict_device_defaults_to_true_and_respects_env() {
947        let _serial = STRICT_DEVICE_ENV_LOCK.lock();
948
949        {
950            let _g = EnvVarGuard::unset("HYPRSTREAM_STRICT_DEVICE");
951
952            assert!(
953                RuntimeConfig::default().strict_device,
954                "strict_device must default to true (safe default for multi-GPU)"
955            );
956        }
957
958        for falsy in ["0", "false", "no", "off"] {
959            let _g = EnvVarGuard::set("HYPRSTREAM_STRICT_DEVICE", falsy);
960
961            assert!(
962                !RuntimeConfig::default().strict_device,
963                "HYPRSTREAM_STRICT_DEVICE={falsy} must disable strict_device"
964            );
965        }
966
967        {
968            let _g = EnvVarGuard::set("HYPRSTREAM_STRICT_DEVICE", "1");
969
970            assert!(RuntimeConfig::default().strict_device);
971        }
972    }
973}