hyprstream_inference/config.rs
1//! Inference-facing configuration types.
2//!
3//! Carved from the main crate's `config` module (Wave B, `hyprstream-inference`
4//! extraction): the runtime/model/generation/training configuration surface the
5//! inference engine and training pipeline own. The main crate re-exports these
6//! from `crate::config` so existing paths keep resolving.
7//!
8//! AGPL-3.0-only. This crate is part of hyprstream's combined AGPL distribution.
9
10use serde::{Deserialize, Serialize};
11use std::path::PathBuf;
12
13use crate::runtime::generation_metrics::GenerationQualityMetrics;
14
15/// Model loading and identification
16#[derive(Debug, Clone, Serialize, Deserialize)]
17pub struct ModelConfig {
18 /// Path to model file
19 pub path: PathBuf,
20 /// Model identifier (e.g., "qwen2-1.5b")
21 pub name: String,
22 /// Architecture type ("llama", "qwen", etc.)
23 pub architecture: String,
24 /// Expected parameter count
25 pub parameters: Option<u64>,
26 /// QUIC/WebTransport port for model service. None = no QUIC, Some(0) = ephemeral, Some(N) = explicit.
27 #[serde(default)]
28 pub quic_port: Option<u16>,
29}
30
31impl Default for ModelConfig {
32 fn default() -> Self {
33 Self {
34 path: PathBuf::new(),
35 name: String::new(),
36 architecture: String::new(),
37 parameters: None,
38 quic_port: None,
39 }
40 }
41}
42
43/// Runtime execution configuration
44#[derive(Debug, Clone, Serialize, Deserialize)]
45pub struct RuntimeConfig {
46 /// Context window size
47 pub context_length: usize,
48 /// Maximum context length override for KV cache allocation.
49 /// None = use model's max_position_embeddings (can be very large, e.g., 40K tokens)
50 /// Some(n) = cap KV cache at n tokens (significantly reduces GPU memory)
51 pub max_context: Option<u32>,
52 /// KV cache quantization type (None, INT8, NF4, FP4).
53 /// Reduces GPU memory by 50-75% at slight quality cost.
54 #[serde(default)]
55 pub kv_quant_type: hyprstream_rpc_std::model_client::KVQuantType,
56 /// Batch processing size
57 pub batch_size: usize,
58 /// CPU threads (None = auto-detect)
59 pub cpu_threads: Option<usize>,
60 /// Use GPU acceleration
61 pub use_gpu: bool,
62 /// GPU device ID (None = auto-detect, typically device 0).
63 ///
64 /// Legacy single-GPU selector. For multi-GPU, prefer [`Self::devices`];
65 /// this field remains the back-compat fallback when `devices` is empty.
66 pub gpu_device_id: Option<usize>,
67 /// Explicit set of GPU device indices for multi-GPU (#313, epic #310).
68 ///
69 /// Empty = unset → fall back to the single [`Self::gpu_device_id`] /
70 /// `HYPRSTREAM_GPU_DEVICE` (existing single-GPU behavior is unchanged).
71 /// Parsed from `HYPRSTREAM_GPU_DEVICES` (comma-separated, e.g. `0,1`).
72 /// Resolution + validation lives in [`Self::resolve_device_indices`] and is
73 /// consumed by `runtime::DevicePool`.
74 #[serde(default)]
75 pub devices: Vec<usize>,
76 /// Fail fast when a *requested* GPU is unavailable instead of silently
77 /// downgrading to CPU (#315, epic #310).
78 ///
79 /// A process told to run on GPU 3 that silently lands on CPU tanks a pipeline
80 /// split, so strictness is the safe default for the multi-GPU path. This only
81 /// affects the case where a GPU was *explicitly* requested (`use_gpu` with an
82 /// explicit `gpu_device_id`/`devices`); pure auto-detect (`use_gpu` with no
83 /// device requested) still falls back to CPU so the legacy single-GPU
84 /// "use a GPU if there is one" behavior is unchanged.
85 /// Defaults to `true`; override with `HYPRSTREAM_STRICT_DEVICE=0`.
86 #[serde(default = "default_strict_device")]
87 pub strict_device: bool,
88 /// GPU layers to offload (None = auto)
89 pub gpu_layers: Option<usize>,
90 /// Use memory mapping for model files
91 pub mmap: bool,
92 /// KV cache size in MB
93 pub kv_cache_size_mb: usize,
94 /// Case-insensitive compute policy: auto, BF16/bfloat16, FP16/float16,
95 /// FP32/float32, or FP8. FP8 describes weight-path selection: compute stays
96 /// BF16 except channel-FP8 CPU activation-QDQ/matmuls, which use Float32.
97 /// Checkpoint quantization remains independently validated from its tensors.
98 pub precision_mode: Option<String>,
99 // NEW: Concurrency and timeout settings
100 pub max_concurrent_loads: usize,
101 pub max_concurrent_generations: usize,
102 pub default_generation_timeout_ms: u64,
103 pub default_model_load_timeout_ms: u64,
104
105 /// Continuous / in-flight batching (#329, epic #310). **Default: off.**
106 ///
107 /// When enabled, the inference scheduler groups concurrent decode steps of
108 /// same-tenant-delta sequences into a single batched forward (Llama only).
109 /// When off, each stream runs the unchanged batch=1 decode path. Override
110 /// with `HYPRSTREAM_CONTINUOUS_BATCH` (truthy = on). Off by default while the
111 /// scheduler wiring lands incrementally — the batched kernel is correctness-
112 /// gated by `batched_ragged_decode_matches_serial`.
113 #[serde(default = "default_continuous_batching")]
114 pub continuous_batching: bool,
115 /// Max sequences fused into one batched decode step when
116 /// [`Self::continuous_batching`] is on (spike default 16). Tunable via
117 /// `HYPRSTREAM_CONTINUOUS_BATCH_MAX`.
118 #[serde(default = "default_continuous_batch_max")]
119 pub continuous_batch_max: usize,
120 /// FP8 GEMM via torch `_scaled_mm` for FP8 (e4m3) weight projections.
121 /// **Default: off.**
122 ///
123 /// When enabled, `LinearProjection::apply` computes FP8-weight matmuls
124 /// with `at::_scaled_mm` instead of the lazy BF16 dequant-then-matmul,
125 /// picking the first recipe the device supports (torch 2.11, verified
126 /// against release/2.11 ScaledBlas.cpp / RowwiseScaledMM.cu):
127 ///
128 /// 1. v2 blockwise (1x128 activation × 128x128 weight blocks, the
129 /// checkpoint's native scales): **NVIDIA Hopper (SM90) + cuBLASLt ≥
130 /// 12.9 only** — `_check_deepseek_support` hard-errors on SM120,
131 /// SM100, SM89, and there is no CPU kernel.
132 /// 2. v1 rowwise (per-token × per-output-channel scales): **SM90+,
133 /// including SM120/Blackwell** — cuBLASLt rowwise at cuBLAS ≥ 12.9,
134 /// otherwise the CUTLASS `f8f8bf16_rowwise` SM120 kernel. Requires a
135 /// load-time requantization of the weight to per-output-channel
136 /// scales (coarser than 128x128 blocks; +1x FP8 weight memory while
137 /// the flag is on).
138 ///
139 /// Each recipe latches off per device after its first kernel error and the
140 /// runtime falls back to lazy dequant, so enabling this on unsupported
141 /// hardware costs one warning per device per recipe, not a failure.
142 ///
143 /// Precedence note: like `mmap`, this field is currently **env-only in
144 /// effect** — model construction has no `RuntimeConfig` plumbing, so the
145 /// runtime reads `HYPRSTREAM_FP8_GEMM` directly (cached per process);
146 /// setting the field programmatically does not reach the model path.
147 #[serde(default = "default_fp8_gemm")]
148 pub fp8_gemm: bool,
149
150 /// Self-speculative decoding via the Qwen3.5 MTP draft head. **Default: off.**
151 ///
152 /// When enabled (and the loaded model is a dense Qwen3.5 checkpoint with an
153 /// MTP head), each decode step drafts 1 token with the MTP head and verifies
154 /// it in the next main-model forward: a greedy exact-match accept emits 2
155 /// tokens, a reject emits the verifier's token and rewinds KV/SSM state.
156 /// v1 restrictions: batch=1 streams only, greedy sampling only
157 /// (temperature ≤ 0.01), no tenant LoRA delta, no quantized KV cache
158 /// (`truncate_to` is a full clear for quantized storage, which would silently
159 /// break the reject rewind), dense (non-MoE) MTP blocks only. Override with
160 /// `HYPRSTREAM_SPECULATIVE_DECODE` (truthy = on). Correctness is gated by
161 /// `mtp_decode_matches_serial`.
162 ///
163 /// # Performance envelope (measured)
164 ///
165 /// **Experimental — measured regression, keep default-off.** GPU validation
166 /// (RTX 5090 / SM120, Qwen3.5-4B BF16) found the path functionally correct
167 /// but 0.237x serial throughput (48.84% draft acceptance). The naive cost
168 /// model — accept = 1 forward per 2 tokens, reject = 2 forwards per 1
169 /// token, breakeven at α ≈ 0.5 acceptance with speedup ≈ (1+α)/(2−α) —
170 /// holds only BEFORE structural per-round overheads:
171 ///
172 /// - a full GDN conv/rec SSM-state deep copy (`snapshot_ssm_states`) every
173 /// round — fixed cost per round, suspected dominant (profile this first
174 /// if this is ever optimized);
175 /// - the reject-path single-token re-forward that re-syncs SSM state;
176 /// - the per-round MTP draft forward itself.
177 ///
178 /// At 4B with k=1 these overheads dominate the verify savings; break-even
179 /// needs the verification cost amortized (larger backbone) or a cheaper
180 /// draft path. Enable only per workload after measuring acceptance via the
181 /// `inference_speculative_tokens_total` counter (kind=accepted|rejected);
182 /// if a workload shows <~60–70% acceptance this flag is a pessimization.
183 #[serde(default = "default_speculative_decoding")]
184 pub speculative_decoding: bool,
185 /// Materialize FP8 weights as BF16 once at load time (applying the
186 /// block-wise `_scale_inv` scales during load) and drop the FP8+scale
187 /// tensors, instead of dequantizing inside every matmul on the hot path.
188 /// Trades ~2x weight VRAM for zero per-matmul dequant work. Tunable via
189 /// `HYPRSTREAM_FP8_DEQUANT_LOAD` (truthy = on). Off by default — off keeps
190 /// the FP8-in-VRAM behavior, which is required when the BF16 equivalent
191 /// would exceed VRAM. Ignored (with a warning) for multi-device pipelines:
192 /// weights are loaded onto the pool's primary device before layers are
193 /// distributed, so full-model BF16 materialization there could OOM.
194 #[serde(default = "default_fp8_dequant_load")]
195 pub fp8_dequant_load: bool,
196}
197
198/// Default for [`RuntimeConfig::continuous_batching`]: off unless
199/// `HYPRSTREAM_CONTINUOUS_BATCH` is set truthy (#329). Off is the safe default —
200/// the batch=1 path is the verified reference.
201fn default_continuous_batching() -> bool {
202 std::env::var("HYPRSTREAM_CONTINUOUS_BATCH")
203 .map(|v| {
204 matches!(
205 v.trim().to_lowercase().as_str(),
206 "1" | "true" | "yes" | "on"
207 )
208 })
209 .unwrap_or(false)
210}
211
212/// Default for [`RuntimeConfig::continuous_batch_max`] (#329): 16 (spike rec),
213/// overridable via `HYPRSTREAM_CONTINUOUS_BATCH_MAX`.
214fn default_continuous_batch_max() -> usize {
215 std::env::var("HYPRSTREAM_CONTINUOUS_BATCH_MAX")
216 .ok()
217 .and_then(|s| s.trim().parse::<usize>().ok())
218 .filter(|&n| n > 0)
219 .unwrap_or(16)
220}
221
222/// Default for [`RuntimeConfig::fp8_gemm`]: off unless `HYPRSTREAM_FP8_GEMM`
223/// is set truthy. Off is the safe default — the lazy BF16 dequant matmul is
224/// the verified reference, and the `_scaled_mm_v2` blockwise path requires
225/// NVIDIA Hopper (SM90) + cuBLASLt ≥ 12.9 (no CPU/ROCm kernel in torch 2.11).
226pub(crate) fn default_fp8_gemm() -> bool {
227 std::env::var("HYPRSTREAM_FP8_GEMM")
228 .map(|v| {
229 matches!(
230 v.trim().to_lowercase().as_str(),
231 "1" | "true" | "yes" | "on"
232 )
233 })
234 .unwrap_or(false)
235}
236
237/// Default for [`RuntimeConfig::speculative_decoding`]: off unless
238/// `HYPRSTREAM_SPECULATIVE_DECODE` is set truthy. Off is the safe default — the
239/// serial batch=1 path is the verified reference; speculation is a v1 perf
240/// experiment (greedy-only, batch=1-only, dense Qwen3.5 MTP only).
241fn default_speculative_decoding() -> bool {
242 std::env::var("HYPRSTREAM_SPECULATIVE_DECODE")
243 .map(|v| {
244 matches!(
245 v.trim().to_lowercase().as_str(),
246 "1" | "true" | "yes" | "on"
247 )
248 })
249 .unwrap_or(false)
250}
251
252/// Default for [`RuntimeConfig::fp8_dequant_load`]: off unless
253/// `HYPRSTREAM_FP8_DEQUANT_LOAD` is set truthy. Off is the safe default — it
254/// keeps FP8 weights at FP8 size in VRAM with lazy per-matmul dequantization.
255pub(crate) fn default_fp8_dequant_load() -> bool {
256 std::env::var("HYPRSTREAM_FP8_DEQUANT_LOAD")
257 .map(|v| {
258 matches!(
259 v.trim().to_lowercase().as_str(),
260 "1" | "true" | "yes" | "on"
261 )
262 })
263 .unwrap_or(false)
264}
265
266/// Default for [`RuntimeConfig::strict_device`]: strict (fail-fast) unless
267/// `HYPRSTREAM_STRICT_DEVICE` is set to a falsy value. Strictness is the safe
268/// default for the multi-GPU path (#315).
269fn default_strict_device() -> bool {
270 std::env::var("HYPRSTREAM_STRICT_DEVICE")
271 .map(|v| {
272 !matches!(
273 v.trim().to_lowercase().as_str(),
274 "0" | "false" | "no" | "off"
275 )
276 })
277 .unwrap_or(true)
278}
279
280impl Default for RuntimeConfig {
281 fn default() -> Self {
282 // Check environment variables for runtime configuration
283 // Precedence: CLI args > env vars (read here) > hardcoded defaults.
284 // Environment variables set initial defaults; CLI args may override them later.
285 let gpu_device_id = std::env::var("HYPRSTREAM_GPU_DEVICE")
286 .ok()
287 .and_then(|s| s.parse::<usize>().ok());
288
289 // `devices` is populated leniently here (Default cannot return errors);
290 // the authoritative strict parse + validation lives in
291 // `RuntimeConfig::resolve_device_indices`, which re-reads the env var and
292 // turns parse errors into hard errors.
293 let devices = std::env::var("HYPRSTREAM_GPU_DEVICES")
294 .ok()
295 .and_then(|s| Self::parse_device_list(&s).ok())
296 .unwrap_or_default();
297
298 let max_context = std::env::var("HYPRSTREAM_MAX_CONTEXT")
299 .ok()
300 .and_then(|s| s.parse::<u32>().ok());
301
302 let kv_quant_type = std::env::var("HYPRSTREAM_KV_QUANT")
303 .ok()
304 .and_then(|s| match s.to_lowercase().as_str() {
305 "int8" => Some(hyprstream_rpc_std::model_client::KVQuantType::Int8),
306 "nf4" => Some(hyprstream_rpc_std::model_client::KVQuantType::Nf4),
307 "fp4" => Some(hyprstream_rpc_std::model_client::KVQuantType::Fp4),
308 "none" | "" => Some(hyprstream_rpc_std::model_client::KVQuantType::None),
309 _ => None,
310 })
311 .unwrap_or(hyprstream_rpc_std::model_client::KVQuantType::None);
312
313 Self {
314 context_length: 4096,
315 max_context,
316 kv_quant_type,
317 batch_size: 512,
318 cpu_threads: None,
319 use_gpu: true,
320 gpu_device_id, // From env or None (auto-detect device 0)
321 devices, // From HYPRSTREAM_GPU_DEVICES or empty (→ fall back to gpu_device_id)
322 strict_device: default_strict_device(),
323 gpu_layers: None,
324 mmap: true,
325 kv_cache_size_mb: 2048,
326 precision_mode: Some("auto".to_owned()),
327 max_concurrent_loads: 2,
328 max_concurrent_generations: 10,
329 default_generation_timeout_ms: 120000, // 2 minutes
330 default_model_load_timeout_ms: 300000, // 5 minutes
331 continuous_batching: default_continuous_batching(),
332 continuous_batch_max: default_continuous_batch_max(),
333 fp8_gemm: default_fp8_gemm(),
334 speculative_decoding: default_speculative_decoding(),
335 fp8_dequant_load: default_fp8_dequant_load(),
336 }
337 }
338}
339
340impl RuntimeConfig {
341 /// Parse a comma-separated GPU device list (e.g. `"0,1"`), strictly.
342 ///
343 /// Whitespace around entries is trimmed. Any non-numeric entry, or a
344 /// trailing/empty field (e.g. `"0,"` or `"0,,1"`), is a hard error — there
345 /// is no silent default. Returns the parsed indices (possibly with
346 /// duplicates; dedup/validation is the caller's job).
347 fn parse_device_list(raw: &str) -> anyhow::Result<Vec<usize>> {
348 raw.split(',')
349 .map(|part| {
350 let trimmed = part.trim();
351 trimmed.parse::<usize>().map_err(|e| {
352 anyhow::anyhow!(
353 "invalid GPU device index {trimmed:?} in HYPRSTREAM_GPU_DEVICES={raw:?}: {e}"
354 )
355 })
356 })
357 .collect()
358 }
359
360 /// Resolve the explicitly-requested *multi-GPU* device set, fail-fast.
361 ///
362 /// This is the seam the multi-GPU foundation uses to decide whether to engage
363 /// the new `DevicePool` path. It considers **only** the explicit multi-GPU
364 /// inputs and deliberately excludes the legacy single [`Self::gpu_device_id`]
365 /// so that existing single-GPU behavior is left entirely on its old code
366 /// path (#313 introduces the pool without changing single-GPU runtime
367 /// behavior). Precedence:
368 ///
369 /// 1. `HYPRSTREAM_GPU_DEVICES` env var (re-parsed here strictly, so a
370 /// malformed value is a hard error rather than a silent fallback).
371 /// 2. The [`Self::devices`] field (e.g. from a config file / CLI).
372 ///
373 /// Returns `Ok(None)` when no explicit multi-GPU set was requested,
374 /// `Ok(Some(indices))` (non-empty, duplicate-free) otherwise, and `Err` on
375 /// parse errors or duplicate indices.
376 pub fn resolve_explicit_multi_device_indices(&self) -> anyhow::Result<Option<Vec<usize>>> {
377 // (1) env var wins and is parsed strictly here. An absent or
378 // empty/whitespace-only value is treated as "unset" so resolution
379 // falls through to (2) the struct field (config file / CLI).
380 let env_raw = std::env::var("HYPRSTREAM_GPU_DEVICES").ok();
381 let env_trimmed = env_raw.as_deref().map(str::trim).filter(|s| !s.is_empty());
382 let indices = match env_trimmed {
383 Some(raw) => Self::parse_device_list(raw)?,
384 None => self.devices.clone(),
385 };
386
387 Self::validate_index_set(indices)
388 }
389
390 /// Resolve the GPU device indices to use, fail-fast, including the legacy
391 /// single-GPU fallback.
392 ///
393 /// This is the full-precedence resolver consumed by
394 /// `runtime::DevicePool::from_config`. It is
395 /// [`Self::resolve_explicit_multi_device_indices`] plus a final fallback to
396 /// the legacy single [`Self::gpu_device_id`] (mapped to a one-element set),
397 /// preserving back-compat for callers that build a pool directly from config.
398 ///
399 /// `Ok(None)` means nothing was requested (caller should auto-detect).
400 pub fn resolve_device_indices(&self) -> anyhow::Result<Option<Vec<usize>>> {
401 if let Some(indices) = self.resolve_explicit_multi_device_indices()? {
402 return Ok(Some(indices));
403 }
404 // Legacy single-GPU selector as the final fallback.
405 match self.gpu_device_id {
406 Some(id) => Self::validate_index_set(vec![id]),
407 None => Ok(None),
408 }
409 }
410
411 /// Shared post-processing for a resolved index list: empty → `None`, reject
412 /// duplicates, otherwise `Some(indices)`.
413 fn validate_index_set(indices: Vec<usize>) -> anyhow::Result<Option<Vec<usize>>> {
414 if indices.is_empty() {
415 return Ok(None);
416 }
417 let mut seen = std::collections::HashSet::with_capacity(indices.len());
418 for &idx in &indices {
419 if !seen.insert(idx) {
420 return Err(anyhow::anyhow!(
421 "duplicate GPU device index {idx} in requested set {indices:?}"
422 ));
423 }
424 }
425 Ok(Some(indices))
426 }
427}
428
429/// Text generation parameters
430#[derive(Debug, Clone, Serialize, Deserialize)]
431pub struct GenerationConfig {
432 /// Maximum tokens to generate
433 pub max_tokens: usize,
434 /// Sampling temperature (0.0-2.0)
435 pub temperature: f32,
436 /// Nucleus sampling threshold
437 pub top_p: f32,
438 /// Top-k sampling limit
439 pub top_k: Option<usize>,
440 /// Repetition penalty
441 pub repeat_penalty: f32,
442 /// Stop sequences
443 pub stop_tokens: Vec<String>,
444 /// Random seed for reproducible generation
445 pub seed: Option<u32>,
446 /// Enable streaming output
447 pub stream: bool,
448}
449
450impl Default for GenerationConfig {
451 fn default() -> Self {
452 Self {
453 max_tokens: 2048,
454 temperature: 0.7,
455 top_p: 0.9,
456 top_k: Some(40),
457 repeat_penalty: 1.1,
458 stop_tokens: vec!["</s>".to_owned(), "<|endoftext|>".to_owned()],
459 seed: None,
460 stream: false,
461 }
462 }
463}
464
465/// Generation result with metadata
466#[derive(Debug, Clone, Serialize, Deserialize)]
467pub struct GenerationResult {
468 pub text: String,
469 pub tokens_generated: usize,
470 pub finish_reason: FinishReason,
471 pub generation_time_ms: u64,
472 pub tokens_per_second: f32,
473 /// Quality metrics for self-supervised training
474 #[serde(skip_serializing_if = "Option::is_none")]
475 pub quality_metrics: Option<GenerationQualityMetrics>,
476
477 // Prefill metrics (processing the prompt)
478 #[serde(default)]
479 pub prefill_tokens: usize,
480 #[serde(default)]
481 pub prefill_time_ms: u64,
482 #[serde(default)]
483 pub prefill_tokens_per_sec: f32,
484
485 // Inference metrics (generating new tokens, excluding prefill)
486 #[serde(default)]
487 pub inference_tokens: usize,
488 #[serde(default)]
489 pub inference_time_ms: u64,
490 #[serde(default)]
491 pub inference_tokens_per_sec: f32,
492
493 /// Online training (TTT) adaptation metrics
494 #[serde(skip_serializing_if = "Option::is_none")]
495 pub ttt_metrics: Option<TTTMetrics>,
496}
497
498/// TTT adaptation metrics (mirrors training::ttt::TTTResult)
499///
500/// Exposed as "Online Training" metrics in user-facing APIs.
501#[derive(Debug, Clone, Serialize, Deserialize)]
502pub struct TTTMetrics {
503 pub avg_loss: f32,
504 pub loss_improvement: f32,
505 pub steps_performed: usize,
506 pub adaptation_time_ms: u64,
507 pub skipped: bool,
508 #[serde(skip_serializing_if = "Option::is_none")]
509 pub skip_reason: Option<String>,
510
511 // Advanced metrics (expert recommendation)
512 pub avg_grad_norm: f32,
513 pub max_grad_norm: f32,
514 pub gradient_clipped: bool,
515 pub tokens_used: usize,
516 pub tokens_provided: usize,
517 pub was_truncated: bool,
518
519 // Tenant-aware TTT metrics
520 /// Initial perplexity before adaptation
521 #[serde(default)]
522 pub initial_perplexity: f32,
523 /// Final perplexity after adaptation
524 #[serde(default)]
525 pub final_perplexity: f32,
526 /// Server's recommendation: true = commit, false = rollback
527 #[serde(default)]
528 pub recommendation: bool,
529 /// Number of steps determined by perplexity gating
530 #[serde(default)]
531 pub gated_steps: usize,
532 /// Whether adaptation is pending client commit/rollback
533 #[serde(default)]
534 pub pending: bool,
535}
536
537impl From<crate::training::ttt::TTTResult> for TTTMetrics {
538 fn from(r: crate::training::ttt::TTTResult) -> Self {
539 Self {
540 avg_loss: r.avg_loss,
541 loss_improvement: r.loss_improvement,
542 steps_performed: r.steps_performed,
543 adaptation_time_ms: r.adaptation_time_ms,
544 skipped: r.skipped,
545 skip_reason: r.skip_reason,
546 avg_grad_norm: r.avg_grad_norm,
547 max_grad_norm: r.max_grad_norm,
548 gradient_clipped: r.gradient_clipped,
549 tokens_used: r.tokens_used,
550 tokens_provided: r.tokens_provided,
551 was_truncated: r.was_truncated,
552 initial_perplexity: r.initial_perplexity,
553 final_perplexity: r.final_perplexity,
554 recommendation: r.recommendation,
555 gated_steps: r.gated_steps,
556 pending: r.pending,
557 }
558 }
559}
560
561/// Why generation stopped
562#[derive(Debug, Clone, Serialize, Deserialize)]
563pub enum FinishReason {
564 MaxTokens,
565 StopToken(String),
566 EndOfSequence,
567 Error(String),
568 Stop,
569}
570
571// =============================================================================
572// Training Mode Configuration (Phase D)
573// =============================================================================
574
575/// Model-level training mode configuration (embedded in config.json under "hyprstream_training")
576///
577/// This allows inference to automatically adapt models when enabled.
578/// The training mode is set via `hyprstream training set test_time_training`.
579#[derive(Debug, Clone, Serialize, Deserialize)]
580pub struct HyprstreamTrainingConfig {
581 /// Training mode: disabled, test_time_training, supervised
582 #[serde(default)]
583 pub mode: TrainingMode,
584
585 /// Target adapter to train (e.g., "01_coding")
586 pub target_adapter: Option<String>,
587
588 /// Learning rate for training
589 #[serde(default = "default_training_learning_rate")]
590 pub learning_rate: f64,
591
592 /// Batch size for training (used by supervised mode)
593 #[serde(default = "default_training_batch_size")]
594 pub batch_size: usize,
595
596 /// Training steps per cycle (used by supervised mode)
597 #[serde(default = "default_training_steps_per_cycle")]
598 pub steps_per_cycle: usize,
599
600 /// Minimum quality score to keep examples (0.0-1.0)
601 #[serde(default = "default_training_min_quality")]
602 pub min_quality_threshold: f32,
603
604 /// Enable training on base model weights (vs LoRA only)
605 #[serde(default)]
606 pub train_base_model: bool,
607
608 /// TTT-specific configuration (for TestTimeTraining mode)
609 #[serde(default)]
610 pub ttt: TTTTrainingConfig,
611
612 /// LoRA rank for TTT delta (default: 8)
613 #[serde(default = "default_lora_rank")]
614 pub lora_rank: usize,
615
616 /// LoRA alpha scaling factor (default: None, which means alpha = rank)
617 #[serde(default)]
618 pub lora_alpha: Option<f32>,
619
620 /// Target modules for LoRA adaptation (default: ["q_proj", "v_proj"])
621 #[serde(default = "default_target_modules")]
622 pub target_modules: Vec<String>,
623}
624
625/// TTT-specific configuration
626#[derive(Debug, Clone, Serialize, Deserialize)]
627pub struct TTTTrainingConfig {
628 /// Learning rate for TTT adaptation (higher than fine-tuning)
629 #[serde(default = "default_ttt_learning_rate")]
630 pub learning_rate: f64,
631
632 /// Number of gradient steps per input
633 #[serde(default = "default_ttt_gradient_steps")]
634 pub gradient_steps: u32,
635
636 /// Maximum gradient norm for clipping
637 #[serde(default = "default_ttt_max_grad_norm")]
638 pub max_grad_norm: f64,
639
640 /// Minimum input length (tokens) to trigger TTT
641 #[serde(default = "default_ttt_min_input_length")]
642 pub min_input_length: u32,
643
644 /// Maximum input length to process for TTT
645 #[serde(default = "default_ttt_max_context")]
646 pub max_ttt_context: u32,
647
648 /// Rank oracle configuration (optional — omit to disable runtime rank adaptation)
649 #[serde(default)]
650 pub rank_oracle: Option<crate::training::RankOracleConfig>,
651
652 /// Per-layer gradient gating (optional — enabled by default)
653 #[serde(default)]
654 pub gradient_gating: Option<crate::training::GradientGatingConfig>,
655}
656
657fn default_ttt_learning_rate() -> f64 {
658 3e-4
659}
660fn default_ttt_gradient_steps() -> u32 {
661 3
662}
663fn default_ttt_max_grad_norm() -> f64 {
664 1.0
665}
666fn default_ttt_min_input_length() -> u32 {
667 32
668}
669fn default_ttt_max_context() -> u32 {
670 512
671}
672
673impl Default for TTTTrainingConfig {
674 fn default() -> Self {
675 Self {
676 learning_rate: default_ttt_learning_rate(),
677 gradient_steps: default_ttt_gradient_steps(),
678 max_grad_norm: default_ttt_max_grad_norm(),
679 min_input_length: default_ttt_min_input_length(),
680 max_ttt_context: default_ttt_max_context(),
681 rank_oracle: None,
682 gradient_gating: None,
683 }
684 }
685}
686
687impl HyprstreamTrainingConfig {
688 /// Check if training is enabled (mode != Disabled)
689 pub fn is_enabled(&self) -> bool {
690 self.mode != TrainingMode::Disabled
691 }
692}
693
694impl Default for HyprstreamTrainingConfig {
695 fn default() -> Self {
696 Self {
697 mode: TrainingMode::default(),
698 target_adapter: None,
699 learning_rate: default_training_learning_rate(),
700 batch_size: default_training_batch_size(),
701 steps_per_cycle: default_training_steps_per_cycle(),
702 min_quality_threshold: default_training_min_quality(),
703 train_base_model: false,
704 ttt: TTTTrainingConfig::default(),
705 lora_rank: default_lora_rank(),
706 lora_alpha: None,
707 target_modules: default_target_modules(),
708 }
709 }
710}
711
712/// Training mode configuration
713#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
714#[serde(rename_all = "snake_case")]
715pub enum TrainingMode {
716 /// Training disabled (default)
717 #[default]
718 Disabled,
719 /// Test-Time Training: adapts to input context before generation
720 /// Research-valid approach based on TTT-E2E
721 TestTimeTraining,
722 /// Supervised training with explicit training data
723 Supervised,
724}
725
726// Default functions for HyprstreamTrainingConfig
727pub fn default_lora_rank() -> usize {
728 8
729}
730pub fn default_target_modules() -> Vec<String> {
731 vec!["q_proj".to_owned(), "v_proj".to_owned()]
732}
733fn default_training_learning_rate() -> f64 {
734 1e-5
735}
736fn default_training_batch_size() -> usize {
737 4
738}
739fn default_training_steps_per_cycle() -> usize {
740 10
741}
742fn default_training_min_quality() -> f32 {
743 0.3
744}
745
746/// Multi-GPU device resolution tests (#313) for the moved `RuntimeConfig`.
747#[cfg(test)]
748mod runtime_config_tests {
749 #![allow(clippy::unwrap_used)]
750
751 use super::*;
752
753 /// RAII guard to set/unset a process env var for the duration of a test
754 /// and restore the previous value. Local copy of the helper that stays
755 /// with the main crate's config tests; test scaffolding only.
756 struct EnvVarGuard {
757 key: String,
758 prev: Option<String>,
759 }
760 impl EnvVarGuard {
761 fn set(key: &str, val: &str) -> Self {
762 let prev = std::env::var(key).ok();
763 std::env::set_var(key, val);
764 Self {
765 key: key.to_owned(),
766 prev,
767 }
768 }
769 fn unset(key: &str) -> Self {
770 let prev = std::env::var(key).ok();
771 std::env::remove_var(key);
772 Self {
773 key: key.to_owned(),
774 prev,
775 }
776 }
777 }
778 impl Drop for EnvVarGuard {
779 fn drop(&mut self) {
780 match &self.prev {
781 Some(v) => std::env::set_var(&self.key, v),
782 None => std::env::remove_var(&self.key),
783 }
784 }
785 }
786
787 #[test]
788 #[allow(clippy::unwrap_used)]
789 fn parse_device_list_basic() {
790 assert_eq!(RuntimeConfig::parse_device_list("0,1").unwrap(), vec![0, 1]);
791
792 // Whitespace around entries is tolerated.
793
794 assert_eq!(
795 RuntimeConfig::parse_device_list(" 0 , 2 ").unwrap(),
796 vec![0, 2]
797 );
798
799 assert_eq!(RuntimeConfig::parse_device_list("3").unwrap(), vec![3]);
800 }
801
802 #[test]
803 fn parse_device_list_rejects_garbage() {
804 // Non-numeric, empty fields, and negatives are hard errors (no silent default).
805
806 assert!(RuntimeConfig::parse_device_list("0,foo").is_err());
807
808 assert!(RuntimeConfig::parse_device_list("0,").is_err());
809
810 assert!(RuntimeConfig::parse_device_list("0,,1").is_err());
811
812 assert!(RuntimeConfig::parse_device_list("-1").is_err());
813 }
814
815 #[test]
816 #[allow(clippy::unwrap_used)]
817 fn validate_index_set_dedup_and_empty() {
818 // Empty → None (auto-detect).
819
820 assert_eq!(RuntimeConfig::validate_index_set(vec![]).unwrap(), None);
821
822 // Duplicates → error.
823
824 assert!(RuntimeConfig::validate_index_set(vec![0, 0]).is_err());
825
826 // Distinct → Some, order preserved.
827
828 assert_eq!(
829 RuntimeConfig::validate_index_set(vec![2, 0, 1]).unwrap(),
830 Some(vec![2, 0, 1])
831 );
832 }
833
834 /// Serializes the two tests that mutate the shared `HYPRSTREAM_GPU_DEVICES`
835 /// process env var so they don't race under the parallel test runner.
836 static GPU_DEVICES_ENV_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
837
838 #[test]
839 #[allow(clippy::unwrap_used)]
840 fn resolve_uses_devices_field_and_legacy_fallback() {
841 let _serial = GPU_DEVICES_ENV_LOCK.lock();
842
843 // Guard against the env var leaking from the ambient environment so the
844
845 // struct-field/legacy precedence is exercised deterministically.
846
847 let _guard = EnvVarGuard::unset("HYPRSTREAM_GPU_DEVICES");
848
849 // Explicit multi-device field wins.
850
851 let mut cfg = RuntimeConfig::default();
852
853 cfg.devices = vec![0, 1];
854
855 cfg.gpu_device_id = Some(7);
856
857 assert_eq!(
858 cfg.resolve_explicit_multi_device_indices().unwrap(),
859 Some(vec![0, 1])
860 );
861
862 assert_eq!(cfg.resolve_device_indices().unwrap(), Some(vec![0, 1]));
863
864 // No explicit multi-device set → explicit resolver is None, but the full
865
866 // resolver falls back to the legacy single gpu_device_id.
867
868 let mut legacy = RuntimeConfig::default();
869
870 legacy.devices = vec![];
871
872 legacy.gpu_device_id = Some(3);
873
874 assert_eq!(
875 legacy.resolve_explicit_multi_device_indices().unwrap(),
876 None
877 );
878
879 assert_eq!(legacy.resolve_device_indices().unwrap(), Some(vec![3]));
880
881 // Nothing requested anywhere → None (auto-detect path preserved).
882
883 let mut none = RuntimeConfig::default();
884
885 none.devices = vec![];
886
887 none.gpu_device_id = None;
888
889 assert_eq!(none.resolve_device_indices().unwrap(), None);
890 }
891
892 #[test]
893 #[allow(clippy::unwrap_used)]
894 fn resolve_env_var_overrides_and_is_strict() {
895 let _serial = GPU_DEVICES_ENV_LOCK.lock();
896
897 let mut cfg = RuntimeConfig::default();
898
899 cfg.devices = vec![5];
900
901 cfg.gpu_device_id = Some(9);
902
903 {
904 let _g = EnvVarGuard::set("HYPRSTREAM_GPU_DEVICES", "0,1,2");
905
906 assert_eq!(
907 cfg.resolve_explicit_multi_device_indices().unwrap(),
908 Some(vec![0, 1, 2]),
909 "env var must override the devices field"
910 );
911 }
912
913 {
914 // Malformed env var is a hard error (not a silent fallback to field).
915
916 let _g = EnvVarGuard::set("HYPRSTREAM_GPU_DEVICES", "0,nope");
917
918 assert!(cfg.resolve_explicit_multi_device_indices().is_err());
919 }
920
921 {
922 // Duplicate in env var → error.
923
924 let _g = EnvVarGuard::set("HYPRSTREAM_GPU_DEVICES", "1,1");
925
926 assert!(cfg.resolve_explicit_multi_device_indices().is_err());
927 }
928
929 {
930 // Explicitly-empty env var is treated as unset (falls back to field).
931
932 let _g = EnvVarGuard::set("HYPRSTREAM_GPU_DEVICES", " ");
933
934 assert_eq!(
935 cfg.resolve_explicit_multi_device_indices().unwrap(),
936 Some(vec![5])
937 );
938 }
939 }
940
941 /// Serializes tests mutating the shared `HYPRSTREAM_STRICT_DEVICE` env var.
942 static STRICT_DEVICE_ENV_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
943
944 /// #315: strict_device defaults to fail-fast, and can be opted out via env.
945 #[test]
946 fn strict_device_defaults_to_true_and_respects_env() {
947 let _serial = STRICT_DEVICE_ENV_LOCK.lock();
948
949 {
950 let _g = EnvVarGuard::unset("HYPRSTREAM_STRICT_DEVICE");
951
952 assert!(
953 RuntimeConfig::default().strict_device,
954 "strict_device must default to true (safe default for multi-GPU)"
955 );
956 }
957
958 for falsy in ["0", "false", "no", "off"] {
959 let _g = EnvVarGuard::set("HYPRSTREAM_STRICT_DEVICE", falsy);
960
961 assert!(
962 !RuntimeConfig::default().strict_device,
963 "HYPRSTREAM_STRICT_DEVICE={falsy} must disable strict_device"
964 );
965 }
966
967 {
968 let _g = EnvVarGuard::set("HYPRSTREAM_STRICT_DEVICE", "1");
969
970 assert!(RuntimeConfig::default().strict_device);
971 }
972 }
973}