Skip to main content

hyprstream_inference/training/
ttt.rs

1//! Test-Time Training (TTT) Module
2//!
3//! Implements research-valid test-time training that adapts the model
4//! to the input distribution BEFORE generation using next-token prediction
5//! loss on the input context itself.
6//!
7//! # Research Foundation
8//!
9//! Based on TTT-E2E (arxiv.org/abs/2512.23675) and TLM approaches:
10//! - Train on INPUT context (not model outputs)
11//! - Use next-token prediction loss (not confidence heuristics)
12//! - Adapt BEFORE generation (not after)
13//!
14//! # Key Difference from SelfSupervisedTrainer
15//!
16//! | Aspect | SelfSupervised (old) | TTT (this) |
17//! |--------|---------------------|------------|
18//! | When | After generation | Before generation |
19//! | Data | Model outputs | User inputs |
20//! | Loss | Confidence heuristic | NTP cross-entropy |
21//! | Weights | Permanent LoRA | Scratch (temporary) |
22
23use anyhow::Result;
24use serde::{Deserialize, Serialize};
25use std::collections::HashMap;
26use std::panic::AssertUnwindSafe;
27use std::time::{Duration, Instant};
28use tch::{Device, Tensor};
29
30use super::tenant_delta::TenantDelta;
31use crate::runtime::TorchEngine;
32
33/// Configuration for Test-Time Training
34#[derive(Debug, Clone, Serialize, Deserialize)]
35pub struct TTTConfig {
36    /// Learning rate for TTT adaptation
37    /// Higher than fine-tuning since we do few steps
38    /// Recommended: 1e-4 to 5e-4
39    #[serde(default = "default_learning_rate")]
40    pub learning_rate: f64,
41
42    /// Number of gradient steps per input
43    /// More steps = better adaptation but higher latency
44    /// Recommended: 1-5
45    #[serde(default = "default_gradient_steps")]
46    pub gradient_steps: u32,
47
48    /// Maximum gradient norm for clipping
49    #[serde(default = "default_max_grad_norm")]
50    pub max_grad_norm: f64,
51
52    /// Minimum input length (tokens) to trigger TTT
53    /// Short inputs don't benefit from adaptation
54    #[serde(default = "default_min_input_length")]
55    pub min_input_length: u32,
56
57    /// Maximum input length to process for TTT
58    /// Truncates very long inputs to control latency
59    #[serde(default = "default_max_ttt_context")]
60    pub max_ttt_context: u32,
61
62    /// Whether TTT is enabled
63    #[serde(default = "default_enabled")]
64    pub enabled: bool,
65
66    // Adaptive TTT thresholds (perplexity gating)
67    /// Perplexity threshold below which adaptation is skipped (input already well-modeled)
68    #[serde(default = "default_tau_skip")]
69    pub tau_skip: f32,
70
71    /// Perplexity threshold for light adaptation (1 step)
72    #[serde(default = "default_tau_light")]
73    pub tau_light: f32,
74
75    /// Perplexity threshold for heavy adaptation (3 steps)
76    #[serde(default = "default_tau_heavy")]
77    pub tau_heavy: f32,
78
79    /// Perplexity ceiling above which adaptation is skipped (likely adversarial/garbage input).
80    /// Very high perplexity indicates the input is far from the model's training distribution
81    /// and gradient steps would be noise or potentially adversarial weight poisoning.
82    #[serde(default = "default_tau_reject")]
83    pub tau_reject: f32,
84
85    /// Minimum loss improvement to consider adaptation beneficial (confidence gate)
86    #[serde(default = "default_delta_min")]
87    pub delta_min: f32,
88
89    /// Whether to use adaptive step counts based on perplexity
90    #[serde(default = "default_adaptive_steps")]
91    pub adaptive_steps: bool,
92
93    /// Maximum wall-clock time for a single TTT adaptation pass (milliseconds).
94    /// When using ZMQ RPC (default timeout 30s), the default leaves headroom for
95    /// serialization, network transfer, and response processing.
96    #[serde(default = "default_max_adaptation_ms")]
97    pub max_adaptation_ms: u64,
98
99    /// Hard ceiling on gradient steps per request, regardless of client override.
100    /// Prevents unbounded compute from malicious or misconfigured clients.
101    #[serde(default = "default_max_gradient_steps")]
102    pub max_gradient_steps: usize,
103
104    /// Auto-rollback timeout for pending adaptations (milliseconds).
105    /// After a non-auto-commit training, the server holds the pre-adaptation
106    /// snapshot for this long before auto-rolling back. Increase for interactive
107    /// workflows (MCP, human-in-the-loop); decrease for programmatic API clients.
108    #[serde(default = "default_pending_rollback_ms")]
109    pub pending_rollback_ms: u64,
110}
111
112fn default_learning_rate() -> f64 {
113    3e-4
114}
115fn default_gradient_steps() -> u32 {
116    3
117}
118fn default_max_grad_norm() -> f64 {
119    1.0
120}
121fn default_min_input_length() -> u32 {
122    32
123}
124fn default_max_ttt_context() -> u32 {
125    512
126}
127fn default_enabled() -> bool {
128    true
129}
130fn default_tau_skip() -> f32 {
131    5.0
132}
133fn default_tau_light() -> f32 {
134    15.0
135}
136fn default_tau_heavy() -> f32 {
137    50.0
138}
139fn default_tau_reject() -> f32 {
140    200.0 // Perplexity > 200 is likely adversarial or garbage — skip TTT entirely
141}
142fn default_delta_min() -> f32 {
143    0.01
144}
145fn default_adaptive_steps() -> bool {
146    true
147}
148fn default_max_adaptation_ms() -> u64 {
149    30_000 // 30 seconds — sufficient for consumer GPUs (RTX 4090: ~7s/step)
150}
151fn default_max_gradient_steps() -> usize {
152    50 // Safety ceiling for client overrides; explicit training needs more steps
153}
154fn default_pending_rollback_ms() -> u64 {
155    60_000
156}
157
158/// Per-layer gradient gating configuration.
159///
160/// After the first gradient step, identifies low-signal layers via gradient
161/// norms and sets `requires_grad_(false)` on their parameters for subsequent
162/// steps. This saves backward-pass FLOPs and prevents Muon momentum
163/// from drifting on frozen layers.
164#[derive(Debug, Clone, Serialize, Deserialize)]
165pub struct GradientGatingConfig {
166    /// Enable per-layer gradient gating (default: true)
167    #[serde(default = "default_grad_gating_enabled")]
168    pub enabled: bool,
169    /// Minimum gradient L2 norm to keep updating a layer (default: 1e-5)
170    #[serde(default = "default_min_grad_norm")]
171    pub min_grad_norm: f64,
172    /// Number of initial adaptation steps before gating activates (default: 1)
173    #[serde(default = "default_grad_warmup")]
174    pub warmup_steps: usize,
175}
176
177fn default_grad_gating_enabled() -> bool {
178    true
179}
180fn default_min_grad_norm() -> f64 {
181    1e-5
182}
183fn default_grad_warmup() -> usize {
184    1
185}
186
187impl Default for GradientGatingConfig {
188    fn default() -> Self {
189        Self {
190            enabled: default_grad_gating_enabled(),
191            min_grad_norm: default_min_grad_norm(),
192            warmup_steps: default_grad_warmup(),
193        }
194    }
195}
196
197#[allow(dead_code)]
198fn compute_per_layer_grad_norms(vs: &tch::nn::VarStore) -> HashMap<String, f64> {
199    let mut norms: HashMap<String, f64> = HashMap::new();
200    for (name, var) in vs.variables() {
201        if !var.grad().defined() {
202            continue;
203        }
204        if let Some(key) = parse_varstore_key_to_delta_key(&name) {
205            let grad_sq = var.grad().norm().double_value(&[]).powi(2);
206            *norms.entry(key).or_insert(0.0) += grad_sq;
207        }
208    }
209    // Take sqrt of accumulated squared norms
210    for v in norms.values_mut() {
211        *v = v.sqrt();
212    }
213    norms
214}
215
216#[allow(dead_code)]
217fn identify_gated_layers(norms: &HashMap<String, f64>, threshold: f64) -> Vec<String> {
218    norms
219        .iter()
220        .filter(|(_, &norm)| norm <= threshold)
221        .map(|(key, _)| key.clone())
222        .collect()
223}
224
225#[allow(dead_code)]
226fn parse_varstore_key_to_delta_key(name: &str) -> Option<String> {
227    let parts: Vec<&str> = name.split('.').collect();
228    if parts.len() >= 3 {
229        let layer_part = parts[0]; // "layer_0"
230        let module = parts[1]; // "q_proj"
231        if let Some(idx_str) = layer_part.strip_prefix("layer_") {
232            if let Ok(idx) = idx_str.parse::<usize>() {
233                return Some(format!("{idx}.{module}"));
234            }
235        }
236    }
237    None
238}
239
240/// Collect per-key rank utilization, narrowed to effective ranks.
241fn collect_rank_utilization(delta: &TenantDelta) -> HashMap<String, f64> {
242    let _guard = tch::no_grad_guard();
243    delta
244        .lora_a
245        .iter()
246        .filter_map(|(key, a)| {
247            delta.lora_b.get(key).map(|b| {
248                let eff_rank = delta
249                    .effective_ranks
250                    .get(key)
251                    .copied()
252                    .unwrap_or_else(|| a.size()[0] as usize) as i64;
253                let a_eff = a.narrow(0, 0, eff_rank);
254                let b_eff = b.narrow(1, 0, eff_rank);
255                (
256                    key.clone(),
257                    crate::runtime::ttn_profile::delta_rank_utilization(&a_eff, &b_eff),
258                )
259            })
260        })
261        .collect()
262}
263
264/// Configuration for runtime rank adaptation oracle.
265#[derive(Debug, Clone, Serialize, Deserialize)]
266pub struct RankOracleConfig {
267    /// How many TTT adaptations between rank evaluations (default: 10)
268    #[serde(default = "default_adaptation_interval")]
269    pub adaptation_interval: usize,
270    /// Whether to auto-adapt effective ranks (default: false — just log recommendations)
271    #[serde(default)]
272    pub auto_adapt: bool,
273    /// Available rank levels for reallocation
274    #[serde(default = "default_rank_levels")]
275    pub rank_levels: Vec<usize>,
276    /// Utilization below this triggers rank decrease (default: 0.25)
277    #[serde(default = "default_low_util")]
278    pub low_utilization_threshold: f64,
279    /// Utilization above this triggers rank increase (default: 0.85)
280    #[serde(default = "default_high_util")]
281    pub high_utilization_threshold: f64,
282}
283
284fn default_adaptation_interval() -> usize {
285    10
286}
287fn default_rank_levels() -> Vec<usize> {
288    vec![1, 2, 4, 8]
289}
290fn default_low_util() -> f64 {
291    0.25
292}
293fn default_high_util() -> f64 {
294    0.85
295}
296
297impl Default for RankOracleConfig {
298    fn default() -> Self {
299        Self {
300            adaptation_interval: default_adaptation_interval(),
301            auto_adapt: false,
302            rank_levels: default_rank_levels(),
303            low_utilization_threshold: default_low_util(),
304            high_utilization_threshold: default_high_util(),
305        }
306    }
307}
308
309/// Per-tenant rank adaptation oracle.
310///
311/// Wraps `RankUtilizationTracker` and produces rank adaptation signals.
312/// Stored per-tenant (inside `TenantDelta`, not in `TestTimeTrainer`)
313/// because utilization history is tenant-specific.
314pub struct RankOracle {
315    pub(crate) config: RankOracleConfig,
316    tracker: crate::runtime::ttn_profile::RankUtilizationTracker,
317    observation_count: usize,
318}
319
320impl RankOracle {
321    pub fn new(config: RankOracleConfig) -> Self {
322        let window = config.adaptation_interval * 2;
323        Self {
324            tracker: crate::runtime::ttn_profile::RankUtilizationTracker::new(window),
325            config,
326            observation_count: 0,
327        }
328    }
329
330    pub fn observe(&mut self, utilizations: &HashMap<String, f64>) {
331        for (key, &val) in utilizations {
332            self.tracker.record(key, val);
333        }
334        self.observation_count += 1;
335    }
336
337    #[allow(clippy::manual_is_multiple_of)]
338    pub fn should_evaluate(&self) -> bool {
339        self.observation_count > 0 && self.observation_count % self.config.adaptation_interval == 0
340    }
341
342    pub fn recommend(&self) -> HashMap<String, crate::runtime::ttn_profile::RankSignal> {
343        self.tracker.rank_adaptation_signals(
344            self.config.low_utilization_threshold,
345            self.config.high_utilization_threshold,
346        )
347    }
348}
349
350impl Default for TTTConfig {
351    fn default() -> Self {
352        Self {
353            learning_rate: default_learning_rate(),
354            gradient_steps: default_gradient_steps(),
355            max_grad_norm: default_max_grad_norm(),
356            min_input_length: default_min_input_length(),
357            max_ttt_context: default_max_ttt_context(),
358            enabled: default_enabled(),
359            tau_skip: default_tau_skip(),
360            tau_light: default_tau_light(),
361            tau_heavy: default_tau_heavy(),
362            tau_reject: default_tau_reject(),
363            delta_min: default_delta_min(),
364            adaptive_steps: default_adaptive_steps(),
365            max_adaptation_ms: default_max_adaptation_ms(),
366            max_gradient_steps: default_max_gradient_steps(),
367            pending_rollback_ms: default_pending_rollback_ms(),
368        }
369    }
370}
371
372/// Result of TTT adaptation
373#[derive(Debug, Clone, serde::Serialize)]
374pub struct TTTResult {
375    /// Average loss across gradient steps
376    pub avg_loss: f32,
377
378    /// Loss improvement (initial - final)
379    pub loss_improvement: f32,
380
381    /// Number of gradient steps actually performed
382    pub steps_performed: usize,
383
384    /// Time spent on TTT (milliseconds)
385    pub adaptation_time_ms: u64,
386
387    /// Whether adaptation was skipped (input too short, etc.)
388    pub skipped: bool,
389
390    /// Reason for skipping (if skipped)
391    pub skip_reason: Option<String>,
392
393    // Advanced metrics (for ML practitioners)
394    /// Average gradient norm across steps
395    pub avg_grad_norm: f32,
396
397    /// Maximum gradient norm observed
398    pub max_grad_norm: f32,
399
400    /// Whether gradients were clipped
401    pub gradient_clipped: bool,
402
403    /// Number of tokens actually used for adaptation (after truncation)
404    pub tokens_used: usize,
405
406    /// Total tokens provided in input
407    pub tokens_provided: usize,
408
409    /// Whether input was truncated (exceeded max_ttt_context)
410    pub was_truncated: bool,
411
412    // Tenant-aware TTT fields
413    /// Initial perplexity before adaptation (exp(initial_loss))
414    pub initial_perplexity: f32,
415
416    /// Final perplexity after adaptation (exp(final_loss))
417    pub final_perplexity: f32,
418
419    /// Server's recommendation: true = commit, false = rollback
420    pub recommendation: bool,
421
422    /// Number of steps determined by perplexity gating
423    pub gated_steps: usize,
424
425    /// Whether adaptation is pending client commit/rollback
426    pub pending: bool,
427
428    /// Wall-clock time used for adaptation (milliseconds)
429    pub time_used_ms: u64,
430
431    /// Time budget that was enforced (milliseconds).
432    /// Client can derive `timed_out = time_used_ms >= time_budget_ms`.
433    pub time_budget_ms: u64,
434
435    /// Per-key rank utilization from this adaptation (empty if skipped)
436    pub rank_utilization: HashMap<String, f64>,
437
438    /// Layers gated (gradients frozen) during this adaptation
439    pub gated_layers: Vec<String>,
440
441    /// Rank adaptation signals from this evaluation (empty if not evaluated)
442    pub rank_signals: HashMap<String, crate::runtime::ttn_profile::RankSignal>,
443}
444
445impl TTTResult {
446    /// Create a result indicating adaptation was skipped
447    pub fn skipped(reason: &str) -> Self {
448        Self {
449            avg_loss: 0.0,
450            loss_improvement: 0.0,
451            steps_performed: 0,
452            adaptation_time_ms: 0,
453            skipped: true,
454            skip_reason: Some(reason.to_owned()),
455            avg_grad_norm: 0.0,
456            max_grad_norm: 0.0,
457            gradient_clipped: false,
458            tokens_used: 0,
459            tokens_provided: 0,
460            was_truncated: false,
461            initial_perplexity: 0.0,
462            final_perplexity: 0.0,
463            recommendation: false,
464            gated_steps: 0,
465            pending: false,
466            time_used_ms: 0,
467            time_budget_ms: 0,
468            rank_utilization: HashMap::new(),
469            gated_layers: Vec::new(),
470            rank_signals: HashMap::new(),
471        }
472    }
473}
474
475/// Per-request TTT overrides from the client
476#[derive(Debug, Clone)]
477pub struct TTTOverrides {
478    /// Override: enable/disable TTT for this request
479    pub enabled: Option<bool>,
480    /// Override: number of gradient steps (0 = skip)
481    pub gradient_steps: Option<u32>,
482    /// Override: learning rate
483    pub learning_rate: Option<f32>,
484    /// How to handle the adaptation result after training
485    pub adaptation_strategy: crate::training::adaptation_state::AdaptationStrategy,
486    /// Override: maximum wall-clock time for adaptation (milliseconds)
487    pub max_adaptation_ms: Option<u64>,
488}
489
490impl Default for TTTOverrides {
491    fn default() -> Self {
492        Self {
493            enabled: None,
494            gradient_steps: None,
495            learning_rate: None,
496            adaptation_strategy: crate::training::adaptation_state::AdaptationStrategy::Speculative,
497            max_adaptation_ms: None,
498        }
499    }
500}
501
502/// Context for TTT adaptation (for future cross-model verification)
503#[derive(Debug, Clone)]
504pub struct TTTContext {
505    /// Original input tokens
506    pub input_tokens: Vec<u32>,
507
508    /// Loss values at each TTT step
509    pub loss_history: Vec<f32>,
510
511    /// Timestamp of adaptation
512    pub adapted_at: Instant,
513}
514
515/// Test-Time Trainer
516///
517/// Adapts the model to input context BEFORE generation using next-token
518/// prediction loss on the input itself. This is the core TTT algorithm.
519///
520/// # Thread Safety
521///
522/// Each adaptation run is isolated - scratch weights are created per-request
523/// and do not persist between calls.
524///
525/// # Example
526///
527/// ```ignore
528/// let trainer = TestTimeTrainer::new(TTTConfig::default(), Device::Cuda(0));
529///
530/// // Adapt to input BEFORE generation
531/// let result = trainer.adapt(&engine, &input_tokens).await?;
532///
533/// // Now generate with adapted model
534/// let response = engine.generate(request)?;
535/// ```
536pub struct TestTimeTrainer {
537    pub config: TTTConfig,
538    pub gradient_gating: GradientGatingConfig,
539    device: Device,
540}
541
542impl TestTimeTrainer {
543    /// Create a new TTT trainer with the given configuration
544    pub fn new(config: TTTConfig, device: Device) -> Self {
545        Self {
546            config,
547            gradient_gating: GradientGatingConfig::default(),
548            device,
549        }
550    }
551
552    /// Create a new TTT trainer with gradient gating configuration
553    pub fn with_gradient_gating(mut self, gating: GradientGatingConfig) -> Self {
554        self.gradient_gating = gating;
555        self
556    }
557
558    /// Check if TTT is enabled
559    pub fn is_enabled(&self) -> bool {
560        self.config.enabled
561    }
562
563    /// Get the configuration
564    pub fn config(&self) -> &TTTConfig {
565        &self.config
566    }
567
568    /// Perform a single gradient step on a tenant delta using Muon
569    ///
570    /// Steps: backward → compute grad_norm → clip → muon_step → zero_grad
571    pub fn ttt_step(
572        &self,
573        loss: &Tensor,
574        delta: &mut TenantDelta,
575        learning_rate: Option<f64>,
576    ) -> Result<(f64, bool)> {
577        // Override learning rate for this step
578        if let Some(lr) = learning_rate {
579            delta.muon_config.lr = lr;
580        }
581
582        // Backward pass
583        loss.backward();
584
585        // Compute gradient norm for metrics
586        let grad_norm = {
587            let mut total_norm_sq = 0.0f64;
588            let variables = delta.vs.trainable_variables();
589
590            for var in &variables {
591                if var.grad().defined() {
592                    let grad = var.grad();
593                    total_norm_sq += grad.norm().double_value(&[]).powi(2);
594                }
595            }
596
597            total_norm_sq.sqrt()
598        };
599
600        let clipped = grad_norm > self.config.max_grad_norm;
601        if clipped {
602            // Manual gradient clipping (replaces optimizer.clip_grad_norm)
603            let clip_coef = self.config.max_grad_norm / (grad_norm + 1e-6);
604            let _guard = tch::no_grad_guard();
605            for var in delta.vs.trainable_variables() {
606                if var.grad().defined() {
607                    let clipped = &var.grad() * clip_coef;
608                    var.grad().copy_(&clipped);
609                }
610            }
611        }
612
613        // Muon step for all trainable parameters + zero gradients
614        self.optimizer_step(delta);
615
616        Ok((grad_norm, clipped))
617    }
618
619    /// Perform Muon optimization step on all trainable parameters and zero gradients.
620    fn optimizer_step(&self, delta: &mut TenantDelta) {
621        use super::muon::muon_step;
622        let _guard = tch::no_grad_guard();
623        let variables = delta.vs.variables();
624        let config = delta.muon_config.clone();
625        for (name, var) in &variables {
626            if var.requires_grad() && var.grad().defined() {
627                let state = delta.muon_states.entry(name.clone()).or_default();
628                muon_step(var, state, &config);
629            }
630        }
631        // Zero gradients
632        for var in variables.values() {
633            if var.grad().defined() {
634                let _ = var.grad().zero_();
635            }
636        }
637    }
638
639    /// Determine number of gradient steps based on perplexity gating
640    fn gate_steps(&self, perplexity: f32, overrides: &TTTOverrides) -> usize {
641        // Client override takes precedence
642        if let Some(steps) = overrides.gradient_steps {
643            return steps as usize;
644        }
645
646        if !self.config.adaptive_steps {
647            return self.config.gradient_steps as usize;
648        }
649
650        if perplexity > self.config.tau_reject {
651            tracing::info!("Skipping TTT: perplexity {:.1} exceeds tau_reject {:.1} (likely adversarial/garbage)",
652                perplexity, self.config.tau_reject);
653            0 // Input too far from distribution — skip
654        } else if perplexity < self.config.tau_skip {
655            0 // Input already well-modeled
656        } else if perplexity < self.config.tau_light {
657            1 // Light adaptation
658        } else if perplexity < self.config.tau_heavy {
659            3 // Moderate adaptation
660        } else {
661            5 // Heavy adaptation
662        }
663    }
664
665    /// Compute server's commit recommendation based on confidence gate
666    fn compute_recommendation(
667        &self,
668        loss_improvement: f32,
669        _gradient_clipped: bool,
670        initial_ppl: f32,
671        final_ppl: f32,
672    ) -> bool {
673        // Gradient clipping is informational, not a veto — clipping is normal
674        // during early adaptation steps, especially with freshly initialized deltas.
675        loss_improvement > self.config.delta_min && final_ppl < initial_ppl
676    }
677
678    /// Adapt a tenant delta to input context using perplexity-gated TTT
679    ///
680    /// This is the tenant-aware entry point for TTT. It:
681    /// 1. Snapshots delta state (for rollback)
682    /// 2. Computes initial NTP loss → perplexity
683    /// 3. Gates step count based on perplexity
684    /// 4. Runs SGD gradient loop on the tenant delta
685    /// 5. Returns metrics (does NOT auto-commit — client decides)
686    ///
687    /// # Arguments
688    /// * `engine` - TorchEngine with loaded model
689    /// * `delta` - Tenant's LoRA delta (mutable for SGD updates)
690    /// * `input_tokens` - Tokenized input context
691    /// * `overrides` - Per-request TTT overrides from client
692    ///
693    /// # Returns
694    /// (TTTResult, pre_snapshot) where pre_snapshot is the state before adaptation
695    pub fn adapt_tenant(
696        &self,
697        engine: &TorchEngine,
698        delta: &mut TenantDelta,
699        input_tokens: &[u32],
700        overrides: &TTTOverrides,
701    ) -> Result<(TTTResult, HashMap<String, Tensor>)> {
702        let start = Instant::now();
703
704        // Check if enabled (respect override)
705        let enabled = overrides.enabled.unwrap_or(self.config.enabled);
706        if !enabled {
707            return Ok((TTTResult::skipped("TTT disabled"), HashMap::new()));
708        }
709
710        // Check minimum length
711        if input_tokens.len() < self.config.min_input_length as usize {
712            return Ok((
713                TTTResult::skipped(&format!(
714                    "Input too short: {} < {} tokens",
715                    input_tokens.len(),
716                    self.config.min_input_length
717                )),
718                HashMap::new(),
719            ));
720        }
721
722        // Snapshot delta state before adaptation (for rollback)
723        let pre_snapshot = delta.extract_state_dict();
724
725        // Snapshot SSM states before TTT loop (C4 fix: prevents training data from
726        // accumulating into the inference recurrent state for Qwen3.5 GDN layers).
727        // No-op for non-Qwen3.5 models.
728        let ssm_snapshot = engine.snapshot_ssm_states();
729
730        // Track whether input was truncated
731        let tokens_provided = input_tokens.len();
732        let was_truncated = tokens_provided > self.config.max_ttt_context as usize;
733
734        // Truncate if too long (use last N tokens for recency)
735        let tokens: Vec<u32> = if was_truncated {
736            input_tokens[input_tokens.len() - self.config.max_ttt_context as usize..].to_vec()
737        } else {
738            input_tokens.to_vec()
739        };
740        let tokens_used = tokens.len();
741
742        // Compute initial NTP loss
743        let initial_loss_tensor = self.compute_ntp_loss_with_delta(engine, delta, &tokens)?;
744        let initial_loss: f32 = initial_loss_tensor.double_value(&[]) as f32;
745        let initial_perplexity = initial_loss.exp();
746
747        // Gate step count based on perplexity, then clamp to max
748        let gated_steps = self
749            .gate_steps(initial_perplexity, overrides)
750            .min(self.config.max_gradient_steps);
751
752        let time_budget_ms = overrides
753            .max_adaptation_ms
754            .unwrap_or(self.config.max_adaptation_ms);
755
756        if gated_steps == 0 {
757            // Restore SSM states — initial NTP loss forward pass may have mutated them.
758            engine.restore_ssm_states(ssm_snapshot);
759            return Ok((
760                TTTResult {
761                    avg_loss: initial_loss,
762                    loss_improvement: 0.0,
763                    steps_performed: 0,
764                    adaptation_time_ms: start.elapsed().as_millis() as u64,
765                    skipped: true,
766                    skip_reason: Some(format!(
767                        "Perplexity {:.1} below skip threshold {:.1}",
768                        initial_perplexity, self.config.tau_skip
769                    )),
770                    avg_grad_norm: 0.0,
771                    max_grad_norm: 0.0,
772                    gradient_clipped: false,
773                    tokens_used,
774                    tokens_provided,
775                    was_truncated,
776                    initial_perplexity,
777                    final_perplexity: initial_perplexity,
778                    recommendation: false,
779                    gated_steps: 0,
780                    pending: false,
781                    time_used_ms: start.elapsed().as_millis() as u64,
782                    time_budget_ms,
783                    rank_utilization: HashMap::new(),
784                    gated_layers: Vec::new(),
785                    rank_signals: HashMap::new(),
786                },
787                pre_snapshot,
788            ));
789        }
790
791        // Determine learning rate (respect override)
792        let lr = overrides
793            .learning_rate
794            .map(|r| r as f64)
795            .unwrap_or(delta.learning_rate);
796
797        let budget = Duration::from_millis(time_budget_ms);
798
799        // Snapshot Muon momentum buffers for rollback (matches weight snapshot above)
800        let muon_snapshot = super::muon::snapshot_muon_states(&delta.muon_states);
801
802        // Run gradient loop with catch_unwind for panic safety.
803        // On panic, delta is restored from pre_snapshot.
804        let loop_result = std::panic::catch_unwind(AssertUnwindSafe(|| {
805            self.adapt_tenant_inner(
806                engine,
807                delta,
808                &tokens,
809                gated_steps,
810                lr,
811                &start,
812                budget,
813                initial_loss_tensor,
814            )
815        }));
816
817        match loop_result {
818            Ok(Ok(inner)) => {
819                let (
820                    actual_steps,
821                    losses,
822                    grad_norms,
823                    any_clipped,
824                    final_loss,
825                    final_perplexity,
826                    rank_utils,
827                    gated_layer_keys,
828                ) = inner;
829
830                let avg_loss = losses.iter().sum::<f32>() / losses.len() as f32;
831                let loss_improvement = initial_loss - final_loss;
832
833                let avg_grad_norm = if !grad_norms.is_empty() {
834                    grad_norms.iter().sum::<f32>() / grad_norms.len() as f32
835                } else {
836                    0.0
837                };
838                let max_grad_norm_val = grad_norms.iter().copied().fold(0.0f32, f32::max);
839
840                let recommendation = self.compute_recommendation(
841                    loss_improvement,
842                    any_clipped,
843                    initial_perplexity,
844                    final_perplexity,
845                );
846
847                // Stats are NOT updated here — they are updated at commit time
848                // by handle_commit_adaptation (pending path) or by train_step
849                // (auto-commit path). Updating here would double-count because
850                // handle_commit_adaptation also increments on commit.
851
852                let time_used_ms = start.elapsed().as_millis() as u64;
853
854                // Per-tenant rank oracle: observe utilization and optionally adapt
855                let rank_signals = if let Some(ref mut oracle) = delta.rank_oracle {
856                    oracle.observe(&rank_utils);
857                    if oracle.should_evaluate() {
858                        let signals = oracle.recommend();
859                        if oracle.config.auto_adapt {
860                            use crate::runtime::ttn_profile::RankSignal;
861                            for (key, signal) in &signals {
862                                if let Some(current) = delta.effective_rank(key) {
863                                    match signal {
864                                        RankSignal::Decrease => {
865                                            let new_rank = (current / 2).max(1);
866                                            delta.set_effective_rank(key, new_rank);
867                                            tracing::info!(key = %key, from = current, to = new_rank, "Rank oracle: decreased");
868                                        }
869                                        RankSignal::Increase => {
870                                            let new_rank = current * 2;
871                                            delta.set_effective_rank(key, new_rank);
872                                            let actual =
873                                                delta.effective_rank(key).unwrap_or(new_rank);
874                                            tracing::info!(key = %key, from = current, to = actual, "Rank oracle: increased");
875                                        }
876                                        RankSignal::Hold => {}
877                                    }
878                                }
879                            }
880                        }
881                        signals
882                    } else {
883                        HashMap::new()
884                    }
885                } else {
886                    HashMap::new()
887                };
888
889                // Restore SSM states so TTT training data does not pollute inference recurrent state.
890                engine.restore_ssm_states(ssm_snapshot);
891
892                // The LoRA weights were mutated in place (same tenant key, new
893                // effective weights) — bump the weight epoch so session KV
894                // computed under the pre-adaptation weights is cleared as stale
895                // on the next reuse (#1254 F1 fix).
896                if actual_steps > 0 {
897                    delta.bump_weight_epoch();
898                }
899
900                Ok((
901                    TTTResult {
902                        avg_loss,
903                        loss_improvement,
904                        steps_performed: actual_steps,
905                        adaptation_time_ms: time_used_ms,
906                        skipped: false,
907                        skip_reason: None,
908                        avg_grad_norm,
909                        max_grad_norm: max_grad_norm_val,
910                        gradient_clipped: any_clipped,
911                        tokens_used,
912                        tokens_provided,
913                        was_truncated,
914                        initial_perplexity,
915                        final_perplexity,
916                        recommendation,
917                        gated_steps,
918                        pending: true, // Awaiting client commit/rollback
919                        time_used_ms,
920                        time_budget_ms,
921                        rank_utilization: rank_utils,
922                        gated_layers: gated_layer_keys,
923                        rank_signals,
924                    },
925                    pre_snapshot,
926                ))
927            }
928            Ok(Err(e)) => {
929                // Gradient loop returned an error — restore delta, momentum, SSM states,
930                // and requires_grad (gradient gating may have frozen some variables).
931                tracing::warn!(
932                    "TTT adapt_tenant gradient loop error, restoring delta: {}",
933                    e
934                );
935                let _ = delta.load_state_dict(&pre_snapshot);
936                super::muon::restore_muon_states(&mut delta.muon_states, &muon_snapshot);
937                Self::restore_requires_grad(delta);
938                delta.zero_grad();
939                engine.restore_ssm_states(ssm_snapshot);
940                Err(e)
941            }
942            Err(panic_info) => {
943                // Panic during gradient loop — restore delta, momentum, SSM states,
944                // and requires_grad. A panic mid-backward/step leaves stale gradient
945                // accumulation and corrupted Muon momentum buffers.
946                tracing::error!("TTT adapt_tenant panicked during gradient loop, restoring delta");
947                let _ = delta.load_state_dict(&pre_snapshot);
948                super::muon::restore_muon_states(&mut delta.muon_states, &muon_snapshot);
949                Self::restore_requires_grad(delta);
950                delta.zero_grad();
951                engine.restore_ssm_states(ssm_snapshot);
952                let msg = if let Some(s) = panic_info.downcast_ref::<String>() {
953                    s.clone()
954                } else if let Some(s) = panic_info.downcast_ref::<&str>() {
955                    (*s).to_owned()
956                } else {
957                    "unknown panic".to_owned()
958                };
959                Err(anyhow::anyhow!("TTT adaptation panicked: {}", msg))
960            }
961        }
962    }
963
964    #[allow(dead_code)]
965    fn apply_gradient_gating(
966        &self,
967        delta: &TenantDelta,
968        step: usize,
969        total_steps: usize,
970    ) -> Vec<String> {
971        if !self.gradient_gating.enabled || total_steps <= self.gradient_gating.warmup_steps {
972            return Vec::new();
973        }
974        if step < self.gradient_gating.warmup_steps {
975            return Vec::new();
976        }
977        let grad_norms_map = compute_per_layer_grad_norms(&delta.vs);
978        let gated = identify_gated_layers(&grad_norms_map, self.gradient_gating.min_grad_norm);
979        if !gated.is_empty() {
980            let _guard = tch::no_grad_guard();
981            for (name, var) in delta.vs.variables() {
982                if let Some(key) = parse_varstore_key_to_delta_key(&name) {
983                    if gated.contains(&key) {
984                        let _ = var.set_requires_grad(false);
985                    }
986                }
987            }
988            tracing::debug!(
989                gated_count = gated.len(),
990                "Gradient gating: froze low-signal layers"
991            );
992        }
993        gated
994    }
995
996    /// Restore requires_grad on all variables (undo gradient gating).
997    fn restore_requires_grad(delta: &TenantDelta) {
998        let _guard = tch::no_grad_guard();
999        for (_, var) in delta.vs.variables() {
1000            if !var.requires_grad() {
1001                let _ = var.set_requires_grad(true);
1002            }
1003        }
1004    }
1005
1006    /// Inner gradient loop for adapt_tenant (extracted for catch_unwind).
1007    /// Receives the pre-computed `initial_loss_tensor` from the caller to avoid
1008    /// a redundant forward pass.
1009    #[allow(clippy::type_complexity)]
1010    fn adapt_tenant_inner(
1011        &self,
1012        engine: &TorchEngine,
1013        delta: &mut TenantDelta,
1014        tokens: &[u32],
1015        gated_steps: usize,
1016        lr: f64,
1017        start: &Instant,
1018        budget: Duration,
1019        initial_loss_tensor: Tensor,
1020    ) -> Result<(
1021        usize,
1022        Vec<f32>,
1023        Vec<f32>,
1024        bool,
1025        f32,
1026        f32,
1027        HashMap<String, f64>,
1028        Vec<String>,
1029    )> {
1030        let initial_loss = initial_loss_tensor.double_value(&[]) as f32;
1031        let mut losses = vec![initial_loss];
1032        let mut grad_norms = Vec::with_capacity(gated_steps);
1033        let mut any_clipped = false;
1034
1035        // First step (always runs at least one)
1036        let (gn, clipped) = self.ttt_step(&initial_loss_tensor, delta, Some(lr))?;
1037        grad_norms.push(gn as f32);
1038        if clipped {
1039            any_clipped = true;
1040        }
1041        let mut actual_steps = 1;
1042
1043        // Skip gradient gating — short TTT runs (1-5 steps) need all layers
1044        // active for meaningful adaptation.
1045        let gated_layer_keys: Vec<String> = Vec::new();
1046
1047        // Remaining steps with time budget check BEFORE each step
1048        for _ in 1..gated_steps {
1049            if start.elapsed() >= budget {
1050                tracing::warn!(
1051                    "TTT: time budget exhausted after {}/{} steps ({:.0}ms >= {}ms)",
1052                    actual_steps,
1053                    gated_steps,
1054                    start.elapsed().as_millis(),
1055                    budget.as_millis()
1056                );
1057                break;
1058            }
1059
1060            let loss = self.compute_ntp_loss_with_delta(engine, delta, tokens)?;
1061            losses.push(loss.double_value(&[]) as f32);
1062
1063            let (gn, clipped) = self.ttt_step(&loss, delta, Some(lr))?;
1064            grad_norms.push(gn as f32);
1065            if clipped {
1066                any_clipped = true;
1067            }
1068            actual_steps += 1;
1069
1070            // Delta rank utilization monitoring on last step + every 10 cumulative steps.
1071            // Last-step monitoring ensures short TTT runs (1-5 steps) always get at least
1072            // one observation. SVD on [rank×rank] is ~μs.
1073            let is_last_step = actual_steps == gated_steps;
1074            if is_last_step || (delta.accumulated_steps + actual_steps as u64).is_multiple_of(10) {
1075                for (key, util) in &collect_rank_utilization(delta) {
1076                    tracing::debug!(key = %key, utilization = %util, "Delta rank utilization");
1077                }
1078            }
1079        }
1080
1081        // Compute final loss for perplexity
1082        let final_loss_tensor = self.compute_ntp_loss_with_delta(engine, delta, tokens)?;
1083        let final_loss = final_loss_tensor.double_value(&[]) as f32;
1084        let final_perplexity = final_loss.exp();
1085
1086        let rank_utils = collect_rank_utilization(delta);
1087
1088        // Restore requires_grad on all vars (undo gradient gating for next TTT call)
1089        if !gated_layer_keys.is_empty() {
1090            Self::restore_requires_grad(delta);
1091        }
1092
1093        Ok((
1094            actual_steps,
1095            losses,
1096            grad_norms,
1097            any_clipped,
1098            final_loss,
1099            final_perplexity,
1100            rank_utils,
1101            gated_layer_keys,
1102        ))
1103    }
1104
1105    /// Compute NTP loss with a tenant delta applied
1106    ///
1107    /// Runs a forward pass that injects the delta's A/B matrices after q_proj/v_proj
1108    /// projections inside each attention layer. This creates a differentiable path
1109    /// from the loss back to the delta parameters, enabling gradient-based training.
1110    fn compute_ntp_loss_with_delta(
1111        &self,
1112        engine: &TorchEngine,
1113        delta: &TenantDelta,
1114        tokens: &[u32],
1115    ) -> Result<Tensor> {
1116        // Defensive: ensure GradMode is enabled for training regardless of leaked state.
1117        // If a previous inference panic inside tch::no_grad leaked disabled GradMode,
1118        // this ensures gradients still flow through delta's A/B matrices.
1119        tch::with_grad(|| {
1120            let tokens_i64: Vec<i64> = tokens.iter().map(|&t| t as i64).collect();
1121
1122            let input_ids = Tensor::from_slice(&tokens_i64)
1123                .to_device(self.device)
1124                .unsqueeze(0); // [1, seq_len]
1125
1126            // Forward pass with delta injection — gradients flow through delta's A/B matrices
1127            let logits = engine.forward_with_delta(&input_ids, delta)?;
1128
1129            let seq_len = tokens.len() as i64;
1130            let vocab_size = logits.size()[2];
1131
1132            // Shift for next-token prediction
1133            let pred_logits = logits.narrow(1, 0, seq_len - 1).reshape([-1, vocab_size]);
1134
1135            let target_ids = input_ids.narrow(1, 1, seq_len - 1).reshape([-1]);
1136
1137            let loss = pred_logits.cross_entropy_loss::<Tensor>(
1138                &target_ids,
1139                None,
1140                tch::Reduction::Mean,
1141                -100,
1142                0.0,
1143            );
1144
1145            Ok(loss)
1146        })
1147    }
1148
1149    /// Run pure training steps on a tenant delta without generation
1150    ///
1151    /// Used by the `trainStep` API endpoint for explicit training.
1152    /// Steps are clamped to `max_gradient_steps` and time-budgeted.
1153    pub fn train_step(
1154        &self,
1155        engine: &TorchEngine,
1156        delta: &mut TenantDelta,
1157        input_tokens: &[u32],
1158        gradient_steps: usize,
1159        learning_rate: Option<f64>,
1160        max_adaptation_ms: Option<u64>,
1161    ) -> Result<TTTResult> {
1162        let start = Instant::now();
1163
1164        let tokens_provided = input_tokens.len();
1165        let was_truncated = tokens_provided > self.config.max_ttt_context as usize;
1166
1167        let tokens: Vec<u32> = if was_truncated {
1168            input_tokens[input_tokens.len() - self.config.max_ttt_context as usize..].to_vec()
1169        } else {
1170            input_tokens.to_vec()
1171        };
1172        let tokens_used = tokens.len();
1173
1174        if tokens_used < 2 {
1175            return Ok(TTTResult::skipped("Input too short for NTP loss"));
1176        }
1177
1178        // Clamp steps to hard ceiling
1179        let clamped_steps = gradient_steps.min(self.config.max_gradient_steps);
1180        let time_budget_ms = max_adaptation_ms.unwrap_or(self.config.max_adaptation_ms);
1181        let budget = Duration::from_millis(time_budget_ms);
1182
1183        let lr = learning_rate.unwrap_or(delta.learning_rate);
1184
1185        // Snapshot for panic recovery
1186        let pre_snapshot = delta.extract_state_dict();
1187        let muon_snapshot = super::muon::snapshot_muon_states(&delta.muon_states);
1188
1189        // Snapshot SSM states before training loop (prevents training data from polluting inference state).
1190        let ssm_snapshot = engine.snapshot_ssm_states();
1191
1192        let loop_result = std::panic::catch_unwind(AssertUnwindSafe(|| {
1193            self.train_step_inner(engine, delta, &tokens, clamped_steps, lr, &start, budget)
1194        }));
1195
1196        match loop_result {
1197            Ok(Ok(inner)) => {
1198                let (
1199                    actual_steps,
1200                    losses,
1201                    grad_norms,
1202                    any_clipped,
1203                    final_loss,
1204                    final_perplexity,
1205                    initial_loss,
1206                    initial_perplexity,
1207                    gated_layer_keys,
1208                ) = inner;
1209
1210                let avg_loss = losses.iter().sum::<f32>() / losses.len() as f32;
1211                let loss_improvement = initial_loss - final_loss;
1212                let avg_grad_norm = if !grad_norms.is_empty() {
1213                    grad_norms.iter().sum::<f32>() / grad_norms.len() as f32
1214                } else {
1215                    0.0
1216                };
1217                let max_grad_norm_val = grad_norms.iter().copied().fold(0.0f32, f32::max);
1218
1219                let recommendation = self.compute_recommendation(
1220                    loss_improvement,
1221                    any_clipped,
1222                    initial_perplexity,
1223                    final_perplexity,
1224                );
1225
1226                let time_used_ms = start.elapsed().as_millis() as u64;
1227
1228                let rank_utils = collect_rank_utilization(delta);
1229
1230                // Feed rank oracle (same logic as adapt_tenant)
1231                let rank_signals = if let Some(ref mut oracle) = delta.rank_oracle {
1232                    oracle.observe(&rank_utils);
1233                    if oracle.should_evaluate() {
1234                        let signals = oracle.recommend();
1235                        if oracle.config.auto_adapt {
1236                            use crate::runtime::ttn_profile::RankSignal;
1237                            for (key, signal) in &signals {
1238                                if let Some(current) = delta.effective_rank(key) {
1239                                    match signal {
1240                                        RankSignal::Decrease => {
1241                                            let new_rank = (current / 2).max(1);
1242                                            delta.set_effective_rank(key, new_rank);
1243                                            tracing::info!(key = %key, from = current, to = new_rank, "Rank oracle: decreased");
1244                                        }
1245                                        RankSignal::Increase => {
1246                                            let new_rank = current * 2;
1247                                            delta.set_effective_rank(key, new_rank);
1248                                            let actual =
1249                                                delta.effective_rank(key).unwrap_or(new_rank);
1250                                            tracing::info!(key = %key, from = current, to = actual, "Rank oracle: increased");
1251                                        }
1252                                        RankSignal::Hold => {}
1253                                    }
1254                                }
1255                            }
1256                        }
1257                        signals
1258                    } else {
1259                        HashMap::new()
1260                    }
1261                } else {
1262                    HashMap::new()
1263                };
1264
1265                // Restore SSM states so training data does not pollute inference recurrent state.
1266                engine.restore_ssm_states(ssm_snapshot);
1267
1268                // The LoRA weights were mutated in place — bump the weight epoch
1269                // so session KV computed under the pre-training weights is
1270                // cleared as stale on the next reuse (#1254 F1 fix).
1271                if actual_steps > 0 {
1272                    delta.bump_weight_epoch();
1273                }
1274
1275                Ok(TTTResult {
1276                    avg_loss,
1277                    loss_improvement,
1278                    steps_performed: actual_steps,
1279                    adaptation_time_ms: time_used_ms,
1280                    skipped: false,
1281                    skip_reason: None,
1282                    avg_grad_norm,
1283                    max_grad_norm: max_grad_norm_val,
1284                    gradient_clipped: any_clipped,
1285                    tokens_used,
1286                    tokens_provided,
1287                    was_truncated,
1288                    initial_perplexity,
1289                    final_perplexity,
1290                    recommendation,
1291                    gated_steps: clamped_steps,
1292                    pending: false, // trainStep commits immediately based on auto_commit
1293                    time_used_ms,
1294                    time_budget_ms,
1295                    rank_utilization: rank_utils,
1296                    gated_layers: gated_layer_keys,
1297                    rank_signals,
1298                })
1299            }
1300            Ok(Err(e)) => {
1301                tracing::warn!("TTT train_step gradient loop error, restoring delta: {}", e);
1302                let _ = delta.load_state_dict(&pre_snapshot);
1303                super::muon::restore_muon_states(&mut delta.muon_states, &muon_snapshot);
1304                Self::restore_requires_grad(delta);
1305                delta.zero_grad();
1306                engine.restore_ssm_states(ssm_snapshot);
1307                Err(e)
1308            }
1309            Err(panic_info) => {
1310                tracing::error!("TTT train_step panicked during gradient loop, restoring delta");
1311                let _ = delta.load_state_dict(&pre_snapshot);
1312                super::muon::restore_muon_states(&mut delta.muon_states, &muon_snapshot);
1313                Self::restore_requires_grad(delta);
1314                delta.zero_grad();
1315                engine.restore_ssm_states(ssm_snapshot);
1316                let msg = if let Some(s) = panic_info.downcast_ref::<String>() {
1317                    s.clone()
1318                } else if let Some(s) = panic_info.downcast_ref::<&str>() {
1319                    (*s).to_owned()
1320                } else {
1321                    "unknown panic".to_owned()
1322                };
1323                Err(anyhow::anyhow!("TTT training panicked: {}", msg))
1324            }
1325        }
1326    }
1327
1328    /// Inner gradient loop for train_step (extracted for catch_unwind)
1329    #[allow(clippy::type_complexity)]
1330    fn train_step_inner(
1331        &self,
1332        engine: &TorchEngine,
1333        delta: &mut TenantDelta,
1334        tokens: &[u32],
1335        clamped_steps: usize,
1336        lr: f64,
1337        start: &Instant,
1338        budget: Duration,
1339    ) -> Result<(
1340        usize,
1341        Vec<f32>,
1342        Vec<f32>,
1343        bool,
1344        f32,
1345        f32,
1346        f32,
1347        f32,
1348        Vec<String>,
1349    )> {
1350        // Compute initial loss
1351        let initial_loss_tensor = self.compute_ntp_loss_with_delta(engine, delta, tokens)?;
1352        let initial_loss = initial_loss_tensor.double_value(&[]) as f32;
1353        let initial_perplexity = initial_loss.exp();
1354
1355        let mut losses = vec![initial_loss];
1356        let mut grad_norms = Vec::with_capacity(clamped_steps);
1357        let mut any_clipped = false;
1358
1359        // First step (always runs)
1360        let (gn, clipped) = self.ttt_step(&initial_loss_tensor, delta, Some(lr))?;
1361        grad_norms.push(gn as f32);
1362        if clipped {
1363            any_clipped = true;
1364        }
1365        let mut actual_steps = 1;
1366
1367        // Skip gradient gating in explicit train_step — short training runs
1368        // (3-5 steps) need all layers active for meaningful adaptation.
1369        let gated_layer_keys: Vec<String> = Vec::new();
1370
1371        // Remaining steps with time budget check
1372        for _ in 1..clamped_steps {
1373            if start.elapsed() >= budget {
1374                tracing::warn!(
1375                    "TTT train_step: time budget exhausted after {}/{} steps ({:.0}ms >= {}ms)",
1376                    actual_steps,
1377                    clamped_steps,
1378                    start.elapsed().as_millis(),
1379                    budget.as_millis()
1380                );
1381                break;
1382            }
1383
1384            let loss = self.compute_ntp_loss_with_delta(engine, delta, tokens)?;
1385            losses.push(loss.double_value(&[]) as f32);
1386            let (gn, clipped) = self.ttt_step(&loss, delta, Some(lr))?;
1387            grad_norms.push(gn as f32);
1388            if clipped {
1389                any_clipped = true;
1390            }
1391            actual_steps += 1;
1392        }
1393
1394        // Final loss
1395        let final_loss_tensor = self.compute_ntp_loss_with_delta(engine, delta, tokens)?;
1396        let final_loss = final_loss_tensor.double_value(&[]) as f32;
1397        let final_perplexity = final_loss.exp();
1398
1399        // Restore requires_grad (undo gradient gating)
1400        if !gated_layer_keys.is_empty() {
1401            Self::restore_requires_grad(delta);
1402        }
1403
1404        Ok((
1405            actual_steps,
1406            losses,
1407            grad_norms,
1408            any_clipped,
1409            final_loss,
1410            final_perplexity,
1411            initial_loss,
1412            initial_perplexity,
1413            gated_layer_keys,
1414        ))
1415    }
1416}
1417
1418// Thread safety - TestTimeTrainer is Send + Sync because it only holds
1419// config data and device info, not mutable state
1420unsafe impl Send for TestTimeTrainer {}
1421unsafe impl Sync for TestTimeTrainer {}
1422
1423/// Trait for TTT verifiers (future extension)
1424///
1425/// This trait allows plugging in different verification strategies:
1426/// - Cross-model verification (use another model to verify)
1427/// - Cycle consistency (for multimodal)
1428/// - Execution verification (for code)
1429#[async_trait::async_trait]
1430pub trait TTTVerifier: Send + Sync {
1431    /// Verify that TTT adaptation is beneficial
1432    ///
1433    /// Returns true if adaptation should be accepted
1434    async fn verify(&self, context: &TTTContext, engine: &TorchEngine) -> Result<bool>;
1435}
1436
1437/// No-op verifier that always accepts adaptation
1438pub struct NoOpVerifier;
1439
1440#[async_trait::async_trait]
1441impl TTTVerifier for NoOpVerifier {
1442    async fn verify(&self, _context: &TTTContext, _engine: &TorchEngine) -> Result<bool> {
1443        Ok(true)
1444    }
1445}
1446
1447#[cfg(test)]
1448#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
1449mod tests {
1450    use super::*;
1451
1452    #[test]
1453    fn test_ttt_config_default() {
1454        let config = TTTConfig::default();
1455        assert!((config.learning_rate - 3e-4).abs() < 1e-10);
1456        assert_eq!(config.gradient_steps, 3);
1457        assert_eq!(config.min_input_length, 32);
1458        assert_eq!(config.max_ttt_context, 512);
1459        assert!(config.enabled);
1460        // Defaults sized for consumer GPUs (per RTX 4090 ~7s/step):
1461        // see `default_max_adaptation_ms` and `default_max_gradient_steps`.
1462        assert_eq!(config.max_adaptation_ms, 30_000);
1463        assert_eq!(config.max_gradient_steps, 50);
1464        assert!((config.tau_reject - 200.0).abs() < 0.1);
1465    }
1466
1467    #[test]
1468    fn test_ttt_result_skipped() {
1469        let result = TTTResult::skipped("test reason");
1470        assert!(result.skipped);
1471        assert_eq!(result.skip_reason, Some("test reason".to_owned()));
1472        assert_eq!(result.steps_performed, 0);
1473    }
1474
1475    #[test]
1476    fn test_ttt_config_serde() {
1477        let json = r#"{
1478            "learning_rate": 0.001,
1479            "gradient_steps": 5,
1480            "enabled": false
1481        }"#;
1482
1483        let config: TTTConfig = serde_json::from_str(json).unwrap();
1484        assert!((config.learning_rate - 0.001).abs() < 1e-10);
1485        assert_eq!(config.gradient_steps, 5);
1486        assert!(!config.enabled);
1487        // Defaults should apply for missing fields
1488        assert_eq!(config.min_input_length, 32);
1489    }
1490
1491    #[test]
1492    fn test_gradient_gating_config_defaults() {
1493        let config = GradientGatingConfig::default();
1494        assert!(config.enabled);
1495        assert!(config.min_grad_norm > 0.0);
1496        assert!(config.warmup_steps > 0);
1497    }
1498
1499    #[test]
1500    fn test_identify_gated_layers() {
1501        let mut norms = HashMap::new();
1502        norms.insert("0.q_proj".to_owned(), 1e-3); // above threshold
1503        norms.insert("0.v_proj".to_owned(), 1e-8); // below threshold
1504        norms.insert("1.q_proj".to_owned(), 5e-6); // at threshold
1505        let gated = identify_gated_layers(&norms, 1e-5);
1506        assert!(!gated.contains(&"0.q_proj".to_owned())); // not gated (high norm)
1507        assert!(gated.contains(&"0.v_proj".to_owned())); // gated (low norm)
1508        assert!(gated.contains(&"1.q_proj".to_owned())); // gated (at/below threshold)
1509    }
1510
1511    #[test]
1512    fn test_gradient_gating_with_real_backward() {
1513        use tch::{nn::VarStore, Kind};
1514        let vs = VarStore::new(Device::Cpu);
1515        let root = vs.root();
1516        let a = root
1517            .sub("layer_0")
1518            .sub("q_proj")
1519            .kaiming_uniform("lora_a", &[4, 8]);
1520        let b = root.sub("layer_0").sub("q_proj").zeros("lora_b", &[8, 4]);
1521        let x = Tensor::randn([2, 8], (Kind::Float, Device::Cpu));
1522
1523        // Forward + backward with requires_grad=true
1524        let out = x.matmul(&a.tr()).matmul(&b.tr());
1525        let loss = out.sum(Kind::Float);
1526        loss.backward();
1527        assert!(a.grad().defined());
1528
1529        // Zero grads, disable requires_grad, forward + backward
1530        for (_, v) in vs.variables() {
1531            if v.grad().defined() {
1532                let _ = v.grad().zero_();
1533            }
1534        }
1535        let _ = a.set_requires_grad(false);
1536        let out2 = x.matmul(&a.tr()).matmul(&b.tr());
1537        let loss2 = out2.sum(Kind::Float);
1538        loss2.backward();
1539        // a should have NO gradient (frozen)
1540        assert!(!a.grad().defined() || a.grad().abs().sum(Kind::Double).double_value(&[]) == 0.0);
1541        // b should still have gradient
1542        assert!(b.grad().defined());
1543
1544        // Re-enable
1545        let _ = a.set_requires_grad(true);
1546    }
1547
1548    #[test]
1549    fn test_parse_varstore_key() {
1550        assert_eq!(
1551            parse_varstore_key_to_delta_key("layer_0.q_proj.lora_a"),
1552            Some("0.q_proj".to_owned())
1553        );
1554        assert_eq!(
1555            parse_varstore_key_to_delta_key("layer_23.v_proj.lora_b"),
1556            Some("23.v_proj".to_owned())
1557        );
1558        assert_eq!(parse_varstore_key_to_delta_key("invalid_key"), None);
1559    }
1560
1561    #[test]
1562    fn test_rank_oracle_config_defaults() {
1563        let config = RankOracleConfig::default();
1564        assert!(config.adaptation_interval > 0);
1565        assert!(!config.auto_adapt); // conservative default
1566        assert!(!config.rank_levels.is_empty());
1567    }
1568
1569    #[test]
1570    fn test_rank_oracle_produces_signals() {
1571        use crate::runtime::ttn_profile::RankSignal;
1572        let mut oracle = RankOracle::new(RankOracleConfig::default());
1573        // Simulate: feed low utilization for 10 steps
1574        for _ in 0..10 {
1575            let mut utils = HashMap::new();
1576            utils.insert("0.q_proj".to_owned(), 0.1);
1577            utils.insert("0.v_proj".to_owned(), 0.9);
1578            oracle.observe(&utils);
1579        }
1580        let signals = oracle.recommend();
1581        assert_eq!(signals.get("0.q_proj"), Some(&RankSignal::Decrease));
1582        assert_eq!(signals.get("0.v_proj"), Some(&RankSignal::Increase));
1583    }
1584
1585    #[test]
1586    fn test_rank_oracle_should_evaluate() {
1587        let mut oracle = RankOracle::new(RankOracleConfig {
1588            adaptation_interval: 3,
1589            ..Default::default()
1590        });
1591        let utils: HashMap<String, f64> = HashMap::new();
1592        oracle.observe(&utils); // 1
1593        assert!(!oracle.should_evaluate());
1594        oracle.observe(&utils); // 2
1595        assert!(!oracle.should_evaluate());
1596        oracle.observe(&utils); // 3
1597        assert!(oracle.should_evaluate());
1598    }
1599}