1use anyhow::Result;
24use serde::{Deserialize, Serialize};
25use std::collections::HashMap;
26use std::panic::AssertUnwindSafe;
27use std::time::{Duration, Instant};
28use tch::{Device, Tensor};
29
30use super::tenant_delta::TenantDelta;
31use crate::runtime::TorchEngine;
32
33#[derive(Debug, Clone, Serialize, Deserialize)]
35pub struct TTTConfig {
36 #[serde(default = "default_learning_rate")]
40 pub learning_rate: f64,
41
42 #[serde(default = "default_gradient_steps")]
46 pub gradient_steps: u32,
47
48 #[serde(default = "default_max_grad_norm")]
50 pub max_grad_norm: f64,
51
52 #[serde(default = "default_min_input_length")]
55 pub min_input_length: u32,
56
57 #[serde(default = "default_max_ttt_context")]
60 pub max_ttt_context: u32,
61
62 #[serde(default = "default_enabled")]
64 pub enabled: bool,
65
66 #[serde(default = "default_tau_skip")]
69 pub tau_skip: f32,
70
71 #[serde(default = "default_tau_light")]
73 pub tau_light: f32,
74
75 #[serde(default = "default_tau_heavy")]
77 pub tau_heavy: f32,
78
79 #[serde(default = "default_tau_reject")]
83 pub tau_reject: f32,
84
85 #[serde(default = "default_delta_min")]
87 pub delta_min: f32,
88
89 #[serde(default = "default_adaptive_steps")]
91 pub adaptive_steps: bool,
92
93 #[serde(default = "default_max_adaptation_ms")]
97 pub max_adaptation_ms: u64,
98
99 #[serde(default = "default_max_gradient_steps")]
102 pub max_gradient_steps: usize,
103
104 #[serde(default = "default_pending_rollback_ms")]
109 pub pending_rollback_ms: u64,
110}
111
112fn default_learning_rate() -> f64 {
113 3e-4
114}
115fn default_gradient_steps() -> u32 {
116 3
117}
118fn default_max_grad_norm() -> f64 {
119 1.0
120}
121fn default_min_input_length() -> u32 {
122 32
123}
124fn default_max_ttt_context() -> u32 {
125 512
126}
127fn default_enabled() -> bool {
128 true
129}
130fn default_tau_skip() -> f32 {
131 5.0
132}
133fn default_tau_light() -> f32 {
134 15.0
135}
136fn default_tau_heavy() -> f32 {
137 50.0
138}
139fn default_tau_reject() -> f32 {
140 200.0 }
142fn default_delta_min() -> f32 {
143 0.01
144}
145fn default_adaptive_steps() -> bool {
146 true
147}
148fn default_max_adaptation_ms() -> u64 {
149 30_000 }
151fn default_max_gradient_steps() -> usize {
152 50 }
154fn default_pending_rollback_ms() -> u64 {
155 60_000
156}
157
158#[derive(Debug, Clone, Serialize, Deserialize)]
165pub struct GradientGatingConfig {
166 #[serde(default = "default_grad_gating_enabled")]
168 pub enabled: bool,
169 #[serde(default = "default_min_grad_norm")]
171 pub min_grad_norm: f64,
172 #[serde(default = "default_grad_warmup")]
174 pub warmup_steps: usize,
175}
176
177fn default_grad_gating_enabled() -> bool {
178 true
179}
180fn default_min_grad_norm() -> f64 {
181 1e-5
182}
183fn default_grad_warmup() -> usize {
184 1
185}
186
187impl Default for GradientGatingConfig {
188 fn default() -> Self {
189 Self {
190 enabled: default_grad_gating_enabled(),
191 min_grad_norm: default_min_grad_norm(),
192 warmup_steps: default_grad_warmup(),
193 }
194 }
195}
196
197#[allow(dead_code)]
198fn compute_per_layer_grad_norms(vs: &tch::nn::VarStore) -> HashMap<String, f64> {
199 let mut norms: HashMap<String, f64> = HashMap::new();
200 for (name, var) in vs.variables() {
201 if !var.grad().defined() {
202 continue;
203 }
204 if let Some(key) = parse_varstore_key_to_delta_key(&name) {
205 let grad_sq = var.grad().norm().double_value(&[]).powi(2);
206 *norms.entry(key).or_insert(0.0) += grad_sq;
207 }
208 }
209 for v in norms.values_mut() {
211 *v = v.sqrt();
212 }
213 norms
214}
215
216#[allow(dead_code)]
217fn identify_gated_layers(norms: &HashMap<String, f64>, threshold: f64) -> Vec<String> {
218 norms
219 .iter()
220 .filter(|(_, &norm)| norm <= threshold)
221 .map(|(key, _)| key.clone())
222 .collect()
223}
224
225#[allow(dead_code)]
226fn parse_varstore_key_to_delta_key(name: &str) -> Option<String> {
227 let parts: Vec<&str> = name.split('.').collect();
228 if parts.len() >= 3 {
229 let layer_part = parts[0]; let module = parts[1]; if let Some(idx_str) = layer_part.strip_prefix("layer_") {
232 if let Ok(idx) = idx_str.parse::<usize>() {
233 return Some(format!("{idx}.{module}"));
234 }
235 }
236 }
237 None
238}
239
240fn collect_rank_utilization(delta: &TenantDelta) -> HashMap<String, f64> {
242 let _guard = tch::no_grad_guard();
243 delta
244 .lora_a
245 .iter()
246 .filter_map(|(key, a)| {
247 delta.lora_b.get(key).map(|b| {
248 let eff_rank = delta
249 .effective_ranks
250 .get(key)
251 .copied()
252 .unwrap_or_else(|| a.size()[0] as usize) as i64;
253 let a_eff = a.narrow(0, 0, eff_rank);
254 let b_eff = b.narrow(1, 0, eff_rank);
255 (
256 key.clone(),
257 crate::runtime::ttn_profile::delta_rank_utilization(&a_eff, &b_eff),
258 )
259 })
260 })
261 .collect()
262}
263
264#[derive(Debug, Clone, Serialize, Deserialize)]
266pub struct RankOracleConfig {
267 #[serde(default = "default_adaptation_interval")]
269 pub adaptation_interval: usize,
270 #[serde(default)]
272 pub auto_adapt: bool,
273 #[serde(default = "default_rank_levels")]
275 pub rank_levels: Vec<usize>,
276 #[serde(default = "default_low_util")]
278 pub low_utilization_threshold: f64,
279 #[serde(default = "default_high_util")]
281 pub high_utilization_threshold: f64,
282}
283
284fn default_adaptation_interval() -> usize {
285 10
286}
287fn default_rank_levels() -> Vec<usize> {
288 vec![1, 2, 4, 8]
289}
290fn default_low_util() -> f64 {
291 0.25
292}
293fn default_high_util() -> f64 {
294 0.85
295}
296
297impl Default for RankOracleConfig {
298 fn default() -> Self {
299 Self {
300 adaptation_interval: default_adaptation_interval(),
301 auto_adapt: false,
302 rank_levels: default_rank_levels(),
303 low_utilization_threshold: default_low_util(),
304 high_utilization_threshold: default_high_util(),
305 }
306 }
307}
308
309pub struct RankOracle {
315 pub(crate) config: RankOracleConfig,
316 tracker: crate::runtime::ttn_profile::RankUtilizationTracker,
317 observation_count: usize,
318}
319
320impl RankOracle {
321 pub fn new(config: RankOracleConfig) -> Self {
322 let window = config.adaptation_interval * 2;
323 Self {
324 tracker: crate::runtime::ttn_profile::RankUtilizationTracker::new(window),
325 config,
326 observation_count: 0,
327 }
328 }
329
330 pub fn observe(&mut self, utilizations: &HashMap<String, f64>) {
331 for (key, &val) in utilizations {
332 self.tracker.record(key, val);
333 }
334 self.observation_count += 1;
335 }
336
337 #[allow(clippy::manual_is_multiple_of)]
338 pub fn should_evaluate(&self) -> bool {
339 self.observation_count > 0 && self.observation_count % self.config.adaptation_interval == 0
340 }
341
342 pub fn recommend(&self) -> HashMap<String, crate::runtime::ttn_profile::RankSignal> {
343 self.tracker.rank_adaptation_signals(
344 self.config.low_utilization_threshold,
345 self.config.high_utilization_threshold,
346 )
347 }
348}
349
350impl Default for TTTConfig {
351 fn default() -> Self {
352 Self {
353 learning_rate: default_learning_rate(),
354 gradient_steps: default_gradient_steps(),
355 max_grad_norm: default_max_grad_norm(),
356 min_input_length: default_min_input_length(),
357 max_ttt_context: default_max_ttt_context(),
358 enabled: default_enabled(),
359 tau_skip: default_tau_skip(),
360 tau_light: default_tau_light(),
361 tau_heavy: default_tau_heavy(),
362 tau_reject: default_tau_reject(),
363 delta_min: default_delta_min(),
364 adaptive_steps: default_adaptive_steps(),
365 max_adaptation_ms: default_max_adaptation_ms(),
366 max_gradient_steps: default_max_gradient_steps(),
367 pending_rollback_ms: default_pending_rollback_ms(),
368 }
369 }
370}
371
372#[derive(Debug, Clone, serde::Serialize)]
374pub struct TTTResult {
375 pub avg_loss: f32,
377
378 pub loss_improvement: f32,
380
381 pub steps_performed: usize,
383
384 pub adaptation_time_ms: u64,
386
387 pub skipped: bool,
389
390 pub skip_reason: Option<String>,
392
393 pub avg_grad_norm: f32,
396
397 pub max_grad_norm: f32,
399
400 pub gradient_clipped: bool,
402
403 pub tokens_used: usize,
405
406 pub tokens_provided: usize,
408
409 pub was_truncated: bool,
411
412 pub initial_perplexity: f32,
415
416 pub final_perplexity: f32,
418
419 pub recommendation: bool,
421
422 pub gated_steps: usize,
424
425 pub pending: bool,
427
428 pub time_used_ms: u64,
430
431 pub time_budget_ms: u64,
434
435 pub rank_utilization: HashMap<String, f64>,
437
438 pub gated_layers: Vec<String>,
440
441 pub rank_signals: HashMap<String, crate::runtime::ttn_profile::RankSignal>,
443}
444
445impl TTTResult {
446 pub fn skipped(reason: &str) -> Self {
448 Self {
449 avg_loss: 0.0,
450 loss_improvement: 0.0,
451 steps_performed: 0,
452 adaptation_time_ms: 0,
453 skipped: true,
454 skip_reason: Some(reason.to_owned()),
455 avg_grad_norm: 0.0,
456 max_grad_norm: 0.0,
457 gradient_clipped: false,
458 tokens_used: 0,
459 tokens_provided: 0,
460 was_truncated: false,
461 initial_perplexity: 0.0,
462 final_perplexity: 0.0,
463 recommendation: false,
464 gated_steps: 0,
465 pending: false,
466 time_used_ms: 0,
467 time_budget_ms: 0,
468 rank_utilization: HashMap::new(),
469 gated_layers: Vec::new(),
470 rank_signals: HashMap::new(),
471 }
472 }
473}
474
475#[derive(Debug, Clone)]
477pub struct TTTOverrides {
478 pub enabled: Option<bool>,
480 pub gradient_steps: Option<u32>,
482 pub learning_rate: Option<f32>,
484 pub adaptation_strategy: crate::training::adaptation_state::AdaptationStrategy,
486 pub max_adaptation_ms: Option<u64>,
488}
489
490impl Default for TTTOverrides {
491 fn default() -> Self {
492 Self {
493 enabled: None,
494 gradient_steps: None,
495 learning_rate: None,
496 adaptation_strategy: crate::training::adaptation_state::AdaptationStrategy::Speculative,
497 max_adaptation_ms: None,
498 }
499 }
500}
501
502#[derive(Debug, Clone)]
504pub struct TTTContext {
505 pub input_tokens: Vec<u32>,
507
508 pub loss_history: Vec<f32>,
510
511 pub adapted_at: Instant,
513}
514
515pub struct TestTimeTrainer {
537 pub config: TTTConfig,
538 pub gradient_gating: GradientGatingConfig,
539 device: Device,
540}
541
542impl TestTimeTrainer {
543 pub fn new(config: TTTConfig, device: Device) -> Self {
545 Self {
546 config,
547 gradient_gating: GradientGatingConfig::default(),
548 device,
549 }
550 }
551
552 pub fn with_gradient_gating(mut self, gating: GradientGatingConfig) -> Self {
554 self.gradient_gating = gating;
555 self
556 }
557
558 pub fn is_enabled(&self) -> bool {
560 self.config.enabled
561 }
562
563 pub fn config(&self) -> &TTTConfig {
565 &self.config
566 }
567
568 pub fn ttt_step(
572 &self,
573 loss: &Tensor,
574 delta: &mut TenantDelta,
575 learning_rate: Option<f64>,
576 ) -> Result<(f64, bool)> {
577 if let Some(lr) = learning_rate {
579 delta.muon_config.lr = lr;
580 }
581
582 loss.backward();
584
585 let grad_norm = {
587 let mut total_norm_sq = 0.0f64;
588 let variables = delta.vs.trainable_variables();
589
590 for var in &variables {
591 if var.grad().defined() {
592 let grad = var.grad();
593 total_norm_sq += grad.norm().double_value(&[]).powi(2);
594 }
595 }
596
597 total_norm_sq.sqrt()
598 };
599
600 let clipped = grad_norm > self.config.max_grad_norm;
601 if clipped {
602 let clip_coef = self.config.max_grad_norm / (grad_norm + 1e-6);
604 let _guard = tch::no_grad_guard();
605 for var in delta.vs.trainable_variables() {
606 if var.grad().defined() {
607 let clipped = &var.grad() * clip_coef;
608 var.grad().copy_(&clipped);
609 }
610 }
611 }
612
613 self.optimizer_step(delta);
615
616 Ok((grad_norm, clipped))
617 }
618
619 fn optimizer_step(&self, delta: &mut TenantDelta) {
621 use super::muon::muon_step;
622 let _guard = tch::no_grad_guard();
623 let variables = delta.vs.variables();
624 let config = delta.muon_config.clone();
625 for (name, var) in &variables {
626 if var.requires_grad() && var.grad().defined() {
627 let state = delta.muon_states.entry(name.clone()).or_default();
628 muon_step(var, state, &config);
629 }
630 }
631 for var in variables.values() {
633 if var.grad().defined() {
634 let _ = var.grad().zero_();
635 }
636 }
637 }
638
639 fn gate_steps(&self, perplexity: f32, overrides: &TTTOverrides) -> usize {
641 if let Some(steps) = overrides.gradient_steps {
643 return steps as usize;
644 }
645
646 if !self.config.adaptive_steps {
647 return self.config.gradient_steps as usize;
648 }
649
650 if perplexity > self.config.tau_reject {
651 tracing::info!("Skipping TTT: perplexity {:.1} exceeds tau_reject {:.1} (likely adversarial/garbage)",
652 perplexity, self.config.tau_reject);
653 0 } else if perplexity < self.config.tau_skip {
655 0 } else if perplexity < self.config.tau_light {
657 1 } else if perplexity < self.config.tau_heavy {
659 3 } else {
661 5 }
663 }
664
665 fn compute_recommendation(
667 &self,
668 loss_improvement: f32,
669 _gradient_clipped: bool,
670 initial_ppl: f32,
671 final_ppl: f32,
672 ) -> bool {
673 loss_improvement > self.config.delta_min && final_ppl < initial_ppl
676 }
677
678 pub fn adapt_tenant(
696 &self,
697 engine: &TorchEngine,
698 delta: &mut TenantDelta,
699 input_tokens: &[u32],
700 overrides: &TTTOverrides,
701 ) -> Result<(TTTResult, HashMap<String, Tensor>)> {
702 let start = Instant::now();
703
704 let enabled = overrides.enabled.unwrap_or(self.config.enabled);
706 if !enabled {
707 return Ok((TTTResult::skipped("TTT disabled"), HashMap::new()));
708 }
709
710 if input_tokens.len() < self.config.min_input_length as usize {
712 return Ok((
713 TTTResult::skipped(&format!(
714 "Input too short: {} < {} tokens",
715 input_tokens.len(),
716 self.config.min_input_length
717 )),
718 HashMap::new(),
719 ));
720 }
721
722 let pre_snapshot = delta.extract_state_dict();
724
725 let ssm_snapshot = engine.snapshot_ssm_states();
729
730 let tokens_provided = input_tokens.len();
732 let was_truncated = tokens_provided > self.config.max_ttt_context as usize;
733
734 let tokens: Vec<u32> = if was_truncated {
736 input_tokens[input_tokens.len() - self.config.max_ttt_context as usize..].to_vec()
737 } else {
738 input_tokens.to_vec()
739 };
740 let tokens_used = tokens.len();
741
742 let initial_loss_tensor = self.compute_ntp_loss_with_delta(engine, delta, &tokens)?;
744 let initial_loss: f32 = initial_loss_tensor.double_value(&[]) as f32;
745 let initial_perplexity = initial_loss.exp();
746
747 let gated_steps = self
749 .gate_steps(initial_perplexity, overrides)
750 .min(self.config.max_gradient_steps);
751
752 let time_budget_ms = overrides
753 .max_adaptation_ms
754 .unwrap_or(self.config.max_adaptation_ms);
755
756 if gated_steps == 0 {
757 engine.restore_ssm_states(ssm_snapshot);
759 return Ok((
760 TTTResult {
761 avg_loss: initial_loss,
762 loss_improvement: 0.0,
763 steps_performed: 0,
764 adaptation_time_ms: start.elapsed().as_millis() as u64,
765 skipped: true,
766 skip_reason: Some(format!(
767 "Perplexity {:.1} below skip threshold {:.1}",
768 initial_perplexity, self.config.tau_skip
769 )),
770 avg_grad_norm: 0.0,
771 max_grad_norm: 0.0,
772 gradient_clipped: false,
773 tokens_used,
774 tokens_provided,
775 was_truncated,
776 initial_perplexity,
777 final_perplexity: initial_perplexity,
778 recommendation: false,
779 gated_steps: 0,
780 pending: false,
781 time_used_ms: start.elapsed().as_millis() as u64,
782 time_budget_ms,
783 rank_utilization: HashMap::new(),
784 gated_layers: Vec::new(),
785 rank_signals: HashMap::new(),
786 },
787 pre_snapshot,
788 ));
789 }
790
791 let lr = overrides
793 .learning_rate
794 .map(|r| r as f64)
795 .unwrap_or(delta.learning_rate);
796
797 let budget = Duration::from_millis(time_budget_ms);
798
799 let muon_snapshot = super::muon::snapshot_muon_states(&delta.muon_states);
801
802 let loop_result = std::panic::catch_unwind(AssertUnwindSafe(|| {
805 self.adapt_tenant_inner(
806 engine,
807 delta,
808 &tokens,
809 gated_steps,
810 lr,
811 &start,
812 budget,
813 initial_loss_tensor,
814 )
815 }));
816
817 match loop_result {
818 Ok(Ok(inner)) => {
819 let (
820 actual_steps,
821 losses,
822 grad_norms,
823 any_clipped,
824 final_loss,
825 final_perplexity,
826 rank_utils,
827 gated_layer_keys,
828 ) = inner;
829
830 let avg_loss = losses.iter().sum::<f32>() / losses.len() as f32;
831 let loss_improvement = initial_loss - final_loss;
832
833 let avg_grad_norm = if !grad_norms.is_empty() {
834 grad_norms.iter().sum::<f32>() / grad_norms.len() as f32
835 } else {
836 0.0
837 };
838 let max_grad_norm_val = grad_norms.iter().copied().fold(0.0f32, f32::max);
839
840 let recommendation = self.compute_recommendation(
841 loss_improvement,
842 any_clipped,
843 initial_perplexity,
844 final_perplexity,
845 );
846
847 let time_used_ms = start.elapsed().as_millis() as u64;
853
854 let rank_signals = if let Some(ref mut oracle) = delta.rank_oracle {
856 oracle.observe(&rank_utils);
857 if oracle.should_evaluate() {
858 let signals = oracle.recommend();
859 if oracle.config.auto_adapt {
860 use crate::runtime::ttn_profile::RankSignal;
861 for (key, signal) in &signals {
862 if let Some(current) = delta.effective_rank(key) {
863 match signal {
864 RankSignal::Decrease => {
865 let new_rank = (current / 2).max(1);
866 delta.set_effective_rank(key, new_rank);
867 tracing::info!(key = %key, from = current, to = new_rank, "Rank oracle: decreased");
868 }
869 RankSignal::Increase => {
870 let new_rank = current * 2;
871 delta.set_effective_rank(key, new_rank);
872 let actual =
873 delta.effective_rank(key).unwrap_or(new_rank);
874 tracing::info!(key = %key, from = current, to = actual, "Rank oracle: increased");
875 }
876 RankSignal::Hold => {}
877 }
878 }
879 }
880 }
881 signals
882 } else {
883 HashMap::new()
884 }
885 } else {
886 HashMap::new()
887 };
888
889 engine.restore_ssm_states(ssm_snapshot);
891
892 if actual_steps > 0 {
897 delta.bump_weight_epoch();
898 }
899
900 Ok((
901 TTTResult {
902 avg_loss,
903 loss_improvement,
904 steps_performed: actual_steps,
905 adaptation_time_ms: time_used_ms,
906 skipped: false,
907 skip_reason: None,
908 avg_grad_norm,
909 max_grad_norm: max_grad_norm_val,
910 gradient_clipped: any_clipped,
911 tokens_used,
912 tokens_provided,
913 was_truncated,
914 initial_perplexity,
915 final_perplexity,
916 recommendation,
917 gated_steps,
918 pending: true, time_used_ms,
920 time_budget_ms,
921 rank_utilization: rank_utils,
922 gated_layers: gated_layer_keys,
923 rank_signals,
924 },
925 pre_snapshot,
926 ))
927 }
928 Ok(Err(e)) => {
929 tracing::warn!(
932 "TTT adapt_tenant gradient loop error, restoring delta: {}",
933 e
934 );
935 let _ = delta.load_state_dict(&pre_snapshot);
936 super::muon::restore_muon_states(&mut delta.muon_states, &muon_snapshot);
937 Self::restore_requires_grad(delta);
938 delta.zero_grad();
939 engine.restore_ssm_states(ssm_snapshot);
940 Err(e)
941 }
942 Err(panic_info) => {
943 tracing::error!("TTT adapt_tenant panicked during gradient loop, restoring delta");
947 let _ = delta.load_state_dict(&pre_snapshot);
948 super::muon::restore_muon_states(&mut delta.muon_states, &muon_snapshot);
949 Self::restore_requires_grad(delta);
950 delta.zero_grad();
951 engine.restore_ssm_states(ssm_snapshot);
952 let msg = if let Some(s) = panic_info.downcast_ref::<String>() {
953 s.clone()
954 } else if let Some(s) = panic_info.downcast_ref::<&str>() {
955 (*s).to_owned()
956 } else {
957 "unknown panic".to_owned()
958 };
959 Err(anyhow::anyhow!("TTT adaptation panicked: {}", msg))
960 }
961 }
962 }
963
964 #[allow(dead_code)]
965 fn apply_gradient_gating(
966 &self,
967 delta: &TenantDelta,
968 step: usize,
969 total_steps: usize,
970 ) -> Vec<String> {
971 if !self.gradient_gating.enabled || total_steps <= self.gradient_gating.warmup_steps {
972 return Vec::new();
973 }
974 if step < self.gradient_gating.warmup_steps {
975 return Vec::new();
976 }
977 let grad_norms_map = compute_per_layer_grad_norms(&delta.vs);
978 let gated = identify_gated_layers(&grad_norms_map, self.gradient_gating.min_grad_norm);
979 if !gated.is_empty() {
980 let _guard = tch::no_grad_guard();
981 for (name, var) in delta.vs.variables() {
982 if let Some(key) = parse_varstore_key_to_delta_key(&name) {
983 if gated.contains(&key) {
984 let _ = var.set_requires_grad(false);
985 }
986 }
987 }
988 tracing::debug!(
989 gated_count = gated.len(),
990 "Gradient gating: froze low-signal layers"
991 );
992 }
993 gated
994 }
995
996 fn restore_requires_grad(delta: &TenantDelta) {
998 let _guard = tch::no_grad_guard();
999 for (_, var) in delta.vs.variables() {
1000 if !var.requires_grad() {
1001 let _ = var.set_requires_grad(true);
1002 }
1003 }
1004 }
1005
1006 #[allow(clippy::type_complexity)]
1010 fn adapt_tenant_inner(
1011 &self,
1012 engine: &TorchEngine,
1013 delta: &mut TenantDelta,
1014 tokens: &[u32],
1015 gated_steps: usize,
1016 lr: f64,
1017 start: &Instant,
1018 budget: Duration,
1019 initial_loss_tensor: Tensor,
1020 ) -> Result<(
1021 usize,
1022 Vec<f32>,
1023 Vec<f32>,
1024 bool,
1025 f32,
1026 f32,
1027 HashMap<String, f64>,
1028 Vec<String>,
1029 )> {
1030 let initial_loss = initial_loss_tensor.double_value(&[]) as f32;
1031 let mut losses = vec![initial_loss];
1032 let mut grad_norms = Vec::with_capacity(gated_steps);
1033 let mut any_clipped = false;
1034
1035 let (gn, clipped) = self.ttt_step(&initial_loss_tensor, delta, Some(lr))?;
1037 grad_norms.push(gn as f32);
1038 if clipped {
1039 any_clipped = true;
1040 }
1041 let mut actual_steps = 1;
1042
1043 let gated_layer_keys: Vec<String> = Vec::new();
1046
1047 for _ in 1..gated_steps {
1049 if start.elapsed() >= budget {
1050 tracing::warn!(
1051 "TTT: time budget exhausted after {}/{} steps ({:.0}ms >= {}ms)",
1052 actual_steps,
1053 gated_steps,
1054 start.elapsed().as_millis(),
1055 budget.as_millis()
1056 );
1057 break;
1058 }
1059
1060 let loss = self.compute_ntp_loss_with_delta(engine, delta, tokens)?;
1061 losses.push(loss.double_value(&[]) as f32);
1062
1063 let (gn, clipped) = self.ttt_step(&loss, delta, Some(lr))?;
1064 grad_norms.push(gn as f32);
1065 if clipped {
1066 any_clipped = true;
1067 }
1068 actual_steps += 1;
1069
1070 let is_last_step = actual_steps == gated_steps;
1074 if is_last_step || (delta.accumulated_steps + actual_steps as u64).is_multiple_of(10) {
1075 for (key, util) in &collect_rank_utilization(delta) {
1076 tracing::debug!(key = %key, utilization = %util, "Delta rank utilization");
1077 }
1078 }
1079 }
1080
1081 let final_loss_tensor = self.compute_ntp_loss_with_delta(engine, delta, tokens)?;
1083 let final_loss = final_loss_tensor.double_value(&[]) as f32;
1084 let final_perplexity = final_loss.exp();
1085
1086 let rank_utils = collect_rank_utilization(delta);
1087
1088 if !gated_layer_keys.is_empty() {
1090 Self::restore_requires_grad(delta);
1091 }
1092
1093 Ok((
1094 actual_steps,
1095 losses,
1096 grad_norms,
1097 any_clipped,
1098 final_loss,
1099 final_perplexity,
1100 rank_utils,
1101 gated_layer_keys,
1102 ))
1103 }
1104
1105 fn compute_ntp_loss_with_delta(
1111 &self,
1112 engine: &TorchEngine,
1113 delta: &TenantDelta,
1114 tokens: &[u32],
1115 ) -> Result<Tensor> {
1116 tch::with_grad(|| {
1120 let tokens_i64: Vec<i64> = tokens.iter().map(|&t| t as i64).collect();
1121
1122 let input_ids = Tensor::from_slice(&tokens_i64)
1123 .to_device(self.device)
1124 .unsqueeze(0); let logits = engine.forward_with_delta(&input_ids, delta)?;
1128
1129 let seq_len = tokens.len() as i64;
1130 let vocab_size = logits.size()[2];
1131
1132 let pred_logits = logits.narrow(1, 0, seq_len - 1).reshape([-1, vocab_size]);
1134
1135 let target_ids = input_ids.narrow(1, 1, seq_len - 1).reshape([-1]);
1136
1137 let loss = pred_logits.cross_entropy_loss::<Tensor>(
1138 &target_ids,
1139 None,
1140 tch::Reduction::Mean,
1141 -100,
1142 0.0,
1143 );
1144
1145 Ok(loss)
1146 })
1147 }
1148
1149 pub fn train_step(
1154 &self,
1155 engine: &TorchEngine,
1156 delta: &mut TenantDelta,
1157 input_tokens: &[u32],
1158 gradient_steps: usize,
1159 learning_rate: Option<f64>,
1160 max_adaptation_ms: Option<u64>,
1161 ) -> Result<TTTResult> {
1162 let start = Instant::now();
1163
1164 let tokens_provided = input_tokens.len();
1165 let was_truncated = tokens_provided > self.config.max_ttt_context as usize;
1166
1167 let tokens: Vec<u32> = if was_truncated {
1168 input_tokens[input_tokens.len() - self.config.max_ttt_context as usize..].to_vec()
1169 } else {
1170 input_tokens.to_vec()
1171 };
1172 let tokens_used = tokens.len();
1173
1174 if tokens_used < 2 {
1175 return Ok(TTTResult::skipped("Input too short for NTP loss"));
1176 }
1177
1178 let clamped_steps = gradient_steps.min(self.config.max_gradient_steps);
1180 let time_budget_ms = max_adaptation_ms.unwrap_or(self.config.max_adaptation_ms);
1181 let budget = Duration::from_millis(time_budget_ms);
1182
1183 let lr = learning_rate.unwrap_or(delta.learning_rate);
1184
1185 let pre_snapshot = delta.extract_state_dict();
1187 let muon_snapshot = super::muon::snapshot_muon_states(&delta.muon_states);
1188
1189 let ssm_snapshot = engine.snapshot_ssm_states();
1191
1192 let loop_result = std::panic::catch_unwind(AssertUnwindSafe(|| {
1193 self.train_step_inner(engine, delta, &tokens, clamped_steps, lr, &start, budget)
1194 }));
1195
1196 match loop_result {
1197 Ok(Ok(inner)) => {
1198 let (
1199 actual_steps,
1200 losses,
1201 grad_norms,
1202 any_clipped,
1203 final_loss,
1204 final_perplexity,
1205 initial_loss,
1206 initial_perplexity,
1207 gated_layer_keys,
1208 ) = inner;
1209
1210 let avg_loss = losses.iter().sum::<f32>() / losses.len() as f32;
1211 let loss_improvement = initial_loss - final_loss;
1212 let avg_grad_norm = if !grad_norms.is_empty() {
1213 grad_norms.iter().sum::<f32>() / grad_norms.len() as f32
1214 } else {
1215 0.0
1216 };
1217 let max_grad_norm_val = grad_norms.iter().copied().fold(0.0f32, f32::max);
1218
1219 let recommendation = self.compute_recommendation(
1220 loss_improvement,
1221 any_clipped,
1222 initial_perplexity,
1223 final_perplexity,
1224 );
1225
1226 let time_used_ms = start.elapsed().as_millis() as u64;
1227
1228 let rank_utils = collect_rank_utilization(delta);
1229
1230 let rank_signals = if let Some(ref mut oracle) = delta.rank_oracle {
1232 oracle.observe(&rank_utils);
1233 if oracle.should_evaluate() {
1234 let signals = oracle.recommend();
1235 if oracle.config.auto_adapt {
1236 use crate::runtime::ttn_profile::RankSignal;
1237 for (key, signal) in &signals {
1238 if let Some(current) = delta.effective_rank(key) {
1239 match signal {
1240 RankSignal::Decrease => {
1241 let new_rank = (current / 2).max(1);
1242 delta.set_effective_rank(key, new_rank);
1243 tracing::info!(key = %key, from = current, to = new_rank, "Rank oracle: decreased");
1244 }
1245 RankSignal::Increase => {
1246 let new_rank = current * 2;
1247 delta.set_effective_rank(key, new_rank);
1248 let actual =
1249 delta.effective_rank(key).unwrap_or(new_rank);
1250 tracing::info!(key = %key, from = current, to = actual, "Rank oracle: increased");
1251 }
1252 RankSignal::Hold => {}
1253 }
1254 }
1255 }
1256 }
1257 signals
1258 } else {
1259 HashMap::new()
1260 }
1261 } else {
1262 HashMap::new()
1263 };
1264
1265 engine.restore_ssm_states(ssm_snapshot);
1267
1268 if actual_steps > 0 {
1272 delta.bump_weight_epoch();
1273 }
1274
1275 Ok(TTTResult {
1276 avg_loss,
1277 loss_improvement,
1278 steps_performed: actual_steps,
1279 adaptation_time_ms: time_used_ms,
1280 skipped: false,
1281 skip_reason: None,
1282 avg_grad_norm,
1283 max_grad_norm: max_grad_norm_val,
1284 gradient_clipped: any_clipped,
1285 tokens_used,
1286 tokens_provided,
1287 was_truncated,
1288 initial_perplexity,
1289 final_perplexity,
1290 recommendation,
1291 gated_steps: clamped_steps,
1292 pending: false, time_used_ms,
1294 time_budget_ms,
1295 rank_utilization: rank_utils,
1296 gated_layers: gated_layer_keys,
1297 rank_signals,
1298 })
1299 }
1300 Ok(Err(e)) => {
1301 tracing::warn!("TTT train_step gradient loop error, restoring delta: {}", e);
1302 let _ = delta.load_state_dict(&pre_snapshot);
1303 super::muon::restore_muon_states(&mut delta.muon_states, &muon_snapshot);
1304 Self::restore_requires_grad(delta);
1305 delta.zero_grad();
1306 engine.restore_ssm_states(ssm_snapshot);
1307 Err(e)
1308 }
1309 Err(panic_info) => {
1310 tracing::error!("TTT train_step panicked during gradient loop, restoring delta");
1311 let _ = delta.load_state_dict(&pre_snapshot);
1312 super::muon::restore_muon_states(&mut delta.muon_states, &muon_snapshot);
1313 Self::restore_requires_grad(delta);
1314 delta.zero_grad();
1315 engine.restore_ssm_states(ssm_snapshot);
1316 let msg = if let Some(s) = panic_info.downcast_ref::<String>() {
1317 s.clone()
1318 } else if let Some(s) = panic_info.downcast_ref::<&str>() {
1319 (*s).to_owned()
1320 } else {
1321 "unknown panic".to_owned()
1322 };
1323 Err(anyhow::anyhow!("TTT training panicked: {}", msg))
1324 }
1325 }
1326 }
1327
1328 #[allow(clippy::type_complexity)]
1330 fn train_step_inner(
1331 &self,
1332 engine: &TorchEngine,
1333 delta: &mut TenantDelta,
1334 tokens: &[u32],
1335 clamped_steps: usize,
1336 lr: f64,
1337 start: &Instant,
1338 budget: Duration,
1339 ) -> Result<(
1340 usize,
1341 Vec<f32>,
1342 Vec<f32>,
1343 bool,
1344 f32,
1345 f32,
1346 f32,
1347 f32,
1348 Vec<String>,
1349 )> {
1350 let initial_loss_tensor = self.compute_ntp_loss_with_delta(engine, delta, tokens)?;
1352 let initial_loss = initial_loss_tensor.double_value(&[]) as f32;
1353 let initial_perplexity = initial_loss.exp();
1354
1355 let mut losses = vec![initial_loss];
1356 let mut grad_norms = Vec::with_capacity(clamped_steps);
1357 let mut any_clipped = false;
1358
1359 let (gn, clipped) = self.ttt_step(&initial_loss_tensor, delta, Some(lr))?;
1361 grad_norms.push(gn as f32);
1362 if clipped {
1363 any_clipped = true;
1364 }
1365 let mut actual_steps = 1;
1366
1367 let gated_layer_keys: Vec<String> = Vec::new();
1370
1371 for _ in 1..clamped_steps {
1373 if start.elapsed() >= budget {
1374 tracing::warn!(
1375 "TTT train_step: time budget exhausted after {}/{} steps ({:.0}ms >= {}ms)",
1376 actual_steps,
1377 clamped_steps,
1378 start.elapsed().as_millis(),
1379 budget.as_millis()
1380 );
1381 break;
1382 }
1383
1384 let loss = self.compute_ntp_loss_with_delta(engine, delta, tokens)?;
1385 losses.push(loss.double_value(&[]) as f32);
1386 let (gn, clipped) = self.ttt_step(&loss, delta, Some(lr))?;
1387 grad_norms.push(gn as f32);
1388 if clipped {
1389 any_clipped = true;
1390 }
1391 actual_steps += 1;
1392 }
1393
1394 let final_loss_tensor = self.compute_ntp_loss_with_delta(engine, delta, tokens)?;
1396 let final_loss = final_loss_tensor.double_value(&[]) as f32;
1397 let final_perplexity = final_loss.exp();
1398
1399 if !gated_layer_keys.is_empty() {
1401 Self::restore_requires_grad(delta);
1402 }
1403
1404 Ok((
1405 actual_steps,
1406 losses,
1407 grad_norms,
1408 any_clipped,
1409 final_loss,
1410 final_perplexity,
1411 initial_loss,
1412 initial_perplexity,
1413 gated_layer_keys,
1414 ))
1415 }
1416}
1417
1418unsafe impl Send for TestTimeTrainer {}
1421unsafe impl Sync for TestTimeTrainer {}
1422
1423#[async_trait::async_trait]
1430pub trait TTTVerifier: Send + Sync {
1431 async fn verify(&self, context: &TTTContext, engine: &TorchEngine) -> Result<bool>;
1435}
1436
1437pub struct NoOpVerifier;
1439
1440#[async_trait::async_trait]
1441impl TTTVerifier for NoOpVerifier {
1442 async fn verify(&self, _context: &TTTContext, _engine: &TorchEngine) -> Result<bool> {
1443 Ok(true)
1444 }
1445}
1446
1447#[cfg(test)]
1448#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
1449mod tests {
1450 use super::*;
1451
1452 #[test]
1453 fn test_ttt_config_default() {
1454 let config = TTTConfig::default();
1455 assert!((config.learning_rate - 3e-4).abs() < 1e-10);
1456 assert_eq!(config.gradient_steps, 3);
1457 assert_eq!(config.min_input_length, 32);
1458 assert_eq!(config.max_ttt_context, 512);
1459 assert!(config.enabled);
1460 assert_eq!(config.max_adaptation_ms, 30_000);
1463 assert_eq!(config.max_gradient_steps, 50);
1464 assert!((config.tau_reject - 200.0).abs() < 0.1);
1465 }
1466
1467 #[test]
1468 fn test_ttt_result_skipped() {
1469 let result = TTTResult::skipped("test reason");
1470 assert!(result.skipped);
1471 assert_eq!(result.skip_reason, Some("test reason".to_owned()));
1472 assert_eq!(result.steps_performed, 0);
1473 }
1474
1475 #[test]
1476 fn test_ttt_config_serde() {
1477 let json = r#"{
1478 "learning_rate": 0.001,
1479 "gradient_steps": 5,
1480 "enabled": false
1481 }"#;
1482
1483 let config: TTTConfig = serde_json::from_str(json).unwrap();
1484 assert!((config.learning_rate - 0.001).abs() < 1e-10);
1485 assert_eq!(config.gradient_steps, 5);
1486 assert!(!config.enabled);
1487 assert_eq!(config.min_input_length, 32);
1489 }
1490
1491 #[test]
1492 fn test_gradient_gating_config_defaults() {
1493 let config = GradientGatingConfig::default();
1494 assert!(config.enabled);
1495 assert!(config.min_grad_norm > 0.0);
1496 assert!(config.warmup_steps > 0);
1497 }
1498
1499 #[test]
1500 fn test_identify_gated_layers() {
1501 let mut norms = HashMap::new();
1502 norms.insert("0.q_proj".to_owned(), 1e-3); norms.insert("0.v_proj".to_owned(), 1e-8); norms.insert("1.q_proj".to_owned(), 5e-6); let gated = identify_gated_layers(&norms, 1e-5);
1506 assert!(!gated.contains(&"0.q_proj".to_owned())); assert!(gated.contains(&"0.v_proj".to_owned())); assert!(gated.contains(&"1.q_proj".to_owned())); }
1510
1511 #[test]
1512 fn test_gradient_gating_with_real_backward() {
1513 use tch::{nn::VarStore, Kind};
1514 let vs = VarStore::new(Device::Cpu);
1515 let root = vs.root();
1516 let a = root
1517 .sub("layer_0")
1518 .sub("q_proj")
1519 .kaiming_uniform("lora_a", &[4, 8]);
1520 let b = root.sub("layer_0").sub("q_proj").zeros("lora_b", &[8, 4]);
1521 let x = Tensor::randn([2, 8], (Kind::Float, Device::Cpu));
1522
1523 let out = x.matmul(&a.tr()).matmul(&b.tr());
1525 let loss = out.sum(Kind::Float);
1526 loss.backward();
1527 assert!(a.grad().defined());
1528
1529 for (_, v) in vs.variables() {
1531 if v.grad().defined() {
1532 let _ = v.grad().zero_();
1533 }
1534 }
1535 let _ = a.set_requires_grad(false);
1536 let out2 = x.matmul(&a.tr()).matmul(&b.tr());
1537 let loss2 = out2.sum(Kind::Float);
1538 loss2.backward();
1539 assert!(!a.grad().defined() || a.grad().abs().sum(Kind::Double).double_value(&[]) == 0.0);
1541 assert!(b.grad().defined());
1543
1544 let _ = a.set_requires_grad(true);
1546 }
1547
1548 #[test]
1549 fn test_parse_varstore_key() {
1550 assert_eq!(
1551 parse_varstore_key_to_delta_key("layer_0.q_proj.lora_a"),
1552 Some("0.q_proj".to_owned())
1553 );
1554 assert_eq!(
1555 parse_varstore_key_to_delta_key("layer_23.v_proj.lora_b"),
1556 Some("23.v_proj".to_owned())
1557 );
1558 assert_eq!(parse_varstore_key_to_delta_key("invalid_key"), None);
1559 }
1560
1561 #[test]
1562 fn test_rank_oracle_config_defaults() {
1563 let config = RankOracleConfig::default();
1564 assert!(config.adaptation_interval > 0);
1565 assert!(!config.auto_adapt); assert!(!config.rank_levels.is_empty());
1567 }
1568
1569 #[test]
1570 fn test_rank_oracle_produces_signals() {
1571 use crate::runtime::ttn_profile::RankSignal;
1572 let mut oracle = RankOracle::new(RankOracleConfig::default());
1573 for _ in 0..10 {
1575 let mut utils = HashMap::new();
1576 utils.insert("0.q_proj".to_owned(), 0.1);
1577 utils.insert("0.v_proj".to_owned(), 0.9);
1578 oracle.observe(&utils);
1579 }
1580 let signals = oracle.recommend();
1581 assert_eq!(signals.get("0.q_proj"), Some(&RankSignal::Decrease));
1582 assert_eq!(signals.get("0.v_proj"), Some(&RankSignal::Increase));
1583 }
1584
1585 #[test]
1586 fn test_rank_oracle_should_evaluate() {
1587 let mut oracle = RankOracle::new(RankOracleConfig {
1588 adaptation_interval: 3,
1589 ..Default::default()
1590 });
1591 let utils: HashMap<String, f64> = HashMap::new();
1592 oracle.observe(&utils); assert!(!oracle.should_evaluate());
1594 oracle.observe(&utils); assert!(!oracle.should_evaluate());
1596 oracle.observe(&utils); assert!(oracle.should_evaluate());
1598 }
1599}