1use super::muon::{MuonConfig, MuonState};
8use anyhow::{anyhow, Result};
9use safetensors::tensor::{SafeTensors, TensorView};
10use serde::{Deserialize, Serialize};
11use std::collections::HashMap;
12use std::time::Instant;
13use tch::nn::VarStore;
14use tch::{Device, Kind, Tensor};
15
16pub fn extract_lora_components(tensor_name: &str) -> Option<(usize, String, String)> {
19 let parts: Vec<&str> = tensor_name.split('.').collect();
20
21 for (i, &part) in parts.iter().enumerate() {
23 let normalized = part.to_lowercase();
24 if (normalized == "lora_a" || normalized == "lora_b") && i > 0 {
25 let module = parts[i - 1];
26 if matches!(
27 module,
28 "q_proj" | "k_proj" | "v_proj" | "o_proj" | "gate_proj" | "up_proj" | "down_proj"
29 ) {
30 let layer_idx = parts
32 .iter()
33 .zip(parts.iter().skip(1))
34 .find_map(|(&a, &b)| {
35 if a == "layers" {
36 b.parse::<usize>().ok()
37 } else {
38 None
39 }
40 })
41 .unwrap_or(0);
42 return Some((layer_idx, module.to_owned(), normalized));
43 }
44 }
45 }
46
47 None
48}
49
50#[derive(Debug, Clone, Serialize, Deserialize)]
55pub struct LayerDeltaConfig {
56 pub rank: usize,
57 pub target_modules: Vec<String>,
58}
59
60#[derive(Debug, Clone, Serialize, Deserialize)]
65pub struct TenantDeltaConfig {
66 #[serde(default = "default_rank")]
68 pub rank: usize,
69
70 #[serde(default = "default_alpha")]
72 pub alpha: f32,
73
74 #[serde(default)]
76 pub dropout: f32,
77
78 #[serde(default = "default_target_modules")]
80 pub target_modules: Vec<String>,
81
82 #[serde(default = "default_learning_rate")]
85 pub learning_rate: f64,
86
87 #[serde(default = "default_max_accumulated_steps")]
89 pub max_accumulated_steps: u64,
90
91 #[serde(default = "default_decay_lambda")]
95 pub decay_lambda: f64,
96
97 #[serde(default = "default_muon_momentum_config")]
100 pub muon_momentum: f64,
101
102 #[serde(default)]
105 pub layer_overrides: Option<HashMap<usize, LayerDeltaConfig>>,
106}
107
108impl TenantDeltaConfig {
109 pub fn from_profile(
113 base: &TenantDeltaConfig,
114 profile: &crate::runtime::ttn_profile::LayerProfile,
115 ) -> Self {
116 Self::from_profile_with_quant_caps(base, profile, None)
117 }
118
119 pub fn from_profile_with_quant_caps(
133 base: &TenantDeltaConfig,
134 profile: &crate::runtime::ttn_profile::LayerProfile,
135 layer_dtype_bytes: Option<&HashMap<usize, usize>>,
136 ) -> Self {
137 let mut overrides = HashMap::new();
138 for la in &profile.layers {
139 let cap = layer_dtype_bytes
140 .and_then(|m| m.get(&la.layer_idx))
141 .map(|&bytes| crate::training::delta_reducer::rank_cap_for_dtype_bytes(bytes))
142 .unwrap_or(usize::MAX);
143 let capped_rank = la.recommended_rank.min(cap).max(1);
144 overrides.insert(
145 la.layer_idx,
146 LayerDeltaConfig {
147 rank: capped_rank,
148 target_modules: la.target_modules.clone(),
149 },
150 );
151 }
152 Self {
153 layer_overrides: Some(overrides),
154 ..base.clone()
155 }
156 }
157}
158
159fn default_rank() -> usize {
160 8
161}
162fn default_alpha() -> f32 {
163 4.0
164}
165fn default_target_modules() -> Vec<String> {
166 vec!["q_proj".to_owned(), "v_proj".to_owned()]
167}
168fn default_learning_rate() -> f64 {
169 1e-3 }
172fn default_max_accumulated_steps() -> u64 {
173 300
174}
175fn default_decay_lambda() -> f64 {
176 0.0 }
179fn default_muon_momentum_config() -> f64 {
180 0.9 }
183
184impl Default for TenantDeltaConfig {
185 fn default() -> Self {
186 Self {
187 rank: default_rank(),
188 alpha: default_alpha(),
189 dropout: 0.0,
190 target_modules: default_target_modules(),
191 learning_rate: default_learning_rate(),
192 max_accumulated_steps: default_max_accumulated_steps(),
193 decay_lambda: default_decay_lambda(),
194 muon_momentum: default_muon_momentum_config(),
195 layer_overrides: None,
196 }
197 }
198}
199
200pub struct TenantDelta {
207 pub lora_a: HashMap<String, Tensor>,
209 pub lora_b: HashMap<String, Tensor>,
211 pub vs: VarStore,
213 pub muon_states: HashMap<String, MuonState>,
215 pub muon_config: MuonConfig,
217 pub scaling_map: HashMap<String, f64>,
220 pub scaling: f64,
222 pub rank: usize,
224 pub device: Device,
226 pub target_modules: Vec<String>,
228 pub num_layers: usize,
230 pub learning_rate: f64,
232
233 pub accumulated_steps: u64,
236 pub max_accumulated_steps: u64,
238 pub request_count: u64,
240 pub avg_loss_improvement: f64,
242 pub last_access: Instant,
244 pub created_at: Instant,
246 pub last_snapshot_hash: Option<String>,
248 pub decay_lambda: f64,
250 pub effective_ranks: HashMap<String, usize>,
254 max_ranks: HashMap<String, usize>,
256 alpha: f32,
258 pub rank_oracle: Option<super::ttt::RankOracle>,
260 pub adaptation_state: crate::training::adaptation_state::DeltaAdaptationState,
262 weight_epoch: u64,
267}
268
269impl TenantDelta {
270 pub fn new(
282 config: &TenantDeltaConfig,
283 module_dims: &HashMap<String, (usize, usize)>,
284 device: Device,
285 num_layers: usize,
286 ) -> Result<Self> {
287 Self::new_with_per_layer_dims(config, module_dims, device, num_layers, None)
288 }
289
290 pub fn new_with_per_layer_dims(
295 config: &TenantDeltaConfig,
296 module_dims: &HashMap<String, (usize, usize)>,
297 device: Device,
298 num_layers: usize,
299 per_layer_dims: Option<&HashMap<usize, HashMap<String, (usize, usize)>>>,
300 ) -> Result<Self> {
301 let vs = VarStore::new(device);
302 let root = vs.root();
303
304 let mut lora_a = HashMap::new();
305 let mut lora_b = HashMap::new();
306 let mut scaling_map = HashMap::new();
307 let mut effective_ranks = HashMap::new();
308 let mut max_ranks = HashMap::new();
309
310 for layer_idx in 0..num_layers {
311 let (layer_rank, layer_modules): (usize, &Vec<String>) =
313 if let Some(overrides) = &config.layer_overrides {
314 if let Some(lc) = overrides.get(&layer_idx) {
315 (lc.rank, &lc.target_modules)
316 } else {
317 (config.rank, &config.target_modules)
318 }
319 } else {
320 (config.rank, &config.target_modules)
321 };
322
323 for module_name in layer_modules {
324 let (in_features, out_features) = per_layer_dims
326 .and_then(|pld| pld.get(&layer_idx))
327 .and_then(|m| m.get(module_name.as_str()))
328 .or_else(|| module_dims.get(module_name.as_str()))
329 .ok_or_else(|| {
330 anyhow!(
331 "Module '{}' not found in model dimensions (layer {})",
332 module_name,
333 layer_idx
334 )
335 })?;
336
337 let key = format!("{}.{}", layer_idx, module_name);
338
339 let layer_path = root.sub(format!("layer_{}", layer_idx)).sub(module_name);
341
342 let a =
344 layer_path.kaiming_uniform("lora_a", &[layer_rank as i64, *in_features as i64]);
345
346 let b = layer_path.zeros("lora_b", &[*out_features as i64, layer_rank as i64]);
348
349 lora_a.insert(key.clone(), a);
350 lora_b.insert(key.clone(), b);
351
352 scaling_map.insert(key.clone(), config.alpha as f64 / layer_rank as f64);
354
355 effective_ranks.insert(key.clone(), layer_rank);
357 max_ranks.insert(key, layer_rank);
358 }
359 }
360
361 let muon_states = HashMap::new();
363 let muon_config = MuonConfig {
364 lr: config.learning_rate,
365 momentum: config.muon_momentum,
366 weight_decay: config.decay_lambda, ..MuonConfig::default()
368 };
369
370 let scaling = config.alpha as f64 / config.rank as f64;
371 let now = Instant::now();
372
373 let mut all_modules: Vec<String> = config.target_modules.clone();
375 if let Some(overrides) = &config.layer_overrides {
376 for lc in overrides.values() {
377 for m in &lc.target_modules {
378 if !all_modules.contains(m) {
379 all_modules.push(m.clone());
380 }
381 }
382 }
383 }
384
385 Ok(Self {
386 lora_a,
387 lora_b,
388 vs,
389 muon_states,
390 muon_config,
391 scaling_map,
392 scaling,
393 rank: config.rank,
394 device,
395 target_modules: all_modules,
396 num_layers,
397 learning_rate: config.learning_rate,
398 accumulated_steps: 0,
399 max_accumulated_steps: config.max_accumulated_steps,
400 request_count: 0,
401 avg_loss_improvement: 0.0,
402 last_access: now,
403 created_at: now,
404 last_snapshot_hash: None,
405 decay_lambda: config.decay_lambda,
406 effective_ranks,
407 max_ranks,
408 alpha: config.alpha,
409 rank_oracle: None,
410 adaptation_state: crate::training::adaptation_state::DeltaAdaptationState::Idle,
411 weight_epoch: 0,
412 })
413 }
414
415 pub fn has_module(&self, module_name: &str, layer_idx: usize) -> bool {
417 self.lora_a
418 .contains_key(&format!("{}.{}", layer_idx, module_name))
419 }
420
421 pub fn forward(&self, x: &Tensor, module_name: &str, layer_idx: usize) -> Result<Tensor> {
431 let key = format!("{}.{}", layer_idx, module_name);
432 let a = self
433 .lora_a
434 .get(&key)
435 .ok_or_else(|| anyhow!("Module '{}' not found in delta", key))?;
436 let b = self
437 .lora_b
438 .get(&key)
439 .ok_or_else(|| anyhow!("Module '{}' B not found in delta", key))?;
440
441 let scaling = self.scaling_map.get(&key).copied().unwrap_or(self.scaling);
443
444 let eff_rank = self
446 .effective_ranks
447 .get(&key)
448 .copied()
449 .unwrap_or_else(|| a.size()[0] as usize) as i64;
450 let a_eff = a.narrow(0, 0, eff_rank); let b_eff = b.narrow(1, 0, eff_rank); let input_kind = x.kind();
459 let x = x.to_kind(a.kind());
460 let intermediate = x
461 .f_matmul(&a_eff.tr())
462 .map_err(|e| anyhow!("Delta forward matmul A failed for '{}': {}", key, e))?;
463 let output = intermediate
464 .f_matmul(&b_eff.tr())
465 .map_err(|e| anyhow!("Delta forward matmul B failed for '{}': {}", key, e))?;
466
467 Ok((output * scaling).to_kind(input_kind))
468 }
469
470 pub fn forward_2d(&self, x: &Tensor, module_name: &str, layer_idx: usize) -> Result<Tensor> {
487 let key = format!("{}.{}", layer_idx, module_name);
488 let a = self
489 .lora_a
490 .get(&key)
491 .ok_or_else(|| anyhow!("Module '{}' not found in delta", key))?;
492 let b = self
493 .lora_b
494 .get(&key)
495 .ok_or_else(|| anyhow!("Module '{}' B not found in delta", key))?;
496
497 let scaling = self.scaling_map.get(&key).copied().unwrap_or(self.scaling);
499
500 let eff_rank = self
502 .effective_ranks
503 .get(&key)
504 .copied()
505 .unwrap_or_else(|| a.size()[0] as usize) as i64;
506 let a_eff = a.narrow(0, 0, eff_rank); let b_eff = b.narrow(1, 0, eff_rank); let input_kind = x.kind();
515 let input_device = x.device();
516 let x = x.to_kind(a.kind());
517 let a_eff = a_eff.to_device(input_device);
525 let b_eff = b_eff.to_device(input_device);
526 let intermediate = x
527 .f_matmul(&a_eff.tr())
528 .map_err(|e| anyhow!("Delta forward_2d matmul A failed for '{}': {}", key, e))?;
529 let output = intermediate
530 .f_matmul(&b_eff.tr())
531 .map_err(|e| anyhow!("Delta forward_2d matmul B failed for '{}': {}", key, e))?;
532
533 Ok((output * scaling).to_kind(input_kind))
534 }
535
536 pub fn delta_norm_ratio(&self, base_norms: &HashMap<String, f64>) -> HashMap<String, f64> {
541 let _guard = tch::no_grad_guard();
542 let mut ratios = HashMap::new();
543
544 for key in self.lora_a.keys() {
546 if let (Some(a), Some(b)) = (self.lora_a.get(key), self.lora_b.get(key)) {
547 let scaling = self.scaling_map.get(key).copied().unwrap_or(self.scaling);
548 let delta = b.matmul(a) * scaling;
549 let delta_norm: f64 = delta.norm().double_value(&[]);
550 let module_name = key.split_once('.').map_or(key.as_str(), |(_, m)| m);
552 let base_norm = base_norms.get(module_name).copied().unwrap_or(1.0);
553 ratios.insert(key.clone(), delta_norm / base_norm.max(1e-8));
554 }
555 }
556
557 ratios
558 }
559
560 pub fn extract_state_dict(&self) -> HashMap<String, Tensor> {
566 let _guard = tch::no_grad_guard();
567 let mut state = HashMap::new();
568
569 for key in self.lora_a.keys() {
571 if let Some(a) = self.lora_a.get(key) {
572 state.insert(format!("{}.lora_a", key), a.copy());
573 }
574 if let Some(b) = self.lora_b.get(key) {
575 state.insert(format!("{}.lora_b", key), b.copy());
576 }
577 }
578
579 for (key, &rank) in &self.effective_ranks {
581 state.insert(
582 format!("{}.__effective_rank", key),
583 Tensor::from_slice(&[rank as i64]),
584 );
585 }
586
587 for (key, muon_state) in &self.muon_states {
589 if let Some(ref buf) = muon_state.momentum_buffer {
590 state.insert(format!("{}.__muon_momentum", key), buf.copy());
591 }
592 }
593
594 state
595 }
596
597 pub fn load_state_dict(&mut self, state: &HashMap<String, Tensor>) -> Result<()> {
602 let _guard = tch::no_grad_guard();
603
604 for key in self.lora_a.keys().cloned().collect::<Vec<_>>() {
605 let a_key = format!("{}.lora_a", key);
606 let b_key = format!("{}.lora_b", key);
607
608 if let Some(a_src) = state.get(&a_key) {
609 if let Some(a_dst) = self.lora_a.get_mut(&key) {
610 a_dst.copy_(a_src);
611 }
612 }
613
614 if let Some(b_src) = state.get(&b_key) {
615 if let Some(b_dst) = self.lora_b.get_mut(&key) {
616 b_dst.copy_(b_src);
617 }
618 }
619 }
620
621 for key in self.effective_ranks.keys().cloned().collect::<Vec<_>>() {
623 let meta_key = format!("{}.__effective_rank", key);
624 if let Some(t) = state.get(&meta_key) {
625 let rank = t.int64_value(&[]) as usize;
626 self.set_effective_rank(&key, rank);
627 }
628 }
629
630 for (key, muon_state) in &mut self.muon_states {
632 let meta_key = format!("{}.__muon_momentum", key);
633 if let Some(t) = state.get(&meta_key) {
634 if let Some(ref mut buf) = muon_state.momentum_buffer {
635 buf.copy_(t);
636 } else {
637 muon_state.momentum_buffer = Some(t.copy());
638 }
639 }
640 }
641
642 self.bump_weight_epoch();
646
647 Ok(())
648 }
649
650 pub fn touch(&mut self) {
652 self.last_access = Instant::now();
653 }
654
655 pub fn weight_epoch(&self) -> u64 {
657 self.weight_epoch
658 }
659
660 pub fn bump_weight_epoch(&mut self) {
665 self.weight_epoch = self.weight_epoch.wrapping_add(1);
666 }
667
668 pub fn memory_bytes(&self) -> usize {
670 let _guard = tch::no_grad_guard();
671 let mut total = 0usize;
672
673 for a in self.lora_a.values() {
674 total += tensor_bytes(a);
675 }
676 for b in self.lora_b.values() {
677 total += tensor_bytes(b);
678 }
679
680 total
681 }
682
683 pub fn is_at_capacity(&self) -> bool {
685 self.accumulated_steps >= self.max_accumulated_steps
686 }
687
688 pub fn reset(&mut self) {
690 let _guard = tch::no_grad_guard();
691 for b in self.lora_b.values_mut() {
692 let _ = b.zero_();
693 }
694 self.accumulated_steps = 0;
695 self.request_count = 0;
696 self.avg_loss_improvement = 0.0;
697 self.adaptation_state = crate::training::adaptation_state::DeltaAdaptationState::Idle;
698 self.bump_weight_epoch();
701 }
702
703 pub fn zero_grad(&self) {
705 for (_, var) in self.vs.variables() {
706 if var.grad().defined() {
707 let _ = var.grad().zero_();
708 }
709 }
710 }
711
712 pub fn effective_rank(&self, key: &str) -> Option<usize> {
714 self.effective_ranks.get(key).copied()
715 }
716
717 pub fn set_effective_rank(&mut self, key: &str, rank: usize) {
720 if let Some(&max_r) = self.max_ranks.get(key) {
721 let clamped = rank.clamp(1, max_r);
722 self.effective_ranks.insert(key.to_owned(), clamped);
723 self.scaling_map
725 .insert(key.to_owned(), self.alpha as f64 / clamped as f64);
726 }
727 }
728}
729
730impl TenantDelta {
731 pub fn compose(
739 base: &std::sync::Arc<parking_lot::Mutex<TenantDelta>>,
740 tenant: &std::sync::Arc<parking_lot::Mutex<TenantDelta>>,
741 ) -> std::sync::Arc<parking_lot::Mutex<TenantDelta>> {
742 let base_guard = base.lock();
743 let tenant_guard = tenant.lock();
744 let _no_grad = tch::no_grad_guard();
745
746 let mut all_keys: Vec<String> = base_guard.lora_a.keys().cloned().collect();
748 for k in tenant_guard.lora_a.keys() {
749 if !all_keys.contains(k) {
750 all_keys.push(k.clone());
751 }
752 }
753
754 let mut composed_a = HashMap::new();
755 let mut composed_b = HashMap::new();
756
757 for key in &all_keys {
758 let base_a = base_guard.lora_a.get(key);
759 let base_b = base_guard.lora_b.get(key);
760 let tenant_a = tenant_guard.lora_a.get(key);
761 let tenant_b = tenant_guard.lora_b.get(key);
762
763 let base_scaling = base_guard
765 .scaling_map
766 .get(key)
767 .copied()
768 .unwrap_or(base_guard.scaling);
769 let tenant_scaling = tenant_guard
770 .scaling_map
771 .get(key)
772 .copied()
773 .unwrap_or(tenant_guard.scaling);
774
775 let narrow_a = |a: &Tensor, guard: &TenantDelta| -> Tensor {
779 let eff = guard
780 .effective_ranks
781 .get(key)
782 .copied()
783 .unwrap_or_else(|| a.size()[0] as usize) as i64;
784 a.narrow(0, 0, eff)
785 };
786 let narrow_b = |b: &Tensor, guard: &TenantDelta| -> Tensor {
787 let eff = guard
788 .effective_ranks
789 .get(key)
790 .copied()
791 .unwrap_or_else(|| b.size()[1] as usize) as i64;
792 b.narrow(1, 0, eff)
793 };
794
795 let base_eff_rank = base_a.map(|a| {
797 base_guard
798 .effective_ranks
799 .get(key)
800 .copied()
801 .unwrap_or_else(|| a.size()[0] as usize)
802 });
803 let tenant_eff_rank = tenant_a.map(|a| {
804 tenant_guard
805 .effective_ranks
806 .get(key)
807 .copied()
808 .unwrap_or_else(|| a.size()[0] as usize)
809 });
810 let ranks_match = base_eff_rank == tenant_eff_rank;
811
812 match (base_a, tenant_a) {
813 (Some(ba), Some(ta)) if ranks_match => {
814 let base_effective_a = narrow_a(ba, &base_guard) * base_scaling;
816 let tenant_effective_a = narrow_a(ta, &tenant_guard) * tenant_scaling;
817 composed_a.insert(key.clone(), base_effective_a + tenant_effective_a);
818 }
819 (Some(ba), Some(ta)) => {
820 #[allow(clippy::expect_used)] let bb = base_b.expect("base B must exist if base A exists");
824 #[allow(clippy::expect_used)]
825 let tb = tenant_b.expect("tenant B must exist if tenant A exists");
826 let ba_eff = narrow_a(ba, &base_guard);
828 let bb_eff = narrow_b(bb, &base_guard);
829 let ta_eff = narrow_a(ta, &tenant_guard);
830 let tb_eff = narrow_b(tb, &tenant_guard);
831 let base_w = bb_eff.matmul(&ba_eff) * base_scaling;
833 let tenant_w = tb_eff.matmul(&ta_eff) * tenant_scaling;
834 let w_eff = base_w + tenant_w;
835 let out_dim = w_eff.size()[0];
840 let identity = Tensor::eye(out_dim, (w_eff.kind(), w_eff.device()));
841 composed_a.insert(key.clone(), w_eff);
842 composed_b.insert(key.clone(), identity);
843 continue; }
845 (Some(ba), None) => {
846 composed_a.insert(key.clone(), narrow_a(ba, &base_guard) * base_scaling);
847 }
848 (None, Some(ta)) => {
849 composed_a.insert(key.clone(), narrow_a(ta, &tenant_guard) * tenant_scaling);
850 }
851 (None, None) => {}
852 }
853
854 match (base_b, tenant_b) {
857 (Some(bb), Some(tb)) => {
858 let bb_eff = narrow_b(bb, &base_guard);
859 let tb_eff = narrow_b(tb, &tenant_guard);
860 composed_b.insert(key.clone(), &bb_eff + &tb_eff);
861 }
862 (Some(bb), None) => {
863 composed_b.insert(key.clone(), narrow_b(bb, &base_guard));
864 }
865 (None, Some(tb)) => {
866 composed_b.insert(key.clone(), narrow_b(tb, &tenant_guard));
867 }
868 (None, None) => {}
869 }
870 }
871
872 let mut all_modules: Vec<String> = base_guard.target_modules.clone();
874 for m in &tenant_guard.target_modules {
875 if !all_modules.contains(m) {
876 all_modules.push(m.clone());
877 }
878 }
879
880 let device = base_guard.device;
881 let rank = base_guard.rank.max(tenant_guard.rank);
882 let num_layers = base_guard.num_layers.max(tenant_guard.num_layers);
883 let vs = VarStore::new(device);
884 let now = Instant::now();
885
886 std::sync::Arc::new(parking_lot::Mutex::new(TenantDelta {
887 lora_a: composed_a,
888 lora_b: composed_b,
889 vs,
890 muon_states: HashMap::new(),
891 muon_config: MuonConfig::default(),
892 scaling_map: HashMap::new(), scaling: 1.0, rank,
895 device,
896 target_modules: all_modules,
897 num_layers,
898 learning_rate: 0.0,
899 accumulated_steps: 0,
900 max_accumulated_steps: u64::MAX,
901 request_count: 0,
902 avg_loss_improvement: 0.0,
903 last_access: now,
904 created_at: now,
905 last_snapshot_hash: None,
906 decay_lambda: 0.0,
907 effective_ranks: HashMap::new(),
908 max_ranks: HashMap::new(),
909 alpha: 1.0, rank_oracle: None,
911 adaptation_state: crate::training::adaptation_state::DeltaAdaptationState::Idle,
912 weight_epoch: 0,
913 }))
914 }
915}
916
917fn tensor_to_safetensors_data(
922 tensor: &Tensor,
923) -> Result<(safetensors::Dtype, Vec<usize>, Vec<u8>)> {
924 let cpu_tensor = tensor.to(tch::Device::Cpu).contiguous();
925 let kind = cpu_tensor.kind();
926 let dtype = match kind {
927 Kind::Float => safetensors::Dtype::F32,
928 Kind::Half => safetensors::Dtype::F16,
929 Kind::BFloat16 => safetensors::Dtype::BF16,
930 Kind::Double => safetensors::Dtype::F64,
931 Kind::Int => safetensors::Dtype::I32,
932 Kind::Int64 => safetensors::Dtype::I64,
933 Kind::Int16 => safetensors::Dtype::I16,
934 Kind::Int8 => safetensors::Dtype::I8,
935 Kind::Uint8 => safetensors::Dtype::U8,
936 other => {
937 return Err(anyhow!(
938 "Unsupported tensor kind for safetensors: {:?}",
939 other
940 ))
941 }
942 };
943 let shape: Vec<usize> = cpu_tensor.size().iter().map(|&d| d as usize).collect();
944 let numel: usize = shape.iter().try_fold(1usize, |acc, &dim| {
945 acc.checked_mul(dim)
946 .ok_or_else(|| anyhow!("Tensor numel overflow: shape {:?}", shape))
947 })?;
948 let elem_size = kind.elt_size_in_bytes();
949 let byte_len = numel
950 .checked_mul(elem_size)
951 .ok_or_else(|| anyhow!("Tensor byte length overflow: {} * {}", numel, elem_size))?;
952 let mut bytes = vec![0u8; byte_len];
953 cpu_tensor.copy_data_u8(&mut bytes, numel);
954 Ok((dtype, shape, bytes))
955}
956
957fn module_to_peft_subpath(module: &str) -> &str {
959 match module {
960 "gate_proj" | "up_proj" | "down_proj" => "mlp",
961 _ => "self_attn", }
963}
964
965impl TenantDelta {
966 pub fn serialize_to_safetensors_bytes(&self) -> Result<Vec<u8>> {
971 let mut pairs: Vec<(String, Tensor)> = Vec::new();
972
973 let mut keys: Vec<&String> = self.lora_a.keys().collect();
975 keys.sort(); for key in keys {
977 let mut parts = key.splitn(2, '.');
979 let layer_idx: usize = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0);
980 let module = parts.next().unwrap_or(key.as_str());
981 let subpath = module_to_peft_subpath(module);
982
983 if let Some(a) = self.lora_a.get(key) {
984 let peft_key = format!(
985 "base_model.model.layers.{}.{}.{}.lora_A.weight",
986 layer_idx, subpath, module
987 );
988 pairs.push((peft_key, a.shallow_clone()));
989 }
990 if let Some(b) = self.lora_b.get(key) {
991 let peft_key = format!(
992 "base_model.model.layers.{}.{}.{}.lora_B.weight",
993 layer_idx, subpath, module
994 );
995 pairs.push((peft_key, b.shallow_clone()));
996 }
997 }
998
999 serialize_tensor_pairs_to_bytes(pairs.into_iter())
1000 }
1001
1002 pub fn load_from_safetensors_bytes(data: &[u8], device: Device) -> Result<Self> {
1006 let tensors = SafeTensors::deserialize(data)
1007 .map_err(|e| anyhow!("safetensors deserialization failed: {}", e))?;
1008
1009 let mut lora_a: HashMap<String, Tensor> = HashMap::new();
1010 let mut lora_b: HashMap<String, Tensor> = HashMap::new();
1011 let mut rank = 0usize;
1012 let mut max_layer = 0usize;
1013 let mut module_names: std::collections::HashSet<String> = std::collections::HashSet::new();
1014
1015 for (name, view) in tensors.tensors() {
1016 if let Some((layer_idx, module, ab)) = extract_lora_components(&name) {
1017 let key = format!("{}.{}", layer_idx, module);
1018 let tensor = safetensors_view_to_tensor(&view, device)?;
1019 max_layer = max_layer.max(layer_idx);
1020 module_names.insert(module.clone());
1021 match ab.as_str() {
1022 "lora_a" => {
1023 if rank == 0 {
1024 rank = tensor.size()[0] as usize;
1025 }
1026 lora_a.insert(key, tensor);
1027 }
1028 "lora_b" => {
1029 lora_b.insert(key, tensor);
1030 }
1031 _ => {}
1032 }
1033 }
1034 }
1035
1036 if lora_a.is_empty() {
1037 return Err(anyhow!("No LoRA A matrices found in safetensors data"));
1038 }
1039
1040 for (key, a_tensor) in &lora_a {
1042 let a_shape = a_tensor.size();
1043 if a_shape.len() != 2 {
1044 return Err(anyhow!(
1045 "LoRA A tensor '{}' has invalid rank: expected 2, got {}",
1046 key,
1047 a_shape.len()
1048 ));
1049 }
1050 if let Some(b_tensor) = lora_b.get(key) {
1051 let b_shape = b_tensor.size();
1052 if b_shape.len() != 2 {
1053 return Err(anyhow!(
1054 "LoRA B tensor '{}' has invalid rank: expected 2, got {}",
1055 key,
1056 b_shape.len()
1057 ));
1058 }
1059 if a_shape[0] != b_shape[1] {
1061 return Err(anyhow!(
1062 "LoRA rank mismatch for '{}': A shape {:?} vs B shape {:?} (A[0]={} != B[1]={})",
1063 key, a_shape, b_shape, a_shape[0], b_shape[1]
1064 ));
1065 }
1066 }
1067 }
1068
1069 let num_layers = max_layer + 1;
1070 let alpha = rank as f32;
1071 let target_modules: Vec<String> = module_names.into_iter().collect();
1072 let scaling = alpha as f64 / rank as f64;
1073 let vs = VarStore::new(device);
1074 let now = Instant::now();
1075
1076 tracing::info!(
1077 "Loaded LoRA adapter from bytes: rank={}, layers={}, modules={:?}, alpha={}",
1078 rank,
1079 num_layers,
1080 target_modules,
1081 alpha
1082 );
1083
1084 let scaling_map: HashMap<String, f64> = lora_a
1086 .iter()
1087 .map(|(key, a)| {
1088 let layer_rank = a.size()[0] as usize;
1089 (key.clone(), alpha as f64 / layer_rank as f64)
1090 })
1091 .collect();
1092
1093 let effective_ranks: HashMap<String, usize> = lora_a
1095 .iter()
1096 .map(|(key, a)| (key.clone(), a.size()[0] as usize))
1097 .collect();
1098 let max_ranks = effective_ranks.clone();
1099
1100 Ok(Self {
1101 lora_a,
1102 lora_b,
1103 vs,
1104 muon_states: HashMap::new(),
1105 muon_config: MuonConfig::default(),
1106 scaling_map,
1107 scaling,
1108 rank,
1109 device,
1110 target_modules,
1111 num_layers,
1112 learning_rate: 0.0,
1113 accumulated_steps: 0,
1114 max_accumulated_steps: u64::MAX,
1115 request_count: 0,
1116 avg_loss_improvement: 0.0,
1117 last_access: now,
1118 created_at: now,
1119 last_snapshot_hash: None,
1120 decay_lambda: 0.0,
1121 effective_ranks,
1122 max_ranks,
1123 alpha,
1124 rank_oracle: None,
1125 adaptation_state: crate::training::adaptation_state::DeltaAdaptationState::Idle,
1126 weight_epoch: 0,
1127 })
1128 }
1129}
1130
1131fn safetensors_view_to_tensor(
1133 view: &safetensors::tensor::TensorView<'_>,
1134 device: Device,
1135) -> Result<Tensor> {
1136 let dtype = view.dtype();
1137 let kind = match dtype {
1138 safetensors::Dtype::F32 => Kind::Float,
1139 safetensors::Dtype::F16 => Kind::Half,
1140 safetensors::Dtype::BF16 => Kind::BFloat16,
1141 safetensors::Dtype::F64 => Kind::Double,
1142 safetensors::Dtype::I32 => Kind::Int,
1143 safetensors::Dtype::I64 => Kind::Int64,
1144 safetensors::Dtype::I16 => Kind::Int16,
1145 safetensors::Dtype::I8 => Kind::Int8,
1146 safetensors::Dtype::U8 => Kind::Uint8,
1147 other => return Err(anyhow!("Unsupported safetensors dtype: {:?}", other)),
1148 };
1149 let shape: Vec<i64> = view.shape().iter().map(|&d| d as i64).collect();
1150 let data = view.data();
1151
1152 let tensor = Tensor::from_data_size(data, &shape, kind);
1154 Ok(tensor.to(device))
1155}
1156
1157fn serialize_tensor_pairs_to_bytes(
1162 pairs: impl Iterator<Item = (String, Tensor)>,
1163) -> Result<Vec<u8>> {
1164 let _guard = tch::no_grad_guard();
1165 let mut tensor_data: Vec<(String, safetensors::Dtype, Vec<usize>, Vec<u8>)> = Vec::new();
1166
1167 for (key, tensor) in pairs {
1168 let cpu = tensor.to_device(Device::Cpu).contiguous();
1169 let (dtype, shape, bytes) = tensor_to_safetensors_data(&cpu)?;
1170 tensor_data.push((key, dtype, shape, bytes));
1171 }
1172
1173 if tensor_data.is_empty() {
1174 return Err(anyhow!("No tensors to serialize"));
1175 }
1176
1177 let views: Vec<(String, TensorView<'_>)> = tensor_data
1178 .iter()
1179 .map(|(key, dtype, shape, bytes)| {
1180 let view = TensorView::new(*dtype, shape.clone(), bytes).unwrap_or_else(|_| {
1181 unreachable!("TensorView construction with valid dtype/shape/bytes")
1182 });
1183 (key.clone(), view)
1184 })
1185 .collect();
1186 let data_refs: Vec<(&str, TensorView<'_>)> =
1187 views.iter().map(|(k, v)| (k.as_str(), v.clone())).collect();
1188
1189 safetensors::tensor::serialize(data_refs, &None)
1190 .map_err(|e| anyhow!("safetensors serialization failed: {}", e))
1191}
1192
1193pub fn serialize_state_dict_to_bytes(state_dict: &HashMap<String, Tensor>) -> Result<Vec<u8>> {
1197 serialize_tensor_pairs_to_bytes(
1198 state_dict
1199 .iter()
1200 .map(|(k, v)| (k.clone(), v.shallow_clone())),
1201 )
1202}
1203
1204pub fn load_state_dict_from_bytes(data: &[u8]) -> Result<HashMap<String, Tensor>> {
1208 let tensors = SafeTensors::deserialize(data)
1209 .map_err(|e| anyhow!("safetensors deserialization failed: {}", e))?;
1210
1211 let mut result = HashMap::new();
1212 for (name, view) in tensors.tensors() {
1213 let tensor = safetensors_view_to_tensor(&view, Device::Cpu)?;
1214 result.insert(name, tensor);
1215 }
1216 Ok(result)
1217}
1218
1219fn tensor_bytes(t: &Tensor) -> usize {
1221 let numel = t.numel();
1222 let elem_size = match t.kind() {
1223 Kind::Half | Kind::BFloat16 => 2,
1224 Kind::Double => 8,
1225 _ => 4, };
1227 numel * elem_size
1228}
1229
1230unsafe impl Send for TenantDelta {}
1238unsafe impl Sync for TenantDelta {}
1239
1240pub type SharedTenantDelta = std::sync::Arc<parking_lot::Mutex<TenantDelta>>;
1243
1244#[cfg(test)]
1245#[allow(clippy::expect_used, clippy::unwrap_used)]
1246mod tests {
1247 use super::*;
1248
1249 const TEST_NUM_LAYERS: usize = 2;
1250
1251 fn test_module_dims() -> HashMap<String, (usize, usize)> {
1252 let mut dims = HashMap::new();
1253 dims.insert("q_proj".to_owned(), (512, 512));
1254 dims.insert("v_proj".to_owned(), (512, 512));
1255 dims
1256 }
1257
1258 #[test]
1259 fn test_tenant_delta_creation() {
1260 let config = TenantDeltaConfig::default();
1261 let dims = test_module_dims();
1262 let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1263
1264 assert_eq!(delta.rank, 8);
1265 assert_eq!(delta.target_modules.len(), 2);
1266 assert_eq!(delta.num_layers, TEST_NUM_LAYERS);
1267 assert_eq!(delta.lora_a.len(), 4);
1269 assert_eq!(delta.accumulated_steps, 0);
1270 assert_eq!(delta.request_count, 0);
1271 }
1272
1273 #[test]
1274 fn test_tenant_delta_forward_shape() {
1275 let config = TenantDeltaConfig::default();
1276 let dims = test_module_dims();
1277 let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1278
1279 let x = Tensor::randn([1, 10, 512], (Kind::Float, Device::Cpu));
1280 let output = delta.forward(&x, "q_proj", 0).unwrap();
1281 assert_eq!(output.size(), vec![1, 10, 512]);
1282
1283 let output1 = delta.forward(&x, "q_proj", 1).unwrap();
1285 assert_eq!(output1.size(), vec![1, 10, 512]);
1286 }
1287
1288 #[test]
1289 fn test_state_dict_roundtrip() {
1290 let config = TenantDeltaConfig::default();
1291 let dims = test_module_dims();
1292 let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1293
1294 let state = delta.extract_state_dict();
1295
1296 let mut delta2 = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1298 delta2.load_state_dict(&state).unwrap();
1299
1300 let diff: f64 = (&delta.lora_a["0.q_proj"] - &delta2.lora_a["0.q_proj"])
1302 .abs()
1303 .sum(Kind::Float)
1304 .double_value(&[]);
1305 assert!(diff < 1e-6, "State dict roundtrip should preserve values");
1306
1307 let diff1: f64 = (&delta.lora_a["1.q_proj"] - &delta2.lora_a["1.q_proj"])
1309 .abs()
1310 .sum(Kind::Float)
1311 .double_value(&[]);
1312 assert!(
1313 diff1 < 1e-6,
1314 "State dict roundtrip should preserve layer 1 values"
1315 );
1316 }
1317
1318 #[test]
1323 fn test_weight_epoch_bumps_on_in_place_mutation() {
1324 let config = TenantDeltaConfig::default();
1325 let dims = test_module_dims();
1326 let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1327 assert_eq!(delta.weight_epoch(), 0, "fresh delta starts at epoch 0");
1328
1329 delta.reset();
1331 assert_eq!(delta.weight_epoch(), 1, "reset bumps the weight epoch");
1332
1333 let state = delta.extract_state_dict();
1335 delta.load_state_dict(&state).unwrap();
1336 assert_eq!(
1337 delta.weight_epoch(),
1338 2,
1339 "state restore bumps the weight epoch"
1340 );
1341 }
1342
1343 #[test]
1344 fn test_memory_bytes() {
1345 let config = TenantDeltaConfig::default();
1346 let dims = test_module_dims();
1347 let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1348
1349 let bytes = delta.memory_bytes();
1350 let expected = TEST_NUM_LAYERS * 2 * (8 * 512 + 512 * 8) * 4;
1352 assert_eq!(bytes, expected);
1353 }
1354
1355 #[test]
1356 fn test_reset() {
1357 let config = TenantDeltaConfig::default();
1358 let dims = test_module_dims();
1359 let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1360
1361 delta.accumulated_steps = 100;
1362 delta.request_count = 50;
1363 delta.reset();
1364
1365 assert_eq!(delta.accumulated_steps, 0);
1366 assert_eq!(delta.request_count, 0);
1367
1368 let b_norm: f64 = delta.lora_b["0.q_proj"].norm().double_value(&[]);
1370 assert!(b_norm < 1e-8, "B should be zeros after reset");
1371 let b_norm1: f64 = delta.lora_b["1.q_proj"].norm().double_value(&[]);
1372 assert!(b_norm1 < 1e-8, "B layer 1 should be zeros after reset");
1373 }
1374
1375 #[test]
1376 fn test_forward_2d_shape() {
1377 let config = TenantDeltaConfig::default();
1378 let dims = test_module_dims();
1379 let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1380
1381 let x = Tensor::randn([10, 512], (Kind::Float, Device::Cpu));
1382 let output = delta.forward_2d(&x, "q_proj", 0).unwrap();
1383 assert_eq!(output.size(), vec![10, 512]);
1384 }
1385
1386 #[test]
1391 fn test_forward_bf16_input_fp32_delta_no_panic() {
1392 let config = TenantDeltaConfig {
1393 rank: 16,
1394 alpha: 32.0,
1395 target_modules: vec!["q_proj".to_owned(), "v_proj".to_owned()],
1396 ..Default::default()
1397 };
1398 let dims = test_module_dims();
1399 let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1400
1401 assert_eq!(delta.lora_a["0.q_proj"].kind(), Kind::Float);
1403 assert_eq!(delta.lora_b["0.q_proj"].kind(), Kind::Float);
1404
1405 let x_2d = Tensor::randn([10, 512], (Kind::Float, Device::Cpu)).to_kind(Kind::BFloat16);
1407
1408 let corr_2d = delta.forward_2d(&x_2d, "q_proj", 0).unwrap();
1410 assert_eq!(corr_2d.kind(), Kind::BFloat16);
1411 assert_eq!(corr_2d.size(), vec![10, 512]);
1412
1413 let base_2d = Tensor::randn([10, 512], (Kind::Float, Device::Cpu)).to_kind(Kind::BFloat16);
1416 let combined = &base_2d + &corr_2d;
1417 assert_eq!(combined.kind(), Kind::BFloat16);
1418 let _ = combined.sum(Kind::Float).double_value(&[]);
1419
1420 let x_3d = Tensor::randn([1, 10, 512], (Kind::Float, Device::Cpu)).to_kind(Kind::BFloat16);
1422 let corr_3d = delta.forward(&x_3d, "v_proj", 0).unwrap();
1423 assert_eq!(corr_3d.kind(), Kind::BFloat16);
1424 assert_eq!(corr_3d.size(), vec![1, 10, 512]);
1425
1426 let x_f32 = Tensor::randn([10, 512], (Kind::Float, Device::Cpu));
1428 let corr_f32 = delta.forward_2d(&x_f32, "q_proj", 0).unwrap();
1429 assert_eq!(corr_f32.kind(), Kind::Float);
1430 }
1431
1432 #[test]
1433 fn test_forward_2d_gradient_flow() {
1434 let config = TenantDeltaConfig {
1435 rank: 4,
1436 alpha: 4.0,
1437 ..Default::default()
1438 };
1439 let dims = test_module_dims();
1440 let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1441
1442 let x = Tensor::randn([5, 512], (Kind::Float, Device::Cpu));
1443
1444 let output = delta.forward_2d(&x, "q_proj", 0).unwrap();
1445
1446 let loss = output.sum(Kind::Float);
1447 loss.backward();
1448
1449 let variables = delta.vs.trainable_variables();
1450 let mut has_grad = false;
1451 for var in &variables {
1452 if var.grad().defined() {
1453 let grad_norm: f64 = var.grad().norm().double_value(&[]);
1454 if grad_norm > 0.0 {
1455 has_grad = true;
1456 }
1457 }
1458 }
1459 assert!(
1460 has_grad,
1461 "Delta A/B matrices should have gradients after backward()"
1462 );
1463 }
1464
1465 #[test]
1466 fn test_has_module() {
1467 let config = TenantDeltaConfig::default();
1468 let dims = test_module_dims();
1469 let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1470
1471 assert!(delta.has_module("q_proj", 0));
1472 assert!(delta.has_module("q_proj", 1));
1473 assert!(delta.has_module("v_proj", 0));
1474 assert!(!delta.has_module("gate_proj", 0)); }
1476
1477 #[test]
1478 fn test_effective_rank_default_is_max() {
1479 let config = TenantDeltaConfig {
1480 rank: 8,
1481 ..Default::default()
1482 };
1483 let dims = test_module_dims();
1484 let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1485
1486 assert_eq!(delta.effective_rank("0.q_proj"), Some(8));
1488 assert_eq!(delta.effective_rank("0.v_proj"), Some(8));
1489 assert_eq!(delta.effective_rank("99.nonexistent"), None);
1490 }
1491
1492 #[test]
1493 fn test_effective_rank_forward() {
1494 let config = TenantDeltaConfig {
1495 rank: 8,
1496 ..Default::default()
1497 };
1498 let dims = test_module_dims();
1499 let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1500
1501 let x = Tensor::randn([2, 512], (Kind::Float, Device::Cpu));
1502
1503 let out_full = delta.forward_2d(&x, "q_proj", 0).unwrap();
1505 assert_eq!(out_full.size(), &[2, 512]);
1506
1507 delta.set_effective_rank("0.q_proj", 4);
1509 let out_narrow = delta.forward_2d(&x, "q_proj", 0).unwrap();
1510 assert_eq!(out_narrow.size(), &[2, 512]); }
1512
1513 #[test]
1514 fn test_set_effective_rank_clamped() {
1515 let config = TenantDeltaConfig {
1516 rank: 8,
1517 ..Default::default()
1518 };
1519 let dims = test_module_dims();
1520 let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1521
1522 delta.set_effective_rank("0.q_proj", 32);
1524 assert_eq!(delta.effective_rank("0.q_proj"), Some(8)); delta.set_effective_rank("0.q_proj", 0);
1528 assert_eq!(delta.effective_rank("0.q_proj"), Some(1));
1529 }
1530
1531 #[test]
1532 fn test_effective_rank_scaling_adjusts() {
1533 let config = TenantDeltaConfig {
1534 rank: 8,
1535 alpha: 4.0,
1536 ..Default::default()
1537 };
1538 let dims = test_module_dims();
1539 let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1540
1541 let s0 = delta.scaling_map["0.q_proj"];
1543 assert!((s0 - 0.5).abs() < 1e-9);
1544
1545 delta.set_effective_rank("0.q_proj", 4);
1547 let s1 = delta.scaling_map["0.q_proj"];
1548 assert!((s1 - 1.0).abs() < 1e-9);
1549 }
1550
1551 #[test]
1552 fn test_effective_rank_with_layer_overrides() {
1553 let mut overrides = HashMap::new();
1554 overrides.insert(
1555 0,
1556 LayerDeltaConfig {
1557 rank: 16,
1558 target_modules: vec!["q_proj".to_owned(), "v_proj".to_owned()],
1559 },
1560 );
1561 let config = TenantDeltaConfig {
1562 rank: 8,
1563 layer_overrides: Some(overrides),
1564 ..Default::default()
1565 };
1566 let dims = test_module_dims();
1567 let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1568
1569 assert_eq!(delta.effective_rank("0.q_proj"), Some(16));
1571 assert_eq!(delta.effective_rank("1.q_proj"), Some(8));
1572 }
1573
1574 #[test]
1575 fn test_state_dict_round_trip_preserves_effective_ranks() {
1576 let config = TenantDeltaConfig {
1577 rank: 8,
1578 alpha: 4.0,
1579 ..Default::default()
1580 };
1581 let dims = test_module_dims();
1582 let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1583
1584 delta.set_effective_rank("0.q_proj", 4);
1586 delta.set_effective_rank("1.v_proj", 2);
1587
1588 let state = delta.extract_state_dict();
1589
1590 assert!(state.contains_key("0.q_proj.__effective_rank"));
1592 assert!(state.contains_key("1.v_proj.__effective_rank"));
1593
1594 let mut delta2 = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1596 assert_eq!(delta2.effective_rank("0.q_proj"), Some(8)); delta2.load_state_dict(&state).unwrap();
1598
1599 assert_eq!(delta2.effective_rank("0.q_proj"), Some(4));
1601 assert_eq!(delta2.effective_rank("1.v_proj"), Some(2));
1602 assert_eq!(delta2.effective_rank("0.v_proj"), Some(8));
1604 }
1605
1606 #[test]
1607 fn test_state_dict_round_trip_preserves_muon_momentum() {
1608 let config = TenantDeltaConfig {
1609 rank: 8,
1610 alpha: 4.0,
1611 ..Default::default()
1612 };
1613 let dims = test_module_dims();
1614 let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1615
1616 let key = "0.q_proj".to_owned();
1618 let fake_momentum = Tensor::randn([8, 512], (Kind::Float, Device::Cpu));
1619 delta.muon_states.insert(
1620 key.clone(),
1621 super::super::muon::MuonState {
1622 momentum_buffer: Some(fake_momentum.copy()),
1623 },
1624 );
1625
1626 let state = delta.extract_state_dict();
1627
1628 assert!(state.contains_key("0.q_proj.__muon_momentum"));
1630
1631 let mut delta2 = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1633 delta2.muon_states.insert(
1634 key.clone(),
1635 super::super::muon::MuonState {
1636 momentum_buffer: None,
1637 },
1638 );
1639 delta2.load_state_dict(&state).unwrap();
1640
1641 let restored = &delta2.muon_states[&key];
1643 assert!(restored.momentum_buffer.is_some());
1644 let diff: f64 = (restored.momentum_buffer.as_ref().unwrap() - &fake_momentum)
1645 .abs()
1646 .sum(Kind::Float)
1647 .double_value(&[]);
1648 assert!(
1649 diff < 1e-6,
1650 "Muon momentum should be preserved through state_dict round-trip"
1651 );
1652 }
1653}