Skip to main content

hyprstream_inference/training/
tenant_delta.rs

1//! Per-tenant LoRA delta for isolated Test-Time Training
2//!
3//! Each tenant gets their own LoRA delta (A/B weight matrices) that accumulates
4//! adaptations across requests. Deltas are trained via SGD and can be persisted
5//! to content-addressed storage or merged into permanent adapter files.
6
7use super::muon::{MuonConfig, MuonState};
8use anyhow::{anyhow, Result};
9use safetensors::tensor::{SafeTensors, TensorView};
10use serde::{Deserialize, Serialize};
11use std::collections::HashMap;
12use std::time::Instant;
13use tch::nn::VarStore;
14use tch::{Device, Kind, Tensor};
15
16/// Extract LoRA components from hierarchical tensor names
17/// e.g., "base_model.model.layers.5.self_attn.q_proj.lora_A.weight" -> Some((5, "q_proj", "lora_a"))
18pub fn extract_lora_components(tensor_name: &str) -> Option<(usize, String, String)> {
19    let parts: Vec<&str> = tensor_name.split('.').collect();
20
21    // Find the lora_a/lora_b/lora_A/lora_B component
22    for (i, &part) in parts.iter().enumerate() {
23        let normalized = part.to_lowercase();
24        if (normalized == "lora_a" || normalized == "lora_b") && i > 0 {
25            let module = parts[i - 1];
26            if matches!(
27                module,
28                "q_proj" | "k_proj" | "v_proj" | "o_proj" | "gate_proj" | "up_proj" | "down_proj"
29            ) {
30                // Find layer index: look for "layers" followed by a number
31                let layer_idx = parts
32                    .iter()
33                    .zip(parts.iter().skip(1))
34                    .find_map(|(&a, &b)| {
35                        if a == "layers" {
36                            b.parse::<usize>().ok()
37                        } else {
38                            None
39                        }
40                    })
41                    .unwrap_or(0);
42                return Some((layer_idx, module.to_owned(), normalized));
43            }
44        }
45    }
46
47    None
48}
49
50/// Per-layer LoRA rank and target module override.
51///
52/// When present in `TenantDeltaConfig::layer_overrides`, these values replace
53/// the top-level `rank` and `target_modules` for the specified layer.
54#[derive(Debug, Clone, Serialize, Deserialize)]
55pub struct LayerDeltaConfig {
56    pub rank: usize,
57    pub target_modules: Vec<String>,
58}
59
60/// Configuration for creating a new tenant delta (also used by the create_lora MCP tool)
61///
62/// This is the single LoRA configuration type. TTT and PEFT differ only in lifecycle
63/// (online vs frozen), not structure.
64#[derive(Debug, Clone, Serialize, Deserialize)]
65pub struct TenantDeltaConfig {
66    /// LoRA rank (default: 8)
67    #[serde(default = "default_rank")]
68    pub rank: usize,
69
70    /// LoRA alpha scaling factor (default: 4.0, < rank for regularization)
71    #[serde(default = "default_alpha")]
72    pub alpha: f32,
73
74    /// Dropout probability (default: 0.0, unused by TTT but accepted by create_lora RPC)
75    #[serde(default)]
76    pub dropout: f32,
77
78    /// Target modules to apply LoRA to (default: ["q_proj", "v_proj"])
79    #[serde(default = "default_target_modules")]
80    pub target_modules: Vec<String>,
81
82    /// Learning rate for Muon optimizer (default: 1e-3).
83    /// Muon's Newton-Schulz amplifies effective step ~22x for rank-8 LoRA B matrices.
84    #[serde(default = "default_learning_rate")]
85    pub learning_rate: f64,
86
87    /// Maximum accumulated gradient steps before requiring save/merge (default: 300)
88    #[serde(default = "default_max_accumulated_steps")]
89    pub max_accumulated_steps: u64,
90
91    /// Weight decay factor for Muon optimizer (default: 0.0).
92    /// Decoupled weight decay applied as `p *= (1 - lr * decay_lambda)` per step.
93    /// Keep at 0.0 for LoRA-B (zero-initialized) — decay fights learned adaptation.
94    #[serde(default = "default_decay_lambda")]
95    pub decay_lambda: f64,
96
97    /// Muon momentum coefficient (default: 0.9).
98    /// Lower values reduce oscillation risk with higher learning rates.
99    #[serde(default = "default_muon_momentum_config")]
100    pub muon_momentum: f64,
101
102    /// Per-layer rank/module override. Key = layer index.
103    /// Layers not in map use the top-level `rank` and `target_modules`.
104    #[serde(default)]
105    pub layer_overrides: Option<HashMap<usize, LayerDeltaConfig>>,
106}
107
108impl TenantDeltaConfig {
109    /// Create config with non-uniform per-layer ranks from a TTN analysis profile.
110    ///
111    /// Works for any model — the profile is produced by `ttn_profile::get_layer_profile()`.
112    pub fn from_profile(
113        base: &TenantDeltaConfig,
114        profile: &crate::runtime::ttn_profile::LayerProfile,
115    ) -> Self {
116        Self::from_profile_with_quant_caps(base, profile, None)
117    }
118
119    /// Like [`from_profile`](Self::from_profile) but applies the **static
120    /// rank↔quant coupling cap** (#326): the oracle's recommended per-layer rank
121    /// is clamped by the layer's stored dtype so a coarse-quant layer (e.g.
122    /// block-wise FP8) is not assigned a rank it cannot usefully express.
123    ///
124    /// `layer_dtype_bytes` maps `layer_idx -> stored-weight element size in
125    /// bytes` (1 = FP8/int8, 2 = FP16/BF16, 4 = FP32). Layers absent from the
126    /// map (or `None` for the whole map) are left uncapped — identical to
127    /// `from_profile`.
128    ///
129    /// This is **static** (applied once, here, when the config is built). It is
130    /// deliberately NOT wired into the live `auto_adapt` rank-oracle path, which
131    /// must stay free to adapt rank at runtime without re-deriving quant caps.
132    pub fn from_profile_with_quant_caps(
133        base: &TenantDeltaConfig,
134        profile: &crate::runtime::ttn_profile::LayerProfile,
135        layer_dtype_bytes: Option<&HashMap<usize, usize>>,
136    ) -> Self {
137        let mut overrides = HashMap::new();
138        for la in &profile.layers {
139            let cap = layer_dtype_bytes
140                .and_then(|m| m.get(&la.layer_idx))
141                .map(|&bytes| crate::training::delta_reducer::rank_cap_for_dtype_bytes(bytes))
142                .unwrap_or(usize::MAX);
143            let capped_rank = la.recommended_rank.min(cap).max(1);
144            overrides.insert(
145                la.layer_idx,
146                LayerDeltaConfig {
147                    rank: capped_rank,
148                    target_modules: la.target_modules.clone(),
149                },
150            );
151        }
152        Self {
153            layer_overrides: Some(overrides),
154            ..base.clone()
155        }
156    }
157}
158
159fn default_rank() -> usize {
160    8
161}
162fn default_alpha() -> f32 {
163    4.0
164}
165fn default_target_modules() -> Vec<String> {
166    vec!["q_proj".to_owned(), "v_proj".to_owned()]
167}
168fn default_learning_rate() -> f64 {
169    1e-3 // Muon's Newton-Schulz amplifies effective step ~22x for rank-8 B matrices;
170         // 1e-3 gives effective step ~0.023, 3x the previous 3e-4 while staying stable.
171}
172fn default_max_accumulated_steps() -> u64 {
173    300
174}
175fn default_decay_lambda() -> f64 {
176    0.0 // Weight decay fights LoRA-B (initialized to zero) — keep off by default.
177        // The field is now wired to MuonConfig.weight_decay so it works if configured.
178}
179fn default_muon_momentum_config() -> f64 {
180    0.9 // Reduced from 0.95 to match higher LR. Product lr*momentum/(1-momentum)
181        // stays controlled: 1e-3 * 0.9/0.1 = 0.009.
182}
183
184impl Default for TenantDeltaConfig {
185    fn default() -> Self {
186        Self {
187            rank: default_rank(),
188            alpha: default_alpha(),
189            dropout: 0.0,
190            target_modules: default_target_modules(),
191            learning_rate: default_learning_rate(),
192            max_accumulated_steps: default_max_accumulated_steps(),
193            decay_lambda: default_decay_lambda(),
194            muon_momentum: default_muon_momentum_config(),
195            layer_overrides: None,
196        }
197    }
198}
199
200/// Per-tenant LoRA delta with per-layer accumulation tracking
201///
202/// Each tenant gets isolated per-layer A/B weight matrices. Keys use the
203/// format `"layer_idx.module_name"` (e.g., `"0.q_proj"`, `"27.v_proj"`).
204/// The delta accumulates across requests and can be periodically saved or
205/// merged into a permanent adapter.
206pub struct TenantDelta {
207    /// LoRA A matrices: "layer_idx.module_name" -> [rank, in_features]
208    pub lora_a: HashMap<String, Tensor>,
209    /// LoRA B matrices: "layer_idx.module_name" -> [out_features, rank]
210    pub lora_b: HashMap<String, Tensor>,
211    /// VarStore owning all trainable parameters
212    pub vs: VarStore,
213    /// Per-parameter Muon optimizer state: VarStore path -> MuonState
214    pub muon_states: HashMap<String, MuonState>,
215    /// Muon optimizer configuration
216    pub muon_config: MuonConfig,
217    /// Per-key scaling: "layer_idx.module_name" -> alpha / layer_rank
218    /// Replaces the single `scaling` field for non-uniform rank support (C7 fix).
219    pub scaling_map: HashMap<String, f64>,
220    /// Default scaling factor: alpha / rank (for uniform-rank deltas or backward compat)
221    pub scaling: f64,
222    /// LoRA rank
223    pub rank: usize,
224    /// Device for tensors
225    pub device: Device,
226    /// Target module names (without layer prefix)
227    pub target_modules: Vec<String>,
228    /// Number of model layers
229    pub num_layers: usize,
230    /// Learning rate for optimizer
231    pub learning_rate: f64,
232
233    // Accumulation tracking
234    /// Total gradient steps accumulated
235    pub accumulated_steps: u64,
236    /// Maximum steps before requiring save/merge
237    pub max_accumulated_steps: u64,
238    /// Number of inference requests served
239    pub request_count: u64,
240    /// Running average of loss improvement per adaptation
241    pub avg_loss_improvement: f64,
242    /// Last time this delta was accessed
243    pub last_access: Instant,
244    /// When this delta was created
245    pub created_at: Instant,
246    /// Hash of last CAS snapshot (if any)
247    pub last_snapshot_hash: Option<String>,
248    /// Weight decay factor (used by Muon's decoupled weight decay)
249    pub decay_lambda: f64,
250    /// Per-key effective rank (for narrow-based rank adaptation).
251    /// Key = "layer_idx.module_name", value = active rank (1..=allocated_rank).
252    /// Defaults to the allocated rank for each key.
253    pub effective_ranks: HashMap<String, usize>,
254    /// Per-key maximum (allocated) rank. Set at creation time, never changes.
255    max_ranks: HashMap<String, usize>,
256    /// Alpha value (stored for scaling recalculation on rank change)
257    alpha: f32,
258    /// Optional per-tenant rank oracle for runtime rank adaptation
259    pub rank_oracle: Option<super::ttt::RankOracle>,
260    /// Lifecycle state of any pending adaptation (Idle or Pending with snapshot).
261    pub adaptation_state: crate::training::adaptation_state::DeltaAdaptationState,
262    /// Weight epoch: a version counter bumped on every in-place weight mutation
263    /// (adaptation, train step, rollback/restore, reset). Session KV caches record
264    /// the epoch they were computed under; a mismatch on reuse means the cached
265    /// KV is weight-stale and must be cleared (#1254 F1 fix).
266    weight_epoch: u64,
267}
268
269impl TenantDelta {
270    /// Create a new per-layer tenant delta with Kaiming init for A and zeros for B.
271    ///
272    /// Creates `num_layers × num_modules` A/B pairs keyed as `"layer_idx.module_name"`.
273    ///
274    /// # Arguments
275    /// * `config` - Delta configuration (rank, alpha, target modules, layer_overrides, etc.)
276    /// * `module_dims` - Map of module_name -> (in_features, out_features) (flat, for uniform layers)
277    /// * `device` - Device for tensor allocation
278    /// * `num_layers` - Number of model layers
279    /// * `per_layer_dims` - Optional per-layer dim overrides: layer_idx -> module_name -> (in, out)
280    ///   Required when different layer types have different `o_proj` dimensions (e.g. Qwen3.5).
281    pub fn new(
282        config: &TenantDeltaConfig,
283        module_dims: &HashMap<String, (usize, usize)>,
284        device: Device,
285        num_layers: usize,
286    ) -> Result<Self> {
287        Self::new_with_per_layer_dims(config, module_dims, device, num_layers, None)
288    }
289
290    /// Create a new per-layer tenant delta with optional per-layer dimension overrides.
291    ///
292    /// Same as `new()` but accepts `per_layer_dims` for architectures where
293    /// `o_proj` (or other modules) have different dimensions across layer types.
294    pub fn new_with_per_layer_dims(
295        config: &TenantDeltaConfig,
296        module_dims: &HashMap<String, (usize, usize)>,
297        device: Device,
298        num_layers: usize,
299        per_layer_dims: Option<&HashMap<usize, HashMap<String, (usize, usize)>>>,
300    ) -> Result<Self> {
301        let vs = VarStore::new(device);
302        let root = vs.root();
303
304        let mut lora_a = HashMap::new();
305        let mut lora_b = HashMap::new();
306        let mut scaling_map = HashMap::new();
307        let mut effective_ranks = HashMap::new();
308        let mut max_ranks = HashMap::new();
309
310        for layer_idx in 0..num_layers {
311            // Determine layer-specific rank and modules
312            let (layer_rank, layer_modules): (usize, &Vec<String>) =
313                if let Some(overrides) = &config.layer_overrides {
314                    if let Some(lc) = overrides.get(&layer_idx) {
315                        (lc.rank, &lc.target_modules)
316                    } else {
317                        (config.rank, &config.target_modules)
318                    }
319                } else {
320                    (config.rank, &config.target_modules)
321                };
322
323            for module_name in layer_modules {
324                // Per-layer dims take priority over flat module_dims (C1 fix)
325                let (in_features, out_features) = per_layer_dims
326                    .and_then(|pld| pld.get(&layer_idx))
327                    .and_then(|m| m.get(module_name.as_str()))
328                    .or_else(|| module_dims.get(module_name.as_str()))
329                    .ok_or_else(|| {
330                        anyhow!(
331                            "Module '{}' not found in model dimensions (layer {})",
332                            module_name,
333                            layer_idx
334                        )
335                    })?;
336
337                let key = format!("{}.{}", layer_idx, module_name);
338
339                // Use Path::sub() for hierarchy — '.' is VarStore's path separator
340                let layer_path = root.sub(format!("layer_{}", layer_idx)).sub(module_name);
341
342                // A: Kaiming uniform initialization [rank, in_features]
343                let a =
344                    layer_path.kaiming_uniform("lora_a", &[layer_rank as i64, *in_features as i64]);
345
346                // B: Zero initialization [out_features, rank]
347                let b = layer_path.zeros("lora_b", &[*out_features as i64, layer_rank as i64]);
348
349                lora_a.insert(key.clone(), a);
350                lora_b.insert(key.clone(), b);
351
352                // Per-key scaling: alpha / layer_rank (C7 fix)
353                scaling_map.insert(key.clone(), config.alpha as f64 / layer_rank as f64);
354
355                // Track effective and max ranks for narrow-based adaptation
356                effective_ranks.insert(key.clone(), layer_rank);
357                max_ranks.insert(key, layer_rank);
358            }
359        }
360
361        // Build Muon optimizer state (created lazily on first step)
362        let muon_states = HashMap::new();
363        let muon_config = MuonConfig {
364            lr: config.learning_rate,
365            momentum: config.muon_momentum,
366            weight_decay: config.decay_lambda, // Wired from config; default 0.0 (off)
367            ..MuonConfig::default()
368        };
369
370        let scaling = config.alpha as f64 / config.rank as f64;
371        let now = Instant::now();
372
373        // Collect unique target modules across all layers for iteration metadata
374        let mut all_modules: Vec<String> = config.target_modules.clone();
375        if let Some(overrides) = &config.layer_overrides {
376            for lc in overrides.values() {
377                for m in &lc.target_modules {
378                    if !all_modules.contains(m) {
379                        all_modules.push(m.clone());
380                    }
381                }
382            }
383        }
384
385        Ok(Self {
386            lora_a,
387            lora_b,
388            vs,
389            muon_states,
390            muon_config,
391            scaling_map,
392            scaling,
393            rank: config.rank,
394            device,
395            target_modules: all_modules,
396            num_layers,
397            learning_rate: config.learning_rate,
398            accumulated_steps: 0,
399            max_accumulated_steps: config.max_accumulated_steps,
400            request_count: 0,
401            avg_loss_improvement: 0.0,
402            last_access: now,
403            created_at: now,
404            last_snapshot_hash: None,
405            decay_lambda: config.decay_lambda,
406            effective_ranks,
407            max_ranks,
408            alpha: config.alpha,
409            rank_oracle: None,
410            adaptation_state: crate::training::adaptation_state::DeltaAdaptationState::Idle,
411            weight_epoch: 0,
412        })
413    }
414
415    /// Check if this delta has a module at the given layer
416    pub fn has_module(&self, module_name: &str, layer_idx: usize) -> bool {
417        self.lora_a
418            .contains_key(&format!("{}.{}", layer_idx, module_name))
419    }
420
421    /// Compute LoRA correction for a given module at a specific layer: output += scaling * (x @ A^T) @ B^T
422    ///
423    /// # Arguments
424    /// * `x` - Input tensor [batch, seq_len, in_features]
425    /// * `module_name` - Name of the target module
426    /// * `layer_idx` - Layer index
427    ///
428    /// # Returns
429    /// LoRA correction tensor [batch, seq_len, out_features]
430    pub fn forward(&self, x: &Tensor, module_name: &str, layer_idx: usize) -> Result<Tensor> {
431        let key = format!("{}.{}", layer_idx, module_name);
432        let a = self
433            .lora_a
434            .get(&key)
435            .ok_or_else(|| anyhow!("Module '{}' not found in delta", key))?;
436        let b = self
437            .lora_b
438            .get(&key)
439            .ok_or_else(|| anyhow!("Module '{}' B not found in delta", key))?;
440
441        // Per-key scaling (C7 fix): use per-key alpha/rank, fall back to global scaling
442        let scaling = self.scaling_map.get(&key).copied().unwrap_or(self.scaling);
443
444        // Narrow to effective rank (view, no copy)
445        let eff_rank = self
446            .effective_ranks
447            .get(&key)
448            .copied()
449            .unwrap_or_else(|| a.size()[0] as usize) as i64;
450        let a_eff = a.narrow(0, 0, eff_rank); // [eff_rank, in_features]
451        let b_eff = b.narrow(1, 0, eff_rank); // [out_features, eff_rank]
452
453        // Compute the LoRA correction in A/B's dtype (fp32, see forward_2d docs), then
454        // cast the result back to the input activation's dtype so callers can add it to
455        // base-dtype tensors. Without this, a BF16 base activation + an fp32 correction
456        // panics on CPU (addmm requires matching operand dtypes). Casting back is a
457        // device-agnostic no-op when x already matches a.kind().
458        let input_kind = x.kind();
459        let x = x.to_kind(a.kind());
460        let intermediate = x
461            .f_matmul(&a_eff.tr())
462            .map_err(|e| anyhow!("Delta forward matmul A failed for '{}': {}", key, e))?;
463        let output = intermediate
464            .f_matmul(&b_eff.tr())
465            .map_err(|e| anyhow!("Delta forward matmul B failed for '{}': {}", key, e))?;
466
467        Ok((output * scaling).to_kind(input_kind))
468    }
469
470    /// Compute LoRA correction for 2D tensors at a specific layer: output += scaling * (x @ A^T) @ B^T
471    ///
472    /// Handles shape [batch*seq_len, features] used inside attention layers.
473    ///
474    /// **Multi-precision:** Input `x` is cast to `a.kind()` (always Float32) via `x.to_kind()`.
475    /// LoRA A/B are kept in Float32 regardless of the base model's dtype. This ensures
476    /// Newton-Schulz convergence in the Muon optimizer and gradient precision within
477    /// TTT's tight 1-5 step budget. See also `muon::ns_compute_kind()`.
478    ///
479    /// # Arguments
480    /// * `x` - Input tensor [tokens, in_features]
481    /// * `module_name` - Name of the target module (e.g., "q_proj", "v_proj")
482    /// * `layer_idx` - Layer index
483    ///
484    /// # Returns
485    /// LoRA correction tensor [tokens, out_features]
486    pub fn forward_2d(&self, x: &Tensor, module_name: &str, layer_idx: usize) -> Result<Tensor> {
487        let key = format!("{}.{}", layer_idx, module_name);
488        let a = self
489            .lora_a
490            .get(&key)
491            .ok_or_else(|| anyhow!("Module '{}' not found in delta", key))?;
492        let b = self
493            .lora_b
494            .get(&key)
495            .ok_or_else(|| anyhow!("Module '{}' B not found in delta", key))?;
496
497        // Per-key scaling (C7 fix): use per-key alpha/rank, fall back to global scaling
498        let scaling = self.scaling_map.get(&key).copied().unwrap_or(self.scaling);
499
500        // Narrow to effective rank (view, no copy)
501        let eff_rank = self
502            .effective_ranks
503            .get(&key)
504            .copied()
505            .unwrap_or_else(|| a.size()[0] as usize) as i64;
506        let a_eff = a.narrow(0, 0, eff_rank); // [eff_rank, in_features]
507        let b_eff = b.narrow(1, 0, eff_rank); // [out_features, eff_rank]
508
509        // Compute in A/B's dtype (fp32 — see doc above for the Muon/NS rationale), then
510        // cast the correction back to the input activation's dtype. The base model is
511        // often BF16; the caller adds this correction to a BF16 tensor, and on CPU a
512        // BF16 + fp32 add panics (addmm requires matching dtypes). The cast back is a
513        // device-agnostic no-op when input_kind already equals a.kind().
514        let input_kind = x.kind();
515        let input_device = x.device();
516        let x = x.to_kind(a.kind());
517        // Pipeline-split (#316) cross-device autograd: the delta's VarStore lives on
518        // a single `self.device`, but under a layer-split TTT the activation `x` may
519        // be on a different device than this layer's delta params. Move A/B onto the
520        // activation's device so the matmuls are device-consistent; `to_device` is
521        // autograd-transparent, so gradients flow back to the params on `self.device`.
522        // On the single-device path both moves are no-op views (source == dest),
523        // keeping the existing TTT behavior byte-identical.
524        let a_eff = a_eff.to_device(input_device);
525        let b_eff = b_eff.to_device(input_device);
526        let intermediate = x
527            .f_matmul(&a_eff.tr())
528            .map_err(|e| anyhow!("Delta forward_2d matmul A failed for '{}': {}", key, e))?;
529        let output = intermediate
530            .f_matmul(&b_eff.tr())
531            .map_err(|e| anyhow!("Delta forward_2d matmul B failed for '{}': {}", key, e))?;
532
533        Ok((output * scaling).to_kind(input_kind))
534    }
535
536    /// Compute the ratio of delta norm to a reference norm for drift monitoring
537    ///
538    /// Returns a map of "layer_idx.module_name" -> ||delta|| / ||base|| where delta is
539    /// the effective weight change (B @ A) and base is the reference norm.
540    pub fn delta_norm_ratio(&self, base_norms: &HashMap<String, f64>) -> HashMap<String, f64> {
541        let _guard = tch::no_grad_guard();
542        let mut ratios = HashMap::new();
543
544        // Iterate lora_a.keys() to handle non-uniform per-layer module sets (I1 fix)
545        for key in self.lora_a.keys() {
546            if let (Some(a), Some(b)) = (self.lora_a.get(key), self.lora_b.get(key)) {
547                let scaling = self.scaling_map.get(key).copied().unwrap_or(self.scaling);
548                let delta = b.matmul(a) * scaling;
549                let delta_norm: f64 = delta.norm().double_value(&[]);
550                // Extract module name from key "layer_idx.module_name"
551                let module_name = key.split_once('.').map_or(key.as_str(), |(_, m)| m);
552                let base_norm = base_norms.get(module_name).copied().unwrap_or(1.0);
553                ratios.insert(key.clone(), delta_norm / base_norm.max(1e-8));
554            }
555        }
556
557        ratios
558    }
559
560    /// Extract state dict (A and B tensors, effective ranks, Muon momentum) for serialization
561    ///
562    /// Keys use the format `"layer_idx.module_name.lora_a"` / `"layer_idx.module_name.lora_b"`.
563    /// Effective ranks are stored as `"layer_idx.module_name.__effective_rank"` (scalar i64).
564    /// Muon momentum buffers are stored as `"layer_idx.module_name.__muon_momentum"`.
565    pub fn extract_state_dict(&self) -> HashMap<String, Tensor> {
566        let _guard = tch::no_grad_guard();
567        let mut state = HashMap::new();
568
569        // Iterate lora_a.keys() to handle non-uniform per-layer module sets (I1 fix)
570        for key in self.lora_a.keys() {
571            if let Some(a) = self.lora_a.get(key) {
572                state.insert(format!("{}.lora_a", key), a.copy());
573            }
574            if let Some(b) = self.lora_b.get(key) {
575                state.insert(format!("{}.lora_b", key), b.copy());
576            }
577        }
578
579        // Persist effective ranks as scalar metadata tensors
580        for (key, &rank) in &self.effective_ranks {
581            state.insert(
582                format!("{}.__effective_rank", key),
583                Tensor::from_slice(&[rank as i64]),
584            );
585        }
586
587        // Persist Muon momentum buffers
588        for (key, muon_state) in &self.muon_states {
589            if let Some(ref buf) = muon_state.momentum_buffer {
590                state.insert(format!("{}.__muon_momentum", key), buf.copy());
591            }
592        }
593
594        state
595    }
596
597    /// Load state dict (restore A/B tensors, effective ranks, and Muon momentum from a snapshot)
598    ///
599    /// Expects keys in format `"layer_idx.module_name.lora_a"` / `"layer_idx.module_name.lora_b"`.
600    /// Also restores `__effective_rank` and `__muon_momentum` metadata if present.
601    pub fn load_state_dict(&mut self, state: &HashMap<String, Tensor>) -> Result<()> {
602        let _guard = tch::no_grad_guard();
603
604        for key in self.lora_a.keys().cloned().collect::<Vec<_>>() {
605            let a_key = format!("{}.lora_a", key);
606            let b_key = format!("{}.lora_b", key);
607
608            if let Some(a_src) = state.get(&a_key) {
609                if let Some(a_dst) = self.lora_a.get_mut(&key) {
610                    a_dst.copy_(a_src);
611                }
612            }
613
614            if let Some(b_src) = state.get(&b_key) {
615                if let Some(b_dst) = self.lora_b.get_mut(&key) {
616                    b_dst.copy_(b_src);
617                }
618            }
619        }
620
621        // Restore effective ranks from metadata tensors
622        for key in self.effective_ranks.keys().cloned().collect::<Vec<_>>() {
623            let meta_key = format!("{}.__effective_rank", key);
624            if let Some(t) = state.get(&meta_key) {
625                let rank = t.int64_value(&[]) as usize;
626                self.set_effective_rank(&key, rank);
627            }
628        }
629
630        // Restore Muon momentum buffers from metadata tensors
631        for (key, muon_state) in &mut self.muon_states {
632            let meta_key = format!("{}.__muon_momentum", key);
633            if let Some(t) = state.get(&meta_key) {
634                if let Some(ref mut buf) = muon_state.momentum_buffer {
635                    buf.copy_(t);
636                } else {
637                    muon_state.momentum_buffer = Some(t.copy());
638                }
639            }
640        }
641
642        // Restoring a state dict changes the effective weights in place
643        // (rollback / snapshot rehydrate) — bump the weight epoch so session
644        // KV computed under the previous weights is treated as stale.
645        self.bump_weight_epoch();
646
647        Ok(())
648    }
649
650    /// Update last access time
651    pub fn touch(&mut self) {
652        self.last_access = Instant::now();
653    }
654
655    /// Current weight epoch (version counter for KV-cache staleness detection).
656    pub fn weight_epoch(&self) -> u64 {
657        self.weight_epoch
658    }
659
660    /// Bump the weight epoch. Must be called on every in-place mutation of the
661    /// effective weights that happens without a remove/re-add of the delta —
662    /// adaptations, train steps, rollback restores, resets. O(1); the session
663    /// KV layer compares epochs on reuse and clears weight-stale caches.
664    pub fn bump_weight_epoch(&mut self) {
665        self.weight_epoch = self.weight_epoch.wrapping_add(1);
666    }
667
668    /// Estimate memory usage in bytes
669    pub fn memory_bytes(&self) -> usize {
670        let _guard = tch::no_grad_guard();
671        let mut total = 0usize;
672
673        for a in self.lora_a.values() {
674            total += tensor_bytes(a);
675        }
676        for b in self.lora_b.values() {
677            total += tensor_bytes(b);
678        }
679
680        total
681    }
682
683    /// Check if the delta has reached its accumulation limit
684    pub fn is_at_capacity(&self) -> bool {
685        self.accumulated_steps >= self.max_accumulated_steps
686    }
687
688    /// Reset the delta to zeros (re-initialize B matrices, keep A)
689    pub fn reset(&mut self) {
690        let _guard = tch::no_grad_guard();
691        for b in self.lora_b.values_mut() {
692            let _ = b.zero_();
693        }
694        self.accumulated_steps = 0;
695        self.request_count = 0;
696        self.avg_loss_improvement = 0.0;
697        self.adaptation_state = crate::training::adaptation_state::DeltaAdaptationState::Idle;
698        // Zeroing the LoRA weights is the largest possible in-place weight
699        // change — bump the epoch so dependent session KV is invalidated.
700        self.bump_weight_epoch();
701    }
702
703    /// Zero all gradients on trainable variables.
704    pub fn zero_grad(&self) {
705        for (_, var) in self.vs.variables() {
706            if var.grad().defined() {
707                let _ = var.grad().zero_();
708            }
709        }
710    }
711
712    /// Get the effective (active) rank for a key. Returns None if key doesn't exist.
713    pub fn effective_rank(&self, key: &str) -> Option<usize> {
714        self.effective_ranks.get(key).copied()
715    }
716
717    /// Set the effective rank for a key. Clamped to [1, max_rank].
718    /// Also updates the scaling factor for this key (alpha / effective_rank).
719    pub fn set_effective_rank(&mut self, key: &str, rank: usize) {
720        if let Some(&max_r) = self.max_ranks.get(key) {
721            let clamped = rank.clamp(1, max_r);
722            self.effective_ranks.insert(key.to_owned(), clamped);
723            // Recalculate scaling: alpha / effective_rank
724            self.scaling_map
725                .insert(key.to_owned(), self.alpha as f64 / clamped as f64);
726        }
727    }
728}
729
730impl TenantDelta {
731    /// Compose two deltas: base + tenant
732    ///
733    /// Creates a new "view" delta whose forward pass sums corrections from both.
734    /// The composed delta is non-trainable (for inference only).
735    ///
736    /// Both deltas' A/B matrices are summed element-wise for shared keys.
737    /// For keys only in one delta, those corrections are used as-is.
738    pub fn compose(
739        base: &std::sync::Arc<parking_lot::Mutex<TenantDelta>>,
740        tenant: &std::sync::Arc<parking_lot::Mutex<TenantDelta>>,
741    ) -> std::sync::Arc<parking_lot::Mutex<TenantDelta>> {
742        let base_guard = base.lock();
743        let tenant_guard = tenant.lock();
744        let _no_grad = tch::no_grad_guard();
745
746        // Collect all per-layer keys from both deltas
747        let mut all_keys: Vec<String> = base_guard.lora_a.keys().cloned().collect();
748        for k in tenant_guard.lora_a.keys() {
749            if !all_keys.contains(k) {
750                all_keys.push(k.clone());
751            }
752        }
753
754        let mut composed_a = HashMap::new();
755        let mut composed_b = HashMap::new();
756
757        for key in &all_keys {
758            let base_a = base_guard.lora_a.get(key);
759            let base_b = base_guard.lora_b.get(key);
760            let tenant_a = tenant_guard.lora_a.get(key);
761            let tenant_b = tenant_guard.lora_b.get(key);
762
763            // Use per-key scaling if available (C7 fix), else fall back to global
764            let base_scaling = base_guard
765                .scaling_map
766                .get(key)
767                .copied()
768                .unwrap_or(base_guard.scaling);
769            let tenant_scaling = tenant_guard
770                .scaling_map
771                .get(key)
772                .copied()
773                .unwrap_or(tenant_guard.scaling);
774
775            // Narrow to effective_rank before composition — matches forward_2d behavior.
776            // Without this, compose operates on dormant rank dimensions (near-zero values
777            // that waste FLOPs and may introduce noise from untrained parameters).
778            let narrow_a = |a: &Tensor, guard: &TenantDelta| -> Tensor {
779                let eff = guard
780                    .effective_ranks
781                    .get(key)
782                    .copied()
783                    .unwrap_or_else(|| a.size()[0] as usize) as i64;
784                a.narrow(0, 0, eff)
785            };
786            let narrow_b = |b: &Tensor, guard: &TenantDelta| -> Tensor {
787                let eff = guard
788                    .effective_ranks
789                    .get(key)
790                    .copied()
791                    .unwrap_or_else(|| b.size()[1] as usize) as i64;
792                b.narrow(1, 0, eff)
793            };
794
795            // Check if effective ranks match for this specific key
796            let base_eff_rank = base_a.map(|a| {
797                base_guard
798                    .effective_ranks
799                    .get(key)
800                    .copied()
801                    .unwrap_or_else(|| a.size()[0] as usize)
802            });
803            let tenant_eff_rank = tenant_a.map(|a| {
804                tenant_guard
805                    .effective_ranks
806                    .get(key)
807                    .copied()
808                    .unwrap_or_else(|| a.size()[0] as usize)
809            });
810            let ranks_match = base_eff_rank == tenant_eff_rank;
811
812            match (base_a, tenant_a) {
813                (Some(ba), Some(ta)) if ranks_match => {
814                    // Same effective rank: narrow then add A matrices directly (pre-scaled)
815                    let base_effective_a = narrow_a(ba, &base_guard) * base_scaling;
816                    let tenant_effective_a = narrow_a(ta, &tenant_guard) * tenant_scaling;
817                    composed_a.insert(key.clone(), base_effective_a + tenant_effective_a);
818                }
819                (Some(ba), Some(ta)) => {
820                    // Different effective ranks: narrow each to its effective rank, compute
821                    // W_eff = s1*(B1_eff @ A1_eff) + s2*(B2_eff @ A2_eff), store as A=W_eff, B=I.
822                    #[allow(clippy::expect_used)] // invariant: B must exist when A exists
823                    let bb = base_b.expect("base B must exist if base A exists");
824                    #[allow(clippy::expect_used)]
825                    let tb = tenant_b.expect("tenant B must exist if tenant A exists");
826                    // Narrow to effective rank before matmul
827                    let ba_eff = narrow_a(ba, &base_guard);
828                    let bb_eff = narrow_b(bb, &base_guard);
829                    let ta_eff = narrow_a(ta, &tenant_guard);
830                    let tb_eff = narrow_b(tb, &tenant_guard);
831                    // B_eff @ A_eff = [out_dim, eff_rank] @ [eff_rank, in_dim] = [out_dim, in_dim]
832                    let base_w = bb_eff.matmul(&ba_eff) * base_scaling;
833                    let tenant_w = tb_eff.matmul(&ta_eff) * tenant_scaling;
834                    let w_eff = base_w + tenant_w;
835                    // forward_2d computes: scaling * (x @ A^T) @ B^T
836                    // We want: x @ W_eff^T  (result shape [tokens, out_dim])
837                    // Store as: A = W_eff [out_dim, in_dim], B = I [out_dim, out_dim]
838                    // Then: (x @ W_eff^T) @ I^T = x @ W_eff^T ✓
839                    let out_dim = w_eff.size()[0];
840                    let identity = Tensor::eye(out_dim, (w_eff.kind(), w_eff.device()));
841                    composed_a.insert(key.clone(), w_eff);
842                    composed_b.insert(key.clone(), identity);
843                    continue; // Skip the B match below, we already handled it
844                }
845                (Some(ba), None) => {
846                    composed_a.insert(key.clone(), narrow_a(ba, &base_guard) * base_scaling);
847                }
848                (None, Some(ta)) => {
849                    composed_a.insert(key.clone(), narrow_a(ta, &tenant_guard) * tenant_scaling);
850                }
851                (None, None) => {}
852            }
853
854            // Only process B separately when we didn't already handle it above
855            // (i.e., ranks matched or only one side had the key)
856            match (base_b, tenant_b) {
857                (Some(bb), Some(tb)) => {
858                    let bb_eff = narrow_b(bb, &base_guard);
859                    let tb_eff = narrow_b(tb, &tenant_guard);
860                    composed_b.insert(key.clone(), &bb_eff + &tb_eff);
861                }
862                (Some(bb), None) => {
863                    composed_b.insert(key.clone(), narrow_b(bb, &base_guard));
864                }
865                (None, Some(tb)) => {
866                    composed_b.insert(key.clone(), narrow_b(tb, &tenant_guard));
867                }
868                (None, None) => {}
869            }
870        }
871
872        // Merge target modules (without layer prefix)
873        let mut all_modules: Vec<String> = base_guard.target_modules.clone();
874        for m in &tenant_guard.target_modules {
875            if !all_modules.contains(m) {
876                all_modules.push(m.clone());
877            }
878        }
879
880        let device = base_guard.device;
881        let rank = base_guard.rank.max(tenant_guard.rank);
882        let num_layers = base_guard.num_layers.max(tenant_guard.num_layers);
883        let vs = VarStore::new(device);
884        let now = Instant::now();
885
886        std::sync::Arc::new(parking_lot::Mutex::new(TenantDelta {
887            lora_a: composed_a,
888            lora_b: composed_b,
889            vs,
890            muon_states: HashMap::new(),
891            muon_config: MuonConfig::default(),
892            scaling_map: HashMap::new(), // scaling=1.0 already baked in during composition
893            scaling: 1.0,                // Already pre-scaled during composition
894            rank,
895            device,
896            target_modules: all_modules,
897            num_layers,
898            learning_rate: 0.0,
899            accumulated_steps: 0,
900            max_accumulated_steps: u64::MAX,
901            request_count: 0,
902            avg_loss_improvement: 0.0,
903            last_access: now,
904            created_at: now,
905            last_snapshot_hash: None,
906            decay_lambda: 0.0,
907            effective_ranks: HashMap::new(),
908            max_ranks: HashMap::new(),
909            alpha: 1.0, // Already pre-scaled during composition
910            rank_oracle: None,
911            adaptation_state: crate::training::adaptation_state::DeltaAdaptationState::Idle,
912            weight_epoch: 0,
913        }))
914    }
915}
916
917/// Extract raw bytes, dtype, and shape from a tch::Tensor for safetensors serialization.
918///
919/// Extracts raw bytes from a tensor for safetensors serialization.
920/// Moves to CPU and makes contiguous if needed.
921fn tensor_to_safetensors_data(
922    tensor: &Tensor,
923) -> Result<(safetensors::Dtype, Vec<usize>, Vec<u8>)> {
924    let cpu_tensor = tensor.to(tch::Device::Cpu).contiguous();
925    let kind = cpu_tensor.kind();
926    let dtype = match kind {
927        Kind::Float => safetensors::Dtype::F32,
928        Kind::Half => safetensors::Dtype::F16,
929        Kind::BFloat16 => safetensors::Dtype::BF16,
930        Kind::Double => safetensors::Dtype::F64,
931        Kind::Int => safetensors::Dtype::I32,
932        Kind::Int64 => safetensors::Dtype::I64,
933        Kind::Int16 => safetensors::Dtype::I16,
934        Kind::Int8 => safetensors::Dtype::I8,
935        Kind::Uint8 => safetensors::Dtype::U8,
936        other => {
937            return Err(anyhow!(
938                "Unsupported tensor kind for safetensors: {:?}",
939                other
940            ))
941        }
942    };
943    let shape: Vec<usize> = cpu_tensor.size().iter().map(|&d| d as usize).collect();
944    let numel: usize = shape.iter().try_fold(1usize, |acc, &dim| {
945        acc.checked_mul(dim)
946            .ok_or_else(|| anyhow!("Tensor numel overflow: shape {:?}", shape))
947    })?;
948    let elem_size = kind.elt_size_in_bytes();
949    let byte_len = numel
950        .checked_mul(elem_size)
951        .ok_or_else(|| anyhow!("Tensor byte length overflow: {} * {}", numel, elem_size))?;
952    let mut bytes = vec![0u8; byte_len];
953    cpu_tensor.copy_data_u8(&mut bytes, numel);
954    Ok((dtype, shape, bytes))
955}
956
957/// Map module name to PEFT subpath
958fn module_to_peft_subpath(module: &str) -> &str {
959    match module {
960        "gate_proj" | "up_proj" | "down_proj" => "mlp",
961        _ => "self_attn", // q_proj, k_proj, v_proj, o_proj, and others
962    }
963}
964
965impl TenantDelta {
966    /// Serialize per-layer LoRA A/B matrices to safetensors bytes (in memory).
967    ///
968    /// Uses standard HuggingFace PEFT naming convention:
969    /// `base_model.model.layers.{N}.{self_attn|mlp}.{module}.lora_{A|B}.weight`
970    pub fn serialize_to_safetensors_bytes(&self) -> Result<Vec<u8>> {
971        let mut pairs: Vec<(String, Tensor)> = Vec::new();
972
973        // Iterate lora_a.keys() to handle non-uniform per-layer module sets (I1 fix)
974        let mut keys: Vec<&String> = self.lora_a.keys().collect();
975        keys.sort(); // deterministic output
976        for key in keys {
977            // key format: "layer_idx.module_name"
978            let mut parts = key.splitn(2, '.');
979            let layer_idx: usize = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0);
980            let module = parts.next().unwrap_or(key.as_str());
981            let subpath = module_to_peft_subpath(module);
982
983            if let Some(a) = self.lora_a.get(key) {
984                let peft_key = format!(
985                    "base_model.model.layers.{}.{}.{}.lora_A.weight",
986                    layer_idx, subpath, module
987                );
988                pairs.push((peft_key, a.shallow_clone()));
989            }
990            if let Some(b) = self.lora_b.get(key) {
991                let peft_key = format!(
992                    "base_model.model.layers.{}.{}.{}.lora_B.weight",
993                    layer_idx, subpath, module
994                );
995                pairs.push((peft_key, b.shallow_clone()));
996            }
997        }
998
999        serialize_tensor_pairs_to_bytes(pairs.into_iter())
1000    }
1001
1002    /// Load per-layer LoRA from safetensors bytes (in memory).
1003    ///
1004    /// Parses standard HuggingFace PEFT naming conventions with layer indices.
1005    pub fn load_from_safetensors_bytes(data: &[u8], device: Device) -> Result<Self> {
1006        let tensors = SafeTensors::deserialize(data)
1007            .map_err(|e| anyhow!("safetensors deserialization failed: {}", e))?;
1008
1009        let mut lora_a: HashMap<String, Tensor> = HashMap::new();
1010        let mut lora_b: HashMap<String, Tensor> = HashMap::new();
1011        let mut rank = 0usize;
1012        let mut max_layer = 0usize;
1013        let mut module_names: std::collections::HashSet<String> = std::collections::HashSet::new();
1014
1015        for (name, view) in tensors.tensors() {
1016            if let Some((layer_idx, module, ab)) = extract_lora_components(&name) {
1017                let key = format!("{}.{}", layer_idx, module);
1018                let tensor = safetensors_view_to_tensor(&view, device)?;
1019                max_layer = max_layer.max(layer_idx);
1020                module_names.insert(module.clone());
1021                match ab.as_str() {
1022                    "lora_a" => {
1023                        if rank == 0 {
1024                            rank = tensor.size()[0] as usize;
1025                        }
1026                        lora_a.insert(key, tensor);
1027                    }
1028                    "lora_b" => {
1029                        lora_b.insert(key, tensor);
1030                    }
1031                    _ => {}
1032                }
1033            }
1034        }
1035
1036        if lora_a.is_empty() {
1037            return Err(anyhow!("No LoRA A matrices found in safetensors data"));
1038        }
1039
1040        // Validate that all A/B pairs have consistent shapes
1041        for (key, a_tensor) in &lora_a {
1042            let a_shape = a_tensor.size();
1043            if a_shape.len() != 2 {
1044                return Err(anyhow!(
1045                    "LoRA A tensor '{}' has invalid rank: expected 2, got {}",
1046                    key,
1047                    a_shape.len()
1048                ));
1049            }
1050            if let Some(b_tensor) = lora_b.get(key) {
1051                let b_shape = b_tensor.size();
1052                if b_shape.len() != 2 {
1053                    return Err(anyhow!(
1054                        "LoRA B tensor '{}' has invalid rank: expected 2, got {}",
1055                        key,
1056                        b_shape.len()
1057                    ));
1058                }
1059                // A is [rank, in_dim], B is [out_dim, rank] — inner dims must match
1060                if a_shape[0] != b_shape[1] {
1061                    return Err(anyhow!(
1062                        "LoRA rank mismatch for '{}': A shape {:?} vs B shape {:?} (A[0]={} != B[1]={})",
1063                        key, a_shape, b_shape, a_shape[0], b_shape[1]
1064                    ));
1065                }
1066            }
1067        }
1068
1069        let num_layers = max_layer + 1;
1070        let alpha = rank as f32;
1071        let target_modules: Vec<String> = module_names.into_iter().collect();
1072        let scaling = alpha as f64 / rank as f64;
1073        let vs = VarStore::new(device);
1074        let now = Instant::now();
1075
1076        tracing::info!(
1077            "Loaded LoRA adapter from bytes: rank={}, layers={}, modules={:?}, alpha={}",
1078            rank,
1079            num_layers,
1080            target_modules,
1081            alpha
1082        );
1083
1084        // Build per-key scaling map from loaded A matrix ranks
1085        let scaling_map: HashMap<String, f64> = lora_a
1086            .iter()
1087            .map(|(key, a)| {
1088                let layer_rank = a.size()[0] as usize;
1089                (key.clone(), alpha as f64 / layer_rank as f64)
1090            })
1091            .collect();
1092
1093        // Build effective_ranks and max_ranks from loaded A matrices
1094        let effective_ranks: HashMap<String, usize> = lora_a
1095            .iter()
1096            .map(|(key, a)| (key.clone(), a.size()[0] as usize))
1097            .collect();
1098        let max_ranks = effective_ranks.clone();
1099
1100        Ok(Self {
1101            lora_a,
1102            lora_b,
1103            vs,
1104            muon_states: HashMap::new(),
1105            muon_config: MuonConfig::default(),
1106            scaling_map,
1107            scaling,
1108            rank,
1109            device,
1110            target_modules,
1111            num_layers,
1112            learning_rate: 0.0,
1113            accumulated_steps: 0,
1114            max_accumulated_steps: u64::MAX,
1115            request_count: 0,
1116            avg_loss_improvement: 0.0,
1117            last_access: now,
1118            created_at: now,
1119            last_snapshot_hash: None,
1120            decay_lambda: 0.0,
1121            effective_ranks,
1122            max_ranks,
1123            alpha,
1124            rank_oracle: None,
1125            adaptation_state: crate::training::adaptation_state::DeltaAdaptationState::Idle,
1126            weight_epoch: 0,
1127        })
1128    }
1129}
1130
1131/// Convert a safetensors TensorView to a tch::Tensor
1132fn safetensors_view_to_tensor(
1133    view: &safetensors::tensor::TensorView<'_>,
1134    device: Device,
1135) -> Result<Tensor> {
1136    let dtype = view.dtype();
1137    let kind = match dtype {
1138        safetensors::Dtype::F32 => Kind::Float,
1139        safetensors::Dtype::F16 => Kind::Half,
1140        safetensors::Dtype::BF16 => Kind::BFloat16,
1141        safetensors::Dtype::F64 => Kind::Double,
1142        safetensors::Dtype::I32 => Kind::Int,
1143        safetensors::Dtype::I64 => Kind::Int64,
1144        safetensors::Dtype::I16 => Kind::Int16,
1145        safetensors::Dtype::I8 => Kind::Int8,
1146        safetensors::Dtype::U8 => Kind::Uint8,
1147        other => return Err(anyhow!("Unsupported safetensors dtype: {:?}", other)),
1148    };
1149    let shape: Vec<i64> = view.shape().iter().map(|&d| d as i64).collect();
1150    let data = view.data();
1151
1152    // Create tensor from raw bytes on CPU, then move to target device
1153    let tensor = Tensor::from_data_size(data, &shape, kind);
1154    Ok(tensor.to(device))
1155}
1156
1157/// Shared safetensors serialization: converts (key, tensor) pairs to bytes.
1158///
1159/// Used by both `TenantDelta::serialize_to_safetensors_bytes()` (PEFT keys)
1160/// and `serialize_state_dict_to_bytes()` (raw keys).
1161fn serialize_tensor_pairs_to_bytes(
1162    pairs: impl Iterator<Item = (String, Tensor)>,
1163) -> Result<Vec<u8>> {
1164    let _guard = tch::no_grad_guard();
1165    let mut tensor_data: Vec<(String, safetensors::Dtype, Vec<usize>, Vec<u8>)> = Vec::new();
1166
1167    for (key, tensor) in pairs {
1168        let cpu = tensor.to_device(Device::Cpu).contiguous();
1169        let (dtype, shape, bytes) = tensor_to_safetensors_data(&cpu)?;
1170        tensor_data.push((key, dtype, shape, bytes));
1171    }
1172
1173    if tensor_data.is_empty() {
1174        return Err(anyhow!("No tensors to serialize"));
1175    }
1176
1177    let views: Vec<(String, TensorView<'_>)> = tensor_data
1178        .iter()
1179        .map(|(key, dtype, shape, bytes)| {
1180            let view = TensorView::new(*dtype, shape.clone(), bytes).unwrap_or_else(|_| {
1181                unreachable!("TensorView construction with valid dtype/shape/bytes")
1182            });
1183            (key.clone(), view)
1184        })
1185        .collect();
1186    let data_refs: Vec<(&str, TensorView<'_>)> =
1187        views.iter().map(|(k, v)| (k.as_str(), v.clone())).collect();
1188
1189    safetensors::tensor::serialize(data_refs, &None)
1190        .map_err(|e| anyhow!("safetensors serialization failed: {}", e))
1191}
1192
1193/// Serialize a state dict (HashMap<String, Tensor>) to safetensors bytes.
1194///
1195/// This replaces direct `Tensor::save_multi()` calls, allowing writes through FsOps.
1196pub fn serialize_state_dict_to_bytes(state_dict: &HashMap<String, Tensor>) -> Result<Vec<u8>> {
1197    serialize_tensor_pairs_to_bytes(
1198        state_dict
1199            .iter()
1200            .map(|(k, v)| (k.clone(), v.shallow_clone())),
1201    )
1202}
1203
1204/// Load a state dict from safetensors bytes.
1205///
1206/// This replaces `Tensor::load_multi()`, allowing reads through FsOps.
1207pub fn load_state_dict_from_bytes(data: &[u8]) -> Result<HashMap<String, Tensor>> {
1208    let tensors = SafeTensors::deserialize(data)
1209        .map_err(|e| anyhow!("safetensors deserialization failed: {}", e))?;
1210
1211    let mut result = HashMap::new();
1212    for (name, view) in tensors.tensors() {
1213        let tensor = safetensors_view_to_tensor(&view, Device::Cpu)?;
1214        result.insert(name, tensor);
1215    }
1216    Ok(result)
1217}
1218
1219/// Estimate tensor memory in bytes
1220fn tensor_bytes(t: &Tensor) -> usize {
1221    let numel = t.numel();
1222    let elem_size = match t.kind() {
1223        Kind::Half | Kind::BFloat16 => 2,
1224        Kind::Double => 8,
1225        _ => 4, // Float and others
1226    };
1227    numel * elem_size
1228}
1229
1230// SAFETY: TenantDelta contains a VarStore backed by tch-rs raw pointers.
1231// VarStore itself is not Send/Sync, but all mutable access to TenantDelta
1232// goes through DeltaPool which wraps it in Arc<parking_lot::Mutex<TenantDelta>>.
1233// The Mutex ensures exclusive access, making cross-thread use safe.
1234//
1235// Invariant: TenantDelta must NEVER be shared across threads without a Mutex.
1236// DeltaPool enforces this — all public APIs return Arc<Mutex<TenantDelta>>.
1237unsafe impl Send for TenantDelta {}
1238unsafe impl Sync for TenantDelta {}
1239
1240/// Type-safe wrapper that enforces Mutex protection for TenantDelta.
1241/// All cross-thread access must go through this wrapper.
1242pub type SharedTenantDelta = std::sync::Arc<parking_lot::Mutex<TenantDelta>>;
1243
1244#[cfg(test)]
1245#[allow(clippy::expect_used, clippy::unwrap_used)]
1246mod tests {
1247    use super::*;
1248
1249    const TEST_NUM_LAYERS: usize = 2;
1250
1251    fn test_module_dims() -> HashMap<String, (usize, usize)> {
1252        let mut dims = HashMap::new();
1253        dims.insert("q_proj".to_owned(), (512, 512));
1254        dims.insert("v_proj".to_owned(), (512, 512));
1255        dims
1256    }
1257
1258    #[test]
1259    fn test_tenant_delta_creation() {
1260        let config = TenantDeltaConfig::default();
1261        let dims = test_module_dims();
1262        let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1263
1264        assert_eq!(delta.rank, 8);
1265        assert_eq!(delta.target_modules.len(), 2);
1266        assert_eq!(delta.num_layers, TEST_NUM_LAYERS);
1267        // Per-layer: 2 layers * 2 modules = 4 entries
1268        assert_eq!(delta.lora_a.len(), 4);
1269        assert_eq!(delta.accumulated_steps, 0);
1270        assert_eq!(delta.request_count, 0);
1271    }
1272
1273    #[test]
1274    fn test_tenant_delta_forward_shape() {
1275        let config = TenantDeltaConfig::default();
1276        let dims = test_module_dims();
1277        let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1278
1279        let x = Tensor::randn([1, 10, 512], (Kind::Float, Device::Cpu));
1280        let output = delta.forward(&x, "q_proj", 0).unwrap();
1281        assert_eq!(output.size(), vec![1, 10, 512]);
1282
1283        // Layer 1 should also work
1284        let output1 = delta.forward(&x, "q_proj", 1).unwrap();
1285        assert_eq!(output1.size(), vec![1, 10, 512]);
1286    }
1287
1288    #[test]
1289    fn test_state_dict_roundtrip() {
1290        let config = TenantDeltaConfig::default();
1291        let dims = test_module_dims();
1292        let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1293
1294        let state = delta.extract_state_dict();
1295
1296        // Create a new delta and load the state
1297        let mut delta2 = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1298        delta2.load_state_dict(&state).unwrap();
1299
1300        // Verify A matrices match for layer 0
1301        let diff: f64 = (&delta.lora_a["0.q_proj"] - &delta2.lora_a["0.q_proj"])
1302            .abs()
1303            .sum(Kind::Float)
1304            .double_value(&[]);
1305        assert!(diff < 1e-6, "State dict roundtrip should preserve values");
1306
1307        // Verify layer 1 too
1308        let diff1: f64 = (&delta.lora_a["1.q_proj"] - &delta2.lora_a["1.q_proj"])
1309            .abs()
1310            .sum(Kind::Float)
1311            .double_value(&[]);
1312        assert!(
1313            diff1 < 1e-6,
1314            "State dict roundtrip should preserve layer 1 values"
1315        );
1316    }
1317
1318    /// The weight epoch starts at 0 and advances on every in-place weight
1319    /// mutation that bypasses remove/re-add — reset() and load_state_dict()
1320    /// (rollback / snapshot rehydrate) here; adapt_tenant/train_step bump it in
1321    /// the trainer. The KV layer keys staleness on this counter (#1254 F1 fix).
1322    #[test]
1323    fn test_weight_epoch_bumps_on_in_place_mutation() {
1324        let config = TenantDeltaConfig::default();
1325        let dims = test_module_dims();
1326        let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1327        assert_eq!(delta.weight_epoch(), 0, "fresh delta starts at epoch 0");
1328
1329        // reset() zeroes the LoRA B weights in place → epoch advances.
1330        delta.reset();
1331        assert_eq!(delta.weight_epoch(), 1, "reset bumps the weight epoch");
1332
1333        // load_state_dict() restores weights in place (rollback path) → epoch advances.
1334        let state = delta.extract_state_dict();
1335        delta.load_state_dict(&state).unwrap();
1336        assert_eq!(
1337            delta.weight_epoch(),
1338            2,
1339            "state restore bumps the weight epoch"
1340        );
1341    }
1342
1343    #[test]
1344    fn test_memory_bytes() {
1345        let config = TenantDeltaConfig::default();
1346        let dims = test_module_dims();
1347        let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1348
1349        let bytes = delta.memory_bytes();
1350        // 2 layers * 2 modules * (rank * in + out * rank) * 4 bytes
1351        let expected = TEST_NUM_LAYERS * 2 * (8 * 512 + 512 * 8) * 4;
1352        assert_eq!(bytes, expected);
1353    }
1354
1355    #[test]
1356    fn test_reset() {
1357        let config = TenantDeltaConfig::default();
1358        let dims = test_module_dims();
1359        let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1360
1361        delta.accumulated_steps = 100;
1362        delta.request_count = 50;
1363        delta.reset();
1364
1365        assert_eq!(delta.accumulated_steps, 0);
1366        assert_eq!(delta.request_count, 0);
1367
1368        // B should be zeros for all layers
1369        let b_norm: f64 = delta.lora_b["0.q_proj"].norm().double_value(&[]);
1370        assert!(b_norm < 1e-8, "B should be zeros after reset");
1371        let b_norm1: f64 = delta.lora_b["1.q_proj"].norm().double_value(&[]);
1372        assert!(b_norm1 < 1e-8, "B layer 1 should be zeros after reset");
1373    }
1374
1375    #[test]
1376    fn test_forward_2d_shape() {
1377        let config = TenantDeltaConfig::default();
1378        let dims = test_module_dims();
1379        let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1380
1381        let x = Tensor::randn([10, 512], (Kind::Float, Device::Cpu));
1382        let output = delta.forward_2d(&x, "q_proj", 0).unwrap();
1383        assert_eq!(output.size(), vec![10, 512]);
1384    }
1385
1386    /// Regression test for the CPU dtype-mismatch panic: a BFloat16 base model uses
1387    /// fp32-stored LoRA deltas. The delta-application matmul must run without panicking
1388    /// and the correction must come back in the base (BF16) dtype so the call site can
1389    /// add it to a BF16 activation (CPU `addmm` rejects mixed float dtypes).
1390    #[test]
1391    fn test_forward_bf16_input_fp32_delta_no_panic() {
1392        let config = TenantDeltaConfig {
1393            rank: 16,
1394            alpha: 32.0,
1395            target_modules: vec!["q_proj".to_owned(), "v_proj".to_owned()],
1396            ..Default::default()
1397        };
1398        let dims = test_module_dims();
1399        let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1400
1401        // Sanity: stored A/B are fp32 (kaiming/zeros default kind).
1402        assert_eq!(delta.lora_a["0.q_proj"].kind(), Kind::Float);
1403        assert_eq!(delta.lora_b["0.q_proj"].kind(), Kind::Float);
1404
1405        // BF16 base activation, mirroring a BF16 model on CPU.
1406        let x_2d = Tensor::randn([10, 512], (Kind::Float, Device::Cpu)).to_kind(Kind::BFloat16);
1407
1408        // forward_2d must not panic and must return a BF16 correction.
1409        let corr_2d = delta.forward_2d(&x_2d, "q_proj", 0).unwrap();
1410        assert_eq!(corr_2d.kind(), Kind::BFloat16);
1411        assert_eq!(corr_2d.size(), vec![10, 512]);
1412
1413        // The exact operation that panicked at the qwen3_5 call site: base + correction,
1414        // both BF16. Forcing evaluation (sum) ensures the add actually executes.
1415        let base_2d = Tensor::randn([10, 512], (Kind::Float, Device::Cpu)).to_kind(Kind::BFloat16);
1416        let combined = &base_2d + &corr_2d;
1417        assert_eq!(combined.kind(), Kind::BFloat16);
1418        let _ = combined.sum(Kind::Float).double_value(&[]);
1419
1420        // The 3D `forward` path must behave identically.
1421        let x_3d = Tensor::randn([1, 10, 512], (Kind::Float, Device::Cpu)).to_kind(Kind::BFloat16);
1422        let corr_3d = delta.forward(&x_3d, "v_proj", 0).unwrap();
1423        assert_eq!(corr_3d.kind(), Kind::BFloat16);
1424        assert_eq!(corr_3d.size(), vec![1, 10, 512]);
1425
1426        // fp32 input still yields fp32 output (cast-back is a no-op when dtypes match).
1427        let x_f32 = Tensor::randn([10, 512], (Kind::Float, Device::Cpu));
1428        let corr_f32 = delta.forward_2d(&x_f32, "q_proj", 0).unwrap();
1429        assert_eq!(corr_f32.kind(), Kind::Float);
1430    }
1431
1432    #[test]
1433    fn test_forward_2d_gradient_flow() {
1434        let config = TenantDeltaConfig {
1435            rank: 4,
1436            alpha: 4.0,
1437            ..Default::default()
1438        };
1439        let dims = test_module_dims();
1440        let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1441
1442        let x = Tensor::randn([5, 512], (Kind::Float, Device::Cpu));
1443
1444        let output = delta.forward_2d(&x, "q_proj", 0).unwrap();
1445
1446        let loss = output.sum(Kind::Float);
1447        loss.backward();
1448
1449        let variables = delta.vs.trainable_variables();
1450        let mut has_grad = false;
1451        for var in &variables {
1452            if var.grad().defined() {
1453                let grad_norm: f64 = var.grad().norm().double_value(&[]);
1454                if grad_norm > 0.0 {
1455                    has_grad = true;
1456                }
1457            }
1458        }
1459        assert!(
1460            has_grad,
1461            "Delta A/B matrices should have gradients after backward()"
1462        );
1463    }
1464
1465    #[test]
1466    fn test_has_module() {
1467        let config = TenantDeltaConfig::default();
1468        let dims = test_module_dims();
1469        let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1470
1471        assert!(delta.has_module("q_proj", 0));
1472        assert!(delta.has_module("q_proj", 1));
1473        assert!(delta.has_module("v_proj", 0));
1474        assert!(!delta.has_module("gate_proj", 0)); // not a target module
1475    }
1476
1477    #[test]
1478    fn test_effective_rank_default_is_max() {
1479        let config = TenantDeltaConfig {
1480            rank: 8,
1481            ..Default::default()
1482        };
1483        let dims = test_module_dims();
1484        let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1485
1486        // Default effective rank should equal allocated rank
1487        assert_eq!(delta.effective_rank("0.q_proj"), Some(8));
1488        assert_eq!(delta.effective_rank("0.v_proj"), Some(8));
1489        assert_eq!(delta.effective_rank("99.nonexistent"), None);
1490    }
1491
1492    #[test]
1493    fn test_effective_rank_forward() {
1494        let config = TenantDeltaConfig {
1495            rank: 8,
1496            ..Default::default()
1497        };
1498        let dims = test_module_dims();
1499        let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1500
1501        let x = Tensor::randn([2, 512], (Kind::Float, Device::Cpu));
1502
1503        // Full rank forward
1504        let out_full = delta.forward_2d(&x, "q_proj", 0).unwrap();
1505        assert_eq!(out_full.size(), &[2, 512]);
1506
1507        // Set effective rank to 4 (half)
1508        delta.set_effective_rank("0.q_proj", 4);
1509        let out_narrow = delta.forward_2d(&x, "q_proj", 0).unwrap();
1510        assert_eq!(out_narrow.size(), &[2, 512]); // output shape unchanged
1511    }
1512
1513    #[test]
1514    fn test_set_effective_rank_clamped() {
1515        let config = TenantDeltaConfig {
1516            rank: 8,
1517            ..Default::default()
1518        };
1519        let dims = test_module_dims();
1520        let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1521
1522        // Setting eff_rank above max_rank should clamp to max
1523        delta.set_effective_rank("0.q_proj", 32);
1524        assert_eq!(delta.effective_rank("0.q_proj"), Some(8)); // clamped to allocated
1525
1526        // Setting eff_rank to 0 should clamp to 1 (minimum)
1527        delta.set_effective_rank("0.q_proj", 0);
1528        assert_eq!(delta.effective_rank("0.q_proj"), Some(1));
1529    }
1530
1531    #[test]
1532    fn test_effective_rank_scaling_adjusts() {
1533        let config = TenantDeltaConfig {
1534            rank: 8,
1535            alpha: 4.0,
1536            ..Default::default()
1537        };
1538        let dims = test_module_dims();
1539        let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1540
1541        // Original scaling: alpha / rank = 4.0 / 8 = 0.5
1542        let s0 = delta.scaling_map["0.q_proj"];
1543        assert!((s0 - 0.5).abs() < 1e-9);
1544
1545        // Set effective rank to 4 -> scaling should be alpha / eff_rank = 4.0 / 4 = 1.0
1546        delta.set_effective_rank("0.q_proj", 4);
1547        let s1 = delta.scaling_map["0.q_proj"];
1548        assert!((s1 - 1.0).abs() < 1e-9);
1549    }
1550
1551    #[test]
1552    fn test_effective_rank_with_layer_overrides() {
1553        let mut overrides = HashMap::new();
1554        overrides.insert(
1555            0,
1556            LayerDeltaConfig {
1557                rank: 16,
1558                target_modules: vec!["q_proj".to_owned(), "v_proj".to_owned()],
1559            },
1560        );
1561        let config = TenantDeltaConfig {
1562            rank: 8,
1563            layer_overrides: Some(overrides),
1564            ..Default::default()
1565        };
1566        let dims = test_module_dims();
1567        let delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1568
1569        // Layer 0 allocated at 16, layer 1 at default 8
1570        assert_eq!(delta.effective_rank("0.q_proj"), Some(16));
1571        assert_eq!(delta.effective_rank("1.q_proj"), Some(8));
1572    }
1573
1574    #[test]
1575    fn test_state_dict_round_trip_preserves_effective_ranks() {
1576        let config = TenantDeltaConfig {
1577            rank: 8,
1578            alpha: 4.0,
1579            ..Default::default()
1580        };
1581        let dims = test_module_dims();
1582        let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1583
1584        // Set non-default effective ranks
1585        delta.set_effective_rank("0.q_proj", 4);
1586        delta.set_effective_rank("1.v_proj", 2);
1587
1588        let state = delta.extract_state_dict();
1589
1590        // Verify metadata keys exist
1591        assert!(state.contains_key("0.q_proj.__effective_rank"));
1592        assert!(state.contains_key("1.v_proj.__effective_rank"));
1593
1594        // Load into a fresh delta
1595        let mut delta2 = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1596        assert_eq!(delta2.effective_rank("0.q_proj"), Some(8)); // default
1597        delta2.load_state_dict(&state).unwrap();
1598
1599        // Verify effective ranks restored
1600        assert_eq!(delta2.effective_rank("0.q_proj"), Some(4));
1601        assert_eq!(delta2.effective_rank("1.v_proj"), Some(2));
1602        // Unchanged keys should keep default
1603        assert_eq!(delta2.effective_rank("0.v_proj"), Some(8));
1604    }
1605
1606    #[test]
1607    fn test_state_dict_round_trip_preserves_muon_momentum() {
1608        let config = TenantDeltaConfig {
1609            rank: 8,
1610            alpha: 4.0,
1611            ..Default::default()
1612        };
1613        let dims = test_module_dims();
1614        let mut delta = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1615
1616        // Simulate a Muon step populating momentum for one key
1617        let key = "0.q_proj".to_owned();
1618        let fake_momentum = Tensor::randn([8, 512], (Kind::Float, Device::Cpu));
1619        delta.muon_states.insert(
1620            key.clone(),
1621            super::super::muon::MuonState {
1622                momentum_buffer: Some(fake_momentum.copy()),
1623            },
1624        );
1625
1626        let state = delta.extract_state_dict();
1627
1628        // Verify momentum key exists
1629        assert!(state.contains_key("0.q_proj.__muon_momentum"));
1630
1631        // Load into a fresh delta that has the same Muon state key
1632        let mut delta2 = TenantDelta::new(&config, &dims, Device::Cpu, TEST_NUM_LAYERS).unwrap();
1633        delta2.muon_states.insert(
1634            key.clone(),
1635            super::super::muon::MuonState {
1636                momentum_buffer: None,
1637            },
1638        );
1639        delta2.load_state_dict(&state).unwrap();
1640
1641        // Verify momentum buffer restored
1642        let restored = &delta2.muon_states[&key];
1643        assert!(restored.momentum_buffer.is_some());
1644        let diff: f64 = (restored.momentum_buffer.as_ref().unwrap() - &fake_momentum)
1645            .abs()
1646            .sum(Kind::Float)
1647            .double_value(&[]);
1648        assert!(
1649            diff < 1e-6,
1650            "Muon momentum should be preserved through state_dict round-trip"
1651        );
1652    }
1653}