Skip to main content

hyprstream_inference/training/
delta_reducer.rs

1//! N-way delta aggregation (param-server reducer) — replaces DO-Merge for
2//! multi-contributor merges.
3//!
4//! # Why this exists
5//!
6//! [`crate::training::merge`]'s DO-Merge (`do_merge`) and `additive_merge`
7//! assert *shape equality* on the raw LoRA A/B factors. The moment two
8//! contributors carry different per-layer ranks — the normal case once the rank
9//! oracle (`ttn_profile`) assigns non-uniform per-layer ranks, or once
10//! different workers run with different `layer_overrides` — those paths
11//! hard-fail. They are also non-associative (path-dependent blend weights +
12//! direction renormalization), so an N-way merge would depend on contributor
13//! order.
14//!
15//! This module replaces the **N-way** aggregation with an order-independent,
16//! rank-heterogeneity-tolerant reducer. The 1-vs-1 save-time DO-Merge path may
17//! stay; this is the path used when *several* independent TTT contributors are
18//! aggregated each round.
19//!
20//! # Algorithm (stack + truncated SVD in delta-space)
21//!
22//! For one layer key, each contributor `i` supplies factors `A_i [r_i, in]`,
23//! `B_i [out, r_i]` and a fold-in weight `w_i` (provenance trust × norm-bound ×
24//! per-key scaling). The aggregate effective weight is
25//!
26//! ```text
27//!   ΔW = Σ_i w_i · (B_i · A_i)
28//!      = B_stack · A_stack
29//! ```
30//!
31//! where `B_stack = [ w_1·B_1 | w_2·B_2 | … ]` is `[out, Σr_i]` and
32//! `A_stack = [ A_1 ; A_2 ; … ]` is `[Σr_i, in]`. This is exact: stacking the
33//! factors *is* the weighted sum of the rank-1 updates, with no dense
34//! `[out, in]` materialization and no requirement that the `r_i` agree.
35//!
36//! We then take **one** rank-`R` truncated SVD of `ΔW` per round, computed on
37//! the tall-skinny stacked factors (`Σr` is the sum of contributor ranks —
38//! small) without ever forming the dense `[out, in]`:
39//!
40//! 1. economy SVD of `A_stack [Σr, in]` → `A_stack = Uₐ·diag(Sₐ)·Vhₐ`
41//!    (factors are `[Σr, Σr]`, `[Σr]`, `[Σr, in]` since `Σr ≪ in`);
42//! 2. `C = B_stack · Uₐ · diag(Sₐ)`  →  `[out, Σr]` (small, no dense `out×in`);
43//! 3. economy SVD of `C [out, Σr]` → `C = U_c·diag(S_c)·Vh_c`;
44//! 4. `ΔW = U_c · diag(S_c) · (Vh_c · Vhₐ)` — an exact SVD of `ΔW`;
45//! 5. truncate to `R`: `B_out = U_c[:, :R]·diag(S_c[:R])` `[out, R]`,
46//!    `A_out = (Vh_c·Vhₐ)[:R, :]` `[R, in]`.
47//!
48//! The reduced factors are stored with `scaling = 1.0` (the weights and
49//! per-contributor scaling are already folded in). Output rank is the fixed,
50//! operational `R` — stable checkpoint shapes.
51//!
52//! # Order-independence
53//!
54//! `ΔW` is a plain sum, so permuting contributors permutes columns of
55//! `B_stack` and rows of `A_stack` identically and leaves the product `ΔW`
56//! unchanged; the SVD of `ΔW` is therefore identical up to fp rounding.
57//!
58//! # Provenance / norm gate (secure multitenant)
59//!
60//! Aggregation is over an *explicit* contributor set — the reducer never
61//! discovers or mixes deltas across tenants. Each contributor carries a
62//! verified host id (Phase-2 mesh trust, #354) and a `delta_norm_ratio`. The
63//! [`ContributionGate`] rejects (does **not** silently drop or clamp-to-zero)
64//! any contributor whose norm exceeds the bound, and weights survivors by
65//! `trust × f(norm_ratio)`. A pluggable [`ValidationHook`] seam is provided for
66//! the deferred held-out-eval host (default impl is a no-op).
67//!
68//! # Send/Sync
69//!
70//! tch `Tensor` is `!Send`. The reducer keeps **all** tensor work on a single
71//! device/thread (`reduce()` takes `&[Contribution]` by reference and returns
72//! freshly-built tensors; it never moves a `Tensor` across a thread boundary).
73//! Contributions cross thread/host boundaries as serialized delta *bytes*
74//! (safetensors) + plain-old-data provenance, deserialized on the reducer
75//! thread — never as live `Tensor`s.
76//!
77//! # Optimizer state
78//!
79//! Per the spike, Muon momentum is **dropped** at aggregation: averaging
80//! optimizer state across independent TTT runs is unsound. The reduced delta
81//! starts with a fresh (empty) optimizer.
82
83use anyhow::{anyhow, Result};
84use std::collections::{HashMap, HashSet};
85use tch::{Kind, Tensor};
86
87/// Provenance for one contributor's delta.
88///
89/// The transport that *verifies* the signature (param-server / mesh) is out of
90/// scope for the reducer (a separate gated follow-on). By the time a
91/// [`Contribution`] reaches the reducer its `host_id` is already an
92/// operator-resolved, verified identity (Phase-2 mesh trust, #354). This struct
93/// is plain-old-data so it can cross a thread/host boundary without moving any
94/// `Tensor`.
95#[derive(Debug, Clone)]
96pub struct Provenance {
97    /// Verified host id (operator label resolved from the signing identity).
98    pub host_id: String,
99    /// Trust weight for this host in `[0, 1]`. `1.0` = full trust. A downweighted
100    /// (e.g. newly-seen) host moves the merge less.
101    pub trust: f64,
102}
103
104impl Provenance {
105    /// Construct a fully-trusted contributor identity.
106    pub fn trusted(host_id: impl Into<String>) -> Self {
107        Self {
108            host_id: host_id.into(),
109            trust: 1.0,
110        }
111    }
112}
113
114/// One contributor in an N-way aggregation round.
115///
116/// Holds the contributor's LoRA factors *for a single layer key already* — the
117/// reducer is invoked per key by [`reduce_state_dicts`]. Factors are borrowed
118/// (shallow clones / views are fine); the reducer does not mutate them.
119pub struct Contribution {
120    /// `A_i [r_i, in]`.
121    pub lora_a: Tensor,
122    /// `B_i [out, r_i]`.
123    pub lora_b: Tensor,
124    /// Per-key LoRA scaling (`alpha / rank`) already associated with these
125    /// factors. Folded into the stack weight so the reduced factors can use
126    /// `scaling = 1.0`.
127    pub scaling: f64,
128    /// `||ΔW_i|| / ||W_base||` for this key (from
129    /// [`crate::training::TenantDelta::delta_norm_ratio`]). Used by the gate.
130    pub delta_norm_ratio: f64,
131    /// Verified provenance.
132    pub provenance: Provenance,
133}
134
135/// Gate + weighting policy applied to each contributor before it enters the
136/// stack. Anomalous / over-norm contributors are **rejected** (error), never
137/// silently dropped or clamped to zero — a quiet drop would let a poisoned host
138/// shrink the merge invisibly.
139#[derive(Debug, Clone)]
140pub struct ContributionGate {
141    /// Reject any contributor whose `delta_norm_ratio` exceeds this bound.
142    /// Defends against norm-inflation poisoning.
143    pub max_norm_ratio: f64,
144    /// Reject any contributor whose resolved trust is below this floor.
145    pub min_trust: f64,
146}
147
148impl Default for ContributionGate {
149    fn default() -> Self {
150        // 2.0 = the aggregate update may be at most ~2x the base-weight norm for
151        // a single contributor; well above legitimate TTT drift, well below a
152        // runaway. min_trust=0 admits any positively-trusted host; raise to
153        // exclude provisional peers.
154        Self {
155            max_norm_ratio: 2.0,
156            min_trust: 0.0,
157        }
158    }
159}
160
161impl ContributionGate {
162    /// Validate one contributor and return its non-negative stack weight
163    /// `trust × f(norm_ratio) × scaling`, or an error if it fails the gate.
164    ///
165    /// `f(norm_ratio) = 1 / (1 + norm_ratio)` softly downweights large-norm
166    /// (more-likely-anomalous) deltas that still pass the hard bound, so a
167    /// borderline contributor moves the merge less than a small clean one.
168    fn weight_for(&self, c: &Contribution) -> Result<f64> {
169        let trust = c.provenance.trust;
170        if !(trust.is_finite()) || trust < 0.0 {
171            return Err(anyhow!(
172                "contributor '{}' has invalid trust {}",
173                c.provenance.host_id,
174                trust
175            ));
176        }
177        if trust < self.min_trust {
178            return Err(anyhow!(
179                "contributor '{}' rejected: trust {} < min_trust {}",
180                c.provenance.host_id,
181                trust,
182                self.min_trust
183            ));
184        }
185        let ratio = c.delta_norm_ratio;
186        if !ratio.is_finite() || ratio < 0.0 {
187            return Err(anyhow!(
188                "contributor '{}' has invalid delta_norm_ratio {}",
189                c.provenance.host_id,
190                ratio
191            ));
192        }
193        if ratio > self.max_norm_ratio {
194            return Err(anyhow!(
195                "contributor '{}' rejected: delta_norm_ratio {:.4} exceeds bound {:.4} (possible norm-inflation poisoning)",
196                c.provenance.host_id,
197                ratio,
198                self.max_norm_ratio
199            ));
200        }
201        let norm_weight = 1.0 / (1.0 + ratio);
202        Ok(trust * norm_weight * c.scaling)
203    }
204}
205
206/// Pluggable post-reduce validation seam.
207///
208/// The real implementation (held-out-eval host) is a deferred, gated follow-on.
209/// Defining the trait now lets the eval-host swap in with zero consumer churn.
210pub trait ValidationHook: Send + Sync {
211    /// Inspect the freshly-reduced factors for one key. Return `Ok(())` to
212    /// accept or an error to reject the round. Default: accept everything.
213    ///
214    /// Tensors are passed by reference and must not be moved across threads.
215    fn validate(&self, _key: &str, _reduced_a: &Tensor, _reduced_b: &Tensor) -> Result<()> {
216        Ok(())
217    }
218}
219
220/// No-op validation hook (the default until the held-out-eval host lands).
221#[derive(Debug, Default, Clone, Copy)]
222pub struct NoopValidation;
223
224impl ValidationHook for NoopValidation {}
225
226/// Order-independent N-way delta reducer.
227///
228/// Implementors merge an explicit contributor set into a fixed-rank delta. The
229/// stack + truncated-SVD implementation is [`StackSvdReducer`]; the trait keeps
230/// it swappable (held-out-eval variants, future native reducers).
231pub trait DeltaReducer {
232    /// Reduce contributors for a **single** layer key into fixed-rank factors
233    /// `(A_out [R, in], B_out [out, R])`, with the result expressed at
234    /// `scaling = 1.0` (weights + per-contributor scaling already folded in).
235    ///
236    /// All tensors stay on `contributions[*].lora_a.device()` (must be uniform).
237    fn reduce_key(&self, key: &str, contributions: &[Contribution]) -> Result<(Tensor, Tensor)>;
238}
239
240/// Stack-in-delta-space + one truncated SVD per round to a fixed rank `R`.
241pub struct StackSvdReducer {
242    /// Fixed output rank — an operational contract (stable checkpoint shapes).
243    pub rank: usize,
244    /// Contributor gate / weighting.
245    pub gate: ContributionGate,
246    /// Pluggable validation hook (default no-op).
247    pub validation: Box<dyn ValidationHook>,
248}
249
250impl StackSvdReducer {
251    /// Construct with a fixed output rank and the default gate / no-op hook.
252    pub fn new(rank: usize) -> Self {
253        Self {
254            rank,
255            gate: ContributionGate::default(),
256            validation: Box::new(NoopValidation),
257        }
258    }
259
260    /// Override the contributor gate.
261    pub fn with_gate(mut self, gate: ContributionGate) -> Self {
262        self.gate = gate;
263        self
264    }
265
266    /// Install a validation hook (e.g. the held-out-eval host once it exists).
267    pub fn with_validation(mut self, validation: Box<dyn ValidationHook>) -> Self {
268        self.validation = validation;
269        self
270    }
271}
272
273impl DeltaReducer for StackSvdReducer {
274    fn reduce_key(&self, key: &str, contributions: &[Contribution]) -> Result<(Tensor, Tensor)> {
275        let _guard = tch::no_grad_guard();
276
277        if contributions.is_empty() {
278            return Err(anyhow!("reduce_key('{}'): no contributors", key));
279        }
280        if self.rank == 0 {
281            return Err(anyhow!("reduce_key('{}'): fixed rank R must be >= 1", key));
282        }
283
284        // Uniform device + matching in/out dims across contributors. The reducer
285        // does all tensor work on this one device/thread (Send/Sync invariant).
286        let device = contributions[0].lora_a.device();
287        let out_dim = contributions[0].lora_b.size()[0];
288        let in_dim = contributions[0].lora_a.size()[1];
289
290        // Build the stacked factors, folding the gate weight into B (one side
291        // only — weighting both A and B would square the weight).
292        //   B_stack = [ w_1·B_1 | … ]  [out, Σr]
293        //   A_stack = [   A_1   ; … ]  [Σr, in]
294        let mut b_cols: Vec<Tensor> = Vec::with_capacity(contributions.len());
295        let mut a_rows: Vec<Tensor> = Vec::with_capacity(contributions.len());
296
297        for c in contributions {
298            // Validate device + shape consistency.
299            if c.lora_a.device() != device || c.lora_b.device() != device {
300                return Err(anyhow!(
301                    "reduce_key('{}'): contributor '{}' on a different device — reducer requires a single device",
302                    key,
303                    c.provenance.host_id
304                ));
305            }
306            let a_size = c.lora_a.size();
307            let b_size = c.lora_b.size();
308            if a_size.len() != 2 || b_size.len() != 2 {
309                return Err(anyhow!("reduce_key('{}'): factors must be 2D", key));
310            }
311            let (ri, in_i) = (a_size[0], a_size[1]);
312            let (out_i, ri_b) = (b_size[0], b_size[1]);
313            if in_i != in_dim || out_i != out_dim {
314                return Err(anyhow!(
315                    "reduce_key('{}'): contributor '{}' dims [out={},in={}] differ from [out={},in={}]",
316                    key, c.provenance.host_id, out_i, in_i, out_dim, in_dim
317                ));
318            }
319            if ri != ri_b {
320                return Err(anyhow!(
321                    "reduce_key('{}'): contributor '{}' rank mismatch A[0]={} vs B[1]={}",
322                    key,
323                    c.provenance.host_id,
324                    ri,
325                    ri_b
326                ));
327            }
328
329            // Gate + weight (rejects over-norm / untrusted; never silent-drops).
330            let weight = self.gate.weight_for(c)?;
331
332            // Compute the reducer in fp32 (deltas are fp32 delta-space).
333            let a = c.lora_a.to_kind(Kind::Float);
334            let b = c.lora_b.to_kind(Kind::Float) * weight;
335            a_rows.push(a);
336            b_cols.push(b);
337        }
338
339        // Σr small (sum of contributor ranks). Concat is the only unavoidable
340        // materialization; both stacks are tall-skinny.
341        let a_refs: Vec<&Tensor> = a_rows.iter().collect();
342        let b_refs: Vec<&Tensor> = b_cols.iter().collect();
343        let a_stack = Tensor::cat(&a_refs, 0); // [Σr, in]
344        let b_stack = Tensor::cat(&b_refs, 1); // [out, Σr]
345        let sigma_r = a_stack.size()[0];
346
347        // If the whole stack is ~zero (all contributors empty/cancelled), return
348        // zero factors at the fixed rank rather than running SVD on zeros.
349        let stack_norm: f64 = a_stack.norm().double_value(&[]) * b_stack.norm().double_value(&[]);
350        let r = self.rank as i64;
351        if stack_norm < 1e-12 {
352            let a_out = Tensor::zeros([r, in_dim], (Kind::Float, device));
353            let b_out = Tensor::zeros([out_dim, r], (Kind::Float, device));
354            self.validation.validate(key, &a_out, &b_out)?;
355            return Ok((a_out, b_out));
356        }
357
358        // Exact rank-Σr SVD of ΔW = B_stack · A_stack, computed via two economy
359        // SVDs on the tall-skinny stacks (never forms the dense [out,in]).
360        //
361        // 1) A_stack = Ua · diag(Sa) · Vha   (economy: some=true, compute_uv=true)
362        let (ua, sa, va) = a_stack.svd(true, true); // Ua [Σr,k], Sa [k], Va [in,k]
363                                                    // tch's `svd` returns V (not Vh); Vha = Va^T.
364        let k = sa.size()[0]; // = min(Σr, in) = Σr
365        let _ = k;
366        // 2) C = B_stack · Ua · diag(Sa)  →  [out, Σr]
367        let c = b_stack.matmul(&ua) * sa.unsqueeze(0); // broadcast diag(Sa) over rows
368                                                       // 3) C = Uc · diag(Sc) · Vhc
369        let (uc, sc, vc) = c.svd(true, true); // Uc [out,m], Sc [m], Vc [Σr,m]
370                                              // 4) Vh_final = Vhc · Vha = Vc^T · Va^T = (Va · Vc)^T  →  [m, in]
371                                              //    ΔW = Uc · diag(Sc) · Vh_final
372        let vh_final = va.matmul(&vc).tr(); // [m, in]
373
374        // 5) Truncate to fixed R (pad with zeros if Σr < R so shapes are stable).
375        let avail = sc.size()[0].min(sigma_r);
376        let take = avail.min(r);
377
378        let uc_t = uc.narrow(1, 0, take); // [out, take]
379        let sc_t = sc.narrow(0, 0, take); // [take]
380        let vh_t = vh_final.narrow(0, 0, take); // [take, in]
381
382        // B_out = Uc[:, :R] · diag(Sc[:R])   [out, take]
383        let b_take = &uc_t * sc_t.unsqueeze(0);
384        // A_out = Vh_final[:R, :]            [take, in]
385        let a_take = vh_t.contiguous();
386        let b_take = b_take.contiguous();
387
388        let (a_out, b_out) = if take < r {
389            // Pad the rank dimension with zeros to the fixed R (zero-energy
390            // directions; keeps checkpoint shapes invariant).
391            let a_pad = Tensor::zeros([r - take, in_dim], (Kind::Float, device));
392            let b_pad = Tensor::zeros([out_dim, r - take], (Kind::Float, device));
393            let a_out = Tensor::cat(&[&a_take, &a_pad], 0);
394            let b_out = Tensor::cat(&[&b_take, &b_pad], 1);
395            (a_out, b_out)
396        } else {
397            (a_take, b_take)
398        };
399
400        self.validation.validate(key, &a_out, &b_out)?;
401        Ok((a_out, b_out))
402    }
403}
404
405/// Aggregate N contributors' full state dicts into one reduced state dict.
406///
407/// Each contributor supplies a `(state_dict, provenance, norm_ratios)` where the
408/// state dict uses the `extract_state_dict` key convention
409/// (`"layer.module.lora_a"` / `"layer.module.lora_b"`). The reducer runs the
410/// stack + truncated-SVD per layer key and emits reduced factors at the fixed
411/// rank `R`. Effective-rank / Muon-momentum metadata are intentionally dropped
412/// (fresh optimizer post-aggregation, per the spike).
413///
414/// Keys present in only some contributors are aggregated over the subset that
415/// has them — this is still order-independent and tolerant of contributors that
416/// trained different module sets.
417pub fn reduce_state_dicts(
418    reducer: &dyn DeltaReducer,
419    contributors: &[ContributorState],
420) -> Result<HashMap<String, Tensor>> {
421    let _guard = tch::no_grad_guard();
422    if contributors.is_empty() {
423        return Err(anyhow!("reduce_state_dicts: no contributors"));
424    }
425
426    // Collect the union of layer keys across all contributors.
427    let mut layer_keys: Vec<String> = Vec::new();
428    let mut seen: HashSet<String> = HashSet::new();
429    for cs in contributors {
430        for k in cs.state.keys() {
431            if let Some(layer_key) = k.strip_suffix(".lora_a") {
432                if seen.insert(layer_key.to_owned()) {
433                    layer_keys.push(layer_key.to_owned());
434                }
435            }
436        }
437    }
438    layer_keys.sort(); // deterministic output ordering
439
440    let mut result: HashMap<String, Tensor> = HashMap::new();
441
442    for layer_key in &layer_keys {
443        let a_key = format!("{}.lora_a", layer_key);
444        let b_key = format!("{}.lora_b", layer_key);
445
446        let mut contribs: Vec<Contribution> = Vec::new();
447        for cs in contributors {
448            let (Some(a), Some(b)) = (cs.state.get(&a_key), cs.state.get(&b_key)) else {
449                continue; // this contributor didn't train this key
450            };
451            let norm_ratio = cs
452                .norm_ratios
453                .get(layer_key)
454                .copied()
455                // module-name fallback ("layer.module" -> "module") mirrors
456                // delta_norm_ratio's own keying tolerance.
457                .or_else(|| {
458                    layer_key
459                        .split_once('.')
460                        .and_then(|(_, m)| cs.norm_ratios.get(m).copied())
461                })
462                .unwrap_or(0.0);
463            contribs.push(Contribution {
464                lora_a: a.shallow_clone(),
465                lora_b: b.shallow_clone(),
466                scaling: cs.scaling.get(layer_key).copied().unwrap_or(1.0),
467                delta_norm_ratio: norm_ratio,
468                provenance: cs.provenance.clone(),
469            });
470        }
471
472        if contribs.is_empty() {
473            continue;
474        }
475
476        let (a_out, b_out) = reducer.reduce_key(layer_key, &contribs)?;
477        result.insert(a_key, a_out);
478        result.insert(b_key, b_out);
479    }
480
481    if result.is_empty() {
482        return Err(anyhow!(
483            "reduce_state_dicts: no aggregatable layer keys across contributors"
484        ));
485    }
486
487    Ok(result)
488}
489
490/// One contributor's full state for [`reduce_state_dicts`].
491///
492/// Plain-old-data + tensors that stay on one thread. The `state` tensors are
493/// expected to already be on the reducer's device (deserialized from bytes on
494/// this thread — see the module-level Send/Sync note).
495pub struct ContributorState {
496    /// `extract_state_dict`-keyed factors (`"layer.module.lora_a"` etc.).
497    pub state: HashMap<String, Tensor>,
498    /// Per-layer-key scaling (`alpha / rank`).
499    pub scaling: HashMap<String, f64>,
500    /// Per-layer-key (or per-module) `delta_norm_ratio`.
501    pub norm_ratios: HashMap<String, f64>,
502    /// Verified provenance for this contributor.
503    pub provenance: Provenance,
504}
505
506/// Static rank↔quant coupling cap.
507///
508/// Coarse quantization (e.g. block-wise FP8) gives a delta limited headroom: a
509/// rank the layer's stored dtype can't actually express is wasted (and adds
510/// noise from untrained directions). This clamps an oracle-recommended rank by
511/// the layer's stored element size. **Static only** — applied once at
512/// `from_profile` time, never in the live `auto_adapt` path.
513///
514/// Returns the rank cap for a given stored element size in bytes:
515/// - 1 byte (FP8 / int8): cap 8
516/// - 2 bytes (FP16 / BF16): cap 16
517/// - otherwise (FP32+, or `0` = unknown): no cap (returns `usize::MAX`)
518pub fn rank_cap_for_dtype_bytes(elem_bytes: usize) -> usize {
519    match elem_bytes {
520        1 => 8,          // FP8 / int8: coarse, low usable rank
521        2 => 16,         // FP16 / BF16
522        _ => usize::MAX, // FP32 and wider (or 0 = unknown): full oracle rank
523    }
524}
525
526#[cfg(test)]
527#[allow(clippy::expect_used, clippy::unwrap_used)]
528mod tests {
529    use super::*;
530    use tch::Device;
531
532    /// Build a contributor for a single key with the given rank and dims.
533    /// `seed` perturbs the values so contributors differ.
534    fn make_contrib(
535        rank: i64,
536        in_dim: i64,
537        out_dim: i64,
538        trust: f64,
539        norm_ratio: f64,
540        host: &str,
541        seed: i64,
542    ) -> Contribution {
543        let a = Tensor::randn([rank, in_dim], (Kind::Float, Device::Cpu)) + seed as f64 * 0.01;
544        let b = Tensor::randn([out_dim, rank], (Kind::Float, Device::Cpu)) + seed as f64 * 0.01;
545        Contribution {
546            lora_a: a,
547            lora_b: b,
548            scaling: 1.0,
549            delta_norm_ratio: norm_ratio,
550            provenance: Provenance {
551                host_id: host.to_owned(),
552                trust,
553            },
554        }
555    }
556
557    /// Dense effective weight ΔW = Σ_i scaling_i·w_i·(B_i·A_i) for a contributor
558    /// set, replicating the gate weighting — used to check the reduced factors
559    /// reconstruct the same ΔW (within the truncation rank).
560    fn dense_delta(gate: &ContributionGate, contribs: &[Contribution]) -> Tensor {
561        let out_dim = contribs[0].lora_b.size()[0];
562        let in_dim = contribs[0].lora_a.size()[1];
563        let mut acc = Tensor::zeros([out_dim, in_dim], (Kind::Float, Device::Cpu));
564        for c in contribs {
565            let w = gate.weight_for(c).unwrap();
566            acc += c.lora_b.matmul(&c.lora_a) * w;
567        }
568        acc
569    }
570
571    fn reconstruct(a: &Tensor, b: &Tensor) -> Tensor {
572        b.matmul(a)
573    }
574
575    #[test]
576    fn test_output_rank_is_fixed_r() {
577        let reducer = StackSvdReducer::new(8);
578        let contribs = vec![
579            make_contrib(4, 32, 16, 1.0, 0.1, "h1", 1),
580            make_contrib(16, 32, 16, 1.0, 0.1, "h2", 2),
581        ];
582        let (a, b) = reducer.reduce_key("0.q_proj", &contribs).unwrap();
583        assert_eq!(a.size(), vec![8, 32]); // [R, in]
584        assert_eq!(b.size(), vec![16, 8]); // [out, R]
585    }
586
587    #[test]
588    fn test_order_independent() {
589        // R large enough to capture the full combined rank (4+5+6=15 -> R=16),
590        // so truncation is lossless and permutation must be exact to fp.
591        let reducer = StackSvdReducer::new(16);
592        let c1 = make_contrib(4, 24, 20, 1.0, 0.1, "h1", 1);
593        let c2 = make_contrib(5, 24, 20, 1.0, 0.1, "h2", 2);
594        let c3 = make_contrib(6, 24, 20, 1.0, 0.1, "h3", 3);
595
596        // Clone the underlying tensors so both orderings see identical inputs.
597        let clone = |c: &Contribution| Contribution {
598            lora_a: c.lora_a.copy(),
599            lora_b: c.lora_b.copy(),
600            scaling: c.scaling,
601            delta_norm_ratio: c.delta_norm_ratio,
602            provenance: c.provenance.clone(),
603        };
604
605        let order_a = vec![clone(&c1), clone(&c2), clone(&c3)];
606        let order_b = vec![clone(&c3), clone(&c1), clone(&c2)];
607
608        let (a1, b1) = reducer.reduce_key("k", &order_a).unwrap();
609        let (a2, b2) = reducer.reduce_key("k", &order_b).unwrap();
610
611        // Compare the reconstructed ΔW (factorization is only unique up to
612        // rotation; the product is the invariant).
613        let w1 = reconstruct(&a1, &b1);
614        let w2 = reconstruct(&a2, &b2);
615        let diff: f64 = (&w1 - &w2).abs().max().double_value(&[]);
616        assert!(
617            diff < 1e-3,
618            "permuting contributors must not change ΔW, max diff = {}",
619            diff
620        );
621    }
622
623    #[test]
624    fn test_tolerates_rank_heterogeneous_contributors() {
625        // The exact case DO-Merge hard-failed on: contributors with different
626        // ranks. Must succeed and reconstruct the weighted sum.
627        let reducer = StackSvdReducer::new(32); // R >= Σr so reconstruction is exact
628        let contribs = vec![
629            make_contrib(4, 20, 16, 1.0, 0.1, "h1", 1),
630            make_contrib(8, 20, 16, 1.0, 0.1, "h2", 2),
631            make_contrib(16, 20, 16, 1.0, 0.1, "h3", 3),
632        ];
633        let expected = dense_delta(&reducer.gate, &contribs);
634        let (a, b) = reducer.reduce_key("0.q_proj", &contribs).unwrap();
635        let got = reconstruct(&a, &b);
636        let diff: f64 = (&got - &expected).abs().max().double_value(&[]);
637        assert!(
638            diff < 1e-3,
639            "rank-heterogeneous reduction must reconstruct the weighted sum, max diff = {}",
640            diff
641        );
642    }
643
644    #[test]
645    fn test_norm_bound_gate_rejects_over_norm() {
646        let gate = ContributionGate {
647            max_norm_ratio: 1.0,
648            min_trust: 0.0,
649        };
650        let reducer = StackSvdReducer::new(8).with_gate(gate);
651        let contribs = vec![
652            make_contrib(4, 16, 16, 1.0, 0.2, "good", 1),
653            make_contrib(4, 16, 16, 1.0, 5.0, "evil", 2), // norm_ratio 5.0 > 1.0
654        ];
655        let err = reducer.reduce_key("k", &contribs).unwrap_err();
656        let msg = err.to_string();
657        assert!(
658            msg.contains("evil"),
659            "error should name the rejected host: {}",
660            msg
661        );
662        assert!(
663            msg.contains("exceeds bound"),
664            "error should explain the norm bound: {}",
665            msg
666        );
667    }
668
669    #[test]
670    fn test_min_trust_gate_rejects_untrusted() {
671        let gate = ContributionGate {
672            max_norm_ratio: 10.0,
673            min_trust: 0.5,
674        };
675        let reducer = StackSvdReducer::new(8).with_gate(gate);
676        let contribs = vec![make_contrib(4, 16, 16, 0.2, 0.1, "provisional", 1)];
677        let err = reducer.reduce_key("k", &contribs).unwrap_err();
678        assert!(err.to_string().contains("min_trust"), "{}", err);
679    }
680
681    #[test]
682    fn test_provenance_weighting_moves_merge_less() {
683        // A down-weighted contributor should move the merge less than a
684        // fully-trusted one. Compare the merge of [base, low-trust X] against
685        // [base, full-trust X]: the latter must be farther from base-alone.
686        let reducer = StackSvdReducer::new(16);
687        let base = make_contrib(8, 24, 24, 1.0, 0.1, "base", 10);
688        let x_full = make_contrib(8, 24, 24, 1.0, 0.1, "x", 99);
689        let x_low = Contribution {
690            lora_a: x_full.lora_a.copy(),
691            lora_b: x_full.lora_b.copy(),
692            scaling: 1.0,
693            delta_norm_ratio: 0.1,
694            provenance: Provenance {
695                host_id: "x".to_owned(),
696                trust: 0.05,
697            },
698        };
699        let base_clone = |seed_marker: &Contribution| Contribution {
700            lora_a: seed_marker.lora_a.copy(),
701            lora_b: seed_marker.lora_b.copy(),
702            scaling: 1.0,
703            delta_norm_ratio: 0.1,
704            provenance: seed_marker.provenance.clone(),
705        };
706
707        let (ba, bb) = reducer.reduce_key("k", &[base_clone(&base)]).unwrap();
708        let w_base = reconstruct(&ba, &bb);
709
710        let (fa, fb) = reducer
711            .reduce_key("k", &[base_clone(&base), x_full])
712            .unwrap();
713        let w_full = reconstruct(&fa, &fb);
714
715        let (la, lb) = reducer
716            .reduce_key("k", &[base_clone(&base), x_low])
717            .unwrap();
718        let w_low = reconstruct(&la, &lb);
719
720        let move_full: f64 = (&w_full - &w_base).abs().sum(Kind::Float).double_value(&[]);
721        let move_low: f64 = (&w_low - &w_base).abs().sum(Kind::Float).double_value(&[]);
722        assert!(
723            move_low < move_full,
724            "down-weighted contributor should move the merge less: low={} full={}",
725            move_low,
726            move_full
727        );
728    }
729
730    #[test]
731    fn test_reduce_state_dicts_end_to_end() {
732        // Two contributors, overlapping + disjoint keys, heterogeneous ranks.
733        let mk_state = |rank: i64, keys: &[&str]| -> HashMap<String, Tensor> {
734            let mut s = HashMap::new();
735            for k in keys {
736                s.insert(
737                    format!("{}.lora_a", k),
738                    Tensor::randn([rank, 32], (Kind::Float, Device::Cpu)),
739                );
740                s.insert(
741                    format!("{}.lora_b", k),
742                    Tensor::randn([16, rank], (Kind::Float, Device::Cpu)),
743                );
744            }
745            s
746        };
747        let c1 = ContributorState {
748            state: mk_state(4, &["0.q_proj", "0.v_proj"]),
749            scaling: HashMap::new(),
750            norm_ratios: HashMap::new(),
751            provenance: Provenance::trusted("h1"),
752        };
753        let c2 = ContributorState {
754            state: mk_state(8, &["0.q_proj", "1.q_proj"]),
755            scaling: HashMap::new(),
756            norm_ratios: HashMap::new(),
757            provenance: Provenance::trusted("h2"),
758        };
759        let reducer = StackSvdReducer::new(8);
760        let out = reduce_state_dicts(&reducer, &[c1, c2]).unwrap();
761
762        // Union of keys: 0.q_proj, 0.v_proj, 1.q_proj — all at fixed R=8.
763        for key in ["0.q_proj", "0.v_proj", "1.q_proj"] {
764            let a = &out[&format!("{}.lora_a", key)];
765            let b = &out[&format!("{}.lora_b", key)];
766            assert_eq!(a.size(), vec![8, 32], "{key} A");
767            assert_eq!(b.size(), vec![16, 8], "{key} B");
768        }
769        // No optimizer/effective-rank metadata leaks into the reduced dict.
770        assert!(out
771            .keys()
772            .all(|k| k.ends_with(".lora_a") || k.ends_with(".lora_b")));
773    }
774
775    #[test]
776    fn test_zero_contributors_errors() {
777        let reducer = StackSvdReducer::new(8);
778        assert!(reducer.reduce_key("k", &[]).is_err());
779    }
780
781    #[test]
782    fn test_zero_stack_returns_zero_factors() {
783        let reducer = StackSvdReducer::new(8);
784        let a = Tensor::zeros([4, 16], (Kind::Float, Device::Cpu));
785        let b = Tensor::zeros([16, 4], (Kind::Float, Device::Cpu));
786        let contribs = vec![Contribution {
787            lora_a: a,
788            lora_b: b,
789            scaling: 1.0,
790            delta_norm_ratio: 0.0,
791            provenance: Provenance::trusted("h1"),
792        }];
793        let (ra, rb) = reducer.reduce_key("k", &contribs).unwrap();
794        assert_eq!(ra.size(), vec![8, 16]);
795        assert_eq!(rb.size(), vec![16, 8]);
796        let norm: f64 = reconstruct(&ra, &rb).norm().double_value(&[]);
797        assert!(norm < 1e-6);
798    }
799
800    #[test]
801    fn test_rank_cap_for_dtype_bytes() {
802        assert_eq!(rank_cap_for_dtype_bytes(1), 8); // FP8/int8
803        assert_eq!(rank_cap_for_dtype_bytes(2), 16); // FP16/BF16
804        assert_eq!(rank_cap_for_dtype_bytes(4), usize::MAX); // FP32
805        assert_eq!(rank_cap_for_dtype_bytes(0), usize::MAX); // unknown
806    }
807}