hyprstream_inference/training/delta_reducer.rs
1//! N-way delta aggregation (param-server reducer) — replaces DO-Merge for
2//! multi-contributor merges.
3//!
4//! # Why this exists
5//!
6//! [`crate::training::merge`]'s DO-Merge (`do_merge`) and `additive_merge`
7//! assert *shape equality* on the raw LoRA A/B factors. The moment two
8//! contributors carry different per-layer ranks — the normal case once the rank
9//! oracle (`ttn_profile`) assigns non-uniform per-layer ranks, or once
10//! different workers run with different `layer_overrides` — those paths
11//! hard-fail. They are also non-associative (path-dependent blend weights +
12//! direction renormalization), so an N-way merge would depend on contributor
13//! order.
14//!
15//! This module replaces the **N-way** aggregation with an order-independent,
16//! rank-heterogeneity-tolerant reducer. The 1-vs-1 save-time DO-Merge path may
17//! stay; this is the path used when *several* independent TTT contributors are
18//! aggregated each round.
19//!
20//! # Algorithm (stack + truncated SVD in delta-space)
21//!
22//! For one layer key, each contributor `i` supplies factors `A_i [r_i, in]`,
23//! `B_i [out, r_i]` and a fold-in weight `w_i` (provenance trust × norm-bound ×
24//! per-key scaling). The aggregate effective weight is
25//!
26//! ```text
27//! ΔW = Σ_i w_i · (B_i · A_i)
28//! = B_stack · A_stack
29//! ```
30//!
31//! where `B_stack = [ w_1·B_1 | w_2·B_2 | … ]` is `[out, Σr_i]` and
32//! `A_stack = [ A_1 ; A_2 ; … ]` is `[Σr_i, in]`. This is exact: stacking the
33//! factors *is* the weighted sum of the rank-1 updates, with no dense
34//! `[out, in]` materialization and no requirement that the `r_i` agree.
35//!
36//! We then take **one** rank-`R` truncated SVD of `ΔW` per round, computed on
37//! the tall-skinny stacked factors (`Σr` is the sum of contributor ranks —
38//! small) without ever forming the dense `[out, in]`:
39//!
40//! 1. economy SVD of `A_stack [Σr, in]` → `A_stack = Uₐ·diag(Sₐ)·Vhₐ`
41//! (factors are `[Σr, Σr]`, `[Σr]`, `[Σr, in]` since `Σr ≪ in`);
42//! 2. `C = B_stack · Uₐ · diag(Sₐ)` → `[out, Σr]` (small, no dense `out×in`);
43//! 3. economy SVD of `C [out, Σr]` → `C = U_c·diag(S_c)·Vh_c`;
44//! 4. `ΔW = U_c · diag(S_c) · (Vh_c · Vhₐ)` — an exact SVD of `ΔW`;
45//! 5. truncate to `R`: `B_out = U_c[:, :R]·diag(S_c[:R])` `[out, R]`,
46//! `A_out = (Vh_c·Vhₐ)[:R, :]` `[R, in]`.
47//!
48//! The reduced factors are stored with `scaling = 1.0` (the weights and
49//! per-contributor scaling are already folded in). Output rank is the fixed,
50//! operational `R` — stable checkpoint shapes.
51//!
52//! # Order-independence
53//!
54//! `ΔW` is a plain sum, so permuting contributors permutes columns of
55//! `B_stack` and rows of `A_stack` identically and leaves the product `ΔW`
56//! unchanged; the SVD of `ΔW` is therefore identical up to fp rounding.
57//!
58//! # Provenance / norm gate (secure multitenant)
59//!
60//! Aggregation is over an *explicit* contributor set — the reducer never
61//! discovers or mixes deltas across tenants. Each contributor carries a
62//! verified host id (Phase-2 mesh trust, #354) and a `delta_norm_ratio`. The
63//! [`ContributionGate`] rejects (does **not** silently drop or clamp-to-zero)
64//! any contributor whose norm exceeds the bound, and weights survivors by
65//! `trust × f(norm_ratio)`. A pluggable [`ValidationHook`] seam is provided for
66//! the deferred held-out-eval host (default impl is a no-op).
67//!
68//! # Send/Sync
69//!
70//! tch `Tensor` is `!Send`. The reducer keeps **all** tensor work on a single
71//! device/thread (`reduce()` takes `&[Contribution]` by reference and returns
72//! freshly-built tensors; it never moves a `Tensor` across a thread boundary).
73//! Contributions cross thread/host boundaries as serialized delta *bytes*
74//! (safetensors) + plain-old-data provenance, deserialized on the reducer
75//! thread — never as live `Tensor`s.
76//!
77//! # Optimizer state
78//!
79//! Per the spike, Muon momentum is **dropped** at aggregation: averaging
80//! optimizer state across independent TTT runs is unsound. The reduced delta
81//! starts with a fresh (empty) optimizer.
82
83use anyhow::{anyhow, Result};
84use std::collections::{HashMap, HashSet};
85use tch::{Kind, Tensor};
86
87/// Provenance for one contributor's delta.
88///
89/// The transport that *verifies* the signature (param-server / mesh) is out of
90/// scope for the reducer (a separate gated follow-on). By the time a
91/// [`Contribution`] reaches the reducer its `host_id` is already an
92/// operator-resolved, verified identity (Phase-2 mesh trust, #354). This struct
93/// is plain-old-data so it can cross a thread/host boundary without moving any
94/// `Tensor`.
95#[derive(Debug, Clone)]
96pub struct Provenance {
97 /// Verified host id (operator label resolved from the signing identity).
98 pub host_id: String,
99 /// Trust weight for this host in `[0, 1]`. `1.0` = full trust. A downweighted
100 /// (e.g. newly-seen) host moves the merge less.
101 pub trust: f64,
102}
103
104impl Provenance {
105 /// Construct a fully-trusted contributor identity.
106 pub fn trusted(host_id: impl Into<String>) -> Self {
107 Self {
108 host_id: host_id.into(),
109 trust: 1.0,
110 }
111 }
112}
113
114/// One contributor in an N-way aggregation round.
115///
116/// Holds the contributor's LoRA factors *for a single layer key already* — the
117/// reducer is invoked per key by [`reduce_state_dicts`]. Factors are borrowed
118/// (shallow clones / views are fine); the reducer does not mutate them.
119pub struct Contribution {
120 /// `A_i [r_i, in]`.
121 pub lora_a: Tensor,
122 /// `B_i [out, r_i]`.
123 pub lora_b: Tensor,
124 /// Per-key LoRA scaling (`alpha / rank`) already associated with these
125 /// factors. Folded into the stack weight so the reduced factors can use
126 /// `scaling = 1.0`.
127 pub scaling: f64,
128 /// `||ΔW_i|| / ||W_base||` for this key (from
129 /// [`crate::training::TenantDelta::delta_norm_ratio`]). Used by the gate.
130 pub delta_norm_ratio: f64,
131 /// Verified provenance.
132 pub provenance: Provenance,
133}
134
135/// Gate + weighting policy applied to each contributor before it enters the
136/// stack. Anomalous / over-norm contributors are **rejected** (error), never
137/// silently dropped or clamped to zero — a quiet drop would let a poisoned host
138/// shrink the merge invisibly.
139#[derive(Debug, Clone)]
140pub struct ContributionGate {
141 /// Reject any contributor whose `delta_norm_ratio` exceeds this bound.
142 /// Defends against norm-inflation poisoning.
143 pub max_norm_ratio: f64,
144 /// Reject any contributor whose resolved trust is below this floor.
145 pub min_trust: f64,
146}
147
148impl Default for ContributionGate {
149 fn default() -> Self {
150 // 2.0 = the aggregate update may be at most ~2x the base-weight norm for
151 // a single contributor; well above legitimate TTT drift, well below a
152 // runaway. min_trust=0 admits any positively-trusted host; raise to
153 // exclude provisional peers.
154 Self {
155 max_norm_ratio: 2.0,
156 min_trust: 0.0,
157 }
158 }
159}
160
161impl ContributionGate {
162 /// Validate one contributor and return its non-negative stack weight
163 /// `trust × f(norm_ratio) × scaling`, or an error if it fails the gate.
164 ///
165 /// `f(norm_ratio) = 1 / (1 + norm_ratio)` softly downweights large-norm
166 /// (more-likely-anomalous) deltas that still pass the hard bound, so a
167 /// borderline contributor moves the merge less than a small clean one.
168 fn weight_for(&self, c: &Contribution) -> Result<f64> {
169 let trust = c.provenance.trust;
170 if !(trust.is_finite()) || trust < 0.0 {
171 return Err(anyhow!(
172 "contributor '{}' has invalid trust {}",
173 c.provenance.host_id,
174 trust
175 ));
176 }
177 if trust < self.min_trust {
178 return Err(anyhow!(
179 "contributor '{}' rejected: trust {} < min_trust {}",
180 c.provenance.host_id,
181 trust,
182 self.min_trust
183 ));
184 }
185 let ratio = c.delta_norm_ratio;
186 if !ratio.is_finite() || ratio < 0.0 {
187 return Err(anyhow!(
188 "contributor '{}' has invalid delta_norm_ratio {}",
189 c.provenance.host_id,
190 ratio
191 ));
192 }
193 if ratio > self.max_norm_ratio {
194 return Err(anyhow!(
195 "contributor '{}' rejected: delta_norm_ratio {:.4} exceeds bound {:.4} (possible norm-inflation poisoning)",
196 c.provenance.host_id,
197 ratio,
198 self.max_norm_ratio
199 ));
200 }
201 let norm_weight = 1.0 / (1.0 + ratio);
202 Ok(trust * norm_weight * c.scaling)
203 }
204}
205
206/// Pluggable post-reduce validation seam.
207///
208/// The real implementation (held-out-eval host) is a deferred, gated follow-on.
209/// Defining the trait now lets the eval-host swap in with zero consumer churn.
210pub trait ValidationHook: Send + Sync {
211 /// Inspect the freshly-reduced factors for one key. Return `Ok(())` to
212 /// accept or an error to reject the round. Default: accept everything.
213 ///
214 /// Tensors are passed by reference and must not be moved across threads.
215 fn validate(&self, _key: &str, _reduced_a: &Tensor, _reduced_b: &Tensor) -> Result<()> {
216 Ok(())
217 }
218}
219
220/// No-op validation hook (the default until the held-out-eval host lands).
221#[derive(Debug, Default, Clone, Copy)]
222pub struct NoopValidation;
223
224impl ValidationHook for NoopValidation {}
225
226/// Order-independent N-way delta reducer.
227///
228/// Implementors merge an explicit contributor set into a fixed-rank delta. The
229/// stack + truncated-SVD implementation is [`StackSvdReducer`]; the trait keeps
230/// it swappable (held-out-eval variants, future native reducers).
231pub trait DeltaReducer {
232 /// Reduce contributors for a **single** layer key into fixed-rank factors
233 /// `(A_out [R, in], B_out [out, R])`, with the result expressed at
234 /// `scaling = 1.0` (weights + per-contributor scaling already folded in).
235 ///
236 /// All tensors stay on `contributions[*].lora_a.device()` (must be uniform).
237 fn reduce_key(&self, key: &str, contributions: &[Contribution]) -> Result<(Tensor, Tensor)>;
238}
239
240/// Stack-in-delta-space + one truncated SVD per round to a fixed rank `R`.
241pub struct StackSvdReducer {
242 /// Fixed output rank — an operational contract (stable checkpoint shapes).
243 pub rank: usize,
244 /// Contributor gate / weighting.
245 pub gate: ContributionGate,
246 /// Pluggable validation hook (default no-op).
247 pub validation: Box<dyn ValidationHook>,
248}
249
250impl StackSvdReducer {
251 /// Construct with a fixed output rank and the default gate / no-op hook.
252 pub fn new(rank: usize) -> Self {
253 Self {
254 rank,
255 gate: ContributionGate::default(),
256 validation: Box::new(NoopValidation),
257 }
258 }
259
260 /// Override the contributor gate.
261 pub fn with_gate(mut self, gate: ContributionGate) -> Self {
262 self.gate = gate;
263 self
264 }
265
266 /// Install a validation hook (e.g. the held-out-eval host once it exists).
267 pub fn with_validation(mut self, validation: Box<dyn ValidationHook>) -> Self {
268 self.validation = validation;
269 self
270 }
271}
272
273impl DeltaReducer for StackSvdReducer {
274 fn reduce_key(&self, key: &str, contributions: &[Contribution]) -> Result<(Tensor, Tensor)> {
275 let _guard = tch::no_grad_guard();
276
277 if contributions.is_empty() {
278 return Err(anyhow!("reduce_key('{}'): no contributors", key));
279 }
280 if self.rank == 0 {
281 return Err(anyhow!("reduce_key('{}'): fixed rank R must be >= 1", key));
282 }
283
284 // Uniform device + matching in/out dims across contributors. The reducer
285 // does all tensor work on this one device/thread (Send/Sync invariant).
286 let device = contributions[0].lora_a.device();
287 let out_dim = contributions[0].lora_b.size()[0];
288 let in_dim = contributions[0].lora_a.size()[1];
289
290 // Build the stacked factors, folding the gate weight into B (one side
291 // only — weighting both A and B would square the weight).
292 // B_stack = [ w_1·B_1 | … ] [out, Σr]
293 // A_stack = [ A_1 ; … ] [Σr, in]
294 let mut b_cols: Vec<Tensor> = Vec::with_capacity(contributions.len());
295 let mut a_rows: Vec<Tensor> = Vec::with_capacity(contributions.len());
296
297 for c in contributions {
298 // Validate device + shape consistency.
299 if c.lora_a.device() != device || c.lora_b.device() != device {
300 return Err(anyhow!(
301 "reduce_key('{}'): contributor '{}' on a different device — reducer requires a single device",
302 key,
303 c.provenance.host_id
304 ));
305 }
306 let a_size = c.lora_a.size();
307 let b_size = c.lora_b.size();
308 if a_size.len() != 2 || b_size.len() != 2 {
309 return Err(anyhow!("reduce_key('{}'): factors must be 2D", key));
310 }
311 let (ri, in_i) = (a_size[0], a_size[1]);
312 let (out_i, ri_b) = (b_size[0], b_size[1]);
313 if in_i != in_dim || out_i != out_dim {
314 return Err(anyhow!(
315 "reduce_key('{}'): contributor '{}' dims [out={},in={}] differ from [out={},in={}]",
316 key, c.provenance.host_id, out_i, in_i, out_dim, in_dim
317 ));
318 }
319 if ri != ri_b {
320 return Err(anyhow!(
321 "reduce_key('{}'): contributor '{}' rank mismatch A[0]={} vs B[1]={}",
322 key,
323 c.provenance.host_id,
324 ri,
325 ri_b
326 ));
327 }
328
329 // Gate + weight (rejects over-norm / untrusted; never silent-drops).
330 let weight = self.gate.weight_for(c)?;
331
332 // Compute the reducer in fp32 (deltas are fp32 delta-space).
333 let a = c.lora_a.to_kind(Kind::Float);
334 let b = c.lora_b.to_kind(Kind::Float) * weight;
335 a_rows.push(a);
336 b_cols.push(b);
337 }
338
339 // Σr small (sum of contributor ranks). Concat is the only unavoidable
340 // materialization; both stacks are tall-skinny.
341 let a_refs: Vec<&Tensor> = a_rows.iter().collect();
342 let b_refs: Vec<&Tensor> = b_cols.iter().collect();
343 let a_stack = Tensor::cat(&a_refs, 0); // [Σr, in]
344 let b_stack = Tensor::cat(&b_refs, 1); // [out, Σr]
345 let sigma_r = a_stack.size()[0];
346
347 // If the whole stack is ~zero (all contributors empty/cancelled), return
348 // zero factors at the fixed rank rather than running SVD on zeros.
349 let stack_norm: f64 = a_stack.norm().double_value(&[]) * b_stack.norm().double_value(&[]);
350 let r = self.rank as i64;
351 if stack_norm < 1e-12 {
352 let a_out = Tensor::zeros([r, in_dim], (Kind::Float, device));
353 let b_out = Tensor::zeros([out_dim, r], (Kind::Float, device));
354 self.validation.validate(key, &a_out, &b_out)?;
355 return Ok((a_out, b_out));
356 }
357
358 // Exact rank-Σr SVD of ΔW = B_stack · A_stack, computed via two economy
359 // SVDs on the tall-skinny stacks (never forms the dense [out,in]).
360 //
361 // 1) A_stack = Ua · diag(Sa) · Vha (economy: some=true, compute_uv=true)
362 let (ua, sa, va) = a_stack.svd(true, true); // Ua [Σr,k], Sa [k], Va [in,k]
363 // tch's `svd` returns V (not Vh); Vha = Va^T.
364 let k = sa.size()[0]; // = min(Σr, in) = Σr
365 let _ = k;
366 // 2) C = B_stack · Ua · diag(Sa) → [out, Σr]
367 let c = b_stack.matmul(&ua) * sa.unsqueeze(0); // broadcast diag(Sa) over rows
368 // 3) C = Uc · diag(Sc) · Vhc
369 let (uc, sc, vc) = c.svd(true, true); // Uc [out,m], Sc [m], Vc [Σr,m]
370 // 4) Vh_final = Vhc · Vha = Vc^T · Va^T = (Va · Vc)^T → [m, in]
371 // ΔW = Uc · diag(Sc) · Vh_final
372 let vh_final = va.matmul(&vc).tr(); // [m, in]
373
374 // 5) Truncate to fixed R (pad with zeros if Σr < R so shapes are stable).
375 let avail = sc.size()[0].min(sigma_r);
376 let take = avail.min(r);
377
378 let uc_t = uc.narrow(1, 0, take); // [out, take]
379 let sc_t = sc.narrow(0, 0, take); // [take]
380 let vh_t = vh_final.narrow(0, 0, take); // [take, in]
381
382 // B_out = Uc[:, :R] · diag(Sc[:R]) [out, take]
383 let b_take = &uc_t * sc_t.unsqueeze(0);
384 // A_out = Vh_final[:R, :] [take, in]
385 let a_take = vh_t.contiguous();
386 let b_take = b_take.contiguous();
387
388 let (a_out, b_out) = if take < r {
389 // Pad the rank dimension with zeros to the fixed R (zero-energy
390 // directions; keeps checkpoint shapes invariant).
391 let a_pad = Tensor::zeros([r - take, in_dim], (Kind::Float, device));
392 let b_pad = Tensor::zeros([out_dim, r - take], (Kind::Float, device));
393 let a_out = Tensor::cat(&[&a_take, &a_pad], 0);
394 let b_out = Tensor::cat(&[&b_take, &b_pad], 1);
395 (a_out, b_out)
396 } else {
397 (a_take, b_take)
398 };
399
400 self.validation.validate(key, &a_out, &b_out)?;
401 Ok((a_out, b_out))
402 }
403}
404
405/// Aggregate N contributors' full state dicts into one reduced state dict.
406///
407/// Each contributor supplies a `(state_dict, provenance, norm_ratios)` where the
408/// state dict uses the `extract_state_dict` key convention
409/// (`"layer.module.lora_a"` / `"layer.module.lora_b"`). The reducer runs the
410/// stack + truncated-SVD per layer key and emits reduced factors at the fixed
411/// rank `R`. Effective-rank / Muon-momentum metadata are intentionally dropped
412/// (fresh optimizer post-aggregation, per the spike).
413///
414/// Keys present in only some contributors are aggregated over the subset that
415/// has them — this is still order-independent and tolerant of contributors that
416/// trained different module sets.
417pub fn reduce_state_dicts(
418 reducer: &dyn DeltaReducer,
419 contributors: &[ContributorState],
420) -> Result<HashMap<String, Tensor>> {
421 let _guard = tch::no_grad_guard();
422 if contributors.is_empty() {
423 return Err(anyhow!("reduce_state_dicts: no contributors"));
424 }
425
426 // Collect the union of layer keys across all contributors.
427 let mut layer_keys: Vec<String> = Vec::new();
428 let mut seen: HashSet<String> = HashSet::new();
429 for cs in contributors {
430 for k in cs.state.keys() {
431 if let Some(layer_key) = k.strip_suffix(".lora_a") {
432 if seen.insert(layer_key.to_owned()) {
433 layer_keys.push(layer_key.to_owned());
434 }
435 }
436 }
437 }
438 layer_keys.sort(); // deterministic output ordering
439
440 let mut result: HashMap<String, Tensor> = HashMap::new();
441
442 for layer_key in &layer_keys {
443 let a_key = format!("{}.lora_a", layer_key);
444 let b_key = format!("{}.lora_b", layer_key);
445
446 let mut contribs: Vec<Contribution> = Vec::new();
447 for cs in contributors {
448 let (Some(a), Some(b)) = (cs.state.get(&a_key), cs.state.get(&b_key)) else {
449 continue; // this contributor didn't train this key
450 };
451 let norm_ratio = cs
452 .norm_ratios
453 .get(layer_key)
454 .copied()
455 // module-name fallback ("layer.module" -> "module") mirrors
456 // delta_norm_ratio's own keying tolerance.
457 .or_else(|| {
458 layer_key
459 .split_once('.')
460 .and_then(|(_, m)| cs.norm_ratios.get(m).copied())
461 })
462 .unwrap_or(0.0);
463 contribs.push(Contribution {
464 lora_a: a.shallow_clone(),
465 lora_b: b.shallow_clone(),
466 scaling: cs.scaling.get(layer_key).copied().unwrap_or(1.0),
467 delta_norm_ratio: norm_ratio,
468 provenance: cs.provenance.clone(),
469 });
470 }
471
472 if contribs.is_empty() {
473 continue;
474 }
475
476 let (a_out, b_out) = reducer.reduce_key(layer_key, &contribs)?;
477 result.insert(a_key, a_out);
478 result.insert(b_key, b_out);
479 }
480
481 if result.is_empty() {
482 return Err(anyhow!(
483 "reduce_state_dicts: no aggregatable layer keys across contributors"
484 ));
485 }
486
487 Ok(result)
488}
489
490/// One contributor's full state for [`reduce_state_dicts`].
491///
492/// Plain-old-data + tensors that stay on one thread. The `state` tensors are
493/// expected to already be on the reducer's device (deserialized from bytes on
494/// this thread — see the module-level Send/Sync note).
495pub struct ContributorState {
496 /// `extract_state_dict`-keyed factors (`"layer.module.lora_a"` etc.).
497 pub state: HashMap<String, Tensor>,
498 /// Per-layer-key scaling (`alpha / rank`).
499 pub scaling: HashMap<String, f64>,
500 /// Per-layer-key (or per-module) `delta_norm_ratio`.
501 pub norm_ratios: HashMap<String, f64>,
502 /// Verified provenance for this contributor.
503 pub provenance: Provenance,
504}
505
506/// Static rank↔quant coupling cap.
507///
508/// Coarse quantization (e.g. block-wise FP8) gives a delta limited headroom: a
509/// rank the layer's stored dtype can't actually express is wasted (and adds
510/// noise from untrained directions). This clamps an oracle-recommended rank by
511/// the layer's stored element size. **Static only** — applied once at
512/// `from_profile` time, never in the live `auto_adapt` path.
513///
514/// Returns the rank cap for a given stored element size in bytes:
515/// - 1 byte (FP8 / int8): cap 8
516/// - 2 bytes (FP16 / BF16): cap 16
517/// - otherwise (FP32+, or `0` = unknown): no cap (returns `usize::MAX`)
518pub fn rank_cap_for_dtype_bytes(elem_bytes: usize) -> usize {
519 match elem_bytes {
520 1 => 8, // FP8 / int8: coarse, low usable rank
521 2 => 16, // FP16 / BF16
522 _ => usize::MAX, // FP32 and wider (or 0 = unknown): full oracle rank
523 }
524}
525
526#[cfg(test)]
527#[allow(clippy::expect_used, clippy::unwrap_used)]
528mod tests {
529 use super::*;
530 use tch::Device;
531
532 /// Build a contributor for a single key with the given rank and dims.
533 /// `seed` perturbs the values so contributors differ.
534 fn make_contrib(
535 rank: i64,
536 in_dim: i64,
537 out_dim: i64,
538 trust: f64,
539 norm_ratio: f64,
540 host: &str,
541 seed: i64,
542 ) -> Contribution {
543 let a = Tensor::randn([rank, in_dim], (Kind::Float, Device::Cpu)) + seed as f64 * 0.01;
544 let b = Tensor::randn([out_dim, rank], (Kind::Float, Device::Cpu)) + seed as f64 * 0.01;
545 Contribution {
546 lora_a: a,
547 lora_b: b,
548 scaling: 1.0,
549 delta_norm_ratio: norm_ratio,
550 provenance: Provenance {
551 host_id: host.to_owned(),
552 trust,
553 },
554 }
555 }
556
557 /// Dense effective weight ΔW = Σ_i scaling_i·w_i·(B_i·A_i) for a contributor
558 /// set, replicating the gate weighting — used to check the reduced factors
559 /// reconstruct the same ΔW (within the truncation rank).
560 fn dense_delta(gate: &ContributionGate, contribs: &[Contribution]) -> Tensor {
561 let out_dim = contribs[0].lora_b.size()[0];
562 let in_dim = contribs[0].lora_a.size()[1];
563 let mut acc = Tensor::zeros([out_dim, in_dim], (Kind::Float, Device::Cpu));
564 for c in contribs {
565 let w = gate.weight_for(c).unwrap();
566 acc += c.lora_b.matmul(&c.lora_a) * w;
567 }
568 acc
569 }
570
571 fn reconstruct(a: &Tensor, b: &Tensor) -> Tensor {
572 b.matmul(a)
573 }
574
575 #[test]
576 fn test_output_rank_is_fixed_r() {
577 let reducer = StackSvdReducer::new(8);
578 let contribs = vec![
579 make_contrib(4, 32, 16, 1.0, 0.1, "h1", 1),
580 make_contrib(16, 32, 16, 1.0, 0.1, "h2", 2),
581 ];
582 let (a, b) = reducer.reduce_key("0.q_proj", &contribs).unwrap();
583 assert_eq!(a.size(), vec![8, 32]); // [R, in]
584 assert_eq!(b.size(), vec![16, 8]); // [out, R]
585 }
586
587 #[test]
588 fn test_order_independent() {
589 // R large enough to capture the full combined rank (4+5+6=15 -> R=16),
590 // so truncation is lossless and permutation must be exact to fp.
591 let reducer = StackSvdReducer::new(16);
592 let c1 = make_contrib(4, 24, 20, 1.0, 0.1, "h1", 1);
593 let c2 = make_contrib(5, 24, 20, 1.0, 0.1, "h2", 2);
594 let c3 = make_contrib(6, 24, 20, 1.0, 0.1, "h3", 3);
595
596 // Clone the underlying tensors so both orderings see identical inputs.
597 let clone = |c: &Contribution| Contribution {
598 lora_a: c.lora_a.copy(),
599 lora_b: c.lora_b.copy(),
600 scaling: c.scaling,
601 delta_norm_ratio: c.delta_norm_ratio,
602 provenance: c.provenance.clone(),
603 };
604
605 let order_a = vec![clone(&c1), clone(&c2), clone(&c3)];
606 let order_b = vec![clone(&c3), clone(&c1), clone(&c2)];
607
608 let (a1, b1) = reducer.reduce_key("k", &order_a).unwrap();
609 let (a2, b2) = reducer.reduce_key("k", &order_b).unwrap();
610
611 // Compare the reconstructed ΔW (factorization is only unique up to
612 // rotation; the product is the invariant).
613 let w1 = reconstruct(&a1, &b1);
614 let w2 = reconstruct(&a2, &b2);
615 let diff: f64 = (&w1 - &w2).abs().max().double_value(&[]);
616 assert!(
617 diff < 1e-3,
618 "permuting contributors must not change ΔW, max diff = {}",
619 diff
620 );
621 }
622
623 #[test]
624 fn test_tolerates_rank_heterogeneous_contributors() {
625 // The exact case DO-Merge hard-failed on: contributors with different
626 // ranks. Must succeed and reconstruct the weighted sum.
627 let reducer = StackSvdReducer::new(32); // R >= Σr so reconstruction is exact
628 let contribs = vec![
629 make_contrib(4, 20, 16, 1.0, 0.1, "h1", 1),
630 make_contrib(8, 20, 16, 1.0, 0.1, "h2", 2),
631 make_contrib(16, 20, 16, 1.0, 0.1, "h3", 3),
632 ];
633 let expected = dense_delta(&reducer.gate, &contribs);
634 let (a, b) = reducer.reduce_key("0.q_proj", &contribs).unwrap();
635 let got = reconstruct(&a, &b);
636 let diff: f64 = (&got - &expected).abs().max().double_value(&[]);
637 assert!(
638 diff < 1e-3,
639 "rank-heterogeneous reduction must reconstruct the weighted sum, max diff = {}",
640 diff
641 );
642 }
643
644 #[test]
645 fn test_norm_bound_gate_rejects_over_norm() {
646 let gate = ContributionGate {
647 max_norm_ratio: 1.0,
648 min_trust: 0.0,
649 };
650 let reducer = StackSvdReducer::new(8).with_gate(gate);
651 let contribs = vec![
652 make_contrib(4, 16, 16, 1.0, 0.2, "good", 1),
653 make_contrib(4, 16, 16, 1.0, 5.0, "evil", 2), // norm_ratio 5.0 > 1.0
654 ];
655 let err = reducer.reduce_key("k", &contribs).unwrap_err();
656 let msg = err.to_string();
657 assert!(
658 msg.contains("evil"),
659 "error should name the rejected host: {}",
660 msg
661 );
662 assert!(
663 msg.contains("exceeds bound"),
664 "error should explain the norm bound: {}",
665 msg
666 );
667 }
668
669 #[test]
670 fn test_min_trust_gate_rejects_untrusted() {
671 let gate = ContributionGate {
672 max_norm_ratio: 10.0,
673 min_trust: 0.5,
674 };
675 let reducer = StackSvdReducer::new(8).with_gate(gate);
676 let contribs = vec![make_contrib(4, 16, 16, 0.2, 0.1, "provisional", 1)];
677 let err = reducer.reduce_key("k", &contribs).unwrap_err();
678 assert!(err.to_string().contains("min_trust"), "{}", err);
679 }
680
681 #[test]
682 fn test_provenance_weighting_moves_merge_less() {
683 // A down-weighted contributor should move the merge less than a
684 // fully-trusted one. Compare the merge of [base, low-trust X] against
685 // [base, full-trust X]: the latter must be farther from base-alone.
686 let reducer = StackSvdReducer::new(16);
687 let base = make_contrib(8, 24, 24, 1.0, 0.1, "base", 10);
688 let x_full = make_contrib(8, 24, 24, 1.0, 0.1, "x", 99);
689 let x_low = Contribution {
690 lora_a: x_full.lora_a.copy(),
691 lora_b: x_full.lora_b.copy(),
692 scaling: 1.0,
693 delta_norm_ratio: 0.1,
694 provenance: Provenance {
695 host_id: "x".to_owned(),
696 trust: 0.05,
697 },
698 };
699 let base_clone = |seed_marker: &Contribution| Contribution {
700 lora_a: seed_marker.lora_a.copy(),
701 lora_b: seed_marker.lora_b.copy(),
702 scaling: 1.0,
703 delta_norm_ratio: 0.1,
704 provenance: seed_marker.provenance.clone(),
705 };
706
707 let (ba, bb) = reducer.reduce_key("k", &[base_clone(&base)]).unwrap();
708 let w_base = reconstruct(&ba, &bb);
709
710 let (fa, fb) = reducer
711 .reduce_key("k", &[base_clone(&base), x_full])
712 .unwrap();
713 let w_full = reconstruct(&fa, &fb);
714
715 let (la, lb) = reducer
716 .reduce_key("k", &[base_clone(&base), x_low])
717 .unwrap();
718 let w_low = reconstruct(&la, &lb);
719
720 let move_full: f64 = (&w_full - &w_base).abs().sum(Kind::Float).double_value(&[]);
721 let move_low: f64 = (&w_low - &w_base).abs().sum(Kind::Float).double_value(&[]);
722 assert!(
723 move_low < move_full,
724 "down-weighted contributor should move the merge less: low={} full={}",
725 move_low,
726 move_full
727 );
728 }
729
730 #[test]
731 fn test_reduce_state_dicts_end_to_end() {
732 // Two contributors, overlapping + disjoint keys, heterogeneous ranks.
733 let mk_state = |rank: i64, keys: &[&str]| -> HashMap<String, Tensor> {
734 let mut s = HashMap::new();
735 for k in keys {
736 s.insert(
737 format!("{}.lora_a", k),
738 Tensor::randn([rank, 32], (Kind::Float, Device::Cpu)),
739 );
740 s.insert(
741 format!("{}.lora_b", k),
742 Tensor::randn([16, rank], (Kind::Float, Device::Cpu)),
743 );
744 }
745 s
746 };
747 let c1 = ContributorState {
748 state: mk_state(4, &["0.q_proj", "0.v_proj"]),
749 scaling: HashMap::new(),
750 norm_ratios: HashMap::new(),
751 provenance: Provenance::trusted("h1"),
752 };
753 let c2 = ContributorState {
754 state: mk_state(8, &["0.q_proj", "1.q_proj"]),
755 scaling: HashMap::new(),
756 norm_ratios: HashMap::new(),
757 provenance: Provenance::trusted("h2"),
758 };
759 let reducer = StackSvdReducer::new(8);
760 let out = reduce_state_dicts(&reducer, &[c1, c2]).unwrap();
761
762 // Union of keys: 0.q_proj, 0.v_proj, 1.q_proj — all at fixed R=8.
763 for key in ["0.q_proj", "0.v_proj", "1.q_proj"] {
764 let a = &out[&format!("{}.lora_a", key)];
765 let b = &out[&format!("{}.lora_b", key)];
766 assert_eq!(a.size(), vec![8, 32], "{key} A");
767 assert_eq!(b.size(), vec![16, 8], "{key} B");
768 }
769 // No optimizer/effective-rank metadata leaks into the reduced dict.
770 assert!(out
771 .keys()
772 .all(|k| k.ends_with(".lora_a") || k.ends_with(".lora_b")));
773 }
774
775 #[test]
776 fn test_zero_contributors_errors() {
777 let reducer = StackSvdReducer::new(8);
778 assert!(reducer.reduce_key("k", &[]).is_err());
779 }
780
781 #[test]
782 fn test_zero_stack_returns_zero_factors() {
783 let reducer = StackSvdReducer::new(8);
784 let a = Tensor::zeros([4, 16], (Kind::Float, Device::Cpu));
785 let b = Tensor::zeros([16, 4], (Kind::Float, Device::Cpu));
786 let contribs = vec![Contribution {
787 lora_a: a,
788 lora_b: b,
789 scaling: 1.0,
790 delta_norm_ratio: 0.0,
791 provenance: Provenance::trusted("h1"),
792 }];
793 let (ra, rb) = reducer.reduce_key("k", &contribs).unwrap();
794 assert_eq!(ra.size(), vec![8, 16]);
795 assert_eq!(rb.size(), vec![16, 8]);
796 let norm: f64 = reconstruct(&ra, &rb).norm().double_value(&[]);
797 assert!(norm < 1e-6);
798 }
799
800 #[test]
801 fn test_rank_cap_for_dtype_bytes() {
802 assert_eq!(rank_cap_for_dtype_bytes(1), 8); // FP8/int8
803 assert_eq!(rank_cap_for_dtype_bytes(2), 16); // FP16/BF16
804 assert_eq!(rank_cap_for_dtype_bytes(4), usize::MAX); // FP32
805 assert_eq!(rank_cap_for_dtype_bytes(0), usize::MAX); // unknown
806 }
807}