hyprstream_inference/training/
merge.rs1use anyhow::{anyhow, Result};
12use std::collections::HashMap;
13use tch::Tensor;
14
15#[derive(Debug, Clone)]
17pub enum MergeStrategy {
18 Replace,
20 Additive { weight: f64 },
22 DoMerge { weight: f64 },
25}
26
27impl Default for MergeStrategy {
28 fn default() -> Self {
29 Self::DoMerge { weight: 0.3 }
30 }
31}
32
33impl MergeStrategy {
34 pub fn from_name(name: &str, weight: f64) -> Result<Self> {
36 match name {
37 "replace" => Ok(Self::Replace),
38 "additive" => Ok(Self::Additive { weight }),
39 "do_merge" | "doMerge" | "DO-Merge" => Ok(Self::DoMerge { weight }),
40 _ => Err(anyhow!(
41 "Unknown merge strategy: '{}'. Use 'replace', 'additive', or 'do_merge'.",
42 name
43 )),
44 }
45 }
46}
47
48pub fn merge_state_dicts(
58 existing: &HashMap<String, Tensor>,
59 new: &HashMap<String, Tensor>,
60 strategy: &MergeStrategy,
61) -> Result<HashMap<String, Tensor>> {
62 let _guard = tch::no_grad_guard();
63 let mut result = HashMap::new();
64
65 for (key, existing_tensor) in existing {
66 let merged = if let Some(new_tensor) = new.get(key) {
67 apply_strategy(existing_tensor, new_tensor, strategy)?
69 } else {
70 existing_tensor.copy()
72 };
73 result.insert(key.clone(), merged);
74 }
75
76 for (key, new_tensor) in new {
78 if !existing.contains_key(key) {
79 result.insert(key.clone(), new_tensor.copy());
80 }
81 }
82
83 Ok(result)
84}
85
86fn apply_strategy(existing: &Tensor, new: &Tensor, strategy: &MergeStrategy) -> Result<Tensor> {
88 match strategy {
89 MergeStrategy::Replace => replace_merge(existing, new),
90 MergeStrategy::Additive { weight } => additive_merge(existing, new, *weight),
91 MergeStrategy::DoMerge { weight } => do_merge(existing, new, *weight),
92 }
93}
94
95pub fn replace_merge(_existing: &Tensor, new: &Tensor) -> Result<Tensor> {
97 Ok(new.copy())
98}
99
100pub fn additive_merge(existing: &Tensor, new: &Tensor, weight: f64) -> Result<Tensor> {
102 if existing.size() != new.size() {
103 return Err(anyhow!(
104 "Shape mismatch in additive merge: {:?} vs {:?}",
105 existing.size(),
106 new.size()
107 ));
108 }
109 let existing = if existing.device() != new.device() {
111 existing.to_device(new.device())
112 } else {
113 existing.shallow_clone()
114 };
115 Ok(&existing * (1.0 - weight) + new * weight)
116}
117
118pub fn do_merge(existing: &Tensor, new: &Tensor, weight: f64) -> Result<Tensor> {
131 if existing.size() != new.size() {
132 return Err(anyhow!(
133 "Shape mismatch in DO-merge: {:?} vs {:?}",
134 existing.size(),
135 new.size()
136 ));
137 }
138 let existing = if existing.device() != new.device() {
140 existing.to_device(new.device())
141 } else {
142 existing.shallow_clone()
143 };
144
145 let mag_existing: f64 = existing.norm().double_value(&[]);
147 let mag_new: f64 = new.norm().double_value(&[]);
148
149 let eps = 1e-8;
151 if mag_existing < eps && mag_new < eps {
152 return Ok(Tensor::zeros(
153 existing.size().as_slice(),
154 (existing.kind(), existing.device()),
155 ));
156 }
157
158 if mag_existing < eps {
159 return Ok(new * weight);
161 }
162 if mag_new < eps {
163 return Ok(existing * (1.0 - weight));
165 }
166
167 let dir_existing = existing / mag_existing;
169 let dir_new = new / mag_new;
170
171 let mag_merged = (1.0 - weight) * mag_existing + weight * mag_new;
173
174 let dir_combined = &dir_existing * (1.0 - weight) + &dir_new * weight;
176 let dir_combined_norm: f64 = dir_combined.norm().double_value(&[]);
177
178 let dir_merged = if dir_combined_norm > eps {
179 dir_combined / dir_combined_norm
180 } else {
181 dir_existing
183 };
184
185 Ok(dir_merged * mag_merged)
187}
188
189#[cfg(test)]
190#[allow(clippy::expect_used, clippy::unwrap_used)]
191mod tests {
192 use super::*;
193 use tch::Kind;
194
195 #[test]
196 fn test_replace_merge() {
197 let a = Tensor::from_slice(&[1.0f32, 2.0, 3.0]);
198 let b = Tensor::from_slice(&[4.0f32, 5.0, 6.0]);
199
200 let result = replace_merge(&a, &b).unwrap();
201 let expected = vec![4.0f32, 5.0, 6.0];
202 let result_vec: Vec<f32> = Vec::try_from(result).unwrap();
203 assert_eq!(result_vec, expected);
204 }
205
206 #[test]
207 fn test_additive_merge_equal_weight() {
208 let a = Tensor::from_slice(&[2.0f32, 4.0, 6.0]);
209 let b = Tensor::from_slice(&[4.0f32, 6.0, 8.0]);
210
211 let result = additive_merge(&a, &b, 0.5).unwrap();
212 let result_vec: Vec<f32> = Vec::try_from(result).unwrap();
213 assert_eq!(result_vec, vec![3.0f32, 5.0, 7.0]);
215 }
216
217 #[test]
218 fn test_additive_merge_zero_weight() {
219 let a = Tensor::from_slice(&[1.0f32, 2.0]);
220 let b = Tensor::from_slice(&[9.0f32, 9.0]);
221
222 let result = additive_merge(&a, &b, 0.0).unwrap();
223 let result_vec: Vec<f32> = Vec::try_from(result).unwrap();
224 assert_eq!(result_vec, vec![1.0f32, 2.0]);
225 }
226
227 #[test]
228 fn test_additive_merge_full_weight() {
229 let a = Tensor::from_slice(&[1.0f32, 2.0]);
230 let b = Tensor::from_slice(&[9.0f32, 9.0]);
231
232 let result = additive_merge(&a, &b, 1.0).unwrap();
233 let result_vec: Vec<f32> = Vec::try_from(result).unwrap();
234 assert_eq!(result_vec, vec![9.0f32, 9.0]);
235 }
236
237 #[test]
238 fn test_do_merge_self_half_weight() {
239 let a = Tensor::from_slice(&[3.0f32, 4.0]); let result = do_merge(&a, &a, 0.5).unwrap();
242
243 let diff: f64 = (&result - &a).abs().sum(Kind::Float).double_value(&[]);
244 assert!(
245 diff < 1e-5,
246 "DO-merge with self at w=0.5 should be identity, diff={}",
247 diff
248 );
249 }
250
251 #[test]
252 fn test_do_merge_full_weight_is_replacement() {
253 let a = Tensor::from_slice(&[1.0f32, 0.0]);
254 let b = Tensor::from_slice(&[0.0f32, 2.0]);
255
256 let result = do_merge(&a, &b, 1.0).unwrap();
257 let diff: f64 = (&result - &b).abs().sum(Kind::Float).double_value(&[]);
258 assert!(
259 diff < 1e-5,
260 "DO-merge at w=1.0 should be replacement, diff={}",
261 diff
262 );
263 }
264
265 #[test]
266 fn test_do_merge_preserves_scale() {
267 let a = Tensor::from_slice(&[3.0f32, 4.0]); let b = Tensor::from_slice(&[6.0f32, 8.0]); let result = do_merge(&a, &b, 0.5).unwrap();
272 let result_norm: f64 = result.norm().double_value(&[]);
273
274 assert!(
276 (result_norm - 7.5).abs() < 0.01,
277 "Expected norm ~7.5, got {}",
278 result_norm
279 );
280 }
281
282 #[test]
283 fn test_do_merge_zero_tensors() {
284 let a = Tensor::zeros([3], (Kind::Float, tch::Device::Cpu));
285 let b = Tensor::zeros([3], (Kind::Float, tch::Device::Cpu));
286
287 let result = do_merge(&a, &b, 0.5).unwrap();
288 let norm: f64 = result.norm().double_value(&[]);
289 assert!(norm < 1e-8, "Merging zeros should produce zeros");
290 }
291
292 #[test]
293 fn test_merge_state_dicts() {
294 let mut existing = HashMap::new();
295 existing.insert("a".to_owned(), Tensor::from_slice(&[1.0f32, 2.0]));
296 existing.insert("b".to_owned(), Tensor::from_slice(&[3.0f32, 4.0]));
297
298 let mut new = HashMap::new();
299 new.insert("a".to_owned(), Tensor::from_slice(&[5.0f32, 6.0]));
300 let strategy = MergeStrategy::Additive { weight: 1.0 };
303 let result = merge_state_dicts(&existing, &new, &strategy).unwrap();
304
305 assert_eq!(result.len(), 2);
306 let a_vec: Vec<f32> = Vec::try_from(result["a"].shallow_clone()).unwrap();
308 assert_eq!(a_vec, vec![5.0f32, 6.0]);
309 let b_vec: Vec<f32> = Vec::try_from(result["b"].shallow_clone()).unwrap();
311 assert_eq!(b_vec, vec![3.0f32, 4.0]);
312 }
313
314 #[test]
315 fn test_shape_mismatch() {
316 let a = Tensor::from_slice(&[1.0f32, 2.0]);
317 let b = Tensor::from_slice(&[1.0f32, 2.0, 3.0]);
318
319 assert!(additive_merge(&a, &b, 0.5).is_err());
320 assert!(do_merge(&a, &b, 0.5).is_err());
321 }
322
323 #[test]
324 fn test_strategy_from_name() {
325 assert!(matches!(
326 MergeStrategy::from_name("replace", 0.5).unwrap(),
327 MergeStrategy::Replace
328 ));
329 assert!(matches!(
330 MergeStrategy::from_name("additive", 0.5).unwrap(),
331 MergeStrategy::Additive { .. }
332 ));
333 assert!(matches!(
334 MergeStrategy::from_name("do_merge", 0.3).unwrap(),
335 MergeStrategy::DoMerge { .. }
336 ));
337 assert!(MergeStrategy::from_name("unknown", 0.5).is_err());
338 }
339}