hyprstream_inference/runtime/
batched_lora.rs1use anyhow::{anyhow, Result};
8use std::sync::Arc;
9use tch::{Device, Tensor};
10
11use crate::training::DeltaPool;
12use hyprstream_rpc::Subject;
13
14pub struct BatchedLoRAForward {
19 delta_pool: Arc<DeltaPool>,
21 #[allow(dead_code)]
23 max_rank: usize,
24 #[allow(dead_code)]
26 scaling: f64,
27 #[allow(dead_code)]
29 device: Device,
30}
31
32impl BatchedLoRAForward {
33 pub fn new(delta_pool: Arc<DeltaPool>, max_rank: usize, scaling: f64, device: Device) -> Self {
41 Self {
42 delta_pool,
43 max_rank,
44 scaling,
45 device,
46 }
47 }
48
49 pub fn batched_lora_forward(
61 &self,
62 x: &[&Tensor],
63 module_name: &str,
64 tenant_ids: &[Subject],
65 layer_idx: usize,
66 ) -> Result<Vec<Tensor>> {
67 if x.len() != tenant_ids.len() {
68 return Err(anyhow!(
69 "Input count ({}) doesn't match tenant count ({})",
70 x.len(),
71 tenant_ids.len()
72 ));
73 }
74
75 if x.is_empty() {
76 return Ok(Vec::new());
77 }
78
79 let batch_size = x.len();
80 let _guard = tch::no_grad_guard();
81
82 let mut a_matrices: Vec<Option<Tensor>> = Vec::with_capacity(batch_size);
84 let mut b_matrices: Vec<Option<Tensor>> = Vec::with_capacity(batch_size);
85 let mut has_any_delta = false;
86
87 let key = format!("{}.{}", layer_idx, module_name);
88 for tenant_id in tenant_ids {
89 if let Some(delta_arc) = self.delta_pool.get(tenant_id) {
90 let delta = delta_arc.lock();
91 if let (Some(a), Some(b)) = (delta.lora_a.get(&key), delta.lora_b.get(&key)) {
92 a_matrices.push(Some(a.shallow_clone()));
94 b_matrices.push(Some(b.shallow_clone()));
95 has_any_delta = true;
96 } else {
97 a_matrices.push(None);
98 b_matrices.push(None);
99 }
100 } else {
101 a_matrices.push(None);
102 b_matrices.push(None);
103 }
104 }
105
106 if !has_any_delta {
108 let results: Vec<Tensor> = x
109 .iter()
110 .map(|xi| {
111 let shape = xi.size();
112 let out_features = shape[shape.len() - 1]; Tensor::zeros(shape.as_slice(), (xi.kind(), xi.device())).narrow(
114 -1,
115 0,
116 out_features,
117 )
118 })
119 .collect();
120 return Ok(results);
121 }
122
123 let mut results = Vec::with_capacity(batch_size);
126
127 for i in 0..batch_size {
128 if let (Some(a), Some(b)) = (&a_matrices[i], &b_matrices[i]) {
129 let input_kind = x[i].kind();
138 let xi = x[i].to_kind(a.kind());
139 let intermediate = xi.matmul(&a.tr()); let output = intermediate.matmul(&b.tr()); results.push((output * self.scaling).to_kind(input_kind));
142 } else {
143 let shape = x[i].size();
145 results.push(Tensor::zeros(
146 shape.as_slice(),
147 (x[i].kind(), x[i].device()),
148 ));
149 }
150 }
151
152 Ok(results)
153 }
154
155 pub fn single_tenant_forward(
168 &self,
169 x: &Tensor,
170 module_name: &str,
171 tenant_id: &Subject,
172 layer_idx: usize,
173 ) -> Result<Tensor> {
174 let _guard = tch::no_grad_guard();
175
176 if let Some(delta_arc) = self.delta_pool.get(tenant_id) {
177 let delta = delta_arc.lock();
178 delta.forward(x, module_name, layer_idx)
179 } else {
180 Ok(Tensor::zeros(x.size().as_slice(), (x.kind(), x.device())))
182 }
183 }
184
185 pub fn delta_pool(&self) -> &Arc<DeltaPool> {
187 &self.delta_pool
188 }
189}
190
191#[cfg(test)]
192#[allow(clippy::expect_used, clippy::unwrap_used)]
193mod tests {
194 use super::*;
195 use crate::training::TenantDeltaConfig;
196 use hyprstream_rpc::Subject;
197 use std::collections::HashMap;
198 use tch::Kind;
199
200 fn create_test_pool() -> (Arc<DeltaPool>, Device) {
201 let device = Device::Cpu;
202 let mut module_dims = HashMap::new();
203 module_dims.insert("q_proj".to_owned(), (64, 64));
204 module_dims.insert("v_proj".to_owned(), (64, 64));
205
206 let config = TenantDeltaConfig {
207 rank: 4,
208 alpha: 2.0,
209 dropout: 0.0,
210 target_modules: vec!["q_proj".to_owned(), "v_proj".to_owned()],
211 learning_rate: 3e-4,
212 max_accumulated_steps: 300,
213 decay_lambda: 0.02,
214 muon_momentum: 0.95,
215 layer_overrides: None,
216 };
217
218 let pool = DeltaPool::new(
219 config,
220 module_dims,
221 device,
222 None,
223 std::env::temp_dir().join("batched_lora_test_snapshots"),
224 None,
225 2,
226 );
227 (Arc::new(pool), device)
228 }
229
230 #[test]
231 fn test_single_tenant_no_delta() {
232 let (pool, device) = create_test_pool();
233 let batched = BatchedLoRAForward::new(pool, 4, 0.5, device);
234
235 let x = Tensor::randn([1, 5, 64], (Kind::Float, device));
236 let tenant = Subject::new("missing");
237
238 let result = batched
239 .single_tenant_forward(&x, "q_proj", &tenant, 0)
240 .unwrap();
241 assert_eq!(result.size(), vec![1, 5, 64]);
242
243 let norm: f64 = result.norm().double_value(&[]);
245 assert!(norm < 1e-8, "Should be zero for missing tenant");
246 }
247
248 #[test]
249 fn test_single_tenant_with_delta() {
250 let (pool, device) = create_test_pool();
251
252 let tenant_a = Subject::new("tenant-a");
254 pool.get_or_create(&tenant_a).unwrap();
255
256 let batched = BatchedLoRAForward::new(pool, 4, 0.5, device);
257 let x = Tensor::randn([1, 5, 64], (Kind::Float, device));
258
259 let result = batched
260 .single_tenant_forward(&x, "q_proj", &tenant_a, 0)
261 .unwrap();
262 assert_eq!(result.size(), vec![1, 5, 64]);
263 }
264
265 #[test]
266 fn test_batched_forward_isolation() {
267 let (pool, device) = create_test_pool();
268
269 let tenant_a = Subject::new("tenant-a");
270 let tenant_b = Subject::new("tenant-b");
271 pool.get_or_create(&tenant_a).unwrap();
272 pool.get_or_create(&tenant_b).unwrap();
273
274 let batched = BatchedLoRAForward::new(pool.clone(), 4, 0.5, device);
275
276 let x1 = Tensor::randn([1, 5, 64], (Kind::Float, device));
277 let x2 = Tensor::randn([1, 5, 64], (Kind::Float, device));
278
279 let results = batched
280 .batched_lora_forward(
281 &[&x1, &x2],
282 "q_proj",
283 &[tenant_a.clone(), tenant_b.clone()],
284 0,
285 )
286 .unwrap();
287
288 assert_eq!(results.len(), 2);
289 assert_eq!(results[0].size(), vec![1, 5, 64]);
290 assert_eq!(results[1].size(), vec![1, 5, 64]);
291 }
292
293 #[test]
294 fn test_batched_matches_sequential() {
295 let (pool, device) = create_test_pool();
296
297 let tenant = Subject::new("tenant-test");
298 pool.get_or_create(&tenant).unwrap();
299
300 let batched = BatchedLoRAForward::new(pool.clone(), 4, 0.5, device);
301
302 let x = Tensor::randn([1, 3, 64], (Kind::Float, device));
303
304 let single_result = batched
306 .single_tenant_forward(&x, "q_proj", &tenant, 0)
307 .unwrap();
308
309 let batch_results = batched
311 .batched_lora_forward(&[&x], "q_proj", std::slice::from_ref(&tenant), 0)
312 .unwrap();
313
314 let diff: f64 = (&single_result - &batch_results[0])
316 .abs()
317 .sum(Kind::Float)
318 .double_value(&[]);
319 assert!(
320 diff < 1e-6,
321 "Batched should match sequential: diff={}",
322 diff
323 );
324 }
325}