Skip to main content

hyprstream_inference/runtime/
batched_lora.rs

1//! Batched multi-tenant LoRA forward pass
2//!
3//! Gathers per-tenant A/B matrices, stacks them into batched tensors,
4//! and uses bmm for efficient parallel LoRA computation. This avoids
5//! sequential per-tenant forward passes.
6
7use anyhow::{anyhow, Result};
8use std::sync::Arc;
9use tch::{Device, Tensor};
10
11use crate::training::DeltaPool;
12use hyprstream_rpc::Subject;
13
14/// Batched LoRA forward pass for multi-tenant inference.
15///
16/// Gathers A/B matrices from each tenant's delta, pads to a common rank,
17/// stacks into batch tensors, and performs bmm for efficient computation.
18pub struct BatchedLoRAForward {
19    /// Reference to the shared delta pool
20    delta_pool: Arc<DeltaPool>,
21    /// Maximum LoRA rank across all tenants (for padding)
22    #[allow(dead_code)]
23    max_rank: usize,
24    /// Scaling factor (alpha / rank)
25    #[allow(dead_code)]
26    scaling: f64,
27    /// Device for tensor allocation
28    #[allow(dead_code)]
29    device: Device,
30}
31
32impl BatchedLoRAForward {
33    /// Create a new batched LoRA forward module.
34    ///
35    /// # Arguments
36    /// * `delta_pool` - Shared delta pool containing per-tenant LoRA weights
37    /// * `max_rank` - Maximum LoRA rank to pad to (default: 8)
38    /// * `scaling` - LoRA scaling factor (alpha / rank)
39    /// * `device` - Device for tensor allocation
40    pub fn new(delta_pool: Arc<DeltaPool>, max_rank: usize, scaling: f64, device: Device) -> Self {
41        Self {
42            delta_pool,
43            max_rank,
44            scaling,
45            device,
46        }
47    }
48
49    /// Compute batched LoRA corrections for a single module across all provided tenants.
50    ///
51    /// # Arguments
52    /// * `x` - Input tensor per tenant: Vec of [1, seq_len, in_features] tensors
53    /// * `module_name` - Name of the target module (e.g., "q_proj")
54    /// * `tenant_ids` - Tenant IDs in the same order as `x`
55    /// * `layer_idx` - Layer index for per-layer delta lookup
56    ///
57    /// # Returns
58    /// Vec of LoRA correction tensors, one per tenant. Each is [1, seq_len, out_features].
59    /// Returns zero tensor for tenants without a delta.
60    pub fn batched_lora_forward(
61        &self,
62        x: &[&Tensor],
63        module_name: &str,
64        tenant_ids: &[Subject],
65        layer_idx: usize,
66    ) -> Result<Vec<Tensor>> {
67        if x.len() != tenant_ids.len() {
68            return Err(anyhow!(
69                "Input count ({}) doesn't match tenant count ({})",
70                x.len(),
71                tenant_ids.len()
72            ));
73        }
74
75        if x.is_empty() {
76            return Ok(Vec::new());
77        }
78
79        let batch_size = x.len();
80        let _guard = tch::no_grad_guard();
81
82        // Gather per-tenant A/B matrices
83        let mut a_matrices: Vec<Option<Tensor>> = Vec::with_capacity(batch_size);
84        let mut b_matrices: Vec<Option<Tensor>> = Vec::with_capacity(batch_size);
85        let mut has_any_delta = false;
86
87        let key = format!("{}.{}", layer_idx, module_name);
88        for tenant_id in tenant_ids {
89            if let Some(delta_arc) = self.delta_pool.get(tenant_id) {
90                let delta = delta_arc.lock();
91                if let (Some(a), Some(b)) = (delta.lora_a.get(&key), delta.lora_b.get(&key)) {
92                    // A: [rank, in_features], B: [out_features, rank]
93                    a_matrices.push(Some(a.shallow_clone()));
94                    b_matrices.push(Some(b.shallow_clone()));
95                    has_any_delta = true;
96                } else {
97                    a_matrices.push(None);
98                    b_matrices.push(None);
99                }
100            } else {
101                a_matrices.push(None);
102                b_matrices.push(None);
103            }
104        }
105
106        // Fast path: no deltas at all
107        if !has_any_delta {
108            let results: Vec<Tensor> = x
109                .iter()
110                .map(|xi| {
111                    let shape = xi.size();
112                    let out_features = shape[shape.len() - 1]; // same as in_features for q/v proj
113                    Tensor::zeros(shape.as_slice(), (xi.kind(), xi.device())).narrow(
114                        -1,
115                        0,
116                        out_features,
117                    )
118                })
119                .collect();
120            return Ok(results);
121        }
122
123        // Process each tenant individually (single-threaded per model)
124        // For batch_size=1 (most common case), this is just one iteration.
125        let mut results = Vec::with_capacity(batch_size);
126
127        for i in 0..batch_size {
128            if let (Some(a), Some(b)) = (&a_matrices[i], &b_matrices[i]) {
129                // x_i: [1, seq_len, in_features]
130                // A: [rank, in_features] -> A^T: [in_features, rank]
131                // B: [out_features, rank] -> B^T: [rank, out_features]
132                // output = scaling * (x_i @ A^T @ B^T)
133                // A/B are stored in fp32; cast x to A's dtype for the matmul (CPU addmm
134                // requires matching operand dtypes), then cast the correction back to x's
135                // dtype so callers can add it to a base-dtype (e.g. BF16) tensor. Both
136                // casts are device-agnostic no-ops when dtypes already match.
137                let input_kind = x[i].kind();
138                let xi = x[i].to_kind(a.kind());
139                let intermediate = xi.matmul(&a.tr()); // [1, seq_len, rank]
140                let output = intermediate.matmul(&b.tr()); // [1, seq_len, out_features]
141                results.push((output * self.scaling).to_kind(input_kind));
142            } else {
143                // No delta for this tenant — zero correction
144                let shape = x[i].size();
145                results.push(Tensor::zeros(
146                    shape.as_slice(),
147                    (x[i].kind(), x[i].device()),
148                ));
149            }
150        }
151
152        Ok(results)
153    }
154
155    /// Compute batched LoRA correction for a single tenant (most common case).
156    ///
157    /// Optimized path for batch_size=1 that avoids allocation overhead.
158    ///
159    /// # Arguments
160    /// * `x` - Input tensor [1, seq_len, in_features]
161    /// * `module_name` - Name of the target module
162    /// * `tenant_id` - The tenant ID
163    /// * `layer_idx` - Layer index for per-layer delta lookup
164    ///
165    /// # Returns
166    /// LoRA correction tensor [1, seq_len, out_features], or zero tensor if no delta.
167    pub fn single_tenant_forward(
168        &self,
169        x: &Tensor,
170        module_name: &str,
171        tenant_id: &Subject,
172        layer_idx: usize,
173    ) -> Result<Tensor> {
174        let _guard = tch::no_grad_guard();
175
176        if let Some(delta_arc) = self.delta_pool.get(tenant_id) {
177            let delta = delta_arc.lock();
178            delta.forward(x, module_name, layer_idx)
179        } else {
180            // No delta for this tenant — return zeros
181            Ok(Tensor::zeros(x.size().as_slice(), (x.kind(), x.device())))
182        }
183    }
184
185    /// Get the delta pool reference.
186    pub fn delta_pool(&self) -> &Arc<DeltaPool> {
187        &self.delta_pool
188    }
189}
190
191#[cfg(test)]
192#[allow(clippy::expect_used, clippy::unwrap_used)]
193mod tests {
194    use super::*;
195    use crate::training::TenantDeltaConfig;
196    use hyprstream_rpc::Subject;
197    use std::collections::HashMap;
198    use tch::Kind;
199
200    fn create_test_pool() -> (Arc<DeltaPool>, Device) {
201        let device = Device::Cpu;
202        let mut module_dims = HashMap::new();
203        module_dims.insert("q_proj".to_owned(), (64, 64));
204        module_dims.insert("v_proj".to_owned(), (64, 64));
205
206        let config = TenantDeltaConfig {
207            rank: 4,
208            alpha: 2.0,
209            dropout: 0.0,
210            target_modules: vec!["q_proj".to_owned(), "v_proj".to_owned()],
211            learning_rate: 3e-4,
212            max_accumulated_steps: 300,
213            decay_lambda: 0.02,
214            muon_momentum: 0.95,
215            layer_overrides: None,
216        };
217
218        let pool = DeltaPool::new(
219            config,
220            module_dims,
221            device,
222            None,
223            std::env::temp_dir().join("batched_lora_test_snapshots"),
224            None,
225            2,
226        );
227        (Arc::new(pool), device)
228    }
229
230    #[test]
231    fn test_single_tenant_no_delta() {
232        let (pool, device) = create_test_pool();
233        let batched = BatchedLoRAForward::new(pool, 4, 0.5, device);
234
235        let x = Tensor::randn([1, 5, 64], (Kind::Float, device));
236        let tenant = Subject::new("missing");
237
238        let result = batched
239            .single_tenant_forward(&x, "q_proj", &tenant, 0)
240            .unwrap();
241        assert_eq!(result.size(), vec![1, 5, 64]);
242
243        // Should be all zeros
244        let norm: f64 = result.norm().double_value(&[]);
245        assert!(norm < 1e-8, "Should be zero for missing tenant");
246    }
247
248    #[test]
249    fn test_single_tenant_with_delta() {
250        let (pool, device) = create_test_pool();
251
252        // Create a delta for tenant A
253        let tenant_a = Subject::new("tenant-a");
254        pool.get_or_create(&tenant_a).unwrap();
255
256        let batched = BatchedLoRAForward::new(pool, 4, 0.5, device);
257        let x = Tensor::randn([1, 5, 64], (Kind::Float, device));
258
259        let result = batched
260            .single_tenant_forward(&x, "q_proj", &tenant_a, 0)
261            .unwrap();
262        assert_eq!(result.size(), vec![1, 5, 64]);
263    }
264
265    #[test]
266    fn test_batched_forward_isolation() {
267        let (pool, device) = create_test_pool();
268
269        let tenant_a = Subject::new("tenant-a");
270        let tenant_b = Subject::new("tenant-b");
271        pool.get_or_create(&tenant_a).unwrap();
272        pool.get_or_create(&tenant_b).unwrap();
273
274        let batched = BatchedLoRAForward::new(pool.clone(), 4, 0.5, device);
275
276        let x1 = Tensor::randn([1, 5, 64], (Kind::Float, device));
277        let x2 = Tensor::randn([1, 5, 64], (Kind::Float, device));
278
279        let results = batched
280            .batched_lora_forward(
281                &[&x1, &x2],
282                "q_proj",
283                &[tenant_a.clone(), tenant_b.clone()],
284                0,
285            )
286            .unwrap();
287
288        assert_eq!(results.len(), 2);
289        assert_eq!(results[0].size(), vec![1, 5, 64]);
290        assert_eq!(results[1].size(), vec![1, 5, 64]);
291    }
292
293    #[test]
294    fn test_batched_matches_sequential() {
295        let (pool, device) = create_test_pool();
296
297        let tenant = Subject::new("tenant-test");
298        pool.get_or_create(&tenant).unwrap();
299
300        let batched = BatchedLoRAForward::new(pool.clone(), 4, 0.5, device);
301
302        let x = Tensor::randn([1, 3, 64], (Kind::Float, device));
303
304        // Single-tenant path
305        let single_result = batched
306            .single_tenant_forward(&x, "q_proj", &tenant, 0)
307            .unwrap();
308
309        // Batched path with batch_size=1
310        let batch_results = batched
311            .batched_lora_forward(&[&x], "q_proj", std::slice::from_ref(&tenant), 0)
312            .unwrap();
313
314        // Should be identical
315        let diff: f64 = (&single_result - &batch_results[0])
316            .abs()
317            .sum(Kind::Float)
318            .double_value(&[]);
319        assert!(
320            diff < 1e-6,
321            "Batched should match sequential: diff={}",
322            diff
323        );
324    }
325}