Skip to main content

hyprstream_inference/runtime/
weight_provider.rs

1//! Weight provider for streaming large models
2//!
3//! This module provides an abstraction for loading model weights on-demand,
4//! allowing models larger than system memory to be loaded.
5
6use anyhow::{anyhow, Result};
7use parking_lot::Mutex;
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11use tch::{Device, Kind as DType, Tensor};
12use tokio::sync::RwLock;
13use tracing::info;
14
15/// Trait for providing weights to models
16pub trait WeightProvider: Send + Sync {
17    /// Get a tensor by name, loading it if necessary
18    fn get_tensor(&self, name: &str) -> Result<Tensor>;
19
20    /// Check if a tensor exists
21    fn has_tensor(&self, name: &str) -> bool;
22
23    /// Get all tensor names (for config detection)
24    fn tensor_names(&self) -> Vec<String>;
25
26    /// Preload a set of tensors (for optimization)
27    fn preload(&mut self, names: &[&str]) -> Result<()>;
28}
29
30/// Simple in-memory weight provider (current behavior)
31/// Wraps tensors in Arc<Mutex<>> for thread safety
32pub struct MemoryWeightProvider {
33    weights: Arc<Mutex<HashMap<String, Tensor>>>,
34}
35
36impl MemoryWeightProvider {
37    pub fn new(weights: HashMap<String, Tensor>) -> Self {
38        Self {
39            weights: Arc::new(Mutex::new(weights)),
40        }
41    }
42}
43
44impl WeightProvider for MemoryWeightProvider {
45    fn get_tensor(&self, name: &str) -> Result<Tensor> {
46        let weights = self.weights.lock();
47        weights
48            .get(name)
49            .map(tch::Tensor::shallow_clone)
50            .ok_or_else(|| anyhow!("Tensor {} not found", name))
51    }
52
53    fn has_tensor(&self, name: &str) -> bool {
54        let weights = self.weights.lock();
55        weights.contains_key(name)
56    }
57
58    fn tensor_names(&self) -> Vec<String> {
59        let weights = self.weights.lock();
60        weights.keys().cloned().collect()
61    }
62
63    fn preload(&mut self, _names: &[&str]) -> Result<()> {
64        // Already in memory, nothing to do
65        Ok(())
66    }
67}
68
69/// Streaming weight provider that loads shards on demand
70///
71/// Note: We wrap non-thread-safe types to ensure Send + Sync
72pub struct StreamingWeightProvider {
73    shard_files: Vec<PathBuf>,
74    device: Device,
75    dtype: DType,
76    /// Currently loaded shard weights
77    current_weights: Arc<RwLock<HashMap<String, Tensor>>>,
78    /// Map from tensor name to shard index
79    tensor_shard_map: HashMap<String, usize>,
80    /// Currently loaded shard index
81    current_shard: Arc<RwLock<Option<usize>>>,
82}
83
84// Manually implement Send + Sync since we ensure thread-safe access
85unsafe impl Send for StreamingWeightProvider {}
86unsafe impl Sync for StreamingWeightProvider {}
87
88impl StreamingWeightProvider {
89    /// Create a new streaming weight provider
90    #[allow(clippy::arc_with_non_send_sync)] // parking_lot::RwLock is !Sync but we impl Send+Sync manually above
91    pub async fn new(shard_files: Vec<PathBuf>, device: Device, dtype: DType) -> Result<Self> {
92        // Build tensor -> shard mapping by reading headers
93        let mut tensor_shard_map = HashMap::new();
94
95        for (shard_idx, shard_path) in shard_files.iter().enumerate() {
96            // Read just the safetensors header to get tensor names
97            let header = Self::read_safetensors_header(shard_path).await?;
98            for tensor_name in header.tensor_names {
99                tensor_shard_map.insert(tensor_name, shard_idx);
100            }
101        }
102
103        Ok(Self {
104            shard_files,
105            device,
106            dtype,
107            current_weights: Arc::new(RwLock::new(HashMap::new())),
108            tensor_shard_map,
109            current_shard: Arc::new(RwLock::new(None)),
110        })
111    }
112
113    /// Read safetensors header without loading tensor data
114    async fn read_safetensors_header(path: &Path) -> Result<SafeTensorsHeader> {
115        use tokio::fs::File;
116        use tokio::io::AsyncReadExt;
117
118        let mut file = File::open(path).await?;
119
120        // Read header size (first 8 bytes)
121        let mut header_size_bytes = [0u8; 8];
122        file.read_exact(&mut header_size_bytes).await?;
123        let header_size = u64::from_le_bytes(header_size_bytes);
124
125        // Read header JSON
126        let mut header_bytes = vec![0u8; header_size as usize];
127        file.read_exact(&mut header_bytes).await?;
128
129        let header_json = String::from_utf8(header_bytes)?;
130        let header: serde_json::Value = serde_json::from_str(&header_json)?;
131
132        // Extract tensor names
133        let tensor_names: Vec<String> = header
134            .as_object()
135            .ok_or_else(|| anyhow!("Invalid safetensors header"))?
136            .keys()
137            .filter(|k| *k != "__metadata__")
138            .cloned()
139            .collect();
140
141        Ok(SafeTensorsHeader { tensor_names })
142    }
143
144    /// Load a specific shard
145    async fn load_shard(&self, shard_idx: usize) -> Result<HashMap<String, Tensor>> {
146        info!("Loading shard {}/{}", shard_idx + 1, self.shard_files.len());
147
148        let shard_path = &self.shard_files[shard_idx];
149        let mut weights = HashMap::new();
150
151        // Use existing load_safetensors_file logic
152        // This would call into model_factory::load_safetensors_file
153        // For now, simplified version:
154        let tensor_data = tokio::fs::read(shard_path).await?;
155        let tensors = safetensors::SafeTensors::deserialize(&tensor_data)?;
156
157        for (name, tensor_view) in tensors.tensors() {
158            // Create tensor (simplified - would use full logic from model_factory)
159            let shape: Vec<i64> = tensor_view.shape().iter().map(|&x| x as i64).collect();
160            let data = tensor_view.data();
161
162            // Support F16, BF16, and F32 models
163            let tensor = match tensor_view.dtype() {
164                safetensors::Dtype::BF16 => {
165                    let cpu_tensor = unsafe {
166                        Tensor::from_blob(
167                            data.as_ptr(),
168                            &shape,
169                            &[],
170                            tch::Kind::BFloat16,
171                            Device::Cpu,
172                        )
173                    };
174
175                    // Convert dtype if needed
176                    let cpu_tensor = if self.dtype == tch::Kind::Half {
177                        cpu_tensor.to_kind(tch::Kind::Half)
178                    } else if self.dtype != tch::Kind::BFloat16 {
179                        cpu_tensor.to_kind(self.dtype)
180                    } else {
181                        cpu_tensor
182                    };
183
184                    if self.device != Device::Cpu {
185                        let gpu_tensor = cpu_tensor.to_device(self.device);
186                        drop(cpu_tensor);
187                        gpu_tensor
188                    } else {
189                        cpu_tensor
190                    }
191                }
192                safetensors::Dtype::F16 => {
193                    let cpu_tensor = unsafe {
194                        Tensor::from_blob(data.as_ptr(), &shape, &[], tch::Kind::Half, Device::Cpu)
195                    };
196
197                    // Convert dtype if needed
198                    let cpu_tensor = if self.dtype == tch::Kind::BFloat16 {
199                        cpu_tensor.to_kind(tch::Kind::BFloat16)
200                    } else if self.dtype != tch::Kind::Half {
201                        cpu_tensor.to_kind(self.dtype)
202                    } else {
203                        cpu_tensor
204                    };
205
206                    if self.device != Device::Cpu {
207                        let gpu_tensor = cpu_tensor.to_device(self.device);
208                        drop(cpu_tensor);
209                        gpu_tensor
210                    } else {
211                        cpu_tensor
212                    }
213                }
214                safetensors::Dtype::F32 => {
215                    let cpu_tensor = unsafe {
216                        Tensor::from_blob(data.as_ptr(), &shape, &[], tch::Kind::Float, Device::Cpu)
217                    };
218
219                    // Convert to target dtype
220                    let cpu_tensor = match self.dtype {
221                        tch::Kind::Half => cpu_tensor.to_kind(tch::Kind::Half),
222                        tch::Kind::BFloat16 => cpu_tensor.to_kind(tch::Kind::BFloat16),
223                        tch::Kind::Float => cpu_tensor,
224                        _ => cpu_tensor.to_kind(self.dtype),
225                    };
226
227                    if self.device != Device::Cpu {
228                        let gpu_tensor = cpu_tensor.to_device(self.device);
229                        drop(cpu_tensor);
230                        gpu_tensor
231                    } else {
232                        cpu_tensor
233                    }
234                }
235                dtype => {
236                    return Err(anyhow::anyhow!(
237                        "Tensor '{}' has unsupported dtype {:?}. Supported: F16, BF16, F32",
238                        name,
239                        dtype
240                    ));
241                }
242            };
243
244            weights.insert(name.clone(), tensor);
245        }
246
247        Ok(weights)
248    }
249}
250
251impl WeightProvider for StreamingWeightProvider {
252    fn get_tensor(&self, name: &str) -> Result<Tensor> {
253        // Check if tensor is in current shard
254        let current_weights = self.current_weights.blocking_read();
255        if let Some(tensor) = current_weights.get(name) {
256            return Ok(tensor.shallow_clone());
257        }
258        drop(current_weights);
259
260        // Find which shard contains this tensor
261        let shard_idx = self
262            .tensor_shard_map
263            .get(name)
264            .ok_or_else(|| anyhow!("Tensor {} not found in any shard", name))?;
265
266        // Load the shard if it's not current
267        let mut current_shard = self.current_shard.blocking_write();
268        if current_shard.as_ref() != Some(shard_idx) {
269            // Load new shard (blocking for simplicity - could be async)
270            let new_weights =
271                tokio::runtime::Handle::current().block_on(self.load_shard(*shard_idx))?;
272
273            let mut current_weights = self.current_weights.blocking_write();
274            *current_weights = new_weights;
275            *current_shard = Some(*shard_idx);
276        }
277        drop(current_shard);
278
279        // Now get the tensor from loaded shard
280        let current_weights = self.current_weights.blocking_read();
281        current_weights
282            .get(name)
283            .map(tch::Tensor::shallow_clone)
284            .ok_or_else(|| anyhow!("Tensor {} not found after loading shard", name))
285    }
286
287    fn has_tensor(&self, name: &str) -> bool {
288        self.tensor_shard_map.contains_key(name)
289    }
290
291    fn tensor_names(&self) -> Vec<String> {
292        self.tensor_shard_map.keys().cloned().collect()
293    }
294
295    fn preload(&mut self, names: &[&str]) -> Result<()> {
296        // Find all shards needed for these tensors
297        let mut needed_shards: Vec<usize> = names
298            .iter()
299            .filter_map(|name| self.tensor_shard_map.get(*name))
300            .copied()
301            .collect();
302        needed_shards.sort_unstable();
303        needed_shards.dedup();
304
305        // For now, just load the first needed shard
306        if let Some(shard_idx) = needed_shards.first() {
307            let new_weights =
308                tokio::runtime::Handle::current().block_on(self.load_shard(*shard_idx))?;
309
310            let mut current_weights = self.current_weights.blocking_write();
311            *current_weights = new_weights;
312            let mut current_shard = self.current_shard.blocking_write();
313            *current_shard = Some(*shard_idx);
314        }
315
316        Ok(())
317    }
318}
319
320struct SafeTensorsHeader {
321    tensor_names: Vec<String>,
322}