hyprstream_inference/runtime/
weight_provider.rs1use anyhow::{anyhow, Result};
7use parking_lot::Mutex;
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11use tch::{Device, Kind as DType, Tensor};
12use tokio::sync::RwLock;
13use tracing::info;
14
15pub trait WeightProvider: Send + Sync {
17 fn get_tensor(&self, name: &str) -> Result<Tensor>;
19
20 fn has_tensor(&self, name: &str) -> bool;
22
23 fn tensor_names(&self) -> Vec<String>;
25
26 fn preload(&mut self, names: &[&str]) -> Result<()>;
28}
29
30pub struct MemoryWeightProvider {
33 weights: Arc<Mutex<HashMap<String, Tensor>>>,
34}
35
36impl MemoryWeightProvider {
37 pub fn new(weights: HashMap<String, Tensor>) -> Self {
38 Self {
39 weights: Arc::new(Mutex::new(weights)),
40 }
41 }
42}
43
44impl WeightProvider for MemoryWeightProvider {
45 fn get_tensor(&self, name: &str) -> Result<Tensor> {
46 let weights = self.weights.lock();
47 weights
48 .get(name)
49 .map(tch::Tensor::shallow_clone)
50 .ok_or_else(|| anyhow!("Tensor {} not found", name))
51 }
52
53 fn has_tensor(&self, name: &str) -> bool {
54 let weights = self.weights.lock();
55 weights.contains_key(name)
56 }
57
58 fn tensor_names(&self) -> Vec<String> {
59 let weights = self.weights.lock();
60 weights.keys().cloned().collect()
61 }
62
63 fn preload(&mut self, _names: &[&str]) -> Result<()> {
64 Ok(())
66 }
67}
68
69pub struct StreamingWeightProvider {
73 shard_files: Vec<PathBuf>,
74 device: Device,
75 dtype: DType,
76 current_weights: Arc<RwLock<HashMap<String, Tensor>>>,
78 tensor_shard_map: HashMap<String, usize>,
80 current_shard: Arc<RwLock<Option<usize>>>,
82}
83
84unsafe impl Send for StreamingWeightProvider {}
86unsafe impl Sync for StreamingWeightProvider {}
87
88impl StreamingWeightProvider {
89 #[allow(clippy::arc_with_non_send_sync)] pub async fn new(shard_files: Vec<PathBuf>, device: Device, dtype: DType) -> Result<Self> {
92 let mut tensor_shard_map = HashMap::new();
94
95 for (shard_idx, shard_path) in shard_files.iter().enumerate() {
96 let header = Self::read_safetensors_header(shard_path).await?;
98 for tensor_name in header.tensor_names {
99 tensor_shard_map.insert(tensor_name, shard_idx);
100 }
101 }
102
103 Ok(Self {
104 shard_files,
105 device,
106 dtype,
107 current_weights: Arc::new(RwLock::new(HashMap::new())),
108 tensor_shard_map,
109 current_shard: Arc::new(RwLock::new(None)),
110 })
111 }
112
113 async fn read_safetensors_header(path: &Path) -> Result<SafeTensorsHeader> {
115 use tokio::fs::File;
116 use tokio::io::AsyncReadExt;
117
118 let mut file = File::open(path).await?;
119
120 let mut header_size_bytes = [0u8; 8];
122 file.read_exact(&mut header_size_bytes).await?;
123 let header_size = u64::from_le_bytes(header_size_bytes);
124
125 let mut header_bytes = vec![0u8; header_size as usize];
127 file.read_exact(&mut header_bytes).await?;
128
129 let header_json = String::from_utf8(header_bytes)?;
130 let header: serde_json::Value = serde_json::from_str(&header_json)?;
131
132 let tensor_names: Vec<String> = header
134 .as_object()
135 .ok_or_else(|| anyhow!("Invalid safetensors header"))?
136 .keys()
137 .filter(|k| *k != "__metadata__")
138 .cloned()
139 .collect();
140
141 Ok(SafeTensorsHeader { tensor_names })
142 }
143
144 async fn load_shard(&self, shard_idx: usize) -> Result<HashMap<String, Tensor>> {
146 info!("Loading shard {}/{}", shard_idx + 1, self.shard_files.len());
147
148 let shard_path = &self.shard_files[shard_idx];
149 let mut weights = HashMap::new();
150
151 let tensor_data = tokio::fs::read(shard_path).await?;
155 let tensors = safetensors::SafeTensors::deserialize(&tensor_data)?;
156
157 for (name, tensor_view) in tensors.tensors() {
158 let shape: Vec<i64> = tensor_view.shape().iter().map(|&x| x as i64).collect();
160 let data = tensor_view.data();
161
162 let tensor = match tensor_view.dtype() {
164 safetensors::Dtype::BF16 => {
165 let cpu_tensor = unsafe {
166 Tensor::from_blob(
167 data.as_ptr(),
168 &shape,
169 &[],
170 tch::Kind::BFloat16,
171 Device::Cpu,
172 )
173 };
174
175 let cpu_tensor = if self.dtype == tch::Kind::Half {
177 cpu_tensor.to_kind(tch::Kind::Half)
178 } else if self.dtype != tch::Kind::BFloat16 {
179 cpu_tensor.to_kind(self.dtype)
180 } else {
181 cpu_tensor
182 };
183
184 if self.device != Device::Cpu {
185 let gpu_tensor = cpu_tensor.to_device(self.device);
186 drop(cpu_tensor);
187 gpu_tensor
188 } else {
189 cpu_tensor
190 }
191 }
192 safetensors::Dtype::F16 => {
193 let cpu_tensor = unsafe {
194 Tensor::from_blob(data.as_ptr(), &shape, &[], tch::Kind::Half, Device::Cpu)
195 };
196
197 let cpu_tensor = if self.dtype == tch::Kind::BFloat16 {
199 cpu_tensor.to_kind(tch::Kind::BFloat16)
200 } else if self.dtype != tch::Kind::Half {
201 cpu_tensor.to_kind(self.dtype)
202 } else {
203 cpu_tensor
204 };
205
206 if self.device != Device::Cpu {
207 let gpu_tensor = cpu_tensor.to_device(self.device);
208 drop(cpu_tensor);
209 gpu_tensor
210 } else {
211 cpu_tensor
212 }
213 }
214 safetensors::Dtype::F32 => {
215 let cpu_tensor = unsafe {
216 Tensor::from_blob(data.as_ptr(), &shape, &[], tch::Kind::Float, Device::Cpu)
217 };
218
219 let cpu_tensor = match self.dtype {
221 tch::Kind::Half => cpu_tensor.to_kind(tch::Kind::Half),
222 tch::Kind::BFloat16 => cpu_tensor.to_kind(tch::Kind::BFloat16),
223 tch::Kind::Float => cpu_tensor,
224 _ => cpu_tensor.to_kind(self.dtype),
225 };
226
227 if self.device != Device::Cpu {
228 let gpu_tensor = cpu_tensor.to_device(self.device);
229 drop(cpu_tensor);
230 gpu_tensor
231 } else {
232 cpu_tensor
233 }
234 }
235 dtype => {
236 return Err(anyhow::anyhow!(
237 "Tensor '{}' has unsupported dtype {:?}. Supported: F16, BF16, F32",
238 name,
239 dtype
240 ));
241 }
242 };
243
244 weights.insert(name.clone(), tensor);
245 }
246
247 Ok(weights)
248 }
249}
250
251impl WeightProvider for StreamingWeightProvider {
252 fn get_tensor(&self, name: &str) -> Result<Tensor> {
253 let current_weights = self.current_weights.blocking_read();
255 if let Some(tensor) = current_weights.get(name) {
256 return Ok(tensor.shallow_clone());
257 }
258 drop(current_weights);
259
260 let shard_idx = self
262 .tensor_shard_map
263 .get(name)
264 .ok_or_else(|| anyhow!("Tensor {} not found in any shard", name))?;
265
266 let mut current_shard = self.current_shard.blocking_write();
268 if current_shard.as_ref() != Some(shard_idx) {
269 let new_weights =
271 tokio::runtime::Handle::current().block_on(self.load_shard(*shard_idx))?;
272
273 let mut current_weights = self.current_weights.blocking_write();
274 *current_weights = new_weights;
275 *current_shard = Some(*shard_idx);
276 }
277 drop(current_shard);
278
279 let current_weights = self.current_weights.blocking_read();
281 current_weights
282 .get(name)
283 .map(tch::Tensor::shallow_clone)
284 .ok_or_else(|| anyhow!("Tensor {} not found after loading shard", name))
285 }
286
287 fn has_tensor(&self, name: &str) -> bool {
288 self.tensor_shard_map.contains_key(name)
289 }
290
291 fn tensor_names(&self) -> Vec<String> {
292 self.tensor_shard_map.keys().cloned().collect()
293 }
294
295 fn preload(&mut self, names: &[&str]) -> Result<()> {
296 let mut needed_shards: Vec<usize> = names
298 .iter()
299 .filter_map(|name| self.tensor_shard_map.get(*name))
300 .copied()
301 .collect();
302 needed_shards.sort_unstable();
303 needed_shards.dedup();
304
305 if let Some(shard_idx) = needed_shards.first() {
307 let new_weights =
308 tokio::runtime::Handle::current().block_on(self.load_shard(*shard_idx))?;
309
310 let mut current_weights = self.current_weights.blocking_write();
311 *current_weights = new_weights;
312 let mut current_shard = self.current_shard.blocking_write();
313 *current_shard = Some(*shard_idx);
314 }
315
316 Ok(())
317 }
318}
319
320struct SafeTensorsHeader {
321 tensor_names: Vec<String>,
322}