1use anyhow::{anyhow, Result};
37use dashmap::DashMap;
38use parking_lot::Mutex;
39use std::collections::HashMap;
40use std::path::PathBuf;
41use std::sync::Arc;
42use tch::{Device, Tensor};
43
44use super::muon::MuonState;
45use super::tenant_delta::{
46 load_state_dict_from_bytes, serialize_state_dict_to_bytes, TenantDelta, TenantDeltaConfig,
47};
48use crate::runtime::kv_cache::KVCacheRegistry;
49use crate::worktree_ext::WorktreeClientExt;
50use hyprstream_rpc::Subject;
51use hyprstream_rpc_std::registry_client::WorktreeClient;
52
53const MAX_SNAPSHOT_BYTES: usize = 512 * 1024 * 1024;
55
56const MAX_TENANTS_DEFAULT: usize = 100;
58
59const META_ACCUMULATED_STEPS: &str = "__meta.accumulated_steps";
68const META_REQUEST_COUNT: &str = "__meta.request_count";
70
71pub struct DeltaPool {
76 deltas: DashMap<Subject, Arc<Mutex<TenantDelta>>>,
78 default_config: Mutex<TenantDeltaConfig>,
80 module_dims: HashMap<String, (usize, usize)>,
82 per_layer_dims: Option<HashMap<usize, HashMap<String, (usize, usize)>>>,
86 device: Device,
88 memory_budget_bytes: Option<usize>,
90 base_weight_norms: HashMap<String, f64>,
92 kv_registry: Option<Arc<KVCacheRegistry>>,
95 snapshots_dir: PathBuf,
97 fs: Option<WorktreeClient>,
99 num_layers: usize,
101 max_tenants: usize,
103 rank_oracle_config: Option<super::ttt::RankOracleConfig>,
105}
106
107impl DeltaPool {
108 pub fn new(
118 config: TenantDeltaConfig,
119 module_dims: HashMap<String, (usize, usize)>,
120 device: Device,
121 kv_registry: Option<Arc<KVCacheRegistry>>,
122 snapshots_dir: PathBuf,
123 fs: Option<WorktreeClient>,
124 num_layers: usize,
125 ) -> Self {
126 Self {
127 deltas: DashMap::new(),
128 default_config: Mutex::new(config),
129 module_dims,
130 per_layer_dims: None,
131 device,
132 memory_budget_bytes: None,
133 base_weight_norms: HashMap::new(),
134 kv_registry,
135 snapshots_dir,
136 fs,
137 num_layers,
138 max_tenants: MAX_TENANTS_DEFAULT,
139 rank_oracle_config: None,
140 }
141 }
142
143 pub fn with_rank_oracle(mut self, config: super::ttt::RankOracleConfig) -> Self {
145 self.rank_oracle_config = Some(config);
146 self
147 }
148
149 pub fn with_per_layer_dims(
151 mut self,
152 per_layer_dims: HashMap<usize, HashMap<String, (usize, usize)>>,
153 ) -> Self {
154 self.per_layer_dims = Some(per_layer_dims);
155 self
156 }
157
158 pub fn with_memory_budget(mut self, budget_bytes: usize) -> Self {
160 self.memory_budget_bytes = Some(budget_bytes);
161 self
162 }
163
164 pub fn with_max_tenants(mut self, max_tenants: usize) -> Self {
166 self.max_tenants = max_tenants;
167 self
168 }
169
170 pub fn update_config(&self, config: TenantDeltaConfig) {
174 self.clear_all();
176 *self.default_config.lock() = config;
177 }
178
179 pub fn set_base_weight_norms(&mut self, norms: HashMap<String, f64>) {
181 self.base_weight_norms = norms;
182 }
183
184 pub fn set_kv_registry(&mut self, registry: Arc<KVCacheRegistry>) {
186 self.kv_registry = Some(registry);
187 }
188
189 fn create_blank_delta(&self) -> Result<TenantDelta> {
195 let config = self.default_config.lock().clone();
196 let mut delta = TenantDelta::new_with_per_layer_dims(
197 &config,
198 &self.module_dims,
199 self.device,
200 self.num_layers,
201 self.per_layer_dims.as_ref(),
202 )?;
203 if let Some(ref oracle_config) = self.rank_oracle_config {
205 delta.rank_oracle = Some(super::ttt::RankOracle::new(oracle_config.clone()));
206 }
207 Ok(delta)
208 }
209
210 pub fn get_or_create(&self, tenant_id: &Subject) -> Result<Arc<Mutex<TenantDelta>>> {
218 if let Some(delta) = self.deltas.get(tenant_id) {
220 delta.lock().touch();
221 return Ok(delta.clone());
222 }
223
224 let delta = Arc::new(Mutex::new(self.create_blank_delta()?));
225
226 Ok(self
228 .deltas
229 .entry(tenant_id.clone())
230 .or_insert(delta.clone())
231 .clone())
232 }
233
234 pub async fn get_or_hydrate(&self, tenant_id: &Subject) -> Result<Arc<Mutex<TenantDelta>>> {
252 if let Some(delta) = self.deltas.get(tenant_id) {
254 delta.lock().touch();
255 return Ok(delta.clone());
256 }
257
258 let mut delta = self.create_blank_delta()?;
259 let id_filename = Self::sanitize_filename(&tenant_id.to_string());
260
261 match self.read_snapshot_bytes(&id_filename).await {
262 Ok(Some(bytes)) => {
263 let loaded = load_state_dict_from_bytes(&bytes)
264 .and_then(|state| Self::apply_snapshot(&mut delta, &state));
265 match loaded {
266 Ok(()) => {
267 tracing::info!(
268 "Rehydrated delta '{}' from snapshot ({} accumulated steps)",
269 tenant_id,
270 delta.accumulated_steps
271 );
272 }
273 Err(e) => {
274 tracing::warn!(
275 "Corrupt/incompatible snapshot for delta '{}': {} — quarantining and starting blank",
276 tenant_id,
277 e
278 );
279 self.quarantine_snapshot(&id_filename).await;
280 delta = self.create_blank_delta()?;
283 }
284 }
285 }
286 Ok(None) => {
287 }
289 Err(e) => {
290 tracing::warn!(
291 "Failed to read snapshot for delta '{}': {} — starting blank",
292 tenant_id,
293 e
294 );
295 }
296 }
297
298 let delta = Arc::new(Mutex::new(delta));
299 Ok(self
300 .deltas
301 .entry(tenant_id.clone())
302 .or_insert(delta.clone())
303 .clone())
304 }
305
306 fn snapshot_state_dict(delta: &TenantDelta) -> HashMap<String, Tensor> {
310 let mut state = delta.extract_state_dict();
311 state.insert(
312 META_ACCUMULATED_STEPS.to_owned(),
313 Tensor::from_slice(&[delta.accumulated_steps as i64]),
314 );
315 state.insert(
316 META_REQUEST_COUNT.to_owned(),
317 Tensor::from_slice(&[delta.request_count as i64]),
318 );
319 state
320 }
321
322 fn apply_snapshot(delta: &mut TenantDelta, state: &HashMap<String, Tensor>) -> Result<()> {
331 for key in delta.lora_a.keys() {
333 if let Some(src) = state.get(&format!("{}.lora_a", key)) {
334 let want = delta.lora_a[key].size();
335 if src.size() != want {
336 return Err(anyhow!(
337 "snapshot lora_a '{}' shape {:?} != expected {:?}",
338 key,
339 src.size(),
340 want
341 ));
342 }
343 }
344 }
345 for key in delta.lora_b.keys() {
346 if let Some(src) = state.get(&format!("{}.lora_b", key)) {
347 let want = delta.lora_b[key].size();
348 if src.size() != want {
349 return Err(anyhow!(
350 "snapshot lora_b '{}' shape {:?} != expected {:?}",
351 key,
352 src.size(),
353 want
354 ));
355 }
356 }
357 }
358
359 let device = delta.device;
361 for (name, t) in state.iter() {
362 if let Some(key) = name.strip_suffix(".__muon_momentum") {
363 delta
364 .muon_states
365 .entry(key.to_owned())
366 .or_insert_with(|| MuonState {
367 momentum_buffer: Some(t.to_device(device)),
368 });
369 }
370 }
371
372 delta.load_state_dict(state)?;
373
374 if let Some(t) = state.get(META_ACCUMULATED_STEPS) {
375 delta.accumulated_steps = t.int64_value(&[]).max(0) as u64;
376 }
377 if let Some(t) = state.get(META_REQUEST_COUNT) {
378 delta.request_count = t.int64_value(&[]).max(0) as u64;
379 }
380 Ok(())
381 }
382
383 async fn read_snapshot_bytes(&self, id_filename: &str) -> Result<Option<Vec<u8>>> {
389 let rel_path = format!("adapters/.snapshots/{}.safetensors", id_filename);
390 if let Some(ref fs) = self.fs {
391 let st = fs
392 .stat_path(&rel_path)
393 .await
394 .map_err(|e| anyhow!("FsOps stat failed: {}", e))?;
395 if !st.exists {
396 return Ok(None);
397 }
398 let bytes = fs
399 .read_file_chunked(&rel_path)
400 .await
401 .map_err(|e| anyhow!("FsOps read failed: {}", e))?;
402 Ok(Some(bytes))
403 } else {
404 let path = self
405 .snapshots_dir
406 .join(format!("{}.safetensors", id_filename));
407 match std::fs::read(&path) {
408 Ok(b) => Ok(Some(b)),
409 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
410 Err(e) => Err(anyhow!("read snapshot {}: {}", path.display(), e)),
411 }
412 }
413 }
414
415 async fn quarantine_snapshot(&self, id_filename: &str) {
419 let rel_path = format!("adapters/.snapshots/{}.safetensors", id_filename);
420 let corrupt_rel = format!("{}.corrupt", rel_path);
421 if let Some(ref fs) = self.fs {
422 match fs.read_file_chunked(&rel_path).await {
424 Ok(bytes) => {
425 if let Err(e) = fs.write_file_chunked(&corrupt_rel, &bytes).await {
426 tracing::warn!(
427 "Failed to write quarantined snapshot {}: {}",
428 corrupt_rel,
429 e
430 );
431 }
432 }
433 Err(e) => {
434 tracing::warn!("Failed to read snapshot for quarantine {}: {}", rel_path, e);
435 }
436 }
437 if let Err(e) = fs.remove_path(&rel_path).await {
438 tracing::warn!("Failed to remove corrupt snapshot {}: {}", rel_path, e);
439 }
440 } else {
441 let path = self
442 .snapshots_dir
443 .join(format!("{}.safetensors", id_filename));
444 let corrupt = self
445 .snapshots_dir
446 .join(format!("{}.safetensors.corrupt", id_filename));
447 if let Err(e) = std::fs::rename(&path, &corrupt) {
448 tracing::warn!(
449 "Failed to quarantine corrupt snapshot {}: {}",
450 path.display(),
451 e
452 );
453 }
454 }
455 tracing::warn!(
456 "Quarantined corrupt snapshot for '{}' -> {}",
457 id_filename,
458 corrupt_rel
459 );
460 }
461
462 pub fn get(&self, tenant_id: &Subject) -> Option<Arc<Mutex<TenantDelta>>> {
464 self.deltas.get(tenant_id).map(|d| {
465 d.lock().touch();
466 d.clone()
467 })
468 }
469
470 pub fn remove(&self, subject: &Subject) -> Option<Arc<Mutex<TenantDelta>>> {
472 let result = self.deltas.remove(subject).map(|(_, d)| d);
473 if result.is_some() {
474 if let Some(kv_reg) = &self.kv_registry {
475 kv_reg.invalidate_for_tenant(&subject.to_string());
476 }
477 }
478 result
479 }
480
481 pub fn list_subjects(&self) -> Vec<String> {
483 self.deltas.iter().map(|e| e.key().to_string()).collect()
484 }
485
486 pub fn tenant_count(&self) -> usize {
488 self.deltas.len()
489 }
490
491 pub fn total_memory_usage(&self) -> usize {
493 self.deltas
494 .iter()
495 .map(|entry| entry.value().lock().memory_bytes())
496 .sum()
497 }
498
499 pub fn default_config(&self) -> TenantDeltaConfig {
501 self.default_config.lock().clone()
502 }
503
504 pub fn rank(&self) -> usize {
506 self.default_config.lock().rank
507 }
508
509 pub fn module_dims(&self) -> &HashMap<String, (usize, usize)> {
511 &self.module_dims
512 }
513
514 pub fn base_weight_norms(&self) -> &HashMap<String, f64> {
516 &self.base_weight_norms
517 }
518
519 pub fn snapshots_dir(&self) -> &PathBuf {
521 &self.snapshots_dir
522 }
523
524 pub fn needs_eviction(&self) -> bool {
528 if self.deltas.len() >= self.max_tenants {
529 return true;
530 }
531 if let Some(budget) = self.memory_budget_bytes {
532 if self.total_memory_usage() > budget {
533 return true;
534 }
535 }
536 false
537 }
538
539 pub async fn ensure_capacity(&self) -> Result<()> {
546 while self.needs_eviction() {
547 if self.evict_lru_async().await?.is_none() {
548 break; }
550 }
551 Ok(())
552 }
553
554 fn sanitize_filename(subject: &str) -> String {
556 subject
557 .replace(['/', '\\', '\0'], "_")
558 .replace("..", "_")
559 .chars()
560 .take(200)
561 .collect()
562 }
563
564 fn find_lru_tenant(&self) -> Option<Subject> {
568 let mut oldest_tenant: Option<Subject> = None;
569 let mut oldest_time = std::time::Instant::now();
570
571 for entry in self.deltas.iter() {
572 let delta = entry.value().lock();
573 if delta.last_access < oldest_time {
574 oldest_time = delta.last_access;
575 oldest_tenant = Some(entry.key().clone());
576 }
577 }
578 oldest_tenant
579 }
580
581 pub async fn evict_lru_async(
589 &self,
590 ) -> Result<Option<(Subject, Arc<Mutex<TenantDelta>>, Option<PathBuf>)>> {
591 if self.deltas.is_empty() {
592 return Ok(None);
593 }
594
595 let Some(tid) = self.find_lru_tenant() else {
596 return Ok(None);
597 };
598
599 let Some((id, delta)) = self.deltas.remove(&tid) else {
600 return Ok(None);
601 };
602
603 let id_filename = Self::sanitize_filename(&id.to_string());
604
605 let snapshot_data = {
608 let d = delta.lock();
609 if d.accumulated_steps > 0 {
610 if self.fs.is_some() {
611 let state_dict = Self::snapshot_state_dict(&d);
612 let memory_bytes = d.memory_bytes();
613 Some((state_dict, memory_bytes))
614 } else {
615 tracing::warn!(
616 "FsOps not available — skipping snapshot for delta '{}' during eviction",
617 id
618 );
619 None
620 }
621 } else {
622 None }
624 }; let snapshot_path = if let Some((state_dict, memory_bytes)) = snapshot_data {
627 let snapshot_file = self
628 .snapshots_dir
629 .join(format!("{}.safetensors", id_filename));
630 let Some(ref fs) = self.fs else {
632 unreachable!()
633 };
634 let rel_path = format!("adapters/.snapshots/{}.safetensors", id_filename);
635
636 if memory_bytes > MAX_SNAPSHOT_BYTES {
637 tracing::warn!(
638 "Delta '{}' exceeds max snapshot size ({} bytes > {}), skipping snapshot",
639 id,
640 memory_bytes,
641 MAX_SNAPSHOT_BYTES
642 );
643 None
644 } else {
645 match async {
646 fs.mkdir_p("adapters/.snapshots")
647 .await
648 .map_err(|e| anyhow::anyhow!("FsOps mkdir failed: {}", e))?;
649 let bytes = serialize_state_dict_to_bytes(&state_dict)?;
650 fs.write_file_chunked(&rel_path, &bytes)
651 .await
652 .map_err(|e| anyhow::anyhow!("FsOps write_file failed: {}", e))?;
653 Ok::<(), anyhow::Error>(())
654 }
655 .await
656 {
657 Ok(()) => {
658 tracing::info!(
659 "Auto-snapshot delta '{}' before eviction via FsOps: {}",
660 id,
661 rel_path
662 );
663 Some(snapshot_file)
664 }
665 Err(e) => {
666 tracing::warn!("Failed to snapshot delta '{}' via FsOps: {}", id, e);
667 None
668 }
669 }
670 }
671 } else {
672 None
673 };
674
675 if let Some(kv_reg) = &self.kv_registry {
677 let invalidated = kv_reg.invalidate_for_tenant(&id.to_string());
678 if invalidated > 0 {
679 tracing::info!(
680 "Invalidated {} KV caches on eviction of delta '{}'",
681 invalidated,
682 id
683 );
684 }
685 }
686
687 Ok(Some((id, delta, snapshot_path)))
688 }
689
690 pub fn evict_lru(&self) -> Option<(Subject, Arc<Mutex<TenantDelta>>, Option<PathBuf>)> {
696 if self.deltas.is_empty() {
699 return None;
700 }
701
702 let tid = self.find_lru_tenant()?;
703 let (id, delta) = self.deltas.remove(&tid)?;
704
705 let id_filename = Self::sanitize_filename(&id.to_string());
706
707 let snapshot_path =
709 {
710 let d = delta.lock();
711 if d.accumulated_steps > 0 {
712 let snapshot_file = self
713 .snapshots_dir
714 .join(format!("{}.safetensors", id_filename));
715 let state_dict = Self::snapshot_state_dict(&d);
718 let memory_bytes = d.memory_bytes();
719 drop(d);
720
721 if memory_bytes > MAX_SNAPSHOT_BYTES {
722 tracing::warn!(
723 "Delta '{}' exceeds max snapshot size ({} bytes > {}), skipping snapshot",
724 id, memory_bytes, MAX_SNAPSHOT_BYTES
725 );
726 None
727 } else {
728 match serialize_state_dict_to_bytes(&state_dict) {
729 Ok(bytes) => {
730 if let Err(e) = std::fs::create_dir_all(&self.snapshots_dir) {
731 tracing::warn!("Failed to create snapshots dir: {}", e);
732 None
733 } else if let Err(e) = std::fs::write(&snapshot_file, &bytes) {
734 tracing::warn!(
735 "Failed to write snapshot for delta '{}': {}",
736 id,
737 e
738 );
739 None
740 } else {
741 tracing::info!(
742 "Auto-snapshot delta '{}' before eviction: {}",
743 id,
744 snapshot_file.display()
745 );
746 Some(snapshot_file)
747 }
748 }
749 Err(e) => {
750 tracing::warn!(
751 "Failed to serialize delta '{}' for snapshot: {}",
752 id,
753 e
754 );
755 None
756 }
757 }
758 }
759 } else {
760 None
761 }
762 };
763
764 if let Some(kv_reg) = &self.kv_registry {
766 let invalidated = kv_reg.invalidate_for_tenant(&id.to_string());
767 if invalidated > 0 {
768 tracing::info!(
769 "Invalidated {} KV caches on eviction of delta '{}'",
770 invalidated,
771 id
772 );
773 }
774 }
775
776 Some((id, delta, snapshot_path))
777 }
778
779 pub async fn evict_to_budget_async(&self) -> Result<Vec<(Subject, Option<PathBuf>)>> {
783 let budget = match self.memory_budget_bytes {
784 Some(b) => b,
785 None => return Ok(Vec::new()),
786 };
787
788 let mut evicted = Vec::new();
789 while self.total_memory_usage() > budget {
790 match self.evict_lru_async().await? {
791 Some((tid, _, path)) => evicted.push((tid, path)),
792 None => break,
793 }
794 }
795 Ok(evicted)
796 }
797
798 pub fn evict_to_budget(&self) -> Vec<(Subject, Option<PathBuf>)> {
802 let budget = match self.memory_budget_bytes {
803 Some(b) => b,
804 None => return Vec::new(),
805 };
806
807 let mut evicted = Vec::new();
808 while self.total_memory_usage() > budget {
809 match self.evict_lru() {
810 Some((tid, _, path)) => evicted.push((tid, path)),
811 None => break,
812 }
813 }
814 evicted
815 }
816
817 pub async fn export_all(&self) -> Result<Vec<(Subject, Option<PathBuf>)>> {
836 let Some(ref fs) = self.fs else {
837 tracing::warn!(
838 "export_all: FsOps not available — {} resident delta(s) NOT persisted (loss on shutdown)",
839 self.deltas.len()
840 );
841 return Ok(Vec::new());
842 };
843
844 if self.deltas.is_empty() {
845 return Ok(Vec::new());
846 }
847
848 fs.mkdir_p("adapters/.snapshots")
849 .await
850 .map_err(|e| anyhow!("FsOps mkdir failed: {}", e))?;
851
852 let ids: Vec<Subject> = self.deltas.iter().map(|e| e.key().clone()).collect();
854 let mut results = Vec::with_capacity(ids.len());
855
856 for id in ids {
857 let Some(entry) = self.deltas.get(&id) else {
858 continue; };
860 let delta = entry.value().clone();
861 drop(entry);
862
863 let snapshot_data = {
865 let d = delta.lock();
866 if d.accumulated_steps == 0 {
867 None } else {
869 let memory_bytes = d.memory_bytes();
870 if memory_bytes > MAX_SNAPSHOT_BYTES {
871 tracing::warn!(
872 "export_all: delta '{}' exceeds max snapshot size ({} > {}), skipping (loss)",
873 id, memory_bytes, MAX_SNAPSHOT_BYTES
874 );
875 None
876 } else {
877 Some(Self::snapshot_state_dict(&d))
878 }
879 }
880 };
881
882 let path = if let Some(state_dict) = snapshot_data {
883 let id_filename = Self::sanitize_filename(&id.to_string());
884 let rel_path = format!("adapters/.snapshots/{}.safetensors", id_filename);
885 match async {
886 let bytes = serialize_state_dict_to_bytes(&state_dict)?;
887 fs.write_file_chunked(&rel_path, &bytes)
888 .await
889 .map_err(|e| anyhow!("FsOps write_file failed: {}", e))?;
890 Ok::<(), anyhow::Error>(())
891 }
892 .await
893 {
894 Ok(()) => {
895 tracing::info!("export_all: snapshotted delta '{}' -> {}", id, rel_path);
896 Some(
897 self.snapshots_dir
898 .join(format!("{}.safetensors", id_filename)),
899 )
900 }
901 Err(e) => {
902 tracing::warn!(
903 "export_all: failed to snapshot delta '{}': {} (loss)",
904 id,
905 e
906 );
907 None
908 }
909 }
910 } else {
911 None
912 };
913
914 results.push((id, path));
915 }
916
917 Ok(results)
918 }
919
920 pub fn clear_all(&self) {
922 if let Some(kv_reg) = &self.kv_registry {
924 for entry in self.deltas.iter() {
925 kv_reg.invalidate_for_tenant(&entry.key().to_string());
926 }
927 }
928 self.deltas.clear();
929 }
930
931 pub fn tenant_ids(&self) -> Vec<Subject> {
933 self.deltas.iter().map(|e| e.key().clone()).collect()
934 }
935}
936
937#[cfg(test)]
938#[allow(clippy::expect_used, clippy::unwrap_used)]
939mod tests {
940 use super::*;
941
942 fn test_module_dims() -> HashMap<String, (usize, usize)> {
943 let mut dims = HashMap::new();
944 dims.insert("q_proj".to_owned(), (512, 512));
945 dims.insert("v_proj".to_owned(), (512, 512));
946 dims
947 }
948
949 fn test_snapshots_dir() -> PathBuf {
950 std::env::temp_dir().join("delta_pool_test_snapshots")
951 }
952
953 #[test]
954 fn test_delta_pool_creation() {
955 let pool = DeltaPool::new(
956 TenantDeltaConfig::default(),
957 test_module_dims(),
958 Device::Cpu,
959 None,
960 test_snapshots_dir(),
961 None,
962 2,
963 );
964 assert_eq!(pool.tenant_count(), 0);
965 }
966
967 #[test]
968 fn test_get_or_create() {
969 let pool = DeltaPool::new(
970 TenantDeltaConfig::default(),
971 test_module_dims(),
972 Device::Cpu,
973 None,
974 test_snapshots_dir(),
975 None,
976 2,
977 );
978
979 let tid_a = Subject::new("tenant-a");
980 let tid_b = Subject::new("tenant-b");
981
982 let delta_a = pool.get_or_create(&tid_a).unwrap();
983 let delta_b = pool.get_or_create(&tid_b).unwrap();
984
985 assert_eq!(pool.tenant_count(), 2);
986
987 let delta_a2 = pool.get_or_create(&tid_a).unwrap();
989 assert!(Arc::ptr_eq(&delta_a, &delta_a2));
990
991 assert!(!Arc::ptr_eq(&delta_a, &delta_b));
993 }
994
995 #[test]
996 fn test_tenant_isolation() {
997 let pool = DeltaPool::new(
998 TenantDeltaConfig::default(),
999 test_module_dims(),
1000 Device::Cpu,
1001 None,
1002 test_snapshots_dir(),
1003 None,
1004 2,
1005 );
1006
1007 let tid_a = Subject::new("tenant-a");
1008 let tid_b = Subject::new("tenant-b");
1009
1010 let delta_a = pool.get_or_create(&tid_a).unwrap();
1011 let delta_b = pool.get_or_create(&tid_b).unwrap();
1012
1013 {
1015 let mut a = delta_a.lock();
1016 a.accumulated_steps = 42;
1017 }
1018
1019 {
1021 let b = delta_b.lock();
1022 assert_eq!(b.accumulated_steps, 0);
1023 }
1024 }
1025
1026 #[test]
1027 fn test_lru_eviction() {
1028 let pool = DeltaPool::new(
1029 TenantDeltaConfig::default(),
1030 test_module_dims(),
1031 Device::Cpu,
1032 None,
1033 test_snapshots_dir(),
1034 None,
1035 2,
1036 );
1037
1038 let tid_a = Subject::new("tenant-a");
1039 let tid_b = Subject::new("tenant-b");
1040 let tid_c = Subject::new("tenant-c");
1041
1042 let _a = pool.get_or_create(&tid_a).unwrap();
1043 std::thread::sleep(std::time::Duration::from_millis(10));
1044 let _b = pool.get_or_create(&tid_b).unwrap();
1045 std::thread::sleep(std::time::Duration::from_millis(10));
1046 let _c = pool.get_or_create(&tid_c).unwrap();
1047
1048 assert_eq!(pool.tenant_count(), 3);
1049
1050 let evicted = pool.evict_lru();
1052 assert!(evicted.is_some());
1053 let (evicted_id, _, _snapshot_path) = evicted.unwrap();
1054 assert_eq!(evicted_id, tid_a);
1055 assert_eq!(pool.tenant_count(), 2);
1056 }
1057
1058 #[test]
1059 fn test_total_memory_usage() {
1060 let pool = DeltaPool::new(
1061 TenantDeltaConfig::default(),
1062 test_module_dims(),
1063 Device::Cpu,
1064 None,
1065 test_snapshots_dir(),
1066 None,
1067 2,
1068 );
1069
1070 let tid = Subject::new("tenant");
1071 let _delta = pool.get_or_create(&tid).unwrap();
1072
1073 let usage = pool.total_memory_usage();
1074 assert!(usage > 0, "Memory usage should be non-zero with a delta");
1075 }
1076
1077 fn unique_snapshots_dir(tag: &str) -> PathBuf {
1081 let nanos = std::time::SystemTime::now()
1082 .duration_since(std::time::UNIX_EPOCH)
1083 .map(|d| d.as_nanos())
1084 .unwrap_or(0);
1085 std::env::temp_dir().join(format!("delta_pool_869_{}_{}", tag, nanos))
1086 }
1087
1088 fn pool_with_dir(dir: PathBuf) -> DeltaPool {
1089 DeltaPool::new(
1090 TenantDeltaConfig::default(),
1091 test_module_dims(),
1092 Device::Cpu,
1093 None,
1094 dir,
1095 None, 2,
1097 )
1098 }
1099
1100 #[tokio::test]
1102 async fn test_hydrate_roundtrip_restores_state() {
1103 let dir = unique_snapshots_dir("roundtrip");
1104 let pool = pool_with_dir(dir.clone());
1105 let tid = Subject::new("tenant-hydrate");
1106
1107 let expected_b: Tensor;
1109 {
1110 let delta_arc = pool.get_or_create(&tid).unwrap();
1111 let mut d = delta_arc.lock();
1112 let _guard = tch::no_grad_guard(); let b = d.lora_b.get_mut("0.q_proj").unwrap();
1114 let _ = b.uniform_(-0.5, 0.5);
1115 expected_b = b.copy();
1116 d.accumulated_steps = 17;
1117 d.request_count = 5;
1118 }
1119
1120 let evicted = pool.evict_lru();
1122 assert!(evicted.is_some());
1123 assert!(
1124 evicted.unwrap().2.is_some(),
1125 "eviction should write a snapshot"
1126 );
1127 assert_eq!(pool.tenant_count(), 0);
1128
1129 let delta_arc = pool.get_or_hydrate(&tid).await.unwrap();
1130 let d = delta_arc.lock();
1131 assert_eq!(d.accumulated_steps, 17, "accumulated_steps rehydrated");
1132 assert_eq!(d.request_count, 5, "request_count rehydrated");
1133 let max_diff: f64 = (&d.lora_b["0.q_proj"] - &expected_b)
1134 .abs()
1135 .max()
1136 .double_value(&[]);
1137 assert!(
1138 max_diff < 1e-6,
1139 "B weights rehydrated (max diff {})",
1140 max_diff
1141 );
1142
1143 drop(d);
1144 let _ = std::fs::remove_dir_all(&dir);
1145 }
1146
1147 #[tokio::test]
1149 async fn test_corrupt_snapshot_quarantined_and_blank() {
1150 let dir = unique_snapshots_dir("corrupt");
1151 std::fs::create_dir_all(&dir).unwrap();
1152 let pool = pool_with_dir(dir.clone());
1153 let tid = Subject::new("tenant-corrupt");
1154
1155 let fname = DeltaPool::sanitize_filename(&tid.to_string());
1157 let snap_path = dir.join(format!("{}.safetensors", fname));
1158 std::fs::write(&snap_path, b"not a safetensors file").unwrap();
1159
1160 let delta_arc = pool.get_or_hydrate(&tid).await.unwrap();
1161 {
1162 let d = delta_arc.lock();
1163 assert_eq!(d.accumulated_steps, 0);
1165 let b_norm: f64 = d.lora_b["0.q_proj"].norm().double_value(&[]);
1166 assert!(b_norm < 1e-8, "fell open to a blank delta");
1167 }
1168 assert!(!snap_path.exists(), "corrupt file moved aside");
1169 assert!(
1170 dir.join(format!("{}.safetensors.corrupt", fname)).exists(),
1171 "corrupt file quarantined"
1172 );
1173
1174 let _ = std::fs::remove_dir_all(&dir);
1175 }
1176
1177 #[tokio::test]
1179 async fn test_incompatible_shape_snapshot_falls_open() {
1180 let dir = unique_snapshots_dir("incompat");
1181 std::fs::create_dir_all(&dir).unwrap();
1182 let pool = pool_with_dir(dir.clone());
1183 let tid = Subject::new("tenant-incompat");
1184
1185 let mut state: HashMap<String, Tensor> = HashMap::new();
1187 state.insert(
1188 "0.q_proj.lora_a".to_owned(),
1189 Tensor::zeros([4, 8], (tch::Kind::Float, Device::Cpu)), );
1191 let bytes = serialize_state_dict_to_bytes(&state).unwrap();
1192 let fname = DeltaPool::sanitize_filename(&tid.to_string());
1193 std::fs::write(dir.join(format!("{}.safetensors", fname)), &bytes).unwrap();
1194
1195 let delta_arc = pool.get_or_hydrate(&tid).await.unwrap();
1197 {
1198 let d = delta_arc.lock();
1199 assert_eq!(d.lora_a["0.q_proj"].size(), vec![8, 512]);
1200 }
1201 assert!(
1202 dir.join(format!("{}.safetensors.corrupt", fname)).exists(),
1203 "incompatible snapshot quarantined"
1204 );
1205
1206 let _ = std::fs::remove_dir_all(&dir);
1207 }
1208
1209 #[tokio::test]
1211 async fn test_hydrate_no_snapshot_is_blank() {
1212 let dir = unique_snapshots_dir("nosnap");
1213 let pool = pool_with_dir(dir.clone());
1214 let tid = Subject::new("tenant-fresh");
1215
1216 let delta_arc = pool.get_or_hydrate(&tid).await.unwrap();
1217 let d = delta_arc.lock();
1218 assert_eq!(d.accumulated_steps, 0);
1219 assert_eq!(pool.tenant_count(), 1);
1220 drop(d);
1221 let _ = std::fs::remove_dir_all(&dir);
1222 }
1223
1224 #[tokio::test]
1226 async fn test_export_all_no_fs_is_noop() {
1227 let dir = unique_snapshots_dir("exportnofs");
1228 let pool = pool_with_dir(dir.clone());
1229 let tid = Subject::new("tenant-x");
1230 {
1231 let d = pool.get_or_create(&tid).unwrap();
1232 d.lock().accumulated_steps = 3;
1233 }
1234 let exported = pool.export_all().await.unwrap();
1236 assert!(exported.is_empty());
1237 assert_eq!(pool.tenant_count(), 1);
1239 let _ = std::fs::remove_dir_all(&dir);
1240 }
1241}