Skip to main content

hyprstream_inference/training/
delta_pool.rs

1//! Per-tenant delta registry with LRU eviction
2//!
3//! Manages a pool of `TenantDelta` instances, one per tenant/session.
4//! Pattern mirrors `KVCacheRegistry` using `DashMap` for concurrent access.
5//!
6//! # Snapshot durability & lifecycle (#869)
7//!
8//! Uncommitted per-tenant TTT adaptation is persisted to
9//! `adapters/.snapshots/{subject}.safetensors` on two occasions:
10//! - **LRU eviction** ([`DeltaPool::evict_lru`] / [`DeltaPool::evict_lru_async`]) — a
11//!   warm-cache spill.
12//! - **Drain-time export** ([`DeltaPool::export_all`]) — a graceful-shutdown "snapshot
13//!   all, now" for pod preStop / scale-in.
14//!
15//! The read half is [`DeltaPool::get_or_hydrate`], which reloads a snapshot before
16//! zero-initializing a fresh delta, restoring weights, effective ranks, Muon momentum,
17//! and `accumulated_steps` / `request_count`.
18//!
19//! **Lifecycle decision — keep, don't delete on load.** A rehydrated snapshot is left
20//! on disk after a successful load. Deleting it on load would open a window where a crash
21//! between hydrate and the next eviction/export loses every step accumulated since — the
22//! exact failure #869 exists to close. The file is instead overwritten by the next
23//! eviction/export snapshot of the same subject, so at worst a reader sees slightly stale
24//! (never absent) state. These staging snapshots stay in `.snapshots/` and are **never**
25//! auto-promoted into a committed `adapters/NN_*.safetensors` adapter — that crossing is
26//! the STEP `Promote` boundary and remains an explicit operation.
27//!
28//! **What a snapshot does NOT preserve** (rehydrated deltas start blank for these; see
29//! [`TenantDelta::extract_state_dict`]): the in-memory STEP `Pending` rollback state
30//! (`DeltaAdaptationState::Pending`) — a speculative, not-yet-resolved adaptation is
31//! folded into the committed weights it was applied on top of, so it survives as
32//! committed rather than as a reversible pending. No SSM/optimizer state beyond Muon
33//! momentum is carried (there is none in `TenantDelta` today). `avg_loss_improvement`
34//! and `last_snapshot_hash` reset.
35
36use anyhow::{anyhow, Result};
37use dashmap::DashMap;
38use parking_lot::Mutex;
39use std::collections::HashMap;
40use std::path::PathBuf;
41use std::sync::Arc;
42use tch::{Device, Tensor};
43
44use super::muon::MuonState;
45use super::tenant_delta::{
46    load_state_dict_from_bytes, serialize_state_dict_to_bytes, TenantDelta, TenantDeltaConfig,
47};
48use crate::runtime::kv_cache::KVCacheRegistry;
49use crate::worktree_ext::WorktreeClientExt;
50use hyprstream_rpc::Subject;
51use hyprstream_rpc_std::registry_client::WorktreeClient;
52
53/// Maximum snapshot size in bytes (512 MB). Deltas exceeding this are not snapshotted on eviction.
54const MAX_SNAPSHOT_BYTES: usize = 512 * 1024 * 1024;
55
56/// Default maximum number of tenants before LRU eviction kicks in.
57const MAX_TENANTS_DEFAULT: usize = 100;
58
59/// Snapshot-file metadata key: accumulated gradient steps (i64 scalar).
60///
61/// These `__meta.*` keys ride inside the safetensors snapshot alongside the LoRA
62/// A/B / `__effective_rank` / `__muon_momentum` tensors written by
63/// [`TenantDelta::extract_state_dict`]. They are injected only at the DeltaPool
64/// snapshot boundary (see [`DeltaPool::snapshot_state_dict`]) so the generic
65/// in-memory rollback state dict (used by the STEP stage/apply machinery) stays
66/// free of pool-lifecycle counters.
67const META_ACCUMULATED_STEPS: &str = "__meta.accumulated_steps";
68/// Snapshot-file metadata key: served request count (i64 scalar).
69const META_REQUEST_COUNT: &str = "__meta.request_count";
70
71/// Registry managing per-tenant LoRA deltas for isolated TTT adaptation
72///
73/// Each `Subject` gets its own isolated `TenantDelta` wrapped in `Arc<Mutex<_>>`.
74/// Uses `DashMap` for lock-free access to different tenants.
75pub struct DeltaPool {
76    /// Active deltas indexed by tenant
77    deltas: DashMap<Subject, Arc<Mutex<TenantDelta>>>,
78    /// Default configuration for new deltas (behind Mutex for interior mutability via update_config)
79    default_config: Mutex<TenantDeltaConfig>,
80    /// Module dimensions from the loaded model: module_name -> (in_features, out_features)
81    module_dims: HashMap<String, (usize, usize)>,
82    /// Per-layer dimension overrides: layer_idx -> module_name -> (in, out)
83    /// Used for hybrid architectures (e.g., Qwen3.5) where GDN and full-attn layers
84    /// have different dimensions for the same LoRA module name (e.g., "o_proj").
85    per_layer_dims: Option<HashMap<usize, HashMap<String, (usize, usize)>>>,
86    /// Device for tensor allocation
87    device: Device,
88    /// Memory budget in bytes (None = unlimited)
89    memory_budget_bytes: Option<usize>,
90    /// Base weight norms for drift monitoring: module_name -> ||W||
91    base_weight_norms: HashMap<String, f64>,
92    /// KV cache registry for dependency-aware eviction.
93    /// When a delta is evicted or reset, dependent KV caches are invalidated.
94    kv_registry: Option<Arc<KVCacheRegistry>>,
95    /// Directory for eviction snapshots
96    snapshots_dir: PathBuf,
97    /// Optional WorktreeClient for worktree-scoped file operations
98    fs: Option<WorktreeClient>,
99    /// Number of model layers for per-layer delta creation
100    num_layers: usize,
101    /// Maximum number of tenants before LRU eviction
102    max_tenants: usize,
103    /// Optional rank oracle config to attach to new deltas
104    rank_oracle_config: Option<super::ttt::RankOracleConfig>,
105}
106
107impl DeltaPool {
108    /// Create a new delta pool
109    ///
110    /// # Arguments
111    /// * `config` - Default configuration for new deltas
112    /// * `module_dims` - Module dimensions from model_info
113    /// * `device` - Device for tensor allocation
114    /// * `kv_registry` - Optional KV cache registry for dependency-aware eviction
115    /// * `snapshots_dir` - Directory to write eviction snapshots
116    /// * `fs` - Optional WorktreeClient for worktree-scoped file operations
117    pub fn new(
118        config: TenantDeltaConfig,
119        module_dims: HashMap<String, (usize, usize)>,
120        device: Device,
121        kv_registry: Option<Arc<KVCacheRegistry>>,
122        snapshots_dir: PathBuf,
123        fs: Option<WorktreeClient>,
124        num_layers: usize,
125    ) -> Self {
126        Self {
127            deltas: DashMap::new(),
128            default_config: Mutex::new(config),
129            module_dims,
130            per_layer_dims: None,
131            device,
132            memory_budget_bytes: None,
133            base_weight_norms: HashMap::new(),
134            kv_registry,
135            snapshots_dir,
136            fs,
137            num_layers,
138            max_tenants: MAX_TENANTS_DEFAULT,
139            rank_oracle_config: None,
140        }
141    }
142
143    /// Set rank oracle config for runtime rank adaptation on new deltas.
144    pub fn with_rank_oracle(mut self, config: super::ttt::RankOracleConfig) -> Self {
145        self.rank_oracle_config = Some(config);
146        self
147    }
148
149    /// Set per-layer dimension overrides for hybrid architectures (e.g., Qwen3.5).
150    pub fn with_per_layer_dims(
151        mut self,
152        per_layer_dims: HashMap<usize, HashMap<String, (usize, usize)>>,
153    ) -> Self {
154        self.per_layer_dims = Some(per_layer_dims);
155        self
156    }
157
158    /// Set memory budget for the pool
159    pub fn with_memory_budget(mut self, budget_bytes: usize) -> Self {
160        self.memory_budget_bytes = Some(budget_bytes);
161        self
162    }
163
164    /// Set maximum number of tenants before LRU eviction
165    pub fn with_max_tenants(mut self, max_tenants: usize) -> Self {
166        self.max_tenants = max_tenants;
167        self
168    }
169
170    /// Update the default config for new deltas (e.g., when createLora changes target modules)
171    ///
172    /// Also clears existing deltas since their module sets would be stale.
173    pub fn update_config(&self, config: TenantDeltaConfig) {
174        // Clear existing deltas — they have the old module set
175        self.clear_all();
176        *self.default_config.lock() = config;
177    }
178
179    /// Set base weight norms for drift monitoring
180    pub fn set_base_weight_norms(&mut self, norms: HashMap<String, f64>) {
181        self.base_weight_norms = norms;
182    }
183
184    /// Set the KV cache registry for dependency-aware eviction
185    pub fn set_kv_registry(&mut self, registry: Arc<KVCacheRegistry>) {
186        self.kv_registry = Some(registry);
187    }
188
189    /// Construct a fresh, zero-initialized delta with the default config.
190    ///
191    /// This is the pre-#869 status quo: B = zeros, no accumulated steps. Shared by
192    /// [`Self::get_or_create`] and [`Self::get_or_hydrate`] (which then overlays a
193    /// persisted snapshot on top, when one exists).
194    fn create_blank_delta(&self) -> Result<TenantDelta> {
195        let config = self.default_config.lock().clone();
196        let mut delta = TenantDelta::new_with_per_layer_dims(
197            &config,
198            &self.module_dims,
199            self.device,
200            self.num_layers,
201            self.per_layer_dims.as_ref(),
202        )?;
203        // Attach rank oracle if configured
204        if let Some(ref oracle_config) = self.rank_oracle_config {
205            delta.rank_oracle = Some(super::ttt::RankOracle::new(oracle_config.clone()));
206        }
207        Ok(delta)
208    }
209
210    /// Get or create a delta for the given tenant (synchronous, **no** snapshot rehydration).
211    ///
212    /// If the delta doesn't exist, creates a new zero-initialized one with the default
213    /// config. This path does **not** reload `adapters/.snapshots/{subject}.safetensors` —
214    /// it cannot, because snapshot reads require async FsOps. Async callers that want
215    /// replica-loss durability (#869) must use [`Self::get_or_hydrate`] instead; this
216    /// sync entry point remains for non-async callers (tests, batched-LoRA construction).
217    pub fn get_or_create(&self, tenant_id: &Subject) -> Result<Arc<Mutex<TenantDelta>>> {
218        // Fast path: existing delta
219        if let Some(delta) = self.deltas.get(tenant_id) {
220            delta.lock().touch();
221            return Ok(delta.clone());
222        }
223
224        let delta = Arc::new(Mutex::new(self.create_blank_delta()?));
225
226        // Insert (handles race condition)
227        Ok(self
228            .deltas
229            .entry(tenant_id.clone())
230            .or_insert(delta.clone())
231            .clone())
232    }
233
234    /// Get an existing delta, or create one **rehydrating from a persisted snapshot**
235    /// if `adapters/.snapshots/{subject}.safetensors` exists (async sibling of
236    /// [`Self::get_or_create`], #869).
237    ///
238    /// This is the read half that makes eviction/drain snapshots durable across replica
239    /// loss and scale-in. On a cold miss it:
240    /// 1. builds a blank delta (as `get_or_create` would),
241    /// 2. looks for a snapshot (via FsOps when a [`WorktreeClient`] is present, else the
242    ///    local `snapshots_dir`),
243    /// 3. on hit, restores the LoRA A/B weights, per-key effective ranks, Muon momentum,
244    ///    and `accumulated_steps` / `request_count` onto the delta (device-mapped to the
245    ///    pool's device),
246    /// 4. on a corrupt/unloadable/shape-incompatible snapshot, **falls open** to a blank
247    ///    delta (the pre-#869 status quo) and quarantines the bad file as `*.corrupt`.
248    ///
249    /// The snapshot file is **kept** after a successful load (not deleted) — see the
250    /// module note on snapshot lifecycle. It is overwritten by the next eviction/export.
251    pub async fn get_or_hydrate(&self, tenant_id: &Subject) -> Result<Arc<Mutex<TenantDelta>>> {
252        // Fast path: already resident.
253        if let Some(delta) = self.deltas.get(tenant_id) {
254            delta.lock().touch();
255            return Ok(delta.clone());
256        }
257
258        let mut delta = self.create_blank_delta()?;
259        let id_filename = Self::sanitize_filename(&tenant_id.to_string());
260
261        match self.read_snapshot_bytes(&id_filename).await {
262            Ok(Some(bytes)) => {
263                let loaded = load_state_dict_from_bytes(&bytes)
264                    .and_then(|state| Self::apply_snapshot(&mut delta, &state));
265                match loaded {
266                    Ok(()) => {
267                        tracing::info!(
268                            "Rehydrated delta '{}' from snapshot ({} accumulated steps)",
269                            tenant_id,
270                            delta.accumulated_steps
271                        );
272                    }
273                    Err(e) => {
274                        tracing::warn!(
275                            "Corrupt/incompatible snapshot for delta '{}': {} — quarantining and starting blank",
276                            tenant_id,
277                            e
278                        );
279                        self.quarantine_snapshot(&id_filename).await;
280                        // Rebuild pristine: apply_snapshot validates before mutating, but a
281                        // rebuild guarantees no partial state leaks through on any error path.
282                        delta = self.create_blank_delta()?;
283                    }
284                }
285            }
286            Ok(None) => {
287                // No snapshot — normal cold start.
288            }
289            Err(e) => {
290                tracing::warn!(
291                    "Failed to read snapshot for delta '{}': {} — starting blank",
292                    tenant_id,
293                    e
294                );
295            }
296        }
297
298        let delta = Arc::new(Mutex::new(delta));
299        Ok(self
300            .deltas
301            .entry(tenant_id.clone())
302            .or_insert(delta.clone())
303            .clone())
304    }
305
306    /// Build the state dict written to a snapshot file: the delta's tensor state
307    /// (A/B, effective ranks, Muon momentum) plus pool-lifecycle counters
308    /// (`accumulated_steps`, `request_count`) as `__meta.*` scalar tensors.
309    fn snapshot_state_dict(delta: &TenantDelta) -> HashMap<String, Tensor> {
310        let mut state = delta.extract_state_dict();
311        state.insert(
312            META_ACCUMULATED_STEPS.to_owned(),
313            Tensor::from_slice(&[delta.accumulated_steps as i64]),
314        );
315        state.insert(
316            META_REQUEST_COUNT.to_owned(),
317            Tensor::from_slice(&[delta.request_count as i64]),
318        );
319        state
320    }
321
322    /// Restore a loaded snapshot state dict onto a (blank) delta.
323    ///
324    /// Validates A/B shapes up front — `load_state_dict`'s `copy_` panics on a shape
325    /// mismatch, so an incompatible snapshot (e.g. the pool's rank/module config changed
326    /// since the snapshot was written) must be rejected as an error here rather than
327    /// aborting the process. Also pre-seeds `muon_states` for any momentum buffers in the
328    /// snapshot (a blank delta starts with none, so `load_state_dict` would otherwise drop
329    /// them), device-mapped to the delta's device.
330    fn apply_snapshot(delta: &mut TenantDelta, state: &HashMap<String, Tensor>) -> Result<()> {
331        // Shape-compatibility gate (fail before any mutation).
332        for key in delta.lora_a.keys() {
333            if let Some(src) = state.get(&format!("{}.lora_a", key)) {
334                let want = delta.lora_a[key].size();
335                if src.size() != want {
336                    return Err(anyhow!(
337                        "snapshot lora_a '{}' shape {:?} != expected {:?}",
338                        key,
339                        src.size(),
340                        want
341                    ));
342                }
343            }
344        }
345        for key in delta.lora_b.keys() {
346            if let Some(src) = state.get(&format!("{}.lora_b", key)) {
347                let want = delta.lora_b[key].size();
348                if src.size() != want {
349                    return Err(anyhow!(
350                        "snapshot lora_b '{}' shape {:?} != expected {:?}",
351                        key,
352                        src.size(),
353                        want
354                    ));
355                }
356            }
357        }
358
359        // Pre-seed Muon momentum slots (device-mapped) so load_state_dict restores them.
360        let device = delta.device;
361        for (name, t) in state.iter() {
362            if let Some(key) = name.strip_suffix(".__muon_momentum") {
363                delta
364                    .muon_states
365                    .entry(key.to_owned())
366                    .or_insert_with(|| MuonState {
367                        momentum_buffer: Some(t.to_device(device)),
368                    });
369            }
370        }
371
372        delta.load_state_dict(state)?;
373
374        if let Some(t) = state.get(META_ACCUMULATED_STEPS) {
375            delta.accumulated_steps = t.int64_value(&[]).max(0) as u64;
376        }
377        if let Some(t) = state.get(META_REQUEST_COUNT) {
378            delta.request_count = t.int64_value(&[]).max(0) as u64;
379        }
380        Ok(())
381    }
382
383    /// Read a snapshot's raw bytes, or `Ok(None)` if it does not exist.
384    ///
385    /// Reads through the [`WorktreeClient`] (9P/FsOps) when present — the same channel
386    /// the async eviction/export path writes through — otherwise falls back to the local
387    /// `snapshots_dir` (used by the sync eviction path and by tests).
388    async fn read_snapshot_bytes(&self, id_filename: &str) -> Result<Option<Vec<u8>>> {
389        let rel_path = format!("adapters/.snapshots/{}.safetensors", id_filename);
390        if let Some(ref fs) = self.fs {
391            let st = fs
392                .stat_path(&rel_path)
393                .await
394                .map_err(|e| anyhow!("FsOps stat failed: {}", e))?;
395            if !st.exists {
396                return Ok(None);
397            }
398            let bytes = fs
399                .read_file_chunked(&rel_path)
400                .await
401                .map_err(|e| anyhow!("FsOps read failed: {}", e))?;
402            Ok(Some(bytes))
403        } else {
404            let path = self
405                .snapshots_dir
406                .join(format!("{}.safetensors", id_filename));
407            match std::fs::read(&path) {
408                Ok(b) => Ok(Some(b)),
409                Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
410                Err(e) => Err(anyhow!("read snapshot {}: {}", path.display(), e)),
411            }
412        }
413    }
414
415    /// Best-effort quarantine of a bad snapshot: move `{id}.safetensors` aside to
416    /// `{id}.safetensors.corrupt` so a later hydrate does not retry the same bad file
417    /// and an operator can inspect it. Never fails the caller.
418    async fn quarantine_snapshot(&self, id_filename: &str) {
419        let rel_path = format!("adapters/.snapshots/{}.safetensors", id_filename);
420        let corrupt_rel = format!("{}.corrupt", rel_path);
421        if let Some(ref fs) = self.fs {
422            // No 9P rename op — copy bytes aside, then remove the original.
423            match fs.read_file_chunked(&rel_path).await {
424                Ok(bytes) => {
425                    if let Err(e) = fs.write_file_chunked(&corrupt_rel, &bytes).await {
426                        tracing::warn!(
427                            "Failed to write quarantined snapshot {}: {}",
428                            corrupt_rel,
429                            e
430                        );
431                    }
432                }
433                Err(e) => {
434                    tracing::warn!("Failed to read snapshot for quarantine {}: {}", rel_path, e);
435                }
436            }
437            if let Err(e) = fs.remove_path(&rel_path).await {
438                tracing::warn!("Failed to remove corrupt snapshot {}: {}", rel_path, e);
439            }
440        } else {
441            let path = self
442                .snapshots_dir
443                .join(format!("{}.safetensors", id_filename));
444            let corrupt = self
445                .snapshots_dir
446                .join(format!("{}.safetensors.corrupt", id_filename));
447            if let Err(e) = std::fs::rename(&path, &corrupt) {
448                tracing::warn!(
449                    "Failed to quarantine corrupt snapshot {}: {}",
450                    path.display(),
451                    e
452                );
453            }
454        }
455        tracing::warn!(
456            "Quarantined corrupt snapshot for '{}' -> {}",
457            id_filename,
458            corrupt_rel
459        );
460    }
461
462    /// Get an existing delta (returns None if not found)
463    pub fn get(&self, tenant_id: &Subject) -> Option<Arc<Mutex<TenantDelta>>> {
464        self.deltas.get(tenant_id).map(|d| {
465            d.lock().touch();
466            d.clone()
467        })
468    }
469
470    /// Remove a subject's delta and invalidate dependent KV caches
471    pub fn remove(&self, subject: &Subject) -> Option<Arc<Mutex<TenantDelta>>> {
472        let result = self.deltas.remove(subject).map(|(_, d)| d);
473        if result.is_some() {
474            if let Some(kv_reg) = &self.kv_registry {
475                kv_reg.invalidate_for_tenant(&subject.to_string());
476            }
477        }
478        result
479    }
480
481    /// List all subjects in the pool (for debugging)
482    pub fn list_subjects(&self) -> Vec<String> {
483        self.deltas.iter().map(|e| e.key().to_string()).collect()
484    }
485
486    /// Get number of active tenants
487    pub fn tenant_count(&self) -> usize {
488        self.deltas.len()
489    }
490
491    /// Get total memory usage across all deltas in bytes
492    pub fn total_memory_usage(&self) -> usize {
493        self.deltas
494            .iter()
495            .map(|entry| entry.value().lock().memory_bytes())
496            .sum()
497    }
498
499    /// Get a clone of the current default config
500    pub fn default_config(&self) -> TenantDeltaConfig {
501        self.default_config.lock().clone()
502    }
503
504    /// Get the LoRA rank used by this pool's deltas
505    pub fn rank(&self) -> usize {
506        self.default_config.lock().rank
507    }
508
509    /// Get module dimensions reference
510    pub fn module_dims(&self) -> &HashMap<String, (usize, usize)> {
511        &self.module_dims
512    }
513
514    /// Get base weight norms reference
515    pub fn base_weight_norms(&self) -> &HashMap<String, f64> {
516        &self.base_weight_norms
517    }
518
519    /// Get snapshots directory reference
520    pub fn snapshots_dir(&self) -> &PathBuf {
521        &self.snapshots_dir
522    }
523
524    /// Check if eviction is needed (tenant count or memory budget exceeded).
525    ///
526    /// This is a fast, non-blocking check with no I/O.
527    pub fn needs_eviction(&self) -> bool {
528        if self.deltas.len() >= self.max_tenants {
529            return true;
530        }
531        if let Some(budget) = self.memory_budget_bytes {
532            if self.total_memory_usage() > budget {
533                return true;
534            }
535        }
536        false
537    }
538
539    /// Ensure the pool has capacity for at least one more tenant.
540    ///
541    /// Call this from async context **before** `get_or_create()` to avoid
542    /// `Handle::block_on()` panics inside `spawn_local` on `current_thread` runtime.
543    ///
544    /// Evicts LRU tenants (with async snapshot I/O) until capacity is available.
545    pub async fn ensure_capacity(&self) -> Result<()> {
546        while self.needs_eviction() {
547            if self.evict_lru_async().await?.is_none() {
548                break; // Pool is empty, nothing left to evict
549            }
550        }
551        Ok(())
552    }
553
554    /// Sanitize a subject string for use as a filename.
555    fn sanitize_filename(subject: &str) -> String {
556        subject
557            .replace(['/', '\\', '\0'], "_")
558            .replace("..", "_")
559            .chars()
560            .take(200)
561            .collect()
562    }
563
564    /// Find the LRU (least recently used) tenant ID.
565    ///
566    /// Returns `None` if the pool is empty.
567    fn find_lru_tenant(&self) -> Option<Subject> {
568        let mut oldest_tenant: Option<Subject> = None;
569        let mut oldest_time = std::time::Instant::now();
570
571        for entry in self.deltas.iter() {
572            let delta = entry.value().lock();
573            if delta.last_access < oldest_time {
574                oldest_time = delta.last_access;
575                oldest_tenant = Some(entry.key().clone());
576            }
577        }
578        oldest_tenant
579    }
580
581    /// Async eviction of the least-recently-used tenant delta.
582    ///
583    /// Before eviction:
584    /// 1. Snapshots the delta to a file (if it has accumulated steps) using async I/O
585    /// 2. Invalidates dependent KV caches (if kv_registry is set)
586    ///
587    /// Returns the evicted tenant ID, delta, and optional snapshot path.
588    pub async fn evict_lru_async(
589        &self,
590    ) -> Result<Option<(Subject, Arc<Mutex<TenantDelta>>, Option<PathBuf>)>> {
591        if self.deltas.is_empty() {
592            return Ok(None);
593        }
594
595        let Some(tid) = self.find_lru_tenant() else {
596            return Ok(None);
597        };
598
599        let Some((id, delta)) = self.deltas.remove(&tid) else {
600            return Ok(None);
601        };
602
603        let id_filename = Self::sanitize_filename(&id.to_string());
604
605        // Auto-snapshot to file before eviction
606        // Extract data under lock, then release before any async I/O
607        let snapshot_data = {
608            let d = delta.lock();
609            if d.accumulated_steps > 0 {
610                if self.fs.is_some() {
611                    let state_dict = Self::snapshot_state_dict(&d);
612                    let memory_bytes = d.memory_bytes();
613                    Some((state_dict, memory_bytes))
614                } else {
615                    tracing::warn!(
616                        "FsOps not available — skipping snapshot for delta '{}' during eviction",
617                        id
618                    );
619                    None
620                }
621            } else {
622                None // No accumulated steps, skip snapshot
623            }
624        }; // lock released here
625
626        let snapshot_path = if let Some((state_dict, memory_bytes)) = snapshot_data {
627            let snapshot_file = self
628                .snapshots_dir
629                .join(format!("{}.safetensors", id_filename));
630            // Safety: snapshot_data is Some only when self.fs.is_some() was true above
631            let Some(ref fs) = self.fs else {
632                unreachable!()
633            };
634            let rel_path = format!("adapters/.snapshots/{}.safetensors", id_filename);
635
636            if memory_bytes > MAX_SNAPSHOT_BYTES {
637                tracing::warn!(
638                    "Delta '{}' exceeds max snapshot size ({} bytes > {}), skipping snapshot",
639                    id,
640                    memory_bytes,
641                    MAX_SNAPSHOT_BYTES
642                );
643                None
644            } else {
645                match async {
646                    fs.mkdir_p("adapters/.snapshots")
647                        .await
648                        .map_err(|e| anyhow::anyhow!("FsOps mkdir failed: {}", e))?;
649                    let bytes = serialize_state_dict_to_bytes(&state_dict)?;
650                    fs.write_file_chunked(&rel_path, &bytes)
651                        .await
652                        .map_err(|e| anyhow::anyhow!("FsOps write_file failed: {}", e))?;
653                    Ok::<(), anyhow::Error>(())
654                }
655                .await
656                {
657                    Ok(()) => {
658                        tracing::info!(
659                            "Auto-snapshot delta '{}' before eviction via FsOps: {}",
660                            id,
661                            rel_path
662                        );
663                        Some(snapshot_file)
664                    }
665                    Err(e) => {
666                        tracing::warn!("Failed to snapshot delta '{}' via FsOps: {}", id, e);
667                        None
668                    }
669                }
670            }
671        } else {
672            None
673        };
674
675        // Invalidate dependent KV caches
676        if let Some(kv_reg) = &self.kv_registry {
677            let invalidated = kv_reg.invalidate_for_tenant(&id.to_string());
678            if invalidated > 0 {
679                tracing::info!(
680                    "Invalidated {} KV caches on eviction of delta '{}'",
681                    invalidated,
682                    id
683                );
684            }
685        }
686
687        Ok(Some((id, delta, snapshot_path)))
688    }
689
690    /// Evict the least-recently-used tenant delta (sync version).
691    ///
692    /// **WARNING**: This uses `futures::executor::block_on()` for snapshot I/O.
693    /// Only safe to call from non-async contexts (tests, standalone threads).
694    /// In async contexts, use `evict_lru_async()` instead.
695    pub fn evict_lru(&self) -> Option<(Subject, Arc<Mutex<TenantDelta>>, Option<PathBuf>)> {
696        // For sync callers, we can't do async I/O. Remove without snapshot
697        // if there's no WorktreeClient, or use block_on only in non-async contexts.
698        if self.deltas.is_empty() {
699            return None;
700        }
701
702        let tid = self.find_lru_tenant()?;
703        let (id, delta) = self.deltas.remove(&tid)?;
704
705        let id_filename = Self::sanitize_filename(&id.to_string());
706
707        // Snapshot without async I/O — extract state dict only, skip FsOps writes
708        let snapshot_path =
709            {
710                let d = delta.lock();
711                if d.accumulated_steps > 0 {
712                    let snapshot_file = self
713                        .snapshots_dir
714                        .join(format!("{}.safetensors", id_filename));
715                    // In sync context, we can only write directly to the snapshots_dir
716                    // (not through WorktreeClient which requires async)
717                    let state_dict = Self::snapshot_state_dict(&d);
718                    let memory_bytes = d.memory_bytes();
719                    drop(d);
720
721                    if memory_bytes > MAX_SNAPSHOT_BYTES {
722                        tracing::warn!(
723                        "Delta '{}' exceeds max snapshot size ({} bytes > {}), skipping snapshot",
724                        id, memory_bytes, MAX_SNAPSHOT_BYTES
725                    );
726                        None
727                    } else {
728                        match serialize_state_dict_to_bytes(&state_dict) {
729                            Ok(bytes) => {
730                                if let Err(e) = std::fs::create_dir_all(&self.snapshots_dir) {
731                                    tracing::warn!("Failed to create snapshots dir: {}", e);
732                                    None
733                                } else if let Err(e) = std::fs::write(&snapshot_file, &bytes) {
734                                    tracing::warn!(
735                                        "Failed to write snapshot for delta '{}': {}",
736                                        id,
737                                        e
738                                    );
739                                    None
740                                } else {
741                                    tracing::info!(
742                                        "Auto-snapshot delta '{}' before eviction: {}",
743                                        id,
744                                        snapshot_file.display()
745                                    );
746                                    Some(snapshot_file)
747                                }
748                            }
749                            Err(e) => {
750                                tracing::warn!(
751                                    "Failed to serialize delta '{}' for snapshot: {}",
752                                    id,
753                                    e
754                                );
755                                None
756                            }
757                        }
758                    }
759                } else {
760                    None
761                }
762            };
763
764        // Invalidate dependent KV caches
765        if let Some(kv_reg) = &self.kv_registry {
766            let invalidated = kv_reg.invalidate_for_tenant(&id.to_string());
767            if invalidated > 0 {
768                tracing::info!(
769                    "Invalidated {} KV caches on eviction of delta '{}'",
770                    invalidated,
771                    id
772                );
773            }
774        }
775
776        Some((id, delta, snapshot_path))
777    }
778
779    /// Async version: evict deltas until total memory is under budget.
780    ///
781    /// Returns a list of evicted (tenant_id, snapshot_path) pairs for audit/recovery.
782    pub async fn evict_to_budget_async(&self) -> Result<Vec<(Subject, Option<PathBuf>)>> {
783        let budget = match self.memory_budget_bytes {
784            Some(b) => b,
785            None => return Ok(Vec::new()),
786        };
787
788        let mut evicted = Vec::new();
789        while self.total_memory_usage() > budget {
790            match self.evict_lru_async().await? {
791                Some((tid, _, path)) => evicted.push((tid, path)),
792                None => break,
793            }
794        }
795        Ok(evicted)
796    }
797
798    /// Evict deltas until total memory is under budget (sync version).
799    ///
800    /// **WARNING**: Only safe in non-async contexts. See `evict_lru()` docs.
801    pub fn evict_to_budget(&self) -> Vec<(Subject, Option<PathBuf>)> {
802        let budget = match self.memory_budget_bytes {
803            Some(b) => b,
804            None => return Vec::new(),
805        };
806
807        let mut evicted = Vec::new();
808        while self.total_memory_usage() > budget {
809            match self.evict_lru() {
810                Some((tid, _, path)) => evicted.push((tid, path)),
811                None => break,
812            }
813        }
814        evicted
815    }
816
817    /// Drain-time export: snapshot **every** resident delta with accumulated steps,
818    /// without evicting any of them (#869, deliverable 2).
819    ///
820    /// This is the explicit "snapshot all, now" entry point for a graceful-shutdown /
821    /// pod-preStop drain hook. It reuses the eviction snapshot format and path
822    /// (`adapters/.snapshots/{subject}.safetensors`) and the same 512 MB cap and
823    /// FsOps-required semantics, so a rehydrating replica reads these back through
824    /// [`Self::get_or_hydrate`] identically to eviction snapshots.
825    ///
826    /// Deltas remain live in the pool after export (the process is about to exit, but
827    /// the delta state must survive to disk, not be cleared). Returns `(subject,
828    /// snapshot_path)` for each resident delta considered — `None` in the path slot
829    /// marks a **loss event**: a delta that was skipped (no accumulated steps, over the
830    /// size cap, or a write failure). Callers/operators should treat `None` with
831    /// `accumulated_steps > 0` as data loss.
832    ///
833    /// Requires a [`WorktreeClient`]; without one this is a logged no-op (FsOps-absent
834    /// skip, matching the eviction path — there is no worktree to write through).
835    pub async fn export_all(&self) -> Result<Vec<(Subject, Option<PathBuf>)>> {
836        let Some(ref fs) = self.fs else {
837            tracing::warn!(
838                "export_all: FsOps not available — {} resident delta(s) NOT persisted (loss on shutdown)",
839                self.deltas.len()
840            );
841            return Ok(Vec::new());
842        };
843
844        if self.deltas.is_empty() {
845            return Ok(Vec::new());
846        }
847
848        fs.mkdir_p("adapters/.snapshots")
849            .await
850            .map_err(|e| anyhow!("FsOps mkdir failed: {}", e))?;
851
852        // Collect ids first so we don't hold DashMap refs across await points.
853        let ids: Vec<Subject> = self.deltas.iter().map(|e| e.key().clone()).collect();
854        let mut results = Vec::with_capacity(ids.len());
855
856        for id in ids {
857            let Some(entry) = self.deltas.get(&id) else {
858                continue; // Evicted concurrently.
859            };
860            let delta = entry.value().clone();
861            drop(entry);
862
863            // Extract snapshot data under lock, release before async I/O.
864            let snapshot_data = {
865                let d = delta.lock();
866                if d.accumulated_steps == 0 {
867                    None // Nothing to persist.
868                } else {
869                    let memory_bytes = d.memory_bytes();
870                    if memory_bytes > MAX_SNAPSHOT_BYTES {
871                        tracing::warn!(
872                            "export_all: delta '{}' exceeds max snapshot size ({} > {}), skipping (loss)",
873                            id, memory_bytes, MAX_SNAPSHOT_BYTES
874                        );
875                        None
876                    } else {
877                        Some(Self::snapshot_state_dict(&d))
878                    }
879                }
880            };
881
882            let path = if let Some(state_dict) = snapshot_data {
883                let id_filename = Self::sanitize_filename(&id.to_string());
884                let rel_path = format!("adapters/.snapshots/{}.safetensors", id_filename);
885                match async {
886                    let bytes = serialize_state_dict_to_bytes(&state_dict)?;
887                    fs.write_file_chunked(&rel_path, &bytes)
888                        .await
889                        .map_err(|e| anyhow!("FsOps write_file failed: {}", e))?;
890                    Ok::<(), anyhow::Error>(())
891                }
892                .await
893                {
894                    Ok(()) => {
895                        tracing::info!("export_all: snapshotted delta '{}' -> {}", id, rel_path);
896                        Some(
897                            self.snapshots_dir
898                                .join(format!("{}.safetensors", id_filename)),
899                        )
900                    }
901                    Err(e) => {
902                        tracing::warn!(
903                            "export_all: failed to snapshot delta '{}': {} (loss)",
904                            id,
905                            e
906                        );
907                        None
908                    }
909                }
910            } else {
911                None
912            };
913
914            results.push((id, path));
915        }
916
917        Ok(results)
918    }
919
920    /// Clear all deltas and invalidate all dependent KV caches
921    pub fn clear_all(&self) {
922        // Invalidate KV caches for all active subjects
923        if let Some(kv_reg) = &self.kv_registry {
924            for entry in self.deltas.iter() {
925                kv_reg.invalidate_for_tenant(&entry.key().to_string());
926            }
927        }
928        self.deltas.clear();
929    }
930
931    /// List all tenant IDs
932    pub fn tenant_ids(&self) -> Vec<Subject> {
933        self.deltas.iter().map(|e| e.key().clone()).collect()
934    }
935}
936
937#[cfg(test)]
938#[allow(clippy::expect_used, clippy::unwrap_used)]
939mod tests {
940    use super::*;
941
942    fn test_module_dims() -> HashMap<String, (usize, usize)> {
943        let mut dims = HashMap::new();
944        dims.insert("q_proj".to_owned(), (512, 512));
945        dims.insert("v_proj".to_owned(), (512, 512));
946        dims
947    }
948
949    fn test_snapshots_dir() -> PathBuf {
950        std::env::temp_dir().join("delta_pool_test_snapshots")
951    }
952
953    #[test]
954    fn test_delta_pool_creation() {
955        let pool = DeltaPool::new(
956            TenantDeltaConfig::default(),
957            test_module_dims(),
958            Device::Cpu,
959            None,
960            test_snapshots_dir(),
961            None,
962            2,
963        );
964        assert_eq!(pool.tenant_count(), 0);
965    }
966
967    #[test]
968    fn test_get_or_create() {
969        let pool = DeltaPool::new(
970            TenantDeltaConfig::default(),
971            test_module_dims(),
972            Device::Cpu,
973            None,
974            test_snapshots_dir(),
975            None,
976            2,
977        );
978
979        let tid_a = Subject::new("tenant-a");
980        let tid_b = Subject::new("tenant-b");
981
982        let delta_a = pool.get_or_create(&tid_a).unwrap();
983        let delta_b = pool.get_or_create(&tid_b).unwrap();
984
985        assert_eq!(pool.tenant_count(), 2);
986
987        // Same tenant returns same delta
988        let delta_a2 = pool.get_or_create(&tid_a).unwrap();
989        assert!(Arc::ptr_eq(&delta_a, &delta_a2));
990
991        // Different tenants are different
992        assert!(!Arc::ptr_eq(&delta_a, &delta_b));
993    }
994
995    #[test]
996    fn test_tenant_isolation() {
997        let pool = DeltaPool::new(
998            TenantDeltaConfig::default(),
999            test_module_dims(),
1000            Device::Cpu,
1001            None,
1002            test_snapshots_dir(),
1003            None,
1004            2,
1005        );
1006
1007        let tid_a = Subject::new("tenant-a");
1008        let tid_b = Subject::new("tenant-b");
1009
1010        let delta_a = pool.get_or_create(&tid_a).unwrap();
1011        let delta_b = pool.get_or_create(&tid_b).unwrap();
1012
1013        // Modify A's delta
1014        {
1015            let mut a = delta_a.lock();
1016            a.accumulated_steps = 42;
1017        }
1018
1019        // B should be unaffected
1020        {
1021            let b = delta_b.lock();
1022            assert_eq!(b.accumulated_steps, 0);
1023        }
1024    }
1025
1026    #[test]
1027    fn test_lru_eviction() {
1028        let pool = DeltaPool::new(
1029            TenantDeltaConfig::default(),
1030            test_module_dims(),
1031            Device::Cpu,
1032            None,
1033            test_snapshots_dir(),
1034            None,
1035            2,
1036        );
1037
1038        let tid_a = Subject::new("tenant-a");
1039        let tid_b = Subject::new("tenant-b");
1040        let tid_c = Subject::new("tenant-c");
1041
1042        let _a = pool.get_or_create(&tid_a).unwrap();
1043        std::thread::sleep(std::time::Duration::from_millis(10));
1044        let _b = pool.get_or_create(&tid_b).unwrap();
1045        std::thread::sleep(std::time::Duration::from_millis(10));
1046        let _c = pool.get_or_create(&tid_c).unwrap();
1047
1048        assert_eq!(pool.tenant_count(), 3);
1049
1050        // Evict LRU should remove tenant-a (oldest)
1051        let evicted = pool.evict_lru();
1052        assert!(evicted.is_some());
1053        let (evicted_id, _, _snapshot_path) = evicted.unwrap();
1054        assert_eq!(evicted_id, tid_a);
1055        assert_eq!(pool.tenant_count(), 2);
1056    }
1057
1058    #[test]
1059    fn test_total_memory_usage() {
1060        let pool = DeltaPool::new(
1061            TenantDeltaConfig::default(),
1062            test_module_dims(),
1063            Device::Cpu,
1064            None,
1065            test_snapshots_dir(),
1066            None,
1067            2,
1068        );
1069
1070        let tid = Subject::new("tenant");
1071        let _delta = pool.get_or_create(&tid).unwrap();
1072
1073        let usage = pool.total_memory_usage();
1074        assert!(usage > 0, "Memory usage should be non-zero with a delta");
1075    }
1076
1077    // ===== #869 rehydration + drain-time export =====
1078
1079    /// Per-test unique snapshots dir under a fresh temp path (isolate file I/O).
1080    fn unique_snapshots_dir(tag: &str) -> PathBuf {
1081        let nanos = std::time::SystemTime::now()
1082            .duration_since(std::time::UNIX_EPOCH)
1083            .map(|d| d.as_nanos())
1084            .unwrap_or(0);
1085        std::env::temp_dir().join(format!("delta_pool_869_{}_{}", tag, nanos))
1086    }
1087
1088    fn pool_with_dir(dir: PathBuf) -> DeltaPool {
1089        DeltaPool::new(
1090            TenantDeltaConfig::default(),
1091            test_module_dims(),
1092            Device::Cpu,
1093            None,
1094            dir,
1095            None, // fs = None -> sync std::fs snapshot path (used by evict_lru + get_or_hydrate)
1096            2,
1097        )
1098    }
1099
1100    /// snapshot -> evict -> get_or_hydrate rehydrates weights + accumulated_steps.
1101    #[tokio::test]
1102    async fn test_hydrate_roundtrip_restores_state() {
1103        let dir = unique_snapshots_dir("roundtrip");
1104        let pool = pool_with_dir(dir.clone());
1105        let tid = Subject::new("tenant-hydrate");
1106
1107        // Train: mutate B and set accumulated_steps so eviction snapshots it.
1108        let expected_b: Tensor;
1109        {
1110            let delta_arc = pool.get_or_create(&tid).unwrap();
1111            let mut d = delta_arc.lock();
1112            let _guard = tch::no_grad_guard(); // B is a grad-requiring leaf var
1113            let b = d.lora_b.get_mut("0.q_proj").unwrap();
1114            let _ = b.uniform_(-0.5, 0.5);
1115            expected_b = b.copy();
1116            d.accumulated_steps = 17;
1117            d.request_count = 5;
1118        }
1119
1120        // Evict (sync path writes snapshot to `dir`), then hydrate a fresh miss.
1121        let evicted = pool.evict_lru();
1122        assert!(evicted.is_some());
1123        assert!(
1124            evicted.unwrap().2.is_some(),
1125            "eviction should write a snapshot"
1126        );
1127        assert_eq!(pool.tenant_count(), 0);
1128
1129        let delta_arc = pool.get_or_hydrate(&tid).await.unwrap();
1130        let d = delta_arc.lock();
1131        assert_eq!(d.accumulated_steps, 17, "accumulated_steps rehydrated");
1132        assert_eq!(d.request_count, 5, "request_count rehydrated");
1133        let max_diff: f64 = (&d.lora_b["0.q_proj"] - &expected_b)
1134            .abs()
1135            .max()
1136            .double_value(&[]);
1137        assert!(
1138            max_diff < 1e-6,
1139            "B weights rehydrated (max diff {})",
1140            max_diff
1141        );
1142
1143        drop(d);
1144        let _ = std::fs::remove_dir_all(&dir);
1145    }
1146
1147    /// A corrupt snapshot file is quarantined (.corrupt) and hydrate falls open to blank.
1148    #[tokio::test]
1149    async fn test_corrupt_snapshot_quarantined_and_blank() {
1150        let dir = unique_snapshots_dir("corrupt");
1151        std::fs::create_dir_all(&dir).unwrap();
1152        let pool = pool_with_dir(dir.clone());
1153        let tid = Subject::new("tenant-corrupt");
1154
1155        // Write garbage where the snapshot would live.
1156        let fname = DeltaPool::sanitize_filename(&tid.to_string());
1157        let snap_path = dir.join(format!("{}.safetensors", fname));
1158        std::fs::write(&snap_path, b"not a safetensors file").unwrap();
1159
1160        let delta_arc = pool.get_or_hydrate(&tid).await.unwrap();
1161        {
1162            let d = delta_arc.lock();
1163            // Blank: B is zeros, no accumulated steps.
1164            assert_eq!(d.accumulated_steps, 0);
1165            let b_norm: f64 = d.lora_b["0.q_proj"].norm().double_value(&[]);
1166            assert!(b_norm < 1e-8, "fell open to a blank delta");
1167        }
1168        assert!(!snap_path.exists(), "corrupt file moved aside");
1169        assert!(
1170            dir.join(format!("{}.safetensors.corrupt", fname)).exists(),
1171            "corrupt file quarantined"
1172        );
1173
1174        let _ = std::fs::remove_dir_all(&dir);
1175    }
1176
1177    /// An incompatible (wrong-shape) snapshot is rejected without panicking.
1178    #[tokio::test]
1179    async fn test_incompatible_shape_snapshot_falls_open() {
1180        let dir = unique_snapshots_dir("incompat");
1181        std::fs::create_dir_all(&dir).unwrap();
1182        let pool = pool_with_dir(dir.clone());
1183        let tid = Subject::new("tenant-incompat");
1184
1185        // Build a valid safetensors snapshot whose lora_a has the wrong shape.
1186        let mut state: HashMap<String, Tensor> = HashMap::new();
1187        state.insert(
1188            "0.q_proj.lora_a".to_owned(),
1189            Tensor::zeros([4, 8], (tch::Kind::Float, Device::Cpu)), // expected [8, 512]
1190        );
1191        let bytes = serialize_state_dict_to_bytes(&state).unwrap();
1192        let fname = DeltaPool::sanitize_filename(&tid.to_string());
1193        std::fs::write(dir.join(format!("{}.safetensors", fname)), &bytes).unwrap();
1194
1195        // Must not panic; falls open to blank and quarantines.
1196        let delta_arc = pool.get_or_hydrate(&tid).await.unwrap();
1197        {
1198            let d = delta_arc.lock();
1199            assert_eq!(d.lora_a["0.q_proj"].size(), vec![8, 512]);
1200        }
1201        assert!(
1202            dir.join(format!("{}.safetensors.corrupt", fname)).exists(),
1203            "incompatible snapshot quarantined"
1204        );
1205
1206        let _ = std::fs::remove_dir_all(&dir);
1207    }
1208
1209    /// Hydrating a subject with no snapshot yields a normal blank delta.
1210    #[tokio::test]
1211    async fn test_hydrate_no_snapshot_is_blank() {
1212        let dir = unique_snapshots_dir("nosnap");
1213        let pool = pool_with_dir(dir.clone());
1214        let tid = Subject::new("tenant-fresh");
1215
1216        let delta_arc = pool.get_or_hydrate(&tid).await.unwrap();
1217        let d = delta_arc.lock();
1218        assert_eq!(d.accumulated_steps, 0);
1219        assert_eq!(pool.tenant_count(), 1);
1220        drop(d);
1221        let _ = std::fs::remove_dir_all(&dir);
1222    }
1223
1224    /// export_all with no FsOps is a logged no-op (no worktree to write through).
1225    #[tokio::test]
1226    async fn test_export_all_no_fs_is_noop() {
1227        let dir = unique_snapshots_dir("exportnofs");
1228        let pool = pool_with_dir(dir.clone());
1229        let tid = Subject::new("tenant-x");
1230        {
1231            let d = pool.get_or_create(&tid).unwrap();
1232            d.lock().accumulated_steps = 3;
1233        }
1234        // fs = None -> export_all returns empty (documented FsOps-absent skip).
1235        let exported = pool.export_all().await.unwrap();
1236        assert!(exported.is_empty());
1237        // Delta stays resident (export never evicts).
1238        assert_eq!(pool.tenant_count(), 1);
1239        let _ = std::fs::remove_dir_all(&dir);
1240    }
1241}