Skip to main content

hyprstream_workers/runtime/
pool.rs

1//! Sandbox Pool for warm sandbox management
2//!
3//! Manages a pool of pre-warmed sandboxes for fast container startup.
4//! Delegates sandbox lifecycle to a pluggable `SandboxBackend`.
5
6use std::collections::{HashMap, VecDeque};
7use std::sync::atomic::{AtomicUsize, Ordering};
8use std::sync::Arc;
9
10use tokio::sync::Mutex;
11use uuid::Uuid;
12
13use crate::config::PoolConfig;
14use crate::error::{Result, WorkerError};
15
16use super::admission::{self, AdmissionTracker};
17use super::backend::SandboxBackend;
18use super::PodSandbox;
19use hyprstream_rpc_std::worker_client::{
20    KeyValue, LinuxContainerResources, PodSandboxConfig, PodSandboxState,
21};
22use hyprstream_vfs::Subject;
23
24/// Sandbox pool for warm sandbox management
25///
26/// Maintains:
27/// - Warm pool: Pre-created sandboxes ready for immediate use
28/// - Active sandboxes: Sandboxes currently in use by workloads
29///
30/// # Admission control (#525 P2)
31///
32/// `acquire` is gated by an [`AdmissionTracker`]: fail-closed identity,
33/// per-Subject/per-group quotas, a bounded wait-queue, and resource
34/// (cpu/memory/GPU) fit — see `runtime::admission` for the full design and
35/// its sign-off-flagged judgment calls.
36///
37/// # Backend placement (#525 P2, scope item 4)
38///
39/// A `SandboxPool` is bound to exactly **one** pre-resolved
40/// [`SandboxBackend`], selected once at construction via
41/// [`super::resolve_backend`] (the fail-closed inventory-registry seam,
42/// #507/#516) — never re-selected per request. Routing *individual*
43/// `acquire()` calls to different backends (e.g. GPU workloads to `kata`,
44/// CPU-only ones to `nspawn`, within the same pool) is **not implemented**:
45/// it would require restructuring `SandboxPool` into a router over multiple
46/// backend-specific pools, a larger architectural change than this ticket's
47/// scope. Flagged as a follow-up, not a silent gap — `acquire` never
48/// bypasses the fail-closed backend-resolution seam; it just doesn't have
49/// more than one backend to choose among.
50pub struct SandboxPool {
51    /// Pre-warmed sandboxes ready for use
52    warm_pool: Mutex<VecDeque<PodSandbox>>,
53
54    /// Active sandboxes (in use)
55    active: Mutex<HashMap<String, PodSandbox>>,
56
57    /// Pool configuration
58    config: PoolConfig,
59
60    /// Pluggable sandbox backend
61    backend: Arc<dyn SandboxBackend>,
62
63    /// Warm sandboxes currently being created (reserved replenishment slots).
64    ///
65    /// Counts creations that have been claimed by `replenish_warm_pool` but
66    /// have not yet been pushed onto `warm_pool`, so concurrent replenish tasks
67    /// can see in-flight work and avoid overshooting `warm_pool_size`.
68    in_flight: AtomicUsize,
69
70    /// Admission decision engine (#525 P2): fail-closed identity,
71    /// per-Subject/per-group quotas, bounded wait-queue, resource-aware fit.
72    admission: AdmissionTracker,
73}
74
75/// Verified task principal consumed by the backend's per-sandbox VFS setup.
76/// This is assigned by [`SandboxPool::acquire`], never accepted from a caller's
77/// untrusted sandbox annotations.
78const SUBJECT_ANNOTATION: &str = "hyprstream.io/subject";
79
80impl SandboxPool {
81    /// Create a new sandbox pool with a backend.
82    ///
83    /// Uses the fail-closed [`admission::DenyUnknownGroupValidator`] — the
84    /// production default until a membership-backed validator is wired: any
85    /// non-empty `hyprstream.io/group` selector is rejected at admission
86    /// (B′); the no-selector case is unaffected.
87    pub fn new(config: PoolConfig, backend: Arc<dyn SandboxBackend>) -> Self {
88        let admission = AdmissionTracker::new(config.admission.clone(), config.max_sandboxes);
89        Self::with_admission(config, backend, admission)
90    }
91
92    /// Create a pool with an explicit [`admission::GroupSelectorValidator`]
93    /// (e.g. one backed by a real membership source such as
94    /// `PlacementIndex::is_member`, or a test double). Mirrors
95    /// [`AdmissionTracker::with_group_validator`].
96    pub fn with_group_validator(
97        config: PoolConfig,
98        backend: Arc<dyn SandboxBackend>,
99        group_validator: Arc<dyn admission::GroupSelectorValidator>,
100    ) -> Self {
101        let admission = AdmissionTracker::with_group_validator(
102            config.admission.clone(),
103            config.max_sandboxes,
104            group_validator,
105        );
106        Self::with_admission(config, backend, admission)
107    }
108
109    fn with_admission(
110        config: PoolConfig,
111        backend: Arc<dyn SandboxBackend>,
112        admission: AdmissionTracker,
113    ) -> Self {
114        Self {
115            warm_pool: Mutex::new(VecDeque::new()),
116            active: Mutex::new(HashMap::new()),
117            config,
118            backend,
119            in_flight: AtomicUsize::new(0),
120            admission,
121        }
122    }
123
124    /// Get a reference to the backend
125    pub fn backend(&self) -> &dyn SandboxBackend {
126        self.backend.as_ref()
127    }
128
129    /// Initialize the warm pool with pre-created sandboxes
130    pub async fn initialize(&self) -> Result<()> {
131        // Let the backend do one-time setup
132        self.backend.initialize(&self.config).await?;
133
134        let mut warm = self.warm_pool.lock().await;
135
136        for _ in 0..self.config.warm_pool_size {
137            let sandbox = self.create_warm_sandbox().await?;
138            warm.push_back(sandbox);
139        }
140
141        tracing::info!(
142            backend = self.backend.backend_type(),
143            "Initialized warm pool with {} sandboxes",
144            self.config.warm_pool_size
145        );
146
147        Ok(())
148    }
149
150    /// Acquire a sandbox from the pool
151    ///
152    /// Prefers warm sandboxes, creates new if none available.
153    ///
154    /// # Admission control (#525 P2)
155    ///
156    /// `subject` must be a verified, non-anonymous [`Subject`] — an
157    /// unauthenticated caller is rejected fail-closed before any capacity or
158    /// quota bookkeeping happens. Admission (capacity + per-Subject/per-group
159    /// quota + cpu/memory/GPU fit) is reserved atomically under one lock
160    /// (see `runtime::admission`) before this method ever touches the warm
161    /// pool or the backend — never check-then-place.
162    ///
163    /// A request that only capacity (not quota) blocks may wait, bounded by
164    /// `PoolConfig::admission`'s queue settings; see [`WorkerError::QueueFull`]
165    /// / [`WorkerError::QueueTimeout`] / [`WorkerError::AdmissionDenied`] for
166    /// the distinct failure modes.
167    pub async fn acquire(
168        self: &Arc<Self>,
169        subject: &Subject,
170        config: &PodSandboxConfig,
171    ) -> Result<String> {
172        let demand = admission::derive_demand(config);
173        let group_selector = admission::derive_group_selector(config);
174        let reservation = self
175            .admission
176            .reserve(subject, group_selector.as_deref(), demand)
177            .await?;
178
179        // The backend constructs the sandbox's namespace during startup. Put
180        // the verified admission principal in the configuration *before* that
181        // point so Kata/OCI/nspawn all serve the task VFS under the same
182        // Subject. A supplied annotation is overwritten rather than trusted.
183        let mut task_config = config.clone();
184        task_config
185            .annotations
186            .retain(|annotation| annotation.key != SUBJECT_ANNOTATION);
187        task_config.annotations.push(KeyValue {
188            key: SUBJECT_ANNOTATION.to_owned(),
189            value: subject.to_string(),
190        });
191
192        let obtained = self.obtain_sandbox(&task_config).await;
193        let mut sandbox = match obtained {
194            Ok(sandbox) => sandbox,
195            Err(e) => {
196                // Admission succeeded but no usable sandbox resulted (backend
197                // create/reconfigure failure) — give the reservation back
198                // rather than leaking capacity/quota.
199                self.admission.rollback(&reservation).await;
200                return Err(e);
201            }
202        };
203
204        sandbox.reservation = Some(reservation);
205        let sandbox_id = sandbox.id.clone();
206
207        // Move to active
208        self.active.lock().await.insert(sandbox_id.clone(), sandbox);
209
210        // Replenish warm pool in background
211        Self::replenish_warm_pool(self.clone());
212
213        Ok(sandbox_id)
214    }
215
216    /// Pop a warm sandbox (reconfiguring it for `config`, including a resize
217    /// via `update_resources` when its previously-applied resources don't
218    /// match — the #519 fix) or cold-create one. Does not touch admission
219    /// bookkeeping; the caller (`acquire`) owns reserve/rollback around this.
220    async fn obtain_sandbox(&self, config: &PodSandboxConfig) -> Result<PodSandbox> {
221        let mut warm = self.warm_pool.lock().await;
222        if let Some(mut sandbox) = warm.pop_front() {
223            drop(warm);
224            // Configure the warm sandbox with the provided config
225            sandbox.metadata = config.metadata.clone();
226            sandbox.labels = config.labels.clone();
227            sandbox.annotations = config.annotations.clone();
228            sandbox.state = PodSandboxState::SandboxReady;
229
230            // #519 fix: a warm sandbox may have been booted at a different
231            // size (or the pool's default) than what THIS caller asked for.
232            // Reconfiguring metadata/labels/annotations above must not
233            // silently leave a mismatched resource envelope in place.
234            let requested = config.linux.resources.clone();
235            if requested != LinuxContainerResources::default()
236                && requested != sandbox.applied_resources
237            {
238                tracing::debug!(
239                    sandbox_id = %sandbox.id,
240                    "warm sandbox resource mismatch on reuse — resizing (#519)"
241                );
242                // F2: this sandbox has already been popped off `warm_pool`, so
243                // if the resize fails the value is about to be dropped and its
244                // VM/container would be orphaned (still running, owned by
245                // nobody, warm pool permanently shrunk). Best-effort destroy it
246                // — its actual size is now unknown/untrusted — before
247                // propagating. `acquire` rolls back the admission reservation
248                // on the returned error, so capacity/quota are not leaked.
249                if let Err(resize_err) = self.backend.update_resources(&sandbox, &requested).await {
250                    if let Err(destroy_err) = self.backend.destroy(&sandbox).await {
251                        tracing::warn!(
252                            sandbox_id = %sandbox.id,
253                            error = %destroy_err,
254                            "failed to destroy warm sandbox after resize failure — possible orphan"
255                        );
256                    }
257                    return Err(resize_err);
258                }
259                sandbox.applied_resources = requested;
260            }
261            Ok(sandbox)
262        } else {
263            drop(warm);
264            self.create_sandbox(config).await
265        }
266    }
267
268    /// Release a sandbox back to the pool or destroy it
269    pub async fn release(self: &Arc<Self>, sandbox_id: &str) -> Result<()> {
270        let mut active = self.active.lock().await;
271
272        let sandbox = active
273            .remove(sandbox_id)
274            .ok_or_else(|| WorkerError::SandboxNotFound(sandbox_id.to_owned()))?;
275        drop(active);
276
277        // F5: give back the admission reservation only *after* the sandbox's
278        // resources are actually released, i.e. after `stop` succeeds. Waking a
279        // queued waiter before `stop` completes would let it cold-create while
280        // this sandbox's VM is still physically holding cpu/memory/GPU —
281        // transiently over-committing the envelope (`max_sandboxes + 1`). On
282        // stop failure, best-effort destroy so we don't orphan the sandbox
283        // (mirroring the shutdown path's tolerance), then still release the
284        // reservation so capacity/quota aren't permanently leaked, and
285        // propagate the error.
286        let release_reservation = || async {
287            if let Some(reservation) = sandbox.reservation.clone() {
288                self.admission.release(&reservation).await;
289            }
290        };
291
292        if let Err(stop_err) = self.backend.stop(&sandbox).await {
293            tracing::warn!(
294                sandbox_id = %sandbox.id,
295                error = %stop_err,
296                "stop failed on release — destroying sandbox to avoid an orphan"
297            );
298            if let Err(destroy_err) = self.backend.destroy(&sandbox).await {
299                tracing::warn!(
300                    sandbox_id = %sandbox.id,
301                    error = %destroy_err,
302                    "destroy also failed after stop failure — possible orphan"
303                );
304            }
305            release_reservation().await;
306            return Err(stop_err);
307        }
308
309        // Stop succeeded — the resource envelope is genuinely free now.
310        release_reservation().await;
311
312        // Decide whether to return to warm pool or destroy
313        let warm = self.warm_pool.lock().await;
314        if warm.len() < self.config.warm_pool_size {
315            drop(warm);
316            // Reset and return to warm pool
317            let reset_sandbox = self.reset_sandbox(sandbox).await?;
318            match reset_sandbox {
319                Some(s) => self.warm_pool.lock().await.push_back(s),
320                // Backend reported the sandbox as non-reusable and destroyed it,
321                // so the warm pool may now be below target — replenish it.
322                None => Self::replenish_warm_pool(self.clone()),
323            }
324        } else {
325            drop(warm);
326            // Destroy the sandbox
327            self.backend.destroy(&sandbox).await?;
328        }
329
330        Ok(())
331    }
332
333    /// Get a sandbox by ID
334    pub async fn get(&self, sandbox_id: &str) -> Option<PodSandbox> {
335        self.active.lock().await.get(sandbox_id).cloned()
336    }
337
338    /// List all active sandboxes
339    pub async fn list_active(&self) -> Vec<PodSandbox> {
340        self.active.lock().await.values().cloned().collect()
341    }
342
343    /// Get pool statistics
344    pub async fn stats(&self) -> PoolStats {
345        let warm_count = self.warm_pool.lock().await.len();
346        let active_count = self.active.lock().await.len();
347
348        PoolStats {
349            warm_count,
350            active_count,
351            max_sandboxes: self.config.max_sandboxes,
352            warm_pool_target: self.config.warm_pool_size,
353        }
354    }
355
356    /// Shutdown the pool, destroying all sandboxes
357    pub async fn shutdown(&self) -> Result<()> {
358        tracing::info!("Shutting down sandbox pool");
359
360        // Destroy warm sandboxes
361        let mut warm = self.warm_pool.lock().await;
362        while let Some(sandbox) = warm.pop_front() {
363            if let Err(e) = self.backend.destroy(&sandbox).await {
364                tracing::warn!("Error destroying warm sandbox: {}", e);
365            }
366        }
367        drop(warm);
368
369        // Destroy active sandboxes
370        let active: Vec<_> = {
371            let mut active = self.active.lock().await;
372            active.drain().map(|(_, s)| s).collect()
373        };
374
375        for sandbox in active {
376            if let Err(e) = self.backend.stop(&sandbox).await {
377                tracing::warn!("Error stopping sandbox {}: {}", sandbox.id, e);
378            }
379            if let Err(e) = self.backend.destroy(&sandbox).await {
380                tracing::warn!("Error destroying sandbox: {}", e);
381            }
382        }
383
384        tracing::info!("Sandbox pool shutdown complete");
385        Ok(())
386    }
387
388    // ─────────────────────────────────────────────────────────────────────────
389    // Internal methods
390    // ─────────────────────────────────────────────────────────────────────────
391
392    /// Create a warm sandbox (minimal configuration)
393    async fn create_warm_sandbox(&self) -> Result<PodSandbox> {
394        let id = format!("warm-{}", Uuid::new_v4());
395        let config = PodSandboxConfig::default();
396        let sandbox_path = self.config.runtime_dir.join(&id);
397
398        let mut sandbox = PodSandbox::new(id, &config, sandbox_path);
399
400        // Start via backend
401        let annotations = HashMap::new();
402        let handle = self
403            .backend
404            .start(&mut sandbox, &config, &self.config, &annotations)
405            .await?;
406        sandbox.set_backend_handle(handle);
407
408        Ok(sandbox)
409    }
410
411    /// Create a sandbox with full configuration
412    async fn create_sandbox(&self, config: &PodSandboxConfig) -> Result<PodSandbox> {
413        let id = Uuid::new_v4().to_string();
414        let sandbox_path = self.config.runtime_dir.join(&id);
415
416        let mut sandbox = PodSandbox::new(id, config, sandbox_path);
417
418        // Convert annotations for backend
419        let annotations: HashMap<String, String> = sandbox
420            .annotations
421            .iter()
422            .map(|kv| (kv.key.clone(), kv.value.clone()))
423            .collect();
424
425        // Start via backend
426        let handle = self
427            .backend
428            .start(&mut sandbox, config, &self.config, &annotations)
429            .await?;
430        sandbox.set_backend_handle(handle);
431
432        sandbox.state = PodSandboxState::SandboxReady;
433        // A cold-created sandbox is started with this exact config, so its
434        // resource envelope is authoritatively `config.linux.resources`.
435        sandbox.applied_resources = config.linux.resources.clone();
436
437        Ok(sandbox)
438    }
439
440    /// Reset a sandbox for reuse in warm pool
441    ///
442    /// Returns `Some(sandbox)` if the backend supports in-place reset,
443    /// `None` if the sandbox is ephemeral and cannot be reused.
444    async fn reset_sandbox(&self, mut sandbox: PodSandbox) -> Result<Option<PodSandbox>> {
445        tracing::debug!(sandbox_id = %sandbox.id, "Resetting sandbox for warm pool");
446
447        // Clear metadata and labels
448        sandbox.metadata = Default::default();
449        sandbox.labels.clear();
450        sandbox.annotations.clear();
451        sandbox.state = PodSandboxState::SandboxNotReady;
452        sandbox.image_id = None;
453        // Admission bookkeeping ends with this acquisition; already released
454        // above in `release()`. Deliberately NOT resetting `applied_resources`
455        // here — see its doc on `PodSandbox` (#519): it must keep reflecting
456        // the sandbox's actual physical size for the next `acquire()`'s
457        // mismatch check.
458        sandbox.reservation = None;
459
460        // Let the backend reset its handle
461        let reusable = match self.backend.reset(&mut sandbox).await {
462            Ok(reusable) => reusable,
463            Err(reset_err) => {
464                tracing::warn!(
465                    sandbox_id = %sandbox.id,
466                    error = %reset_err,
467                    "reset failed on release — destroying sandbox to avoid an orphan"
468                );
469                if let Err(destroy_err) = self.backend.destroy(&sandbox).await {
470                    tracing::warn!(
471                        sandbox_id = %sandbox.id,
472                        error = %destroy_err,
473                        "destroy also failed after reset failure — possible orphan"
474                    );
475                }
476                return Err(reset_err);
477            }
478        };
479        if reusable {
480            Ok(Some(sandbox))
481        } else {
482            // Backend says sandbox is ephemeral — destroy it
483            self.backend.destroy(&sandbox).await?;
484            Ok(None)
485        }
486    }
487
488    /// Replenish warm pool in background when it drops below target.
489    ///
490    /// Concurrency-safe: each invocation atomically claims the slots it intends
491    /// to fill (the deficit between `warm_pool_size` and the warm count plus any
492    /// in-flight creations) while holding the warm-pool lock, then creates that
493    /// many sandboxes. Reserving slots up front prevents the TOCTOU race where N
494    /// concurrent tasks each observe the same deficit and overshoot the target,
495    /// and creating the full deficit (rather than a single sandbox) lets the
496    /// pool recover quickly after a burst drains it.
497    fn replenish_warm_pool(pool: Arc<Self>) {
498        let target = pool.config.warm_pool_size;
499        tokio::spawn(async move {
500            // Reserve slots while holding the lock so concurrent replenish tasks
501            // observe each other's claims via `in_flight`.
502            let to_create = {
503                let warm = pool.warm_pool.lock().await;
504                let in_flight = pool.in_flight.load(Ordering::SeqCst);
505                let deficit = target.saturating_sub(warm.len() + in_flight);
506                pool.in_flight.fetch_add(deficit, Ordering::SeqCst);
507                deficit
508            };
509
510            for _ in 0..to_create {
511                match pool.create_warm_sandbox().await {
512                    Ok(sandbox) => {
513                        pool.warm_pool.lock().await.push_back(sandbox);
514                        tracing::debug!(target, "Warm pool replenished");
515                    }
516                    Err(e) => {
517                        tracing::warn!(error = %e, "Failed to replenish warm pool");
518                    }
519                }
520                // Release the reserved slot once this creation completes,
521                // whether it succeeded or failed.
522                pool.in_flight.fetch_sub(1, Ordering::SeqCst);
523            }
524        });
525    }
526}
527
528/// Pool statistics
529#[derive(Debug, Clone)]
530pub struct PoolStats {
531    /// Number of warm sandboxes ready
532    pub warm_count: usize,
533    /// Number of active sandboxes in use
534    pub active_count: usize,
535    /// Maximum allowed sandboxes
536    pub max_sandboxes: usize,
537    /// Target warm pool size
538    pub warm_pool_target: usize,
539}
540
541// The pool tests build a KataBackend + RafsStore, so they only run under `kata`.
542#[cfg(all(test, feature = "kata"))]
543mod tests {
544    use super::*;
545    use crate::config::{ImageConfig, PoolConfig};
546    use crate::image::RafsStore;
547    use crate::runtime::kata_backend::KataBackend;
548    use hyprstream_rpc::auth::mac::{ObjectRef, SecurityContext};
549    use tempfile::TempDir;
550
551    struct FixtureAccessDecider;
552
553    impl hyprstream_9p::AccessDecider for FixtureAccessDecider {
554        fn check(
555            &self,
556            _ctx: &SecurityContext,
557            _object: ObjectRef<'_>,
558            _action: hyprstream_9p::Action,
559        ) -> bool {
560            true
561        }
562
563        fn audit_denial(
564            &self,
565            _ctx: &SecurityContext,
566            _object: ObjectRef<'_>,
567            _object_label: Option<hyprstream_rpc::auth::mac::SecurityLabel>,
568            _action: hyprstream_9p::Action,
569            _reason: hyprstream_9p::ReferenceMonitorDenyReason,
570        ) {
571        }
572    }
573
574    /// Create a test pool with Kata backend and minimal configuration
575    async fn create_test_pool(
576        max_sandboxes: usize,
577        warm_pool_size: usize,
578    ) -> Result<(Arc<SandboxPool>, TempDir)> {
579        let temp_dir = TempDir::new()?;
580        let base_path = temp_dir.path();
581
582        let pool_config = PoolConfig {
583            max_sandboxes,
584            warm_pool_size,
585            runtime_dir: base_path.join("runtime"),
586            ..Default::default()
587        };
588
589        let image_config = ImageConfig {
590            blobs_dir: base_path.join("blobs"),
591            bootstrap_dir: base_path.join("bootstrap"),
592            refs_dir: base_path.join("refs"),
593            cache_dir: base_path.join("cache"),
594            runtime_dir: base_path.join("nydus-runtime"),
595            ..Default::default()
596        };
597
598        // Create the required directories
599        std::fs::create_dir_all(&image_config.blobs_dir)?;
600        std::fs::create_dir_all(&image_config.bootstrap_dir)?;
601        std::fs::create_dir_all(&image_config.refs_dir)?;
602        std::fs::create_dir_all(&image_config.cache_dir)?;
603        std::fs::create_dir_all(&pool_config.runtime_dir)?;
604
605        let rafs_store = Arc::new(RafsStore::new(image_config.clone())?);
606        let backend: Arc<dyn SandboxBackend> = Arc::new(KataBackend::new(
607            image_config,
608            rafs_store,
609            Arc::new(FixtureAccessDecider),
610            None,
611        ));
612        let pool = Arc::new(SandboxPool::new(pool_config, backend));
613
614        Ok((pool, temp_dir))
615    }
616
617    #[tokio::test]
618    async fn test_pool_acquire_release() -> Result<()> {
619        let (pool, _temp_dir) = create_test_pool(5, 0).await?;
620
621        let sandbox_config = PodSandboxConfig::default();
622        let subject = Subject::new("test-user");
623
624        // Note: This will fail in CI since cloud-hypervisor isn't installed
625        let result = pool.acquire(&subject, &sandbox_config).await;
626
627        match result {
628            Ok(sandbox_id) => {
629                assert!(pool.get(&sandbox_id).await.is_some());
630                let stats = pool.stats().await;
631                assert_eq!(stats.active_count, 1);
632
633                pool.release(&sandbox_id).await?;
634                assert!(pool.get(&sandbox_id).await.is_none());
635            }
636            Err(WorkerError::VmStartFailed(_)) => {
637                // Expected when cloud-hypervisor is not installed
638                let stats = pool.stats().await;
639                assert_eq!(stats.active_count, 0);
640            }
641            Err(e) => panic!("Unexpected error: {e:?}"),
642        }
643        Ok(())
644    }
645
646    #[tokio::test]
647    async fn test_pool_stats() -> Result<()> {
648        let (pool, _temp_dir) = create_test_pool(10, 2).await?;
649
650        let stats = pool.stats().await;
651        assert_eq!(stats.max_sandboxes, 10);
652        assert_eq!(stats.warm_pool_target, 2);
653        assert_eq!(stats.active_count, 0);
654        assert_eq!(stats.warm_count, 0);
655        Ok(())
656    }
657
658    #[tokio::test]
659    async fn test_pool_shutdown() -> Result<()> {
660        let (pool, _temp_dir) = create_test_pool(5, 0).await?;
661
662        // Shutdown should succeed even with no sandboxes
663        pool.shutdown().await?;
664
665        let stats = pool.stats().await;
666        assert_eq!(stats.active_count, 0);
667        assert_eq!(stats.warm_count, 0);
668        Ok(())
669    }
670}
671
672// #525 P2 — admission/queue/resource-aware placement tests. These use a
673// minimal in-memory `FakeBackend` (no kata/nspawn/wasm dependency), so they
674// run in the default feature set (unlike the `kata-vm`-gated suite above,
675// which needs a real cloud-hypervisor).
676#[cfg(test)]
677#[allow(clippy::unwrap_used, clippy::expect_used)]
678mod admission_tests {
679    use super::*;
680    use crate::runtime::backend::SandboxHandle;
681    use hyprstream_rpc_std::worker_client::KeyValue;
682    use parking_lot::Mutex as StdMutex;
683    use std::any::Any;
684    use std::collections::HashMap as StdHashMap;
685    use std::sync::atomic::AtomicBool as StdAtomicBool;
686    use std::sync::atomic::AtomicUsize as StdAtomicUsize;
687
688    #[derive(Debug)]
689    struct FakeHandle;
690    impl SandboxHandle for FakeHandle {
691        fn as_any(&self) -> &dyn Any {
692            self
693        }
694    }
695
696    /// Minimal in-memory fake backend. `reset()` returns `true` (reusable,
697    /// mirroring Kata) so warm-pool reuse — and the #519 resize path — is
698    /// actually exercised. Tracks concurrently-active starts so tests can
699    /// assert the admission layer never over-places (the #489/#519 TOCTOU
700    /// lesson), and records every `update_resources` call so the #519
701    /// regression test can assert a warm-sandbox resize actually happened.
702    #[derive(Default)]
703    struct FakeBackend {
704        current: StdAtomicUsize,
705        peak: StdAtomicUsize,
706        applied: StdMutex<StdHashMap<String, LinuxContainerResources>>,
707        /// When set, `update_resources` fails (F2 resize-leak test).
708        fail_resize: StdAtomicBool,
709        /// When set, `stop` fails (F5 release-ordering test).
710        fail_stop: StdAtomicBool,
711        /// When set, `reset` fails (reset-failure orphan-avoidance test).
712        fail_reset: StdAtomicBool,
713        /// Ids passed to `destroy`, in call order (F2/F5 orphan-avoidance).
714        destroyed: StdMutex<Vec<String>>,
715    }
716
717    impl FakeBackend {
718        fn bump(&self) {
719            let cur = self.current.fetch_add(1, Ordering::SeqCst) + 1;
720            self.peak.fetch_max(cur, Ordering::SeqCst);
721        }
722        fn was_destroyed(&self, id: &str) -> bool {
723            self.destroyed.lock().iter().any(|d| d == id)
724        }
725    }
726
727    #[async_trait::async_trait]
728    impl SandboxBackend for FakeBackend {
729        fn backend_type(&self) -> &'static str {
730            "fake"
731        }
732        fn is_available(&self) -> bool {
733            true
734        }
735        async fn initialize(&self, _config: &crate::config::PoolConfig) -> Result<()> {
736            Ok(())
737        }
738        async fn start(
739            &self,
740            _sandbox: &mut PodSandbox,
741            _config: &PodSandboxConfig,
742            _pool_config: &crate::config::PoolConfig,
743            _annotations: &HashMap<String, String>,
744        ) -> Result<Arc<dyn SandboxHandle>> {
745            self.bump();
746            Ok(Arc::new(FakeHandle))
747        }
748        async fn stop(&self, _sandbox: &PodSandbox) -> Result<()> {
749            if self.fail_stop.load(Ordering::SeqCst) {
750                // Did not stop → do not decrement the live count.
751                return Err(WorkerError::SandboxTimeout {
752                    operation: "stop".to_owned(),
753                    timeout_secs: 0,
754                });
755            }
756            self.current.fetch_sub(1, Ordering::SeqCst);
757            Ok(())
758        }
759        async fn destroy(&self, sandbox: &PodSandbox) -> Result<()> {
760            self.destroyed.lock().push(sandbox.id.clone());
761            Ok(())
762        }
763        async fn reset(&self, _sandbox: &mut PodSandbox) -> Result<bool> {
764            if self.fail_reset.load(Ordering::SeqCst) {
765                return Err(WorkerError::SandboxTimeout {
766                    operation: "reset".to_owned(),
767                    timeout_secs: 0,
768                });
769            }
770            // Reusable (Kata-like) — required to exercise the warm-pool /
771            // #519 resize path below.
772            Ok(true)
773        }
774        async fn get_pids(&self, _sandbox: &PodSandbox) -> Result<Vec<u32>> {
775            Ok(vec![])
776        }
777        fn supports_exec(&self) -> bool {
778            false
779        }
780        async fn exec_sync(
781            &self,
782            _sandbox: &PodSandbox,
783            _command: &[String],
784            _timeout_secs: u64,
785        ) -> Result<(i32, Vec<u8>, Vec<u8>)> {
786            Err(WorkerError::ExecFailed("not supported".into()))
787        }
788        async fn update_resources(
789            &self,
790            sandbox: &PodSandbox,
791            resources: &LinuxContainerResources,
792        ) -> Result<()> {
793            if self.fail_resize.load(Ordering::SeqCst) {
794                return Err(WorkerError::SandboxTimeout {
795                    operation: "update_resources".to_owned(),
796                    timeout_secs: 0,
797                });
798            }
799            self.applied
800                .lock()
801                .insert(sandbox.id.clone(), resources.clone());
802            Ok(())
803        }
804    }
805
806    fn make_pool_with(
807        max_sandboxes: usize,
808        warm_pool_size: usize,
809    ) -> (Arc<SandboxPool>, Arc<FakeBackend>) {
810        let fake = Arc::new(FakeBackend::default());
811        let backend: Arc<dyn SandboxBackend> = fake.clone();
812        let config = PoolConfig {
813            max_sandboxes,
814            warm_pool_size,
815            ..Default::default()
816        };
817        (Arc::new(SandboxPool::new(config, backend)), fake)
818    }
819
820    fn make_pool_with_admission(
821        max_sandboxes: usize,
822        warm_pool_size: usize,
823        admission: crate::runtime::AdmissionConfig,
824    ) -> (Arc<SandboxPool>, Arc<FakeBackend>) {
825        let fake = Arc::new(FakeBackend::default());
826        let backend: Arc<dyn SandboxBackend> = fake.clone();
827        let config = PoolConfig {
828            max_sandboxes,
829            warm_pool_size,
830            admission,
831            ..Default::default()
832        };
833        (Arc::new(SandboxPool::new(config, backend)), fake)
834    }
835
836    // ── Solo/no-regression baseline ──────────────────────────────────────
837
838    #[tokio::test]
839    async fn solo_acquire_release_unchanged() {
840        let (pool, _fake) = make_pool_with(5, 0);
841        let subject = Subject::new("alice");
842        let sandbox_config = PodSandboxConfig::default();
843
844        let id = pool
845            .acquire(&subject, &sandbox_config)
846            .await
847            .expect("solo acquire must succeed");
848        assert!(pool.get(&id).await.is_some());
849        assert_eq!(pool.stats().await.active_count, 1);
850
851        pool.release(&id).await.expect("release must succeed");
852        assert!(pool.get(&id).await.is_none());
853        assert_eq!(pool.stats().await.active_count, 0);
854    }
855
856    #[tokio::test]
857    async fn solo_capacity_exhausted_without_queue_room_rejects_clearly() {
858        let cfg = crate::runtime::AdmissionConfig {
859            queue_capacity: 0,
860            ..Default::default()
861        };
862        let (pool, _fake) = make_pool_with_admission(1, 0, cfg);
863        let subject = Subject::new("alice");
864        let sandbox_config = PodSandboxConfig::default();
865
866        let _id = pool
867            .acquire(&subject, &sandbox_config)
868            .await
869            .expect("first acquire admits");
870        let err = pool.acquire(&subject, &sandbox_config).await.unwrap_err();
871        assert!(matches!(err, WorkerError::QueueFull { .. }), "got: {err:?}");
872    }
873
874    // ── Fail-closed identity ─────────────────────────────────────────────
875
876    #[tokio::test]
877    async fn anonymous_caller_rejected_fail_closed() {
878        let (pool, _fake) = make_pool_with(5, 0);
879        let sandbox_config = PodSandboxConfig::default();
880        let err = pool
881            .acquire(&Subject::anonymous(), &sandbox_config)
882            .await
883            .unwrap_err();
884        assert!(
885            matches!(err, WorkerError::Unauthorized { .. }),
886            "got: {err:?}"
887        );
888        // Must not have consumed any capacity.
889        assert_eq!(pool.stats().await.active_count, 0);
890    }
891
892    // ── Per-Subject / per-group quota ────────────────────────────────────
893
894    #[tokio::test]
895    async fn per_subject_quota_rejects_over_quota_caller() {
896        let cfg = crate::runtime::AdmissionConfig {
897            max_per_subject: Some(1),
898            ..Default::default()
899        };
900        let (pool, _fake) = make_pool_with_admission(10, 0, cfg);
901        let alice = Subject::new("alice");
902        let sandbox_config = PodSandboxConfig::default();
903
904        let _id = pool
905            .acquire(&alice, &sandbox_config)
906            .await
907            .expect("first request admitted");
908        let err = pool.acquire(&alice, &sandbox_config).await.unwrap_err();
909        assert!(
910            matches!(err, WorkerError::AdmissionDenied { .. }),
911            "got: {err:?}"
912        );
913    }
914
915    // ── GPU-aware placement (fit over the #628 scheduling substrate) ─────
916
917    #[tokio::test]
918    async fn gpu_request_placed_only_when_capacity_declared() {
919        let cfg = crate::runtime::AdmissionConfig {
920            gpu_total: 0,
921            ..Default::default()
922        };
923        let (pool, _fake) = make_pool_with_admission(10, 0, cfg);
924        let subject = Subject::new("alice");
925        let mut sandbox_config = PodSandboxConfig::default();
926        sandbox_config.annotations.push(KeyValue {
927            key: crate::runtime::ANN_GPU_REQUEST.to_owned(),
928            value: "1".to_owned(),
929        });
930
931        // No GPU capacity declared → fail-closed reject, never a silent
932        // CPU-only placement. F1: this is *infeasible* (exceeds the node total),
933        // so it must reject immediately rather than queue out the timeout.
934        let start = std::time::Instant::now();
935        let err = pool.acquire(&subject, &sandbox_config).await.unwrap_err();
936        assert!(
937            matches!(err, WorkerError::AdmissionInfeasible { .. }),
938            "got: {err:?}"
939        );
940        assert!(
941            start.elapsed() < std::time::Duration::from_secs(1),
942            "infeasible GPU request must reject immediately, not wait the queue timeout"
943        );
944    }
945
946    #[tokio::test]
947    async fn gpu_request_placed_when_capacity_available() {
948        let cfg = crate::runtime::AdmissionConfig {
949            gpu_total: 1,
950            ..Default::default()
951        };
952        let (pool, _fake) = make_pool_with_admission(10, 0, cfg);
953        let subject = Subject::new("alice");
954        let mut sandbox_config = PodSandboxConfig::default();
955        sandbox_config.annotations.push(KeyValue {
956            key: crate::runtime::ANN_GPU_REQUEST.to_owned(),
957            value: "1".to_owned(),
958        });
959
960        let id = pool
961            .acquire(&subject, &sandbox_config)
962            .await
963            .expect("gpu request should be admitted");
964        assert!(pool.get(&id).await.is_some());
965    }
966
967    #[tokio::test]
968    async fn cpu_and_memory_request_honored_within_capacity() {
969        let cfg = crate::runtime::AdmissionConfig {
970            cpu_millis_total: Some(1000),
971            memory_bytes_total: Some(1024 * 1024 * 1024),
972            ..Default::default()
973        };
974        let (pool, _fake) = make_pool_with_admission(10, 0, cfg);
975        let subject = Subject::new("alice");
976        let mut sandbox_config = PodSandboxConfig::default();
977        sandbox_config.linux.resources.cpu_period = 100_000;
978        sandbox_config.linux.resources.cpu_quota = 50_000; // 500m
979        sandbox_config.linux.resources.memory_limit_in_bytes = 512 * 1024 * 1024;
980
981        let id = pool
982            .acquire(&subject, &sandbox_config)
983            .await
984            .expect("fits within declared capacity");
985        assert!(pool.get(&id).await.is_some());
986    }
987
988    #[tokio::test]
989    async fn memory_request_exceeding_capacity_is_not_silently_placed() {
990        let cfg = crate::runtime::AdmissionConfig {
991            memory_bytes_total: Some(256 * 1024 * 1024),
992            queue_capacity: 0,
993            ..Default::default()
994        };
995        let (pool, _fake) = make_pool_with_admission(10, 0, cfg);
996        let subject = Subject::new("alice");
997        let mut sandbox_config = PodSandboxConfig::default();
998        sandbox_config.linux.resources.memory_limit_in_bytes = 512 * 1024 * 1024; // exceeds declared 256Mi
999
1000        // F1: demand above the declared *total* is infeasible (no release can
1001        // ever satisfy it), so it rejects immediately rather than queueing —
1002        // still never a silent placement.
1003        let err = pool.acquire(&subject, &sandbox_config).await.unwrap_err();
1004        assert!(
1005            matches!(err, WorkerError::AdmissionInfeasible { .. }),
1006            "got: {err:?}"
1007        );
1008    }
1009
1010    // ── #519 regression: warm-sandbox reuse must resize, never silently
1011    //    hand out the wrong size ──────────────────────────────────────────
1012
1013    #[tokio::test]
1014    async fn warm_sandbox_reuse_resizes_mismatched_resources() {
1015        let (pool, fake) = make_pool_with(5, 1);
1016        pool.initialize().await.expect("prewarm one sandbox");
1017        assert_eq!(pool.stats().await.warm_count, 1);
1018
1019        let subject = Subject::new("alice");
1020        let mut sandbox_config = PodSandboxConfig::default();
1021        sandbox_config.linux.resources.cpu_period = 100_000;
1022        sandbox_config.linux.resources.cpu_quota = 75_000;
1023        sandbox_config.linux.resources.memory_limit_in_bytes = 256 * 1024 * 1024;
1024
1025        // The one warm sandbox was booted with `PodSandboxConfig::default()`
1026        // (all-zero resources) — this request asks for something different,
1027        // so acquire() must detect the mismatch and resize it rather than
1028        // silently handing out the warm sandbox's original (wrong) size.
1029        let id = pool
1030            .acquire(&subject, &sandbox_config)
1031            .await
1032            .expect("warm reuse should succeed");
1033
1034        let applied = fake.applied.lock();
1035        let recorded = applied
1036            .get(&id)
1037            .expect("update_resources must have been called for this sandbox");
1038        assert_eq!(recorded.memory_limit_in_bytes, 256 * 1024 * 1024);
1039        assert_eq!(recorded.cpu_quota, 75_000);
1040    }
1041
1042    #[tokio::test]
1043    async fn cold_created_sandbox_needs_no_extra_resize_call() {
1044        // warm_pool_size 0 → always cold-creates. `start()` receives the
1045        // full config directly, so there should be no separate
1046        // `update_resources` call layered on top.
1047        let (pool, fake) = make_pool_with(5, 0);
1048        let subject = Subject::new("alice");
1049        let mut sandbox_config = PodSandboxConfig::default();
1050        sandbox_config.linux.resources.memory_limit_in_bytes = 128 * 1024 * 1024;
1051
1052        let id = pool
1053            .acquire(&subject, &sandbox_config)
1054            .await
1055            .expect("cold create should succeed");
1056        assert!(fake.applied.lock().get(&id).is_none());
1057    }
1058
1059    // ── Concurrency / TOCTOU (#489/#519 lesson): never over-place ─────────
1060
1061    #[tokio::test]
1062    async fn concurrent_acquires_never_exceed_capacity() {
1063        let capacity = 2usize;
1064        let n = 6usize;
1065        let cfg = crate::runtime::AdmissionConfig {
1066            queue_capacity: n,
1067            queue_timeout_secs: 1,
1068            ..Default::default()
1069        };
1070        let (pool, fake) = make_pool_with_admission(capacity, 0, cfg);
1071
1072        let mut handles = Vec::new();
1073        for i in 0..n {
1074            let pool = pool.clone();
1075            handles.push(tokio::spawn(async move {
1076                let subject = Subject::new(format!("user-{i}"));
1077                pool.acquire(&subject, &PodSandboxConfig::default()).await
1078            }));
1079        }
1080
1081        let mut ok = 0usize;
1082        let mut failed = 0usize;
1083        for h in handles {
1084            match h.await.expect("task must not panic") {
1085                Ok(_) => ok += 1,
1086                Err(e) => {
1087                    failed += 1;
1088                    assert!(
1089                        matches!(
1090                            e,
1091                            WorkerError::QueueFull { .. } | WorkerError::QueueTimeout { .. }
1092                        ),
1093                        "unexpected error: {e:?}"
1094                    );
1095                }
1096            }
1097        }
1098
1099        assert_eq!(ok, capacity, "exactly `capacity` requests must be admitted");
1100        assert_eq!(
1101            failed,
1102            n - capacity,
1103            "the rest must queue-then-timeout or reject, never silently succeed"
1104        );
1105        // The backend itself must never have seen more concurrently-started
1106        // sandboxes than the declared capacity — the actual TOCTOU check.
1107        assert!(
1108            fake.peak.load(Ordering::SeqCst) <= capacity,
1109            "backend saw more concurrent starts than capacity allows: {}",
1110            fake.peak.load(Ordering::SeqCst)
1111        );
1112        assert_eq!(pool.stats().await.active_count, capacity);
1113    }
1114
1115    #[tokio::test]
1116    async fn queued_request_is_admitted_once_capacity_frees() {
1117        let cfg = crate::runtime::AdmissionConfig {
1118            queue_timeout_secs: 5,
1119            ..Default::default()
1120        };
1121        let (pool, _fake) = make_pool_with_admission(1, 0, cfg);
1122        let subject = Subject::new("alice");
1123        let sandbox_config = PodSandboxConfig::default();
1124
1125        let first = pool
1126            .acquire(&subject, &sandbox_config)
1127            .await
1128            .expect("first request admitted");
1129
1130        let pool2 = pool.clone();
1131        let waiter = tokio::spawn(async move {
1132            let bob = Subject::new("bob");
1133            pool2.acquire(&bob, &PodSandboxConfig::default()).await
1134        });
1135
1136        // Give the waiter a moment to actually enter the queue, then free
1137        // capacity by releasing the first sandbox.
1138        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1139        pool.release(&first).await.expect("release must succeed");
1140
1141        let result = tokio::time::timeout(std::time::Duration::from_secs(2), waiter)
1142            .await
1143            .expect("queued waiter should be woken well within 2s")
1144            .expect("task must not panic");
1145        assert!(
1146            result.is_ok(),
1147            "queued request should be admitted once capacity frees: {result:?}"
1148        );
1149    }
1150
1151    // ── F2: resize failure must not orphan the popped warm sandbox ─────────
1152
1153    #[tokio::test]
1154    async fn resize_failure_destroys_warm_sandbox_and_rolls_back_reservation() {
1155        // queue_capacity 0 so any leaked reservation shows up as an immediate
1156        // QueueFull on the follow-up acquire.
1157        let cfg = crate::runtime::AdmissionConfig {
1158            queue_capacity: 0,
1159            ..Default::default()
1160        };
1161        let (pool, fake) = make_pool_with_admission(1, 1, cfg);
1162        pool.initialize().await.expect("prewarm one sandbox");
1163        assert_eq!(pool.stats().await.warm_count, 1);
1164
1165        fake.fail_resize.store(true, Ordering::SeqCst);
1166
1167        let subject = Subject::new("alice");
1168        let mut sandbox_config = PodSandboxConfig::default();
1169        // Force a mismatch vs the warm sandbox's default (all-zero) resources so
1170        // the #519 resize path runs — and fails.
1171        sandbox_config.linux.resources.memory_limit_in_bytes = 256 * 1024 * 1024;
1172
1173        let err = pool.acquire(&subject, &sandbox_config).await.unwrap_err();
1174        assert!(
1175            matches!(err, WorkerError::SandboxTimeout { .. }),
1176            "resize error propagates: {err:?}"
1177        );
1178        // The popped warm sandbox must be destroyed, not orphaned.
1179        assert!(
1180            !fake.destroyed.lock().is_empty(),
1181            "warm sandbox must be destroyed on resize failure (no orphan)"
1182        );
1183        assert_eq!(pool.stats().await.active_count, 0, "no active-sandbox leak");
1184
1185        // Reservation rolled back: a fresh cold acquire fits the single slot.
1186        fake.fail_resize.store(false, Ordering::SeqCst);
1187        let id = pool
1188            .acquire(&subject, &PodSandboxConfig::default())
1189            .await
1190            .expect("capacity/quota must have been rolled back after resize failure");
1191        assert!(pool.get(&id).await.is_some());
1192    }
1193
1194    #[tokio::test]
1195    async fn reset_failure_destroys_released_sandbox() {
1196        let (pool, fake) = make_pool_with(1, 1);
1197        pool.initialize().await.expect("prewarm one sandbox");
1198
1199        let subject = Subject::new("alice");
1200        let id = pool
1201            .acquire(&subject, &PodSandboxConfig::default())
1202            .await
1203            .expect("warm acquire should succeed");
1204
1205        fake.fail_reset.store(true, Ordering::SeqCst);
1206        let err = pool.release(&id).await.unwrap_err();
1207        assert!(
1208            matches!(err, WorkerError::SandboxTimeout { .. }),
1209            "reset error propagates: {err:?}"
1210        );
1211        assert!(
1212            fake.was_destroyed(&id),
1213            "sandbox must be destroyed after reset failure to avoid an orphan"
1214        );
1215        assert_eq!(
1216            pool.stats().await.active_count,
1217            0,
1218            "release removed active sandbox"
1219        );
1220    }
1221
1222    // ── F5: stop failure on release must not orphan / over-commit ──────────
1223
1224    #[tokio::test]
1225    async fn stop_failure_on_release_destroys_sandbox_and_still_frees_reservation() {
1226        let cfg = crate::runtime::AdmissionConfig {
1227            queue_capacity: 0,
1228            ..Default::default()
1229        };
1230        let (pool, fake) = make_pool_with_admission(1, 0, cfg);
1231        let subject = Subject::new("alice");
1232        let id = pool
1233            .acquire(&subject, &PodSandboxConfig::default())
1234            .await
1235            .expect("acquire");
1236
1237        fake.fail_stop.store(true, Ordering::SeqCst);
1238        let err = pool.release(&id).await.unwrap_err();
1239        assert!(
1240            matches!(err, WorkerError::SandboxTimeout { .. }),
1241            "stop error propagates: {err:?}"
1242        );
1243        // Orphan avoided: a destroy was attempted after the stop failure.
1244        assert!(
1245            fake.was_destroyed(&id),
1246            "sandbox must be destroyed after stop failure to avoid an orphan"
1247        );
1248
1249        // The reservation is still freed (only after the stop attempt), so
1250        // capacity recovers — a leak would make the next acquire QueueFull.
1251        fake.fail_stop.store(false, Ordering::SeqCst);
1252        let id2 = pool
1253            .acquire(&subject, &PodSandboxConfig::default())
1254            .await
1255            .expect("reservation must be freed even when stop fails");
1256        assert!(pool.get(&id2).await.is_some());
1257    }
1258}