1use std::collections::{HashMap, VecDeque};
7use std::sync::atomic::{AtomicUsize, Ordering};
8use std::sync::Arc;
9
10use tokio::sync::Mutex;
11use uuid::Uuid;
12
13use crate::config::PoolConfig;
14use crate::error::{Result, WorkerError};
15
16use super::admission::{self, AdmissionTracker};
17use super::backend::SandboxBackend;
18use super::PodSandbox;
19use hyprstream_rpc_std::worker_client::{
20 KeyValue, LinuxContainerResources, PodSandboxConfig, PodSandboxState,
21};
22use hyprstream_vfs::Subject;
23
24pub struct SandboxPool {
51 warm_pool: Mutex<VecDeque<PodSandbox>>,
53
54 active: Mutex<HashMap<String, PodSandbox>>,
56
57 config: PoolConfig,
59
60 backend: Arc<dyn SandboxBackend>,
62
63 in_flight: AtomicUsize,
69
70 admission: AdmissionTracker,
73}
74
75const SUBJECT_ANNOTATION: &str = "hyprstream.io/subject";
79
80impl SandboxPool {
81 pub fn new(config: PoolConfig, backend: Arc<dyn SandboxBackend>) -> Self {
88 let admission = AdmissionTracker::new(config.admission.clone(), config.max_sandboxes);
89 Self::with_admission(config, backend, admission)
90 }
91
92 pub fn with_group_validator(
97 config: PoolConfig,
98 backend: Arc<dyn SandboxBackend>,
99 group_validator: Arc<dyn admission::GroupSelectorValidator>,
100 ) -> Self {
101 let admission = AdmissionTracker::with_group_validator(
102 config.admission.clone(),
103 config.max_sandboxes,
104 group_validator,
105 );
106 Self::with_admission(config, backend, admission)
107 }
108
109 fn with_admission(
110 config: PoolConfig,
111 backend: Arc<dyn SandboxBackend>,
112 admission: AdmissionTracker,
113 ) -> Self {
114 Self {
115 warm_pool: Mutex::new(VecDeque::new()),
116 active: Mutex::new(HashMap::new()),
117 config,
118 backend,
119 in_flight: AtomicUsize::new(0),
120 admission,
121 }
122 }
123
124 pub fn backend(&self) -> &dyn SandboxBackend {
126 self.backend.as_ref()
127 }
128
129 pub async fn initialize(&self) -> Result<()> {
131 self.backend.initialize(&self.config).await?;
133
134 let mut warm = self.warm_pool.lock().await;
135
136 for _ in 0..self.config.warm_pool_size {
137 let sandbox = self.create_warm_sandbox().await?;
138 warm.push_back(sandbox);
139 }
140
141 tracing::info!(
142 backend = self.backend.backend_type(),
143 "Initialized warm pool with {} sandboxes",
144 self.config.warm_pool_size
145 );
146
147 Ok(())
148 }
149
150 pub async fn acquire(
168 self: &Arc<Self>,
169 subject: &Subject,
170 config: &PodSandboxConfig,
171 ) -> Result<String> {
172 let demand = admission::derive_demand(config);
173 let group_selector = admission::derive_group_selector(config);
174 let reservation = self
175 .admission
176 .reserve(subject, group_selector.as_deref(), demand)
177 .await?;
178
179 let mut task_config = config.clone();
184 task_config
185 .annotations
186 .retain(|annotation| annotation.key != SUBJECT_ANNOTATION);
187 task_config.annotations.push(KeyValue {
188 key: SUBJECT_ANNOTATION.to_owned(),
189 value: subject.to_string(),
190 });
191
192 let obtained = self.obtain_sandbox(&task_config).await;
193 let mut sandbox = match obtained {
194 Ok(sandbox) => sandbox,
195 Err(e) => {
196 self.admission.rollback(&reservation).await;
200 return Err(e);
201 }
202 };
203
204 sandbox.reservation = Some(reservation);
205 let sandbox_id = sandbox.id.clone();
206
207 self.active.lock().await.insert(sandbox_id.clone(), sandbox);
209
210 Self::replenish_warm_pool(self.clone());
212
213 Ok(sandbox_id)
214 }
215
216 async fn obtain_sandbox(&self, config: &PodSandboxConfig) -> Result<PodSandbox> {
221 let mut warm = self.warm_pool.lock().await;
222 if let Some(mut sandbox) = warm.pop_front() {
223 drop(warm);
224 sandbox.metadata = config.metadata.clone();
226 sandbox.labels = config.labels.clone();
227 sandbox.annotations = config.annotations.clone();
228 sandbox.state = PodSandboxState::SandboxReady;
229
230 let requested = config.linux.resources.clone();
235 if requested != LinuxContainerResources::default()
236 && requested != sandbox.applied_resources
237 {
238 tracing::debug!(
239 sandbox_id = %sandbox.id,
240 "warm sandbox resource mismatch on reuse — resizing (#519)"
241 );
242 if let Err(resize_err) = self.backend.update_resources(&sandbox, &requested).await {
250 if let Err(destroy_err) = self.backend.destroy(&sandbox).await {
251 tracing::warn!(
252 sandbox_id = %sandbox.id,
253 error = %destroy_err,
254 "failed to destroy warm sandbox after resize failure — possible orphan"
255 );
256 }
257 return Err(resize_err);
258 }
259 sandbox.applied_resources = requested;
260 }
261 Ok(sandbox)
262 } else {
263 drop(warm);
264 self.create_sandbox(config).await
265 }
266 }
267
268 pub async fn release(self: &Arc<Self>, sandbox_id: &str) -> Result<()> {
270 let mut active = self.active.lock().await;
271
272 let sandbox = active
273 .remove(sandbox_id)
274 .ok_or_else(|| WorkerError::SandboxNotFound(sandbox_id.to_owned()))?;
275 drop(active);
276
277 let release_reservation = || async {
287 if let Some(reservation) = sandbox.reservation.clone() {
288 self.admission.release(&reservation).await;
289 }
290 };
291
292 if let Err(stop_err) = self.backend.stop(&sandbox).await {
293 tracing::warn!(
294 sandbox_id = %sandbox.id,
295 error = %stop_err,
296 "stop failed on release — destroying sandbox to avoid an orphan"
297 );
298 if let Err(destroy_err) = self.backend.destroy(&sandbox).await {
299 tracing::warn!(
300 sandbox_id = %sandbox.id,
301 error = %destroy_err,
302 "destroy also failed after stop failure — possible orphan"
303 );
304 }
305 release_reservation().await;
306 return Err(stop_err);
307 }
308
309 release_reservation().await;
311
312 let warm = self.warm_pool.lock().await;
314 if warm.len() < self.config.warm_pool_size {
315 drop(warm);
316 let reset_sandbox = self.reset_sandbox(sandbox).await?;
318 match reset_sandbox {
319 Some(s) => self.warm_pool.lock().await.push_back(s),
320 None => Self::replenish_warm_pool(self.clone()),
323 }
324 } else {
325 drop(warm);
326 self.backend.destroy(&sandbox).await?;
328 }
329
330 Ok(())
331 }
332
333 pub async fn get(&self, sandbox_id: &str) -> Option<PodSandbox> {
335 self.active.lock().await.get(sandbox_id).cloned()
336 }
337
338 pub async fn list_active(&self) -> Vec<PodSandbox> {
340 self.active.lock().await.values().cloned().collect()
341 }
342
343 pub async fn stats(&self) -> PoolStats {
345 let warm_count = self.warm_pool.lock().await.len();
346 let active_count = self.active.lock().await.len();
347
348 PoolStats {
349 warm_count,
350 active_count,
351 max_sandboxes: self.config.max_sandboxes,
352 warm_pool_target: self.config.warm_pool_size,
353 }
354 }
355
356 pub async fn shutdown(&self) -> Result<()> {
358 tracing::info!("Shutting down sandbox pool");
359
360 let mut warm = self.warm_pool.lock().await;
362 while let Some(sandbox) = warm.pop_front() {
363 if let Err(e) = self.backend.destroy(&sandbox).await {
364 tracing::warn!("Error destroying warm sandbox: {}", e);
365 }
366 }
367 drop(warm);
368
369 let active: Vec<_> = {
371 let mut active = self.active.lock().await;
372 active.drain().map(|(_, s)| s).collect()
373 };
374
375 for sandbox in active {
376 if let Err(e) = self.backend.stop(&sandbox).await {
377 tracing::warn!("Error stopping sandbox {}: {}", sandbox.id, e);
378 }
379 if let Err(e) = self.backend.destroy(&sandbox).await {
380 tracing::warn!("Error destroying sandbox: {}", e);
381 }
382 }
383
384 tracing::info!("Sandbox pool shutdown complete");
385 Ok(())
386 }
387
388 async fn create_warm_sandbox(&self) -> Result<PodSandbox> {
394 let id = format!("warm-{}", Uuid::new_v4());
395 let config = PodSandboxConfig::default();
396 let sandbox_path = self.config.runtime_dir.join(&id);
397
398 let mut sandbox = PodSandbox::new(id, &config, sandbox_path);
399
400 let annotations = HashMap::new();
402 let handle = self
403 .backend
404 .start(&mut sandbox, &config, &self.config, &annotations)
405 .await?;
406 sandbox.set_backend_handle(handle);
407
408 Ok(sandbox)
409 }
410
411 async fn create_sandbox(&self, config: &PodSandboxConfig) -> Result<PodSandbox> {
413 let id = Uuid::new_v4().to_string();
414 let sandbox_path = self.config.runtime_dir.join(&id);
415
416 let mut sandbox = PodSandbox::new(id, config, sandbox_path);
417
418 let annotations: HashMap<String, String> = sandbox
420 .annotations
421 .iter()
422 .map(|kv| (kv.key.clone(), kv.value.clone()))
423 .collect();
424
425 let handle = self
427 .backend
428 .start(&mut sandbox, config, &self.config, &annotations)
429 .await?;
430 sandbox.set_backend_handle(handle);
431
432 sandbox.state = PodSandboxState::SandboxReady;
433 sandbox.applied_resources = config.linux.resources.clone();
436
437 Ok(sandbox)
438 }
439
440 async fn reset_sandbox(&self, mut sandbox: PodSandbox) -> Result<Option<PodSandbox>> {
445 tracing::debug!(sandbox_id = %sandbox.id, "Resetting sandbox for warm pool");
446
447 sandbox.metadata = Default::default();
449 sandbox.labels.clear();
450 sandbox.annotations.clear();
451 sandbox.state = PodSandboxState::SandboxNotReady;
452 sandbox.image_id = None;
453 sandbox.reservation = None;
459
460 let reusable = match self.backend.reset(&mut sandbox).await {
462 Ok(reusable) => reusable,
463 Err(reset_err) => {
464 tracing::warn!(
465 sandbox_id = %sandbox.id,
466 error = %reset_err,
467 "reset failed on release — destroying sandbox to avoid an orphan"
468 );
469 if let Err(destroy_err) = self.backend.destroy(&sandbox).await {
470 tracing::warn!(
471 sandbox_id = %sandbox.id,
472 error = %destroy_err,
473 "destroy also failed after reset failure — possible orphan"
474 );
475 }
476 return Err(reset_err);
477 }
478 };
479 if reusable {
480 Ok(Some(sandbox))
481 } else {
482 self.backend.destroy(&sandbox).await?;
484 Ok(None)
485 }
486 }
487
488 fn replenish_warm_pool(pool: Arc<Self>) {
498 let target = pool.config.warm_pool_size;
499 tokio::spawn(async move {
500 let to_create = {
503 let warm = pool.warm_pool.lock().await;
504 let in_flight = pool.in_flight.load(Ordering::SeqCst);
505 let deficit = target.saturating_sub(warm.len() + in_flight);
506 pool.in_flight.fetch_add(deficit, Ordering::SeqCst);
507 deficit
508 };
509
510 for _ in 0..to_create {
511 match pool.create_warm_sandbox().await {
512 Ok(sandbox) => {
513 pool.warm_pool.lock().await.push_back(sandbox);
514 tracing::debug!(target, "Warm pool replenished");
515 }
516 Err(e) => {
517 tracing::warn!(error = %e, "Failed to replenish warm pool");
518 }
519 }
520 pool.in_flight.fetch_sub(1, Ordering::SeqCst);
523 }
524 });
525 }
526}
527
528#[derive(Debug, Clone)]
530pub struct PoolStats {
531 pub warm_count: usize,
533 pub active_count: usize,
535 pub max_sandboxes: usize,
537 pub warm_pool_target: usize,
539}
540
541#[cfg(all(test, feature = "kata"))]
543mod tests {
544 use super::*;
545 use crate::config::{ImageConfig, PoolConfig};
546 use crate::image::RafsStore;
547 use crate::runtime::kata_backend::KataBackend;
548 use hyprstream_rpc::auth::mac::{ObjectRef, SecurityContext};
549 use tempfile::TempDir;
550
551 struct FixtureAccessDecider;
552
553 impl hyprstream_9p::AccessDecider for FixtureAccessDecider {
554 fn check(
555 &self,
556 _ctx: &SecurityContext,
557 _object: ObjectRef<'_>,
558 _action: hyprstream_9p::Action,
559 ) -> bool {
560 true
561 }
562
563 fn audit_denial(
564 &self,
565 _ctx: &SecurityContext,
566 _object: ObjectRef<'_>,
567 _object_label: Option<hyprstream_rpc::auth::mac::SecurityLabel>,
568 _action: hyprstream_9p::Action,
569 _reason: hyprstream_9p::ReferenceMonitorDenyReason,
570 ) {
571 }
572 }
573
574 async fn create_test_pool(
576 max_sandboxes: usize,
577 warm_pool_size: usize,
578 ) -> Result<(Arc<SandboxPool>, TempDir)> {
579 let temp_dir = TempDir::new()?;
580 let base_path = temp_dir.path();
581
582 let pool_config = PoolConfig {
583 max_sandboxes,
584 warm_pool_size,
585 runtime_dir: base_path.join("runtime"),
586 ..Default::default()
587 };
588
589 let image_config = ImageConfig {
590 blobs_dir: base_path.join("blobs"),
591 bootstrap_dir: base_path.join("bootstrap"),
592 refs_dir: base_path.join("refs"),
593 cache_dir: base_path.join("cache"),
594 runtime_dir: base_path.join("nydus-runtime"),
595 ..Default::default()
596 };
597
598 std::fs::create_dir_all(&image_config.blobs_dir)?;
600 std::fs::create_dir_all(&image_config.bootstrap_dir)?;
601 std::fs::create_dir_all(&image_config.refs_dir)?;
602 std::fs::create_dir_all(&image_config.cache_dir)?;
603 std::fs::create_dir_all(&pool_config.runtime_dir)?;
604
605 let rafs_store = Arc::new(RafsStore::new(image_config.clone())?);
606 let backend: Arc<dyn SandboxBackend> = Arc::new(KataBackend::new(
607 image_config,
608 rafs_store,
609 Arc::new(FixtureAccessDecider),
610 None,
611 ));
612 let pool = Arc::new(SandboxPool::new(pool_config, backend));
613
614 Ok((pool, temp_dir))
615 }
616
617 #[tokio::test]
618 async fn test_pool_acquire_release() -> Result<()> {
619 let (pool, _temp_dir) = create_test_pool(5, 0).await?;
620
621 let sandbox_config = PodSandboxConfig::default();
622 let subject = Subject::new("test-user");
623
624 let result = pool.acquire(&subject, &sandbox_config).await;
626
627 match result {
628 Ok(sandbox_id) => {
629 assert!(pool.get(&sandbox_id).await.is_some());
630 let stats = pool.stats().await;
631 assert_eq!(stats.active_count, 1);
632
633 pool.release(&sandbox_id).await?;
634 assert!(pool.get(&sandbox_id).await.is_none());
635 }
636 Err(WorkerError::VmStartFailed(_)) => {
637 let stats = pool.stats().await;
639 assert_eq!(stats.active_count, 0);
640 }
641 Err(e) => panic!("Unexpected error: {e:?}"),
642 }
643 Ok(())
644 }
645
646 #[tokio::test]
647 async fn test_pool_stats() -> Result<()> {
648 let (pool, _temp_dir) = create_test_pool(10, 2).await?;
649
650 let stats = pool.stats().await;
651 assert_eq!(stats.max_sandboxes, 10);
652 assert_eq!(stats.warm_pool_target, 2);
653 assert_eq!(stats.active_count, 0);
654 assert_eq!(stats.warm_count, 0);
655 Ok(())
656 }
657
658 #[tokio::test]
659 async fn test_pool_shutdown() -> Result<()> {
660 let (pool, _temp_dir) = create_test_pool(5, 0).await?;
661
662 pool.shutdown().await?;
664
665 let stats = pool.stats().await;
666 assert_eq!(stats.active_count, 0);
667 assert_eq!(stats.warm_count, 0);
668 Ok(())
669 }
670}
671
672#[cfg(test)]
677#[allow(clippy::unwrap_used, clippy::expect_used)]
678mod admission_tests {
679 use super::*;
680 use crate::runtime::backend::SandboxHandle;
681 use hyprstream_rpc_std::worker_client::KeyValue;
682 use parking_lot::Mutex as StdMutex;
683 use std::any::Any;
684 use std::collections::HashMap as StdHashMap;
685 use std::sync::atomic::AtomicBool as StdAtomicBool;
686 use std::sync::atomic::AtomicUsize as StdAtomicUsize;
687
688 #[derive(Debug)]
689 struct FakeHandle;
690 impl SandboxHandle for FakeHandle {
691 fn as_any(&self) -> &dyn Any {
692 self
693 }
694 }
695
696 #[derive(Default)]
703 struct FakeBackend {
704 current: StdAtomicUsize,
705 peak: StdAtomicUsize,
706 applied: StdMutex<StdHashMap<String, LinuxContainerResources>>,
707 fail_resize: StdAtomicBool,
709 fail_stop: StdAtomicBool,
711 fail_reset: StdAtomicBool,
713 destroyed: StdMutex<Vec<String>>,
715 }
716
717 impl FakeBackend {
718 fn bump(&self) {
719 let cur = self.current.fetch_add(1, Ordering::SeqCst) + 1;
720 self.peak.fetch_max(cur, Ordering::SeqCst);
721 }
722 fn was_destroyed(&self, id: &str) -> bool {
723 self.destroyed.lock().iter().any(|d| d == id)
724 }
725 }
726
727 #[async_trait::async_trait]
728 impl SandboxBackend for FakeBackend {
729 fn backend_type(&self) -> &'static str {
730 "fake"
731 }
732 fn is_available(&self) -> bool {
733 true
734 }
735 async fn initialize(&self, _config: &crate::config::PoolConfig) -> Result<()> {
736 Ok(())
737 }
738 async fn start(
739 &self,
740 _sandbox: &mut PodSandbox,
741 _config: &PodSandboxConfig,
742 _pool_config: &crate::config::PoolConfig,
743 _annotations: &HashMap<String, String>,
744 ) -> Result<Arc<dyn SandboxHandle>> {
745 self.bump();
746 Ok(Arc::new(FakeHandle))
747 }
748 async fn stop(&self, _sandbox: &PodSandbox) -> Result<()> {
749 if self.fail_stop.load(Ordering::SeqCst) {
750 return Err(WorkerError::SandboxTimeout {
752 operation: "stop".to_owned(),
753 timeout_secs: 0,
754 });
755 }
756 self.current.fetch_sub(1, Ordering::SeqCst);
757 Ok(())
758 }
759 async fn destroy(&self, sandbox: &PodSandbox) -> Result<()> {
760 self.destroyed.lock().push(sandbox.id.clone());
761 Ok(())
762 }
763 async fn reset(&self, _sandbox: &mut PodSandbox) -> Result<bool> {
764 if self.fail_reset.load(Ordering::SeqCst) {
765 return Err(WorkerError::SandboxTimeout {
766 operation: "reset".to_owned(),
767 timeout_secs: 0,
768 });
769 }
770 Ok(true)
773 }
774 async fn get_pids(&self, _sandbox: &PodSandbox) -> Result<Vec<u32>> {
775 Ok(vec![])
776 }
777 fn supports_exec(&self) -> bool {
778 false
779 }
780 async fn exec_sync(
781 &self,
782 _sandbox: &PodSandbox,
783 _command: &[String],
784 _timeout_secs: u64,
785 ) -> Result<(i32, Vec<u8>, Vec<u8>)> {
786 Err(WorkerError::ExecFailed("not supported".into()))
787 }
788 async fn update_resources(
789 &self,
790 sandbox: &PodSandbox,
791 resources: &LinuxContainerResources,
792 ) -> Result<()> {
793 if self.fail_resize.load(Ordering::SeqCst) {
794 return Err(WorkerError::SandboxTimeout {
795 operation: "update_resources".to_owned(),
796 timeout_secs: 0,
797 });
798 }
799 self.applied
800 .lock()
801 .insert(sandbox.id.clone(), resources.clone());
802 Ok(())
803 }
804 }
805
806 fn make_pool_with(
807 max_sandboxes: usize,
808 warm_pool_size: usize,
809 ) -> (Arc<SandboxPool>, Arc<FakeBackend>) {
810 let fake = Arc::new(FakeBackend::default());
811 let backend: Arc<dyn SandboxBackend> = fake.clone();
812 let config = PoolConfig {
813 max_sandboxes,
814 warm_pool_size,
815 ..Default::default()
816 };
817 (Arc::new(SandboxPool::new(config, backend)), fake)
818 }
819
820 fn make_pool_with_admission(
821 max_sandboxes: usize,
822 warm_pool_size: usize,
823 admission: crate::runtime::AdmissionConfig,
824 ) -> (Arc<SandboxPool>, Arc<FakeBackend>) {
825 let fake = Arc::new(FakeBackend::default());
826 let backend: Arc<dyn SandboxBackend> = fake.clone();
827 let config = PoolConfig {
828 max_sandboxes,
829 warm_pool_size,
830 admission,
831 ..Default::default()
832 };
833 (Arc::new(SandboxPool::new(config, backend)), fake)
834 }
835
836 #[tokio::test]
839 async fn solo_acquire_release_unchanged() {
840 let (pool, _fake) = make_pool_with(5, 0);
841 let subject = Subject::new("alice");
842 let sandbox_config = PodSandboxConfig::default();
843
844 let id = pool
845 .acquire(&subject, &sandbox_config)
846 .await
847 .expect("solo acquire must succeed");
848 assert!(pool.get(&id).await.is_some());
849 assert_eq!(pool.stats().await.active_count, 1);
850
851 pool.release(&id).await.expect("release must succeed");
852 assert!(pool.get(&id).await.is_none());
853 assert_eq!(pool.stats().await.active_count, 0);
854 }
855
856 #[tokio::test]
857 async fn solo_capacity_exhausted_without_queue_room_rejects_clearly() {
858 let cfg = crate::runtime::AdmissionConfig {
859 queue_capacity: 0,
860 ..Default::default()
861 };
862 let (pool, _fake) = make_pool_with_admission(1, 0, cfg);
863 let subject = Subject::new("alice");
864 let sandbox_config = PodSandboxConfig::default();
865
866 let _id = pool
867 .acquire(&subject, &sandbox_config)
868 .await
869 .expect("first acquire admits");
870 let err = pool.acquire(&subject, &sandbox_config).await.unwrap_err();
871 assert!(matches!(err, WorkerError::QueueFull { .. }), "got: {err:?}");
872 }
873
874 #[tokio::test]
877 async fn anonymous_caller_rejected_fail_closed() {
878 let (pool, _fake) = make_pool_with(5, 0);
879 let sandbox_config = PodSandboxConfig::default();
880 let err = pool
881 .acquire(&Subject::anonymous(), &sandbox_config)
882 .await
883 .unwrap_err();
884 assert!(
885 matches!(err, WorkerError::Unauthorized { .. }),
886 "got: {err:?}"
887 );
888 assert_eq!(pool.stats().await.active_count, 0);
890 }
891
892 #[tokio::test]
895 async fn per_subject_quota_rejects_over_quota_caller() {
896 let cfg = crate::runtime::AdmissionConfig {
897 max_per_subject: Some(1),
898 ..Default::default()
899 };
900 let (pool, _fake) = make_pool_with_admission(10, 0, cfg);
901 let alice = Subject::new("alice");
902 let sandbox_config = PodSandboxConfig::default();
903
904 let _id = pool
905 .acquire(&alice, &sandbox_config)
906 .await
907 .expect("first request admitted");
908 let err = pool.acquire(&alice, &sandbox_config).await.unwrap_err();
909 assert!(
910 matches!(err, WorkerError::AdmissionDenied { .. }),
911 "got: {err:?}"
912 );
913 }
914
915 #[tokio::test]
918 async fn gpu_request_placed_only_when_capacity_declared() {
919 let cfg = crate::runtime::AdmissionConfig {
920 gpu_total: 0,
921 ..Default::default()
922 };
923 let (pool, _fake) = make_pool_with_admission(10, 0, cfg);
924 let subject = Subject::new("alice");
925 let mut sandbox_config = PodSandboxConfig::default();
926 sandbox_config.annotations.push(KeyValue {
927 key: crate::runtime::ANN_GPU_REQUEST.to_owned(),
928 value: "1".to_owned(),
929 });
930
931 let start = std::time::Instant::now();
935 let err = pool.acquire(&subject, &sandbox_config).await.unwrap_err();
936 assert!(
937 matches!(err, WorkerError::AdmissionInfeasible { .. }),
938 "got: {err:?}"
939 );
940 assert!(
941 start.elapsed() < std::time::Duration::from_secs(1),
942 "infeasible GPU request must reject immediately, not wait the queue timeout"
943 );
944 }
945
946 #[tokio::test]
947 async fn gpu_request_placed_when_capacity_available() {
948 let cfg = crate::runtime::AdmissionConfig {
949 gpu_total: 1,
950 ..Default::default()
951 };
952 let (pool, _fake) = make_pool_with_admission(10, 0, cfg);
953 let subject = Subject::new("alice");
954 let mut sandbox_config = PodSandboxConfig::default();
955 sandbox_config.annotations.push(KeyValue {
956 key: crate::runtime::ANN_GPU_REQUEST.to_owned(),
957 value: "1".to_owned(),
958 });
959
960 let id = pool
961 .acquire(&subject, &sandbox_config)
962 .await
963 .expect("gpu request should be admitted");
964 assert!(pool.get(&id).await.is_some());
965 }
966
967 #[tokio::test]
968 async fn cpu_and_memory_request_honored_within_capacity() {
969 let cfg = crate::runtime::AdmissionConfig {
970 cpu_millis_total: Some(1000),
971 memory_bytes_total: Some(1024 * 1024 * 1024),
972 ..Default::default()
973 };
974 let (pool, _fake) = make_pool_with_admission(10, 0, cfg);
975 let subject = Subject::new("alice");
976 let mut sandbox_config = PodSandboxConfig::default();
977 sandbox_config.linux.resources.cpu_period = 100_000;
978 sandbox_config.linux.resources.cpu_quota = 50_000; sandbox_config.linux.resources.memory_limit_in_bytes = 512 * 1024 * 1024;
980
981 let id = pool
982 .acquire(&subject, &sandbox_config)
983 .await
984 .expect("fits within declared capacity");
985 assert!(pool.get(&id).await.is_some());
986 }
987
988 #[tokio::test]
989 async fn memory_request_exceeding_capacity_is_not_silently_placed() {
990 let cfg = crate::runtime::AdmissionConfig {
991 memory_bytes_total: Some(256 * 1024 * 1024),
992 queue_capacity: 0,
993 ..Default::default()
994 };
995 let (pool, _fake) = make_pool_with_admission(10, 0, cfg);
996 let subject = Subject::new("alice");
997 let mut sandbox_config = PodSandboxConfig::default();
998 sandbox_config.linux.resources.memory_limit_in_bytes = 512 * 1024 * 1024; let err = pool.acquire(&subject, &sandbox_config).await.unwrap_err();
1004 assert!(
1005 matches!(err, WorkerError::AdmissionInfeasible { .. }),
1006 "got: {err:?}"
1007 );
1008 }
1009
1010 #[tokio::test]
1014 async fn warm_sandbox_reuse_resizes_mismatched_resources() {
1015 let (pool, fake) = make_pool_with(5, 1);
1016 pool.initialize().await.expect("prewarm one sandbox");
1017 assert_eq!(pool.stats().await.warm_count, 1);
1018
1019 let subject = Subject::new("alice");
1020 let mut sandbox_config = PodSandboxConfig::default();
1021 sandbox_config.linux.resources.cpu_period = 100_000;
1022 sandbox_config.linux.resources.cpu_quota = 75_000;
1023 sandbox_config.linux.resources.memory_limit_in_bytes = 256 * 1024 * 1024;
1024
1025 let id = pool
1030 .acquire(&subject, &sandbox_config)
1031 .await
1032 .expect("warm reuse should succeed");
1033
1034 let applied = fake.applied.lock();
1035 let recorded = applied
1036 .get(&id)
1037 .expect("update_resources must have been called for this sandbox");
1038 assert_eq!(recorded.memory_limit_in_bytes, 256 * 1024 * 1024);
1039 assert_eq!(recorded.cpu_quota, 75_000);
1040 }
1041
1042 #[tokio::test]
1043 async fn cold_created_sandbox_needs_no_extra_resize_call() {
1044 let (pool, fake) = make_pool_with(5, 0);
1048 let subject = Subject::new("alice");
1049 let mut sandbox_config = PodSandboxConfig::default();
1050 sandbox_config.linux.resources.memory_limit_in_bytes = 128 * 1024 * 1024;
1051
1052 let id = pool
1053 .acquire(&subject, &sandbox_config)
1054 .await
1055 .expect("cold create should succeed");
1056 assert!(fake.applied.lock().get(&id).is_none());
1057 }
1058
1059 #[tokio::test]
1062 async fn concurrent_acquires_never_exceed_capacity() {
1063 let capacity = 2usize;
1064 let n = 6usize;
1065 let cfg = crate::runtime::AdmissionConfig {
1066 queue_capacity: n,
1067 queue_timeout_secs: 1,
1068 ..Default::default()
1069 };
1070 let (pool, fake) = make_pool_with_admission(capacity, 0, cfg);
1071
1072 let mut handles = Vec::new();
1073 for i in 0..n {
1074 let pool = pool.clone();
1075 handles.push(tokio::spawn(async move {
1076 let subject = Subject::new(format!("user-{i}"));
1077 pool.acquire(&subject, &PodSandboxConfig::default()).await
1078 }));
1079 }
1080
1081 let mut ok = 0usize;
1082 let mut failed = 0usize;
1083 for h in handles {
1084 match h.await.expect("task must not panic") {
1085 Ok(_) => ok += 1,
1086 Err(e) => {
1087 failed += 1;
1088 assert!(
1089 matches!(
1090 e,
1091 WorkerError::QueueFull { .. } | WorkerError::QueueTimeout { .. }
1092 ),
1093 "unexpected error: {e:?}"
1094 );
1095 }
1096 }
1097 }
1098
1099 assert_eq!(ok, capacity, "exactly `capacity` requests must be admitted");
1100 assert_eq!(
1101 failed,
1102 n - capacity,
1103 "the rest must queue-then-timeout or reject, never silently succeed"
1104 );
1105 assert!(
1108 fake.peak.load(Ordering::SeqCst) <= capacity,
1109 "backend saw more concurrent starts than capacity allows: {}",
1110 fake.peak.load(Ordering::SeqCst)
1111 );
1112 assert_eq!(pool.stats().await.active_count, capacity);
1113 }
1114
1115 #[tokio::test]
1116 async fn queued_request_is_admitted_once_capacity_frees() {
1117 let cfg = crate::runtime::AdmissionConfig {
1118 queue_timeout_secs: 5,
1119 ..Default::default()
1120 };
1121 let (pool, _fake) = make_pool_with_admission(1, 0, cfg);
1122 let subject = Subject::new("alice");
1123 let sandbox_config = PodSandboxConfig::default();
1124
1125 let first = pool
1126 .acquire(&subject, &sandbox_config)
1127 .await
1128 .expect("first request admitted");
1129
1130 let pool2 = pool.clone();
1131 let waiter = tokio::spawn(async move {
1132 let bob = Subject::new("bob");
1133 pool2.acquire(&bob, &PodSandboxConfig::default()).await
1134 });
1135
1136 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1139 pool.release(&first).await.expect("release must succeed");
1140
1141 let result = tokio::time::timeout(std::time::Duration::from_secs(2), waiter)
1142 .await
1143 .expect("queued waiter should be woken well within 2s")
1144 .expect("task must not panic");
1145 assert!(
1146 result.is_ok(),
1147 "queued request should be admitted once capacity frees: {result:?}"
1148 );
1149 }
1150
1151 #[tokio::test]
1154 async fn resize_failure_destroys_warm_sandbox_and_rolls_back_reservation() {
1155 let cfg = crate::runtime::AdmissionConfig {
1158 queue_capacity: 0,
1159 ..Default::default()
1160 };
1161 let (pool, fake) = make_pool_with_admission(1, 1, cfg);
1162 pool.initialize().await.expect("prewarm one sandbox");
1163 assert_eq!(pool.stats().await.warm_count, 1);
1164
1165 fake.fail_resize.store(true, Ordering::SeqCst);
1166
1167 let subject = Subject::new("alice");
1168 let mut sandbox_config = PodSandboxConfig::default();
1169 sandbox_config.linux.resources.memory_limit_in_bytes = 256 * 1024 * 1024;
1172
1173 let err = pool.acquire(&subject, &sandbox_config).await.unwrap_err();
1174 assert!(
1175 matches!(err, WorkerError::SandboxTimeout { .. }),
1176 "resize error propagates: {err:?}"
1177 );
1178 assert!(
1180 !fake.destroyed.lock().is_empty(),
1181 "warm sandbox must be destroyed on resize failure (no orphan)"
1182 );
1183 assert_eq!(pool.stats().await.active_count, 0, "no active-sandbox leak");
1184
1185 fake.fail_resize.store(false, Ordering::SeqCst);
1187 let id = pool
1188 .acquire(&subject, &PodSandboxConfig::default())
1189 .await
1190 .expect("capacity/quota must have been rolled back after resize failure");
1191 assert!(pool.get(&id).await.is_some());
1192 }
1193
1194 #[tokio::test]
1195 async fn reset_failure_destroys_released_sandbox() {
1196 let (pool, fake) = make_pool_with(1, 1);
1197 pool.initialize().await.expect("prewarm one sandbox");
1198
1199 let subject = Subject::new("alice");
1200 let id = pool
1201 .acquire(&subject, &PodSandboxConfig::default())
1202 .await
1203 .expect("warm acquire should succeed");
1204
1205 fake.fail_reset.store(true, Ordering::SeqCst);
1206 let err = pool.release(&id).await.unwrap_err();
1207 assert!(
1208 matches!(err, WorkerError::SandboxTimeout { .. }),
1209 "reset error propagates: {err:?}"
1210 );
1211 assert!(
1212 fake.was_destroyed(&id),
1213 "sandbox must be destroyed after reset failure to avoid an orphan"
1214 );
1215 assert_eq!(
1216 pool.stats().await.active_count,
1217 0,
1218 "release removed active sandbox"
1219 );
1220 }
1221
1222 #[tokio::test]
1225 async fn stop_failure_on_release_destroys_sandbox_and_still_frees_reservation() {
1226 let cfg = crate::runtime::AdmissionConfig {
1227 queue_capacity: 0,
1228 ..Default::default()
1229 };
1230 let (pool, fake) = make_pool_with_admission(1, 0, cfg);
1231 let subject = Subject::new("alice");
1232 let id = pool
1233 .acquire(&subject, &PodSandboxConfig::default())
1234 .await
1235 .expect("acquire");
1236
1237 fake.fail_stop.store(true, Ordering::SeqCst);
1238 let err = pool.release(&id).await.unwrap_err();
1239 assert!(
1240 matches!(err, WorkerError::SandboxTimeout { .. }),
1241 "stop error propagates: {err:?}"
1242 );
1243 assert!(
1245 fake.was_destroyed(&id),
1246 "sandbox must be destroyed after stop failure to avoid an orphan"
1247 );
1248
1249 fake.fail_stop.store(false, Ordering::SeqCst);
1252 let id2 = pool
1253 .acquire(&subject, &PodSandboxConfig::default())
1254 .await
1255 .expect("reservation must be freed even when stop fails");
1256 assert!(pool.get(&id2).await.is_some());
1257 }
1258}