1use std::collections::{HashMap, HashSet, VecDeque};
66use std::sync::Arc;
67use std::time::Duration;
68
69use async_trait::async_trait;
70use serde::{Deserialize, Serialize};
71use tokio::sync::{Mutex, Notify};
72use tokio::time::Instant;
73
74use hyprstream_discovery::scheduling::{self, Predicate, RejectionReason, ResourceRequest};
75use hyprstream_vfs::Subject;
76
77use crate::error::{Result, WorkerError};
78
79use hyprstream_rpc_std::worker_client::{KeyValue, PodSandboxConfig};
80
81pub const ANN_GPU_REQUEST: &str = "hyprstream.io/gpu-request";
85
86pub const ANN_GROUP: &str = "hyprstream.io/group";
93
94fn annotation<'a>(annotations: &'a [KeyValue], key: &str) -> Option<&'a str> {
95 annotations
96 .iter()
97 .find(|kv| kv.key == key)
98 .map(|kv| kv.value.as_str())
99}
100
101#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
103pub struct Demand {
104 pub cpu_millis: u64,
105 pub memory_bytes: u64,
106 pub gpu: usize,
107}
108
109pub fn derive_demand(config: &PodSandboxConfig) -> Demand {
118 let res = &config.linux.resources;
119 let cpu_millis = if res.cpu_period > 0 && res.cpu_quota > 0 {
120 ((res.cpu_quota as f64 / res.cpu_period as f64) * 1000.0).round() as u64
121 } else {
122 0
123 };
124 let memory_bytes = res.memory_limit_in_bytes.max(0) as u64;
125 let gpu = annotation(&config.annotations, ANN_GPU_REQUEST)
126 .and_then(|v| v.parse::<usize>().ok())
127 .unwrap_or(0);
128 Demand {
129 cpu_millis,
130 memory_bytes,
131 gpu,
132 }
133}
134
135pub fn derive_group_selector(config: &PodSandboxConfig) -> Option<String> {
141 annotation(&config.annotations, ANN_GROUP).map(str::to_owned)
142}
143
144#[derive(Debug, Clone, Serialize, Deserialize)]
156#[serde(default)]
157pub struct AdmissionConfig {
158 #[serde(skip_serializing_if = "Option::is_none")]
165 pub max_per_subject: Option<usize>,
166 #[serde(skip_serializing_if = "Option::is_none")]
177 pub max_per_group: Option<usize>,
178 pub queue_capacity: usize,
182 pub queue_timeout_secs: u64,
185 pub cpu_millis_total: Option<u64>,
188 pub memory_bytes_total: Option<u64>,
191 pub gpu_total: usize,
197}
198
199impl Default for AdmissionConfig {
200 fn default() -> Self {
201 Self {
202 max_per_subject: None,
203 max_per_group: None,
204 queue_capacity: 64,
205 queue_timeout_secs: 30,
206 cpu_millis_total: None,
207 memory_bytes_total: None,
208 gpu_total: 0,
209 }
210 }
211}
212
213#[derive(Debug, Clone, PartialEq, Eq)]
216pub struct ReservationRecord {
217 subject: String,
218 group: Option<String>,
219 demand: Demand,
220}
221
222#[derive(Debug, Clone, PartialEq, Eq)]
230struct LocalCandidate {
231 name: &'static str,
232 cpu_millis_free: u64,
233 memory_bytes_free: u64,
234 gpu_free: usize,
235}
236
237fn fit_predicates(demand: Demand) -> Vec<Predicate<LocalCandidate>> {
242 let mut preds: Vec<Predicate<LocalCandidate>> = Vec::new();
243 if demand.cpu_millis > 0 {
244 preds.push(Box::new(move |c: &LocalCandidate| {
245 let req = ResourceRequest::new("cpu", format!("{}m", demand.cpu_millis));
246 if req.satisfied_by(&format!("{}m", c.cpu_millis_free)) {
247 None
248 } else {
249 Some(RejectionReason(format!(
250 "insufficient cpu: need {}m, {}m free",
251 demand.cpu_millis, c.cpu_millis_free
252 )))
253 }
254 }));
255 }
256 if demand.memory_bytes > 0 {
257 preds.push(Box::new(move |c: &LocalCandidate| {
258 let req = ResourceRequest::new("memory", demand.memory_bytes.to_string());
259 if req.satisfied_by(&c.memory_bytes_free.to_string()) {
260 None
261 } else {
262 Some(RejectionReason(format!(
263 "insufficient memory: need {} bytes, {} bytes free",
264 demand.memory_bytes, c.memory_bytes_free
265 )))
266 }
267 }));
268 }
269 if demand.gpu > 0 {
270 preds.push(Box::new(move |c: &LocalCandidate| {
271 let req = ResourceRequest::new("gpu", demand.gpu.to_string());
272 if req.satisfied_by(&c.gpu_free.to_string()) {
273 None
274 } else {
275 Some(RejectionReason(format!(
276 "insufficient gpu: need {}, {} free",
277 demand.gpu, c.gpu_free
278 )))
279 }
280 }));
281 }
282 preds
283}
284
285fn fit_over_local_candidate(
289 cpu_millis_free: u64,
290 memory_bytes_free: u64,
291 gpu_free: usize,
292 demand: Demand,
293) -> std::result::Result<(), String> {
294 let candidate = LocalCandidate {
295 name: "self",
296 cpu_millis_free,
297 memory_bytes_free,
298 gpu_free,
299 };
300 let preds = fit_predicates(demand);
301 let report = scheduling::select(&[candidate], &preds, |_, _| std::cmp::Ordering::Equal, 1);
305 if report.selected.is_some() {
306 Ok(())
307 } else {
308 let reasons: Vec<String> = report
309 .candidates
310 .iter()
311 .filter_map(|c| c.rejection_reason.clone())
312 .collect();
313 Err(reasons.join("; "))
314 }
315}
316
317#[derive(Debug, Default)]
318struct AdmissionState {
319 active_total: usize,
320 active_by_subject: HashMap<String, usize>,
321 active_by_group: HashMap<String, usize>,
322 cpu_millis_reserved: u64,
323 memory_bytes_reserved: u64,
324 gpu_reserved: usize,
325 queue: VecDeque<Arc<Notify>>,
330}
331
332enum AdmitReject {
340 Infeasible(String),
341 Quota(String),
342 Capacity(String),
343}
344
345fn check_feasible(
346 demand: Demand,
347 cfg: &AdmissionConfig,
348 max_sandboxes: usize,
349) -> std::result::Result<(), String> {
350 if max_sandboxes == 0 {
351 return Err("pool declares zero schedulable capacity (max_sandboxes = 0)".to_owned());
352 }
353 if let Some(total) = cfg.cpu_millis_total {
354 if demand.cpu_millis > total {
355 return Err(format!(
356 "cpu demand {}m exceeds node total {total}m",
357 demand.cpu_millis
358 ));
359 }
360 }
361 if let Some(total) = cfg.memory_bytes_total {
362 if demand.memory_bytes > total {
363 return Err(format!(
364 "memory demand {} bytes exceeds node total {total} bytes",
365 demand.memory_bytes
366 ));
367 }
368 }
369 if demand.gpu > cfg.gpu_total {
370 return Err(format!(
371 "gpu demand {} exceeds node total {}",
372 demand.gpu, cfg.gpu_total
373 ));
374 }
375
376 Ok(())
377}
378
379#[allow(clippy::too_many_arguments)]
384fn try_admit_locked(
385 state: &mut AdmissionState,
386 subject_key: &str,
387 group_key: Option<&str>,
388 demand: Demand,
389 cfg: &AdmissionConfig,
390 max_sandboxes: usize,
391) -> std::result::Result<(), AdmitReject> {
392 check_feasible(demand, cfg, max_sandboxes).map_err(AdmitReject::Infeasible)?;
397
398 if let Some(max) = cfg.max_per_subject {
400 let subj_count = *state.active_by_subject.get(subject_key).unwrap_or(&0);
401 if subj_count >= max {
402 return Err(AdmitReject::Quota(format!(
403 "subject '{subject_key}' at quota ({subj_count}/{max})"
404 )));
405 }
406 }
407 if let Some(g) = group_key {
408 if let Some(max) = cfg.max_per_group {
409 let gcount = *state.active_by_group.get(g).unwrap_or(&0);
410 if gcount >= max {
411 return Err(AdmitReject::Quota(format!(
412 "group '{g}' at quota ({gcount}/{max})"
413 )));
414 }
415 }
416 }
417
418 if state.active_total >= max_sandboxes {
420 return Err(AdmitReject::Capacity(format!(
421 "pool at capacity ({}/{max_sandboxes})",
422 state.active_total
423 )));
424 }
425 let cpu_free = cfg
426 .cpu_millis_total
427 .map(|t| t.saturating_sub(state.cpu_millis_reserved))
428 .unwrap_or(u64::MAX);
429 let memory_free = cfg
430 .memory_bytes_total
431 .map(|t| t.saturating_sub(state.memory_bytes_reserved))
432 .unwrap_or(u64::MAX);
433 let gpu_free = cfg.gpu_total.saturating_sub(state.gpu_reserved);
434 if let Err(reason) = fit_over_local_candidate(cpu_free, memory_free, gpu_free, demand) {
435 return Err(AdmitReject::Capacity(reason));
436 }
437
438 state.active_total += 1;
440 *state
441 .active_by_subject
442 .entry(subject_key.to_owned())
443 .or_insert(0) += 1;
444 if let Some(g) = group_key {
445 *state.active_by_group.entry(g.to_owned()).or_insert(0) += 1;
446 }
447 state.cpu_millis_reserved += demand.cpu_millis;
448 state.memory_bytes_reserved += demand.memory_bytes;
449 state.gpu_reserved += demand.gpu;
450 Ok(())
451}
452
453fn wake_head(state: &AdmissionState) {
458 if let Some(head) = state.queue.front() {
459 head.notify_one();
460 }
461}
462
463fn release_locked(state: &mut AdmissionState, record: &ReservationRecord) {
464 state.active_total = state.active_total.saturating_sub(1);
465 if let Some(c) = state.active_by_subject.get_mut(&record.subject) {
466 *c = c.saturating_sub(1);
467 if *c == 0 {
468 state.active_by_subject.remove(&record.subject);
469 }
470 }
471 if let Some(g) = &record.group {
472 if let Some(c) = state.active_by_group.get_mut(g) {
473 *c = c.saturating_sub(1);
474 if *c == 0 {
475 state.active_by_group.remove(g);
476 }
477 }
478 }
479 state.cpu_millis_reserved = state
480 .cpu_millis_reserved
481 .saturating_sub(record.demand.cpu_millis);
482 state.memory_bytes_reserved = state
483 .memory_bytes_reserved
484 .saturating_sub(record.demand.memory_bytes);
485 state.gpu_reserved = state.gpu_reserved.saturating_sub(record.demand.gpu);
486}
487
488#[async_trait]
510pub trait GroupSelectorValidator: Send + Sync + std::fmt::Debug {
511 async fn validate(&self, subject: &Subject, selector: Option<&str>) -> Result<Option<String>>;
517}
518
519#[derive(Debug, Default)]
525pub struct DenyUnknownGroupValidator;
526
527#[async_trait]
528impl GroupSelectorValidator for DenyUnknownGroupValidator {
529 async fn validate(&self, _subject: &Subject, selector: Option<&str>) -> Result<Option<String>> {
530 match selector {
531 None => Ok(None),
532 Some(sel) => Err(WorkerError::AdmissionDenied {
533 reason: format!(
534 "group selector '{sel}' cannot be verified: no membership source wired (fail-closed, B′)"
535 ),
536 }),
537 }
538 }
539}
540
541#[derive(Debug, Default)]
546pub struct StaticGroupMembership {
547 members: HashMap<String, HashSet<String>>,
548}
549
550impl StaticGroupMembership {
551 pub fn new() -> Self {
552 Self::default()
553 }
554
555 pub fn with_member(mut self, group: &str, subject: &str) -> Self {
557 self.members
558 .entry(group.to_owned())
559 .or_default()
560 .insert(subject.to_owned());
561 self
562 }
563}
564
565#[async_trait]
566impl GroupSelectorValidator for StaticGroupMembership {
567 async fn validate(&self, subject: &Subject, selector: Option<&str>) -> Result<Option<String>> {
568 match selector {
569 None => Ok(None),
570 Some(sel) => {
571 let subj = subject.name().unwrap_or("");
572 if self.members.get(sel).is_some_and(|m| m.contains(subj)) {
573 Ok(Some(sel.to_owned()))
574 } else {
575 Err(WorkerError::AdmissionDenied {
576 reason: format!(
577 "subject '{subj}' is not a consented member of group '{sel}'"
578 ),
579 })
580 }
581 }
582 }
583 }
584}
585
586#[derive(Debug)]
588pub struct AdmissionTracker {
589 state: Mutex<AdmissionState>,
590 config: AdmissionConfig,
591 max_sandboxes: usize,
592 group_validator: Arc<dyn GroupSelectorValidator>,
593}
594
595impl AdmissionTracker {
596 pub fn new(config: AdmissionConfig, max_sandboxes: usize) -> Self {
600 Self::with_group_validator(config, max_sandboxes, Arc::new(DenyUnknownGroupValidator))
601 }
602
603 pub fn with_group_validator(
606 config: AdmissionConfig,
607 max_sandboxes: usize,
608 group_validator: Arc<dyn GroupSelectorValidator>,
609 ) -> Self {
610 Self {
611 state: Mutex::new(AdmissionState::default()),
612 config,
613 max_sandboxes,
614 group_validator,
615 }
616 }
617
618 pub async fn reserve(
631 &self,
632 subject: &Subject,
633 group_selector: Option<&str>,
634 demand: Demand,
635 ) -> Result<ReservationRecord> {
636 if subject.is_anonymous() {
639 return Err(WorkerError::Unauthorized {
640 subject: "anonymous".to_owned(),
641 operation: "acquire".to_owned(),
642 resource: "sandbox-pool".to_owned(),
643 });
644 }
645 let subject_key = subject.name().unwrap_or("").to_owned();
646
647 let group: Option<String> = self
651 .group_validator
652 .validate(subject, group_selector)
653 .await?;
654 let group_key = group.as_deref();
655
656 let deadline = Instant::now() + Duration::from_secs(self.config.queue_timeout_secs);
657 let mut ticket: Option<Arc<Notify>> = None;
661
662 loop {
663 {
664 let mut state = self.state.lock().await;
665
666 let may_attempt = match &ticket {
670 Some(t) => state.queue.front().is_some_and(|h| Arc::ptr_eq(h, t)),
671 None => state.queue.is_empty(),
672 };
673
674 if may_attempt {
675 match try_admit_locked(
676 &mut state,
677 &subject_key,
678 group_key,
679 demand,
680 &self.config,
681 self.max_sandboxes,
682 ) {
683 Ok(()) => {
684 if ticket.is_some() {
685 state.queue.pop_front();
686 }
687 wake_head(&state);
688 return Ok(ReservationRecord {
689 subject: subject_key,
690 group,
691 demand,
692 });
693 }
694 Err(AdmitReject::Infeasible(reason)) => {
695 if ticket.is_some() {
696 state.queue.pop_front();
697 wake_head(&state);
698 }
699 return Err(WorkerError::AdmissionInfeasible { reason });
700 }
701 Err(AdmitReject::Quota(reason)) => {
702 if ticket.is_some() {
703 state.queue.pop_front();
704 wake_head(&state);
705 }
706 return Err(WorkerError::AdmissionDenied { reason });
707 }
708 Err(AdmitReject::Capacity(reason)) => {
709 if ticket.is_none() {
710 if state.queue.len() >= self.config.queue_capacity {
711 return Err(WorkerError::QueueFull {
712 waiting: state.queue.len(),
713 bound: self.config.queue_capacity,
714 });
715 }
716 tracing::debug!(subject = %subject_key, reason = %reason, "acquire: queueing for capacity (FIFO)");
717 let t = Arc::new(Notify::new());
718 state.queue.push_back(t.clone());
719 ticket = Some(t);
720 }
721 }
724 }
725 } else if ticket.is_none() {
726 if let Err(reason) = check_feasible(demand, &self.config, self.max_sandboxes) {
731 return Err(WorkerError::AdmissionInfeasible { reason });
732 }
733 if state.queue.len() >= self.config.queue_capacity {
734 return Err(WorkerError::QueueFull {
735 waiting: state.queue.len(),
736 bound: self.config.queue_capacity,
737 });
738 }
739 let t = Arc::new(Notify::new());
740 state.queue.push_back(t.clone());
741 ticket = Some(t);
742 }
743 }
744
745 let t = match &ticket {
749 Some(t) => t.clone(),
750 None => continue,
751 };
752 let remaining = deadline.saturating_duration_since(Instant::now());
753 let timed_out =
754 remaining.is_zero() || tokio::time::timeout(remaining, t.notified()).await.is_err();
755 if timed_out {
756 let mut state = self.state.lock().await;
757 let was_head = state.queue.front().is_some_and(|h| Arc::ptr_eq(h, &t));
758 state.queue.retain(|x| !Arc::ptr_eq(x, &t));
759 if was_head {
761 wake_head(&state);
762 }
763 return Err(WorkerError::QueueTimeout {
764 timeout_secs: self.config.queue_timeout_secs,
765 });
766 }
767 }
769 }
770
771 pub async fn release(&self, record: &ReservationRecord) {
775 let mut state = self.state.lock().await;
776 release_locked(&mut state, record);
777 wake_head(&state);
778 }
779
780 pub async fn rollback(&self, record: &ReservationRecord) {
785 self.release(record).await;
786 }
787}
788
789#[cfg(test)]
790#[allow(clippy::unwrap_used, clippy::expect_used)]
791mod tests {
792 use super::*;
793
794 fn demand(cpu_millis: u64, memory_bytes: u64, gpu: usize) -> Demand {
795 Demand {
796 cpu_millis,
797 memory_bytes,
798 gpu,
799 }
800 }
801
802 #[tokio::test]
803 async fn anonymous_subject_rejected_fail_closed() {
804 let tracker = AdmissionTracker::new(AdmissionConfig::default(), 10);
805 let err = tracker
806 .reserve(&Subject::anonymous(), None, demand(0, 0, 0))
807 .await
808 .unwrap_err();
809 assert!(
810 matches!(err, WorkerError::Unauthorized { .. }),
811 "got: {err:?}"
812 );
813 }
814
815 #[tokio::test]
816 async fn zero_demand_never_trips_resource_capacity() {
817 let tracker = AdmissionTracker::new(AdmissionConfig::default(), 10);
821 let r = tracker
822 .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
823 .await;
824 assert!(r.is_ok());
825 }
826
827 fn team_a_membership() -> Arc<dyn GroupSelectorValidator> {
831 Arc::new(
832 StaticGroupMembership::new()
833 .with_member("team-a", "alice")
834 .with_member("team-a", "bob"),
835 )
836 }
837
838 #[tokio::test]
839 async fn per_subject_quota_rejects_not_queues() {
840 let cfg = AdmissionConfig {
841 max_per_subject: Some(1),
842 ..Default::default()
843 };
844 let tracker = AdmissionTracker::new(cfg, 10);
845 let alice = Subject::new("alice");
846 let r1 = tracker.reserve(&alice, None, demand(0, 0, 0)).await;
847 assert!(r1.is_ok());
848 let r2 = tracker.reserve(&alice, None, demand(0, 0, 0)).await;
849 assert!(
850 matches!(r2, Err(WorkerError::AdmissionDenied { .. })),
851 "got: {r2:?}"
852 );
853 }
854
855 #[tokio::test]
856 async fn per_group_quota_rejects_across_subjects() {
857 let cfg = AdmissionConfig {
858 max_per_group: Some(1),
859 ..Default::default()
860 };
861 let tracker = AdmissionTracker::with_group_validator(cfg, 10, team_a_membership());
862 let r1 = tracker
863 .reserve(&Subject::new("alice"), Some("team-a"), demand(0, 0, 0))
864 .await;
865 assert!(r1.is_ok());
866 let r2 = tracker
867 .reserve(&Subject::new("bob"), Some("team-a"), demand(0, 0, 0))
868 .await;
869 assert!(
870 matches!(r2, Err(WorkerError::AdmissionDenied { .. })),
871 "got: {r2:?}"
872 );
873 }
874
875 #[tokio::test]
879 async fn group_selector_from_non_member_is_rejected() {
880 let validator = Arc::new(StaticGroupMembership::new().with_member("team-a", "alice"));
882 let tracker =
883 AdmissionTracker::with_group_validator(AdmissionConfig::default(), 10, validator);
884 let r_alice = tracker
886 .reserve(&Subject::new("alice"), Some("team-a"), demand(0, 0, 0))
887 .await;
888 assert!(
889 r_alice.is_ok(),
890 "member's selector must validate: {r_alice:?}"
891 );
892 let r_bob = tracker
895 .reserve(&Subject::new("bob"), Some("team-a"), demand(0, 0, 0))
896 .await;
897 assert!(
898 matches!(r_bob, Err(WorkerError::AdmissionDenied { .. })),
899 "non-member selector must be rejected: {r_bob:?}"
900 );
901 }
902
903 #[tokio::test]
904 async fn deny_unknown_group_validator_rejects_any_selector_but_allows_none() {
905 let tracker = AdmissionTracker::new(AdmissionConfig::default(), 10);
907 let r_none = tracker
909 .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
910 .await;
911 assert!(
912 r_none.is_ok(),
913 "absent selector must not partition: {r_none:?}"
914 );
915 let r_sel = tracker
917 .reserve(&Subject::new("alice"), Some("team-a"), demand(0, 0, 0))
918 .await;
919 assert!(
920 matches!(r_sel, Err(WorkerError::AdmissionDenied { .. })),
921 "fail-closed: unverifiable selector must be rejected: {r_sel:?}"
922 );
923 }
924
925 #[tokio::test]
926 async fn gpu_capacity_enforced_only_when_requested() {
927 let cfg = AdmissionConfig {
928 gpu_total: 1,
929 ..Default::default()
930 };
931 let tracker = AdmissionTracker::new(cfg, 10);
932 let r = tracker
934 .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
935 .await;
936 assert!(r.is_ok());
937 let r1 = tracker
939 .reserve(&Subject::new("bob"), None, demand(0, 0, 1))
940 .await;
941 assert!(r1.is_ok());
942 }
945
946 #[tokio::test]
947 async fn gpu_over_capacity_queues_then_times_out() {
948 let cfg = AdmissionConfig {
949 gpu_total: 1,
950 queue_timeout_secs: 0,
951 ..Default::default()
952 };
953 let tracker = AdmissionTracker::new(cfg, 10);
954 let r1 = tracker
955 .reserve(&Subject::new("alice"), None, demand(0, 0, 1))
956 .await;
957 assert!(r1.is_ok());
958 let r2 = tracker
959 .reserve(&Subject::new("bob"), None, demand(0, 0, 1))
960 .await;
961 assert!(
962 matches!(r2, Err(WorkerError::QueueTimeout { .. })),
963 "got: {r2:?}"
964 );
965 }
966
967 #[tokio::test]
971 async fn gpu_demand_exceeding_total_rejects_immediately_infeasible() {
972 let cfg = AdmissionConfig {
975 queue_timeout_secs: 30,
977 ..Default::default()
978 };
979 let tracker = AdmissionTracker::new(cfg, 10);
980 let start = Instant::now();
981 let r = tracker
982 .reserve(&Subject::new("alice"), None, demand(0, 0, 1))
983 .await;
984 assert!(
985 matches!(r, Err(WorkerError::AdmissionInfeasible { .. })),
986 "got: {r:?}"
987 );
988 assert!(
989 start.elapsed() < Duration::from_secs(1),
990 "infeasible demand must reject immediately, not wait out the timeout"
991 );
992 }
993
994 #[tokio::test]
995 async fn memory_demand_exceeding_total_is_infeasible() {
996 let cfg = AdmissionConfig {
997 memory_bytes_total: Some(256),
998 queue_timeout_secs: 30,
999 ..Default::default()
1000 };
1001 let tracker = AdmissionTracker::new(cfg, 10);
1002 let start = Instant::now();
1003 let r = tracker
1004 .reserve(&Subject::new("alice"), None, demand(0, 512, 0))
1005 .await;
1006 assert!(
1007 matches!(r, Err(WorkerError::AdmissionInfeasible { .. })),
1008 "got: {r:?}"
1009 );
1010 assert!(start.elapsed() < Duration::from_secs(1));
1011 }
1012
1013 #[tokio::test]
1014 async fn cpu_demand_exceeding_total_is_infeasible() {
1015 let cfg = AdmissionConfig {
1016 cpu_millis_total: Some(1000),
1017 queue_timeout_secs: 30,
1018 ..Default::default()
1019 };
1020 let tracker = AdmissionTracker::new(cfg, 10);
1021 let r = tracker
1022 .reserve(&Subject::new("alice"), None, demand(2000, 0, 0))
1023 .await;
1024 assert!(
1025 matches!(r, Err(WorkerError::AdmissionInfeasible { .. })),
1026 "got: {r:?}"
1027 );
1028 }
1029
1030 #[tokio::test]
1031 async fn demand_equal_to_total_is_feasible_not_infeasible() {
1032 let cfg = AdmissionConfig {
1036 gpu_total: 1,
1037 queue_timeout_secs: 0,
1038 ..Default::default()
1039 };
1040 let tracker = AdmissionTracker::new(cfg, 10);
1041 let r1 = tracker
1042 .reserve(&Subject::new("alice"), None, demand(0, 0, 1))
1043 .await;
1044 assert!(r1.is_ok(), "demand == total must fit: {r1:?}");
1045 let r2 = tracker
1046 .reserve(&Subject::new("bob"), None, demand(0, 0, 1))
1047 .await;
1048 assert!(
1049 matches!(r2, Err(WorkerError::QueueTimeout { .. })),
1050 "second equal request is busy (capacity), not infeasible: {r2:?}"
1051 );
1052 }
1053
1054 #[tokio::test]
1055 async fn infeasible_request_behind_waiter_rejects_without_taking_queue_slot() {
1056 let cfg = AdmissionConfig {
1057 gpu_total: 0,
1058 queue_capacity: 2,
1059 queue_timeout_secs: 5,
1060 ..Default::default()
1061 };
1062 let tracker = Arc::new(AdmissionTracker::new(cfg, 1));
1063 let held = tracker
1064 .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
1065 .await
1066 .expect("first request should occupy the only sandbox slot");
1067
1068 let t_bob = tracker.clone();
1069 let bob = tokio::spawn(async move {
1070 t_bob
1071 .reserve(&Subject::new("bob"), None, demand(0, 0, 0))
1072 .await
1073 });
1074 tokio::time::sleep(Duration::from_millis(50)).await;
1075
1076 let start = Instant::now();
1077 let carol = tracker
1078 .reserve(&Subject::new("carol"), None, demand(0, 0, 1))
1079 .await;
1080 assert!(
1081 matches!(carol, Err(WorkerError::AdmissionInfeasible { .. })),
1082 "infeasible demand behind a waiter must fail immediately: {carol:?}"
1083 );
1084 assert!(
1085 start.elapsed() < Duration::from_secs(1),
1086 "infeasible demand must not wait behind the existing queue"
1087 );
1088
1089 let t_dave = tracker.clone();
1090 let dave = tokio::spawn(async move {
1091 t_dave
1092 .reserve(&Subject::new("dave"), None, demand(0, 0, 0))
1093 .await
1094 });
1095 tokio::time::sleep(Duration::from_millis(50)).await;
1096 assert!(
1097 !dave.is_finished(),
1098 "feasible follow-up should fit in the queue slot not consumed by the infeasible request"
1099 );
1100
1101 tracker.release(&held).await;
1102 let bob_res = tokio::time::timeout(Duration::from_secs(2), bob)
1103 .await
1104 .expect("bob should be admitted")
1105 .expect("task ok")
1106 .expect("bob reservation ok");
1107 tracker.release(&bob_res).await;
1108 let dave_res = tokio::time::timeout(Duration::from_secs(2), dave)
1109 .await
1110 .expect("dave should be admitted")
1111 .expect("task ok");
1112 assert!(
1113 dave_res.is_ok(),
1114 "dave should have remained queued: {dave_res:?}"
1115 );
1116 }
1117
1118 #[tokio::test]
1121 async fn fifo_earlier_waiter_is_not_starved_by_later_arrivals() {
1122 let cfg = AdmissionConfig {
1125 queue_timeout_secs: 5,
1126 ..Default::default()
1127 };
1128 let tracker = Arc::new(AdmissionTracker::new(cfg, 1));
1129 let held = tracker
1130 .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
1131 .await
1132 .unwrap();
1133
1134 let t_bob = tracker.clone();
1135 let bob = tokio::spawn(async move {
1136 t_bob
1137 .reserve(&Subject::new("bob"), None, demand(0, 0, 0))
1138 .await
1139 });
1140 tokio::time::sleep(Duration::from_millis(50)).await;
1142
1143 let t_carol = tracker.clone();
1144 let carol = tokio::spawn(async move {
1145 t_carol
1146 .reserve(&Subject::new("carol"), None, demand(0, 0, 0))
1147 .await
1148 });
1149 tokio::time::sleep(Duration::from_millis(50)).await;
1150
1151 tracker.release(&held).await;
1153
1154 let bob_res = tokio::time::timeout(Duration::from_secs(2), bob)
1156 .await
1157 .expect("bob should be woken")
1158 .expect("task ok");
1159 assert!(
1160 bob_res.is_ok(),
1161 "earlier waiter bob must be admitted: {bob_res:?}"
1162 );
1163
1164 tokio::time::sleep(Duration::from_millis(50)).await;
1167 assert!(
1168 !carol.is_finished(),
1169 "later arrival carol must not barge ahead of / with bob"
1170 );
1171
1172 tracker.release(&bob_res.unwrap()).await;
1174 let carol_res = tokio::time::timeout(Duration::from_secs(2), carol)
1175 .await
1176 .expect("carol should now be woken")
1177 .expect("task ok");
1178 assert!(
1179 carol_res.is_ok(),
1180 "carol admitted once capacity frees: {carol_res:?}"
1181 );
1182 }
1183
1184 #[tokio::test]
1185 async fn queue_full_rejects_immediately() {
1186 let cfg = AdmissionConfig {
1187 queue_capacity: 0,
1188 queue_timeout_secs: 5,
1189 ..Default::default()
1190 };
1191 let tracker = AdmissionTracker::new(cfg, 1);
1194 let r1 = tracker
1195 .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
1196 .await;
1197 assert!(r1.is_ok());
1198 let start = Instant::now();
1199 let r2 = tracker
1200 .reserve(&Subject::new("bob"), None, demand(0, 0, 0))
1201 .await;
1202 assert!(
1203 matches!(r2, Err(WorkerError::QueueFull { .. })),
1204 "got: {r2:?}"
1205 );
1206 assert!(
1207 start.elapsed() < Duration::from_secs(1),
1208 "must reject immediately, not wait out the timeout"
1209 );
1210 }
1211
1212 #[tokio::test]
1213 async fn release_wakes_a_queued_waiter() {
1214 let cfg = AdmissionConfig {
1215 queue_timeout_secs: 5,
1216 ..Default::default()
1217 };
1218 let tracker = Arc::new(AdmissionTracker::new(cfg, 1));
1219 let r1 = tracker
1220 .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
1221 .await
1222 .unwrap();
1223
1224 let t2 = tracker.clone();
1225 let waiter = tokio::spawn(async move {
1226 t2.reserve(&Subject::new("bob"), None, demand(0, 0, 0))
1227 .await
1228 });
1229
1230 tokio::time::sleep(Duration::from_millis(50)).await;
1232 tracker.release(&r1).await;
1233
1234 let r2 = tokio::time::timeout(Duration::from_secs(2), waiter)
1235 .await
1236 .expect("waiter should be woken well within 2s")
1237 .expect("task should not panic");
1238 assert!(
1239 r2.is_ok(),
1240 "queued waiter should be admitted once capacity frees: {r2:?}"
1241 );
1242 }
1243
1244 #[tokio::test]
1245 async fn derive_demand_is_zero_for_default_config() {
1246 let config = PodSandboxConfig::default();
1247 assert_eq!(derive_demand(&config), Demand::default());
1248 assert_eq!(derive_group_selector(&config), None);
1249 }
1250
1251 #[tokio::test]
1252 async fn derive_demand_reads_cpu_memory_and_gpu_annotation() {
1253 let mut config = PodSandboxConfig::default();
1254 config.linux.resources.cpu_period = 100_000;
1255 config.linux.resources.cpu_quota = 50_000;
1256 config.linux.resources.memory_limit_in_bytes = 1024;
1257 config.annotations.push(KeyValue {
1258 key: ANN_GPU_REQUEST.to_owned(),
1259 value: "2".to_owned(),
1260 });
1261 config.annotations.push(KeyValue {
1262 key: ANN_GROUP.to_owned(),
1263 value: "team-a".to_owned(),
1264 });
1265
1266 let d = derive_demand(&config);
1267 assert_eq!(d.cpu_millis, 500);
1268 assert_eq!(d.memory_bytes, 1024);
1269 assert_eq!(d.gpu, 2);
1270 assert_eq!(derive_group_selector(&config).as_deref(), Some("team-a"));
1271 }
1272}