Skip to main content

hyprstream_workers/runtime/
admission.rs

1//! Admission control for `SandboxPool::acquire` (#525, P2 of epic #523).
2//!
3//! Replaces the old "check `active.len()`, then either pop/create" shape
4//! (immediate `PoolExhausted`, no queue, no identity, no resource dimension)
5//! with a real decision engine:
6//!
7//! - **Fail-closed identity**: no verified [`Subject`] → reject before doing
8//!   any bookkeeping. There is no "anonymous but allowed" path.
9//! - **Reserve-then-place, never check-then-place** (the #489/#519 TOCTOU
10//!   lesson): every admission decision — capacity, per-Subject quota,
11//!   per-group quota, and cpu/memory/GPU capacity — is checked *and*
12//!   committed atomically under one lock in [`try_admit_locked`]. There is no
13//!   window between "checked OK" and "reserved" where a second caller can
14//!   slip in and overshoot.
15//! - **Bounded wait-queue with backpressure**: a request that cannot be
16//!   admitted immediately because of *capacity* (never quota — see below)
17//!   waits, bounded by [`AdmissionConfig::queue_capacity`] and
18//!   [`AdmissionConfig::queue_timeout_secs`]. Over the bound, or past the
19//!   timeout, it fails clearly rather than blocking forever or silently
20//!   degrading.
21//!
22//! ## 🔒 NEEDS REVIEWER SIGN-OFF
23//!
24//! This module makes several judgment calls the issue explicitly flags as
25//! needing sign-off rather than being final:
26//!
27//! 1. **Quota model shape**: a simple in-memory per-Subject/per-group counter
28//!    with an operator-configured numeric ceiling ([`AdmissionConfig`]). This
29//!    is *not* Casbin-policy-driven — there is no policy lookup, no dynamic
30//!    per-tenant configuration source, just a static number in `PoolConfig`.
31//!    Full Casbin-backed quota configuration (e.g. deriving `max_per_subject`
32//!    from a Casbin role/policy rather than a flat config value) is a
33//!    follow-up.
34//! 2. **"Group" derivation (B′, ratified in #921 decision 5)**: the group is
35//!    derived from the **verified [`Subject`], never trusted from the
36//!    annotation**. The `hyprstream.io/group` annotation is only a *selector*
37//!    among groups the Subject provably belongs to; a
38//!    [`GroupSelectorValidator`] resolves it against the authoritative
39//!    membership source (the bidirectional-consent placement group,
40//!    `ai.hyprstream.placement.{group,groupItem}` / `PlacementIndex::is_member`).
41//!    A selector naming a group the Subject is not a member of is **rejected**;
42//!    an absent selector means **no group partition** (never a quota bypass).
43//!    The default [`DenyUnknownGroupValidator`] is fail-closed: it denies every
44//!    non-empty selector until a membership-backed validator is wired, so an
45//!    unverified group can never partition capacity. See point 3 for what the
46//!    counter itself means.
47//! 3. **Wait-queue fairness**: the queue is a single bounded **FIFO** — new
48//!    arrivals never barge past a non-empty queue, and each release hands
49//!    capacity to the queue *head* (rather than waking all waiters to
50//!    thundering-herd re-race). This eliminates the starvation the blocking
51//!    review flagged (a queued waiter losing every re-race under sustained
52//!    arrival). The tradeoff is **head-of-line blocking**: a large demand at
53//!    the head can block a smaller demand behind it even when capacity for the
54//!    smaller one exists. That is the conservative choice; a per-Subject/
55//!    per-group fair sub-queue + weighted round-robin is the documented (not
56//!    implemented) follow-up. Quota- and infeasibility-rejected requests never
57//!    queue (they fail fast).
58//! 4. **GPU/resource vocabulary**: cpu/memory/GPU demand is read from
59//!    `config.linux.resources` (existing CRI-shaped fields) plus a new,
60//!    separate annotation `hyprstream.io/gpu-request` (a plain integer count,
61//!    not a GPU *class*) — independent of the existing boolean
62//!    `hyprstream.io/gpu` passthrough annotation consumed by `oci_backend`/
63//!    `nspawn`. The two are intentionally not unified in this change; see the
64//!    module docs on `pool.rs` for the full rationale.
65use std::collections::{HashMap, HashSet, VecDeque};
66use std::sync::Arc;
67use std::time::Duration;
68
69use async_trait::async_trait;
70use serde::{Deserialize, Serialize};
71use tokio::sync::{Mutex, Notify};
72use tokio::time::Instant;
73
74use hyprstream_discovery::scheduling::{self, Predicate, RejectionReason, ResourceRequest};
75use hyprstream_vfs::Subject;
76
77use crate::error::{Result, WorkerError};
78
79use hyprstream_rpc_std::worker_client::{KeyValue, PodSandboxConfig};
80
81/// Annotation key: per-request GPU count demand (a plain non-negative
82/// integer string, e.g. `"1"`). Independent of `oci_backend`'s existing
83/// boolean `hyprstream.io/gpu` passthrough flag — see module docs, point 4.
84pub const ANN_GPU_REQUEST: &str = "hyprstream.io/gpu-request";
85
86/// Annotation key: the group *selector* for this request (B′, #921 decision
87/// 5). This is **not** a trusted group claim — it only names which of the
88/// groups the verified [`Subject`] *provably belongs to* this request should
89/// be partitioned under. A [`GroupSelectorValidator`] resolves it against the
90/// authoritative membership source; a selector the Subject is not a member of
91/// is rejected. See module docs, point 2.
92pub const ANN_GROUP: &str = "hyprstream.io/group";
93
94fn annotation<'a>(annotations: &'a [KeyValue], key: &str) -> Option<&'a str> {
95    annotations
96        .iter()
97        .find(|kv| kv.key == key)
98        .map(|kv| kv.value.as_str())
99}
100
101/// Resource demand derived from a [`PodSandboxConfig`] for admission purposes.
102#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
103pub struct Demand {
104    pub cpu_millis: u64,
105    pub memory_bytes: u64,
106    pub gpu: usize,
107}
108
109/// Derive the resource `Demand` this request represents.
110///
111/// CRI convention: `0` means "unspecified" for `cpu_period`/`cpu_quota`/
112/// `memory_limit_in_bytes`, so an all-zero (`Default`) `LinuxContainerResources`
113/// — what every pre-#525 caller passes — derives a zero `Demand`, and a zero
114/// `Demand` never trips a capacity check. This is what keeps the solo/no-op
115/// case regression-free (acceptance criterion: "no regression for the
116/// existing solo case").
117pub fn derive_demand(config: &PodSandboxConfig) -> Demand {
118    let res = &config.linux.resources;
119    let cpu_millis = if res.cpu_period > 0 && res.cpu_quota > 0 {
120        ((res.cpu_quota as f64 / res.cpu_period as f64) * 1000.0).round() as u64
121    } else {
122        0
123    };
124    let memory_bytes = res.memory_limit_in_bytes.max(0) as u64;
125    let gpu = annotation(&config.annotations, ANN_GPU_REQUEST)
126        .and_then(|v| v.parse::<usize>().ok())
127        .unwrap_or(0);
128    Demand {
129        cpu_millis,
130        memory_bytes,
131        gpu,
132    }
133}
134
135/// Derive the group *selector* for a request, if any (the raw
136/// `hyprstream.io/group` annotation). This is an **untrusted** selector, not
137/// an authoritative group: it must be validated against the verified Subject's
138/// membership by a [`GroupSelectorValidator`] before it can partition capacity
139/// (B′ — see module docs, point 2).
140pub fn derive_group_selector(config: &PodSandboxConfig) -> Option<String> {
141    annotation(&config.annotations, ANN_GROUP).map(str::to_owned)
142}
143
144/// Per-Subject / per-group admission quotas + wait-queue bounds + declared
145/// local schedulable capacity (#525 P2).
146///
147/// 🔒 See the module-level "NEEDS REVIEWER SIGN-OFF" doc for the judgment
148/// calls baked into this shape.
149///
150/// All limits default to **unconstrained** so that a `PoolConfig` with no
151/// explicit `admission` section behaves exactly as before this change (no
152/// regression for the existing solo/no-quota case): only an operator who
153/// opts in by lowering these gets admission-control behavior beyond the
154/// pre-existing `max_sandboxes` capacity check.
155#[derive(Debug, Clone, Serialize, Deserialize)]
156#[serde(default)]
157pub struct AdmissionConfig {
158    /// Maximum concurrently-active sandboxes for a single Subject.
159    /// `None` (default) = unconstrained.
160    ///
161    /// Serialized as *absent* when unlimited (rather than a sentinel like
162    /// `usize::MAX`, which is not representable as a TOML i64 and would break
163    /// `Config::save()`).
164    #[serde(skip_serializing_if = "Option::is_none")]
165    pub max_per_subject: Option<usize>,
166    /// Maximum concurrently-active sandboxes for a single group (see
167    /// [`derive_group_selector`]). `None` (default) = unconstrained.
168    ///
169    /// This counter is **node-local capacity partitioning** — a fairness /
170    /// organization mechanism, *not* a trust or billing boundary. The
171    /// authoritative quota is the ledger/capability path (#922/#925: verify
172    /// presented grant → debit cell ledger → emit receipt), which this module
173    /// deliberately does **not** implement. See module docs, point 2.
174    ///
175    /// Serialized as *absent* when unlimited (see [`Self::max_per_subject`]).
176    #[serde(skip_serializing_if = "Option::is_none")]
177    pub max_per_group: Option<usize>,
178    /// Bound on the number of callers allowed to wait in the admission queue
179    /// at once. A request that would exceed this is rejected immediately
180    /// (`QueueFull`) rather than queued.
181    pub queue_capacity: usize,
182    /// How long a queued request waits for capacity before giving up
183    /// (`QueueTimeout`).
184    pub queue_timeout_secs: u64,
185    /// Total schedulable CPU (millicores) for this node's local capacity
186    /// overlay. `None` (default) = the CPU dimension is not enforced.
187    pub cpu_millis_total: Option<u64>,
188    /// Total schedulable memory (bytes) for this node's local capacity
189    /// overlay. `None` (default) = the memory dimension is not enforced.
190    pub memory_bytes_total: Option<u64>,
191    /// Total GPU count available for scheduling. Default `0`: a request that
192    /// asks for a GPU (`Demand::gpu > 0`) is fail-closed rejected unless an
193    /// operator explicitly declares GPU capacity here. Requests that don't
194    /// ask for a GPU (`Demand::gpu == 0`, the default) are never affected by
195    /// this value.
196    pub gpu_total: usize,
197}
198
199impl Default for AdmissionConfig {
200    fn default() -> Self {
201        Self {
202            max_per_subject: None,
203            max_per_group: None,
204            queue_capacity: 64,
205            queue_timeout_secs: 30,
206            cpu_millis_total: None,
207            memory_bytes_total: None,
208            gpu_total: 0,
209        }
210    }
211}
212
213/// What a committed reservation consumed, so [`AdmissionTracker::release`] /
214/// [`AdmissionTracker::rollback`] can give it back precisely.
215#[derive(Debug, Clone, PartialEq, Eq)]
216pub struct ReservationRecord {
217    subject: String,
218    group: Option<String>,
219    demand: Demand,
220}
221
222/// A local resource-capacity candidate for the #628 scheduling substrate's
223/// `filter → rank → select` pipeline (see module docs on GPU/resource
224/// vocabulary). In the single-node/solo case this is the *only* candidate;
225/// the extension point for cross-node placement is `rank_by` in
226/// [`fit_over_local_candidate`] — feeding `queryCandidates` results in as additional
227/// candidates here (with a real least-loaded/best-fit `rank_by`) is P2's
228/// documented (not implemented) cross-node extension.
229#[derive(Debug, Clone, PartialEq, Eq)]
230struct LocalCandidate {
231    name: &'static str,
232    cpu_millis_free: u64,
233    memory_bytes_free: u64,
234    gpu_free: usize,
235}
236
237/// Build the fit predicates for `demand` over a [`LocalCandidate`]. Only
238/// dimensions actually requested (or actually enforced by config) produce a
239/// predicate — an all-zero `Demand` (the default/no-op case) yields no
240/// predicates, so it always "fits."
241fn fit_predicates(demand: Demand) -> Vec<Predicate<LocalCandidate>> {
242    let mut preds: Vec<Predicate<LocalCandidate>> = Vec::new();
243    if demand.cpu_millis > 0 {
244        preds.push(Box::new(move |c: &LocalCandidate| {
245            let req = ResourceRequest::new("cpu", format!("{}m", demand.cpu_millis));
246            if req.satisfied_by(&format!("{}m", c.cpu_millis_free)) {
247                None
248            } else {
249                Some(RejectionReason(format!(
250                    "insufficient cpu: need {}m, {}m free",
251                    demand.cpu_millis, c.cpu_millis_free
252                )))
253            }
254        }));
255    }
256    if demand.memory_bytes > 0 {
257        preds.push(Box::new(move |c: &LocalCandidate| {
258            let req = ResourceRequest::new("memory", demand.memory_bytes.to_string());
259            if req.satisfied_by(&c.memory_bytes_free.to_string()) {
260                None
261            } else {
262                Some(RejectionReason(format!(
263                    "insufficient memory: need {} bytes, {} bytes free",
264                    demand.memory_bytes, c.memory_bytes_free
265                )))
266            }
267        }));
268    }
269    if demand.gpu > 0 {
270        preds.push(Box::new(move |c: &LocalCandidate| {
271            let req = ResourceRequest::new("gpu", demand.gpu.to_string());
272            if req.satisfied_by(&c.gpu_free.to_string()) {
273                None
274            } else {
275                Some(RejectionReason(format!(
276                    "insufficient gpu: need {}, {} free",
277                    demand.gpu, c.gpu_free
278                )))
279            }
280        }));
281    }
282    preds
283}
284
285/// Run the #628 scheduling substrate's `filter → rank → select` over the
286/// (today: single) local candidate for `demand`. Returns `Ok(())` if it fits,
287/// `Err(reason)` with the joined rejection reasons otherwise.
288fn fit_over_local_candidate(
289    cpu_millis_free: u64,
290    memory_bytes_free: u64,
291    gpu_free: usize,
292    demand: Demand,
293) -> std::result::Result<(), String> {
294    let candidate = LocalCandidate {
295        name: "self",
296        cpu_millis_free,
297        memory_bytes_free,
298        gpu_free,
299    };
300    let preds = fit_predicates(demand);
301    // Single candidate today, so ranking is a no-op (`Ordering::Equal`); the
302    // rank_by closure is the seam a cross-node extension plugs a real
303    // least-loaded/best-fit comparator into.
304    let report = scheduling::select(&[candidate], &preds, |_, _| std::cmp::Ordering::Equal, 1);
305    if report.selected.is_some() {
306        Ok(())
307    } else {
308        let reasons: Vec<String> = report
309            .candidates
310            .iter()
311            .filter_map(|c| c.rejection_reason.clone())
312            .collect();
313        Err(reasons.join("; "))
314    }
315}
316
317#[derive(Debug, Default)]
318struct AdmissionState {
319    active_total: usize,
320    active_by_subject: HashMap<String, usize>,
321    active_by_group: HashMap<String, usize>,
322    cpu_millis_reserved: u64,
323    memory_bytes_reserved: u64,
324    gpu_reserved: usize,
325    /// Bounded **FIFO** wait-queue. A release hands capacity to the *head*
326    /// (front) only; a new arrival never barges past a non-empty queue. See
327    /// module docs, point 3, for the head-of-line-blocking tradeoff and why
328    /// this is not yet a per-key fair sub-queue.
329    queue: VecDeque<Arc<Notify>>,
330}
331
332/// Why [`try_admit_locked`] refused to admit. Three failure classes with
333/// different queueing semantics:
334/// - `Infeasible`: demand exceeds the node's *configured totals* — no release
335///   can ever satisfy it. Never queues; immediate hard reject.
336/// - `Quota`: the Subject/group is at its per-key ceiling. Never queues;
337///   retrying won't help until that Subject/group frees up on its own.
338/// - `Capacity`: transient — a release elsewhere can free room. May queue.
339enum AdmitReject {
340    Infeasible(String),
341    Quota(String),
342    Capacity(String),
343}
344
345fn check_feasible(
346    demand: Demand,
347    cfg: &AdmissionConfig,
348    max_sandboxes: usize,
349) -> std::result::Result<(), String> {
350    if max_sandboxes == 0 {
351        return Err("pool declares zero schedulable capacity (max_sandboxes = 0)".to_owned());
352    }
353    if let Some(total) = cfg.cpu_millis_total {
354        if demand.cpu_millis > total {
355            return Err(format!(
356                "cpu demand {}m exceeds node total {total}m",
357                demand.cpu_millis
358            ));
359        }
360    }
361    if let Some(total) = cfg.memory_bytes_total {
362        if demand.memory_bytes > total {
363            return Err(format!(
364                "memory demand {} bytes exceeds node total {total} bytes",
365                demand.memory_bytes
366            ));
367        }
368    }
369    if demand.gpu > cfg.gpu_total {
370        return Err(format!(
371            "gpu demand {} exceeds node total {}",
372            demand.gpu, cfg.gpu_total
373        ));
374    }
375
376    Ok(())
377}
378
379/// Check admission for `(subject_key, group_key, demand)` and, if it fits,
380/// commit the reservation — atomically, under the caller's lock. This is the
381/// entire "reserve-then-place" critical section; there is no gap between
382/// "checked OK" and "reserved" (the #489/#519 TOCTOU lesson).
383#[allow(clippy::too_many_arguments)]
384fn try_admit_locked(
385    state: &mut AdmissionState,
386    subject_key: &str,
387    group_key: Option<&str>,
388    demand: Demand,
389    cfg: &AdmissionConfig,
390    max_sandboxes: usize,
391) -> std::result::Result<(), AdmitReject> {
392    // ── Feasibility first: demand exceeding the configured TOTALS (not just
393    //    the currently-free amount) can never be satisfied by any release, so
394    //    it must reject immediately rather than burn the whole queue_timeout.
395    //    Checked independently of `*_reserved`. ──
396    check_feasible(demand, cfg, max_sandboxes).map_err(AdmitReject::Infeasible)?;
397
398    // ── Quota checks: fail-closed, never queue ──
399    if let Some(max) = cfg.max_per_subject {
400        let subj_count = *state.active_by_subject.get(subject_key).unwrap_or(&0);
401        if subj_count >= max {
402            return Err(AdmitReject::Quota(format!(
403                "subject '{subject_key}' at quota ({subj_count}/{max})"
404            )));
405        }
406    }
407    if let Some(g) = group_key {
408        if let Some(max) = cfg.max_per_group {
409            let gcount = *state.active_by_group.get(g).unwrap_or(&0);
410            if gcount >= max {
411                return Err(AdmitReject::Quota(format!(
412                    "group '{g}' at quota ({gcount}/{max})"
413                )));
414            }
415        }
416    }
417
418    // ── Capacity checks: may queue ──
419    if state.active_total >= max_sandboxes {
420        return Err(AdmitReject::Capacity(format!(
421            "pool at capacity ({}/{max_sandboxes})",
422            state.active_total
423        )));
424    }
425    let cpu_free = cfg
426        .cpu_millis_total
427        .map(|t| t.saturating_sub(state.cpu_millis_reserved))
428        .unwrap_or(u64::MAX);
429    let memory_free = cfg
430        .memory_bytes_total
431        .map(|t| t.saturating_sub(state.memory_bytes_reserved))
432        .unwrap_or(u64::MAX);
433    let gpu_free = cfg.gpu_total.saturating_sub(state.gpu_reserved);
434    if let Err(reason) = fit_over_local_candidate(cpu_free, memory_free, gpu_free, demand) {
435        return Err(AdmitReject::Capacity(reason));
436    }
437
438    // ── Admit: commit the reservation ──
439    state.active_total += 1;
440    *state
441        .active_by_subject
442        .entry(subject_key.to_owned())
443        .or_insert(0) += 1;
444    if let Some(g) = group_key {
445        *state.active_by_group.entry(g.to_owned()).or_insert(0) += 1;
446    }
447    state.cpu_millis_reserved += demand.cpu_millis;
448    state.memory_bytes_reserved += demand.memory_bytes;
449    state.gpu_reserved += demand.gpu;
450    Ok(())
451}
452
453/// Wake the current FIFO head (if any) so it retries admission. `Notify`
454/// stores one permit if the head isn't yet awaiting, so this never loses a
455/// wakeup. Idempotent-ish: a spurious extra wake just makes the head re-check
456/// under the lock.
457fn wake_head(state: &AdmissionState) {
458    if let Some(head) = state.queue.front() {
459        head.notify_one();
460    }
461}
462
463fn release_locked(state: &mut AdmissionState, record: &ReservationRecord) {
464    state.active_total = state.active_total.saturating_sub(1);
465    if let Some(c) = state.active_by_subject.get_mut(&record.subject) {
466        *c = c.saturating_sub(1);
467        if *c == 0 {
468            state.active_by_subject.remove(&record.subject);
469        }
470    }
471    if let Some(g) = &record.group {
472        if let Some(c) = state.active_by_group.get_mut(g) {
473            *c = c.saturating_sub(1);
474            if *c == 0 {
475                state.active_by_group.remove(g);
476            }
477        }
478    }
479    state.cpu_millis_reserved = state
480        .cpu_millis_reserved
481        .saturating_sub(record.demand.cpu_millis);
482    state.memory_bytes_reserved = state
483        .memory_bytes_reserved
484        .saturating_sub(record.demand.memory_bytes);
485    state.gpu_reserved = state.gpu_reserved.saturating_sub(record.demand.gpu);
486}
487
488/// Resolves an untrusted group *selector* against the verified [`Subject`]'s
489/// membership (B′, ratified in #921 decision 5).
490///
491/// The group is **never** trusted from the annotation. The annotation is only
492/// a selector naming which of the groups the Subject *provably belongs to* a
493/// request should be partitioned under. This trait is the seam that resolves
494/// that selector against the authoritative membership source — the
495/// bidirectional-consent placement group
496/// (`ai.hyprstream.placement.{group,groupItem}` / `PlacementIndex::is_member`,
497/// in `hyprstream-pds`/`hyprstream-discovery`).
498///
499/// # Fail-closed
500///
501/// The default impl ([`DenyUnknownGroupValidator`]) denies **every** non-empty
502/// selector, so a pool constructed without an explicit membership-backed
503/// validator can never partition capacity by an unverified group. Wire a
504/// real membership-source-backed impl at construction (where the discovery /
505/// PDS client is available) to enable group partitioning.
506///
507/// Returning `Ok(None)` for an absent selector means **no group partition** —
508/// never a quota bypass.
509#[async_trait]
510pub trait GroupSelectorValidator: Send + Sync + std::fmt::Debug {
511    /// Resolve `selector` for `subject`:
512    /// - `Ok(None)` — no selector present → no group partition.
513    /// - `Ok(Some(group))` — selector validated: `subject` is a consented
514    ///   member of `group`, which becomes the authoritative partition key.
515    /// - `Err(_)` — selector present but `subject` is not a member → reject.
516    async fn validate(&self, subject: &Subject, selector: Option<&str>) -> Result<Option<String>>;
517}
518
519/// Fail-closed default [`GroupSelectorValidator`]: denies every non-empty
520/// group selector (no membership source is wired), permits the absent-selector
521/// (no-partition) case. This is the production default until a real
522/// membership-backed validator is wired — an unverified group can never
523/// partition capacity.
524#[derive(Debug, Default)]
525pub struct DenyUnknownGroupValidator;
526
527#[async_trait]
528impl GroupSelectorValidator for DenyUnknownGroupValidator {
529    async fn validate(&self, _subject: &Subject, selector: Option<&str>) -> Result<Option<String>> {
530        match selector {
531            None => Ok(None),
532            Some(sel) => Err(WorkerError::AdmissionDenied {
533                reason: format!(
534                    "group selector '{sel}' cannot be verified: no membership source wired (fail-closed, B′)"
535                ),
536            }),
537        }
538    }
539}
540
541/// Reference [`GroupSelectorValidator`] backed by a static in-memory
542/// membership set (group → member Subject keys). Models the authoritative
543/// bidirectional-consent membership fact without the PDS/discovery plumbing;
544/// a production deployment wires an `is_member`-backed impl instead.
545#[derive(Debug, Default)]
546pub struct StaticGroupMembership {
547    members: HashMap<String, HashSet<String>>,
548}
549
550impl StaticGroupMembership {
551    pub fn new() -> Self {
552        Self::default()
553    }
554
555    /// Record that `subject` is a consented member of `group`.
556    pub fn with_member(mut self, group: &str, subject: &str) -> Self {
557        self.members
558            .entry(group.to_owned())
559            .or_default()
560            .insert(subject.to_owned());
561        self
562    }
563}
564
565#[async_trait]
566impl GroupSelectorValidator for StaticGroupMembership {
567    async fn validate(&self, subject: &Subject, selector: Option<&str>) -> Result<Option<String>> {
568        match selector {
569            None => Ok(None),
570            Some(sel) => {
571                let subj = subject.name().unwrap_or("");
572                if self.members.get(sel).is_some_and(|m| m.contains(subj)) {
573                    Ok(Some(sel.to_owned()))
574                } else {
575                    Err(WorkerError::AdmissionDenied {
576                        reason: format!(
577                            "subject '{subj}' is not a consented member of group '{sel}'"
578                        ),
579                    })
580                }
581            }
582        }
583    }
584}
585
586/// Admission decision engine backing `SandboxPool::acquire` (#525 P2).
587#[derive(Debug)]
588pub struct AdmissionTracker {
589    state: Mutex<AdmissionState>,
590    config: AdmissionConfig,
591    max_sandboxes: usize,
592    group_validator: Arc<dyn GroupSelectorValidator>,
593}
594
595impl AdmissionTracker {
596    /// Construct with the fail-closed [`DenyUnknownGroupValidator`] — the
597    /// production default until a membership-backed validator is wired
598    /// (any group selector is rejected; the no-selector case is unaffected).
599    pub fn new(config: AdmissionConfig, max_sandboxes: usize) -> Self {
600        Self::with_group_validator(config, max_sandboxes, Arc::new(DenyUnknownGroupValidator))
601    }
602
603    /// Construct with an explicit [`GroupSelectorValidator`] (e.g. one backed
604    /// by a real membership source, or a test double).
605    pub fn with_group_validator(
606        config: AdmissionConfig,
607        max_sandboxes: usize,
608        group_validator: Arc<dyn GroupSelectorValidator>,
609    ) -> Self {
610        Self {
611            state: Mutex::new(AdmissionState::default()),
612            config,
613            max_sandboxes,
614            group_validator,
615        }
616    }
617
618    /// Reserve capacity for `(subject, group_selector, demand)`, fail-closed on
619    /// unauthenticated callers, queueing (bounded, **FIFO**) when only
620    /// *capacity* (not quota/infeasibility) blocks admission.
621    ///
622    /// `group_selector` is the untrusted `hyprstream.io/group` annotation; it
623    /// is resolved against the verified Subject's membership by the configured
624    /// [`GroupSelectorValidator`] (B′). A selector the Subject is not a member
625    /// of is rejected; an absent selector means no group partition.
626    ///
627    /// FIFO (F4): a new arrival never barges past a non-empty queue, and each
628    /// release hands capacity to the queue head. See module docs, point 3, for
629    /// the head-of-line-blocking tradeoff.
630    pub async fn reserve(
631        &self,
632        subject: &Subject,
633        group_selector: Option<&str>,
634        demand: Demand,
635    ) -> Result<ReservationRecord> {
636        // Fail-closed identity: no verified Subject → reject before any
637        // bookkeeping, queueing, or lock acquisition whatsoever.
638        if subject.is_anonymous() {
639            return Err(WorkerError::Unauthorized {
640                subject: "anonymous".to_owned(),
641                operation: "acquire".to_owned(),
642                resource: "sandbox-pool".to_owned(),
643            });
644        }
645        let subject_key = subject.name().unwrap_or("").to_owned();
646
647        // B′: derive the authoritative group from the verified Subject, never
648        // trust the annotation. Rejects a selector the Subject is not a member
649        // of; `None` means no group partition.
650        let group: Option<String> = self
651            .group_validator
652            .validate(subject, group_selector)
653            .await?;
654        let group_key = group.as_deref();
655
656        let deadline = Instant::now() + Duration::from_secs(self.config.queue_timeout_secs);
657        // Our place in the FIFO queue, once we join it. `None` until we first
658        // fail an admission attempt on *capacity* (or find the queue non-empty
659        // ahead of us).
660        let mut ticket: Option<Arc<Notify>> = None;
661
662        loop {
663            {
664                let mut state = self.state.lock().await;
665
666                // A caller may attempt admission only if nobody is ahead of it
667                // in the FIFO: it holds the head ticket, or it holds no ticket
668                // and the queue is empty (no barging past waiters — F4).
669                let may_attempt = match &ticket {
670                    Some(t) => state.queue.front().is_some_and(|h| Arc::ptr_eq(h, t)),
671                    None => state.queue.is_empty(),
672                };
673
674                if may_attempt {
675                    match try_admit_locked(
676                        &mut state,
677                        &subject_key,
678                        group_key,
679                        demand,
680                        &self.config,
681                        self.max_sandboxes,
682                    ) {
683                        Ok(()) => {
684                            if ticket.is_some() {
685                                state.queue.pop_front();
686                            }
687                            wake_head(&state);
688                            return Ok(ReservationRecord {
689                                subject: subject_key,
690                                group,
691                                demand,
692                            });
693                        }
694                        Err(AdmitReject::Infeasible(reason)) => {
695                            if ticket.is_some() {
696                                state.queue.pop_front();
697                                wake_head(&state);
698                            }
699                            return Err(WorkerError::AdmissionInfeasible { reason });
700                        }
701                        Err(AdmitReject::Quota(reason)) => {
702                            if ticket.is_some() {
703                                state.queue.pop_front();
704                                wake_head(&state);
705                            }
706                            return Err(WorkerError::AdmissionDenied { reason });
707                        }
708                        Err(AdmitReject::Capacity(reason)) => {
709                            if ticket.is_none() {
710                                if state.queue.len() >= self.config.queue_capacity {
711                                    return Err(WorkerError::QueueFull {
712                                        waiting: state.queue.len(),
713                                        bound: self.config.queue_capacity,
714                                    });
715                                }
716                                tracing::debug!(subject = %subject_key, reason = %reason, "acquire: queueing for capacity (FIFO)");
717                                let t = Arc::new(Notify::new());
718                                state.queue.push_back(t.clone());
719                                ticket = Some(t);
720                            }
721                            // else: we are the head and capacity still isn't
722                            // free — stay queued and wait again.
723                        }
724                    }
725                } else if ticket.is_none() {
726                    // Someone is ahead of us: join the back of the queue
727                    // (bounded) and wait our turn rather than barging. Demand
728                    // infeasible against configured totals never queues, even
729                    // behind existing waiters.
730                    if let Err(reason) = check_feasible(demand, &self.config, self.max_sandboxes) {
731                        return Err(WorkerError::AdmissionInfeasible { reason });
732                    }
733                    if state.queue.len() >= self.config.queue_capacity {
734                        return Err(WorkerError::QueueFull {
735                            waiting: state.queue.len(),
736                            bound: self.config.queue_capacity,
737                        });
738                    }
739                    let t = Arc::new(Notify::new());
740                    state.queue.push_back(t.clone());
741                    ticket = Some(t);
742                }
743            }
744
745            // Wait to be handed capacity (or to reach the head), bounded by the
746            // deadline. Every path that falls through to here has set `ticket`;
747            // the `None` arm is unreachable, so re-loop rather than panic.
748            let t = match &ticket {
749                Some(t) => t.clone(),
750                None => continue,
751            };
752            let remaining = deadline.saturating_duration_since(Instant::now());
753            let timed_out =
754                remaining.is_zero() || tokio::time::timeout(remaining, t.notified()).await.is_err();
755            if timed_out {
756                let mut state = self.state.lock().await;
757                let was_head = state.queue.front().is_some_and(|h| Arc::ptr_eq(h, &t));
758                state.queue.retain(|x| !Arc::ptr_eq(x, &t));
759                // If the head gave up, hand its turn to the next waiter.
760                if was_head {
761                    wake_head(&state);
762                }
763                return Err(WorkerError::QueueTimeout {
764                    timeout_secs: self.config.queue_timeout_secs,
765                });
766            }
767            // Woken: loop back and retry admission under the lock.
768        }
769    }
770
771    /// Give back a committed reservation (a sandbox using it was released or
772    /// destroyed) and hand the freed capacity to the FIFO queue head (F4 — no
773    /// thundering-herd re-race).
774    pub async fn release(&self, record: &ReservationRecord) {
775        let mut state = self.state.lock().await;
776        release_locked(&mut state, record);
777        wake_head(&state);
778    }
779
780    /// Give back a reservation that was committed but never resulted in a
781    /// usable sandbox (e.g. the backend failed to create/reconfigure one
782    /// after admission succeeded). Identical to [`Self::release`] — kept as
783    /// a distinctly-named entry point for readability at call sites.
784    pub async fn rollback(&self, record: &ReservationRecord) {
785        self.release(record).await;
786    }
787}
788
789#[cfg(test)]
790#[allow(clippy::unwrap_used, clippy::expect_used)]
791mod tests {
792    use super::*;
793
794    fn demand(cpu_millis: u64, memory_bytes: u64, gpu: usize) -> Demand {
795        Demand {
796            cpu_millis,
797            memory_bytes,
798            gpu,
799        }
800    }
801
802    #[tokio::test]
803    async fn anonymous_subject_rejected_fail_closed() {
804        let tracker = AdmissionTracker::new(AdmissionConfig::default(), 10);
805        let err = tracker
806            .reserve(&Subject::anonymous(), None, demand(0, 0, 0))
807            .await
808            .unwrap_err();
809        assert!(
810            matches!(err, WorkerError::Unauthorized { .. }),
811            "got: {err:?}"
812        );
813    }
814
815    #[tokio::test]
816    async fn zero_demand_never_trips_resource_capacity() {
817        // No cpu/mem/gpu totals configured (the default) → a zero-demand
818        // request (the CRI "unspecified" convention) always fits, matching
819        // pre-#525 behavior exactly.
820        let tracker = AdmissionTracker::new(AdmissionConfig::default(), 10);
821        let r = tracker
822            .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
823            .await;
824        assert!(r.is_ok());
825    }
826
827    /// Membership validator with alice + bob both consented members of
828    /// team-a — models the authoritative bidirectional-consent membership
829    /// fact so the group tests exercise quota, not the fail-closed default.
830    fn team_a_membership() -> Arc<dyn GroupSelectorValidator> {
831        Arc::new(
832            StaticGroupMembership::new()
833                .with_member("team-a", "alice")
834                .with_member("team-a", "bob"),
835        )
836    }
837
838    #[tokio::test]
839    async fn per_subject_quota_rejects_not_queues() {
840        let cfg = AdmissionConfig {
841            max_per_subject: Some(1),
842            ..Default::default()
843        };
844        let tracker = AdmissionTracker::new(cfg, 10);
845        let alice = Subject::new("alice");
846        let r1 = tracker.reserve(&alice, None, demand(0, 0, 0)).await;
847        assert!(r1.is_ok());
848        let r2 = tracker.reserve(&alice, None, demand(0, 0, 0)).await;
849        assert!(
850            matches!(r2, Err(WorkerError::AdmissionDenied { .. })),
851            "got: {r2:?}"
852        );
853    }
854
855    #[tokio::test]
856    async fn per_group_quota_rejects_across_subjects() {
857        let cfg = AdmissionConfig {
858            max_per_group: Some(1),
859            ..Default::default()
860        };
861        let tracker = AdmissionTracker::with_group_validator(cfg, 10, team_a_membership());
862        let r1 = tracker
863            .reserve(&Subject::new("alice"), Some("team-a"), demand(0, 0, 0))
864            .await;
865        assert!(r1.is_ok());
866        let r2 = tracker
867            .reserve(&Subject::new("bob"), Some("team-a"), demand(0, 0, 0))
868            .await;
869        assert!(
870            matches!(r2, Err(WorkerError::AdmissionDenied { .. })),
871            "got: {r2:?}"
872        );
873    }
874
875    // ── B′: group is derived from the verified Subject, never trusted from
876    //    the annotation (F6) ────────────────────────────────────────────────
877
878    #[tokio::test]
879    async fn group_selector_from_non_member_is_rejected() {
880        // Only alice is a member of team-a.
881        let validator = Arc::new(StaticGroupMembership::new().with_member("team-a", "alice"));
882        let tracker =
883            AdmissionTracker::with_group_validator(AdmissionConfig::default(), 10, validator);
884        // A member's selector validates and partitions.
885        let r_alice = tracker
886            .reserve(&Subject::new("alice"), Some("team-a"), demand(0, 0, 0))
887            .await;
888        assert!(
889            r_alice.is_ok(),
890            "member's selector must validate: {r_alice:?}"
891        );
892        // A non-member asserting team-a is rejected — no cross-tenant quota
893        // consumption.
894        let r_bob = tracker
895            .reserve(&Subject::new("bob"), Some("team-a"), demand(0, 0, 0))
896            .await;
897        assert!(
898            matches!(r_bob, Err(WorkerError::AdmissionDenied { .. })),
899            "non-member selector must be rejected: {r_bob:?}"
900        );
901    }
902
903    #[tokio::test]
904    async fn deny_unknown_group_validator_rejects_any_selector_but_allows_none() {
905        // Default tracker uses the fail-closed DenyUnknownGroupValidator.
906        let tracker = AdmissionTracker::new(AdmissionConfig::default(), 10);
907        // No selector → no group partition, admitted.
908        let r_none = tracker
909            .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
910            .await;
911        assert!(
912            r_none.is_ok(),
913            "absent selector must not partition: {r_none:?}"
914        );
915        // Any selector is rejected fail-closed (no membership source wired).
916        let r_sel = tracker
917            .reserve(&Subject::new("alice"), Some("team-a"), demand(0, 0, 0))
918            .await;
919        assert!(
920            matches!(r_sel, Err(WorkerError::AdmissionDenied { .. })),
921            "fail-closed: unverifiable selector must be rejected: {r_sel:?}"
922        );
923    }
924
925    #[tokio::test]
926    async fn gpu_capacity_enforced_only_when_requested() {
927        let cfg = AdmissionConfig {
928            gpu_total: 1,
929            ..Default::default()
930        };
931        let tracker = AdmissionTracker::new(cfg, 10);
932        // No GPU asked for → unaffected even though gpu_total is small.
933        let r = tracker
934            .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
935            .await;
936        assert!(r.is_ok());
937        // One GPU asked for and available → admitted.
938        let r1 = tracker
939            .reserve(&Subject::new("bob"), None, demand(0, 0, 1))
940            .await;
941        assert!(r1.is_ok());
942        // Second GPU request with none free → queues then times out (queue
943        // bound left at default; timeout shortened for the test).
944    }
945
946    #[tokio::test]
947    async fn gpu_over_capacity_queues_then_times_out() {
948        let cfg = AdmissionConfig {
949            gpu_total: 1,
950            queue_timeout_secs: 0,
951            ..Default::default()
952        };
953        let tracker = AdmissionTracker::new(cfg, 10);
954        let r1 = tracker
955            .reserve(&Subject::new("alice"), None, demand(0, 0, 1))
956            .await;
957        assert!(r1.is_ok());
958        let r2 = tracker
959            .reserve(&Subject::new("bob"), None, demand(0, 0, 1))
960            .await;
961        assert!(
962            matches!(r2, Err(WorkerError::QueueTimeout { .. })),
963            "got: {r2:?}"
964        );
965    }
966
967    // ── F1: demand exceeding configured TOTALS rejects immediately
968    //    (infeasible), never burns the queue timeout ─────────────────────────
969
970    #[tokio::test]
971    async fn gpu_demand_exceeding_total_rejects_immediately_infeasible() {
972        // gpu_total defaults to 0. A GPU request can never be satisfied, so it
973        // must reject *immediately* (Infeasible), not queue for 30s.
974        let cfg = AdmissionConfig {
975            // Long timeout so a wrong "queue" answer would be obvious/slow.
976            queue_timeout_secs: 30,
977            ..Default::default()
978        };
979        let tracker = AdmissionTracker::new(cfg, 10);
980        let start = Instant::now();
981        let r = tracker
982            .reserve(&Subject::new("alice"), None, demand(0, 0, 1))
983            .await;
984        assert!(
985            matches!(r, Err(WorkerError::AdmissionInfeasible { .. })),
986            "got: {r:?}"
987        );
988        assert!(
989            start.elapsed() < Duration::from_secs(1),
990            "infeasible demand must reject immediately, not wait out the timeout"
991        );
992    }
993
994    #[tokio::test]
995    async fn memory_demand_exceeding_total_is_infeasible() {
996        let cfg = AdmissionConfig {
997            memory_bytes_total: Some(256),
998            queue_timeout_secs: 30,
999            ..Default::default()
1000        };
1001        let tracker = AdmissionTracker::new(cfg, 10);
1002        let start = Instant::now();
1003        let r = tracker
1004            .reserve(&Subject::new("alice"), None, demand(0, 512, 0))
1005            .await;
1006        assert!(
1007            matches!(r, Err(WorkerError::AdmissionInfeasible { .. })),
1008            "got: {r:?}"
1009        );
1010        assert!(start.elapsed() < Duration::from_secs(1));
1011    }
1012
1013    #[tokio::test]
1014    async fn cpu_demand_exceeding_total_is_infeasible() {
1015        let cfg = AdmissionConfig {
1016            cpu_millis_total: Some(1000),
1017            queue_timeout_secs: 30,
1018            ..Default::default()
1019        };
1020        let tracker = AdmissionTracker::new(cfg, 10);
1021        let r = tracker
1022            .reserve(&Subject::new("alice"), None, demand(2000, 0, 0))
1023            .await;
1024        assert!(
1025            matches!(r, Err(WorkerError::AdmissionInfeasible { .. })),
1026            "got: {r:?}"
1027        );
1028    }
1029
1030    #[tokio::test]
1031    async fn demand_equal_to_total_is_feasible_not_infeasible() {
1032        // Boundary: demand == total fits (busy, not infeasible). First takes
1033        // the GPU; a second equal request queues on *capacity* (would time
1034        // out), proving it wasn't classified infeasible.
1035        let cfg = AdmissionConfig {
1036            gpu_total: 1,
1037            queue_timeout_secs: 0,
1038            ..Default::default()
1039        };
1040        let tracker = AdmissionTracker::new(cfg, 10);
1041        let r1 = tracker
1042            .reserve(&Subject::new("alice"), None, demand(0, 0, 1))
1043            .await;
1044        assert!(r1.is_ok(), "demand == total must fit: {r1:?}");
1045        let r2 = tracker
1046            .reserve(&Subject::new("bob"), None, demand(0, 0, 1))
1047            .await;
1048        assert!(
1049            matches!(r2, Err(WorkerError::QueueTimeout { .. })),
1050            "second equal request is busy (capacity), not infeasible: {r2:?}"
1051        );
1052    }
1053
1054    #[tokio::test]
1055    async fn infeasible_request_behind_waiter_rejects_without_taking_queue_slot() {
1056        let cfg = AdmissionConfig {
1057            gpu_total: 0,
1058            queue_capacity: 2,
1059            queue_timeout_secs: 5,
1060            ..Default::default()
1061        };
1062        let tracker = Arc::new(AdmissionTracker::new(cfg, 1));
1063        let held = tracker
1064            .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
1065            .await
1066            .expect("first request should occupy the only sandbox slot");
1067
1068        let t_bob = tracker.clone();
1069        let bob = tokio::spawn(async move {
1070            t_bob
1071                .reserve(&Subject::new("bob"), None, demand(0, 0, 0))
1072                .await
1073        });
1074        tokio::time::sleep(Duration::from_millis(50)).await;
1075
1076        let start = Instant::now();
1077        let carol = tracker
1078            .reserve(&Subject::new("carol"), None, demand(0, 0, 1))
1079            .await;
1080        assert!(
1081            matches!(carol, Err(WorkerError::AdmissionInfeasible { .. })),
1082            "infeasible demand behind a waiter must fail immediately: {carol:?}"
1083        );
1084        assert!(
1085            start.elapsed() < Duration::from_secs(1),
1086            "infeasible demand must not wait behind the existing queue"
1087        );
1088
1089        let t_dave = tracker.clone();
1090        let dave = tokio::spawn(async move {
1091            t_dave
1092                .reserve(&Subject::new("dave"), None, demand(0, 0, 0))
1093                .await
1094        });
1095        tokio::time::sleep(Duration::from_millis(50)).await;
1096        assert!(
1097            !dave.is_finished(),
1098            "feasible follow-up should fit in the queue slot not consumed by the infeasible request"
1099        );
1100
1101        tracker.release(&held).await;
1102        let bob_res = tokio::time::timeout(Duration::from_secs(2), bob)
1103            .await
1104            .expect("bob should be admitted")
1105            .expect("task ok")
1106            .expect("bob reservation ok");
1107        tracker.release(&bob_res).await;
1108        let dave_res = tokio::time::timeout(Duration::from_secs(2), dave)
1109            .await
1110            .expect("dave should be admitted")
1111            .expect("task ok");
1112        assert!(
1113            dave_res.is_ok(),
1114            "dave should have remained queued: {dave_res:?}"
1115        );
1116    }
1117
1118    // ── F4: FIFO — arrivals never barge past queued waiters ────────────────
1119
1120    #[tokio::test]
1121    async fn fifo_earlier_waiter_is_not_starved_by_later_arrivals() {
1122        // Capacity 1. alice holds it; bob queues first, carol queues after.
1123        // Freeing one slot must admit bob (the earlier waiter), never carol.
1124        let cfg = AdmissionConfig {
1125            queue_timeout_secs: 5,
1126            ..Default::default()
1127        };
1128        let tracker = Arc::new(AdmissionTracker::new(cfg, 1));
1129        let held = tracker
1130            .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
1131            .await
1132            .unwrap();
1133
1134        let t_bob = tracker.clone();
1135        let bob = tokio::spawn(async move {
1136            t_bob
1137                .reserve(&Subject::new("bob"), None, demand(0, 0, 0))
1138                .await
1139        });
1140        // Ensure bob is queued at the head before carol arrives.
1141        tokio::time::sleep(Duration::from_millis(50)).await;
1142
1143        let t_carol = tracker.clone();
1144        let carol = tokio::spawn(async move {
1145            t_carol
1146                .reserve(&Subject::new("carol"), None, demand(0, 0, 0))
1147                .await
1148        });
1149        tokio::time::sleep(Duration::from_millis(50)).await;
1150
1151        // Free exactly one slot.
1152        tracker.release(&held).await;
1153
1154        // bob (earlier) must be the one admitted.
1155        let bob_res = tokio::time::timeout(Duration::from_secs(2), bob)
1156            .await
1157            .expect("bob should be woken")
1158            .expect("task ok");
1159        assert!(
1160            bob_res.is_ok(),
1161            "earlier waiter bob must be admitted: {bob_res:?}"
1162        );
1163
1164        // carol must still be waiting (only one slot freed, bob took it — no
1165        // barging).
1166        tokio::time::sleep(Duration::from_millis(50)).await;
1167        assert!(
1168            !carol.is_finished(),
1169            "later arrival carol must not barge ahead of / with bob"
1170        );
1171
1172        // Clean up: release bob's slot so carol can finish.
1173        tracker.release(&bob_res.unwrap()).await;
1174        let carol_res = tokio::time::timeout(Duration::from_secs(2), carol)
1175            .await
1176            .expect("carol should now be woken")
1177            .expect("task ok");
1178        assert!(
1179            carol_res.is_ok(),
1180            "carol admitted once capacity frees: {carol_res:?}"
1181        );
1182    }
1183
1184    #[tokio::test]
1185    async fn queue_full_rejects_immediately() {
1186        let cfg = AdmissionConfig {
1187            queue_capacity: 0,
1188            queue_timeout_secs: 5,
1189            ..Default::default()
1190        };
1191        // Capacity 1, already taken; queue_capacity 0 means the very next
1192        // over-capacity request must reject immediately, not wait 5s.
1193        let tracker = AdmissionTracker::new(cfg, 1);
1194        let r1 = tracker
1195            .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
1196            .await;
1197        assert!(r1.is_ok());
1198        let start = Instant::now();
1199        let r2 = tracker
1200            .reserve(&Subject::new("bob"), None, demand(0, 0, 0))
1201            .await;
1202        assert!(
1203            matches!(r2, Err(WorkerError::QueueFull { .. })),
1204            "got: {r2:?}"
1205        );
1206        assert!(
1207            start.elapsed() < Duration::from_secs(1),
1208            "must reject immediately, not wait out the timeout"
1209        );
1210    }
1211
1212    #[tokio::test]
1213    async fn release_wakes_a_queued_waiter() {
1214        let cfg = AdmissionConfig {
1215            queue_timeout_secs: 5,
1216            ..Default::default()
1217        };
1218        let tracker = Arc::new(AdmissionTracker::new(cfg, 1));
1219        let r1 = tracker
1220            .reserve(&Subject::new("alice"), None, demand(0, 0, 0))
1221            .await
1222            .unwrap();
1223
1224        let t2 = tracker.clone();
1225        let waiter = tokio::spawn(async move {
1226            t2.reserve(&Subject::new("bob"), None, demand(0, 0, 0))
1227                .await
1228        });
1229
1230        // Give the waiter task a moment to actually enter the queue.
1231        tokio::time::sleep(Duration::from_millis(50)).await;
1232        tracker.release(&r1).await;
1233
1234        let r2 = tokio::time::timeout(Duration::from_secs(2), waiter)
1235            .await
1236            .expect("waiter should be woken well within 2s")
1237            .expect("task should not panic");
1238        assert!(
1239            r2.is_ok(),
1240            "queued waiter should be admitted once capacity frees: {r2:?}"
1241        );
1242    }
1243
1244    #[tokio::test]
1245    async fn derive_demand_is_zero_for_default_config() {
1246        let config = PodSandboxConfig::default();
1247        assert_eq!(derive_demand(&config), Demand::default());
1248        assert_eq!(derive_group_selector(&config), None);
1249    }
1250
1251    #[tokio::test]
1252    async fn derive_demand_reads_cpu_memory_and_gpu_annotation() {
1253        let mut config = PodSandboxConfig::default();
1254        config.linux.resources.cpu_period = 100_000;
1255        config.linux.resources.cpu_quota = 50_000;
1256        config.linux.resources.memory_limit_in_bytes = 1024;
1257        config.annotations.push(KeyValue {
1258            key: ANN_GPU_REQUEST.to_owned(),
1259            value: "2".to_owned(),
1260        });
1261        config.annotations.push(KeyValue {
1262            key: ANN_GROUP.to_owned(),
1263            value: "team-a".to_owned(),
1264        });
1265
1266        let d = derive_demand(&config);
1267        assert_eq!(d.cpu_millis, 500);
1268        assert_eq!(d.memory_bytes, 1024);
1269        assert_eq!(d.gpu, 2);
1270        assert_eq!(derive_group_selector(&config).as_deref(), Some("team-a"));
1271    }
1272}