Skip to main content

hyprstream_workers/workflow/
scheduler.rs

1//! Job → scheduled workload placement (#527).
2//!
3//! Maps a job's `runs_on:` label + `resources:` hints onto a placement
4//! decision and, for jobs that request isolation, acquires/releases a real
5//! sandbox reservation through the #525 P2 admission engine
6//! (`SandboxPool::acquire`) instead of the pre-#527 fire-and-forget
7//! `tokio::spawn` that ignored both.
8//!
9//! ## 🔒 Mapping convention (ratified: B′ group semantics, #921 decision 5)
10//!
11//! `runs_on: <label>` is matched, case-insensitively, against a small
12//! reserved alias set ([`IN_PROC_LABELS`]) meaning "no isolation — run
13//! in-process" (today's VFS/Tcl execution path, unchanged). Any other label
14//! routes the job onto the [`JobScheduler`]'s configured [`SandboxPool`] and
15//! is recorded as the `hyprstream_workers::runtime::ANN_GROUP` annotation.
16//!
17//! Under the ratified B′ semantics that annotation is a **capacity-partition
18//! selector, never an authoritative quota/trust/billing key**: admission
19//! (`runtime::admission`) resolves it against the verified Subject's
20//! bidirectional-consent placement-group membership via the pool's
21//! `GroupSelectorValidator` — a workflow naming a group its Subject is not a
22//! member of is **rejected at admission**, not silently accepted. The
23//! production default validator (`DenyUnknownGroupValidator`) fail-closes on
24//! every non-empty selector until a membership source
25//! (`PlacementIndex::is_member`) is wired, so an isolated `runs_on:` label is
26//! deny-by-default today. Validated labels partition node-local capacity via
27//! `AdmissionConfig::max_per_group` (fairness/organization only —
28//! authoritative quota arrives via the ledger path, #922/#925). This module
29//! adds **no group accounting of its own**: the label's only consumer is the
30//! `ANN_GROUP` annotation read back by `admission::derive_group_selector`
31//! inside `SandboxPool::acquire`.
32//!
33//! This is a **label → group** mapping, not a **label → backend** mapping:
34//! one `JobScheduler` wraps exactly one pre-resolved [`SandboxPool`] (bound
35//! to one [`crate::runtime::SandboxBackend`] at construction — see
36//! `pool.rs`'s own documented invariant that a pool never re-selects its
37//! backend per-request). `runs_on: kata` therefore does NOT pick a different
38//! backend than `runs_on: nspawn` within a single scheduler; both go to
39//! *this scheduler's* one backend, merely tagged with a different admission
40//! group. Routing different labels to genuinely different backends would
41//! mean a router over multiple backend-specific pools — the same
42//! architectural change `pool.rs` already flags as a follow-up, not
43//! something this ticket re-derives. Flagged here explicitly for reviewer
44//! sign-off, per the issue's disclosure ask.
45//!
46//! ## What "runs in a sandbox" means today (also flagged)
47//!
48//! Placement acquires a *real* sandbox reservation — admission is checked,
49//! the backend's `start()` actually runs (a real Kata VM boots when the pool
50//! is Kata-backed) — closing the #520 "no isolation" gap structurally: a
51//! job that requests isolation now really holds a sandbox for its duration.
52//! Step *content* execution (the `uses:`/`run:` bodies) still runs through
53//! the existing VFS `/bin/` ctl + TclShell bridge on the host, not inside the
54//! guest's own namespace — routing individual step exec into the guest (via
55//! `kata_agent::ExecProcess`, `nspawn`, etc.) needs the streaming/fd plane
56//! `exec_mount.rs` itself documents as not yet built. That is a materially
57//! larger follow-up, not attempted here.
58
59use std::sync::Arc;
60
61use hyprstream_vfs::Subject;
62
63use crate::error::Result;
64use crate::runtime::{SandboxPool, ANN_GPU_REQUEST, ANN_GROUP};
65use hyprstream_rpc_std::worker_client::{KeyValue, PodSandboxConfig, PodSandboxMetadata};
66
67use super::parser::{JobResources, RunsOn};
68
69/// `runs_on:` labels (case-insensitive) meaning "no isolation — run
70/// in-process" via the existing VFS/Tcl execution path. Anything else is
71/// treated as a request for this scheduler's sandbox (see module docs).
72pub const IN_PROC_LABELS: &[&str] = &["local", "self-hosted", "in-process", "tcl"];
73
74/// The label that drives a job's placement decision.
75///
76/// GHA's `runs-on: [a, b]` means "match ALL labels" against a runner's
77/// capability set. Hyprstream has no such per-runner capability set to
78/// intersect against (yet) — `runs_on` here selects an admission *group*
79/// (see module docs), a mapping for which multi-label AND-composition has no
80/// defined meaning. Only the first label is consulted; additional labels in
81/// `RunsOn::Labels` are accepted (not a parse error) but otherwise ignored.
82/// This is a documented scope-limit, not a silent drop.
83pub fn primary_label(runs_on: &RunsOn) -> &str {
84    match runs_on {
85        RunsOn::Label(l) => l.as_str(),
86        RunsOn::Labels(v) => v.first().map(String::as_str).unwrap_or(""),
87    }
88}
89
90/// True when `runs_on`'s primary label requests in-process (no sandbox)
91/// execution — either explicitly (an [`IN_PROC_LABELS`] alias) or because no
92/// label was usefully provided (empty string).
93pub fn wants_in_proc(runs_on: &RunsOn) -> bool {
94    let label = primary_label(runs_on);
95    label.is_empty() || IN_PROC_LABELS.iter().any(|l| l.eq_ignore_ascii_case(label))
96}
97
98/// Build the `PodSandboxConfig` a job's `resources:` hints + `runs_on` label
99/// translate to for admission (#525 P2): cpu/memory become
100/// `linux.resources`, `gpu` becomes the `ANN_GPU_REQUEST` annotation (the
101/// same vocabulary `runtime::admission::derive_demand` already reads), and
102/// the `runs_on` label becomes the `ANN_GROUP` annotation — a B′ group
103/// *selector*, validated against the verified Subject's membership by the
104/// pool's `GroupSelectorValidator` at admission (see module docs).
105pub fn job_pod_sandbox_config(
106    job_name: &str,
107    runs_on_label: &str,
108    resources: &JobResources,
109) -> PodSandboxConfig {
110    let mut config = PodSandboxConfig {
111        metadata: PodSandboxMetadata {
112            name: format!("workflow-job-{job_name}"),
113            ..Default::default()
114        },
115        ..Default::default()
116    };
117
118    if resources.cpu_millis > 0 {
119        // CRI convention: a 100ms period is the common default; quota scales
120        // with it so `(quota / period) * 1000` round-trips through
121        // `derive_demand` back to `cpu_millis`.
122        config.linux.resources.cpu_period = 100_000;
123        config.linux.resources.cpu_quota = (resources.cpu_millis as i64) * 100;
124    }
125    if resources.memory_bytes > 0 {
126        config.linux.resources.memory_limit_in_bytes = resources.memory_bytes as i64;
127    }
128    if resources.gpu > 0 {
129        config.annotations.push(KeyValue {
130            key: ANN_GPU_REQUEST.to_owned(),
131            value: resources.gpu.to_string(),
132        });
133    }
134    config.annotations.push(KeyValue {
135        key: ANN_GROUP.to_owned(),
136        value: runs_on_label.to_owned(),
137    });
138
139    config
140}
141
142/// A job's resolved placement. `Sandbox` must be released via
143/// [`Placement::release`] once the job finishes (success, failure, timeout,
144/// or cancellation) so the admission reservation and warm-pool slot are
145/// given back — see `SandboxPool::release`'s own reservation-rollback
146/// discipline.
147pub enum Placement {
148    /// No isolation — the job's steps run on the existing host VFS/Tcl path.
149    InProc,
150    /// A live sandbox reservation acquired from `pool`.
151    Sandbox {
152        pool: Arc<SandboxPool>,
153        sandbox_id: String,
154    },
155}
156
157impl Placement {
158    /// Release a `Sandbox` placement back to its pool. A no-op for `InProc`.
159    /// Errors are logged, not propagated: by the time a job finishes, its
160    /// result is already decided, and swallowing a release failure here
161    /// matches `SandboxPool::release`'s own callers elsewhere (best-effort
162    /// cleanup, never fails the job it releases after).
163    pub async fn release(self) {
164        if let Placement::Sandbox { pool, sandbox_id } = self {
165            if let Err(e) = pool.release(&sandbox_id).await {
166                tracing::warn!(
167                    sandbox_id = %sandbox_id,
168                    error = %e,
169                    "failed to release workflow job sandbox"
170                );
171            }
172        }
173    }
174}
175
176/// Routes workflow jobs through the #525 P2 scheduler.
177///
178/// See the module docs for the `runs_on` → group mapping convention and its
179/// documented scope limits.
180pub struct JobScheduler {
181    pool: Arc<SandboxPool>,
182}
183
184impl JobScheduler {
185    /// Wrap an already-constructed [`SandboxPool`] (bound to one backend,
186    /// e.g. via `runtime::resolve_backend` — the same #516 fail-closed
187    /// registry `worker.backend` uses) as the target for isolated jobs.
188    pub fn new(pool: Arc<SandboxPool>) -> Self {
189        Self { pool }
190    }
191
192    /// Decide placement for a job and, if it requests isolation, acquire a
193    /// sandbox reservation for it (admitted via `subject`'s identity/quota —
194    /// fail-closed if `subject` is anonymous, see `runtime::admission`).
195    ///
196    /// The `runs_on` label rides along as a B′ group selector; admission
197    /// rejects it unless the pool's `GroupSelectorValidator` proves `subject`
198    /// a consented member of that group (deny-unknown by default).
199    pub async fn place(
200        &self,
201        job_name: &str,
202        runs_on: &RunsOn,
203        resources: &JobResources,
204        subject: &Subject,
205    ) -> Result<Placement> {
206        if wants_in_proc(runs_on) {
207            return Ok(Placement::InProc);
208        }
209        let label = primary_label(runs_on);
210        let config = job_pod_sandbox_config(job_name, label, resources);
211        let sandbox_id = self.pool.acquire(subject, &config).await?;
212        Ok(Placement::Sandbox {
213            pool: Arc::clone(&self.pool),
214            sandbox_id,
215        })
216    }
217}
218
219#[cfg(test)]
220#[allow(clippy::unwrap_used, clippy::expect_used)]
221mod tests {
222    use super::*;
223    use crate::config::PoolConfig;
224    use crate::runtime::{
225        AdmissionConfig, PodSandbox, SandboxBackend, SandboxHandle, StaticGroupMembership,
226    };
227    use async_trait::async_trait;
228    use hyprstream_rpc_std::worker_client::LinuxContainerResources;
229    use std::any::Any;
230    use std::collections::HashMap;
231    use std::sync::atomic::{AtomicUsize, Ordering};
232
233    #[test]
234    fn primary_label_picks_first_of_list() {
235        assert_eq!(primary_label(&RunsOn::Label("kata".into())), "kata");
236        assert_eq!(
237            primary_label(&RunsOn::Labels(vec!["gpu".into(), "linux".into()])),
238            "gpu"
239        );
240        assert_eq!(primary_label(&RunsOn::Labels(vec![])), "");
241    }
242
243    #[test]
244    fn in_proc_labels_are_case_insensitive() {
245        assert!(wants_in_proc(&RunsOn::Label("LOCAL".into())));
246        assert!(wants_in_proc(&RunsOn::Label("Self-Hosted".into())));
247        assert!(!wants_in_proc(&RunsOn::Label("kata".into())));
248        assert!(!wants_in_proc(&RunsOn::Label("ubuntu-latest".into())));
249    }
250
251    /// Read back an annotation by key (mirrors
252    /// `runtime::admission::annotation`, which is private to that module).
253    fn find_annotation<'a>(config: &'a PodSandboxConfig, key: &str) -> Option<&'a str> {
254        config
255            .annotations
256            .iter()
257            .find(|kv| kv.key == key)
258            .map(|kv| kv.value.as_str())
259    }
260
261    #[test]
262    fn job_pod_sandbox_config_maps_resources_and_group() {
263        let resources = JobResources {
264            cpu_millis: 500,
265            memory_bytes: 1024,
266            gpu: 2,
267        };
268        let config = job_pod_sandbox_config("build", "gpu-workers", &resources);
269
270        // CRI convention: (quota / period) * 1000 == requested cpu_millis
271        // (see `runtime::admission::derive_demand`, which reads it back the
272        // same way from a real acquire() call).
273        assert_eq!(config.linux.resources.cpu_period, 100_000);
274        assert_eq!(config.linux.resources.cpu_quota, 50_000);
275        assert_eq!(config.linux.resources.memory_limit_in_bytes, 1024);
276        assert_eq!(find_annotation(&config, ANN_GPU_REQUEST), Some("2"));
277        assert_eq!(find_annotation(&config, ANN_GROUP), Some("gpu-workers"));
278    }
279
280    #[test]
281    fn job_pod_sandbox_config_no_resources_is_zero_demand() {
282        let config = job_pod_sandbox_config("noop", "kata", &JobResources::default());
283        assert_eq!(config.linux.resources.cpu_period, 0);
284        assert_eq!(config.linux.resources.cpu_quota, 0);
285        assert_eq!(config.linux.resources.memory_limit_in_bytes, 0);
286        assert_eq!(find_annotation(&config, ANN_GPU_REQUEST), None);
287        // Group is still set even with no resource demand — `runs_on`
288        // influences placement (quota) independent of resource requests.
289        assert_eq!(find_annotation(&config, ANN_GROUP), Some("kata"));
290    }
291
292    #[derive(Debug)]
293    struct FakeHandle;
294    impl SandboxHandle for FakeHandle {
295        fn as_any(&self) -> &dyn Any {
296            self
297        }
298    }
299
300    /// Minimal in-memory fake backend so scheduler tests don't need a real
301    /// Kata/nspawn toolchain. Tracks how many times `start()` was actually
302    /// invoked, so tests can prove an isolated job really acquires a sandbox
303    /// (#520) and an in-proc job never touches the backend at all.
304    #[derive(Default)]
305    struct FakeBackend {
306        starts: AtomicUsize,
307    }
308
309    #[async_trait]
310    impl SandboxBackend for FakeBackend {
311        fn backend_type(&self) -> &'static str {
312            "fake"
313        }
314        fn is_available(&self) -> bool {
315            true
316        }
317        async fn initialize(&self, _config: &PoolConfig) -> Result<()> {
318            Ok(())
319        }
320        async fn start(
321            &self,
322            _sandbox: &mut PodSandbox,
323            _config: &PodSandboxConfig,
324            _pool_config: &PoolConfig,
325            _annotations: &HashMap<String, String>,
326        ) -> Result<Arc<dyn SandboxHandle>> {
327            self.starts.fetch_add(1, Ordering::SeqCst);
328            Ok(Arc::new(FakeHandle))
329        }
330        async fn stop(&self, _sandbox: &PodSandbox) -> Result<()> {
331            Ok(())
332        }
333        async fn destroy(&self, _sandbox: &PodSandbox) -> Result<()> {
334            Ok(())
335        }
336        async fn reset(&self, _sandbox: &mut PodSandbox) -> Result<bool> {
337            Ok(true)
338        }
339        async fn get_pids(&self, _sandbox: &PodSandbox) -> Result<Vec<u32>> {
340            Ok(vec![])
341        }
342        fn supports_exec(&self) -> bool {
343            false
344        }
345        async fn exec_sync(
346            &self,
347            _sandbox: &PodSandbox,
348            _command: &[String],
349            _timeout_secs: u64,
350        ) -> Result<(i32, Vec<u8>, Vec<u8>)> {
351            Err(crate::error::WorkerError::ExecFailed(
352                "not supported".into(),
353            ))
354        }
355        async fn update_resources(
356            &self,
357            _sandbox: &PodSandbox,
358            _resources: &LinuxContainerResources,
359        ) -> Result<()> {
360            Ok(())
361        }
362    }
363
364    /// Scheduler over a pool with the production-default (fail-closed,
365    /// deny-unknown-group) validator — what `SandboxPool::new` gives you.
366    fn make_scheduler(admission: AdmissionConfig) -> (Arc<JobScheduler>, Arc<FakeBackend>) {
367        let backend = Arc::new(FakeBackend::default());
368        let pool_config = PoolConfig {
369            warm_pool_size: 0,
370            admission,
371            ..Default::default()
372        };
373        let pool = Arc::new(SandboxPool::new(
374            pool_config,
375            backend.clone() as Arc<dyn SandboxBackend>,
376        ));
377        (Arc::new(JobScheduler::new(pool)), backend)
378    }
379
380    /// Scheduler over a pool with a membership-backed validator (B′): the
381    /// runs_on selector is only honored for groups `subject` provably belongs
382    /// to.
383    fn make_scheduler_with_membership(
384        admission: AdmissionConfig,
385        membership: StaticGroupMembership,
386    ) -> (Arc<JobScheduler>, Arc<FakeBackend>) {
387        let backend = Arc::new(FakeBackend::default());
388        let pool_config = PoolConfig {
389            warm_pool_size: 0,
390            admission,
391            ..Default::default()
392        };
393        let pool = Arc::new(SandboxPool::with_group_validator(
394            pool_config,
395            backend.clone() as Arc<dyn SandboxBackend>,
396            Arc::new(membership),
397        ));
398        (Arc::new(JobScheduler::new(pool)), backend)
399    }
400
401    #[tokio::test]
402    async fn in_proc_runs_on_never_touches_the_backend() {
403        let (scheduler, backend) = make_scheduler(AdmissionConfig::default());
404        let subject = Subject::new("test-user");
405
406        let placement = scheduler
407            .place(
408                "job-a",
409                &RunsOn::Label("local".into()),
410                &JobResources::default(),
411                &subject,
412            )
413            .await
414            .expect("in-proc placement never fails");
415
416        assert!(matches!(placement, Placement::InProc));
417        assert_eq!(backend.starts.load(Ordering::SeqCst), 0);
418    }
419
420    #[tokio::test]
421    async fn isolated_runs_on_acquires_a_real_sandbox() {
422        // B′: the runs_on label is a group selector, so the Subject must be a
423        // consented member of that group for admission to honor it.
424        let (scheduler, backend) = make_scheduler_with_membership(
425            AdmissionConfig::default(),
426            StaticGroupMembership::new().with_member("kata", "test-user"),
427        );
428        let subject = Subject::new("test-user");
429
430        let placement = scheduler
431            .place(
432                "job-b",
433                &RunsOn::Label("kata".into()),
434                &JobResources::default(),
435                &subject,
436            )
437            .await
438            .expect("sandbox placement should succeed");
439
440        assert!(matches!(placement, Placement::Sandbox { .. }));
441        assert_eq!(backend.starts.load(Ordering::SeqCst), 1);
442
443        placement.release().await;
444    }
445
446    #[tokio::test]
447    async fn unverified_runs_on_group_is_rejected_at_admission() {
448        // B′ fail-closed default: with no membership source wired
449        // (`DenyUnknownGroupValidator`, what `SandboxPool::new` installs), a
450        // runs_on label — an unverifiable group selector — must be REJECTED
451        // at admission, never silently accepted as a fresh quota bucket. The
452        // backend is never touched.
453        let (scheduler, backend) = make_scheduler(AdmissionConfig::default());
454        let subject = Subject::new("test-user");
455
456        let placement = scheduler
457            .place(
458                "job-x",
459                &RunsOn::Label("kata".into()),
460                &JobResources::default(),
461                &subject,
462            )
463            .await;
464
465        assert!(
466            placement.is_err(),
467            "unverifiable group selector must be rejected fail-closed (B′)"
468        );
469        assert_eq!(backend.starts.load(Ordering::SeqCst), 0);
470    }
471
472    #[tokio::test]
473    async fn runs_on_label_influences_admission_via_group_quota() {
474        // max_per_group: 1 means a second job under the SAME runs_on label
475        // cannot be admitted concurrently, while a job under a DIFFERENT
476        // label is unaffected — proving the runs_on value actually reaches
477        // and influences the scheduler's placement decision, not just that
478        // *a* sandbox gets created. Both labels are consented memberships of
479        // the Subject (B′) — the quota partition only applies to validated
480        // selectors.
481        let (scheduler, _backend) = make_scheduler_with_membership(
482            AdmissionConfig {
483                max_per_group: Some(1),
484                ..AdmissionConfig::default()
485            },
486            StaticGroupMembership::new()
487                .with_member("gpu-only", "test-user")
488                .with_member("cpu-only", "test-user"),
489        );
490        let subject = Subject::new("test-user");
491
492        let first = scheduler
493            .place(
494                "job-1",
495                &RunsOn::Label("gpu-only".into()),
496                &JobResources::default(),
497                &subject,
498            )
499            .await
500            .expect("first gpu-only job admitted");
501
502        // Second job, SAME label, same subject: over the per-group quota.
503        let second = scheduler
504            .place(
505                "job-2",
506                &RunsOn::Label("gpu-only".into()),
507                &JobResources::default(),
508                &subject,
509            )
510            .await;
511        assert!(
512            second.is_err(),
513            "second job under the same maxed-out group should be rejected"
514        );
515
516        // Third job, DIFFERENT label: its own group quota, unaffected by gpu-only's.
517        let third = scheduler
518            .place(
519                "job-3",
520                &RunsOn::Label("cpu-only".into()),
521                &JobResources::default(),
522                &subject,
523            )
524            .await
525            .expect("different runs_on label has its own group quota");
526
527        first.release().await;
528        third.release().await;
529    }
530}