hyprstream_workers/workflow/scheduler.rs
1//! Job → scheduled workload placement (#527).
2//!
3//! Maps a job's `runs_on:` label + `resources:` hints onto a placement
4//! decision and, for jobs that request isolation, acquires/releases a real
5//! sandbox reservation through the #525 P2 admission engine
6//! (`SandboxPool::acquire`) instead of the pre-#527 fire-and-forget
7//! `tokio::spawn` that ignored both.
8//!
9//! ## 🔒 Mapping convention (ratified: B′ group semantics, #921 decision 5)
10//!
11//! `runs_on: <label>` is matched, case-insensitively, against a small
12//! reserved alias set ([`IN_PROC_LABELS`]) meaning "no isolation — run
13//! in-process" (today's VFS/Tcl execution path, unchanged). Any other label
14//! routes the job onto the [`JobScheduler`]'s configured [`SandboxPool`] and
15//! is recorded as the `hyprstream_workers::runtime::ANN_GROUP` annotation.
16//!
17//! Under the ratified B′ semantics that annotation is a **capacity-partition
18//! selector, never an authoritative quota/trust/billing key**: admission
19//! (`runtime::admission`) resolves it against the verified Subject's
20//! bidirectional-consent placement-group membership via the pool's
21//! `GroupSelectorValidator` — a workflow naming a group its Subject is not a
22//! member of is **rejected at admission**, not silently accepted. The
23//! production default validator (`DenyUnknownGroupValidator`) fail-closes on
24//! every non-empty selector until a membership source
25//! (`PlacementIndex::is_member`) is wired, so an isolated `runs_on:` label is
26//! deny-by-default today. Validated labels partition node-local capacity via
27//! `AdmissionConfig::max_per_group` (fairness/organization only —
28//! authoritative quota arrives via the ledger path, #922/#925). This module
29//! adds **no group accounting of its own**: the label's only consumer is the
30//! `ANN_GROUP` annotation read back by `admission::derive_group_selector`
31//! inside `SandboxPool::acquire`.
32//!
33//! This is a **label → group** mapping, not a **label → backend** mapping:
34//! one `JobScheduler` wraps exactly one pre-resolved [`SandboxPool`] (bound
35//! to one [`crate::runtime::SandboxBackend`] at construction — see
36//! `pool.rs`'s own documented invariant that a pool never re-selects its
37//! backend per-request). `runs_on: kata` therefore does NOT pick a different
38//! backend than `runs_on: nspawn` within a single scheduler; both go to
39//! *this scheduler's* one backend, merely tagged with a different admission
40//! group. Routing different labels to genuinely different backends would
41//! mean a router over multiple backend-specific pools — the same
42//! architectural change `pool.rs` already flags as a follow-up, not
43//! something this ticket re-derives. Flagged here explicitly for reviewer
44//! sign-off, per the issue's disclosure ask.
45//!
46//! ## What "runs in a sandbox" means today (also flagged)
47//!
48//! Placement acquires a *real* sandbox reservation — admission is checked,
49//! the backend's `start()` actually runs (a real Kata VM boots when the pool
50//! is Kata-backed) — closing the #520 "no isolation" gap structurally: a
51//! job that requests isolation now really holds a sandbox for its duration.
52//! Step *content* execution (the `uses:`/`run:` bodies) still runs through
53//! the existing VFS `/bin/` ctl + TclShell bridge on the host, not inside the
54//! guest's own namespace — routing individual step exec into the guest (via
55//! `kata_agent::ExecProcess`, `nspawn`, etc.) needs the streaming/fd plane
56//! `exec_mount.rs` itself documents as not yet built. That is a materially
57//! larger follow-up, not attempted here.
58
59use std::sync::Arc;
60
61use hyprstream_vfs::Subject;
62
63use crate::error::Result;
64use crate::runtime::{SandboxPool, ANN_GPU_REQUEST, ANN_GROUP};
65use hyprstream_rpc_std::worker_client::{KeyValue, PodSandboxConfig, PodSandboxMetadata};
66
67use super::parser::{JobResources, RunsOn};
68
69/// `runs_on:` labels (case-insensitive) meaning "no isolation — run
70/// in-process" via the existing VFS/Tcl execution path. Anything else is
71/// treated as a request for this scheduler's sandbox (see module docs).
72pub const IN_PROC_LABELS: &[&str] = &["local", "self-hosted", "in-process", "tcl"];
73
74/// The label that drives a job's placement decision.
75///
76/// GHA's `runs-on: [a, b]` means "match ALL labels" against a runner's
77/// capability set. Hyprstream has no such per-runner capability set to
78/// intersect against (yet) — `runs_on` here selects an admission *group*
79/// (see module docs), a mapping for which multi-label AND-composition has no
80/// defined meaning. Only the first label is consulted; additional labels in
81/// `RunsOn::Labels` are accepted (not a parse error) but otherwise ignored.
82/// This is a documented scope-limit, not a silent drop.
83pub fn primary_label(runs_on: &RunsOn) -> &str {
84 match runs_on {
85 RunsOn::Label(l) => l.as_str(),
86 RunsOn::Labels(v) => v.first().map(String::as_str).unwrap_or(""),
87 }
88}
89
90/// True when `runs_on`'s primary label requests in-process (no sandbox)
91/// execution — either explicitly (an [`IN_PROC_LABELS`] alias) or because no
92/// label was usefully provided (empty string).
93pub fn wants_in_proc(runs_on: &RunsOn) -> bool {
94 let label = primary_label(runs_on);
95 label.is_empty() || IN_PROC_LABELS.iter().any(|l| l.eq_ignore_ascii_case(label))
96}
97
98/// Build the `PodSandboxConfig` a job's `resources:` hints + `runs_on` label
99/// translate to for admission (#525 P2): cpu/memory become
100/// `linux.resources`, `gpu` becomes the `ANN_GPU_REQUEST` annotation (the
101/// same vocabulary `runtime::admission::derive_demand` already reads), and
102/// the `runs_on` label becomes the `ANN_GROUP` annotation — a B′ group
103/// *selector*, validated against the verified Subject's membership by the
104/// pool's `GroupSelectorValidator` at admission (see module docs).
105pub fn job_pod_sandbox_config(
106 job_name: &str,
107 runs_on_label: &str,
108 resources: &JobResources,
109) -> PodSandboxConfig {
110 let mut config = PodSandboxConfig {
111 metadata: PodSandboxMetadata {
112 name: format!("workflow-job-{job_name}"),
113 ..Default::default()
114 },
115 ..Default::default()
116 };
117
118 if resources.cpu_millis > 0 {
119 // CRI convention: a 100ms period is the common default; quota scales
120 // with it so `(quota / period) * 1000` round-trips through
121 // `derive_demand` back to `cpu_millis`.
122 config.linux.resources.cpu_period = 100_000;
123 config.linux.resources.cpu_quota = (resources.cpu_millis as i64) * 100;
124 }
125 if resources.memory_bytes > 0 {
126 config.linux.resources.memory_limit_in_bytes = resources.memory_bytes as i64;
127 }
128 if resources.gpu > 0 {
129 config.annotations.push(KeyValue {
130 key: ANN_GPU_REQUEST.to_owned(),
131 value: resources.gpu.to_string(),
132 });
133 }
134 config.annotations.push(KeyValue {
135 key: ANN_GROUP.to_owned(),
136 value: runs_on_label.to_owned(),
137 });
138
139 config
140}
141
142/// A job's resolved placement. `Sandbox` must be released via
143/// [`Placement::release`] once the job finishes (success, failure, timeout,
144/// or cancellation) so the admission reservation and warm-pool slot are
145/// given back — see `SandboxPool::release`'s own reservation-rollback
146/// discipline.
147pub enum Placement {
148 /// No isolation — the job's steps run on the existing host VFS/Tcl path.
149 InProc,
150 /// A live sandbox reservation acquired from `pool`.
151 Sandbox {
152 pool: Arc<SandboxPool>,
153 sandbox_id: String,
154 },
155}
156
157impl Placement {
158 /// Release a `Sandbox` placement back to its pool. A no-op for `InProc`.
159 /// Errors are logged, not propagated: by the time a job finishes, its
160 /// result is already decided, and swallowing a release failure here
161 /// matches `SandboxPool::release`'s own callers elsewhere (best-effort
162 /// cleanup, never fails the job it releases after).
163 pub async fn release(self) {
164 if let Placement::Sandbox { pool, sandbox_id } = self {
165 if let Err(e) = pool.release(&sandbox_id).await {
166 tracing::warn!(
167 sandbox_id = %sandbox_id,
168 error = %e,
169 "failed to release workflow job sandbox"
170 );
171 }
172 }
173 }
174}
175
176/// Routes workflow jobs through the #525 P2 scheduler.
177///
178/// See the module docs for the `runs_on` → group mapping convention and its
179/// documented scope limits.
180pub struct JobScheduler {
181 pool: Arc<SandboxPool>,
182}
183
184impl JobScheduler {
185 /// Wrap an already-constructed [`SandboxPool`] (bound to one backend,
186 /// e.g. via `runtime::resolve_backend` — the same #516 fail-closed
187 /// registry `worker.backend` uses) as the target for isolated jobs.
188 pub fn new(pool: Arc<SandboxPool>) -> Self {
189 Self { pool }
190 }
191
192 /// Decide placement for a job and, if it requests isolation, acquire a
193 /// sandbox reservation for it (admitted via `subject`'s identity/quota —
194 /// fail-closed if `subject` is anonymous, see `runtime::admission`).
195 ///
196 /// The `runs_on` label rides along as a B′ group selector; admission
197 /// rejects it unless the pool's `GroupSelectorValidator` proves `subject`
198 /// a consented member of that group (deny-unknown by default).
199 pub async fn place(
200 &self,
201 job_name: &str,
202 runs_on: &RunsOn,
203 resources: &JobResources,
204 subject: &Subject,
205 ) -> Result<Placement> {
206 if wants_in_proc(runs_on) {
207 return Ok(Placement::InProc);
208 }
209 let label = primary_label(runs_on);
210 let config = job_pod_sandbox_config(job_name, label, resources);
211 let sandbox_id = self.pool.acquire(subject, &config).await?;
212 Ok(Placement::Sandbox {
213 pool: Arc::clone(&self.pool),
214 sandbox_id,
215 })
216 }
217}
218
219#[cfg(test)]
220#[allow(clippy::unwrap_used, clippy::expect_used)]
221mod tests {
222 use super::*;
223 use crate::config::PoolConfig;
224 use crate::runtime::{
225 AdmissionConfig, PodSandbox, SandboxBackend, SandboxHandle, StaticGroupMembership,
226 };
227 use async_trait::async_trait;
228 use hyprstream_rpc_std::worker_client::LinuxContainerResources;
229 use std::any::Any;
230 use std::collections::HashMap;
231 use std::sync::atomic::{AtomicUsize, Ordering};
232
233 #[test]
234 fn primary_label_picks_first_of_list() {
235 assert_eq!(primary_label(&RunsOn::Label("kata".into())), "kata");
236 assert_eq!(
237 primary_label(&RunsOn::Labels(vec!["gpu".into(), "linux".into()])),
238 "gpu"
239 );
240 assert_eq!(primary_label(&RunsOn::Labels(vec![])), "");
241 }
242
243 #[test]
244 fn in_proc_labels_are_case_insensitive() {
245 assert!(wants_in_proc(&RunsOn::Label("LOCAL".into())));
246 assert!(wants_in_proc(&RunsOn::Label("Self-Hosted".into())));
247 assert!(!wants_in_proc(&RunsOn::Label("kata".into())));
248 assert!(!wants_in_proc(&RunsOn::Label("ubuntu-latest".into())));
249 }
250
251 /// Read back an annotation by key (mirrors
252 /// `runtime::admission::annotation`, which is private to that module).
253 fn find_annotation<'a>(config: &'a PodSandboxConfig, key: &str) -> Option<&'a str> {
254 config
255 .annotations
256 .iter()
257 .find(|kv| kv.key == key)
258 .map(|kv| kv.value.as_str())
259 }
260
261 #[test]
262 fn job_pod_sandbox_config_maps_resources_and_group() {
263 let resources = JobResources {
264 cpu_millis: 500,
265 memory_bytes: 1024,
266 gpu: 2,
267 };
268 let config = job_pod_sandbox_config("build", "gpu-workers", &resources);
269
270 // CRI convention: (quota / period) * 1000 == requested cpu_millis
271 // (see `runtime::admission::derive_demand`, which reads it back the
272 // same way from a real acquire() call).
273 assert_eq!(config.linux.resources.cpu_period, 100_000);
274 assert_eq!(config.linux.resources.cpu_quota, 50_000);
275 assert_eq!(config.linux.resources.memory_limit_in_bytes, 1024);
276 assert_eq!(find_annotation(&config, ANN_GPU_REQUEST), Some("2"));
277 assert_eq!(find_annotation(&config, ANN_GROUP), Some("gpu-workers"));
278 }
279
280 #[test]
281 fn job_pod_sandbox_config_no_resources_is_zero_demand() {
282 let config = job_pod_sandbox_config("noop", "kata", &JobResources::default());
283 assert_eq!(config.linux.resources.cpu_period, 0);
284 assert_eq!(config.linux.resources.cpu_quota, 0);
285 assert_eq!(config.linux.resources.memory_limit_in_bytes, 0);
286 assert_eq!(find_annotation(&config, ANN_GPU_REQUEST), None);
287 // Group is still set even with no resource demand — `runs_on`
288 // influences placement (quota) independent of resource requests.
289 assert_eq!(find_annotation(&config, ANN_GROUP), Some("kata"));
290 }
291
292 #[derive(Debug)]
293 struct FakeHandle;
294 impl SandboxHandle for FakeHandle {
295 fn as_any(&self) -> &dyn Any {
296 self
297 }
298 }
299
300 /// Minimal in-memory fake backend so scheduler tests don't need a real
301 /// Kata/nspawn toolchain. Tracks how many times `start()` was actually
302 /// invoked, so tests can prove an isolated job really acquires a sandbox
303 /// (#520) and an in-proc job never touches the backend at all.
304 #[derive(Default)]
305 struct FakeBackend {
306 starts: AtomicUsize,
307 }
308
309 #[async_trait]
310 impl SandboxBackend for FakeBackend {
311 fn backend_type(&self) -> &'static str {
312 "fake"
313 }
314 fn is_available(&self) -> bool {
315 true
316 }
317 async fn initialize(&self, _config: &PoolConfig) -> Result<()> {
318 Ok(())
319 }
320 async fn start(
321 &self,
322 _sandbox: &mut PodSandbox,
323 _config: &PodSandboxConfig,
324 _pool_config: &PoolConfig,
325 _annotations: &HashMap<String, String>,
326 ) -> Result<Arc<dyn SandboxHandle>> {
327 self.starts.fetch_add(1, Ordering::SeqCst);
328 Ok(Arc::new(FakeHandle))
329 }
330 async fn stop(&self, _sandbox: &PodSandbox) -> Result<()> {
331 Ok(())
332 }
333 async fn destroy(&self, _sandbox: &PodSandbox) -> Result<()> {
334 Ok(())
335 }
336 async fn reset(&self, _sandbox: &mut PodSandbox) -> Result<bool> {
337 Ok(true)
338 }
339 async fn get_pids(&self, _sandbox: &PodSandbox) -> Result<Vec<u32>> {
340 Ok(vec![])
341 }
342 fn supports_exec(&self) -> bool {
343 false
344 }
345 async fn exec_sync(
346 &self,
347 _sandbox: &PodSandbox,
348 _command: &[String],
349 _timeout_secs: u64,
350 ) -> Result<(i32, Vec<u8>, Vec<u8>)> {
351 Err(crate::error::WorkerError::ExecFailed(
352 "not supported".into(),
353 ))
354 }
355 async fn update_resources(
356 &self,
357 _sandbox: &PodSandbox,
358 _resources: &LinuxContainerResources,
359 ) -> Result<()> {
360 Ok(())
361 }
362 }
363
364 /// Scheduler over a pool with the production-default (fail-closed,
365 /// deny-unknown-group) validator — what `SandboxPool::new` gives you.
366 fn make_scheduler(admission: AdmissionConfig) -> (Arc<JobScheduler>, Arc<FakeBackend>) {
367 let backend = Arc::new(FakeBackend::default());
368 let pool_config = PoolConfig {
369 warm_pool_size: 0,
370 admission,
371 ..Default::default()
372 };
373 let pool = Arc::new(SandboxPool::new(
374 pool_config,
375 backend.clone() as Arc<dyn SandboxBackend>,
376 ));
377 (Arc::new(JobScheduler::new(pool)), backend)
378 }
379
380 /// Scheduler over a pool with a membership-backed validator (B′): the
381 /// runs_on selector is only honored for groups `subject` provably belongs
382 /// to.
383 fn make_scheduler_with_membership(
384 admission: AdmissionConfig,
385 membership: StaticGroupMembership,
386 ) -> (Arc<JobScheduler>, Arc<FakeBackend>) {
387 let backend = Arc::new(FakeBackend::default());
388 let pool_config = PoolConfig {
389 warm_pool_size: 0,
390 admission,
391 ..Default::default()
392 };
393 let pool = Arc::new(SandboxPool::with_group_validator(
394 pool_config,
395 backend.clone() as Arc<dyn SandboxBackend>,
396 Arc::new(membership),
397 ));
398 (Arc::new(JobScheduler::new(pool)), backend)
399 }
400
401 #[tokio::test]
402 async fn in_proc_runs_on_never_touches_the_backend() {
403 let (scheduler, backend) = make_scheduler(AdmissionConfig::default());
404 let subject = Subject::new("test-user");
405
406 let placement = scheduler
407 .place(
408 "job-a",
409 &RunsOn::Label("local".into()),
410 &JobResources::default(),
411 &subject,
412 )
413 .await
414 .expect("in-proc placement never fails");
415
416 assert!(matches!(placement, Placement::InProc));
417 assert_eq!(backend.starts.load(Ordering::SeqCst), 0);
418 }
419
420 #[tokio::test]
421 async fn isolated_runs_on_acquires_a_real_sandbox() {
422 // B′: the runs_on label is a group selector, so the Subject must be a
423 // consented member of that group for admission to honor it.
424 let (scheduler, backend) = make_scheduler_with_membership(
425 AdmissionConfig::default(),
426 StaticGroupMembership::new().with_member("kata", "test-user"),
427 );
428 let subject = Subject::new("test-user");
429
430 let placement = scheduler
431 .place(
432 "job-b",
433 &RunsOn::Label("kata".into()),
434 &JobResources::default(),
435 &subject,
436 )
437 .await
438 .expect("sandbox placement should succeed");
439
440 assert!(matches!(placement, Placement::Sandbox { .. }));
441 assert_eq!(backend.starts.load(Ordering::SeqCst), 1);
442
443 placement.release().await;
444 }
445
446 #[tokio::test]
447 async fn unverified_runs_on_group_is_rejected_at_admission() {
448 // B′ fail-closed default: with no membership source wired
449 // (`DenyUnknownGroupValidator`, what `SandboxPool::new` installs), a
450 // runs_on label — an unverifiable group selector — must be REJECTED
451 // at admission, never silently accepted as a fresh quota bucket. The
452 // backend is never touched.
453 let (scheduler, backend) = make_scheduler(AdmissionConfig::default());
454 let subject = Subject::new("test-user");
455
456 let placement = scheduler
457 .place(
458 "job-x",
459 &RunsOn::Label("kata".into()),
460 &JobResources::default(),
461 &subject,
462 )
463 .await;
464
465 assert!(
466 placement.is_err(),
467 "unverifiable group selector must be rejected fail-closed (B′)"
468 );
469 assert_eq!(backend.starts.load(Ordering::SeqCst), 0);
470 }
471
472 #[tokio::test]
473 async fn runs_on_label_influences_admission_via_group_quota() {
474 // max_per_group: 1 means a second job under the SAME runs_on label
475 // cannot be admitted concurrently, while a job under a DIFFERENT
476 // label is unaffected — proving the runs_on value actually reaches
477 // and influences the scheduler's placement decision, not just that
478 // *a* sandbox gets created. Both labels are consented memberships of
479 // the Subject (B′) — the quota partition only applies to validated
480 // selectors.
481 let (scheduler, _backend) = make_scheduler_with_membership(
482 AdmissionConfig {
483 max_per_group: Some(1),
484 ..AdmissionConfig::default()
485 },
486 StaticGroupMembership::new()
487 .with_member("gpu-only", "test-user")
488 .with_member("cpu-only", "test-user"),
489 );
490 let subject = Subject::new("test-user");
491
492 let first = scheduler
493 .place(
494 "job-1",
495 &RunsOn::Label("gpu-only".into()),
496 &JobResources::default(),
497 &subject,
498 )
499 .await
500 .expect("first gpu-only job admitted");
501
502 // Second job, SAME label, same subject: over the per-group quota.
503 let second = scheduler
504 .place(
505 "job-2",
506 &RunsOn::Label("gpu-only".into()),
507 &JobResources::default(),
508 &subject,
509 )
510 .await;
511 assert!(
512 second.is_err(),
513 "second job under the same maxed-out group should be rejected"
514 );
515
516 // Third job, DIFFERENT label: its own group quota, unaffected by gpu-only's.
517 let third = scheduler
518 .place(
519 "job-3",
520 &RunsOn::Label("cpu-only".into()),
521 &JobResources::default(),
522 &subject,
523 )
524 .await
525 .expect("different runs_on label has its own group quota");
526
527 first.release().await;
528 third.release().await;
529 }
530}