hyprstream_workers/runtime/selection.rs
1//! Inventory-based sandbox backend registry + fail-closed selection (#507).
2//!
3//! This is the *spine* of the unified sandbox taxonomy: one [`SandboxBackend`]
4//! seam, one selection model shared by every call site (no per-site ad-hoc
5//! `match` with a silent `_ => nspawn` catch-all).
6//!
7//! Backends are **not** a hardcoded enum. Each concrete backend self-registers a
8//! [`BackendRegistration`] via `inventory::submit!` next to its implementation,
9//! feature-gated by the `#[cfg]` that compiles that backend in (#518). The
10//! registry is therefore *exactly* the set of backends built into this binary:
11//!
12//! * `nspawn` — registered only under the `nspawn` feature (systemd-nspawn
13//! lightweight container).
14//! * `kata` — registered only under the `kata` feature (full VM isolation;
15//! `kata-vm` is a backward-compat alias for `kata`).
16//! * `oci` — registered only under the `oci` feature (rootless OCI/podman
17//! container, #346).
18//! * `wasm` — registered only under the `wasm` feature (in-process WebAssembly,
19//! explicit-name-only).
20//! * `cri` — registered only under the `cri` feature (#510: a gRPC *client*
21//! of an external, already-running CRI runtime service — containerd's CRI
22//! plugin, or CRI-O — not an embedded runtime). Explicit-name-only, like
23//! `wasm`: driving an external runtime is an auth-surface decision `"auto"`
24//! must never reach for on its own.
25//!
26//! YAGNI: still-speculative tiers are intentionally **not** registered. They
27//! gain a `submit!` when their phase actually builds them; until then they
28//! simply do not exist as selectable names (an explicit request errors).
29//!
30//! Selection is **config-driven** via `worker.backend` (a string: a registered
31//! backend name, or `"auto"`):
32//!
33//! * `"auto"` → among registrations that are
34//! [`auto_selectable`](BackendRegistration::auto_selectable), pick the
35//! highest-[`priority`](BackendRegistration::priority) one whose
36//! [`is_available`](BackendRegistration::is_available) probe passes. The choice
37//! is logged. If none qualifies, error — never run a workload without isolation,
38//! and never auto-pick a backend that opted out of auto-selection (a silent
39//! isolation downgrade; see the in-process `wasm` tier, #547 ZSP).
40//! * a concrete name → that registration, **iff** it is registered *and* its
41//! prerequisites are present. Otherwise error. We never substitute a different
42//! (weaker) backend (the #486 fail-open bug). An explicit name resolves
43//! regardless of `auto_selectable`, so an auto-excluded backend (e.g. `wasm`)
44//! is still reachable when deliberately requested by name.
45//!
46//! The cardinal rule is **fail-closed**: an unavailable or unknown backend
47//! returns an error, never a silent downgrade to weaker isolation.
48
49use std::sync::Arc;
50
51use hyprstream_discovery::scheduling::{
52 self, CapabilitySource, Predicate, RejectionReason, Resource, SelectionReport,
53};
54
55use crate::config::PoolConfig;
56use crate::error::{Result, WorkerError};
57
58#[cfg(feature = "oci-image")]
59use crate::config::ImageConfig;
60#[cfg(feature = "oci-image")]
61use crate::image::RafsStore;
62
63use super::SandboxBackend;
64
65/// Per-call construction context threaded to a backend's `construct` fn.
66///
67/// `inventory` items are `'static`, so [`BackendRegistration::construct`] must be
68/// a bare `fn` pointer that cannot capture per-call state. This struct carries
69/// everything a backend constructor needs at selection time.
70pub struct BackendCtx {
71 /// Sandbox pool configuration (paths, hypervisor, limits).
72 pub pool_config: PoolConfig,
73
74 /// Mandatory resolver-backed, audited policy-enforcement point for every
75 /// 9P export constructed by a backend.
76 #[cfg(not(target_arch = "wasm32"))]
77 pub ninep_decider: Arc<dyn hyprstream_9p::AccessDecider>,
78
79 /// #1269: full MAC reference monitor for 9P exports. `Some` at every
80 /// production constructor — the `anonymous_floor()` fallback is not
81 /// reachable. `None` in tests only.
82 #[cfg(not(target_arch = "wasm32"))]
83 pub ninep_monitor: Option<Arc<hyprstream_9p::ReferenceMonitor>>,
84
85 /// Image storage configuration. Any backend that composes the RAFS image
86 /// filesystem (kata over virtio-fs, or nspawn over FUSE — Model B #715)
87 /// consumes it, so it exists on the `oci-image` build, not just `kata-vm`.
88 /// The Cargo manifest is explicit that the RAFS image service "lives on
89 /// oci-image, not kata-vm".
90 #[cfg(feature = "oci-image")]
91 pub image_config: ImageConfig,
92
93 /// RAFS/nydus image store — available whenever the image filesystem service
94 /// is compiled in (`oci-image`), so both the kata (virtio-fs) and nspawn
95 /// (FUSE, #715) backends can compose a per-sandbox VFS from it.
96 #[cfg(feature = "oci-image")]
97 pub rafs_store: Arc<RafsStore>,
98}
99
100/// A compile-time sandbox-backend registration, collected via `inventory`.
101///
102/// Each backend submits one of these next to its implementation, gated by the
103/// `#[cfg]` feature that compiles the backend in. The registry is enumerated by
104/// [`resolve_backend`]; there is no central enum to keep in sync.
105#[derive(Debug)]
106pub struct BackendRegistration {
107 /// Stable selector name, e.g. `"kata"`, `"nspawn"`. Matched against
108 /// `worker.backend` (case-insensitively).
109 pub name: &'static str,
110
111 /// Auto-selection precedence: higher is preferred. `"auto"` picks the
112 /// highest-priority *available* registration (among `auto_selectable` ones).
113 pub priority: i32,
114
115 /// Whether this backend is eligible for `"auto"` selection.
116 ///
117 /// `true` → `"auto"` may pick it (subject to `priority` and `is_available`).
118 /// `false` → **explicit-name-only**: `"auto"` must never choose it, even if it
119 /// is the highest-priority available (or only) registration. It remains
120 /// selectable by its concrete name (fail-closed).
121 ///
122 /// This exists so weaker-isolation tiers cannot become a silent `"auto"`
123 /// fallback when stronger backends are absent — auto-selecting them would be a
124 /// silent isolation downgrade, which the #547 MAC/ZSP model forbids. The
125 /// in-process `wasm` sandbox (shared host address space) sets this `false`.
126 pub auto_selectable: bool,
127
128 /// Whether this backend can **inject a 9P socket endpoint** — a host Unix
129 /// domain socket — into a workload's mount namespace (#506).
130 ///
131 /// The Wanix-guest workload (`runtime::wanix_workload`) works by having
132 /// hyprstream serve a tenant's Subject-scoped VFS `Mount` as a 9P2000.L
133 /// server on a host UDS, then making that socket reachable *inside* the
134 /// sandbox (bind-mounting the host socket into the container's mount
135 /// namespace and setting `HYPRSTREAM_9P_SOCK`) so the guest dials back. That
136 /// requires the backend to be able to bind-mount an arbitrary host path (the
137 /// socket) into the workload, which the tiers do differently:
138 ///
139 /// * `oci` / `nspawn` → `true`: both bind-mount host paths into the
140 /// container (`podman --volume` / `nspawn --bind`), so a host UDS injects
141 /// cleanly.
142 /// * `kata` → `false`: a full VM does not share the host mount namespace, so
143 /// a host UDS cannot be bind-injected the same way (the VM path uses
144 /// virtio-fs, a different mechanism — see `sandbox_fs`); it therefore does
145 /// **not** advertise this capability.
146 /// * `wasm` → `false`: in-process, no separate mount namespace to inject a
147 /// host socket into.
148 ///
149 /// Selection is **fail-closed** on this flag: a workload that requires 9P
150 /// socket injection ([`resolve_backend_9p_capable`] /
151 /// [`require_9p_socket_capability`]) never resolves to a backend that does
152 /// not advertise it — it errors rather than silently dropping the injection.
153 pub injects_9p_socket: bool,
154 /// Whether this backend **FUSE-mounts the composed per-sandbox tenant VFS**
155 /// as a real host directory and POSIX-roots its workload there (Model B,
156 /// #715 — the non-VM sibling of the kata virtio-fs path).
157 ///
158 /// `true` → the backend consumes `SandboxFs::serve_local_at` to serve the
159 /// composed namespace over `/dev/fuse` and give that host directory to the
160 /// container. `nspawn` sets this. `kata` does **not** — it already shares the
161 /// composed VFS as a virtio-fs device, not a host FUSE mount. In-process
162 /// `wasm` has no host filesystem to mount into. `oci` is deferred pending a
163 /// rootless `/dev/fuse` feasibility spike and advertises `false` for now.
164 ///
165 /// [`require_fuse_mount_capability`] gates the mount fail-closed: a backend
166 /// that does not advertise this capability is never handed a FUSE-rooted
167 /// namespace (no silent POSIX-rooting in a namespace a backend can't mount).
168 pub mounts_fuse_vfs: bool,
169
170 /// Runtime prerequisite probe (PATH lookups, socket existence, …). Returns
171 /// `true` when this backend can actually run on this host *right now*. Used
172 /// to inform `"auto"` and to fail-close explicit requests whose prereqs are
173 /// missing.
174 pub is_available: fn() -> bool,
175
176 /// Construct the concrete backend from per-call [`BackendCtx`] state.
177 pub construct: fn(&BackendCtx) -> anyhow::Result<Arc<dyn SandboxBackend>>,
178}
179
180inventory::collect!(BackendRegistration);
181
182/// Comma-separated registered backend names, for error messages.
183fn registered_names(regs: &[&BackendRegistration]) -> String {
184 let mut names: Vec<&str> = regs.iter().map(|r| r.name).collect();
185 names.sort_unstable();
186 names.join(", ")
187}
188
189/// Pure selection over a registration set, with an injectable availability
190/// oracle. Factored out of [`resolve_backend`] so the fail-closed / auto-priority
191/// / unknown-name logic is unit-testable without depending on which runtimes
192/// happen to be installed on the test host.
193///
194/// Capability-agnostic; delegates to [`select_registration_with_cap`] with no
195/// required capability.
196fn select_registration<'a>(
197 name: &str,
198 regs: &[&'a BackendRegistration],
199 is_available: impl Fn(&BackendRegistration) -> bool,
200) -> Result<&'a BackendRegistration> {
201 select_registration_with_cap(name, regs, is_available, false)
202}
203
204/// Pure selection with an additional **required-capability** gate for 9P socket
205/// injection (#506), fail-closed.
206///
207/// When `require_9p_socket` is `true`, the chosen registration must advertise
208/// [`BackendRegistration::injects_9p_socket`]:
209///
210/// * `"auto"` → only backends that are *both* `auto_selectable` **and**
211/// `injects_9p_socket` are candidates; the highest-priority available one
212/// wins. If none qualifies, error — never fall back to a backend that cannot
213/// inject the socket (that would silently drop the tenant's namespace export).
214/// * a concrete name → resolves the named backend, then rejects it if it does
215/// not advertise the capability. We never substitute a different (capable)
216/// backend for an explicitly-named incapable one — the explicit request is
217/// authoritative, and the correct answer is a clean error, not a swap.
218///
219/// With `require_9p_socket == false` this is exactly the prior behaviour.
220fn select_registration_with_cap<'a>(
221 name: &str,
222 regs: &[&'a BackendRegistration],
223 is_available: impl Fn(&BackendRegistration) -> bool,
224 require_9p_socket: bool,
225) -> Result<&'a BackendRegistration> {
226 // ── Auto: highest-priority *available* registration wins ──
227 //
228 // Only `auto_selectable` registrations are eligible. A backend that opts out
229 // (e.g. the in-process `wasm` tier) must never be auto-picked, even as a last
230 // resort when nothing stronger is available — that would be a silent isolation
231 // downgrade (#547 ZSP). Such backends remain reachable by explicit name below.
232 if name.eq_ignore_ascii_case("auto") {
233 let mut candidates: Vec<&'a BackendRegistration> = regs
234 .iter()
235 .copied()
236 .filter(|r| r.auto_selectable)
237 // Capability gate: when a 9P-socket-injecting workload is being
238 // placed, an incapable backend is not even a candidate — auto must
239 // never downgrade to one that would silently drop the injection.
240 .filter(|r| !require_9p_socket || r.injects_9p_socket)
241 .collect();
242 // Highest priority first; ties broken by name for determinism.
243 candidates.sort_by(|a, b| b.priority.cmp(&a.priority).then_with(|| a.name.cmp(b.name)));
244 for reg in candidates {
245 if is_available(reg) {
246 return Ok(reg);
247 }
248 }
249 let cap_note = if require_9p_socket {
250 " that can inject a 9P socket endpoint (host UDS bind-mount)"
251 } else {
252 ""
253 };
254 return Err(WorkerError::ConfigError(format!(
255 "auto backend selection found no available sandbox backend{cap_note} among [{}]. \
256 Install a supported runtime or set `worker.backend` to an explicit \
257 backend; refusing to run a workload without isolation (fail-closed).",
258 registered_names(regs)
259 )));
260 }
261
262 // ── Explicit request: authoritative, fail-closed ──
263 match regs.iter().find(|r| r.name.eq_ignore_ascii_case(name)) {
264 // Named + available, but cannot inject the required 9P socket → reject.
265 // We do NOT substitute a capable backend for an explicitly-named one.
266 Some(reg) if require_9p_socket && !reg.injects_9p_socket => {
267 Err(WorkerError::ConfigError(format!(
268 "sandbox backend '{}' was requested for a workload that requires 9P \
269 socket injection (a host Unix socket bind-mounted into the sandbox), \
270 but '{}' cannot inject a host UDS into a workload's mount namespace. \
271 Choose a backend that can (e.g. oci, nspawn); refusing to silently \
272 drop the namespace export (fail-closed).",
273 reg.name, reg.name
274 )))
275 }
276 Some(reg) if is_available(reg) => Ok(reg),
277 Some(reg) => Err(WorkerError::ConfigError(format!(
278 "sandbox backend '{}' was requested but its runtime prerequisites are \
279 missing. Refusing to silently downgrade isolation to a weaker backend \
280 (fail-closed).",
281 reg.name
282 ))),
283 None => {
284 let mut msg = format!(
285 "unknown sandbox backend '{name}'; registered backends are [{}] (or \"auto\")",
286 registered_names(regs)
287 );
288 // Helpful hint for the most common misconfiguration: requesting the
289 // VM backend in a binary built without it.
290 if name.eq_ignore_ascii_case("kata") {
291 msg.push_str(
292 ". The `kata` backend is only present when built with \
293 `--features kata` (or the `kata-vm` alias)",
294 );
295 }
296 Err(WorkerError::ConfigError(msg))
297 }
298 }
299}
300
301/// Resolve `worker.backend` to a constructed [`SandboxBackend`], fail-closed.
302///
303/// `name` is either a registered backend name or `"auto"`. This is the single
304/// function both `factories.rs` and `bin/main.rs` route through — no scattered
305/// `#[cfg]`, no `_ => nspawn` fallback.
306///
307/// * **`"auto"`** → among [`auto_selectable`](BackendRegistration::auto_selectable)
308/// registrations, the highest-[`priority`](BackendRegistration::priority) one
309/// whose [`is_available`](BackendRegistration::is_available) is true; error if
310/// none (auto-excluded backends are never picked here — no silent downgrade).
311/// * **concrete name** → that registration if present *and* available, regardless
312/// of `auto_selectable`; otherwise error (unavailable → fail-closed; unknown →
313/// error listing the registry).
314pub fn resolve_backend(name: &str, ctx: &BackendCtx) -> Result<Arc<dyn SandboxBackend>> {
315 let regs: Vec<&'static BackendRegistration> =
316 inventory::iter::<BackendRegistration>().collect();
317
318 let reg = select_registration(name, ®s, |r| (r.is_available)())?;
319
320 tracing::info!(
321 backend = reg.name,
322 requested = name,
323 priority = reg.priority,
324 "sandbox backend selected (fail-closed)"
325 );
326
327 (reg.construct)(ctx).map_err(|e| {
328 WorkerError::ConfigError(format!(
329 "failed to construct sandbox backend '{}': {e:#}",
330 reg.name
331 ))
332 })
333}
334
335/// Resolve `worker.backend` to a [`SandboxBackend`] that is **guaranteed capable
336/// of 9P socket injection** (#506), fail-closed.
337///
338/// Identical to [`resolve_backend`] but with the capability gate engaged: the
339/// resolved backend always advertises
340/// [`injects_9p_socket`](BackendRegistration::injects_9p_socket). Use this when
341/// placing a workload (e.g. the Wanix guest) that needs a host UDS injected into
342/// its mount namespace. `"auto"` picks the highest-priority *capable* backend;
343/// an explicit name that is incapable errors rather than being swapped for a
344/// capable one.
345pub fn resolve_backend_9p_capable(name: &str, ctx: &BackendCtx) -> Result<Arc<dyn SandboxBackend>> {
346 let regs: Vec<&'static BackendRegistration> =
347 inventory::iter::<BackendRegistration>().collect();
348
349 let reg = select_registration_with_cap(name, ®s, |r| (r.is_available)(), true)?;
350
351 tracing::info!(
352 backend = reg.name,
353 requested = name,
354 priority = reg.priority,
355 "sandbox backend selected for 9P-socket-injecting workload (fail-closed)"
356 );
357
358 (reg.construct)(ctx).map_err(|e| {
359 WorkerError::ConfigError(format!(
360 "failed to construct sandbox backend '{}': {e:#}",
361 reg.name
362 ))
363 })
364}
365
366/// Does the registered backend named `backend_type` advertise 9P socket
367/// injection? Returns `false` for an unknown/unregistered name.
368///
369/// This queries the *registry* (not a live backend instance), so a caller
370/// holding an already-constructed `Arc<dyn SandboxBackend>` can check its
371/// capability via [`SandboxBackend::backend_type`](super::SandboxBackend::backend_type).
372pub fn backend_injects_9p_socket(backend_type: &str) -> bool {
373 inventory::iter::<BackendRegistration>()
374 .find(|r| r.name.eq_ignore_ascii_case(backend_type))
375 .is_some_and(|r| r.injects_9p_socket)
376}
377
378/// Fail-closed guard used at the workload seam: error unless `backend_type`
379/// advertises 9P socket injection (#506).
380///
381/// A caller that already holds a constructed backend (e.g. from the pool) uses
382/// this to refuse — cleanly, before spawning any 9P server or building any
383/// injection annotations — to place a socket-injecting workload on a backend
384/// that cannot receive the socket. It never downgrades; it only reports.
385pub fn require_9p_socket_capability(backend_type: &str) -> Result<()> {
386 if backend_injects_9p_socket(backend_type) {
387 Ok(())
388 } else {
389 Err(WorkerError::ConfigError(format!(
390 "sandbox backend '{backend_type}' cannot inject a 9P socket endpoint \
391 (a host Unix socket bind-mounted into the sandbox), which this workload \
392 requires. Select a capable backend (e.g. oci, nspawn); refusing to run \
393 the workload without its namespace export (fail-closed)."
394 )))
395 }
396}
397
398/// Pure capability check over a registration set, with the same fail-closed
399/// contract as [`select_registration`]. Factored out so the Model B (#715)
400/// FUSE-mount gate is unit-testable without depending on which backends the
401/// build's inventory happens to contain.
402fn check_fuse_mount_capability(name: &str, regs: &[&BackendRegistration]) -> Result<()> {
403 match regs.iter().find(|r| r.name.eq_ignore_ascii_case(name)) {
404 Some(reg) if reg.mounts_fuse_vfs => Ok(()),
405 Some(reg) => Err(WorkerError::ConfigError(format!(
406 "sandbox backend '{}' does not advertise the FUSE tenant-VFS mount \
407 capability (mounts_fuse_vfs); refusing to POSIX-root a workload in a \
408 namespace this backend was not declared able to mount (fail-closed).",
409 reg.name
410 ))),
411 None => Err(WorkerError::ConfigError(format!(
412 "unknown sandbox backend '{name}'; cannot check FUSE tenant-VFS mount \
413 capability"
414 ))),
415 }
416}
417
418/// Fail-closed gate for Model B (#715): assert `backend_type` advertises the
419/// [`mounts_fuse_vfs`](BackendRegistration::mounts_fuse_vfs) capability before a
420/// caller FUSE-mounts the composed tenant VFS as that backend's root.
421///
422/// Parallels [`resolve_backend`]'s fail-closed contract: an unknown backend, or
423/// a registered one that does not advertise the capability, errors rather than
424/// proceeding — we never POSIX-root a workload in a namespace a backend was not
425/// declared able to mount, and never silently substitute a different transport.
426pub fn require_fuse_mount_capability(backend_type: &str) -> Result<()> {
427 let regs: Vec<&'static BackendRegistration> =
428 inventory::iter::<BackendRegistration>().collect();
429 check_fuse_mount_capability(backend_type, ®s)
430}
431
432// ─────────────────────────────────────────────────────────────────────────────
433// Selection diagnostics (#348)
434// ─────────────────────────────────────────────────────────────────────────────
435//
436// `resolve_backend` answers "which backend do I get"; the function below adds
437// the backend-local "why not" knowledge the #628 scheduling substrate's explain
438// trace needs — a per-backend prerequisite sub-reason. This does not change
439// selection semantics; it is read-only introspection over the same registry.
440//
441// The generic filter→rank→select explain trace (`SelectionReport<C>`) is owned
442// by the #628 scheduling substrate (`hyprstream-discovery::scheduling`);
443// `explain_selection` below composes on it (see the "Scheduling-substrate
444// adoption" section) rather than carrying a second, backend-specific copy.
445
446/// Backend-specific sub-reason for why `is_available()` returned `false`,
447/// e.g. "cloud-hypervisor not on PATH" for `kata`. Falls back to a generic
448/// message for backends this function doesn't have specific knowledge of.
449///
450/// This mirrors the prerequisite probes in each backend's
451/// `registry_is_available()` (see `kata_backend.rs`, `nspawn.rs`,
452/// `wasm_backend.rs`) but is allowed to drift from them in detail since it is
453/// diagnostic-only — it never gates selection itself.
454fn unavailable_reason(name: &str) -> String {
455 match name {
456 "kata" => "runtime prerequisites missing: `cloud-hypervisor` not found on PATH".to_owned(),
457 "nspawn" => {
458 "runtime prerequisites missing: `systemd-nspawn` and/or `machinectl` not found on PATH"
459 .to_owned()
460 }
461 "podman" => "runtime prerequisites missing: `podman` not found on PATH".to_owned(),
462 "wasm" => "wasm runtime prerequisites are not satisfied".to_owned(),
463 other => format!("runtime prerequisites for '{other}' are not satisfied"),
464 }
465}
466
467// ─────────────────────────────────────────────────────────────────────────────
468// Scheduling-substrate adoption (#628)
469// ─────────────────────────────────────────────────────────────────────────────
470//
471// The explain surface #348 needs is the #628 substrate's generic
472// `filter → rank → select → SelectionReport<C>`. Backend selection is one
473// consumer of that shape: candidates are the compiled-in registrations,
474// availability is a `Predicate` whose `RejectionReason` is sourced from
475// `unavailable_reason`, and rank is by descending priority. `explain_selection`
476// runs the substrate pipeline and returns its `SelectionReport`, so the trace
477// falls out of `select` for free — no bespoke report type here.
478//
479// `resolve_backend`/`select_registration` above remain the authoritative
480// construction path (they encode explicit-name and unknown-name semantics that
481// are not a single generic `select` call); the substrate governs the "auto"
482// explain trace, keeping one explain shape across every scheduling surface.
483//
484// Full resource-aware ranking (GPU/memory/NUMA via `ResourceRequest`) plugs in
485// here as additional predicates + a resource-aware `rank_by`; that is downstream
486// consumer work (#341/#525). This adoption wires availability-as-predicate +
487// explain + the capability-source seam.
488
489/// A cloneable projection of a [`BackendRegistration`] for the substrate's
490/// generic pipeline (`SelectionReport<C>` requires `C: Clone`; a registration
491/// holds `fn` pointers and is not cloneable). Carries exactly what selection
492/// filtering, ranking, and the explain trace need.
493#[derive(Debug, Clone, PartialEq, Eq)]
494pub struct BackendCandidate {
495 /// Registered backend name (`"kata"`, `"nspawn"`, …).
496 pub name: &'static str,
497 /// Auto-selection precedence (higher preferred).
498 pub priority: i32,
499 /// Whether `"auto"` may pick this backend.
500 pub auto_selectable: bool,
501 /// Live availability probe result at the time the candidate was captured.
502 pub available: bool,
503}
504
505impl BackendCandidate {
506 fn from_registration(reg: &BackendRegistration) -> Self {
507 Self {
508 name: reg.name,
509 priority: reg.priority,
510 auto_selectable: reg.auto_selectable,
511 available: (reg.is_available)(),
512 }
513 }
514}
515
516/// Runtime availability + capability truth a backend contributes to the shared
517/// scheduling vocabulary. A node-record publisher aggregates these across every
518/// registered backend so a placement record's `runtime=<name>` labels reflect
519/// what is actually available here (no hand-declaration, no drift — #628).
520impl CapabilitySource for BackendCandidate {
521 fn capability_labels(&self) -> Vec<(String, String)> {
522 vec![
523 ("runtime".to_owned(), self.name.to_owned()),
524 (
525 format!("runtime.{}.available", self.name),
526 self.available.to_string(),
527 ),
528 ]
529 }
530
531 fn capability_resources(&self) -> Vec<Resource> {
532 // Backend selection contributes runtime-availability labels, not
533 // countable capacity; GPU/memory/NUMA capacity comes from other sources
534 // (DevicePool, node liveness). See the module note on resource-aware rank.
535 Vec::new()
536 }
537}
538
539/// Availability predicate: rejects a candidate whose runtime prerequisites are
540/// missing, carrying the backend-specific [`unavailable_reason`] as the
541/// [`RejectionReason`] so it lands in the [`SelectionReport`] trace.
542fn availability_predicate() -> Predicate<BackendCandidate> {
543 Box::new(|c: &BackendCandidate| {
544 if c.available {
545 None
546 } else {
547 Some(RejectionReason(unavailable_reason(c.name)))
548 }
549 })
550}
551
552/// Auto-eligibility predicate: rejects backends excluded from `"auto"` (the
553/// isolation-downgrade guard, #547 ZSP), with the reason recorded in the trace.
554fn auto_eligibility_predicate() -> Predicate<BackendCandidate> {
555 Box::new(|c: &BackendCandidate| {
556 if c.auto_selectable {
557 None
558 } else {
559 Some(RejectionReason(format!(
560 "backend '{}' is explicit-name-only (excluded from auto-selection; \
561 no silent isolation downgrade)",
562 c.name
563 )))
564 }
565 })
566}
567
568/// Explain what `"auto"` selection would decide over the compiled-in registry,
569/// as the shared [`SelectionReport`]. The winning candidate is the highest-
570/// priority backend that is both auto-eligible and available; every other
571/// candidate carries the [`RejectionReason`] that excluded it.
572///
573/// This is diagnostics only (a wizard/CLI "why this backend?" trace); the
574/// authoritative construction path is [`resolve_backend`]. Both agree on the
575/// same winner for `"auto"` because both rank by descending priority over the
576/// auto-eligible, available set.
577pub fn explain_selection() -> SelectionReport<BackendCandidate> {
578 let candidates: Vec<BackendCandidate> = inventory::iter::<BackendRegistration>()
579 .map(BackendCandidate::from_registration)
580 .collect();
581
582 let predicates = vec![auto_eligibility_predicate(), availability_predicate()];
583
584 let mut report = scheduling::select(
585 &candidates,
586 &predicates,
587 // Highest priority first; name ascending as a deterministic tiebreak —
588 // matching `select_registration`'s "auto" ordering exactly.
589 |a, b| b.priority.cmp(&a.priority).then_with(|| a.name.cmp(b.name)),
590 1, // "auto" selects a single backend
591 );
592 report.requested = "auto".to_owned();
593 report
594}
595
596#[cfg(test)]
597#[allow(clippy::unwrap_used, clippy::expect_used)]
598mod tests {
599 use super::*;
600
601 // ── Synthetic registrations for deterministic selection-logic tests ──
602 //
603 // These never touch the real `is_available` probes or `construct` fns; they
604 // exercise the pure `select_registration` spine (auto-priority, fail-closed,
605 // unknown-name) independent of what runtimes are installed.
606
607 fn never_available(_: &BackendCtx) -> anyhow::Result<Arc<dyn SandboxBackend>> {
608 unreachable!("construct is not called in selection-logic tests")
609 }
610
611 const HIGH: BackendRegistration = BackendRegistration {
612 name: "high-tier",
613 priority: 100,
614 auto_selectable: true,
615 injects_9p_socket: false,
616 mounts_fuse_vfs: false,
617 is_available: || true,
618 construct: never_available,
619 };
620 const LOW: BackendRegistration = BackendRegistration {
621 name: "low-tier",
622 priority: 10,
623 auto_selectable: true,
624 injects_9p_socket: false,
625 mounts_fuse_vfs: false,
626 is_available: || true,
627 construct: never_available,
628 };
629
630 /// A FUSE-mount-capable backend, modeling the Model B (#715) nspawn target:
631 /// it advertises `mounts_fuse_vfs: true`.
632 const FUSE_CAP: BackendRegistration = BackendRegistration {
633 name: "fuse-tier",
634 priority: 50,
635 auto_selectable: true,
636 injects_9p_socket: false,
637 mounts_fuse_vfs: true,
638 is_available: || true,
639 construct: never_available,
640 };
641
642 /// An auto-excluded (explicit-name-only) backend, modeling the in-process
643 /// `wasm` tier: even at the highest priority and always available, `"auto"`
644 /// must never pick it.
645 const EXPLICIT_ONLY: BackendRegistration = BackendRegistration {
646 name: "explicit-only",
647 priority: 1000,
648 auto_selectable: false,
649 injects_9p_socket: false,
650 mounts_fuse_vfs: false,
651 is_available: || true,
652 construct: never_available,
653 };
654
655 /// A high-priority backend that CAN inject a 9P socket (models `oci`).
656 const NINEP_HIGH: BackendRegistration = BackendRegistration {
657 name: "ninep-high",
658 priority: 100,
659 auto_selectable: true,
660 injects_9p_socket: true,
661 mounts_fuse_vfs: false,
662 is_available: || true,
663 construct: never_available,
664 };
665 /// A lower-priority backend that CAN inject a 9P socket (models `nspawn`).
666 const NINEP_LOW: BackendRegistration = BackendRegistration {
667 name: "ninep-low",
668 priority: 10,
669 auto_selectable: true,
670 injects_9p_socket: true,
671 mounts_fuse_vfs: false,
672 is_available: || true,
673 construct: never_available,
674 };
675
676 fn regs() -> Vec<&'static BackendRegistration> {
677 vec![&HIGH, &LOW]
678 }
679
680 #[test]
681 fn auto_picks_highest_priority_available() {
682 // Both available → strongest (highest priority) wins.
683 let r = select_registration("auto", ®s(), |_| true).unwrap();
684 assert_eq!(r.name, "high-tier");
685 }
686
687 #[test]
688 fn auto_skips_unavailable_to_next_priority() {
689 // Strongest unavailable → auto must fall to the next available tier,
690 // never error while a usable backend remains.
691 let r = select_registration("auto", ®s(), |reg| reg.name != "high-tier").unwrap();
692 assert_eq!(r.name, "low-tier");
693 }
694
695 #[test]
696 fn auto_errors_when_nothing_available() {
697 let err = select_registration("auto", ®s(), |_| false).unwrap_err();
698 let msg = err.to_string();
699 assert!(msg.contains("no available sandbox backend"), "got: {msg}");
700 assert!(msg.contains("fail-closed"), "got: {msg}");
701 }
702
703 #[test]
704 fn auto_never_picks_explicit_only_even_as_only_backend() {
705 // An auto-excluded backend that is the *only* registration (and available)
706 // must NOT be auto-selected — `"auto"` errors instead of downgrading.
707 let only_explicit = vec![&EXPLICIT_ONLY];
708 let err = select_registration("auto", &only_explicit, |_| true).unwrap_err();
709 let msg = err.to_string();
710 assert!(msg.contains("no available sandbox backend"), "got: {msg}");
711 assert!(msg.contains("fail-closed"), "got: {msg}");
712 }
713
714 #[test]
715 fn auto_skips_explicit_only_for_a_weaker_auto_eligible_backend() {
716 // Even though EXPLICIT_ONLY has the highest priority and is available,
717 // `"auto"` must prefer a lower-priority *auto-eligible* backend over it —
718 // never an isolation downgrade to the auto-excluded tier.
719 let regs = vec![&EXPLICIT_ONLY, &LOW];
720 let r = select_registration("auto", ®s, |_| true).unwrap();
721 assert_eq!(
722 r.name, "low-tier",
723 "auto must skip the auto-excluded backend"
724 );
725 }
726
727 #[test]
728 fn explicit_only_still_selectable_by_name() {
729 // Auto-exclusion does not remove the backend from the registry: an
730 // explicit request resolves it (fail-closed when unavailable).
731 let regs = vec![&EXPLICIT_ONLY, &LOW];
732 let r = select_registration("explicit-only", ®s, |_| true).unwrap();
733 assert_eq!(r.name, "explicit-only");
734 }
735
736 #[test]
737 fn explicit_available_resolves_exactly() {
738 let r = select_registration("low-tier", ®s(), |_| true).unwrap();
739 assert_eq!(r.name, "low-tier");
740 // Case-insensitive.
741 let r = select_registration("LOW-TIER", ®s(), |_| true).unwrap();
742 assert_eq!(r.name, "low-tier");
743 }
744
745 #[test]
746 fn explicit_unavailable_fails_closed_no_downgrade() {
747 // "high-tier" requested but unavailable → error, must NOT downgrade to
748 // the available "low-tier" backend.
749 let err =
750 select_registration("high-tier", ®s(), |reg| reg.name == "low-tier").unwrap_err();
751 let msg = err.to_string();
752 assert!(msg.contains("prerequisites are missing"), "got: {msg}");
753 assert!(msg.contains("fail-closed"), "got: {msg}");
754 // Must not name the weaker backend it could have silently fallen back to.
755 assert!(
756 !msg.contains("low-tier"),
757 "must not mention a downgrade target: {msg}"
758 );
759 }
760
761 #[test]
762 fn unknown_name_errors_listing_registry() {
763 let err = select_registration("bogus", ®s(), |_| true).unwrap_err();
764 let msg = err.to_string();
765 assert!(
766 msg.contains("unknown sandbox backend 'bogus'"),
767 "got: {msg}"
768 );
769 assert!(
770 msg.contains("high-tier"),
771 "should list registered names: {msg}"
772 );
773 assert!(
774 msg.contains("low-tier"),
775 "should list registered names: {msg}"
776 );
777 }
778
779 // ── 9P-socket-injection capability gate (#506), fail-closed ──
780
781 #[test]
782 fn cap_auto_picks_highest_priority_capable_backend() {
783 // Both capable → strongest (highest priority) wins.
784 let regs = vec![&NINEP_HIGH, &NINEP_LOW];
785 let r = select_registration_with_cap("auto", ®s, |_| true, true).unwrap();
786 assert_eq!(r.name, "ninep-high");
787 }
788
789 #[test]
790 fn cap_auto_skips_incapable_backend_even_if_higher_priority() {
791 // HIGH (priority 100) cannot inject; NINEP_LOW (priority 10) can. With
792 // the capability required, auto MUST pick the lower-priority *capable*
793 // backend, never the higher-priority incapable one.
794 let regs = vec![&HIGH, &NINEP_LOW];
795 let r = select_registration_with_cap("auto", ®s, |_| true, true).unwrap();
796 assert_eq!(r.name, "ninep-low", "auto must pick the capable backend");
797 }
798
799 #[test]
800 fn cap_auto_errors_when_no_capable_backend_available() {
801 // Only incapable backends present → auto errors rather than downgrading
802 // to one that would silently drop the 9P injection.
803 let regs = vec![&HIGH, &LOW];
804 let err = select_registration_with_cap("auto", ®s, |_| true, true).unwrap_err();
805 let msg = err.to_string();
806 assert!(msg.contains("9P socket endpoint"), "got: {msg}");
807 assert!(msg.contains("fail-closed"), "got: {msg}");
808 }
809
810 #[test]
811 fn cap_auto_errors_when_capable_backend_unavailable() {
812 // The only capable backend is unavailable → auto fails closed (no
813 // fallback to an available-but-incapable backend).
814 let regs = vec![&NINEP_HIGH, &LOW];
815 let err = select_registration_with_cap("auto", ®s, |r| r.name == "low-tier", true)
816 .unwrap_err();
817 assert!(err.to_string().contains("9P socket endpoint"), "got: {err}");
818 }
819
820 #[test]
821 fn cap_explicit_capable_resolves() {
822 let regs = vec![&NINEP_HIGH, &LOW];
823 let r = select_registration_with_cap("ninep-high", ®s, |_| true, true).unwrap();
824 assert_eq!(r.name, "ninep-high");
825 }
826
827 #[test]
828 fn cap_explicit_incapable_errors_no_substitution() {
829 // Explicitly asking for an incapable backend when the capability is
830 // required errors — and must NOT silently swap in a capable one.
831 let regs = vec![&HIGH, &NINEP_LOW];
832 let err = select_registration_with_cap("high-tier", ®s, |_| true, true).unwrap_err();
833 let msg = err.to_string();
834 assert!(msg.contains("9P socket injection"), "got: {msg}");
835 assert!(msg.contains("fail-closed"), "got: {msg}");
836 // Must not name the capable backend it could have swapped to.
837 assert!(
838 !msg.contains("ninep-low"),
839 "must not suggest a silent swap: {msg}"
840 );
841 }
842
843 #[test]
844 fn cap_off_ignores_incapable_flag() {
845 // With the capability NOT required, an incapable backend resolves fine
846 // (the gate is opt-in; default behaviour is unchanged).
847 let regs = vec![&HIGH, &LOW];
848 let r = select_registration_with_cap("high-tier", ®s, |_| true, false).unwrap();
849 assert_eq!(r.name, "high-tier");
850 let r = select_registration_with_cap("auto", ®s, |_| true, false).unwrap();
851 assert_eq!(r.name, "high-tier");
852 }
853
854 #[test]
855 fn require_capability_helper_is_fail_closed() {
856 // The registry-backed helper the workload seam uses: real registered
857 // backends reflect reality (oci/nspawn capable; kata/wasm not), and an
858 // unknown name is treated as incapable (fail-closed).
859 assert!(!backend_injects_9p_socket("does-not-exist"));
860 assert!(require_9p_socket_capability("does-not-exist").is_err());
861
862 // nspawn is registered under the `nspawn` feature and IS capable (host --bind).
863 #[cfg(feature = "nspawn")]
864 {
865 assert!(
866 backend_injects_9p_socket("nspawn"),
867 "nspawn can bind a host UDS"
868 );
869 assert!(require_9p_socket_capability("nspawn").is_ok());
870 // Case-insensitive, mirroring name matching.
871 assert!(backend_injects_9p_socket("NSPAWN"));
872 }
873 }
874
875 #[cfg(feature = "oci")]
876 #[test]
877 fn oci_advertises_9p_socket_injection() {
878 let oci = inventory::iter::<BackendRegistration>()
879 .find(|r| r.name == "oci")
880 .expect("oci registered under the oci feature");
881 assert!(
882 oci.injects_9p_socket,
883 "oci bind-mounts a host UDS → capable"
884 );
885 }
886
887 #[cfg(feature = "nspawn")]
888 #[test]
889 fn nspawn_advertises_9p_socket_injection() {
890 let nspawn = inventory::iter::<BackendRegistration>()
891 .find(|r| r.name == "nspawn")
892 .expect("nspawn registered under the nspawn feature");
893 assert!(
894 nspawn.injects_9p_socket,
895 "nspawn --bind can inject a host UDS"
896 );
897 }
898
899 #[cfg(feature = "kata")]
900 #[test]
901 fn kata_does_not_advertise_9p_socket_injection() {
902 // A VM does not share the host mount namespace, so it cannot bind-inject
903 // a host UDS the way oci/nspawn do — it must NOT advertise the capability.
904 let kata = inventory::iter::<BackendRegistration>()
905 .find(|r| r.name == "kata")
906 .expect("kata registered under the kata feature");
907 assert!(
908 !kata.injects_9p_socket,
909 "kata (VM, no shared host mount ns) must not advertise host-UDS injection"
910 );
911 }
912
913 #[cfg(feature = "wasm")]
914 #[test]
915 fn wasm_does_not_advertise_9p_socket_injection() {
916 let wasm = inventory::iter::<BackendRegistration>()
917 .find(|r| r.name == "wasm")
918 .expect("wasm registered under the wasm feature");
919 assert!(
920 !wasm.injects_9p_socket,
921 "in-process wasm has no separate mount ns to inject a host socket into"
922 );
923 }
924
925 #[test]
926 fn unknown_kata_hints_about_feature() {
927 // Asking for kata when it isn't registered (e.g. kata off) gives a
928 // build-feature hint rather than a bare unknown error.
929 let err = select_registration("kata", ®s(), |_| true).unwrap_err();
930 assert!(err.to_string().contains("kata"), "got: {err}");
931 assert!(err.to_string().contains("kata-vm"), "got: {err}");
932 }
933
934 // ── The real inventory must reflect the build's feature set ──
935
936 // ── Model B (#715): FUSE tenant-VFS mount capability gate ──
937
938 #[test]
939 fn fuse_capability_gate_allows_advertised_backend() {
940 let regs = vec![&FUSE_CAP, &LOW];
941 assert!(check_fuse_mount_capability("fuse-tier", ®s).is_ok());
942 // Case-insensitive, mirroring backend selection.
943 assert!(check_fuse_mount_capability("FUSE-TIER", ®s).is_ok());
944 }
945
946 #[test]
947 fn fuse_capability_gate_fails_closed_for_non_advertising_backend() {
948 // A registered backend that does NOT advertise the capability must be
949 // refused — no silent POSIX-rooting in a namespace it can't mount.
950 let regs = vec![&FUSE_CAP, &LOW];
951 let err = check_fuse_mount_capability("low-tier", ®s).unwrap_err();
952 let msg = err.to_string();
953 assert!(msg.contains("does not advertise"), "got: {msg}");
954 assert!(msg.contains("fail-closed"), "got: {msg}");
955 }
956
957 #[test]
958 fn fuse_capability_gate_unknown_backend_errors() {
959 let regs = vec![&FUSE_CAP];
960 let err = check_fuse_mount_capability("bogus", ®s).unwrap_err();
961 assert!(
962 err.to_string().contains("unknown sandbox backend"),
963 "got: {err}"
964 );
965 }
966
967 #[cfg(feature = "nspawn")]
968 #[test]
969 fn nspawn_advertises_fuse_mount_capability_in_real_inventory() {
970 // The real inventory: nspawn is the Model B FUSE-root target (#715), so
971 // the public gate resolves it Ok. nspawn registers under its feature.
972 assert!(
973 require_fuse_mount_capability("nspawn").is_ok(),
974 "nspawn must advertise mounts_fuse_vfs"
975 );
976 }
977
978 #[cfg(feature = "kata")]
979 #[test]
980 fn kata_does_not_advertise_fuse_mount_capability() {
981 // kata already shares the composed VFS via virtio-fs, not a host FUSE
982 // mount, so it must NOT advertise the capability (fail-closed).
983 let err = require_fuse_mount_capability("kata").unwrap_err();
984 assert!(err.to_string().contains("does not advertise"), "got: {err}");
985 }
986
987 #[test]
988 fn registry_contains_nspawn_only_under_feature() {
989 // nspawn registers iff built with `--features nspawn` (#518: per-
990 // BackendType feature, mirroring kata/oci/cri/wasm). With the feature off
991 // it must NOT be a selectable name (fail-closed).
992 let has_nspawn = inventory::iter::<BackendRegistration>().any(|r| r.name == "nspawn");
993 assert_eq!(
994 has_nspawn,
995 cfg!(feature = "nspawn"),
996 "nspawn registration must track the nspawn feature"
997 );
998 }
999
1000 #[test]
1001 fn registry_contains_kata_only_under_feature() {
1002 let has_kata = inventory::iter::<BackendRegistration>().any(|r| r.name == "kata");
1003 assert_eq!(
1004 has_kata,
1005 cfg!(feature = "kata"),
1006 "kata registration must track the kata feature (kata-vm is its alias)"
1007 );
1008 }
1009
1010 // YAGNI note: still-speculative tiers must not be in the registry until
1011 // built. `wasm` (#505 P2), `oci` (#346) and `cri` (#510) are now real
1012 // backends, each tracked by its own feature-gated test below; nothing
1013 // remains speculative at present.
1014
1015 #[test]
1016 fn registry_contains_cri_only_under_feature() {
1017 // The CRI-client backend (#510) registers iff built with `--features
1018 // cri`. With the feature off it must NOT be a selectable name
1019 // (fail-closed), mirroring `wasm`/`oci`/`kata`.
1020 let has_cri = inventory::iter::<BackendRegistration>().any(|r| r.name == "cri");
1021 assert_eq!(
1022 has_cri,
1023 cfg!(feature = "cri"),
1024 "cri registration must track the cri feature"
1025 );
1026 }
1027
1028 #[test]
1029 fn registry_contains_k8s_only_under_feature() {
1030 // The Kubernetes API-server backend (#781) registers iff built with
1031 // `--features k8s`. With the feature off it must NOT be a selectable
1032 // name (fail-closed), mirroring `cri`/`wasm`/`oci`/`kata`.
1033 let has_k8s = inventory::iter::<BackendRegistration>().any(|r| r.name == "k8s");
1034 assert_eq!(
1035 has_k8s,
1036 cfg!(feature = "k8s"),
1037 "k8s registration must track the k8s feature"
1038 );
1039 }
1040
1041 #[cfg(feature = "k8s")]
1042 #[test]
1043 fn k8s_is_explicit_name_only() {
1044 // Handing a workload to a cluster (scheduler-placed, cross-node,
1045 // quota-charged) is an auth-surface decision (#781/#778): `"auto"` must
1046 // never reach for `k8s`, mirroring `cri`.
1047 let k8s = inventory::iter::<BackendRegistration>()
1048 .find(|r| r.name == "k8s")
1049 .expect("k8s registered under the k8s feature");
1050 assert!(!k8s.auto_selectable, "k8s must be explicit-name-only");
1051 }
1052
1053 #[cfg(feature = "cri")]
1054 #[test]
1055 fn cri_is_explicit_name_only() {
1056 // Driving an external runtime is an auth-surface decision (#510):
1057 // `"auto"` must never reach for `cri` on its own, even if it were the
1058 // only registered backend — mirrors `wasm`'s explicit-name-only stance.
1059 let cri = inventory::iter::<BackendRegistration>()
1060 .find(|r| r.name == "cri")
1061 .expect("cri registered under the cri feature");
1062 assert!(!cri.auto_selectable, "cri must be explicit-name-only");
1063 }
1064
1065 #[test]
1066 fn registry_contains_oci_only_under_feature() {
1067 // The rootless OCI (podman) backend (#346) registers iff built with
1068 // `--features oci`. With the feature off it must NOT be a selectable name
1069 // (fail-closed), mirroring `wasm`/`kata`.
1070 let has_oci = inventory::iter::<BackendRegistration>().any(|r| r.name == "oci");
1071 assert_eq!(
1072 has_oci,
1073 cfg!(feature = "oci"),
1074 "oci registration must track the oci feature"
1075 );
1076 }
1077
1078 #[cfg(all(feature = "oci", feature = "nspawn"))]
1079 #[test]
1080 fn oci_is_auto_selectable_and_outranks_nspawn() {
1081 // A rootless container (user namespaces + seccomp + its own image rootfs)
1082 // is a real isolation tier, so — unlike the in-process `wasm` backend — it
1083 // is auto-selectable and outranks nspawn (host-rootfs sharing).
1084 let oci = inventory::iter::<BackendRegistration>()
1085 .find(|r| r.name == "oci")
1086 .expect("oci registered under the oci feature");
1087 let nspawn = inventory::iter::<BackendRegistration>()
1088 .find(|r| r.name == "nspawn")
1089 .expect("nspawn registered under the nspawn feature");
1090 assert!(
1091 oci.auto_selectable,
1092 "oci is a real isolation tier → auto-selectable"
1093 );
1094 assert!(
1095 oci.priority > nspawn.priority,
1096 "rootless OCI must outrank nspawn for auto-selection"
1097 );
1098 }
1099
1100 #[test]
1101 fn registry_contains_wasm_only_under_feature() {
1102 // The in-process wasm backend (#505 P2) registers iff built with
1103 // `--features wasm` (which vendors the wasmtime substrate). With the
1104 // feature off it must NOT be a selectable name (fail-closed).
1105 let has_wasm = inventory::iter::<BackendRegistration>().any(|r| r.name == "wasm");
1106 assert_eq!(
1107 has_wasm,
1108 cfg!(feature = "wasm"),
1109 "wasm registration must track the wasm feature"
1110 );
1111 }
1112
1113 #[cfg(feature = "wasm")]
1114 #[test]
1115 fn wasm_is_auto_excluded() {
1116 // The in-process wasm backend (shared host address space) must be flagged
1117 // explicit-name-only so it can never be auto-picked (#547 ZSP — no silent
1118 // isolation downgrade).
1119 let wasm = inventory::iter::<BackendRegistration>()
1120 .find(|r| r.name == "wasm")
1121 .expect("wasm registered under the wasm feature");
1122 assert!(
1123 !wasm.auto_selectable,
1124 "wasm (in-process, shared address space) must be excluded from auto-selection"
1125 );
1126 }
1127
1128 #[cfg(feature = "wasm")]
1129 #[test]
1130 fn auto_never_returns_wasm_even_when_only_available_backend() {
1131 // With *only* the real wasm registration present (registered + available),
1132 // `"auto"` must still error rather than downgrade to it.
1133 let wasm = inventory::iter::<BackendRegistration>()
1134 .find(|r| r.name == "wasm")
1135 .expect("wasm registered under the wasm feature");
1136 let only_wasm = vec![wasm];
1137 let err = select_registration("auto", &only_wasm, |r| (r.is_available)()).unwrap_err();
1138 assert!(
1139 err.to_string().contains("no available sandbox backend"),
1140 "auto must not pick wasm; got: {err}"
1141 );
1142 }
1143
1144 // The wasm-vs-nspawn comparison needs both backends compiled in: wasm (the
1145 // auto-excluded tier) and nspawn (the auto-eligible tier that must win).
1146 #[cfg(all(feature = "wasm", feature = "nspawn"))]
1147 #[test]
1148 fn auto_prefers_nspawn_over_auto_excluded_wasm() {
1149 // With nspawn alongside wasm, `"auto"` resolves to nspawn — never wasm.
1150 // Inject a deterministic availability oracle (everything available) so this
1151 // does NOT depend on the runner actually having nspawn installed: nspawn is
1152 // auto_selectable (and outranks), wasm is auto_selectable:false (excluded), so
1153 // auto must return nspawn. (Real-runtime availability is exercised elsewhere;
1154 // here we assert the selection *logic*, env-independently.)
1155 let wasm = inventory::iter::<BackendRegistration>()
1156 .find(|r| r.name == "wasm")
1157 .expect("wasm registered under the wasm feature");
1158 let nspawn = inventory::iter::<BackendRegistration>()
1159 .find(|r| r.name == "nspawn")
1160 .expect("nspawn registered under the nspawn feature");
1161 let both = vec![wasm, nspawn];
1162 let r = select_registration("auto", &both, |_| true).unwrap();
1163 assert_eq!(r.name, "nspawn", "auto must pick nspawn, never wasm");
1164 }
1165
1166 #[cfg(feature = "wasm")]
1167 #[test]
1168 fn wasm_selectable_by_name_and_available() {
1169 // Explicit `wasm` request resolves to the wasm registration, and it is
1170 // always available once compiled in (wasmtime is vendored in-process).
1171 let regs: Vec<&'static BackendRegistration> =
1172 inventory::iter::<BackendRegistration>().collect();
1173 let reg = select_registration("wasm", ®s, |r| (r.is_available)())
1174 .expect("wasm selectable by name when feature on");
1175 assert_eq!(reg.name, "wasm");
1176 assert!((reg.is_available)(), "wasm is always available once built");
1177 }
1178
1179 #[cfg(all(feature = "kata", feature = "nspawn"))]
1180 #[test]
1181 fn kata_outranks_nspawn_in_priority() {
1182 let kata = inventory::iter::<BackendRegistration>()
1183 .find(|r| r.name == "kata")
1184 .unwrap();
1185 let nspawn = inventory::iter::<BackendRegistration>()
1186 .find(|r| r.name == "nspawn")
1187 .unwrap();
1188 assert!(
1189 kata.priority > nspawn.priority,
1190 "kata (VM) must outrank nspawn for auto-selection"
1191 );
1192 }
1193
1194 // ── N-backend fallback-chain semantics (#348) ──
1195 //
1196 // Synthetic 3-tier registry modeling a future kata > podman > nspawn world
1197 // (podman lands under #346; until then this just proves the *logic*
1198 // generalizes past two backends — it does not depend on #346 landing).
1199
1200 const MID: BackendRegistration = BackendRegistration {
1201 name: "mid-tier",
1202 priority: 50,
1203 auto_selectable: true,
1204 injects_9p_socket: false,
1205 mounts_fuse_vfs: false,
1206 is_available: || true,
1207 construct: never_available,
1208 };
1209
1210 fn three_tier_regs() -> Vec<&'static BackendRegistration> {
1211 vec![&HIGH, &MID, &LOW]
1212 }
1213
1214 #[test]
1215 fn auto_three_tier_all_available_picks_highest() {
1216 let r = select_registration("auto", &three_tier_regs(), |_| true).unwrap();
1217 assert_eq!(r.name, "high-tier");
1218 }
1219
1220 #[test]
1221 fn auto_three_tier_degrades_through_priority_order_one_by_one() {
1222 // Simulate backends disappearing one at a time, strongest first —
1223 // exactly the "kata missing -> try podman -> try nspawn" chain a
1224 // wizard cares about. Each step must fall exactly one tier, never
1225 // skip ahead and never error while something remains available.
1226 let regs = three_tier_regs();
1227
1228 let r = select_registration("auto", ®s, |reg| reg.name != "high-tier").unwrap();
1229 assert_eq!(
1230 r.name, "mid-tier",
1231 "first degradation: high unavailable -> mid"
1232 );
1233
1234 let r = select_registration("auto", ®s, |reg| reg.name == "low-tier").unwrap();
1235 assert_eq!(r.name, "low-tier", "second degradation: only low remains");
1236 }
1237
1238 #[test]
1239 fn auto_three_tier_errors_only_when_all_unavailable() {
1240 let err = select_registration("auto", &three_tier_regs(), |_| false).unwrap_err();
1241 let msg = err.to_string();
1242 assert!(msg.contains("no available sandbox backend"), "got: {msg}");
1243 // Must list all three registered names so the operator knows what to install.
1244 assert!(msg.contains("high-tier") && msg.contains("mid-tier") && msg.contains("low-tier"));
1245 }
1246
1247 #[test]
1248 fn auto_three_tier_skips_middle_unavailable_tier() {
1249 // Mid-tier specifically down (e.g. podman not installed) must not
1250 // wrongly fall all the way to low; high still wins if available, and
1251 // if high is also down it must land on low (skipping mid), not error.
1252 let regs = three_tier_regs();
1253 let r = select_registration("auto", ®s, |reg| reg.name != "mid-tier").unwrap();
1254 assert_eq!(
1255 r.name, "high-tier",
1256 "high still available, mid down -> high wins"
1257 );
1258
1259 let r = select_registration("auto", ®s, |reg| reg.name == "low-tier").unwrap();
1260 assert_eq!(
1261 r.name, "low-tier",
1262 "high and mid down -> falls through mid to low"
1263 );
1264 }
1265
1266 // ── unavailable_reason (#348 selection diagnostics) ──
1267
1268 #[test]
1269 fn unavailable_reason_has_per_backend_specifics() {
1270 assert!(unavailable_reason("kata").contains("cloud-hypervisor"));
1271 assert!(unavailable_reason("nspawn").contains("systemd-nspawn"));
1272 assert!(unavailable_reason("podman").contains("podman"));
1273 // Unknown backend still gets a non-empty, non-panicking message.
1274 assert!(unavailable_reason("future-backend").contains("future-backend"));
1275 }
1276
1277 // ── Scheduling-substrate explain adoption (#628) ──
1278 //
1279 // These exercise the substrate pipeline over `BackendCandidate` directly (a
1280 // deterministic candidate set), proving the migrated explain produces the
1281 // shared `SelectionReport<C>` and that `unavailable_reason` flows through as
1282 // the `RejectionReason`. `explain_selection()` itself is a thin wrapper that
1283 // feeds the real inventory into this same pipeline.
1284
1285 fn candidate(name: &'static str, priority: i32, auto: bool, avail: bool) -> BackendCandidate {
1286 BackendCandidate {
1287 name,
1288 priority,
1289 auto_selectable: auto,
1290 available: avail,
1291 }
1292 }
1293
1294 /// Reusable copy of `explain_selection`'s pipeline over an explicit candidate
1295 /// set (so tests don't depend on the host's installed runtimes).
1296 fn explain_over(candidates: &[BackendCandidate]) -> SelectionReport<BackendCandidate> {
1297 let predicates = vec![auto_eligibility_predicate(), availability_predicate()];
1298 let mut report = scheduling::select(
1299 candidates,
1300 &predicates,
1301 |a, b| b.priority.cmp(&a.priority).then_with(|| a.name.cmp(b.name)),
1302 1,
1303 );
1304 report.requested = "auto".to_owned();
1305 report
1306 }
1307
1308 #[test]
1309 fn explain_selects_highest_priority_available_auto_eligible() {
1310 let cands = vec![
1311 candidate("kata", 100, true, true),
1312 candidate("nspawn", 10, true, true),
1313 ];
1314 let report = explain_over(&cands);
1315 let selected: Vec<&str> = report
1316 .candidates
1317 .iter()
1318 .filter(|o| o.selected)
1319 .map(|o| o.candidate.name)
1320 .collect();
1321 assert_eq!(selected, vec!["kata"], "highest-priority available wins");
1322 assert_eq!(report.requested, "auto");
1323 }
1324
1325 #[test]
1326 fn explain_reports_unavailable_reason_as_rejection() {
1327 // kata unavailable → excluded, and its RejectionReason must be the
1328 // backend-specific `unavailable_reason` (the #628 integration point).
1329 let cands = vec![
1330 candidate("kata", 100, true, false),
1331 candidate("nspawn", 10, true, true),
1332 ];
1333 let report = explain_over(&cands);
1334 let kata = report
1335 .candidates
1336 .iter()
1337 .find(|o| o.candidate.name == "kata")
1338 .unwrap();
1339 assert!(!kata.selected);
1340 assert_eq!(
1341 kata.rejection_reason.as_deref(),
1342 Some(unavailable_reason("kata").as_str())
1343 );
1344 assert!(kata
1345 .rejection_reason
1346 .as_deref()
1347 .unwrap()
1348 .contains("cloud-hypervisor"));
1349 // nspawn (available, auto-eligible) is the winner.
1350 let nspawn = report
1351 .candidates
1352 .iter()
1353 .find(|o| o.candidate.name == "nspawn")
1354 .unwrap();
1355 assert!(nspawn.selected);
1356 }
1357
1358 #[test]
1359 fn explain_excludes_auto_ineligible_with_reason() {
1360 // An explicit-name-only backend (e.g. wasm) at the highest priority and
1361 // available must be rejected in the trace, and a weaker auto-eligible
1362 // backend selected — matching `select_registration`'s auto semantics.
1363 let cands = vec![
1364 candidate("wasm", 1000, false, true),
1365 candidate("nspawn", 10, true, true),
1366 ];
1367 let report = explain_over(&cands);
1368 let wasm = report
1369 .candidates
1370 .iter()
1371 .find(|o| o.candidate.name == "wasm")
1372 .unwrap();
1373 assert!(!wasm.selected);
1374 assert!(wasm
1375 .rejection_reason
1376 .as_deref()
1377 .unwrap()
1378 .contains("explicit-name-only"));
1379 let nspawn = report
1380 .candidates
1381 .iter()
1382 .find(|o| o.candidate.name == "nspawn")
1383 .unwrap();
1384 assert!(
1385 nspawn.selected,
1386 "auto must pick the auto-eligible backend, never the excluded one"
1387 );
1388 }
1389
1390 #[test]
1391 fn explain_agrees_with_select_registration_on_the_winner() {
1392 // The explain trace and the authoritative `select_registration` path must
1393 // agree on which backend `"auto"` yields, over the same candidate set.
1394 const A: BackendRegistration = BackendRegistration {
1395 name: "high-tier",
1396 priority: 100,
1397 auto_selectable: true,
1398 injects_9p_socket: false,
1399 mounts_fuse_vfs: false,
1400 is_available: || true,
1401 construct: never_available,
1402 };
1403 const B: BackendRegistration = BackendRegistration {
1404 name: "low-tier",
1405 priority: 10,
1406 auto_selectable: true,
1407 injects_9p_socket: false,
1408 mounts_fuse_vfs: false,
1409 is_available: || true,
1410 construct: never_available,
1411 };
1412 let regs = vec![&A, &B];
1413 let winner = select_registration("auto", ®s, |_| true).unwrap();
1414
1415 let cands: Vec<BackendCandidate> = regs
1416 .iter()
1417 .map(|r| BackendCandidate::from_registration(r))
1418 .collect();
1419 let report = explain_over(&cands);
1420 let explained = report
1421 .candidates
1422 .iter()
1423 .find(|o| o.selected)
1424 .map(|o| o.candidate.name)
1425 .unwrap();
1426 assert_eq!(
1427 explained, winner.name,
1428 "explain winner must match select_registration"
1429 );
1430 }
1431
1432 #[test]
1433 fn backend_candidate_capability_labels_emit_runtime_truth() {
1434 // The capability-source seam: a backend contributes `runtime=<name>` +
1435 // availability so a node record can aggregate it (#628, no drift).
1436 let c = candidate("kata", 100, true, false);
1437 let labels = c.capability_labels();
1438 assert!(labels.contains(&("runtime".to_owned(), "kata".to_owned())));
1439 assert!(labels.contains(&("runtime.kata.available".to_owned(), "false".to_owned())));
1440 assert!(c.capability_resources().is_empty());
1441 }
1442}