Skip to main content

hyprstream_workers/runtime/
selection.rs

1//! Inventory-based sandbox backend registry + fail-closed selection (#507).
2//!
3//! This is the *spine* of the unified sandbox taxonomy: one [`SandboxBackend`]
4//! seam, one selection model shared by every call site (no per-site ad-hoc
5//! `match` with a silent `_ => nspawn` catch-all).
6//!
7//! Backends are **not** a hardcoded enum. Each concrete backend self-registers a
8//! [`BackendRegistration`] via `inventory::submit!` next to its implementation,
9//! feature-gated by the `#[cfg]` that compiles that backend in (#518). The
10//! registry is therefore *exactly* the set of backends built into this binary:
11//!
12//! * `nspawn` — registered only under the `nspawn` feature (systemd-nspawn
13//!   lightweight container).
14//! * `kata`   — registered only under the `kata` feature (full VM isolation;
15//!   `kata-vm` is a backward-compat alias for `kata`).
16//! * `oci`    — registered only under the `oci` feature (rootless OCI/podman
17//!   container, #346).
18//! * `wasm`   — registered only under the `wasm` feature (in-process WebAssembly,
19//!   explicit-name-only).
20//! * `cri`    — registered only under the `cri` feature (#510: a gRPC *client*
21//!   of an external, already-running CRI runtime service — containerd's CRI
22//!   plugin, or CRI-O — not an embedded runtime). Explicit-name-only, like
23//!   `wasm`: driving an external runtime is an auth-surface decision `"auto"`
24//!   must never reach for on its own.
25//!
26//! YAGNI: still-speculative tiers are intentionally **not** registered. They
27//! gain a `submit!` when their phase actually builds them; until then they
28//! simply do not exist as selectable names (an explicit request errors).
29//!
30//! Selection is **config-driven** via `worker.backend` (a string: a registered
31//! backend name, or `"auto"`):
32//!
33//! * `"auto"` → among registrations that are
34//!   [`auto_selectable`](BackendRegistration::auto_selectable), pick the
35//!   highest-[`priority`](BackendRegistration::priority) one whose
36//!   [`is_available`](BackendRegistration::is_available) probe passes. The choice
37//!   is logged. If none qualifies, error — never run a workload without isolation,
38//!   and never auto-pick a backend that opted out of auto-selection (a silent
39//!   isolation downgrade; see the in-process `wasm` tier, #547 ZSP).
40//! * a concrete name → that registration, **iff** it is registered *and* its
41//!   prerequisites are present. Otherwise error. We never substitute a different
42//!   (weaker) backend (the #486 fail-open bug). An explicit name resolves
43//!   regardless of `auto_selectable`, so an auto-excluded backend (e.g. `wasm`)
44//!   is still reachable when deliberately requested by name.
45//!
46//! The cardinal rule is **fail-closed**: an unavailable or unknown backend
47//! returns an error, never a silent downgrade to weaker isolation.
48
49use std::sync::Arc;
50
51use hyprstream_discovery::scheduling::{
52    self, CapabilitySource, Predicate, RejectionReason, Resource, SelectionReport,
53};
54
55use crate::config::PoolConfig;
56use crate::error::{Result, WorkerError};
57
58#[cfg(feature = "oci-image")]
59use crate::config::ImageConfig;
60#[cfg(feature = "oci-image")]
61use crate::image::RafsStore;
62
63use super::SandboxBackend;
64
65/// Per-call construction context threaded to a backend's `construct` fn.
66///
67/// `inventory` items are `'static`, so [`BackendRegistration::construct`] must be
68/// a bare `fn` pointer that cannot capture per-call state. This struct carries
69/// everything a backend constructor needs at selection time.
70pub struct BackendCtx {
71    /// Sandbox pool configuration (paths, hypervisor, limits).
72    pub pool_config: PoolConfig,
73
74    /// Mandatory resolver-backed, audited policy-enforcement point for every
75    /// 9P export constructed by a backend.
76    #[cfg(not(target_arch = "wasm32"))]
77    pub ninep_decider: Arc<dyn hyprstream_9p::AccessDecider>,
78
79    /// #1269: full MAC reference monitor for 9P exports. `Some` at every
80    /// production constructor — the `anonymous_floor()` fallback is not
81    /// reachable. `None` in tests only.
82    #[cfg(not(target_arch = "wasm32"))]
83    pub ninep_monitor: Option<Arc<hyprstream_9p::ReferenceMonitor>>,
84
85    /// Image storage configuration. Any backend that composes the RAFS image
86    /// filesystem (kata over virtio-fs, or nspawn over FUSE — Model B #715)
87    /// consumes it, so it exists on the `oci-image` build, not just `kata-vm`.
88    /// The Cargo manifest is explicit that the RAFS image service "lives on
89    /// oci-image, not kata-vm".
90    #[cfg(feature = "oci-image")]
91    pub image_config: ImageConfig,
92
93    /// RAFS/nydus image store — available whenever the image filesystem service
94    /// is compiled in (`oci-image`), so both the kata (virtio-fs) and nspawn
95    /// (FUSE, #715) backends can compose a per-sandbox VFS from it.
96    #[cfg(feature = "oci-image")]
97    pub rafs_store: Arc<RafsStore>,
98}
99
100/// A compile-time sandbox-backend registration, collected via `inventory`.
101///
102/// Each backend submits one of these next to its implementation, gated by the
103/// `#[cfg]` feature that compiles the backend in. The registry is enumerated by
104/// [`resolve_backend`]; there is no central enum to keep in sync.
105#[derive(Debug)]
106pub struct BackendRegistration {
107    /// Stable selector name, e.g. `"kata"`, `"nspawn"`. Matched against
108    /// `worker.backend` (case-insensitively).
109    pub name: &'static str,
110
111    /// Auto-selection precedence: higher is preferred. `"auto"` picks the
112    /// highest-priority *available* registration (among `auto_selectable` ones).
113    pub priority: i32,
114
115    /// Whether this backend is eligible for `"auto"` selection.
116    ///
117    /// `true` → `"auto"` may pick it (subject to `priority` and `is_available`).
118    /// `false` → **explicit-name-only**: `"auto"` must never choose it, even if it
119    /// is the highest-priority available (or only) registration. It remains
120    /// selectable by its concrete name (fail-closed).
121    ///
122    /// This exists so weaker-isolation tiers cannot become a silent `"auto"`
123    /// fallback when stronger backends are absent — auto-selecting them would be a
124    /// silent isolation downgrade, which the #547 MAC/ZSP model forbids. The
125    /// in-process `wasm` sandbox (shared host address space) sets this `false`.
126    pub auto_selectable: bool,
127
128    /// Whether this backend can **inject a 9P socket endpoint** — a host Unix
129    /// domain socket — into a workload's mount namespace (#506).
130    ///
131    /// The Wanix-guest workload (`runtime::wanix_workload`) works by having
132    /// hyprstream serve a tenant's Subject-scoped VFS `Mount` as a 9P2000.L
133    /// server on a host UDS, then making that socket reachable *inside* the
134    /// sandbox (bind-mounting the host socket into the container's mount
135    /// namespace and setting `HYPRSTREAM_9P_SOCK`) so the guest dials back. That
136    /// requires the backend to be able to bind-mount an arbitrary host path (the
137    /// socket) into the workload, which the tiers do differently:
138    ///
139    /// * `oci` / `nspawn` → `true`: both bind-mount host paths into the
140    ///   container (`podman --volume` / `nspawn --bind`), so a host UDS injects
141    ///   cleanly.
142    /// * `kata` → `false`: a full VM does not share the host mount namespace, so
143    ///   a host UDS cannot be bind-injected the same way (the VM path uses
144    ///   virtio-fs, a different mechanism — see `sandbox_fs`); it therefore does
145    ///   **not** advertise this capability.
146    /// * `wasm` → `false`: in-process, no separate mount namespace to inject a
147    ///   host socket into.
148    ///
149    /// Selection is **fail-closed** on this flag: a workload that requires 9P
150    /// socket injection ([`resolve_backend_9p_capable`] /
151    /// [`require_9p_socket_capability`]) never resolves to a backend that does
152    /// not advertise it — it errors rather than silently dropping the injection.
153    pub injects_9p_socket: bool,
154    /// Whether this backend **FUSE-mounts the composed per-sandbox tenant VFS**
155    /// as a real host directory and POSIX-roots its workload there (Model B,
156    /// #715 — the non-VM sibling of the kata virtio-fs path).
157    ///
158    /// `true` → the backend consumes `SandboxFs::serve_local_at` to serve the
159    /// composed namespace over `/dev/fuse` and give that host directory to the
160    /// container. `nspawn` sets this. `kata` does **not** — it already shares the
161    /// composed VFS as a virtio-fs device, not a host FUSE mount. In-process
162    /// `wasm` has no host filesystem to mount into. `oci` is deferred pending a
163    /// rootless `/dev/fuse` feasibility spike and advertises `false` for now.
164    ///
165    /// [`require_fuse_mount_capability`] gates the mount fail-closed: a backend
166    /// that does not advertise this capability is never handed a FUSE-rooted
167    /// namespace (no silent POSIX-rooting in a namespace a backend can't mount).
168    pub mounts_fuse_vfs: bool,
169
170    /// Runtime prerequisite probe (PATH lookups, socket existence, …). Returns
171    /// `true` when this backend can actually run on this host *right now*. Used
172    /// to inform `"auto"` and to fail-close explicit requests whose prereqs are
173    /// missing.
174    pub is_available: fn() -> bool,
175
176    /// Construct the concrete backend from per-call [`BackendCtx`] state.
177    pub construct: fn(&BackendCtx) -> anyhow::Result<Arc<dyn SandboxBackend>>,
178}
179
180inventory::collect!(BackendRegistration);
181
182/// Comma-separated registered backend names, for error messages.
183fn registered_names(regs: &[&BackendRegistration]) -> String {
184    let mut names: Vec<&str> = regs.iter().map(|r| r.name).collect();
185    names.sort_unstable();
186    names.join(", ")
187}
188
189/// Pure selection over a registration set, with an injectable availability
190/// oracle. Factored out of [`resolve_backend`] so the fail-closed / auto-priority
191/// / unknown-name logic is unit-testable without depending on which runtimes
192/// happen to be installed on the test host.
193///
194/// Capability-agnostic; delegates to [`select_registration_with_cap`] with no
195/// required capability.
196fn select_registration<'a>(
197    name: &str,
198    regs: &[&'a BackendRegistration],
199    is_available: impl Fn(&BackendRegistration) -> bool,
200) -> Result<&'a BackendRegistration> {
201    select_registration_with_cap(name, regs, is_available, false)
202}
203
204/// Pure selection with an additional **required-capability** gate for 9P socket
205/// injection (#506), fail-closed.
206///
207/// When `require_9p_socket` is `true`, the chosen registration must advertise
208/// [`BackendRegistration::injects_9p_socket`]:
209///
210/// * `"auto"` → only backends that are *both* `auto_selectable` **and**
211///   `injects_9p_socket` are candidates; the highest-priority available one
212///   wins. If none qualifies, error — never fall back to a backend that cannot
213///   inject the socket (that would silently drop the tenant's namespace export).
214/// * a concrete name → resolves the named backend, then rejects it if it does
215///   not advertise the capability. We never substitute a different (capable)
216///   backend for an explicitly-named incapable one — the explicit request is
217///   authoritative, and the correct answer is a clean error, not a swap.
218///
219/// With `require_9p_socket == false` this is exactly the prior behaviour.
220fn select_registration_with_cap<'a>(
221    name: &str,
222    regs: &[&'a BackendRegistration],
223    is_available: impl Fn(&BackendRegistration) -> bool,
224    require_9p_socket: bool,
225) -> Result<&'a BackendRegistration> {
226    // ── Auto: highest-priority *available* registration wins ──
227    //
228    // Only `auto_selectable` registrations are eligible. A backend that opts out
229    // (e.g. the in-process `wasm` tier) must never be auto-picked, even as a last
230    // resort when nothing stronger is available — that would be a silent isolation
231    // downgrade (#547 ZSP). Such backends remain reachable by explicit name below.
232    if name.eq_ignore_ascii_case("auto") {
233        let mut candidates: Vec<&'a BackendRegistration> = regs
234            .iter()
235            .copied()
236            .filter(|r| r.auto_selectable)
237            // Capability gate: when a 9P-socket-injecting workload is being
238            // placed, an incapable backend is not even a candidate — auto must
239            // never downgrade to one that would silently drop the injection.
240            .filter(|r| !require_9p_socket || r.injects_9p_socket)
241            .collect();
242        // Highest priority first; ties broken by name for determinism.
243        candidates.sort_by(|a, b| b.priority.cmp(&a.priority).then_with(|| a.name.cmp(b.name)));
244        for reg in candidates {
245            if is_available(reg) {
246                return Ok(reg);
247            }
248        }
249        let cap_note = if require_9p_socket {
250            " that can inject a 9P socket endpoint (host UDS bind-mount)"
251        } else {
252            ""
253        };
254        return Err(WorkerError::ConfigError(format!(
255            "auto backend selection found no available sandbox backend{cap_note} among [{}]. \
256             Install a supported runtime or set `worker.backend` to an explicit \
257             backend; refusing to run a workload without isolation (fail-closed).",
258            registered_names(regs)
259        )));
260    }
261
262    // ── Explicit request: authoritative, fail-closed ──
263    match regs.iter().find(|r| r.name.eq_ignore_ascii_case(name)) {
264        // Named + available, but cannot inject the required 9P socket → reject.
265        // We do NOT substitute a capable backend for an explicitly-named one.
266        Some(reg) if require_9p_socket && !reg.injects_9p_socket => {
267            Err(WorkerError::ConfigError(format!(
268                "sandbox backend '{}' was requested for a workload that requires 9P \
269                 socket injection (a host Unix socket bind-mounted into the sandbox), \
270                 but '{}' cannot inject a host UDS into a workload's mount namespace. \
271                 Choose a backend that can (e.g. oci, nspawn); refusing to silently \
272                 drop the namespace export (fail-closed).",
273                reg.name, reg.name
274            )))
275        }
276        Some(reg) if is_available(reg) => Ok(reg),
277        Some(reg) => Err(WorkerError::ConfigError(format!(
278            "sandbox backend '{}' was requested but its runtime prerequisites are \
279             missing. Refusing to silently downgrade isolation to a weaker backend \
280             (fail-closed).",
281            reg.name
282        ))),
283        None => {
284            let mut msg = format!(
285                "unknown sandbox backend '{name}'; registered backends are [{}] (or \"auto\")",
286                registered_names(regs)
287            );
288            // Helpful hint for the most common misconfiguration: requesting the
289            // VM backend in a binary built without it.
290            if name.eq_ignore_ascii_case("kata") {
291                msg.push_str(
292                    ". The `kata` backend is only present when built with \
293                     `--features kata` (or the `kata-vm` alias)",
294                );
295            }
296            Err(WorkerError::ConfigError(msg))
297        }
298    }
299}
300
301/// Resolve `worker.backend` to a constructed [`SandboxBackend`], fail-closed.
302///
303/// `name` is either a registered backend name or `"auto"`. This is the single
304/// function both `factories.rs` and `bin/main.rs` route through — no scattered
305/// `#[cfg]`, no `_ => nspawn` fallback.
306///
307/// * **`"auto"`** → among [`auto_selectable`](BackendRegistration::auto_selectable)
308///   registrations, the highest-[`priority`](BackendRegistration::priority) one
309///   whose [`is_available`](BackendRegistration::is_available) is true; error if
310///   none (auto-excluded backends are never picked here — no silent downgrade).
311/// * **concrete name** → that registration if present *and* available, regardless
312///   of `auto_selectable`; otherwise error (unavailable → fail-closed; unknown →
313///   error listing the registry).
314pub fn resolve_backend(name: &str, ctx: &BackendCtx) -> Result<Arc<dyn SandboxBackend>> {
315    let regs: Vec<&'static BackendRegistration> =
316        inventory::iter::<BackendRegistration>().collect();
317
318    let reg = select_registration(name, &regs, |r| (r.is_available)())?;
319
320    tracing::info!(
321        backend = reg.name,
322        requested = name,
323        priority = reg.priority,
324        "sandbox backend selected (fail-closed)"
325    );
326
327    (reg.construct)(ctx).map_err(|e| {
328        WorkerError::ConfigError(format!(
329            "failed to construct sandbox backend '{}': {e:#}",
330            reg.name
331        ))
332    })
333}
334
335/// Resolve `worker.backend` to a [`SandboxBackend`] that is **guaranteed capable
336/// of 9P socket injection** (#506), fail-closed.
337///
338/// Identical to [`resolve_backend`] but with the capability gate engaged: the
339/// resolved backend always advertises
340/// [`injects_9p_socket`](BackendRegistration::injects_9p_socket). Use this when
341/// placing a workload (e.g. the Wanix guest) that needs a host UDS injected into
342/// its mount namespace. `"auto"` picks the highest-priority *capable* backend;
343/// an explicit name that is incapable errors rather than being swapped for a
344/// capable one.
345pub fn resolve_backend_9p_capable(name: &str, ctx: &BackendCtx) -> Result<Arc<dyn SandboxBackend>> {
346    let regs: Vec<&'static BackendRegistration> =
347        inventory::iter::<BackendRegistration>().collect();
348
349    let reg = select_registration_with_cap(name, &regs, |r| (r.is_available)(), true)?;
350
351    tracing::info!(
352        backend = reg.name,
353        requested = name,
354        priority = reg.priority,
355        "sandbox backend selected for 9P-socket-injecting workload (fail-closed)"
356    );
357
358    (reg.construct)(ctx).map_err(|e| {
359        WorkerError::ConfigError(format!(
360            "failed to construct sandbox backend '{}': {e:#}",
361            reg.name
362        ))
363    })
364}
365
366/// Does the registered backend named `backend_type` advertise 9P socket
367/// injection? Returns `false` for an unknown/unregistered name.
368///
369/// This queries the *registry* (not a live backend instance), so a caller
370/// holding an already-constructed `Arc<dyn SandboxBackend>` can check its
371/// capability via [`SandboxBackend::backend_type`](super::SandboxBackend::backend_type).
372pub fn backend_injects_9p_socket(backend_type: &str) -> bool {
373    inventory::iter::<BackendRegistration>()
374        .find(|r| r.name.eq_ignore_ascii_case(backend_type))
375        .is_some_and(|r| r.injects_9p_socket)
376}
377
378/// Fail-closed guard used at the workload seam: error unless `backend_type`
379/// advertises 9P socket injection (#506).
380///
381/// A caller that already holds a constructed backend (e.g. from the pool) uses
382/// this to refuse — cleanly, before spawning any 9P server or building any
383/// injection annotations — to place a socket-injecting workload on a backend
384/// that cannot receive the socket. It never downgrades; it only reports.
385pub fn require_9p_socket_capability(backend_type: &str) -> Result<()> {
386    if backend_injects_9p_socket(backend_type) {
387        Ok(())
388    } else {
389        Err(WorkerError::ConfigError(format!(
390            "sandbox backend '{backend_type}' cannot inject a 9P socket endpoint \
391             (a host Unix socket bind-mounted into the sandbox), which this workload \
392             requires. Select a capable backend (e.g. oci, nspawn); refusing to run \
393             the workload without its namespace export (fail-closed)."
394        )))
395    }
396}
397
398/// Pure capability check over a registration set, with the same fail-closed
399/// contract as [`select_registration`]. Factored out so the Model B (#715)
400/// FUSE-mount gate is unit-testable without depending on which backends the
401/// build's inventory happens to contain.
402fn check_fuse_mount_capability(name: &str, regs: &[&BackendRegistration]) -> Result<()> {
403    match regs.iter().find(|r| r.name.eq_ignore_ascii_case(name)) {
404        Some(reg) if reg.mounts_fuse_vfs => Ok(()),
405        Some(reg) => Err(WorkerError::ConfigError(format!(
406            "sandbox backend '{}' does not advertise the FUSE tenant-VFS mount \
407             capability (mounts_fuse_vfs); refusing to POSIX-root a workload in a \
408             namespace this backend was not declared able to mount (fail-closed).",
409            reg.name
410        ))),
411        None => Err(WorkerError::ConfigError(format!(
412            "unknown sandbox backend '{name}'; cannot check FUSE tenant-VFS mount \
413             capability"
414        ))),
415    }
416}
417
418/// Fail-closed gate for Model B (#715): assert `backend_type` advertises the
419/// [`mounts_fuse_vfs`](BackendRegistration::mounts_fuse_vfs) capability before a
420/// caller FUSE-mounts the composed tenant VFS as that backend's root.
421///
422/// Parallels [`resolve_backend`]'s fail-closed contract: an unknown backend, or
423/// a registered one that does not advertise the capability, errors rather than
424/// proceeding — we never POSIX-root a workload in a namespace a backend was not
425/// declared able to mount, and never silently substitute a different transport.
426pub fn require_fuse_mount_capability(backend_type: &str) -> Result<()> {
427    let regs: Vec<&'static BackendRegistration> =
428        inventory::iter::<BackendRegistration>().collect();
429    check_fuse_mount_capability(backend_type, &regs)
430}
431
432// ─────────────────────────────────────────────────────────────────────────────
433// Selection diagnostics (#348)
434// ─────────────────────────────────────────────────────────────────────────────
435//
436// `resolve_backend` answers "which backend do I get"; the function below adds
437// the backend-local "why not" knowledge the #628 scheduling substrate's explain
438// trace needs — a per-backend prerequisite sub-reason. This does not change
439// selection semantics; it is read-only introspection over the same registry.
440//
441// The generic filter→rank→select explain trace (`SelectionReport<C>`) is owned
442// by the #628 scheduling substrate (`hyprstream-discovery::scheduling`);
443// `explain_selection` below composes on it (see the "Scheduling-substrate
444// adoption" section) rather than carrying a second, backend-specific copy.
445
446/// Backend-specific sub-reason for why `is_available()` returned `false`,
447/// e.g. "cloud-hypervisor not on PATH" for `kata`. Falls back to a generic
448/// message for backends this function doesn't have specific knowledge of.
449///
450/// This mirrors the prerequisite probes in each backend's
451/// `registry_is_available()` (see `kata_backend.rs`, `nspawn.rs`,
452/// `wasm_backend.rs`) but is allowed to drift from them in detail since it is
453/// diagnostic-only — it never gates selection itself.
454fn unavailable_reason(name: &str) -> String {
455    match name {
456        "kata" => "runtime prerequisites missing: `cloud-hypervisor` not found on PATH".to_owned(),
457        "nspawn" => {
458            "runtime prerequisites missing: `systemd-nspawn` and/or `machinectl` not found on PATH"
459                .to_owned()
460        }
461        "podman" => "runtime prerequisites missing: `podman` not found on PATH".to_owned(),
462        "wasm" => "wasm runtime prerequisites are not satisfied".to_owned(),
463        other => format!("runtime prerequisites for '{other}' are not satisfied"),
464    }
465}
466
467// ─────────────────────────────────────────────────────────────────────────────
468// Scheduling-substrate adoption (#628)
469// ─────────────────────────────────────────────────────────────────────────────
470//
471// The explain surface #348 needs is the #628 substrate's generic
472// `filter → rank → select → SelectionReport<C>`. Backend selection is one
473// consumer of that shape: candidates are the compiled-in registrations,
474// availability is a `Predicate` whose `RejectionReason` is sourced from
475// `unavailable_reason`, and rank is by descending priority. `explain_selection`
476// runs the substrate pipeline and returns its `SelectionReport`, so the trace
477// falls out of `select` for free — no bespoke report type here.
478//
479// `resolve_backend`/`select_registration` above remain the authoritative
480// construction path (they encode explicit-name and unknown-name semantics that
481// are not a single generic `select` call); the substrate governs the "auto"
482// explain trace, keeping one explain shape across every scheduling surface.
483//
484// Full resource-aware ranking (GPU/memory/NUMA via `ResourceRequest`) plugs in
485// here as additional predicates + a resource-aware `rank_by`; that is downstream
486// consumer work (#341/#525). This adoption wires availability-as-predicate +
487// explain + the capability-source seam.
488
489/// A cloneable projection of a [`BackendRegistration`] for the substrate's
490/// generic pipeline (`SelectionReport<C>` requires `C: Clone`; a registration
491/// holds `fn` pointers and is not cloneable). Carries exactly what selection
492/// filtering, ranking, and the explain trace need.
493#[derive(Debug, Clone, PartialEq, Eq)]
494pub struct BackendCandidate {
495    /// Registered backend name (`"kata"`, `"nspawn"`, …).
496    pub name: &'static str,
497    /// Auto-selection precedence (higher preferred).
498    pub priority: i32,
499    /// Whether `"auto"` may pick this backend.
500    pub auto_selectable: bool,
501    /// Live availability probe result at the time the candidate was captured.
502    pub available: bool,
503}
504
505impl BackendCandidate {
506    fn from_registration(reg: &BackendRegistration) -> Self {
507        Self {
508            name: reg.name,
509            priority: reg.priority,
510            auto_selectable: reg.auto_selectable,
511            available: (reg.is_available)(),
512        }
513    }
514}
515
516/// Runtime availability + capability truth a backend contributes to the shared
517/// scheduling vocabulary. A node-record publisher aggregates these across every
518/// registered backend so a placement record's `runtime=<name>` labels reflect
519/// what is actually available here (no hand-declaration, no drift — #628).
520impl CapabilitySource for BackendCandidate {
521    fn capability_labels(&self) -> Vec<(String, String)> {
522        vec![
523            ("runtime".to_owned(), self.name.to_owned()),
524            (
525                format!("runtime.{}.available", self.name),
526                self.available.to_string(),
527            ),
528        ]
529    }
530
531    fn capability_resources(&self) -> Vec<Resource> {
532        // Backend selection contributes runtime-availability labels, not
533        // countable capacity; GPU/memory/NUMA capacity comes from other sources
534        // (DevicePool, node liveness). See the module note on resource-aware rank.
535        Vec::new()
536    }
537}
538
539/// Availability predicate: rejects a candidate whose runtime prerequisites are
540/// missing, carrying the backend-specific [`unavailable_reason`] as the
541/// [`RejectionReason`] so it lands in the [`SelectionReport`] trace.
542fn availability_predicate() -> Predicate<BackendCandidate> {
543    Box::new(|c: &BackendCandidate| {
544        if c.available {
545            None
546        } else {
547            Some(RejectionReason(unavailable_reason(c.name)))
548        }
549    })
550}
551
552/// Auto-eligibility predicate: rejects backends excluded from `"auto"` (the
553/// isolation-downgrade guard, #547 ZSP), with the reason recorded in the trace.
554fn auto_eligibility_predicate() -> Predicate<BackendCandidate> {
555    Box::new(|c: &BackendCandidate| {
556        if c.auto_selectable {
557            None
558        } else {
559            Some(RejectionReason(format!(
560                "backend '{}' is explicit-name-only (excluded from auto-selection; \
561                 no silent isolation downgrade)",
562                c.name
563            )))
564        }
565    })
566}
567
568/// Explain what `"auto"` selection would decide over the compiled-in registry,
569/// as the shared [`SelectionReport`]. The winning candidate is the highest-
570/// priority backend that is both auto-eligible and available; every other
571/// candidate carries the [`RejectionReason`] that excluded it.
572///
573/// This is diagnostics only (a wizard/CLI "why this backend?" trace); the
574/// authoritative construction path is [`resolve_backend`]. Both agree on the
575/// same winner for `"auto"` because both rank by descending priority over the
576/// auto-eligible, available set.
577pub fn explain_selection() -> SelectionReport<BackendCandidate> {
578    let candidates: Vec<BackendCandidate> = inventory::iter::<BackendRegistration>()
579        .map(BackendCandidate::from_registration)
580        .collect();
581
582    let predicates = vec![auto_eligibility_predicate(), availability_predicate()];
583
584    let mut report = scheduling::select(
585        &candidates,
586        &predicates,
587        // Highest priority first; name ascending as a deterministic tiebreak —
588        // matching `select_registration`'s "auto" ordering exactly.
589        |a, b| b.priority.cmp(&a.priority).then_with(|| a.name.cmp(b.name)),
590        1, // "auto" selects a single backend
591    );
592    report.requested = "auto".to_owned();
593    report
594}
595
596#[cfg(test)]
597#[allow(clippy::unwrap_used, clippy::expect_used)]
598mod tests {
599    use super::*;
600
601    // ── Synthetic registrations for deterministic selection-logic tests ──
602    //
603    // These never touch the real `is_available` probes or `construct` fns; they
604    // exercise the pure `select_registration` spine (auto-priority, fail-closed,
605    // unknown-name) independent of what runtimes are installed.
606
607    fn never_available(_: &BackendCtx) -> anyhow::Result<Arc<dyn SandboxBackend>> {
608        unreachable!("construct is not called in selection-logic tests")
609    }
610
611    const HIGH: BackendRegistration = BackendRegistration {
612        name: "high-tier",
613        priority: 100,
614        auto_selectable: true,
615        injects_9p_socket: false,
616        mounts_fuse_vfs: false,
617        is_available: || true,
618        construct: never_available,
619    };
620    const LOW: BackendRegistration = BackendRegistration {
621        name: "low-tier",
622        priority: 10,
623        auto_selectable: true,
624        injects_9p_socket: false,
625        mounts_fuse_vfs: false,
626        is_available: || true,
627        construct: never_available,
628    };
629
630    /// A FUSE-mount-capable backend, modeling the Model B (#715) nspawn target:
631    /// it advertises `mounts_fuse_vfs: true`.
632    const FUSE_CAP: BackendRegistration = BackendRegistration {
633        name: "fuse-tier",
634        priority: 50,
635        auto_selectable: true,
636        injects_9p_socket: false,
637        mounts_fuse_vfs: true,
638        is_available: || true,
639        construct: never_available,
640    };
641
642    /// An auto-excluded (explicit-name-only) backend, modeling the in-process
643    /// `wasm` tier: even at the highest priority and always available, `"auto"`
644    /// must never pick it.
645    const EXPLICIT_ONLY: BackendRegistration = BackendRegistration {
646        name: "explicit-only",
647        priority: 1000,
648        auto_selectable: false,
649        injects_9p_socket: false,
650        mounts_fuse_vfs: false,
651        is_available: || true,
652        construct: never_available,
653    };
654
655    /// A high-priority backend that CAN inject a 9P socket (models `oci`).
656    const NINEP_HIGH: BackendRegistration = BackendRegistration {
657        name: "ninep-high",
658        priority: 100,
659        auto_selectable: true,
660        injects_9p_socket: true,
661        mounts_fuse_vfs: false,
662        is_available: || true,
663        construct: never_available,
664    };
665    /// A lower-priority backend that CAN inject a 9P socket (models `nspawn`).
666    const NINEP_LOW: BackendRegistration = BackendRegistration {
667        name: "ninep-low",
668        priority: 10,
669        auto_selectable: true,
670        injects_9p_socket: true,
671        mounts_fuse_vfs: false,
672        is_available: || true,
673        construct: never_available,
674    };
675
676    fn regs() -> Vec<&'static BackendRegistration> {
677        vec![&HIGH, &LOW]
678    }
679
680    #[test]
681    fn auto_picks_highest_priority_available() {
682        // Both available → strongest (highest priority) wins.
683        let r = select_registration("auto", &regs(), |_| true).unwrap();
684        assert_eq!(r.name, "high-tier");
685    }
686
687    #[test]
688    fn auto_skips_unavailable_to_next_priority() {
689        // Strongest unavailable → auto must fall to the next available tier,
690        // never error while a usable backend remains.
691        let r = select_registration("auto", &regs(), |reg| reg.name != "high-tier").unwrap();
692        assert_eq!(r.name, "low-tier");
693    }
694
695    #[test]
696    fn auto_errors_when_nothing_available() {
697        let err = select_registration("auto", &regs(), |_| false).unwrap_err();
698        let msg = err.to_string();
699        assert!(msg.contains("no available sandbox backend"), "got: {msg}");
700        assert!(msg.contains("fail-closed"), "got: {msg}");
701    }
702
703    #[test]
704    fn auto_never_picks_explicit_only_even_as_only_backend() {
705        // An auto-excluded backend that is the *only* registration (and available)
706        // must NOT be auto-selected — `"auto"` errors instead of downgrading.
707        let only_explicit = vec![&EXPLICIT_ONLY];
708        let err = select_registration("auto", &only_explicit, |_| true).unwrap_err();
709        let msg = err.to_string();
710        assert!(msg.contains("no available sandbox backend"), "got: {msg}");
711        assert!(msg.contains("fail-closed"), "got: {msg}");
712    }
713
714    #[test]
715    fn auto_skips_explicit_only_for_a_weaker_auto_eligible_backend() {
716        // Even though EXPLICIT_ONLY has the highest priority and is available,
717        // `"auto"` must prefer a lower-priority *auto-eligible* backend over it —
718        // never an isolation downgrade to the auto-excluded tier.
719        let regs = vec![&EXPLICIT_ONLY, &LOW];
720        let r = select_registration("auto", &regs, |_| true).unwrap();
721        assert_eq!(
722            r.name, "low-tier",
723            "auto must skip the auto-excluded backend"
724        );
725    }
726
727    #[test]
728    fn explicit_only_still_selectable_by_name() {
729        // Auto-exclusion does not remove the backend from the registry: an
730        // explicit request resolves it (fail-closed when unavailable).
731        let regs = vec![&EXPLICIT_ONLY, &LOW];
732        let r = select_registration("explicit-only", &regs, |_| true).unwrap();
733        assert_eq!(r.name, "explicit-only");
734    }
735
736    #[test]
737    fn explicit_available_resolves_exactly() {
738        let r = select_registration("low-tier", &regs(), |_| true).unwrap();
739        assert_eq!(r.name, "low-tier");
740        // Case-insensitive.
741        let r = select_registration("LOW-TIER", &regs(), |_| true).unwrap();
742        assert_eq!(r.name, "low-tier");
743    }
744
745    #[test]
746    fn explicit_unavailable_fails_closed_no_downgrade() {
747        // "high-tier" requested but unavailable → error, must NOT downgrade to
748        // the available "low-tier" backend.
749        let err =
750            select_registration("high-tier", &regs(), |reg| reg.name == "low-tier").unwrap_err();
751        let msg = err.to_string();
752        assert!(msg.contains("prerequisites are missing"), "got: {msg}");
753        assert!(msg.contains("fail-closed"), "got: {msg}");
754        // Must not name the weaker backend it could have silently fallen back to.
755        assert!(
756            !msg.contains("low-tier"),
757            "must not mention a downgrade target: {msg}"
758        );
759    }
760
761    #[test]
762    fn unknown_name_errors_listing_registry() {
763        let err = select_registration("bogus", &regs(), |_| true).unwrap_err();
764        let msg = err.to_string();
765        assert!(
766            msg.contains("unknown sandbox backend 'bogus'"),
767            "got: {msg}"
768        );
769        assert!(
770            msg.contains("high-tier"),
771            "should list registered names: {msg}"
772        );
773        assert!(
774            msg.contains("low-tier"),
775            "should list registered names: {msg}"
776        );
777    }
778
779    // ── 9P-socket-injection capability gate (#506), fail-closed ──
780
781    #[test]
782    fn cap_auto_picks_highest_priority_capable_backend() {
783        // Both capable → strongest (highest priority) wins.
784        let regs = vec![&NINEP_HIGH, &NINEP_LOW];
785        let r = select_registration_with_cap("auto", &regs, |_| true, true).unwrap();
786        assert_eq!(r.name, "ninep-high");
787    }
788
789    #[test]
790    fn cap_auto_skips_incapable_backend_even_if_higher_priority() {
791        // HIGH (priority 100) cannot inject; NINEP_LOW (priority 10) can. With
792        // the capability required, auto MUST pick the lower-priority *capable*
793        // backend, never the higher-priority incapable one.
794        let regs = vec![&HIGH, &NINEP_LOW];
795        let r = select_registration_with_cap("auto", &regs, |_| true, true).unwrap();
796        assert_eq!(r.name, "ninep-low", "auto must pick the capable backend");
797    }
798
799    #[test]
800    fn cap_auto_errors_when_no_capable_backend_available() {
801        // Only incapable backends present → auto errors rather than downgrading
802        // to one that would silently drop the 9P injection.
803        let regs = vec![&HIGH, &LOW];
804        let err = select_registration_with_cap("auto", &regs, |_| true, true).unwrap_err();
805        let msg = err.to_string();
806        assert!(msg.contains("9P socket endpoint"), "got: {msg}");
807        assert!(msg.contains("fail-closed"), "got: {msg}");
808    }
809
810    #[test]
811    fn cap_auto_errors_when_capable_backend_unavailable() {
812        // The only capable backend is unavailable → auto fails closed (no
813        // fallback to an available-but-incapable backend).
814        let regs = vec![&NINEP_HIGH, &LOW];
815        let err = select_registration_with_cap("auto", &regs, |r| r.name == "low-tier", true)
816            .unwrap_err();
817        assert!(err.to_string().contains("9P socket endpoint"), "got: {err}");
818    }
819
820    #[test]
821    fn cap_explicit_capable_resolves() {
822        let regs = vec![&NINEP_HIGH, &LOW];
823        let r = select_registration_with_cap("ninep-high", &regs, |_| true, true).unwrap();
824        assert_eq!(r.name, "ninep-high");
825    }
826
827    #[test]
828    fn cap_explicit_incapable_errors_no_substitution() {
829        // Explicitly asking for an incapable backend when the capability is
830        // required errors — and must NOT silently swap in a capable one.
831        let regs = vec![&HIGH, &NINEP_LOW];
832        let err = select_registration_with_cap("high-tier", &regs, |_| true, true).unwrap_err();
833        let msg = err.to_string();
834        assert!(msg.contains("9P socket injection"), "got: {msg}");
835        assert!(msg.contains("fail-closed"), "got: {msg}");
836        // Must not name the capable backend it could have swapped to.
837        assert!(
838            !msg.contains("ninep-low"),
839            "must not suggest a silent swap: {msg}"
840        );
841    }
842
843    #[test]
844    fn cap_off_ignores_incapable_flag() {
845        // With the capability NOT required, an incapable backend resolves fine
846        // (the gate is opt-in; default behaviour is unchanged).
847        let regs = vec![&HIGH, &LOW];
848        let r = select_registration_with_cap("high-tier", &regs, |_| true, false).unwrap();
849        assert_eq!(r.name, "high-tier");
850        let r = select_registration_with_cap("auto", &regs, |_| true, false).unwrap();
851        assert_eq!(r.name, "high-tier");
852    }
853
854    #[test]
855    fn require_capability_helper_is_fail_closed() {
856        // The registry-backed helper the workload seam uses: real registered
857        // backends reflect reality (oci/nspawn capable; kata/wasm not), and an
858        // unknown name is treated as incapable (fail-closed).
859        assert!(!backend_injects_9p_socket("does-not-exist"));
860        assert!(require_9p_socket_capability("does-not-exist").is_err());
861
862        // nspawn is registered under the `nspawn` feature and IS capable (host --bind).
863        #[cfg(feature = "nspawn")]
864        {
865            assert!(
866                backend_injects_9p_socket("nspawn"),
867                "nspawn can bind a host UDS"
868            );
869            assert!(require_9p_socket_capability("nspawn").is_ok());
870            // Case-insensitive, mirroring name matching.
871            assert!(backend_injects_9p_socket("NSPAWN"));
872        }
873    }
874
875    #[cfg(feature = "oci")]
876    #[test]
877    fn oci_advertises_9p_socket_injection() {
878        let oci = inventory::iter::<BackendRegistration>()
879            .find(|r| r.name == "oci")
880            .expect("oci registered under the oci feature");
881        assert!(
882            oci.injects_9p_socket,
883            "oci bind-mounts a host UDS → capable"
884        );
885    }
886
887    #[cfg(feature = "nspawn")]
888    #[test]
889    fn nspawn_advertises_9p_socket_injection() {
890        let nspawn = inventory::iter::<BackendRegistration>()
891            .find(|r| r.name == "nspawn")
892            .expect("nspawn registered under the nspawn feature");
893        assert!(
894            nspawn.injects_9p_socket,
895            "nspawn --bind can inject a host UDS"
896        );
897    }
898
899    #[cfg(feature = "kata")]
900    #[test]
901    fn kata_does_not_advertise_9p_socket_injection() {
902        // A VM does not share the host mount namespace, so it cannot bind-inject
903        // a host UDS the way oci/nspawn do — it must NOT advertise the capability.
904        let kata = inventory::iter::<BackendRegistration>()
905            .find(|r| r.name == "kata")
906            .expect("kata registered under the kata feature");
907        assert!(
908            !kata.injects_9p_socket,
909            "kata (VM, no shared host mount ns) must not advertise host-UDS injection"
910        );
911    }
912
913    #[cfg(feature = "wasm")]
914    #[test]
915    fn wasm_does_not_advertise_9p_socket_injection() {
916        let wasm = inventory::iter::<BackendRegistration>()
917            .find(|r| r.name == "wasm")
918            .expect("wasm registered under the wasm feature");
919        assert!(
920            !wasm.injects_9p_socket,
921            "in-process wasm has no separate mount ns to inject a host socket into"
922        );
923    }
924
925    #[test]
926    fn unknown_kata_hints_about_feature() {
927        // Asking for kata when it isn't registered (e.g. kata off) gives a
928        // build-feature hint rather than a bare unknown error.
929        let err = select_registration("kata", &regs(), |_| true).unwrap_err();
930        assert!(err.to_string().contains("kata"), "got: {err}");
931        assert!(err.to_string().contains("kata-vm"), "got: {err}");
932    }
933
934    // ── The real inventory must reflect the build's feature set ──
935
936    // ── Model B (#715): FUSE tenant-VFS mount capability gate ──
937
938    #[test]
939    fn fuse_capability_gate_allows_advertised_backend() {
940        let regs = vec![&FUSE_CAP, &LOW];
941        assert!(check_fuse_mount_capability("fuse-tier", &regs).is_ok());
942        // Case-insensitive, mirroring backend selection.
943        assert!(check_fuse_mount_capability("FUSE-TIER", &regs).is_ok());
944    }
945
946    #[test]
947    fn fuse_capability_gate_fails_closed_for_non_advertising_backend() {
948        // A registered backend that does NOT advertise the capability must be
949        // refused — no silent POSIX-rooting in a namespace it can't mount.
950        let regs = vec![&FUSE_CAP, &LOW];
951        let err = check_fuse_mount_capability("low-tier", &regs).unwrap_err();
952        let msg = err.to_string();
953        assert!(msg.contains("does not advertise"), "got: {msg}");
954        assert!(msg.contains("fail-closed"), "got: {msg}");
955    }
956
957    #[test]
958    fn fuse_capability_gate_unknown_backend_errors() {
959        let regs = vec![&FUSE_CAP];
960        let err = check_fuse_mount_capability("bogus", &regs).unwrap_err();
961        assert!(
962            err.to_string().contains("unknown sandbox backend"),
963            "got: {err}"
964        );
965    }
966
967    #[cfg(feature = "nspawn")]
968    #[test]
969    fn nspawn_advertises_fuse_mount_capability_in_real_inventory() {
970        // The real inventory: nspawn is the Model B FUSE-root target (#715), so
971        // the public gate resolves it Ok. nspawn registers under its feature.
972        assert!(
973            require_fuse_mount_capability("nspawn").is_ok(),
974            "nspawn must advertise mounts_fuse_vfs"
975        );
976    }
977
978    #[cfg(feature = "kata")]
979    #[test]
980    fn kata_does_not_advertise_fuse_mount_capability() {
981        // kata already shares the composed VFS via virtio-fs, not a host FUSE
982        // mount, so it must NOT advertise the capability (fail-closed).
983        let err = require_fuse_mount_capability("kata").unwrap_err();
984        assert!(err.to_string().contains("does not advertise"), "got: {err}");
985    }
986
987    #[test]
988    fn registry_contains_nspawn_only_under_feature() {
989        // nspawn registers iff built with `--features nspawn` (#518: per-
990        // BackendType feature, mirroring kata/oci/cri/wasm). With the feature off
991        // it must NOT be a selectable name (fail-closed).
992        let has_nspawn = inventory::iter::<BackendRegistration>().any(|r| r.name == "nspawn");
993        assert_eq!(
994            has_nspawn,
995            cfg!(feature = "nspawn"),
996            "nspawn registration must track the nspawn feature"
997        );
998    }
999
1000    #[test]
1001    fn registry_contains_kata_only_under_feature() {
1002        let has_kata = inventory::iter::<BackendRegistration>().any(|r| r.name == "kata");
1003        assert_eq!(
1004            has_kata,
1005            cfg!(feature = "kata"),
1006            "kata registration must track the kata feature (kata-vm is its alias)"
1007        );
1008    }
1009
1010    // YAGNI note: still-speculative tiers must not be in the registry until
1011    // built. `wasm` (#505 P2), `oci` (#346) and `cri` (#510) are now real
1012    // backends, each tracked by its own feature-gated test below; nothing
1013    // remains speculative at present.
1014
1015    #[test]
1016    fn registry_contains_cri_only_under_feature() {
1017        // The CRI-client backend (#510) registers iff built with `--features
1018        // cri`. With the feature off it must NOT be a selectable name
1019        // (fail-closed), mirroring `wasm`/`oci`/`kata`.
1020        let has_cri = inventory::iter::<BackendRegistration>().any(|r| r.name == "cri");
1021        assert_eq!(
1022            has_cri,
1023            cfg!(feature = "cri"),
1024            "cri registration must track the cri feature"
1025        );
1026    }
1027
1028    #[test]
1029    fn registry_contains_k8s_only_under_feature() {
1030        // The Kubernetes API-server backend (#781) registers iff built with
1031        // `--features k8s`. With the feature off it must NOT be a selectable
1032        // name (fail-closed), mirroring `cri`/`wasm`/`oci`/`kata`.
1033        let has_k8s = inventory::iter::<BackendRegistration>().any(|r| r.name == "k8s");
1034        assert_eq!(
1035            has_k8s,
1036            cfg!(feature = "k8s"),
1037            "k8s registration must track the k8s feature"
1038        );
1039    }
1040
1041    #[cfg(feature = "k8s")]
1042    #[test]
1043    fn k8s_is_explicit_name_only() {
1044        // Handing a workload to a cluster (scheduler-placed, cross-node,
1045        // quota-charged) is an auth-surface decision (#781/#778): `"auto"` must
1046        // never reach for `k8s`, mirroring `cri`.
1047        let k8s = inventory::iter::<BackendRegistration>()
1048            .find(|r| r.name == "k8s")
1049            .expect("k8s registered under the k8s feature");
1050        assert!(!k8s.auto_selectable, "k8s must be explicit-name-only");
1051    }
1052
1053    #[cfg(feature = "cri")]
1054    #[test]
1055    fn cri_is_explicit_name_only() {
1056        // Driving an external runtime is an auth-surface decision (#510):
1057        // `"auto"` must never reach for `cri` on its own, even if it were the
1058        // only registered backend — mirrors `wasm`'s explicit-name-only stance.
1059        let cri = inventory::iter::<BackendRegistration>()
1060            .find(|r| r.name == "cri")
1061            .expect("cri registered under the cri feature");
1062        assert!(!cri.auto_selectable, "cri must be explicit-name-only");
1063    }
1064
1065    #[test]
1066    fn registry_contains_oci_only_under_feature() {
1067        // The rootless OCI (podman) backend (#346) registers iff built with
1068        // `--features oci`. With the feature off it must NOT be a selectable name
1069        // (fail-closed), mirroring `wasm`/`kata`.
1070        let has_oci = inventory::iter::<BackendRegistration>().any(|r| r.name == "oci");
1071        assert_eq!(
1072            has_oci,
1073            cfg!(feature = "oci"),
1074            "oci registration must track the oci feature"
1075        );
1076    }
1077
1078    #[cfg(all(feature = "oci", feature = "nspawn"))]
1079    #[test]
1080    fn oci_is_auto_selectable_and_outranks_nspawn() {
1081        // A rootless container (user namespaces + seccomp + its own image rootfs)
1082        // is a real isolation tier, so — unlike the in-process `wasm` backend — it
1083        // is auto-selectable and outranks nspawn (host-rootfs sharing).
1084        let oci = inventory::iter::<BackendRegistration>()
1085            .find(|r| r.name == "oci")
1086            .expect("oci registered under the oci feature");
1087        let nspawn = inventory::iter::<BackendRegistration>()
1088            .find(|r| r.name == "nspawn")
1089            .expect("nspawn registered under the nspawn feature");
1090        assert!(
1091            oci.auto_selectable,
1092            "oci is a real isolation tier → auto-selectable"
1093        );
1094        assert!(
1095            oci.priority > nspawn.priority,
1096            "rootless OCI must outrank nspawn for auto-selection"
1097        );
1098    }
1099
1100    #[test]
1101    fn registry_contains_wasm_only_under_feature() {
1102        // The in-process wasm backend (#505 P2) registers iff built with
1103        // `--features wasm` (which vendors the wasmtime substrate). With the
1104        // feature off it must NOT be a selectable name (fail-closed).
1105        let has_wasm = inventory::iter::<BackendRegistration>().any(|r| r.name == "wasm");
1106        assert_eq!(
1107            has_wasm,
1108            cfg!(feature = "wasm"),
1109            "wasm registration must track the wasm feature"
1110        );
1111    }
1112
1113    #[cfg(feature = "wasm")]
1114    #[test]
1115    fn wasm_is_auto_excluded() {
1116        // The in-process wasm backend (shared host address space) must be flagged
1117        // explicit-name-only so it can never be auto-picked (#547 ZSP — no silent
1118        // isolation downgrade).
1119        let wasm = inventory::iter::<BackendRegistration>()
1120            .find(|r| r.name == "wasm")
1121            .expect("wasm registered under the wasm feature");
1122        assert!(
1123            !wasm.auto_selectable,
1124            "wasm (in-process, shared address space) must be excluded from auto-selection"
1125        );
1126    }
1127
1128    #[cfg(feature = "wasm")]
1129    #[test]
1130    fn auto_never_returns_wasm_even_when_only_available_backend() {
1131        // With *only* the real wasm registration present (registered + available),
1132        // `"auto"` must still error rather than downgrade to it.
1133        let wasm = inventory::iter::<BackendRegistration>()
1134            .find(|r| r.name == "wasm")
1135            .expect("wasm registered under the wasm feature");
1136        let only_wasm = vec![wasm];
1137        let err = select_registration("auto", &only_wasm, |r| (r.is_available)()).unwrap_err();
1138        assert!(
1139            err.to_string().contains("no available sandbox backend"),
1140            "auto must not pick wasm; got: {err}"
1141        );
1142    }
1143
1144    // The wasm-vs-nspawn comparison needs both backends compiled in: wasm (the
1145    // auto-excluded tier) and nspawn (the auto-eligible tier that must win).
1146    #[cfg(all(feature = "wasm", feature = "nspawn"))]
1147    #[test]
1148    fn auto_prefers_nspawn_over_auto_excluded_wasm() {
1149        // With nspawn alongside wasm, `"auto"` resolves to nspawn — never wasm.
1150        // Inject a deterministic availability oracle (everything available) so this
1151        // does NOT depend on the runner actually having nspawn installed: nspawn is
1152        // auto_selectable (and outranks), wasm is auto_selectable:false (excluded), so
1153        // auto must return nspawn. (Real-runtime availability is exercised elsewhere;
1154        // here we assert the selection *logic*, env-independently.)
1155        let wasm = inventory::iter::<BackendRegistration>()
1156            .find(|r| r.name == "wasm")
1157            .expect("wasm registered under the wasm feature");
1158        let nspawn = inventory::iter::<BackendRegistration>()
1159            .find(|r| r.name == "nspawn")
1160            .expect("nspawn registered under the nspawn feature");
1161        let both = vec![wasm, nspawn];
1162        let r = select_registration("auto", &both, |_| true).unwrap();
1163        assert_eq!(r.name, "nspawn", "auto must pick nspawn, never wasm");
1164    }
1165
1166    #[cfg(feature = "wasm")]
1167    #[test]
1168    fn wasm_selectable_by_name_and_available() {
1169        // Explicit `wasm` request resolves to the wasm registration, and it is
1170        // always available once compiled in (wasmtime is vendored in-process).
1171        let regs: Vec<&'static BackendRegistration> =
1172            inventory::iter::<BackendRegistration>().collect();
1173        let reg = select_registration("wasm", &regs, |r| (r.is_available)())
1174            .expect("wasm selectable by name when feature on");
1175        assert_eq!(reg.name, "wasm");
1176        assert!((reg.is_available)(), "wasm is always available once built");
1177    }
1178
1179    #[cfg(all(feature = "kata", feature = "nspawn"))]
1180    #[test]
1181    fn kata_outranks_nspawn_in_priority() {
1182        let kata = inventory::iter::<BackendRegistration>()
1183            .find(|r| r.name == "kata")
1184            .unwrap();
1185        let nspawn = inventory::iter::<BackendRegistration>()
1186            .find(|r| r.name == "nspawn")
1187            .unwrap();
1188        assert!(
1189            kata.priority > nspawn.priority,
1190            "kata (VM) must outrank nspawn for auto-selection"
1191        );
1192    }
1193
1194    // ── N-backend fallback-chain semantics (#348) ──
1195    //
1196    // Synthetic 3-tier registry modeling a future kata > podman > nspawn world
1197    // (podman lands under #346; until then this just proves the *logic*
1198    // generalizes past two backends — it does not depend on #346 landing).
1199
1200    const MID: BackendRegistration = BackendRegistration {
1201        name: "mid-tier",
1202        priority: 50,
1203        auto_selectable: true,
1204        injects_9p_socket: false,
1205        mounts_fuse_vfs: false,
1206        is_available: || true,
1207        construct: never_available,
1208    };
1209
1210    fn three_tier_regs() -> Vec<&'static BackendRegistration> {
1211        vec![&HIGH, &MID, &LOW]
1212    }
1213
1214    #[test]
1215    fn auto_three_tier_all_available_picks_highest() {
1216        let r = select_registration("auto", &three_tier_regs(), |_| true).unwrap();
1217        assert_eq!(r.name, "high-tier");
1218    }
1219
1220    #[test]
1221    fn auto_three_tier_degrades_through_priority_order_one_by_one() {
1222        // Simulate backends disappearing one at a time, strongest first —
1223        // exactly the "kata missing -> try podman -> try nspawn" chain a
1224        // wizard cares about. Each step must fall exactly one tier, never
1225        // skip ahead and never error while something remains available.
1226        let regs = three_tier_regs();
1227
1228        let r = select_registration("auto", &regs, |reg| reg.name != "high-tier").unwrap();
1229        assert_eq!(
1230            r.name, "mid-tier",
1231            "first degradation: high unavailable -> mid"
1232        );
1233
1234        let r = select_registration("auto", &regs, |reg| reg.name == "low-tier").unwrap();
1235        assert_eq!(r.name, "low-tier", "second degradation: only low remains");
1236    }
1237
1238    #[test]
1239    fn auto_three_tier_errors_only_when_all_unavailable() {
1240        let err = select_registration("auto", &three_tier_regs(), |_| false).unwrap_err();
1241        let msg = err.to_string();
1242        assert!(msg.contains("no available sandbox backend"), "got: {msg}");
1243        // Must list all three registered names so the operator knows what to install.
1244        assert!(msg.contains("high-tier") && msg.contains("mid-tier") && msg.contains("low-tier"));
1245    }
1246
1247    #[test]
1248    fn auto_three_tier_skips_middle_unavailable_tier() {
1249        // Mid-tier specifically down (e.g. podman not installed) must not
1250        // wrongly fall all the way to low; high still wins if available, and
1251        // if high is also down it must land on low (skipping mid), not error.
1252        let regs = three_tier_regs();
1253        let r = select_registration("auto", &regs, |reg| reg.name != "mid-tier").unwrap();
1254        assert_eq!(
1255            r.name, "high-tier",
1256            "high still available, mid down -> high wins"
1257        );
1258
1259        let r = select_registration("auto", &regs, |reg| reg.name == "low-tier").unwrap();
1260        assert_eq!(
1261            r.name, "low-tier",
1262            "high and mid down -> falls through mid to low"
1263        );
1264    }
1265
1266    // ── unavailable_reason (#348 selection diagnostics) ──
1267
1268    #[test]
1269    fn unavailable_reason_has_per_backend_specifics() {
1270        assert!(unavailable_reason("kata").contains("cloud-hypervisor"));
1271        assert!(unavailable_reason("nspawn").contains("systemd-nspawn"));
1272        assert!(unavailable_reason("podman").contains("podman"));
1273        // Unknown backend still gets a non-empty, non-panicking message.
1274        assert!(unavailable_reason("future-backend").contains("future-backend"));
1275    }
1276
1277    // ── Scheduling-substrate explain adoption (#628) ──
1278    //
1279    // These exercise the substrate pipeline over `BackendCandidate` directly (a
1280    // deterministic candidate set), proving the migrated explain produces the
1281    // shared `SelectionReport<C>` and that `unavailable_reason` flows through as
1282    // the `RejectionReason`. `explain_selection()` itself is a thin wrapper that
1283    // feeds the real inventory into this same pipeline.
1284
1285    fn candidate(name: &'static str, priority: i32, auto: bool, avail: bool) -> BackendCandidate {
1286        BackendCandidate {
1287            name,
1288            priority,
1289            auto_selectable: auto,
1290            available: avail,
1291        }
1292    }
1293
1294    /// Reusable copy of `explain_selection`'s pipeline over an explicit candidate
1295    /// set (so tests don't depend on the host's installed runtimes).
1296    fn explain_over(candidates: &[BackendCandidate]) -> SelectionReport<BackendCandidate> {
1297        let predicates = vec![auto_eligibility_predicate(), availability_predicate()];
1298        let mut report = scheduling::select(
1299            candidates,
1300            &predicates,
1301            |a, b| b.priority.cmp(&a.priority).then_with(|| a.name.cmp(b.name)),
1302            1,
1303        );
1304        report.requested = "auto".to_owned();
1305        report
1306    }
1307
1308    #[test]
1309    fn explain_selects_highest_priority_available_auto_eligible() {
1310        let cands = vec![
1311            candidate("kata", 100, true, true),
1312            candidate("nspawn", 10, true, true),
1313        ];
1314        let report = explain_over(&cands);
1315        let selected: Vec<&str> = report
1316            .candidates
1317            .iter()
1318            .filter(|o| o.selected)
1319            .map(|o| o.candidate.name)
1320            .collect();
1321        assert_eq!(selected, vec!["kata"], "highest-priority available wins");
1322        assert_eq!(report.requested, "auto");
1323    }
1324
1325    #[test]
1326    fn explain_reports_unavailable_reason_as_rejection() {
1327        // kata unavailable → excluded, and its RejectionReason must be the
1328        // backend-specific `unavailable_reason` (the #628 integration point).
1329        let cands = vec![
1330            candidate("kata", 100, true, false),
1331            candidate("nspawn", 10, true, true),
1332        ];
1333        let report = explain_over(&cands);
1334        let kata = report
1335            .candidates
1336            .iter()
1337            .find(|o| o.candidate.name == "kata")
1338            .unwrap();
1339        assert!(!kata.selected);
1340        assert_eq!(
1341            kata.rejection_reason.as_deref(),
1342            Some(unavailable_reason("kata").as_str())
1343        );
1344        assert!(kata
1345            .rejection_reason
1346            .as_deref()
1347            .unwrap()
1348            .contains("cloud-hypervisor"));
1349        // nspawn (available, auto-eligible) is the winner.
1350        let nspawn = report
1351            .candidates
1352            .iter()
1353            .find(|o| o.candidate.name == "nspawn")
1354            .unwrap();
1355        assert!(nspawn.selected);
1356    }
1357
1358    #[test]
1359    fn explain_excludes_auto_ineligible_with_reason() {
1360        // An explicit-name-only backend (e.g. wasm) at the highest priority and
1361        // available must be rejected in the trace, and a weaker auto-eligible
1362        // backend selected — matching `select_registration`'s auto semantics.
1363        let cands = vec![
1364            candidate("wasm", 1000, false, true),
1365            candidate("nspawn", 10, true, true),
1366        ];
1367        let report = explain_over(&cands);
1368        let wasm = report
1369            .candidates
1370            .iter()
1371            .find(|o| o.candidate.name == "wasm")
1372            .unwrap();
1373        assert!(!wasm.selected);
1374        assert!(wasm
1375            .rejection_reason
1376            .as_deref()
1377            .unwrap()
1378            .contains("explicit-name-only"));
1379        let nspawn = report
1380            .candidates
1381            .iter()
1382            .find(|o| o.candidate.name == "nspawn")
1383            .unwrap();
1384        assert!(
1385            nspawn.selected,
1386            "auto must pick the auto-eligible backend, never the excluded one"
1387        );
1388    }
1389
1390    #[test]
1391    fn explain_agrees_with_select_registration_on_the_winner() {
1392        // The explain trace and the authoritative `select_registration` path must
1393        // agree on which backend `"auto"` yields, over the same candidate set.
1394        const A: BackendRegistration = BackendRegistration {
1395            name: "high-tier",
1396            priority: 100,
1397            auto_selectable: true,
1398            injects_9p_socket: false,
1399            mounts_fuse_vfs: false,
1400            is_available: || true,
1401            construct: never_available,
1402        };
1403        const B: BackendRegistration = BackendRegistration {
1404            name: "low-tier",
1405            priority: 10,
1406            auto_selectable: true,
1407            injects_9p_socket: false,
1408            mounts_fuse_vfs: false,
1409            is_available: || true,
1410            construct: never_available,
1411        };
1412        let regs = vec![&A, &B];
1413        let winner = select_registration("auto", &regs, |_| true).unwrap();
1414
1415        let cands: Vec<BackendCandidate> = regs
1416            .iter()
1417            .map(|r| BackendCandidate::from_registration(r))
1418            .collect();
1419        let report = explain_over(&cands);
1420        let explained = report
1421            .candidates
1422            .iter()
1423            .find(|o| o.selected)
1424            .map(|o| o.candidate.name)
1425            .unwrap();
1426        assert_eq!(
1427            explained, winner.name,
1428            "explain winner must match select_registration"
1429        );
1430    }
1431
1432    #[test]
1433    fn backend_candidate_capability_labels_emit_runtime_truth() {
1434        // The capability-source seam: a backend contributes `runtime=<name>` +
1435        // availability so a node record can aggregate it (#628, no drift).
1436        let c = candidate("kata", 100, true, false);
1437        let labels = c.capability_labels();
1438        assert!(labels.contains(&("runtime".to_owned(), "kata".to_owned())));
1439        assert!(labels.contains(&("runtime.kata.available".to_owned(), "false".to_owned())));
1440        assert!(c.capability_resources().is_empty());
1441    }
1442}