Skip to main content

hyprstream_workers/runtime/
nspawn.rs

1//! NspawnBackend — systemd-nspawn based sandbox isolation
2//!
3//! Runs each sandbox as a lightweight container via `systemd-nspawn`,
4//! bind-mounting the host root filesystem (`--directory=/`) with ephemeral
5//! `/tmp` and `/run`.  Provides network isolation via `--network-veth` and
6//! GPU pass-through via device bind-mounts.
7//!
8//! Service discovery uses host-side IPC paths namespaced by
9//! `HYPRSTREAM_INSTANCE` rather than network IP, avoiding timing issues
10//! with veth IP assignment.
11
12use std::any::Any;
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::sync::Arc;
16use std::time::Duration;
17
18use async_trait::async_trait;
19use tracing::{debug, info, warn};
20
21use crate::config::PoolConfig;
22use crate::error::{Result, WorkerError};
23
24use super::backend::{SandboxBackend, SandboxHandle};
25use super::sandbox::PodSandbox;
26#[cfg(feature = "oci-image")]
27use crate::image::RafsStore;
28use hyprstream_rpc_std::worker_client::{LinuxContainerResources, PodSandboxConfig};
29
30/// Sub-directory of the per-sandbox runtime dir where the tenant VFS is
31/// FUSE-mounted for Model B (#715) POSIX-rooting.
32#[cfg(feature = "oci-image")]
33const VFS_ROOT_DIR: &str = "vfs-root";
34
35// ─────────────────────────────────────────────────────────────────────────────
36// Configuration
37// ─────────────────────────────────────────────────────────────────────────────
38
39/// Configuration for the nspawn backend.
40#[derive(Debug, Clone)]
41pub struct NspawnConfig {
42    /// Path to libtorch (bind-mounted read-only into the container).
43    pub libtorch_path: PathBuf,
44    /// `LD_LIBRARY_PATH` forwarded into the container.
45    pub ld_library_path: String,
46    /// Run in rootless / user mode (default: true).
47    pub user_mode: bool,
48    /// Enable `--network-veth` for network isolation.
49    pub network_veth: bool,
50    /// GPU device paths to bind-mount (auto-detected from `/dev/dri`).
51    pub gpu_devices: Vec<PathBuf>,
52    /// Readiness poll timeout.
53    pub ready_timeout: Duration,
54    /// Readiness poll interval.
55    pub ready_interval: Duration,
56}
57
58impl Default for NspawnConfig {
59    fn default() -> Self {
60        let libtorch_path = std::env::var("LIBTORCH")
61            .map(PathBuf::from)
62            .unwrap_or_else(|_| PathBuf::from("/usr/lib/libtorch"));
63
64        let ld_library_path = std::env::var("LD_LIBRARY_PATH").unwrap_or_default();
65
66        Self {
67            libtorch_path,
68            ld_library_path,
69            user_mode: !nix::unistd::geteuid().is_root(),
70            network_veth: true,
71            gpu_devices: detect_gpu_devices(),
72            ready_timeout: Duration::from_secs(10),
73            ready_interval: Duration::from_millis(100),
74        }
75    }
76}
77
78/// Auto-detect GPU render/card nodes under `/dev/dri`.
79fn detect_gpu_devices() -> Vec<PathBuf> {
80    let dri = Path::new("/dev/dri");
81    if !dri.exists() {
82        return Vec::new();
83    }
84    let mut devices = Vec::new();
85    if let Ok(entries) = std::fs::read_dir(dri) {
86        for entry in entries.flatten() {
87            let name = entry.file_name();
88            let name = name.to_string_lossy();
89            if name.starts_with("card") || name.starts_with("renderD") {
90                devices.push(entry.path());
91            }
92        }
93    }
94    devices
95}
96
97// ─────────────────────────────────────────────────────────────────────────────
98// Handle
99// ─────────────────────────────────────────────────────────────────────────────
100
101/// Backend-specific state stored on each `PodSandbox`.
102#[derive(Debug, Clone)]
103pub struct NspawnHandle {
104    /// Sandbox identifier (matches `PodSandbox::id`).
105    pub sandbox_id: String,
106    /// Machine name passed to `--machine=`.
107    pub machine_name: String,
108    /// PID of the nspawn leader process (populated after start).
109    pub leader_pid: Option<u32>,
110    /// Host-side sandbox directory for bind-mounts.
111    pub sandbox_path: PathBuf,
112    /// Per-sandbox tenant-VFS FUSE mount (Model B, #715). When set, the
113    /// container is `--directory`-rooted at this mount's host directory. Held
114    /// here (behind `Arc`) for the sandbox's lifetime so the mount — and its
115    /// serving thread + injected registries — outlive container start; it is
116    /// unmounted when the last handle clone drops (RAII), i.e. on sandbox
117    /// stop/destroy.
118    #[cfg(feature = "oci-image")]
119    pub vfs_root_mount: Option<Arc<super::sandbox_fs::SandboxFsLocalMount>>,
120}
121
122impl SandboxHandle for NspawnHandle {
123    fn as_any(&self) -> &dyn Any {
124        self
125    }
126}
127
128// ─────────────────────────────────────────────────────────────────────────────
129// Backend
130// ─────────────────────────────────────────────────────────────────────────────
131
132/// systemd-nspawn sandbox backend.
133pub struct NspawnBackend {
134    config: NspawnConfig,
135    /// RAFS image store for composing the per-sandbox tenant VFS that Model B
136    /// (#715) FUSE-mounts as the container root. `None` when no store was
137    /// provided; the field exists only when the image service (`oci-image`) is
138    /// compiled in.
139    #[cfg(feature = "oci-image")]
140    rafs_store: Option<Arc<RafsStore>>,
141}
142
143impl NspawnBackend {
144    pub fn new(config: NspawnConfig) -> Self {
145        Self {
146            config,
147            #[cfg(feature = "oci-image")]
148            rafs_store: None,
149        }
150    }
151
152    /// Attach the RAFS image store used to compose the per-sandbox tenant VFS
153    /// for Model B (#715) FUSE-rooting. Threaded in from [`BackendCtx`] at
154    /// selection time (see the inventory registration below). Without a store,
155    /// nspawn behaves exactly as before — a host-root (`--directory=/`)
156    /// container, no tenant-VFS root.
157    ///
158    /// [`BackendCtx`]: super::selection::BackendCtx
159    #[cfg(feature = "oci-image")]
160    pub fn with_rafs_store(mut self, rafs_store: Option<Arc<RafsStore>>) -> Self {
161        self.rafs_store = rafs_store;
162        self
163    }
164
165    /// Runtime prerequisite probe for the backend registry: both
166    /// `systemd-nspawn` and `machinectl` must be on PATH. Mirrors the
167    /// instance-level [`SandboxBackend::is_available`] check.
168    fn registry_is_available() -> bool {
169        which::which("systemd-nspawn").is_ok() && which::which("machinectl").is_ok()
170    }
171
172    /// Build the `systemd-nspawn` argument list for a sandbox.
173    ///
174    /// `vfs_root` is the host directory the tenant VFS was FUSE-mounted at
175    /// (Model B, #715). When `Some`, the container is rooted there — its
176    /// workload sees the tenant VFS as its filesystem — instead of sharing the
177    /// host root (`--directory=/`).
178    fn build_nspawn_args(
179        &self,
180        sandbox: &PodSandbox,
181        machine_name: &str,
182        annotations: &HashMap<String, String>,
183        resources: Option<&LinuxContainerResources>,
184        vfs_root: Option<&Path>,
185    ) -> Vec<String> {
186        let mut args = Vec::new();
187
188        // Core flags. Root at the tenant-VFS FUSE mount when Model B (#715) has
189        // composed one for this sandbox; otherwise share the host root as before.
190        args.push(format!("--machine={machine_name}"));
191        match vfs_root {
192            Some(root) => args.push(format!("--directory={}", root.display())),
193            None => args.push("--directory=/".into()),
194        }
195        args.push("--tmpfs=/tmp".into());
196        args.push("--tmpfs=/run".into());
197
198        // Bind-mount sandbox runtime directory
199        args.push(format!("--bind={}", sandbox.sandbox_path().display()));
200
201        // Bind-mount libtorch read-only
202        if self.config.libtorch_path.exists() {
203            args.push(format!("--bind-ro={}", self.config.libtorch_path.display()));
204        }
205
206        // Network isolation
207        if self.config.network_veth {
208            args.push("--network-veth".into());
209        }
210
211        // GPU pass-through when requested via annotation
212        let wants_gpu = annotations
213            .get("hyprstream.io/gpu")
214            .is_some_and(|v| v == "true");
215        if wants_gpu {
216            for dev in &self.config.gpu_devices {
217                args.push(format!("--bind={}", dev.display()));
218            }
219        }
220
221        // Environment variables
222        args.push(format!("--setenv=HYPRSTREAM_INSTANCE={}", sandbox.id));
223        if !self.config.ld_library_path.is_empty() {
224            args.push(format!(
225                "--setenv=LD_LIBRARY_PATH={}",
226                self.config.ld_library_path
227            ));
228        }
229        args.push(format!(
230            "--setenv=LIBTORCH={}",
231            self.config.libtorch_path.display()
232        ));
233
234        // Resource limits → systemd property flags
235        if let Some(res) = resources {
236            if res.cpu_quota > 0 {
237                args.push(format!("--property=CPUQuota={}%", res.cpu_quota));
238            }
239            if res.memory_limit_in_bytes > 0 {
240                args.push(format!(
241                    "--property=MemoryMax={}",
242                    res.memory_limit_in_bytes
243                ));
244            }
245        }
246
247        // Separator + boot command
248        args.push("--".into());
249        let hyprstream_bin = hyprstream_rpc::paths::executable_path()
250            .unwrap_or_else(|_| PathBuf::from("hyprstream"));
251        args.push(hyprstream_bin.to_string_lossy().into());
252        args.push("service".into());
253        args.push("start".into());
254        args.push("--all".into());
255        args.push("--foreground".into());
256
257        args
258    }
259
260    /// Annotation key carrying the sandbox's policy principal (the [`Subject`]
261    /// at the VFS Mount boundary, #353/#319/#328) — same key the kata backend
262    /// reads. When absent, the sandbox id is used as a stable per-sandbox
263    /// principal so isolation still holds.
264    ///
265    /// [`Subject`]: hyprstream_vfs::Subject
266    #[cfg(feature = "oci-image")]
267    const SUBJECT_ANNOTATION: &'static str = "hyprstream.io/subject";
268
269    /// Resolve the [`Subject`](hyprstream_vfs::Subject) this sandbox's tenant
270    /// VFS is served under (mirrors `KataBackend::sandbox_subject`).
271    #[cfg(feature = "oci-image")]
272    fn sandbox_subject(
273        sandbox: &PodSandbox,
274        annotations: &HashMap<String, String>,
275    ) -> hyprstream_vfs::Subject {
276        annotations
277            .get(Self::SUBJECT_ANNOTATION)
278            .filter(|s| !s.is_empty())
279            .map(hyprstream_vfs::Subject::new)
280            .unwrap_or_else(|| hyprstream_vfs::Subject::new(sandbox.id.clone()))
281    }
282
283    /// Compose the per-sandbox tenant VFS (image rootfs + injected mounts) and
284    /// FUSE-mount it at a per-sandbox host directory (Model B, #715) — the
285    /// non-VM sibling of `KataBackend::compose_and_serve_vfs`.
286    ///
287    /// Returns the RAII [`SandboxFsLocalMount`](super::sandbox_fs::SandboxFsLocalMount);
288    /// the container is then `--directory`-rooted at its `mountpoint()`. Compose
289    /// is CPU/IO-bound (RAFS load), so it runs on a blocking thread; the mount's
290    /// own serving thread is spawned inside `serve_local_at`. The
291    /// [`require_fuse_mount_capability`](super::selection::require_fuse_mount_capability)
292    /// gate asserts nspawn is declared able to FUSE-root before we mount
293    /// (fail-closed).
294    #[cfg(feature = "oci-image")]
295    async fn compose_and_mount_vfs_root(
296        &self,
297        sandbox: &PodSandbox,
298        image_id: &str,
299        subject: hyprstream_vfs::Subject,
300    ) -> Result<super::sandbox_fs::SandboxFsLocalMount> {
301        use super::sandbox_fs::SandboxFs;
302        use hyprstream_vfs::injected::SyntheticNode;
303
304        let rafs_store = self.rafs_store.clone().ok_or_else(|| {
305            WorkerError::SandboxCreationFailed(
306                "nspawn tenant-VFS root requested but no RAFS image store is configured".into(),
307            )
308        })?;
309
310        // Fail-closed capability gate (#715): only POSIX-root the composed VFS
311        // for a backend that declares it can FUSE-mount it. nspawn does.
312        super::selection::require_fuse_mount_capability(self.backend_type())?;
313
314        let mountpoint = sandbox.sandbox_path().join(VFS_ROOT_DIR);
315        let rt = tokio::runtime::Handle::current();
316        let image_id = image_id.to_owned();
317        let sandbox_dir = sandbox.sandbox_path().clone();
318        let mp = mountpoint.clone();
319
320        // RAFS load + overlay setup is blocking; do it off the async executor.
321        tokio::task::spawn_blocking(move || {
322            let fs = SandboxFs::compose(
323                &rafs_store,
324                &image_id,
325                &sandbox_dir,
326                subject,
327                // Models / deltas injected listings: empty mount points for now
328                // (the worker runtime populates them per tenant), mirroring kata.
329                SyntheticNode::dir(),
330                SyntheticNode::dir(),
331            )?;
332            fs.serve_local_at(rt, mp)
333        })
334        .await
335        .map_err(|e| {
336            WorkerError::SandboxCreationFailed(format!("VFS compose/mount task join: {e}"))
337        })?
338    }
339
340    /// Discover leader PID via `machinectl show`.
341    async fn discover_leader_pid(machine_name: &str) -> Option<u32> {
342        let output = tokio::process::Command::new("machinectl")
343            .args(["show", "-p", "Leader", "--value", machine_name])
344            .output()
345            .await
346            .ok()?;
347
348        if !output.status.success() {
349            return None;
350        }
351
352        let stdout = String::from_utf8_lossy(&output.stdout);
353        stdout.trim().parse().ok()
354    }
355
356    /// Wait for the inner hyprstream to write its discovery socket.
357    async fn wait_for_ready(&self, sandbox_id: &str) -> Result<()> {
358        let runtime_dir = hyprstream_rpc::paths::runtime_dir();
359        // The inner hyprstream uses HYPRSTREAM_INSTANCE, so its sockets
360        // land under `instances/{sandbox_id}/` on the host filesystem.
361        let discovery_sock = runtime_dir
362            .join("instances")
363            .join(sandbox_id)
364            .join("discovery.sock");
365
366        let deadline = tokio::time::Instant::now() + self.config.ready_timeout;
367
368        while tokio::time::Instant::now() < deadline {
369            if discovery_sock.exists() {
370                debug!(
371                    sandbox_id,
372                    path = %discovery_sock.display(),
373                    "Sandbox ready (discovery socket exists)"
374                );
375                return Ok(());
376            }
377            tokio::time::sleep(self.config.ready_interval).await;
378        }
379
380        Err(WorkerError::SandboxTimeout {
381            operation: format!("nspawn readiness ({sandbox_id})"),
382            timeout_secs: self.config.ready_timeout.as_secs(),
383        })
384    }
385}
386
387impl std::fmt::Debug for NspawnBackend {
388    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
389        f.debug_struct("NspawnBackend")
390            .field("config", &self.config)
391            .finish()
392    }
393}
394
395#[async_trait]
396impl SandboxBackend for NspawnBackend {
397    fn backend_type(&self) -> &'static str {
398        "nspawn"
399    }
400
401    fn is_available(&self) -> bool {
402        which::which("systemd-nspawn").is_ok() && which::which("machinectl").is_ok()
403    }
404
405    async fn initialize(&self, _config: &PoolConfig) -> Result<()> {
406        if !self.is_available() {
407            return Err(WorkerError::ConfigError(
408                "systemd-nspawn or machinectl not found in PATH".into(),
409            ));
410        }
411        if !self.config.libtorch_path.exists() {
412            warn!(
413                path = %self.config.libtorch_path.display(),
414                "libtorch path does not exist — containers may fail to start"
415            );
416        }
417        Ok(())
418    }
419
420    async fn start(
421        &self,
422        sandbox: &mut PodSandbox,
423        config: &PodSandboxConfig,
424        pool_config: &PoolConfig,
425        annotations: &HashMap<String, String>,
426    ) -> Result<Arc<dyn SandboxHandle>> {
427        let machine_name = format!("hyprstream-{}", sandbox.id);
428
429        // Ensure sandbox runtime directory exists
430        let sandbox_path = pool_config.runtime_dir.join(&sandbox.id);
431        tokio::fs::create_dir_all(&sandbox_path).await?;
432        sandbox.sandbox_path = sandbox_path.clone();
433
434        // Model B (#715): when an image store is configured and the sandbox
435        // names an image, compose the per-sandbox tenant VFS and FUSE-mount it as
436        // the container root, so the workload POSIX-roots in the tenant's VFS —
437        // the non-VM sibling of kata's virtio-fs path. Held on the handle so it
438        // outlives start and unmounts on teardown (RAII). Without a store (or
439        // image), nspawn stays a host-root container exactly as before.
440        #[cfg(feature = "oci-image")]
441        let vfs_root_mount: Option<Arc<super::sandbox_fs::SandboxFsLocalMount>> =
442            if self.rafs_store.is_some() {
443                if let Some(image_id) = sandbox.image_id.clone() {
444                    let subject = Self::sandbox_subject(sandbox, annotations);
445                    let mount = self
446                        .compose_and_mount_vfs_root(sandbox, &image_id, subject)
447                        .await?;
448                    info!(
449                        sandbox_id = %sandbox.id,
450                        mountpoint = %mount.mountpoint().display(),
451                        "Mounted tenant VFS as nspawn container root (Model B, #715)"
452                    );
453                    Some(Arc::new(mount))
454                } else {
455                    None
456                }
457            } else {
458                None
459            };
460
461        #[cfg(feature = "oci-image")]
462        let vfs_root_mountpoint: Option<PathBuf> = vfs_root_mount
463            .as_ref()
464            .map(|m| m.mountpoint().to_path_buf());
465        #[cfg(not(feature = "oci-image"))]
466        let vfs_root_mountpoint: Option<PathBuf> = None;
467
468        // Build argument list
469        let nspawn_args = self.build_nspawn_args(
470            sandbox,
471            &machine_name,
472            annotations,
473            Some(&config.linux.resources),
474            vfs_root_mountpoint.as_deref(),
475        );
476
477        info!(
478            sandbox_id = %sandbox.id,
479            machine = %machine_name,
480            "Starting nspawn sandbox"
481        );
482
483        // Spawn detached — nspawn runs as a daemon
484        let mut cmd = tokio::process::Command::new("systemd-nspawn");
485        cmd.args(&nspawn_args);
486        cmd.stdin(std::process::Stdio::null())
487            .stdout(std::process::Stdio::null())
488            .stderr(std::process::Stdio::null());
489
490        let _child = cmd.spawn().map_err(|e| {
491            WorkerError::VmStartFailed(format!("failed to spawn systemd-nspawn: {e}"))
492        })?;
493
494        // Wait for the inner hyprstream to become ready
495        self.wait_for_ready(&sandbox.id).await?;
496
497        // Discover leader PID
498        let leader_pid = Self::discover_leader_pid(&machine_name).await;
499
500        let handle = Arc::new(NspawnHandle {
501            sandbox_id: sandbox.id.clone(),
502            machine_name: machine_name.clone(),
503            leader_pid,
504            sandbox_path,
505            #[cfg(feature = "oci-image")]
506            vfs_root_mount,
507        });
508
509        info!(
510            sandbox_id = %sandbox.id,
511            machine = %machine_name,
512            leader_pid = ?leader_pid,
513            "Nspawn sandbox started"
514        );
515
516        Ok(handle)
517    }
518
519    async fn stop(&self, sandbox: &PodSandbox) -> Result<()> {
520        let machine_name = format!("hyprstream-{}", sandbox.id);
521        info!(sandbox_id = %sandbox.id, machine = %machine_name, "Stopping nspawn sandbox");
522
523        let status = tokio::process::Command::new("machinectl")
524            .args(["stop", &machine_name])
525            .status()
526            .await
527            .map_err(|e| {
528                WorkerError::VmStopFailed(format!("failed to run machinectl stop: {e}"))
529            })?;
530
531        if !status.success() {
532            warn!(
533                sandbox_id = %sandbox.id,
534                "machinectl stop returned non-zero (machine may already be stopped)"
535            );
536        }
537
538        Ok(())
539    }
540
541    async fn destroy(&self, sandbox: &PodSandbox) -> Result<()> {
542        let machine_name = format!("hyprstream-{}", sandbox.id);
543        info!(sandbox_id = %sandbox.id, machine = %machine_name, "Destroying nspawn sandbox");
544
545        // Forcefully terminate
546        let _ = tokio::process::Command::new("machinectl")
547            .args(["terminate", &machine_name])
548            .status()
549            .await;
550
551        // Clean up sandbox directory
552        let sandbox_path = sandbox.sandbox_path();
553        if sandbox_path.exists() {
554            if let Err(e) = tokio::fs::remove_dir_all(sandbox_path).await {
555                warn!(
556                    sandbox_id = %sandbox.id,
557                    error = %e,
558                    "Failed to remove sandbox directory"
559                );
560            }
561        }
562
563        // Clean up instance runtime directory
564        let instance_dir = hyprstream_rpc::paths::runtime_dir()
565            .join("instances")
566            .join(&sandbox.id);
567        if instance_dir.exists() {
568            let _ = tokio::fs::remove_dir_all(&instance_dir).await;
569        }
570
571        Ok(())
572    }
573
574    async fn reset(&self, _sandbox: &mut PodSandbox) -> Result<bool> {
575        // Nspawn sandboxes are ephemeral — must be recreated
576        Ok(false)
577    }
578
579    async fn get_pids(&self, sandbox: &PodSandbox) -> Result<Vec<u32>> {
580        let machine_name = format!("hyprstream-{}", sandbox.id);
581        match Self::discover_leader_pid(&machine_name).await {
582            Some(pid) => Ok(vec![pid]),
583            None => Ok(Vec::new()),
584        }
585    }
586
587    async fn deliver_namespace(
588        &self,
589        sandbox: &PodSandbox,
590        _namespace: hyprstream_vfs::Namespace,
591        _subject: hyprstream_vfs::Subject,
592        transport: super::backend::NamespaceTransport,
593    ) -> Result<super::backend::NamespaceDelivery> {
594        use super::backend::{NamespaceDelivery, NamespaceTransport};
595
596        // Provisional (#635): nspawn doesn't need FUSE to see a composed
597        // namespace — it can bind a directory straight into a running
598        // container via `machinectl bind`. Until #653's local-FUSE-mount
599        // serve mode lands, there is no generic way to materialize an
600        // in-process `Namespace` at a host directory, so this only supports
601        // the case where the caller has *already* materialized `_namespace`
602        // at `target` (e.g. by hand today; by #653's serve mode once it
603        // exists) and just wants it attached to the guest. The `Namespace`
604        // value itself is intentionally unused here — that is the gap #653
605        // closes, not something this backend can paper over on its own.
606        let NamespaceTransport::BindMount { target } = transport else {
607            return Err(WorkerError::Unsupported(format!(
608                "nspawn backend only supports the BindMount namespace transport (needs #653's \
609                 local-FUSE-mount serve mode for a real VirtioFs/HostImports equivalent), got {transport:?}"
610            )));
611        };
612
613        if !target.exists() {
614            return Err(WorkerError::SandboxCreationFailed(format!(
615                "deliver_namespace: bind-mount source {} does not exist",
616                target.display()
617            )));
618        }
619
620        let machine_name = format!("hyprstream-{}", sandbox.id);
621        let target_str = target.to_string_lossy().into_owned();
622
623        // `machinectl bind` runtime-attaches a host path into an already
624        // running container/VM — the nspawn analogue of Kata's hot-attach
625        // ShareFs device.
626        let status = tokio::process::Command::new("machinectl")
627            .args(["bind", "--mkdir", &machine_name, &target_str])
628            .status()
629            .await
630            .map_err(|e| {
631                WorkerError::SandboxCreationFailed(format!("failed to run machinectl bind: {e}"))
632            })?;
633
634        if !status.success() {
635            return Err(WorkerError::SandboxCreationFailed(format!(
636                "machinectl bind {machine_name} {target_str} failed"
637            )));
638        }
639
640        Ok(NamespaceDelivery::BindMount { target })
641    }
642
643    fn supports_exec(&self) -> bool {
644        true
645    }
646
647    async fn exec_sync(
648        &self,
649        sandbox: &PodSandbox,
650        command: &[String],
651        timeout_secs: u64,
652    ) -> Result<(i32, Vec<u8>, Vec<u8>)> {
653        let machine_name = format!("hyprstream-{}", sandbox.id);
654
655        let mut args = vec!["shell".to_owned(), machine_name, "--".to_owned()];
656        args.extend_from_slice(command);
657
658        let output = tokio::time::timeout(
659            Duration::from_secs(timeout_secs),
660            tokio::process::Command::new("machinectl")
661                .args(&args)
662                .output(),
663        )
664        .await
665        .map_err(|_| WorkerError::SandboxTimeout {
666            operation: format!("exec_sync in {}", sandbox.id),
667            timeout_secs,
668        })?
669        .map_err(|e| WorkerError::ExecFailed(format!("machinectl shell failed: {e}")))?;
670
671        let exit_code = output.status.code().unwrap_or(-1);
672        Ok((exit_code, output.stdout, output.stderr))
673    }
674
675    async fn update_resources(
676        &self,
677        sandbox: &PodSandbox,
678        resources: &LinuxContainerResources,
679    ) -> Result<()> {
680        let machine_name = format!("hyprstream-{}", sandbox.id);
681
682        // Use systemctl set-property on the machine's scope unit
683        let scope = format!("machine-{machine_name}.scope");
684
685        if resources.cpu_quota > 0 {
686            let _ = tokio::process::Command::new("systemctl")
687                .args([
688                    "set-property",
689                    &scope,
690                    &format!("CPUQuota={}%", resources.cpu_quota),
691                ])
692                .status()
693                .await;
694        }
695        if resources.memory_limit_in_bytes > 0 {
696            let _ = tokio::process::Command::new("systemctl")
697                .args([
698                    "set-property",
699                    &scope,
700                    &format!("MemoryMax={}", resources.memory_limit_in_bytes),
701                ])
702                .status()
703                .await;
704        }
705
706        Ok(())
707    }
708}
709
710// ─────────────────────────────────────────────────────────────────────────────
711// Backend registry self-registration (#507 / #518)
712// ─────────────────────────────────────────────────────────────────────────────
713
714// systemd-nspawn is always compiled in (no feature gate), so it is always a
715// registered selectable backend. It is the lowest-priority auto pick — chosen
716// only when no stronger isolation tier is available. Selection still fails
717// closed: if nspawn's prerequisites are missing, an explicit `nspawn` request or
718// an `auto` with nothing else available errors rather than running unisolated.
719inventory::submit! {
720    crate::runtime::selection::BackendRegistration {
721        name: "nspawn",
722        priority: 10,
723        // Kernel-namespace isolation (real container) → eligible for `"auto"`.
724        auto_selectable: true,
725        // systemd-nspawn `--bind`s host paths into the container, so hyprstream
726        // can inject a host 9P Unix socket into the workload's mount namespace
727        // (#506). (Running the Wanix guest *as the boot command* under nspawn is
728        // a separate follow-up — nspawn currently boots the inner hyprstream
729        // service — but the socket-injection capability itself is real.)
730        injects_9p_socket: true,
731        // Model B (#715): nspawn POSIX-roots the workload in the tenant VFS by
732        // FUSE-mounting the composed per-sandbox namespace at a host directory
733        // and handing it to the container — the non-VM sibling of kata's
734        // virtio-fs path. It advertises the capability so the fail-closed gate
735        // ([`require_fuse_mount_capability`]) permits the mount.
736        mounts_fuse_vfs: true,
737        is_available: NspawnBackend::registry_is_available,
738        construct: |_ctx| {
739            #[cfg(feature = "oci-image")]
740            let backend = NspawnBackend::new(NspawnConfig::default())
741                .with_rafs_store(Some(std::sync::Arc::clone(&_ctx.rafs_store)));
742            #[cfg(not(feature = "oci-image"))]
743            let backend = NspawnBackend::new(NspawnConfig::default());
744            Ok(std::sync::Arc::new(backend) as std::sync::Arc<dyn crate::runtime::SandboxBackend>)
745        },
746    }
747}
748
749#[cfg(test)]
750#[allow(clippy::unwrap_used)]
751mod tests {
752    use super::*;
753
754    #[test]
755    fn test_nspawn_config_default() {
756        let config = NspawnConfig::default();
757        assert!(config.network_veth);
758        assert_eq!(config.ready_timeout, Duration::from_secs(10));
759        assert_eq!(config.ready_interval, Duration::from_millis(100));
760    }
761
762    #[test]
763    fn test_nspawn_handle_as_any() {
764        let handle = NspawnHandle {
765            sandbox_id: "test-123".into(),
766            machine_name: "hyprstream-test-123".into(),
767            leader_pid: Some(42),
768            sandbox_path: PathBuf::from("/tmp/test"),
769            #[cfg(feature = "oci-image")]
770            vfs_root_mount: None,
771        };
772
773        let handle: Arc<dyn SandboxHandle> = Arc::new(handle);
774        let downcast = handle.as_any().downcast_ref::<NspawnHandle>();
775        assert!(downcast.is_some());
776        assert_eq!(downcast.unwrap().sandbox_id, "test-123");
777        assert_eq!(downcast.unwrap().leader_pid, Some(42));
778    }
779
780    #[test]
781    fn test_detect_gpu_devices() {
782        // Just verify it doesn't panic — actual devices depend on hardware
783        let devices = detect_gpu_devices();
784        for d in &devices {
785            assert!(
786                d.to_string_lossy().contains("card") || d.to_string_lossy().contains("renderD")
787            );
788        }
789    }
790
791    #[test]
792    fn test_backend_type() {
793        let backend = NspawnBackend::new(NspawnConfig::default());
794        assert_eq!(backend.backend_type(), "nspawn");
795    }
796
797    // ─────────────────────────────────────────────────────────────────────
798    // deliver_namespace (#635)
799    // ─────────────────────────────────────────────────────────────────────
800
801    fn test_sandbox(sandbox_path: PathBuf) -> PodSandbox {
802        let config = PodSandboxConfig::default();
803        PodSandbox::new("test-sandbox-001".to_owned(), &config, sandbox_path)
804    }
805
806    #[tokio::test]
807    async fn test_deliver_namespace_rejects_non_bindmount_transport() {
808        use super::super::backend::NamespaceTransport;
809
810        let backend = NspawnBackend::new(NspawnConfig::default());
811        let sandbox = test_sandbox(PathBuf::from("/tmp/nonexistent-sandbox"));
812
813        let result = backend
814            .deliver_namespace(
815                &sandbox,
816                hyprstream_vfs::Namespace::new(),
817                hyprstream_vfs::Subject::new("test"),
818                NamespaceTransport::HostImports,
819            )
820            .await;
821
822        match result {
823            Err(WorkerError::Unsupported(msg)) => {
824                assert!(msg.contains("BindMount"), "unexpected message: {msg}");
825            }
826            other => panic!("expected Unsupported error, got: {other:?}"),
827        }
828    }
829
830    #[tokio::test]
831    async fn test_deliver_namespace_rejects_missing_bindmount_source() {
832        use super::super::backend::NamespaceTransport;
833
834        let backend = NspawnBackend::new(NspawnConfig::default());
835        let sandbox = test_sandbox(PathBuf::from("/tmp/nonexistent-sandbox"));
836
837        // No `machinectl` call should even be attempted: the source path
838        // doesn't exist, so this must fail before shelling out.
839        let result = backend
840            .deliver_namespace(
841                &sandbox,
842                hyprstream_vfs::Namespace::new(),
843                hyprstream_vfs::Subject::new("test"),
844                NamespaceTransport::BindMount {
845                    target: PathBuf::from("/nonexistent/path/for/635/test"),
846                },
847            )
848            .await;
849
850        assert!(result.is_err());
851        match result.unwrap_err() {
852            WorkerError::SandboxCreationFailed(msg) => {
853                assert!(msg.contains("does not exist"), "unexpected message: {msg}");
854            }
855            other => panic!("expected SandboxCreationFailed, got: {other:?}"),
856        }
857    }
858
859    #[test]
860    fn test_is_available() {
861        let backend = NspawnBackend::new(NspawnConfig::default());
862        // Just check it runs without panic — availability depends on system
863        let _available = backend.is_available();
864    }
865}