1use std::any::Any;
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use std::sync::Arc;
16use std::time::Duration;
17
18use async_trait::async_trait;
19use tracing::{debug, info, warn};
20
21use crate::config::PoolConfig;
22use crate::error::{Result, WorkerError};
23
24use super::backend::{SandboxBackend, SandboxHandle};
25use super::sandbox::PodSandbox;
26#[cfg(feature = "oci-image")]
27use crate::image::RafsStore;
28use hyprstream_rpc_std::worker_client::{LinuxContainerResources, PodSandboxConfig};
29
30#[cfg(feature = "oci-image")]
33const VFS_ROOT_DIR: &str = "vfs-root";
34
35#[derive(Debug, Clone)]
41pub struct NspawnConfig {
42 pub libtorch_path: PathBuf,
44 pub ld_library_path: String,
46 pub user_mode: bool,
48 pub network_veth: bool,
50 pub gpu_devices: Vec<PathBuf>,
52 pub ready_timeout: Duration,
54 pub ready_interval: Duration,
56}
57
58impl Default for NspawnConfig {
59 fn default() -> Self {
60 let libtorch_path = std::env::var("LIBTORCH")
61 .map(PathBuf::from)
62 .unwrap_or_else(|_| PathBuf::from("/usr/lib/libtorch"));
63
64 let ld_library_path = std::env::var("LD_LIBRARY_PATH").unwrap_or_default();
65
66 Self {
67 libtorch_path,
68 ld_library_path,
69 user_mode: !nix::unistd::geteuid().is_root(),
70 network_veth: true,
71 gpu_devices: detect_gpu_devices(),
72 ready_timeout: Duration::from_secs(10),
73 ready_interval: Duration::from_millis(100),
74 }
75 }
76}
77
78fn detect_gpu_devices() -> Vec<PathBuf> {
80 let dri = Path::new("/dev/dri");
81 if !dri.exists() {
82 return Vec::new();
83 }
84 let mut devices = Vec::new();
85 if let Ok(entries) = std::fs::read_dir(dri) {
86 for entry in entries.flatten() {
87 let name = entry.file_name();
88 let name = name.to_string_lossy();
89 if name.starts_with("card") || name.starts_with("renderD") {
90 devices.push(entry.path());
91 }
92 }
93 }
94 devices
95}
96
97#[derive(Debug, Clone)]
103pub struct NspawnHandle {
104 pub sandbox_id: String,
106 pub machine_name: String,
108 pub leader_pid: Option<u32>,
110 pub sandbox_path: PathBuf,
112 #[cfg(feature = "oci-image")]
119 pub vfs_root_mount: Option<Arc<super::sandbox_fs::SandboxFsLocalMount>>,
120}
121
122impl SandboxHandle for NspawnHandle {
123 fn as_any(&self) -> &dyn Any {
124 self
125 }
126}
127
128pub struct NspawnBackend {
134 config: NspawnConfig,
135 #[cfg(feature = "oci-image")]
140 rafs_store: Option<Arc<RafsStore>>,
141}
142
143impl NspawnBackend {
144 pub fn new(config: NspawnConfig) -> Self {
145 Self {
146 config,
147 #[cfg(feature = "oci-image")]
148 rafs_store: None,
149 }
150 }
151
152 #[cfg(feature = "oci-image")]
160 pub fn with_rafs_store(mut self, rafs_store: Option<Arc<RafsStore>>) -> Self {
161 self.rafs_store = rafs_store;
162 self
163 }
164
165 fn registry_is_available() -> bool {
169 which::which("systemd-nspawn").is_ok() && which::which("machinectl").is_ok()
170 }
171
172 fn build_nspawn_args(
179 &self,
180 sandbox: &PodSandbox,
181 machine_name: &str,
182 annotations: &HashMap<String, String>,
183 resources: Option<&LinuxContainerResources>,
184 vfs_root: Option<&Path>,
185 ) -> Vec<String> {
186 let mut args = Vec::new();
187
188 args.push(format!("--machine={machine_name}"));
191 match vfs_root {
192 Some(root) => args.push(format!("--directory={}", root.display())),
193 None => args.push("--directory=/".into()),
194 }
195 args.push("--tmpfs=/tmp".into());
196 args.push("--tmpfs=/run".into());
197
198 args.push(format!("--bind={}", sandbox.sandbox_path().display()));
200
201 if self.config.libtorch_path.exists() {
203 args.push(format!("--bind-ro={}", self.config.libtorch_path.display()));
204 }
205
206 if self.config.network_veth {
208 args.push("--network-veth".into());
209 }
210
211 let wants_gpu = annotations
213 .get("hyprstream.io/gpu")
214 .is_some_and(|v| v == "true");
215 if wants_gpu {
216 for dev in &self.config.gpu_devices {
217 args.push(format!("--bind={}", dev.display()));
218 }
219 }
220
221 args.push(format!("--setenv=HYPRSTREAM_INSTANCE={}", sandbox.id));
223 if !self.config.ld_library_path.is_empty() {
224 args.push(format!(
225 "--setenv=LD_LIBRARY_PATH={}",
226 self.config.ld_library_path
227 ));
228 }
229 args.push(format!(
230 "--setenv=LIBTORCH={}",
231 self.config.libtorch_path.display()
232 ));
233
234 if let Some(res) = resources {
236 if res.cpu_quota > 0 {
237 args.push(format!("--property=CPUQuota={}%", res.cpu_quota));
238 }
239 if res.memory_limit_in_bytes > 0 {
240 args.push(format!(
241 "--property=MemoryMax={}",
242 res.memory_limit_in_bytes
243 ));
244 }
245 }
246
247 args.push("--".into());
249 let hyprstream_bin = hyprstream_rpc::paths::executable_path()
250 .unwrap_or_else(|_| PathBuf::from("hyprstream"));
251 args.push(hyprstream_bin.to_string_lossy().into());
252 args.push("service".into());
253 args.push("start".into());
254 args.push("--all".into());
255 args.push("--foreground".into());
256
257 args
258 }
259
260 #[cfg(feature = "oci-image")]
267 const SUBJECT_ANNOTATION: &'static str = "hyprstream.io/subject";
268
269 #[cfg(feature = "oci-image")]
272 fn sandbox_subject(
273 sandbox: &PodSandbox,
274 annotations: &HashMap<String, String>,
275 ) -> hyprstream_vfs::Subject {
276 annotations
277 .get(Self::SUBJECT_ANNOTATION)
278 .filter(|s| !s.is_empty())
279 .map(hyprstream_vfs::Subject::new)
280 .unwrap_or_else(|| hyprstream_vfs::Subject::new(sandbox.id.clone()))
281 }
282
283 #[cfg(feature = "oci-image")]
295 async fn compose_and_mount_vfs_root(
296 &self,
297 sandbox: &PodSandbox,
298 image_id: &str,
299 subject: hyprstream_vfs::Subject,
300 ) -> Result<super::sandbox_fs::SandboxFsLocalMount> {
301 use super::sandbox_fs::SandboxFs;
302 use hyprstream_vfs::injected::SyntheticNode;
303
304 let rafs_store = self.rafs_store.clone().ok_or_else(|| {
305 WorkerError::SandboxCreationFailed(
306 "nspawn tenant-VFS root requested but no RAFS image store is configured".into(),
307 )
308 })?;
309
310 super::selection::require_fuse_mount_capability(self.backend_type())?;
313
314 let mountpoint = sandbox.sandbox_path().join(VFS_ROOT_DIR);
315 let rt = tokio::runtime::Handle::current();
316 let image_id = image_id.to_owned();
317 let sandbox_dir = sandbox.sandbox_path().clone();
318 let mp = mountpoint.clone();
319
320 tokio::task::spawn_blocking(move || {
322 let fs = SandboxFs::compose(
323 &rafs_store,
324 &image_id,
325 &sandbox_dir,
326 subject,
327 SyntheticNode::dir(),
330 SyntheticNode::dir(),
331 )?;
332 fs.serve_local_at(rt, mp)
333 })
334 .await
335 .map_err(|e| {
336 WorkerError::SandboxCreationFailed(format!("VFS compose/mount task join: {e}"))
337 })?
338 }
339
340 async fn discover_leader_pid(machine_name: &str) -> Option<u32> {
342 let output = tokio::process::Command::new("machinectl")
343 .args(["show", "-p", "Leader", "--value", machine_name])
344 .output()
345 .await
346 .ok()?;
347
348 if !output.status.success() {
349 return None;
350 }
351
352 let stdout = String::from_utf8_lossy(&output.stdout);
353 stdout.trim().parse().ok()
354 }
355
356 async fn wait_for_ready(&self, sandbox_id: &str) -> Result<()> {
358 let runtime_dir = hyprstream_rpc::paths::runtime_dir();
359 let discovery_sock = runtime_dir
362 .join("instances")
363 .join(sandbox_id)
364 .join("discovery.sock");
365
366 let deadline = tokio::time::Instant::now() + self.config.ready_timeout;
367
368 while tokio::time::Instant::now() < deadline {
369 if discovery_sock.exists() {
370 debug!(
371 sandbox_id,
372 path = %discovery_sock.display(),
373 "Sandbox ready (discovery socket exists)"
374 );
375 return Ok(());
376 }
377 tokio::time::sleep(self.config.ready_interval).await;
378 }
379
380 Err(WorkerError::SandboxTimeout {
381 operation: format!("nspawn readiness ({sandbox_id})"),
382 timeout_secs: self.config.ready_timeout.as_secs(),
383 })
384 }
385}
386
387impl std::fmt::Debug for NspawnBackend {
388 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
389 f.debug_struct("NspawnBackend")
390 .field("config", &self.config)
391 .finish()
392 }
393}
394
395#[async_trait]
396impl SandboxBackend for NspawnBackend {
397 fn backend_type(&self) -> &'static str {
398 "nspawn"
399 }
400
401 fn is_available(&self) -> bool {
402 which::which("systemd-nspawn").is_ok() && which::which("machinectl").is_ok()
403 }
404
405 async fn initialize(&self, _config: &PoolConfig) -> Result<()> {
406 if !self.is_available() {
407 return Err(WorkerError::ConfigError(
408 "systemd-nspawn or machinectl not found in PATH".into(),
409 ));
410 }
411 if !self.config.libtorch_path.exists() {
412 warn!(
413 path = %self.config.libtorch_path.display(),
414 "libtorch path does not exist — containers may fail to start"
415 );
416 }
417 Ok(())
418 }
419
420 async fn start(
421 &self,
422 sandbox: &mut PodSandbox,
423 config: &PodSandboxConfig,
424 pool_config: &PoolConfig,
425 annotations: &HashMap<String, String>,
426 ) -> Result<Arc<dyn SandboxHandle>> {
427 let machine_name = format!("hyprstream-{}", sandbox.id);
428
429 let sandbox_path = pool_config.runtime_dir.join(&sandbox.id);
431 tokio::fs::create_dir_all(&sandbox_path).await?;
432 sandbox.sandbox_path = sandbox_path.clone();
433
434 #[cfg(feature = "oci-image")]
441 let vfs_root_mount: Option<Arc<super::sandbox_fs::SandboxFsLocalMount>> =
442 if self.rafs_store.is_some() {
443 if let Some(image_id) = sandbox.image_id.clone() {
444 let subject = Self::sandbox_subject(sandbox, annotations);
445 let mount = self
446 .compose_and_mount_vfs_root(sandbox, &image_id, subject)
447 .await?;
448 info!(
449 sandbox_id = %sandbox.id,
450 mountpoint = %mount.mountpoint().display(),
451 "Mounted tenant VFS as nspawn container root (Model B, #715)"
452 );
453 Some(Arc::new(mount))
454 } else {
455 None
456 }
457 } else {
458 None
459 };
460
461 #[cfg(feature = "oci-image")]
462 let vfs_root_mountpoint: Option<PathBuf> = vfs_root_mount
463 .as_ref()
464 .map(|m| m.mountpoint().to_path_buf());
465 #[cfg(not(feature = "oci-image"))]
466 let vfs_root_mountpoint: Option<PathBuf> = None;
467
468 let nspawn_args = self.build_nspawn_args(
470 sandbox,
471 &machine_name,
472 annotations,
473 Some(&config.linux.resources),
474 vfs_root_mountpoint.as_deref(),
475 );
476
477 info!(
478 sandbox_id = %sandbox.id,
479 machine = %machine_name,
480 "Starting nspawn sandbox"
481 );
482
483 let mut cmd = tokio::process::Command::new("systemd-nspawn");
485 cmd.args(&nspawn_args);
486 cmd.stdin(std::process::Stdio::null())
487 .stdout(std::process::Stdio::null())
488 .stderr(std::process::Stdio::null());
489
490 let _child = cmd.spawn().map_err(|e| {
491 WorkerError::VmStartFailed(format!("failed to spawn systemd-nspawn: {e}"))
492 })?;
493
494 self.wait_for_ready(&sandbox.id).await?;
496
497 let leader_pid = Self::discover_leader_pid(&machine_name).await;
499
500 let handle = Arc::new(NspawnHandle {
501 sandbox_id: sandbox.id.clone(),
502 machine_name: machine_name.clone(),
503 leader_pid,
504 sandbox_path,
505 #[cfg(feature = "oci-image")]
506 vfs_root_mount,
507 });
508
509 info!(
510 sandbox_id = %sandbox.id,
511 machine = %machine_name,
512 leader_pid = ?leader_pid,
513 "Nspawn sandbox started"
514 );
515
516 Ok(handle)
517 }
518
519 async fn stop(&self, sandbox: &PodSandbox) -> Result<()> {
520 let machine_name = format!("hyprstream-{}", sandbox.id);
521 info!(sandbox_id = %sandbox.id, machine = %machine_name, "Stopping nspawn sandbox");
522
523 let status = tokio::process::Command::new("machinectl")
524 .args(["stop", &machine_name])
525 .status()
526 .await
527 .map_err(|e| {
528 WorkerError::VmStopFailed(format!("failed to run machinectl stop: {e}"))
529 })?;
530
531 if !status.success() {
532 warn!(
533 sandbox_id = %sandbox.id,
534 "machinectl stop returned non-zero (machine may already be stopped)"
535 );
536 }
537
538 Ok(())
539 }
540
541 async fn destroy(&self, sandbox: &PodSandbox) -> Result<()> {
542 let machine_name = format!("hyprstream-{}", sandbox.id);
543 info!(sandbox_id = %sandbox.id, machine = %machine_name, "Destroying nspawn sandbox");
544
545 let _ = tokio::process::Command::new("machinectl")
547 .args(["terminate", &machine_name])
548 .status()
549 .await;
550
551 let sandbox_path = sandbox.sandbox_path();
553 if sandbox_path.exists() {
554 if let Err(e) = tokio::fs::remove_dir_all(sandbox_path).await {
555 warn!(
556 sandbox_id = %sandbox.id,
557 error = %e,
558 "Failed to remove sandbox directory"
559 );
560 }
561 }
562
563 let instance_dir = hyprstream_rpc::paths::runtime_dir()
565 .join("instances")
566 .join(&sandbox.id);
567 if instance_dir.exists() {
568 let _ = tokio::fs::remove_dir_all(&instance_dir).await;
569 }
570
571 Ok(())
572 }
573
574 async fn reset(&self, _sandbox: &mut PodSandbox) -> Result<bool> {
575 Ok(false)
577 }
578
579 async fn get_pids(&self, sandbox: &PodSandbox) -> Result<Vec<u32>> {
580 let machine_name = format!("hyprstream-{}", sandbox.id);
581 match Self::discover_leader_pid(&machine_name).await {
582 Some(pid) => Ok(vec![pid]),
583 None => Ok(Vec::new()),
584 }
585 }
586
587 async fn deliver_namespace(
588 &self,
589 sandbox: &PodSandbox,
590 _namespace: hyprstream_vfs::Namespace,
591 _subject: hyprstream_vfs::Subject,
592 transport: super::backend::NamespaceTransport,
593 ) -> Result<super::backend::NamespaceDelivery> {
594 use super::backend::{NamespaceDelivery, NamespaceTransport};
595
596 let NamespaceTransport::BindMount { target } = transport else {
607 return Err(WorkerError::Unsupported(format!(
608 "nspawn backend only supports the BindMount namespace transport (needs #653's \
609 local-FUSE-mount serve mode for a real VirtioFs/HostImports equivalent), got {transport:?}"
610 )));
611 };
612
613 if !target.exists() {
614 return Err(WorkerError::SandboxCreationFailed(format!(
615 "deliver_namespace: bind-mount source {} does not exist",
616 target.display()
617 )));
618 }
619
620 let machine_name = format!("hyprstream-{}", sandbox.id);
621 let target_str = target.to_string_lossy().into_owned();
622
623 let status = tokio::process::Command::new("machinectl")
627 .args(["bind", "--mkdir", &machine_name, &target_str])
628 .status()
629 .await
630 .map_err(|e| {
631 WorkerError::SandboxCreationFailed(format!("failed to run machinectl bind: {e}"))
632 })?;
633
634 if !status.success() {
635 return Err(WorkerError::SandboxCreationFailed(format!(
636 "machinectl bind {machine_name} {target_str} failed"
637 )));
638 }
639
640 Ok(NamespaceDelivery::BindMount { target })
641 }
642
643 fn supports_exec(&self) -> bool {
644 true
645 }
646
647 async fn exec_sync(
648 &self,
649 sandbox: &PodSandbox,
650 command: &[String],
651 timeout_secs: u64,
652 ) -> Result<(i32, Vec<u8>, Vec<u8>)> {
653 let machine_name = format!("hyprstream-{}", sandbox.id);
654
655 let mut args = vec!["shell".to_owned(), machine_name, "--".to_owned()];
656 args.extend_from_slice(command);
657
658 let output = tokio::time::timeout(
659 Duration::from_secs(timeout_secs),
660 tokio::process::Command::new("machinectl")
661 .args(&args)
662 .output(),
663 )
664 .await
665 .map_err(|_| WorkerError::SandboxTimeout {
666 operation: format!("exec_sync in {}", sandbox.id),
667 timeout_secs,
668 })?
669 .map_err(|e| WorkerError::ExecFailed(format!("machinectl shell failed: {e}")))?;
670
671 let exit_code = output.status.code().unwrap_or(-1);
672 Ok((exit_code, output.stdout, output.stderr))
673 }
674
675 async fn update_resources(
676 &self,
677 sandbox: &PodSandbox,
678 resources: &LinuxContainerResources,
679 ) -> Result<()> {
680 let machine_name = format!("hyprstream-{}", sandbox.id);
681
682 let scope = format!("machine-{machine_name}.scope");
684
685 if resources.cpu_quota > 0 {
686 let _ = tokio::process::Command::new("systemctl")
687 .args([
688 "set-property",
689 &scope,
690 &format!("CPUQuota={}%", resources.cpu_quota),
691 ])
692 .status()
693 .await;
694 }
695 if resources.memory_limit_in_bytes > 0 {
696 let _ = tokio::process::Command::new("systemctl")
697 .args([
698 "set-property",
699 &scope,
700 &format!("MemoryMax={}", resources.memory_limit_in_bytes),
701 ])
702 .status()
703 .await;
704 }
705
706 Ok(())
707 }
708}
709
710inventory::submit! {
720 crate::runtime::selection::BackendRegistration {
721 name: "nspawn",
722 priority: 10,
723 auto_selectable: true,
725 injects_9p_socket: true,
731 mounts_fuse_vfs: true,
737 is_available: NspawnBackend::registry_is_available,
738 construct: |_ctx| {
739 #[cfg(feature = "oci-image")]
740 let backend = NspawnBackend::new(NspawnConfig::default())
741 .with_rafs_store(Some(std::sync::Arc::clone(&_ctx.rafs_store)));
742 #[cfg(not(feature = "oci-image"))]
743 let backend = NspawnBackend::new(NspawnConfig::default());
744 Ok(std::sync::Arc::new(backend) as std::sync::Arc<dyn crate::runtime::SandboxBackend>)
745 },
746 }
747}
748
749#[cfg(test)]
750#[allow(clippy::unwrap_used)]
751mod tests {
752 use super::*;
753
754 #[test]
755 fn test_nspawn_config_default() {
756 let config = NspawnConfig::default();
757 assert!(config.network_veth);
758 assert_eq!(config.ready_timeout, Duration::from_secs(10));
759 assert_eq!(config.ready_interval, Duration::from_millis(100));
760 }
761
762 #[test]
763 fn test_nspawn_handle_as_any() {
764 let handle = NspawnHandle {
765 sandbox_id: "test-123".into(),
766 machine_name: "hyprstream-test-123".into(),
767 leader_pid: Some(42),
768 sandbox_path: PathBuf::from("/tmp/test"),
769 #[cfg(feature = "oci-image")]
770 vfs_root_mount: None,
771 };
772
773 let handle: Arc<dyn SandboxHandle> = Arc::new(handle);
774 let downcast = handle.as_any().downcast_ref::<NspawnHandle>();
775 assert!(downcast.is_some());
776 assert_eq!(downcast.unwrap().sandbox_id, "test-123");
777 assert_eq!(downcast.unwrap().leader_pid, Some(42));
778 }
779
780 #[test]
781 fn test_detect_gpu_devices() {
782 let devices = detect_gpu_devices();
784 for d in &devices {
785 assert!(
786 d.to_string_lossy().contains("card") || d.to_string_lossy().contains("renderD")
787 );
788 }
789 }
790
791 #[test]
792 fn test_backend_type() {
793 let backend = NspawnBackend::new(NspawnConfig::default());
794 assert_eq!(backend.backend_type(), "nspawn");
795 }
796
797 fn test_sandbox(sandbox_path: PathBuf) -> PodSandbox {
802 let config = PodSandboxConfig::default();
803 PodSandbox::new("test-sandbox-001".to_owned(), &config, sandbox_path)
804 }
805
806 #[tokio::test]
807 async fn test_deliver_namespace_rejects_non_bindmount_transport() {
808 use super::super::backend::NamespaceTransport;
809
810 let backend = NspawnBackend::new(NspawnConfig::default());
811 let sandbox = test_sandbox(PathBuf::from("/tmp/nonexistent-sandbox"));
812
813 let result = backend
814 .deliver_namespace(
815 &sandbox,
816 hyprstream_vfs::Namespace::new(),
817 hyprstream_vfs::Subject::new("test"),
818 NamespaceTransport::HostImports,
819 )
820 .await;
821
822 match result {
823 Err(WorkerError::Unsupported(msg)) => {
824 assert!(msg.contains("BindMount"), "unexpected message: {msg}");
825 }
826 other => panic!("expected Unsupported error, got: {other:?}"),
827 }
828 }
829
830 #[tokio::test]
831 async fn test_deliver_namespace_rejects_missing_bindmount_source() {
832 use super::super::backend::NamespaceTransport;
833
834 let backend = NspawnBackend::new(NspawnConfig::default());
835 let sandbox = test_sandbox(PathBuf::from("/tmp/nonexistent-sandbox"));
836
837 let result = backend
840 .deliver_namespace(
841 &sandbox,
842 hyprstream_vfs::Namespace::new(),
843 hyprstream_vfs::Subject::new("test"),
844 NamespaceTransport::BindMount {
845 target: PathBuf::from("/nonexistent/path/for/635/test"),
846 },
847 )
848 .await;
849
850 assert!(result.is_err());
851 match result.unwrap_err() {
852 WorkerError::SandboxCreationFailed(msg) => {
853 assert!(msg.contains("does not exist"), "unexpected message: {msg}");
854 }
855 other => panic!("expected SandboxCreationFailed, got: {other:?}"),
856 }
857 }
858
859 #[test]
860 fn test_is_available() {
861 let backend = NspawnBackend::new(NspawnConfig::default());
862 let _available = backend.is_available();
864 }
865}