Skip to main content

hyprstream_workers/runtime/
kata_backend.rs

1//! KataBackend — Kata Containers VM-based sandbox isolation
2//!
3//! Implements `SandboxBackend` using Kata's `Hypervisor` trait for full VM
4//! isolation.  Supports Cloud Hypervisor and Dragonball hypervisors.
5
6use std::any::Any;
7use std::collections::HashMap;
8use std::path::{Path, PathBuf};
9use std::sync::Arc;
10
11use async_trait::async_trait;
12use kata_hypervisor::ch::CloudHypervisor;
13use kata_hypervisor::device::DeviceType;
14#[cfg(feature = "dragonball")]
15use kata_hypervisor::dragonball::Dragonball;
16use kata_hypervisor::{Hypervisor, ShareFsConfig, ShareFsDevice};
17use kata_types::config::hypervisor::{Hypervisor as HypervisorConfig, RootlessUser};
18use kata_types::rootless;
19use protocols::{agent, oci};
20
21use crate::config::{HypervisorType, ImageConfig, PoolConfig};
22use crate::error::{Result, WorkerError};
23use crate::image::RafsStore;
24
25use super::backend::{SandboxBackend, SandboxHandle};
26use super::kata_agent::{AgentAddress, ContainerRootfs, KataAgentClient};
27use super::sandbox::PodSandbox;
28use super::sandbox_fs::{SandboxFs, SandboxFsServer, VFS_SOCKET_NAME};
29use hyprstream_rpc_std::worker_client::{
30    CpuUsage, LinuxContainerResources, MemoryUsage, PodSandboxConfig,
31};
32use hyprstream_vfs::{Mount, Subject, SyntheticNode};
33
34/// virtio-fs mount tag under which `start()` attaches the composed per-sandbox
35/// tenant VFS to Cloud Hypervisor. The guest's kata-agent mounts the share by
36/// this tag (see [`kata_agent::ContainerRootfs`](super::kata_agent::ContainerRootfs))
37/// as the container's rootfs (#721). A single, stable tag is used because
38/// exactly one VFS share is attached per sandbox.
39const ROOTFS_MOUNT_TAG: &str = "hyprstream-vfs";
40
41/// Derive the user-writable runtime base directory that rootless Kata must use
42/// for its Cloud Hypervisor virtio-fs (`ShareFs`) jailer root (#743).
43///
44/// **Why this exists / what actually drives the jailer base.** In rootless mode
45/// kata-hypervisor 4.0.0 builds the sharefs jailer root as
46/// `get_rootless_symlink_sandbox_jailer_root(id)` →
47/// `get_rootless_symlink_sandbox_path(id)` → `kata_types::build_path(id)` →
48/// `rootless_dir()`, i.e. its base is **`$XDG_RUNTIME_DIR`**, *not* `KATA_PATH`.
49/// `KATA_PATH` is a compile-time `const` (`"/run/kata"`, root-owned) with **no
50/// env override**, so `std::env::set_var("KATA_PATH", …)` would have zero effect
51/// on the vendored jailer path. The real seam is `XDG_RUNTIME_DIR`, which
52/// `kata_types::rootless::rootless_dir()` reads once via a `lazy_static` and
53/// `.unwrap()`s — so if it is unset the rootless jailer code panics, and if it
54/// points at a non-writable dir (or the caller lacks `mkdir` on it) VM start
55/// fails with `failed to create rootless sharefs symlink jailer root dir:
56/// Permission denied`.
57///
58/// This returns a directory we can `mkdir_all` and own, to be assigned to
59/// `XDG_RUNTIME_DIR` before any VM/hypervisor config is built (`rootless_dir()`
60/// is a one-shot `lazy_static`, so it must be set first).
61///
62/// Pure/deterministic so it can be unit-tested:
63/// - `euid == 0` (rootful) → `None`: behaviour is left unchanged.
64/// - otherwise, prefer `<XDG_RUNTIME_DIR>/kata` (user-owned, e.g.
65///   `/run/user/1000/kata`); if `XDG_RUNTIME_DIR` is unset/empty, fall back to
66///   `<TMPDIR or /tmp>/kata-<uid>`.
67fn rootless_kata_runtime_dir(
68    euid: u32,
69    xdg_runtime_dir: Option<&str>,
70    tmpdir: Option<&str>,
71) -> Option<PathBuf> {
72    if euid == 0 {
73        return None;
74    }
75    let base = match xdg_runtime_dir.map(str::trim).filter(|s| !s.is_empty()) {
76        Some(xdg) => PathBuf::from(xdg).join("kata"),
77        None => {
78            let tmp = tmpdir
79                .map(str::trim)
80                .filter(|s| !s.is_empty())
81                .unwrap_or("/tmp");
82            PathBuf::from(tmp).join(format!("kata-{euid}"))
83        }
84    };
85    Some(base)
86}
87
88// ─────────────────────────────────────────────────────────────────────────────
89// Handle
90// ─────────────────────────────────────────────────────────────────────────────
91
92/// Kata-specific state stored on each `PodSandbox`.
93pub struct KataHandle {
94    /// The Kata hypervisor handle for VM lifecycle management.
95    pub hypervisor: Arc<dyn Hypervisor>,
96    /// Path to the VM API socket.
97    pub api_socket: PathBuf,
98    /// Path to the virtio-fs socket the guest mounts. Served by the composed
99    /// VFS namespace (`vfs_server`).
100    pub virtiofs_socket: Option<PathBuf>,
101    /// Per-sandbox composed VFS server (FS-D, #365). When set, the guest's
102    /// filesystem is the hyprstream VFS (rootfs + injected mounts) served by
103    /// `hyprstream-vfs-server`. Held for the VM's lifetime so the serving thread
104    /// + injected registries outlive the guest; dropped on sandbox teardown.
105    pub vfs_server: Option<SandboxFsServer>,
106    /// Per-sandbox tenant-VFS **9P-over-vsock** channel (V2, #731). A second
107    /// vsock port (`VFS_9P_VSOCK_PORT`), distinct from the kata-agent's 1024,
108    /// serving the same Subject-scoped tenant Mount as native 9P so an in-guest
109    /// 9P client (V3, #732) can dial the host. Held for the VM's lifetime; its
110    /// [`Drop`] aborts the serve task and removes the host UDS on sandbox
111    /// teardown. `None` when the sandbox has no tenant VFS (no `image_id`) or the
112    /// channel could not be stood up.
113    pub vfs_9p: Option<Vfs9pVsockServer>,
114    /// virtio-fs mount tag of the composed tenant VFS share attached to this
115    /// sandbox's VM, when one was attached (#721). `exec_sync` passes this to
116    /// the kata-agent so `CreateContainer` mounts the share, by tag, as the
117    /// container's rootfs. `None` when the VM booted its own rootfs with no
118    /// tenant VFS (e.g. `sandbox.image_id` unset).
119    ///
120    /// Interior-mutable (`parking_lot::Mutex`) because the handle is shared
121    /// (`Arc`) once the sandbox has started: the post-start `deliver_namespace`
122    /// path (#742) composes+attaches a tenant VFS over virtio-fs *after*
123    /// construction and must record the resulting rootfs tag through a shared
124    /// `&self` — it has no `&mut KataHandle`. Both the boot path
125    /// (`start()`) and the delivery path write through
126    /// [`record_rootfs_mount_tag`](KataHandle::record_rootfs_mount_tag); readers
127    /// use [`rootfs_mount_tag`](KataHandle::rootfs_mount_tag). `parking_lot`
128    /// (not `std::sync`) so the lock is infallible — no poisoning, so no
129    /// `unwrap`/`expect` at the (`-D clippy::unwrap_used`) call sites.
130    pub rootfs_mount_tag: parking_lot::Mutex<Option<String>>,
131    /// kata-agent ttrpc/vsock client (#344), connected lazily on first
132    /// `exec_sync` call and cached for subsequent calls. `None` until then,
133    /// or if the guest agent connection could not be established (e.g. the
134    /// VM image has no `kata-agent`, or it hasn't finished booting yet).
135    ///
136    /// Guarded by a `tokio::sync::Mutex` rather than stored as a plain
137    /// `Option` because `exec_sync` takes `&PodSandbox` (shared ref) and
138    /// must be able to lazily populate the connection on first use.
139    pub agent: tokio::sync::Mutex<Option<Arc<KataAgentClient>>>,
140}
141
142impl KataHandle {
143    /// Read the recorded container rootfs virtio-fs mount tag, if any.
144    ///
145    /// Set either at boot (`start()` → the `ROOTFS_MOUNT_TAG` share) or by a
146    /// post-start `deliver_namespace` (#742). `exec_sync` reads this to tell the
147    /// kata-agent which virtio-fs share to mount, by tag, as the container
148    /// rootfs. Returns an owned clone so the lock is not held across the caller's
149    /// `.await`s.
150    pub fn rootfs_mount_tag(&self) -> Option<String> {
151        self.rootfs_mount_tag.lock().clone()
152    }
153
154    /// Record the container rootfs virtio-fs mount tag through the shared
155    /// (`Arc`) handle.
156    ///
157    /// Called from both the boot path and — the point of #742 — the post-start
158    /// `deliver_namespace` path, which composes+attaches the tenant VFS after
159    /// the handle is already `Arc`-shared and so cannot take `&mut`. A later
160    /// `exec_sync` then mounts the delivered share as the container rootfs.
161    /// Last write wins (a re-delivery replaces the prior tag).
162    pub fn record_rootfs_mount_tag(&self, tag: String) {
163        *self.rootfs_mount_tag.lock() = Some(tag);
164    }
165}
166
167impl std::fmt::Debug for KataHandle {
168    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
169        f.debug_struct("KataHandle")
170            .field("api_socket", &self.api_socket)
171            .field("virtiofs_socket", &self.virtiofs_socket)
172            .field(
173                "vfs_socket",
174                &self.vfs_server.as_ref().map(SandboxFsServer::socket_path),
175            )
176            .field(
177                "vfs_9p_socket",
178                &self.vfs_9p.as_ref().map(Vfs9pVsockServer::socket_path),
179            )
180            .field("rootfs_mount_tag", &self.rootfs_mount_tag())
181            .finish_non_exhaustive()
182    }
183}
184
185impl SandboxHandle for KataHandle {
186    fn as_any(&self) -> &dyn Any {
187        self
188    }
189}
190
191// ─────────────────────────────────────────────────────────────────────────────
192// Tenant-VFS 9P-over-vsock channel (V2, #731)
193// ─────────────────────────────────────────────────────────────────────────────
194
195/// Guest vsock port the tenant-VFS 9P server listens on — the port an in-guest
196/// 9P client (V3, #732) dials to reach the host VFS over native 9P.
197///
198/// **564** is the IANA-registered port for the Plan 9 file service (9P); reusing
199/// it as the vsock port keeps "9P lives on 564" true across transports and makes
200/// the channel self-documenting. It is deliberately distinct from the kata-agent's
201/// ttrpc port **1024** ([`KATA_AGENT_VSOCK_PORT`](super::kata_agent::KATA_AGENT_VSOCK_PORT)),
202/// so the two vsock channels never collide.
203pub const VFS_9P_VSOCK_PORT: u32 = 564;
204
205/// Derive the host-side Unix socket a Cloud-Hypervisor **hybrid-vsock** guest
206/// reaches by dialing vsock port `port`.
207///
208/// CH (like Firecracker, whose hybrid-vsock design it inherits) routes a
209/// *guest-initiated* connection to vsock port `N` to a host Unix socket named
210/// `<vsock-uds>_<N>`, where `<vsock-uds>` is the base UDS path CH is configured
211/// with (the same path the host writes `connect <port>\n` to for the reverse,
212/// host-initiated direction — e.g. the kata-agent on 1024). The host service
213/// must be listening on `<vsock-uds>_<N>` before the guest dials.
214fn vfs_9p_vsock_uds(vsock_base: &str, port: u32) -> PathBuf {
215    PathBuf::from(format!("{vsock_base}_{port}"))
216}
217
218/// A running per-sandbox tenant-VFS 9P-over-vsock server (V2, #731).
219///
220/// RAII handle mirroring `wanix_workload::Injected9pServer`: owns the host UDS
221/// path and the background 9P accept task. [`Drop`] aborts the task and removes
222/// the socket, so the channel is torn down with the sandbox.
223pub struct Vfs9pVsockServer {
224    /// Host-side UDS the guest's vsock port maps to (`<vsock-uds>_<port>`).
225    socket_path: PathBuf,
226    /// The background 9P serve task ([`hyprstream_9p::serve_mount_vsock_raw`]).
227    task: tokio::task::JoinHandle<()>,
228}
229
230impl Vfs9pVsockServer {
231    /// Host-side UDS path the tenant-VFS 9P is served on.
232    pub fn socket_path(&self) -> &Path {
233        &self.socket_path
234    }
235
236    /// Abort the serve task and remove the host UDS now (idempotent; `Drop` also
237    /// does this).
238    pub fn shutdown(&self) {
239        self.task.abort();
240        if let Err(e) = std::fs::remove_file(&self.socket_path) {
241            if e.kind() != std::io::ErrorKind::NotFound {
242                tracing::debug!(
243                    socket = %self.socket_path.display(),
244                    error = %e,
245                    "remove tenant-VFS 9P vsock socket"
246                );
247            }
248        }
249    }
250}
251
252impl Drop for Vfs9pVsockServer {
253    fn drop(&mut self) {
254        self.shutdown();
255    }
256}
257
258impl std::fmt::Debug for Vfs9pVsockServer {
259    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
260        f.debug_struct("Vfs9pVsockServer")
261            .field("socket_path", &self.socket_path)
262            .finish()
263    }
264}
265
266// ─────────────────────────────────────────────────────────────────────────────
267// Backend
268// ─────────────────────────────────────────────────────────────────────────────
269
270/// Kata Containers sandbox backend.
271pub struct KataBackend {
272    image_config: ImageConfig,
273    rafs_store: Arc<RafsStore>,
274    ninep_decider: Arc<dyn hyprstream_9p::AccessDecider>,
275    ninep_monitor: Option<Arc<hyprstream_9p::ReferenceMonitor>>,
276}
277
278impl KataBackend {
279    pub fn new(
280        image_config: ImageConfig,
281        rafs_store: Arc<RafsStore>,
282        ninep_decider: Arc<dyn hyprstream_9p::AccessDecider>,
283        ninep_monitor: Option<Arc<hyprstream_9p::ReferenceMonitor>>,
284    ) -> Self {
285        Self {
286            image_config,
287            rafs_store,
288            ninep_decider,
289            ninep_monitor,
290        }
291    }
292
293    /// Runtime prerequisite probe for the backend registry: a `cloud-hypervisor`
294    /// binary must be on PATH. Mirrors the instance-level
295    /// [`SandboxBackend::is_available`] check.
296    fn registry_is_available() -> bool {
297        which::which("cloud-hypervisor").is_ok()
298    }
299
300    /// Build a `HypervisorConfig` from `PoolConfig`.
301    fn build_hypervisor_config(pool_config: &PoolConfig) -> HypervisorConfig {
302        let mut config = HypervisorConfig::default();
303
304        config.path = if pool_config.hypervisor_path.as_os_str().is_empty() {
305            match pool_config.hypervisor {
306                HypervisorType::CloudHypervisor => which::which("cloud-hypervisor")
307                    .map(|p| p.to_string_lossy().to_string())
308                    .unwrap_or_else(|e| {
309                        tracing::warn!("Failed to find cloud-hypervisor in PATH: {}", e);
310                        "cloud-hypervisor".to_owned()
311                    }),
312                #[cfg(feature = "dragonball")]
313                HypervisorType::Dragonball => String::new(),
314            }
315        } else {
316            pool_config.hypervisor_path.to_string_lossy().to_string()
317        };
318
319        config.boot_info.kernel = pool_config.kernel_path.to_string_lossy().to_string();
320        config.boot_info.image = pool_config.vm_image.to_string_lossy().to_string();
321        config.cpu_info.default_vcpus = pool_config.vm_cpus as f32;
322        config.cpu_info.default_maxvcpus = pool_config.vm_cpus;
323        config.memory_info.default_memory =
324            u32::try_from(pool_config.vm_memory_mb).unwrap_or(u32::MAX);
325
326        if rootless::is_rootless() {
327            let uid = nix::unistd::getuid().as_raw();
328            let gid = nix::unistd::getgid().as_raw();
329            let username = std::env::var("USER").unwrap_or_else(|_| "user".to_owned());
330            let groups = nix::unistd::getgroups()
331                .map(|gs| gs.into_iter().map(nix::unistd::Gid::as_raw).collect())
332                .unwrap_or_else(|_| vec![gid]);
333
334            config.security_info.rootless = true;
335            config.security_info.rootless_user = Some(RootlessUser {
336                uid,
337                gid,
338                groups,
339                user_name: username,
340            });
341
342            tracing::debug!(uid, gid, "Configured rootless user for hypervisor");
343        }
344
345        config
346    }
347
348    /// Create a hypervisor instance for the given sandbox.
349    async fn create_hypervisor(
350        pool_config: &PoolConfig,
351        sandbox: &PodSandbox,
352        api_socket: &Path,
353        virtiofs_socket: &Path,
354    ) -> Result<Arc<dyn Hypervisor>> {
355        let config = Self::build_hypervisor_config(pool_config);
356
357        let hypervisor: Arc<dyn Hypervisor> = match pool_config.hypervisor {
358            HypervisorType::CloudHypervisor => {
359                let ch = CloudHypervisor::new();
360                ch.set_hypervisor_config(config).await;
361                tracing::debug!(
362                    sandbox_id = %sandbox.id,
363                    api_socket = %api_socket.display(),
364                    virtiofs_socket = %virtiofs_socket.display(),
365                    "Created Cloud Hypervisor instance"
366                );
367                Arc::new(ch)
368            }
369            #[cfg(feature = "dragonball")]
370            HypervisorType::Dragonball => {
371                let db = Dragonball::new();
372                db.set_hypervisor_config(config).await;
373                tracing::debug!(
374                    sandbox_id = %sandbox.id,
375                    "Created Dragonball hypervisor instance"
376                );
377                Arc::new(db)
378            }
379        };
380
381        Ok(hypervisor)
382    }
383
384    /// Attach a vhost-user-fs share to the hypervisor as a ShareFs device.
385    ///
386    /// This closes the gap noted in the FS spike (the virtiofs socket was created
387    /// but never wired into the VM): it adds the existing vhost-user-fs socket as
388    /// a Cloud Hypervisor `ShareFs` (virtio-fs) device via the **embedded**
389    /// runtime-rs hypervisor crate (#343 = embed), so the guest can mount the
390    /// share. The same path is used whether the socket is served by `nydusd`
391    /// (RAFS image rootfs) or by hyprstream's own `hyprstream-vfs-server`
392    /// (the VFS down-adapter).
393    ///
394    /// # Ordering
395    ///
396    /// Must be called **after `prepare_vm`** (which sets the hypervisor's
397    /// `vm_path`) and **before `start_vm`**. When the VM is not yet running, the
398    /// CH backend queues the device into its `pending_devices` and folds it into
399    /// the `VmConfig.fs` at boot — no virtiofsd is spawned by the crate; CH
400    /// connects to the socket we hand it. An **absolute** `sock_path` is used so
401    /// CH does not re-root it under the sandbox dir.
402    ///
403    /// `fs_type` must be the literal `"virtio-fs"` or the CH backend rejects the
404    /// device.
405    async fn attach_share_fs(
406        hypervisor: &Arc<dyn Hypervisor>,
407        sandbox_id: &str,
408        socket_path: &Path,
409        mount_tag: &str,
410    ) -> Result<()> {
411        let sock_path = socket_path.to_str().ok_or_else(|| {
412            WorkerError::SandboxCreationFailed("virtiofs socket path contains invalid UTF-8".into())
413        })?;
414
415        let config = ShareFsConfig {
416            // CH never reads host_shared_path/options/mount_config — the daemon
417            // that serves `sock_path` owns the exported tree. Only the socket,
418            // tag, type, and queue geometry matter for the boot-time attach.
419            fs_type: "virtio-fs".to_owned(),
420            sock_path: sock_path.to_owned(),
421            mount_tag: mount_tag.to_owned(),
422            queue_num: 1,
423            queue_size: 1024,
424            ..Default::default()
425        };
426        let device = ShareFsDevice::new(&format!("share-fs-{sandbox_id}"), &config);
427
428        hypervisor
429            .add_device(DeviceType::ShareFs(device))
430            .await
431            .map_err(|e| {
432                WorkerError::VmStartFailed(format!("failed to attach vhost-user-fs share: {e}"))
433            })?;
434
435        tracing::info!(
436            sandbox_id = %sandbox_id,
437            socket = %socket_path.display(),
438            mount_tag = %mount_tag,
439            "Attached vhost-user-fs ShareFs device to hypervisor"
440        );
441        Ok(())
442    }
443
444    /// Annotation key carrying the sandbox's policy principal (the [`Subject`]
445    /// at the VFS Mount boundary, #353/#319/#328). When absent, the sandbox id is
446    /// used as a stable per-sandbox principal so isolation still holds.
447    const SUBJECT_ANNOTATION: &'static str = "hyprstream.io/subject";
448
449    /// Resolve the [`Subject`] this sandbox's VFS is served under.
450    fn sandbox_subject(sandbox: &PodSandbox, annotations: &HashMap<String, String>) -> Subject {
451        annotations
452            .get(Self::SUBJECT_ANNOTATION)
453            .filter(|s| !s.is_empty())
454            .map(Subject::new)
455            .unwrap_or_else(|| Subject::new(sandbox.id.clone()))
456    }
457
458    /// Compose the per-sandbox VFS namespace (rootfs + injected mounts) and serve
459    /// it over a per-sandbox Unix socket (FS-D, #365).
460    ///
461    /// Returns the running [`SandboxFsServer`] whose socket the caller attaches to
462    /// CH. The namespace is forked per sandbox and bound to `subject`, so it
463    /// exposes only this sandbox's rootfs + injected paths — never another
464    /// sandbox's tree.
465    ///
466    /// `serve` blocks, so it runs on a dedicated thread inside `serve_on`; we
467    /// pass the current tokio runtime handle for the down-adapter's async→sync
468    /// bridge. Compose is CPU/IO-bound (RAFS load), so we run it on a blocking
469    /// thread to avoid stalling the async runtime.
470    ///
471    /// Also returns a clone of the tenant **root** [`Mount`] so the *same*
472    /// Subject-scoped Mount can be re-served over the guest's 9P-over-vsock
473    /// channel (#731) — the virtio-fs namespace and the 9P channel share one Arc,
474    /// so there is no second writable-upper CoW layer and no new principal.
475    async fn compose_and_serve_vfs(
476        &self,
477        sandbox: &PodSandbox,
478        image_id: &str,
479        subject: Subject,
480    ) -> Result<(SandboxFsServer, Arc<dyn Mount>)> {
481        let socket_path = sandbox.sandbox_path().join(VFS_SOCKET_NAME);
482        let rt = tokio::runtime::Handle::current();
483        let rafs_store = self.rafs_store.clone();
484        let image_id = image_id.to_owned();
485        let sandbox_dir = sandbox.sandbox_path().clone();
486
487        // RAFS load + overlay setup is blocking; do it off the async executor.
488        tokio::task::spawn_blocking(move || {
489            let fs = SandboxFs::compose(
490                &rafs_store,
491                &image_id,
492                &sandbox_dir,
493                subject,
494                // Models / deltas injected listings: empty mount points for now
495                // (the worker runtime populates them per tenant). The mount
496                // points exist so the guest sees /models and /deltas.
497                SyntheticNode::dir(),
498                SyntheticNode::dir(),
499            )?;
500            // Grab the root export root before `serve_on` consumes `fs`.
501            let root = fs.root_mount();
502            let server = fs.serve_on(socket_path, rt)?;
503            Ok((server, root))
504        })
505        .await
506        .map_err(|e| WorkerError::SandboxCreationFailed(format!("VFS compose task join: {e}")))?
507    }
508
509    /// Generate cloud-init ISO for a sandbox.
510    async fn generate_cloud_init_iso(sandbox: &PodSandbox, iso_path: &Path) -> Result<()> {
511        let sandbox_runtime_dir = iso_path
512            .parent()
513            .ok_or_else(|| anyhow::anyhow!("ISO path has no parent directory"))?;
514
515        let hostname = if sandbox.metadata.name.is_empty() {
516            sandbox.id.clone()
517        } else {
518            sandbox.metadata.name.clone()
519        };
520
521        let user_data = format!(
522            "#cloud-config\nhostname: {hostname}\nusers:\n  - name: root\n    lock_passwd: false\nwrite_files:\n  - path: /etc/sandbox-id\n    content: {id}\nruncmd:\n  - echo \"Sandbox {id} initialized\"\n",
523            hostname = hostname,
524            id = sandbox.id,
525        );
526
527        let user_data_path = sandbox_runtime_dir.join("user-data");
528        tokio::fs::write(&user_data_path, user_data).await?;
529
530        let meta_data = format!(
531            "instance-id: {id}\nlocal-hostname: {hostname}\n",
532            id = sandbox.id,
533            hostname = hostname,
534        );
535
536        let meta_data_path = sandbox_runtime_dir.join("meta-data");
537        tokio::fs::write(&meta_data_path, meta_data).await?;
538
539        let iso_path_str = iso_path.to_str().ok_or_else(|| {
540            WorkerError::CloudInitFailed("ISO path contains invalid UTF-8".into())
541        })?;
542        let user_data_str = user_data_path.to_str().ok_or_else(|| {
543            WorkerError::CloudInitFailed("User data path contains invalid UTF-8".into())
544        })?;
545        let meta_data_str = meta_data_path.to_str().ok_or_else(|| {
546            WorkerError::CloudInitFailed("Meta data path contains invalid UTF-8".into())
547        })?;
548
549        let status = tokio::process::Command::new("genisoimage")
550            .args([
551                "-output",
552                iso_path_str,
553                "-volid",
554                "cidata",
555                "-joliet",
556                "-rock",
557                user_data_str,
558                meta_data_str,
559            ])
560            .status()
561            .await
562            .map_err(|e| WorkerError::CloudInitFailed(format!("failed to run genisoimage: {e}")))?;
563
564        if !status.success() {
565            return Err(WorkerError::CloudInitFailed("genisoimage failed".into()));
566        }
567
568        Ok(())
569    }
570}
571
572impl std::fmt::Debug for KataBackend {
573    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
574        f.debug_struct("KataBackend")
575            .field("image_config", &self.image_config)
576            .finish()
577    }
578}
579
580#[async_trait]
581impl SandboxBackend for KataBackend {
582    fn backend_type(&self) -> &'static str {
583        "kata"
584    }
585
586    fn is_available(&self) -> bool {
587        which::which("cloud-hypervisor").is_ok()
588    }
589
590    async fn initialize(&self, _config: &PoolConfig) -> Result<()> {
591        let euid = nix::unistd::geteuid();
592        if !euid.is_root() {
593            // Rootless: kata-hypervisor 4.0.0 derives the Cloud Hypervisor
594            // sharefs jailer root from `$XDG_RUNTIME_DIR` (via
595            // `kata_types::build_path` → `rootless_dir()`), NOT from `KATA_PATH`
596            // (a compile-time const "/run/kata" with no env override). A rootless
597            // user cannot `mkdir /run/kata`, and if `XDG_RUNTIME_DIR` is unset
598            // `rootless_dir()` panics on `.unwrap()`. Pin `XDG_RUNTIME_DIR` at a
599            // directory we create and own, before `set_rootless` and before any
600            // VM/hypervisor config is built (`rootless_dir()` is a one-shot
601            // lazy_static, so this must happen first). See #743.
602            let xdg = std::env::var("XDG_RUNTIME_DIR").ok();
603            let tmpdir = std::env::var("TMPDIR").ok();
604            if let Some(kata_runtime_dir) =
605                rootless_kata_runtime_dir(euid.as_raw(), xdg.as_deref(), tmpdir.as_deref())
606            {
607                tokio::fs::create_dir_all(&kata_runtime_dir)
608                    .await
609                    .map_err(|e| {
610                        WorkerError::VmStartFailed(format!(
611                            "failed to create rootless kata runtime dir {}: {e}",
612                            kata_runtime_dir.display()
613                        ))
614                    })?;
615                std::env::set_var("XDG_RUNTIME_DIR", &kata_runtime_dir);
616                tracing::debug!(
617                    dir = %kata_runtime_dir.display(),
618                    "Rootless Kata: pinned XDG_RUNTIME_DIR to a user-writable sharefs jailer base"
619                );
620            }
621            rootless::set_rootless(true);
622            tracing::debug!("Enabled Kata rootless mode for non-root user");
623        }
624        Ok(())
625    }
626
627    async fn start(
628        &self,
629        sandbox: &mut PodSandbox,
630        _config: &PodSandboxConfig,
631        pool_config: &PoolConfig,
632        annotations: &HashMap<String, String>,
633    ) -> Result<Arc<dyn SandboxHandle>> {
634        tracing::info!(sandbox_id = %sandbox.id, "Starting Kata VM for sandbox");
635
636        let sandbox_path = sandbox.sandbox_path().clone();
637        tokio::fs::create_dir_all(&sandbox_path).await?;
638
639        let api_socket = sandbox_path.join("cloud-hypervisor.sock");
640        let virtiofs_socket = sandbox_path.join("virtiofs.sock");
641        let cloud_init_iso = sandbox_path.join("cloud-init.iso");
642
643        if pool_config.cloud_init_dir.exists() {
644            Self::generate_cloud_init_iso(sandbox, &cloud_init_iso).await?;
645        }
646
647        // FS-D (#365): compose a per-sandbox VFS (rootfs at `/` from FS-B's
648        // OverlayFs(RAFS lower + per-sandbox writable upper) + the native
649        // injected mounts `/stream`/`/models`/`/deltas`), forked into its own
650        // Namespace bound to this sandbox's Subject, and serve it over a
651        // per-sandbox Unix socket via FS-A's `hyprstream-vfs-server`. The guest
652        // mounts this composed VFS — no external `nydusd`. Each sandbox's
653        // namespace/socket/Subject/writable-upper is private, so sandbox A's
654        // namespace cannot expose sandbox B's rootfs or injected paths.
655        let mut vfs_server: Option<SandboxFsServer> = None;
656        let mut share_socket: Option<PathBuf> = None;
657        // The tenant root Mount + Subject, captured so the *same* Subject-scoped
658        // Mount can be re-served over the guest's 9P-over-vsock channel (#731)
659        // once the VM (and thus its vsock UDS) exists.
660        let mut tenant_vfs: Option<(Arc<dyn Mount>, Subject)> = None;
661        if let Some(ref image_id) = sandbox.image_id {
662            let subject = Self::sandbox_subject(sandbox, annotations);
663            let (server, root_mount) = self
664                .compose_and_serve_vfs(sandbox, image_id, subject.clone())
665                .await?;
666            share_socket = Some(server.socket_path().to_path_buf());
667            tenant_vfs = Some((root_mount, subject));
668            vfs_server = Some(server);
669        }
670        let virtiofs_sock = share_socket.clone();
671
672        let hypervisor =
673            Self::create_hypervisor(pool_config, sandbox, &api_socket, &virtiofs_socket).await?;
674
675        hypervisor
676            .prepare_vm(&sandbox.id, None, annotations, None)
677            .await
678            .map_err(|e| WorkerError::VmStartFailed(format!("failed to prepare VM: {e}")))?;
679
680        // Attach the composed VFS socket as a CH ShareFs device, after prepare_vm
681        // and before start_vm so it is folded into the boot VmConfig. Record the
682        // mount tag so `exec_sync` can tell the kata-agent to mount this share as
683        // the container's rootfs (#721).
684        let rootfs_mount_tag = if let Some(ref socket) = share_socket {
685            Self::attach_share_fs(&hypervisor, &sandbox.id, socket, ROOTFS_MOUNT_TAG).await?;
686            Some(ROOTFS_MOUNT_TAG.to_owned())
687        } else {
688            None
689        };
690
691        let timeout_secs = i32::try_from(pool_config.create_timeout_secs).unwrap_or(i32::MAX);
692        tracing::debug!(sandbox_id = %sandbox.id, timeout_secs, "Starting VM");
693        hypervisor
694            .start_vm(timeout_secs)
695            .await
696            .map_err(|e| WorkerError::VmStartFailed(format!("failed to start VM: {e}")))?;
697
698        let pids = hypervisor.get_pids().await.ok();
699        tracing::info!(
700            sandbox_id = %sandbox.id,
701            pids = ?pids,
702            "VM started successfully via Kata Hypervisor trait"
703        );
704
705        // V2 (#731): stand up a *second* vsock channel — distinct from the
706        // kata-agent's port 1024 — backed by the tenant VFS 9P server, so an
707        // in-guest 9P client (V3, #732) can dial `VFS_9P_VSOCK_PORT` and operate
708        // the hyprstream VFS over native 9P. Purely additive to the boot path, so
709        // a failure here is logged, not fatal (the virtio-fs rootfs the guest
710        // boots from is unaffected).
711        let mut vfs_9p: Option<Vfs9pVsockServer> = None;
712        if let Some((tenant_mount, subject)) = tenant_vfs {
713            match Self::serve_tenant_vfs_9p(
714                &hypervisor,
715                &sandbox.id,
716                tenant_mount,
717                subject,
718                Arc::clone(&self.ninep_decider),
719                self.ninep_monitor.clone(),
720            )
721            .await
722            {
723                Ok(server) => vfs_9p = Some(server),
724                Err(e) => tracing::warn!(
725                    sandbox_id = %sandbox.id,
726                    error = %e,
727                    "failed to stand up tenant-VFS 9P vsock channel (#731); \
728                     sandbox boots without it"
729                ),
730            }
731        }
732
733        let handle = Arc::new(KataHandle {
734            hypervisor,
735            api_socket,
736            virtiofs_socket: virtiofs_sock,
737            vfs_server,
738            vfs_9p,
739            rootfs_mount_tag: parking_lot::Mutex::new(rootfs_mount_tag),
740            agent: tokio::sync::Mutex::new(None),
741        });
742
743        Ok(handle)
744    }
745
746    async fn stop(&self, sandbox: &PodSandbox) -> Result<()> {
747        tracing::info!(sandbox_id = %sandbox.id, "Stopping sandbox via Kata backend");
748
749        if let Some(handle) = sandbox.backend_handle.as_ref() {
750            if let Some(kata) = handle.as_any().downcast_ref::<KataHandle>() {
751                kata.hypervisor
752                    .stop_vm()
753                    .await
754                    .map_err(|e| WorkerError::VmStopFailed(format!("failed to stop VM: {e}")))?;
755                tracing::debug!(sandbox_id = %sandbox.id, "VM stopped via Kata Hypervisor trait");
756            }
757        } else {
758            tracing::warn!(
759                sandbox_id = %sandbox.id,
760                "No backend handle - sandbox may not have been started"
761            );
762        }
763
764        tracing::info!(sandbox_id = %sandbox.id, "Sandbox stopped");
765        Ok(())
766    }
767
768    async fn destroy(&self, sandbox: &PodSandbox) -> Result<()> {
769        tracing::info!(sandbox_id = %sandbox.id, "Destroying sandbox via Kata backend");
770
771        if let Some(handle) = sandbox.backend_handle.as_ref() {
772            if let Some(kata) = handle.as_any().downcast_ref::<KataHandle>() {
773                if let Err(e) = kata.hypervisor.stop_vm().await {
774                    tracing::warn!(sandbox_id = %sandbox.id, error = %e, "Error stopping VM during destroy");
775                }
776                if let Err(e) = kata.hypervisor.cleanup().await {
777                    tracing::warn!(sandbox_id = %sandbox.id, error = %e, "Error cleaning up hypervisor resources");
778                }
779
780                if kata.api_socket.exists() {
781                    let _ = tokio::fs::remove_file(&kata.api_socket).await;
782                }
783                if let Some(ref vs) = kata.virtiofs_socket {
784                    if vs.exists() {
785                        let _ = tokio::fs::remove_file(vs).await;
786                    }
787                }
788            }
789        }
790
791        let sandbox_path = sandbox.sandbox_path();
792        if sandbox_path.exists() {
793            tracing::debug!(
794                sandbox_id = %sandbox.id,
795                dir = %sandbox_path.display(),
796                "Removing sandbox runtime directory"
797            );
798            if let Err(e) = tokio::fs::remove_dir_all(sandbox_path).await {
799                tracing::warn!(sandbox_id = %sandbox.id, error = %e, "Failed to remove sandbox runtime directory");
800            }
801        }
802
803        tracing::info!(sandbox_id = %sandbox.id, "Sandbox destroyed");
804        Ok(())
805    }
806
807    async fn reset(&self, sandbox: &mut PodSandbox) -> Result<bool> {
808        tracing::debug!(sandbox_id = %sandbox.id, "Resetting Kata sandbox for warm pool");
809
810        if let Some(handle) = sandbox.backend_handle.take() {
811            if let Some(kata) = handle.as_any().downcast_ref::<KataHandle>() {
812                let fresh = Arc::new(KataHandle {
813                    hypervisor: Arc::clone(&kata.hypervisor),
814                    api_socket: kata.api_socket.clone(),
815                    virtiofs_socket: None,
816                    // The per-sandbox VFS server is not reusable across resets;
817                    // a recycled sandbox composes a fresh one on next start.
818                    vfs_server: None,
819                    // Ditto the 9P-over-vsock channel (#731): dropping the old
820                    // handle aborts its serve task + removes the UDS; a recycled
821                    // sandbox stands up a fresh one on next start.
822                    vfs_9p: None,
823                    // Fresh compose+attach on next start re-establishes the tag;
824                    // the old container (and its rootfs mount) is torn down.
825                    rootfs_mount_tag: parking_lot::Mutex::new(None),
826                    // A reused (warm-pool) sandbox keeps the same VM, but the
827                    // container(s) inside it are torn down — drop any cached
828                    // agent connection so the next `exec_sync` reconnects
829                    // against a clean guest state rather than reusing a
830                    // client whose container/exec ids may now be stale.
831                    agent: tokio::sync::Mutex::new(None),
832                });
833                sandbox.backend_handle = Some(fresh);
834            }
835        }
836
837        Ok(true)
838    }
839
840    async fn get_pids(&self, sandbox: &PodSandbox) -> Result<Vec<u32>> {
841        if let Some(handle) = sandbox.backend_handle.as_ref() {
842            if let Some(kata) = handle.as_any().downcast_ref::<KataHandle>() {
843                return kata
844                    .hypervisor
845                    .get_pids()
846                    .await
847                    .map_err(|e| WorkerError::Internal(format!("get_pids failed: {e}")));
848            }
849        }
850        Ok(Vec::new())
851    }
852
853    async fn deliver_namespace(
854        &self,
855        sandbox: &PodSandbox,
856        namespace: hyprstream_vfs::Namespace,
857        subject: Subject,
858        transport: super::backend::NamespaceTransport,
859    ) -> Result<super::backend::NamespaceDelivery> {
860        use super::backend::{NamespaceDelivery, NamespaceTransport};
861
862        let (socket_path, mount_tag) = match transport {
863            NamespaceTransport::VirtioFs {
864                socket_path,
865                mount_tag,
866            } => (socket_path, mount_tag),
867            other => {
868                return Err(WorkerError::Unsupported(format!(
869                    "kata backend only supports the VirtioFs namespace transport, got {other:?}"
870                )))
871            }
872        };
873
874        // Serve the already-composed namespace over a per-sandbox Unix socket,
875        // exactly as `compose_and_serve_vfs` does for the boot-time path — the
876        // difference is that composition already happened in the caller (#635:
877        // the backend's job is delivery, not composition).
878        let rt = tokio::runtime::Handle::current();
879        let sandbox_fs = SandboxFs::from_namespace(namespace, subject);
880        let socket_for_serve = socket_path.clone();
881        let server = tokio::task::spawn_blocking(move || sandbox_fs.serve_on(socket_for_serve, rt))
882            .await
883            .map_err(|e| {
884                WorkerError::SandboxCreationFailed(format!("VFS serve task join: {e}"))
885            })??;
886
887        // Hot-attach when a hypervisor handle already exists for this sandbox
888        // (VM already prepared/running). If the sandbox hasn't started yet,
889        // the caller is expected to attach the returned socket itself via
890        // `attach_share_fs` before `start_vm` (mirrors `KataBackend::start`'s
891        // own compose-then-attach ordering).
892        if let Some(handle) = sandbox.backend_handle.as_ref() {
893            if let Some(kata) = handle.as_any().downcast_ref::<KataHandle>() {
894                Self::attach_share_fs(
895                    &kata.hypervisor,
896                    &sandbox.id,
897                    server.socket_path(),
898                    &mount_tag,
899                )
900                .await?;
901                // #742: record the delivered share's tag as the container
902                // rootfs tag on the (already-`Arc`-shared) `KataHandle` via
903                // interior mutability. The delivered namespace is the fully
904                // composed tenant VFS (rootfs + injected mounts — see the
905                // `deliver_namespace` trait contract), so a later `exec_sync`
906                // passes this tag to the kata-agent's `CreateContainer` and the
907                // guest mounts it, by tag, as the container's rootfs — exactly
908                // as the boot-time `start()` path does. Last write wins, so a
909                // re-delivery re-roots the container on the new share.
910                kata.record_rootfs_mount_tag(mount_tag.clone());
911            }
912        }
913
914        Ok(NamespaceDelivery::VirtioFs {
915            socket_path: server.socket_path().to_path_buf(),
916            mount_tag,
917            guard: Some(Arc::new(server)),
918        })
919    }
920
921    fn supports_exec(&self) -> bool {
922        // Real, via the kata-agent ttrpc/vsock client (#344) — see
923        // `exec_sync` below. This is no longer the "host is a black box"
924        // limitation the original stub described.
925        true
926    }
927
928    async fn exec_sync(
929        &self,
930        sandbox: &PodSandbox,
931        command: &[String],
932        timeout_secs: u64,
933    ) -> Result<(i32, Vec<u8>, Vec<u8>)> {
934        if command.is_empty() {
935            return Err(WorkerError::ExecFailed("empty command".into()));
936        }
937
938        let handle = sandbox
939            .backend_handle
940            .as_ref()
941            .ok_or_else(|| WorkerError::ExecFailed("sandbox has no backend handle".into()))?;
942        let kata = handle
943            .as_any()
944            .downcast_ref::<KataHandle>()
945            .ok_or_else(|| WorkerError::ExecFailed("backend handle is not a KataHandle".into()))?;
946
947        let client = self.guest_agent_client(kata).await?;
948
949        // The container id is the sandbox id: `exec_sync` here targets the
950        // single workload container running in the sandbox's VM (the
951        // CRI-shaped subset this issue scopes — multi-container pods inside
952        // one Kata VM are a separate concern from #344's vsock/ttrpc client).
953        let container_id = sandbox.id.clone();
954
955        // When a tenant VFS was attached at boot (#721), tell the kata-agent to
956        // mount that virtio-fs share, by tag, as the container's rootfs — so
957        // `CreateContainer` finds a real rootfs at `<CONTAINER_BASE>/<cid>/rootfs`
958        // (the tenant VFS) instead of failing `No such file or directory`.
959        let rootfs = kata.rootfs_mount_tag().map(|tag| ContainerRootfs {
960            virtiofs_mount_tag: tag,
961        });
962
963        match client
964            .exec(
965                &container_id,
966                command,
967                std::time::Duration::from_secs(timeout_secs),
968                rootfs.as_ref(),
969            )
970            .await
971        {
972            Ok(out) => Ok(out),
973            Err(e) => {
974                // The cached ttrpc connection may be stale (the VM died or was
975                // restarted externally without a `reset_sandbox` that would
976                // have cleared it). Drop the cached client so the *next*
977                // `exec_sync` redials against a fresh guest state instead of
978                // reusing a dead connection and failing every subsequent call
979                // until an explicit reset. We do NOT retry in-line: the caller
980                // asked for one exec (not a reconnect loop), and surfacing the
981                // original failure — broken pipe / RpcStatus / timeout — is
982                // more honest than masking it behind a retry that may hit the
983                // same dead VM. The next call re-establishes the connection
984                // via `guest_agent_client`.
985                //
986                // `try_lock`: if another exec is in flight on this sandbox,
987                // leave the cache alone — clearing it under them would race.
988                // Only clear when it still points at the same client we just
989                // used (a concurrent exec may have already swapped in a fresh
990                // one); compare by `Arc::ptr_eq`.
991                if let Ok(mut guard) = kata.agent.try_lock() {
992                    // Only clear when the cache still holds the same client we
993                    // just used (compared by `Arc` pointer identity): a
994                    // concurrent exec on another task may already have swapped
995                    // in a fresh client, and clearing that one would discard a
996                    // good connection.
997                    if guard
998                        .as_ref()
999                        .map(|c| Arc::ptr_eq(c, &client))
1000                        .unwrap_or(false)
1001                    {
1002                        *guard = None;
1003                    }
1004                }
1005                Err(WorkerError::ExecFailed(format!(
1006                    "kata-agent exec failed: {e:#}"
1007                )))
1008            }
1009        }
1010    }
1011
1012    /// Apply new CPU/memory limits to the sandbox's running container by
1013    /// driving the kata-agent `UpdateContainer` RPC (the CRI
1014    /// `UpdateContainerResources` path). The CRI-shaped
1015    /// [`LinuxContainerResources`] is mapped to the guest's
1016    /// [`oci::LinuxResources`] and the agent rewrites the in-guest cgroups.
1017    ///
1018    /// Uses `sandbox.id` as the guest container id — the same single-workload
1019    /// convention `exec_sync` uses (see there). Requires a started sandbox
1020    /// with a live agent connection; without a backend handle this fails
1021    /// rather than silently no-op'ing (unlike the trait default), because a
1022    /// caller asking to resize an unstarted sandbox has made a mistake.
1023    async fn update_resources(
1024        &self,
1025        sandbox: &PodSandbox,
1026        resources: &LinuxContainerResources,
1027    ) -> Result<()> {
1028        let handle = sandbox.backend_handle.as_ref().ok_or_else(|| {
1029            WorkerError::Internal("update_resources: sandbox has no backend handle".into())
1030        })?;
1031        let kata = handle
1032            .as_any()
1033            .downcast_ref::<KataHandle>()
1034            .ok_or_else(|| {
1035                WorkerError::Internal("update_resources: backend handle is not a KataHandle".into())
1036            })?;
1037
1038        let client = self.guest_agent_client(kata).await?;
1039        let container_id = sandbox.id.clone();
1040        let oci_resources = Self::map_linux_resources(resources);
1041
1042        client
1043            .update_container(&container_id, oci_resources)
1044            .await
1045            .map_err(|e| {
1046                WorkerError::Internal(format!("kata-agent update_container failed: {e:#}"))
1047            })?;
1048
1049        tracing::info!(
1050            sandbox_id = %sandbox.id,
1051            cpu_shares = resources.cpu_shares,
1052            cpu_quota = resources.cpu_quota,
1053            cpu_period = resources.cpu_period,
1054            memory_limit_in_bytes = resources.memory_limit_in_bytes,
1055            "Updated container resources via kata-agent UpdateContainer"
1056        );
1057        Ok(())
1058    }
1059
1060    /// Fetch live CPU/memory usage for the sandbox's container from the guest
1061    /// via the kata-agent `StatsContainer` RPC, mapped into the CRI
1062    /// [`CpuUsage`]/[`MemoryUsage`] shape.
1063    ///
1064    /// Returns `Ok(None)` when the sandbox has no backend handle yet (never
1065    /// started) so the caller can fall back to placeholder stats rather than
1066    /// erroring on a not-yet-running sandbox. A live agent that then fails the
1067    /// RPC surfaces as `Err`.
1068    async fn container_stats(
1069        &self,
1070        sandbox: &PodSandbox,
1071    ) -> Result<Option<(CpuUsage, MemoryUsage)>> {
1072        let Some(handle) = sandbox.backend_handle.as_ref() else {
1073            return Ok(None);
1074        };
1075        let Some(kata) = handle.as_any().downcast_ref::<KataHandle>() else {
1076            return Ok(None);
1077        };
1078
1079        let client = self.guest_agent_client(kata).await?;
1080        let container_id = sandbox.id.clone();
1081
1082        let resp = client.stats_container(&container_id).await.map_err(|e| {
1083            WorkerError::Internal(format!("kata-agent stats_container failed: {e:#}"))
1084        })?;
1085
1086        Ok(Some(Self::map_container_stats(&resp)))
1087    }
1088}
1089
1090impl KataBackend {
1091    /// Map the CRI [`LinuxContainerResources`] to the guest
1092    /// [`oci::LinuxResources`] the kata-agent's `UpdateContainer` expects.
1093    ///
1094    /// Only the fields with a meaningful CRI→OCI correspondence are set; the
1095    /// rest are left at their protobuf defaults so the agent treats them as
1096    /// "unchanged". `memory_swap_limit_in_bytes` maps to OCI `Swap`, and the
1097    /// cpuset strings to `Cpus`/`Mems`.
1098    fn map_linux_resources(resources: &LinuxContainerResources) -> oci::LinuxResources {
1099        let cpu = oci::LinuxCPU {
1100            // CRI cpu_shares is i64; OCI Shares is u64. Clamp negatives to 0.
1101            Shares: u64::try_from(resources.cpu_shares).unwrap_or(0),
1102            Quota: resources.cpu_quota,
1103            Period: u64::try_from(resources.cpu_period).unwrap_or(0),
1104            Cpus: resources.cpuset_cpus.clone(),
1105            Mems: resources.cpuset_mems.clone(),
1106            ..Default::default()
1107        };
1108        let memory = oci::LinuxMemory {
1109            Limit: resources.memory_limit_in_bytes,
1110            Swap: resources.memory_swap_limit_in_bytes,
1111            ..Default::default()
1112        };
1113        oci::LinuxResources {
1114            CPU: protobuf::MessageField::some(cpu),
1115            Memory: protobuf::MessageField::some(memory),
1116            ..Default::default()
1117        }
1118    }
1119
1120    /// Map a kata-agent [`agent::StatsContainerResponse`] (guest cgroup
1121    /// counters) into the CRI [`CpuUsage`]/[`MemoryUsage`] shape.
1122    ///
1123    /// `usage_nano_cores` is left 0: it is a *rate* that requires two samples
1124    /// (a delta of `usage_core_nano_seconds` over wall-clock) which a single
1125    /// stats call cannot compute — kubelet's own CRI stats provider derives
1126    /// it host-side from successive samples, so emitting a fabricated value
1127    /// here would be wrong. `working_set_bytes` follows the CRI definition
1128    /// (`usage − total_inactive_file`); other fields come straight from the
1129    /// cgroup memory counters, reading `rss`/`pgfault`/`pgmajfault` from the
1130    /// per-cgroup `stats` map (falling back to 0 when a controller does not
1131    /// export them).
1132    fn map_container_stats(resp: &agent::StatsContainerResponse) -> (CpuUsage, MemoryUsage) {
1133        let now_ns = chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0);
1134        let cgroup = &resp.cgroup_stats;
1135
1136        let cpu = CpuUsage {
1137            timestamp: now_ns,
1138            usage_core_nano_seconds: cgroup.cpu_stats.cpu_usage.total_usage,
1139            usage_nano_cores: 0,
1140        };
1141
1142        let mem = &cgroup.memory_stats;
1143        let usage_bytes = mem.usage.usage;
1144        let inactive_file = mem
1145            .stats
1146            .get("total_inactive_file")
1147            .or_else(|| mem.stats.get("inactive_file"))
1148            .copied()
1149            .unwrap_or(0);
1150        let rss_bytes = mem
1151            .stats
1152            .get("total_rss")
1153            .or_else(|| mem.stats.get("rss"))
1154            .copied()
1155            .unwrap_or(0);
1156        let page_faults = mem
1157            .stats
1158            .get("total_pgfault")
1159            .or_else(|| mem.stats.get("pgfault"))
1160            .copied()
1161            .unwrap_or(0);
1162        let major_page_faults = mem
1163            .stats
1164            .get("total_pgmajfault")
1165            .or_else(|| mem.stats.get("pgmajfault"))
1166            .copied()
1167            .unwrap_or(0);
1168        let limit = mem.usage.limit;
1169
1170        let memory = MemoryUsage {
1171            timestamp: now_ns,
1172            working_set_bytes: usage_bytes.saturating_sub(inactive_file),
1173            available_bytes: limit.saturating_sub(usage_bytes),
1174            usage_bytes,
1175            rss_bytes,
1176            page_faults,
1177            major_page_faults,
1178        };
1179
1180        (cpu, memory)
1181    }
1182
1183    /// Get (lazily connecting if needed) the cached kata-agent ttrpc client
1184    /// for this sandbox's VM.
1185    ///
1186    /// The connection address comes straight from
1187    /// `Hypervisor::get_agent_socket()` — the same call upstream
1188    /// kata-containers' own runtime uses
1189    /// (`runtime-rs/crates/agent/src/kata/agent.rs::start`) — so this does
1190    /// not hardcode the hybrid-vsock UDS path; it asks the hypervisor
1191    /// abstraction for it, which keeps Cloud Hypervisor/Dragonball both
1192    /// working without an `if hypervisor_type == ...` branch here.
1193    async fn guest_agent_client(&self, kata: &KataHandle) -> Result<Arc<KataAgentClient>> {
1194        let mut guard = kata.agent.lock().await;
1195        if let Some(client) = guard.as_ref() {
1196            return Ok(Arc::clone(client));
1197        }
1198
1199        let address = kata
1200            .hypervisor
1201            .get_agent_socket()
1202            .await
1203            .map_err(|e| WorkerError::ExecFailed(format!("get_agent_socket failed: {e}")))?;
1204        let address = AgentAddress::parse(&address).map_err(|e| {
1205            WorkerError::ExecFailed(format!("unparseable agent socket address: {e}"))
1206        })?;
1207
1208        let client = KataAgentClient::connect(&address).await.map_err(|e| {
1209            WorkerError::ExecFailed(format!("failed to connect to kata-agent: {e:#}"))
1210        })?;
1211        let client = Arc::new(client);
1212        *guard = Some(Arc::clone(&client));
1213        Ok(client)
1214    }
1215
1216    /// Stand up the tenant-VFS **9P-over-vsock** channel for a booted VM (V2,
1217    /// #731).
1218    ///
1219    /// Serves `mount` (scoped to `subject` — the same per-sandbox Subject the
1220    /// virtio-fs namespace uses; no new principal) as native 9P on the host UDS
1221    /// the guest reaches by dialing [`VFS_9P_VSOCK_PORT`]. The base vsock UDS is
1222    /// taken from `Hypervisor::get_agent_socket()` (the CH hybrid-vsock path,
1223    /// `hvsock://<base>`), so the listener path is not hardcoded; the per-port
1224    /// host UDS is `<base>_<VFS_9P_VSOCK_PORT>` ([`vfs_9p_vsock_uds`]).
1225    ///
1226    /// The serve loop runs on a background task held by the returned
1227    /// [`Vfs9pVsockServer`], whose `Drop` tears it down with the sandbox.
1228    async fn serve_tenant_vfs_9p(
1229        hypervisor: &Arc<dyn Hypervisor>,
1230        sandbox_id: &str,
1231        mount: Arc<dyn Mount>,
1232        subject: Subject,
1233        decider: Arc<dyn hyprstream_9p::AccessDecider>,
1234        monitor: Option<Arc<hyprstream_9p::ReferenceMonitor>>,
1235    ) -> Result<Vfs9pVsockServer> {
1236        // The CH hybrid-vsock base UDS (`hvsock://<base>`), same source the
1237        // kata-agent client uses — keeps CH/Dragonball working without a branch.
1238        let address = hypervisor.get_agent_socket().await.map_err(|e| {
1239            WorkerError::SandboxCreationFailed(format!("get_agent_socket for 9P vsock: {e}"))
1240        })?;
1241        let address = AgentAddress::parse(&address).map_err(|e| {
1242            WorkerError::SandboxCreationFailed(format!("unparseable agent socket address: {e}"))
1243        })?;
1244        let base = match &address {
1245            AgentAddress::HybridVsock { path } => path.clone(),
1246            // Real AF_VSOCK (Firecracker/Dragonball) has no host-UDS-per-port
1247            // convention; the 9P-over-vsock channel is CH hybrid-vsock only for now.
1248            AgentAddress::Vsock { cid } => {
1249                return Err(WorkerError::SandboxCreationFailed(format!(
1250                    "tenant-VFS 9P vsock channel requires CH hybrid-vsock; \
1251                     got real AF_VSOCK (cid={cid})"
1252                )))
1253            }
1254        };
1255        let socket_path = vfs_9p_vsock_uds(&base, VFS_9P_VSOCK_PORT);
1256
1257        // Remove any stale socket from a crashed predecessor (bind would EADDRINUSE).
1258        if let Err(e) = std::fs::remove_file(&socket_path) {
1259            if e.kind() != std::io::ErrorKind::NotFound {
1260                tracing::warn!(
1261                    socket = %socket_path.display(),
1262                    error = %e,
1263                    "remove stale tenant-VFS 9P vsock socket"
1264                );
1265            }
1266        }
1267
1268        let sock_for_task = socket_path.clone();
1269        let sandbox_id = sandbox_id.to_owned();
1270        // `serve_mount_vsock_raw` binds the per-port host UDS and runs the 9P
1271        // accept loop until the socket errors/closes (or the task is aborted on
1272        // teardown). It serves the same Subject-scoped Mount as native 9P; only
1273        // the transport differs.
1274        //
1275        // RAW (no-handshake) mode is required here (#741): this is a **per-port**
1276        // host listener (`<vsock-base>_<VFS_9P_VSOCK_PORT>`) that the **guest
1277        // dials**. Per the Firecracker/CH hybrid-vsock spec, guest-initiated
1278        // connections to a per-port host UDS arrive **raw** — the port is encoded
1279        // in the socket path, not in an in-band `connect <port>\n` preamble. The
1280        // preamble-stripping `serve_mount_vsock` is for the opposite (host-
1281        // initiated, CH-multiplexer) direction; using it here would consume the
1282        // guest's first 9P `Tversion` bytes and break the handshake.
1283        let task = tokio::spawn(async move {
1284            if let Err(e) = hyprstream_9p::serve_mount_vsock_raw(
1285                mount,
1286                subject,
1287                decider,
1288                monitor,
1289                &sock_for_task,
1290            )
1291            .await
1292            {
1293                tracing::warn!(
1294                    sandbox_id = %sandbox_id,
1295                    socket = %sock_for_task.display(),
1296                    error = %e,
1297                    "tenant-VFS 9P vsock server exited with error"
1298                );
1299            }
1300        });
1301
1302        tracing::info!(
1303            socket = %socket_path.display(),
1304            port = VFS_9P_VSOCK_PORT,
1305            "tenant-VFS 9P vsock channel listening (#731)"
1306        );
1307        Ok(Vfs9pVsockServer { socket_path, task })
1308    }
1309}
1310
1311// ─────────────────────────────────────────────────────────────────────────────
1312// Backend registry self-registration (#507 / #518)
1313// ─────────────────────────────────────────────────────────────────────────────
1314
1315// This whole file only compiles under `kata-vm` (see runtime/mod.rs), so this
1316// `submit!` is feature-gated by construction: the `kata` backend is a selectable
1317// name *only* in builds that include the VM toolchain. In a non-`kata` binary
1318// (the `kata-vm` alias is off too) there is no registration, and selecting
1319// `kata` fails closed with a build hint.
1320//
1321// Highest priority → preferred by `auto` (strongest isolation). Construction
1322// pulls the RAFS image store from the per-call BackendCtx.
1323inventory::submit! {
1324    crate::runtime::selection::BackendRegistration {
1325        name: "kata",
1326        priority: 100,
1327        // Full VM isolation (strongest tier) → eligible for `"auto"`.
1328        auto_selectable: true,
1329        // A VM does NOT share the host mount namespace, so a host Unix socket
1330        // cannot be bind-injected the way oci/nspawn do (#506). The VM path
1331        // exposes a tenant namespace over virtio-fs instead (see `sandbox_fs`),
1332        // a different mechanism — so kata does not advertise host-UDS injection.
1333        injects_9p_socket: false,
1334        // kata already shares the composed per-sandbox VFS with its guest as a
1335        // virtio-fs (vhost-user-fs) device, NOT a host FUSE mount, so it does not
1336        // advertise the Model B (#715) host-FUSE-mount capability.
1337        mounts_fuse_vfs: false,
1338        is_available: KataBackend::registry_is_available,
1339        construct: |ctx| {
1340            Ok(Arc::new(KataBackend::new(
1341                ctx.image_config.clone(),
1342                Arc::clone(&ctx.rafs_store),
1343                Arc::clone(&ctx.ninep_decider),
1344                ctx.ninep_monitor.clone(),
1345            )) as Arc<dyn crate::runtime::SandboxBackend>)
1346        },
1347    }
1348}
1349
1350#[cfg(test)]
1351#[allow(clippy::unwrap_used, clippy::expect_used)]
1352mod tests {
1353    use super::*;
1354    use crate::config::{ImageConfig, PoolConfig};
1355    use crate::error::WorkerError;
1356    use crate::image::RafsStore;
1357    use hyprstream_rpc::auth::mac::{ObjectRef, SecurityContext};
1358    use tempfile::TempDir;
1359
1360    struct FixtureAccessDecider;
1361
1362    impl hyprstream_9p::AccessDecider for FixtureAccessDecider {
1363        fn check(
1364            &self,
1365            _ctx: &SecurityContext,
1366            _object: ObjectRef<'_>,
1367            _action: hyprstream_9p::Action,
1368        ) -> bool {
1369            true
1370        }
1371
1372        fn audit_denial(
1373            &self,
1374            _ctx: &SecurityContext,
1375            _object: ObjectRef<'_>,
1376            _object_label: Option<hyprstream_rpc::auth::mac::SecurityLabel>,
1377            _action: hyprstream_9p::Action,
1378            _reason: hyprstream_9p::ReferenceMonitorDenyReason,
1379        ) {
1380        }
1381    }
1382
1383    #[test]
1384    fn vfs_9p_vsock_port_distinct_from_agent() {
1385        // The tenant-VFS 9P channel must never collide with the kata-agent's
1386        // ttrpc port (1024): they share one CH hybrid-vsock base UDS, so equal
1387        // ports would map to the same host socket.
1388        assert_ne!(
1389            VFS_9P_VSOCK_PORT,
1390            super::super::kata_agent::KATA_AGENT_VSOCK_PORT
1391        );
1392        // 564 is the IANA-registered Plan 9 (9P) port.
1393        assert_eq!(VFS_9P_VSOCK_PORT, 564);
1394    }
1395
1396    #[test]
1397    fn vfs_9p_vsock_uds_appends_underscore_port() {
1398        // CH routes a guest connection to vsock port N to the host UDS
1399        // `<base>_<N>`; the derivation must produce exactly that.
1400        let base = "/run/sandbox/abc/ch-vm.sock";
1401        let uds = super::vfs_9p_vsock_uds(base, VFS_9P_VSOCK_PORT);
1402        assert_eq!(uds, PathBuf::from("/run/sandbox/abc/ch-vm.sock_564"));
1403
1404        // And it composes with an arbitrary base + port, underscore-joined.
1405        assert_eq!(
1406            super::vfs_9p_vsock_uds("/tmp/vm.sock", 1024),
1407            PathBuf::from("/tmp/vm.sock_1024")
1408        );
1409    }
1410
1411    /// Create a KataBackend with temporary directories.
1412    fn create_test_backend() -> (KataBackend, Arc<RafsStore>, TempDir) {
1413        let temp_dir = TempDir::new().unwrap();
1414        let base = temp_dir.path();
1415
1416        let image_config = ImageConfig {
1417            blobs_dir: base.join("blobs"),
1418            bootstrap_dir: base.join("bootstrap"),
1419            refs_dir: base.join("refs"),
1420            cache_dir: base.join("cache"),
1421            runtime_dir: base.join("nydus-runtime"),
1422            ..Default::default()
1423        };
1424
1425        std::fs::create_dir_all(&image_config.blobs_dir).unwrap();
1426        std::fs::create_dir_all(&image_config.bootstrap_dir).unwrap();
1427        std::fs::create_dir_all(&image_config.refs_dir).unwrap();
1428        std::fs::create_dir_all(&image_config.cache_dir).unwrap();
1429
1430        let rafs_store = Arc::new(RafsStore::new(image_config.clone()).unwrap());
1431        let backend = KataBackend::new(
1432            image_config,
1433            Arc::clone(&rafs_store),
1434            Arc::new(FixtureAccessDecider),
1435            None,
1436        );
1437        (backend, rafs_store, temp_dir)
1438    }
1439
1440    /// Create a PodSandbox with a temp directory as sandbox_path.
1441    fn create_test_sandbox(sandbox_path: PathBuf) -> PodSandbox {
1442        PodSandbox {
1443            id: "test-sandbox-001".to_owned(),
1444            metadata: hyprstream_rpc_std::worker_client::PodSandboxMetadata {
1445                name: String::new(),
1446                uid: String::new(),
1447                namespace: "default".to_owned(),
1448                attempt: 0,
1449            },
1450            state: hyprstream_rpc_std::worker_client::PodSandboxState::SandboxNotReady,
1451            created_at: chrono::Utc::now(),
1452            labels: vec![],
1453            annotations: vec![],
1454            runtime_handler: "kata".to_owned(),
1455            backend_handle: None,
1456            sandbox_path,
1457            image_id: None,
1458            console_socket: None,
1459            applied_resources: hyprstream_rpc_std::worker_client::LinuxContainerResources::default(
1460            ),
1461            reservation: None,
1462        }
1463    }
1464
1465    /// Create a KataHandle with a real CloudHypervisor instance (no VM started).
1466    fn create_test_handle(sandbox_path: &Path) -> Arc<KataHandle> {
1467        let ch = CloudHypervisor::new();
1468        Arc::new(KataHandle {
1469            hypervisor: Arc::new(ch),
1470            api_socket: sandbox_path.join("test.sock"),
1471            virtiofs_socket: Some(sandbox_path.join("virtiofs.sock")),
1472            vfs_server: None,
1473            vfs_9p: None,
1474            rootfs_mount_tag: parking_lot::Mutex::new(None),
1475            agent: tokio::sync::Mutex::new(None),
1476        })
1477    }
1478
1479    // ─────────────────────────────────────────────────────────────────────
1480    // SandboxBackend trait method tests
1481    // ─────────────────────────────────────────────────────────────────────
1482
1483    #[test]
1484    fn test_backend_type() {
1485        let (backend, _, _temp) = create_test_backend();
1486        assert_eq!(backend.backend_type(), "kata");
1487    }
1488
1489    #[test]
1490    fn test_supports_exec() {
1491        // #344: exec is real now (kata-agent ttrpc/vsock client), so the
1492        // backend advertises support — connection failures surface from
1493        // `exec_sync` itself, not from this capability flag.
1494        let (backend, _, _temp) = create_test_backend();
1495        assert!(backend.supports_exec());
1496    }
1497
1498    // ─────────────────────────────────────────────────────────────────────
1499    // deliver_namespace (#635)
1500    // ─────────────────────────────────────────────────────────────────────
1501
1502    #[tokio::test]
1503    async fn test_deliver_namespace_rejects_non_virtiofs_transport() {
1504        use super::super::backend::NamespaceTransport;
1505        use hyprstream_vfs::{Namespace, Subject};
1506
1507        let (backend, _, temp) = create_test_backend();
1508        let sandbox = create_test_sandbox(temp.path().join("sandbox"));
1509
1510        let result = backend
1511            .deliver_namespace(
1512                &sandbox,
1513                Namespace::new(),
1514                Subject::new("test"),
1515                NamespaceTransport::HostImports,
1516            )
1517            .await;
1518
1519        match result {
1520            Err(WorkerError::Unsupported(msg)) => {
1521                assert!(msg.contains("VirtioFs"), "unexpected message: {msg}");
1522            }
1523            other => panic!("expected Unsupported error, got: {other:?}"),
1524        }
1525    }
1526
1527    #[tokio::test]
1528    async fn test_deliver_namespace_serves_composed_namespace_over_virtiofs() {
1529        use super::super::backend::{NamespaceDelivery, NamespaceTransport};
1530        use hyprstream_vfs::{Namespace, Subject};
1531
1532        let (backend, _, temp) = create_test_backend();
1533        let sandbox_path = temp.path().join("sandbox");
1534        std::fs::create_dir_all(&sandbox_path).unwrap();
1535        // No `backend_handle` set — this exercises the "sandbox hasn't
1536        // started yet" path (serve, but skip the hot-attach).
1537        let sandbox = create_test_sandbox(sandbox_path.clone());
1538
1539        let socket_path = sandbox_path.join("deliver-test.sock");
1540        let result = backend
1541            .deliver_namespace(
1542                &sandbox,
1543                Namespace::new(),
1544                Subject::new("test"),
1545                NamespaceTransport::VirtioFs {
1546                    socket_path: socket_path.clone(),
1547                    mount_tag: "hyprstream-vfs".to_owned(),
1548                },
1549            )
1550            .await
1551            .expect("deliver_namespace should serve the namespace");
1552
1553        match result {
1554            NamespaceDelivery::VirtioFs {
1555                socket_path: served,
1556                mount_tag,
1557                guard,
1558            } => {
1559                assert_eq!(served, socket_path);
1560                assert_eq!(mount_tag, "hyprstream-vfs");
1561                assert!(guard.is_some(), "kata should return a serving-thread guard");
1562            }
1563            other => panic!("expected VirtioFs delivery, got: {other:?}"),
1564        }
1565    }
1566
1567    /// #742: the container rootfs mount tag is recorded and read back through
1568    /// the interior-mutable `rootfs_mount_tag` field — including through a
1569    /// *shared* (`Arc`, no `&mut`) handle, which is exactly the constraint the
1570    /// post-start `deliver_namespace` path faces. A fresh handle reads `None`;
1571    /// after `record_rootfs_mount_tag` (the write `deliver_namespace` performs),
1572    /// `rootfs_mount_tag()` (the read `exec_sync` performs) observes the tag,
1573    /// and a later re-record replaces it (last write wins).
1574    #[test]
1575    fn test_rootfs_mount_tag_recorded_through_shared_handle() {
1576        let temp = TempDir::new().unwrap();
1577        let handle = create_test_handle(temp.path());
1578
1579        // Boot with no tenant VFS → no rootfs tag; `exec_sync` would send no
1580        // rootfs storage.
1581        assert_eq!(handle.rootfs_mount_tag(), None);
1582
1583        // Record through a *shared* clone — no `&mut` available, mirroring
1584        // `deliver_namespace` recording on the already-`Arc`-shared handle.
1585        let shared = Arc::clone(&handle);
1586        shared.record_rootfs_mount_tag(ROOTFS_MOUNT_TAG.to_owned());
1587
1588        // The original `Arc` (what `exec_sync` reads) sees the delivered tag.
1589        assert_eq!(handle.rootfs_mount_tag().as_deref(), Some(ROOTFS_MOUNT_TAG));
1590
1591        // Re-delivery re-roots the container: last write wins.
1592        handle.record_rootfs_mount_tag("re-delivered-tag".to_owned());
1593        assert_eq!(
1594            handle.rootfs_mount_tag().as_deref(),
1595            Some("re-delivered-tag")
1596        );
1597    }
1598
1599    #[tokio::test]
1600    async fn test_exec_sync_empty_command_rejected() {
1601        let (backend, _, temp) = create_test_backend();
1602        let sandbox = create_test_sandbox(temp.path().join("sandbox"));
1603
1604        let result = backend.exec_sync(&sandbox, &[], 5).await;
1605
1606        assert!(result.is_err());
1607        match result.unwrap_err() {
1608            WorkerError::ExecFailed(msg) => assert!(msg.contains("empty command")),
1609            other => panic!("Expected ExecFailed, got: {other:?}"),
1610        }
1611    }
1612
1613    #[tokio::test]
1614    async fn test_exec_sync_no_backend_handle_fails() {
1615        // No `backend_handle` set (sandbox never `start()`ed) — exec_sync
1616        // must fail fast rather than panic or hang trying to dial an agent.
1617        let (backend, _, temp) = create_test_backend();
1618        let sandbox = create_test_sandbox(temp.path().join("sandbox"));
1619
1620        let result = backend
1621            .exec_sync(&sandbox, &["echo".into(), "hello".into()], 5)
1622            .await;
1623
1624        assert!(result.is_err());
1625        match result.unwrap_err() {
1626            WorkerError::ExecFailed(msg) => assert!(msg.contains("no backend handle")),
1627            other => panic!("Expected ExecFailed, got: {other:?}"),
1628        }
1629    }
1630
1631    #[tokio::test]
1632    async fn test_exec_sync_unreachable_agent_fails_cleanly() {
1633        // A KataHandle with a real (but not started) CloudHypervisor: calling
1634        // `get_agent_socket()` on a VM that was never `prepare_vm`'d /
1635        // `start_vm`'d should error out (no vsock path configured yet)
1636        // rather than hang — exec_sync must surface that as ExecFailed.
1637        let (backend, _, temp) = create_test_backend();
1638        let sandbox_path = temp.path().join("sandbox");
1639        std::fs::create_dir_all(&sandbox_path).unwrap();
1640        let mut sandbox = create_test_sandbox(sandbox_path.clone());
1641        sandbox.backend_handle = Some(create_test_handle(&sandbox_path));
1642
1643        let result = backend
1644            .exec_sync(&sandbox, &["echo".into(), "hello".into()], 1)
1645            .await;
1646
1647        assert!(
1648            result.is_err(),
1649            "exec against an unprepared VM must fail, not hang"
1650        );
1651    }
1652
1653    #[tokio::test]
1654    async fn test_exec_sync_clears_cached_client_on_failure() {
1655        // Regression: if `KataHandle.agent` caches a ttrpc client whose
1656        // connection is dead (VM restarted/died without a `reset_sandbox`),
1657        // `exec_sync` must NOT keep reusing that stale client on every
1658        // subsequent call. It should clear the cache on failure so the next
1659        // call redials. We seed the cache with a client connected to a
1660        // listener that is then dropped — its first RPC deterministically
1661        // fails — and assert the cache is `None` afterward.
1662        let (backend, _, temp) = create_test_backend();
1663        let sandbox_path = temp.path().join("sandbox");
1664        std::fs::create_dir_all(&sandbox_path).unwrap();
1665        let mut sandbox = create_test_sandbox(sandbox_path.clone());
1666        let handle = create_test_handle(&sandbox_path);
1667
1668        // Build a client over a connection to a transient listener, then
1669        // drop the listener so the connection has no peer — any RPC on it
1670        // fails. This mirrors a cached client whose VM has gone away.
1671        let dead_sock = sandbox_path.join("dead-agent.sock");
1672        let listener = tokio::net::UnixListener::bind(&dead_sock).unwrap();
1673        let client = KataAgentClient::from_ttrpc_client(
1674            ttrpc::r#async::Client::connect(&format!("unix://{}", dead_sock.display()))
1675                .await
1676                .unwrap(),
1677        );
1678        drop(listener);
1679
1680        handle.agent.lock().await.replace(Arc::new(client));
1681        // Keep a clone of the `Arc<KataHandle>` for post-call assertions:
1682        // `backend_handle` takes ownership of the reference we stored, but we
1683        // still need to inspect `agent` afterward (the `Arc` is shared, so
1684        // both observe the same `Mutex`).
1685        let handle_for_assert = Arc::clone(&handle);
1686        sandbox.backend_handle = Some(handle);
1687
1688        // Bounded timeout: a stale connection must not hang the call. The
1689        // underlying ttrpc context timeout (30s) bounds a silent server, but
1690        // a closed peer should fail much faster; cap at 35s to also catch a
1691        // regression where the cache-clear path itself wedges.
1692        let result = tokio::time::timeout(
1693            std::time::Duration::from_secs(35),
1694            backend.exec_sync(&sandbox, &["echo".into(), "hi".into()], 1),
1695        )
1696        .await;
1697        assert!(
1698            result.is_ok(),
1699            "exec_sync against a dead cached client must not hang"
1700        );
1701        assert!(
1702            result.unwrap().is_err(),
1703            "exec against a dead cached client must fail, not silently succeed"
1704        );
1705
1706        // The fix's core guarantee: the stale client was evicted so the next
1707        // call reconnects instead of reusing the dead connection forever.
1708        assert!(
1709            handle_for_assert.agent.lock().await.is_none(),
1710            "cached dead agent client must be cleared on exec failure"
1711        );
1712    }
1713
1714    #[test]
1715    fn test_is_available() {
1716        let (backend, _, _temp) = create_test_backend();
1717        let expected = which::which("cloud-hypervisor").is_ok();
1718        assert_eq!(backend.is_available(), expected);
1719    }
1720
1721    #[tokio::test]
1722    async fn test_initialize_sets_rootless() {
1723        let (backend, _, _temp) = create_test_backend();
1724        let pool_config = PoolConfig::default();
1725
1726        backend.initialize(&pool_config).await.unwrap();
1727
1728        // We're running as non-root in CI/dev, so rootless should be set.
1729        if !nix::unistd::geteuid().is_root() {
1730            assert!(rootless::is_rootless());
1731        }
1732    }
1733
1734    // ─────────────────────────────────────────────────────────────────────
1735    // rootless_kata_runtime_dir — pure path derivation (#743)
1736    // ─────────────────────────────────────────────────────────────────────
1737
1738    #[test]
1739    fn test_rootless_kata_runtime_dir_root_is_none() {
1740        // Rootful (euid 0): behaviour unchanged, no XDG rewrite.
1741        assert_eq!(
1742            rootless_kata_runtime_dir(0, Some("/run/user/0"), Some("/tmp")),
1743            None
1744        );
1745    }
1746
1747    #[test]
1748    fn test_rootless_kata_runtime_dir_uses_xdg() {
1749        // The seam that actually drives kata's rootless jailer base is
1750        // XDG_RUNTIME_DIR (not KATA_PATH); nest kata files under it.
1751        assert_eq!(
1752            rootless_kata_runtime_dir(1000, Some("/run/user/1000"), None),
1753            Some(PathBuf::from("/run/user/1000/kata"))
1754        );
1755    }
1756
1757    #[test]
1758    fn test_rootless_kata_runtime_dir_falls_back_to_tmpdir() {
1759        assert_eq!(
1760            rootless_kata_runtime_dir(1000, None, Some("/var/tmp")),
1761            Some(PathBuf::from("/var/tmp/kata-1000"))
1762        );
1763    }
1764
1765    #[test]
1766    fn test_rootless_kata_runtime_dir_defaults_to_slash_tmp() {
1767        // XDG and TMPDIR both unset/empty → /tmp/kata-<uid> (avoids the
1768        // rootless_dir() unwrap panic on a missing XDG_RUNTIME_DIR).
1769        assert_eq!(
1770            rootless_kata_runtime_dir(1000, None, None),
1771            Some(PathBuf::from("/tmp/kata-1000"))
1772        );
1773        assert_eq!(
1774            rootless_kata_runtime_dir(1000, Some("  "), Some("")),
1775            Some(PathBuf::from("/tmp/kata-1000"))
1776        );
1777    }
1778
1779    // ─────────────────────────────────────────────────────────────────────
1780    // build_hypervisor_config tests
1781    // ─────────────────────────────────────────────────────────────────────
1782
1783    #[test]
1784    fn test_build_hypervisor_config_defaults() {
1785        let pool_config = PoolConfig {
1786            kernel_path: PathBuf::from("/boot/vmlinux"),
1787            vm_image: PathBuf::from("/images/rootfs.img"),
1788            vm_cpus: 4,
1789            vm_memory_mb: 2048,
1790            ..Default::default()
1791        };
1792
1793        let config = KataBackend::build_hypervisor_config(&pool_config);
1794
1795        assert_eq!(config.boot_info.kernel, "/boot/vmlinux");
1796        assert_eq!(config.boot_info.image, "/images/rootfs.img");
1797        assert_eq!(config.cpu_info.default_vcpus, 4.0);
1798        assert_eq!(config.cpu_info.default_maxvcpus, 4);
1799        assert_eq!(config.memory_info.default_memory, 2048);
1800    }
1801
1802    #[test]
1803    fn test_build_hypervisor_config_explicit_path() {
1804        let pool_config = PoolConfig {
1805            hypervisor_path: PathBuf::from("/opt/custom/cloud-hypervisor"),
1806            ..Default::default()
1807        };
1808
1809        let config = KataBackend::build_hypervisor_config(&pool_config);
1810        assert_eq!(config.path, "/opt/custom/cloud-hypervisor");
1811    }
1812
1813    #[test]
1814    fn test_build_hypervisor_config_memory_overflow() {
1815        let pool_config = PoolConfig {
1816            vm_memory_mb: u64::MAX,
1817            ..Default::default()
1818        };
1819
1820        let config = KataBackend::build_hypervisor_config(&pool_config);
1821        assert_eq!(config.memory_info.default_memory, u32::MAX);
1822    }
1823
1824    // ─────────────────────────────────────────────────────────────────────
1825    // KataHandle tests
1826    // ─────────────────────────────────────────────────────────────────────
1827
1828    #[test]
1829    fn test_kata_handle_as_any_downcast() {
1830        let temp = TempDir::new().unwrap();
1831        let handle: Arc<dyn SandboxHandle> = create_test_handle(temp.path());
1832
1833        let kata = handle.as_any().downcast_ref::<KataHandle>();
1834        assert!(kata.is_some(), "downcast to KataHandle should succeed");
1835
1836        let kata = kata.unwrap();
1837        assert_eq!(kata.api_socket, temp.path().join("test.sock"));
1838        assert_eq!(
1839            kata.virtiofs_socket.as_deref(),
1840            Some(temp.path().join("virtiofs.sock").as_path())
1841        );
1842    }
1843
1844    // ─────────────────────────────────────────────────────────────────────
1845    // stop / get_pids / destroy with no handle
1846    // ─────────────────────────────────────────────────────────────────────
1847
1848    #[tokio::test]
1849    async fn test_stop_no_handle() {
1850        let (backend, _, temp) = create_test_backend();
1851        let sandbox = create_test_sandbox(temp.path().join("sandbox"));
1852
1853        // Should succeed (logs warning, no error)
1854        backend.stop(&sandbox).await.unwrap();
1855    }
1856
1857    #[tokio::test]
1858    async fn test_get_pids_no_handle() {
1859        let (backend, _, temp) = create_test_backend();
1860        let sandbox = create_test_sandbox(temp.path().join("sandbox"));
1861
1862        let pids = backend.get_pids(&sandbox).await.unwrap();
1863        assert!(pids.is_empty());
1864    }
1865
1866    #[tokio::test]
1867    async fn test_destroy_no_handle_cleans_dir() {
1868        let (backend, _, temp) = create_test_backend();
1869        let sandbox_path = temp.path().join("sandbox-to-destroy");
1870        std::fs::create_dir_all(&sandbox_path).unwrap();
1871        assert!(sandbox_path.exists());
1872
1873        let sandbox = create_test_sandbox(sandbox_path.clone());
1874        backend.destroy(&sandbox).await.unwrap();
1875
1876        assert!(!sandbox_path.exists(), "sandbox_path should be removed");
1877    }
1878
1879    #[tokio::test]
1880    async fn test_destroy_no_handle_missing_dir() {
1881        let (backend, _, temp) = create_test_backend();
1882        let sandbox_path = temp.path().join("does-not-exist");
1883        assert!(!sandbox_path.exists());
1884
1885        let sandbox = create_test_sandbox(sandbox_path);
1886        // Should succeed even though directory doesn't exist
1887        backend.destroy(&sandbox).await.unwrap();
1888    }
1889
1890    // ─────────────────────────────────────────────────────────────────────
1891    // reset tests
1892    // ─────────────────────────────────────────────────────────────────────
1893
1894    #[tokio::test]
1895    async fn test_reset_no_handle() {
1896        let (backend, _, temp) = create_test_backend();
1897        let mut sandbox = create_test_sandbox(temp.path().join("sandbox"));
1898
1899        let reusable = backend.reset(&mut sandbox).await.unwrap();
1900        assert!(reusable);
1901        assert!(sandbox.backend_handle.is_none());
1902    }
1903
1904    #[tokio::test]
1905    async fn test_reset_preserves_hypervisor() {
1906        let (backend, _, temp) = create_test_backend();
1907        let sandbox_path = temp.path().join("sandbox");
1908        std::fs::create_dir_all(&sandbox_path).unwrap();
1909        let mut sandbox = create_test_sandbox(sandbox_path.clone());
1910
1911        // Set a handle with virtiofs fields populated
1912        let original_handle = create_test_handle(&sandbox_path);
1913        let original_hypervisor_ptr =
1914            Arc::as_ptr(&original_handle.hypervisor) as *const () as usize;
1915        sandbox.backend_handle = Some(original_handle);
1916
1917        let reusable = backend.reset(&mut sandbox).await.unwrap();
1918        assert!(reusable, "Kata reset should return true (reusable)");
1919
1920        let new_handle = sandbox
1921            .backend_handle
1922            .as_ref()
1923            .expect("handle should still be set after reset");
1924        let kata = new_handle
1925            .as_any()
1926            .downcast_ref::<KataHandle>()
1927            .expect("should downcast to KataHandle");
1928
1929        // Hypervisor Arc should point to the same allocation
1930        let new_hypervisor_ptr = Arc::as_ptr(&kata.hypervisor) as *const () as usize;
1931        assert_eq!(
1932            original_hypervisor_ptr, new_hypervisor_ptr,
1933            "reset should preserve the same hypervisor instance"
1934        );
1935
1936        // virtiofs socket should be cleared on reset (the composed VFS server
1937        // is not reused; a recycled sandbox composes a fresh one on next start).
1938        assert!(
1939            kata.virtiofs_socket.is_none(),
1940            "virtiofs_socket should be cleared"
1941        );
1942
1943        // api_socket should be preserved
1944        assert_eq!(kata.api_socket, sandbox_path.join("test.sock"));
1945    }
1946
1947    // ─────────────────────────────────────────────────────────────────────
1948    // Debug impl
1949    // ─────────────────────────────────────────────────────────────────────
1950
1951    #[test]
1952    fn test_debug_impl() {
1953        let (backend, _, _temp) = create_test_backend();
1954        let debug = format!("{backend:?}");
1955        assert!(
1956            debug.contains("KataBackend"),
1957            "debug should contain struct name"
1958        );
1959        assert!(
1960            debug.contains("image_config"),
1961            "debug should contain image_config field"
1962        );
1963    }
1964
1965    // ─────────────────────────────────────────────────────────────────────
1966    // Cloud-init ISO generation (requires genisoimage)
1967    // ─────────────────────────────────────────────────────────────────────
1968
1969    // ─────────────────────────────────────────────────────────────────────
1970    // Resource / stats mapping (T1-D #345) — pure functions, no VM needed
1971    // ─────────────────────────────────────────────────────────────────────
1972
1973    #[test]
1974    fn test_map_linux_resources_maps_cpu_and_memory() {
1975        let resources = LinuxContainerResources {
1976            cpu_shares: 512,
1977            cpu_quota: 50_000,
1978            cpu_period: 100_000,
1979            memory_limit_in_bytes: 256 * 1024 * 1024,
1980            memory_swap_limit_in_bytes: 512 * 1024 * 1024,
1981            cpuset_cpus: "0-1".to_owned(),
1982            cpuset_mems: "0".to_owned(),
1983            ..Default::default()
1984        };
1985
1986        let oci = KataBackend::map_linux_resources(&resources);
1987
1988        assert_eq!(oci.CPU.Shares, 512);
1989        assert_eq!(oci.CPU.Quota, 50_000);
1990        assert_eq!(oci.CPU.Period, 100_000);
1991        assert_eq!(oci.CPU.Cpus, "0-1");
1992        assert_eq!(oci.CPU.Mems, "0");
1993        assert_eq!(oci.Memory.Limit, 256 * 1024 * 1024);
1994        assert_eq!(oci.Memory.Swap, 512 * 1024 * 1024);
1995    }
1996
1997    #[test]
1998    fn test_map_linux_resources_clamps_negative_cpu_to_zero() {
1999        // CRI cpu_shares/cpu_period are i64; OCI Shares/Period are u64. A
2000        // negative "unset" sentinel must not wrap around to a huge u64.
2001        let resources = LinuxContainerResources {
2002            cpu_shares: -1,
2003            cpu_period: -1,
2004            ..Default::default()
2005        };
2006
2007        let oci = KataBackend::map_linux_resources(&resources);
2008        assert_eq!(oci.CPU.Shares, 0);
2009        assert_eq!(oci.CPU.Period, 0);
2010    }
2011
2012    #[test]
2013    fn test_map_container_stats_maps_cgroup_counters() {
2014        let mut cpu_usage = agent::CpuUsage::new();
2015        cpu_usage.total_usage = 123_456;
2016        let mut cpu_stats = agent::CpuStats::new();
2017        cpu_stats.cpu_usage = protobuf::MessageField::some(cpu_usage);
2018
2019        let mut mem_data = agent::MemoryData::new();
2020        mem_data.usage = 1000;
2021        mem_data.limit = 4000;
2022        let mut mem_stats = agent::MemoryStats::new();
2023        mem_stats.usage = protobuf::MessageField::some(mem_data);
2024        mem_stats
2025            .stats
2026            .insert("total_inactive_file".to_owned(), 200);
2027        mem_stats.stats.insert("total_rss".to_owned(), 700);
2028        mem_stats.stats.insert("total_pgfault".to_owned(), 42);
2029        mem_stats.stats.insert("total_pgmajfault".to_owned(), 7);
2030
2031        let mut cgroup = agent::CgroupStats::new();
2032        cgroup.cpu_stats = protobuf::MessageField::some(cpu_stats);
2033        cgroup.memory_stats = protobuf::MessageField::some(mem_stats);
2034
2035        let mut resp = agent::StatsContainerResponse::new();
2036        resp.cgroup_stats = protobuf::MessageField::some(cgroup);
2037
2038        let (cpu, mem) = KataBackend::map_container_stats(&resp);
2039
2040        assert_eq!(cpu.usage_core_nano_seconds, 123_456);
2041        // usage_nano_cores is a rate needing two samples; single-sample = 0.
2042        assert_eq!(cpu.usage_nano_cores, 0);
2043        assert_eq!(mem.usage_bytes, 1000);
2044        assert_eq!(mem.working_set_bytes, 800, "usage - total_inactive_file");
2045        assert_eq!(mem.available_bytes, 3000, "limit - usage");
2046        assert_eq!(mem.rss_bytes, 700);
2047        assert_eq!(mem.page_faults, 42);
2048        assert_eq!(mem.major_page_faults, 7);
2049    }
2050
2051    #[test]
2052    fn test_map_container_stats_empty_response_is_zeroed() {
2053        // A response with no cgroup_stats (default instances) must not panic
2054        // and must yield all-zero counters.
2055        let resp = agent::StatsContainerResponse::new();
2056        let (cpu, mem) = KataBackend::map_container_stats(&resp);
2057        assert_eq!(cpu.usage_core_nano_seconds, 0);
2058        assert_eq!(mem.usage_bytes, 0);
2059        assert_eq!(mem.working_set_bytes, 0);
2060        assert_eq!(mem.available_bytes, 0);
2061    }
2062
2063    #[tokio::test]
2064    #[ignore] // requires genisoimage binary
2065    async fn test_cloud_init_iso_generation() {
2066        let temp = TempDir::new().unwrap();
2067        let sandbox_dir = temp.path().join("sandbox-iso");
2068        std::fs::create_dir_all(&sandbox_dir).unwrap();
2069
2070        let sandbox = create_test_sandbox(sandbox_dir.clone());
2071        let iso_path = sandbox_dir.join("cloud-init.iso");
2072
2073        KataBackend::generate_cloud_init_iso(&sandbox, &iso_path)
2074            .await
2075            .unwrap();
2076
2077        assert!(iso_path.exists(), "ISO file should be created");
2078
2079        // Verify user-data was written with sandbox ID
2080        let user_data = tokio::fs::read_to_string(sandbox_dir.join("user-data"))
2081            .await
2082            .unwrap();
2083        assert!(user_data.contains("#cloud-config"));
2084        assert!(user_data.contains(&sandbox.id));
2085
2086        // Verify meta-data was written with sandbox ID
2087        let meta_data = tokio::fs::read_to_string(sandbox_dir.join("meta-data"))
2088            .await
2089            .unwrap();
2090        assert!(meta_data.contains(&format!("instance-id: {}", sandbox.id)));
2091    }
2092
2093    #[tokio::test]
2094    #[ignore] // requires genisoimage binary
2095    async fn test_cloud_init_uses_metadata_name() {
2096        let temp = TempDir::new().unwrap();
2097        let sandbox_dir = temp.path().join("sandbox-named");
2098        std::fs::create_dir_all(&sandbox_dir).unwrap();
2099
2100        let mut sandbox = create_test_sandbox(sandbox_dir.clone());
2101        sandbox.metadata.name = "my-pod".to_owned();
2102
2103        let iso_path = sandbox_dir.join("cloud-init.iso");
2104        KataBackend::generate_cloud_init_iso(&sandbox, &iso_path)
2105            .await
2106            .unwrap();
2107
2108        let user_data = tokio::fs::read_to_string(sandbox_dir.join("user-data"))
2109            .await
2110            .unwrap();
2111        assert!(
2112            user_data.contains("hostname: my-pod"),
2113            "should use metadata name as hostname, got: {user_data}"
2114        );
2115
2116        let meta_data = tokio::fs::read_to_string(sandbox_dir.join("meta-data"))
2117            .await
2118            .unwrap();
2119        assert!(
2120            meta_data.contains("local-hostname: my-pod"),
2121            "should use metadata name as local-hostname, got: {meta_data}"
2122        );
2123    }
2124}