Skip to main content

hyprstream_workers/runtime/
sandbox_fs.rs

1//! Per-sandbox VFS composition + serve (FS-D, #365).
2//!
3//! This is the end-to-end integration that ties FS-A/FS-B/FS-C together for the
4//! Worker GA filesystem: for **each** sandbox it builds a private VFS
5//! [`Namespace`], composes the image filesystem + injected mounts into it, and
6//! serves it over a per-sandbox Unix socket that the Cloud Hypervisor guest
7//! attaches as a virtio-fs ShareFs device.
8//!
9//! ```text
10//!   per-sandbox flow (one per sandbox, fully isolated)
11//!
12//!   RafsStore (shared, RO)         injected mounts (per-sandbox)
13//!        │                          /stream  (StreamMount)
14//!   image_fs_for(...)      ──┐      /models  (SyntheticMount)
15//!   (ImageFs: RAFS lower +  │      /deltas  (SyntheticMount)
16//!    per-sandbox writable   │          │
17//!    upper, OverlayFs)      ▼          ▼
18//!                       Namespace::fork() + bind_mount  (FS-C)
19//!                              │  bound to this sandbox's Subject
20//!                              ▼
21//!                       VfsFileSystem (FS-A down-adapter, Subject-per-call)
22//!                              ▼
23//!                       serve() on  <sandbox>/vfs.sock   (FS-A vhost-user-fs)
24//!                              ▼
25//!                       attach_share_fs  →  CH ShareFs device  (kata_backend)
26//! ```
27//!
28//! ## Tenant isolation (#365 / #353 / #319 / #328)
29//!
30//! Each sandbox gets its **own** forked `Namespace`, its **own** writable rootfs
31//! upper, its **own** injected-mount registries, its **own** socket, and is
32//! served under its **own** [`Subject`]. Sandbox A's namespace contains only
33//! A's rootfs and injected paths; it has no mount for B's rootfs or B's
34//! injected registries, so A cannot name — let alone read — B's tree. The
35//! `Subject` is threaded into every VFS op as the policy/audit principal at the
36//! Mount boundary.
37//!
38//! ## Fail-closed
39//!
40//! Composition hard-errors if the rootfs cannot be built (missing RAFS
41//! bootstrap, etc.); we never serve a partially-composed namespace. The
42//! injected mounts are read-only except `/stream/{topic}/ctl`.
43
44#![cfg(not(target_arch = "wasm32"))]
45
46use std::path::{Path, PathBuf};
47use std::sync::Arc;
48
49use hyprstream_vfs::injected::{StreamMount, StreamRegistry, SyntheticMount, SyntheticNode};
50use hyprstream_vfs::{Mount, Namespace, Subject};
51use hyprstream_vfs_server::{serve, serve_local, VfsFileSystem};
52
53use crate::error::{Result, WorkerError};
54use crate::image::{image_fs_for, RafsStore};
55
56/// Default guest mount points for the injected mounts.
57const ROOTFS_PREFIX: &str = "/";
58const STREAM_PREFIX: &str = "/stream";
59const MODELS_PREFIX: &str = "/models";
60const DELTAS_PREFIX: &str = "/deltas";
61
62/// File name of the per-sandbox VFS socket under the sandbox dir.
63pub const VFS_SOCKET_NAME: &str = "vfs.sock";
64
65/// The injected mounts composed into a sandbox's namespace, plus the handles
66/// the runtime keeps to feed them at runtime.
67pub struct InjectedMounts {
68    /// `/stream` registry — push blocks / mark complete here at runtime.
69    pub stream_registry: Arc<StreamRegistry>,
70}
71
72/// A fully composed, per-sandbox VFS: the namespace, its Subject, and the
73/// runtime handles for the injected mounts.
74pub struct SandboxFs {
75    /// The composed namespace (rootfs + injected mounts), private to one sandbox.
76    namespace: Namespace,
77    /// The policy/audit principal this namespace is served under.
78    subject: Subject,
79    /// Runtime handles for the injected mounts.
80    injected: InjectedMounts,
81    /// A clone of the tenant **root** `Mount` (the `/` image filesystem), kept so
82    /// the same Subject-scoped Mount can be exported over a *second* transport
83    /// (e.g. the kata guest's 9P-over-vsock channel, #731) without re-composing
84    /// or duplicating the writable-upper CoW layer. The virtio-fs path still
85    /// serves the whole `namespace`; this is only the root export root.
86    root: Arc<dyn Mount>,
87}
88
89impl SandboxFs {
90    /// Compose a per-sandbox VFS namespace for `image_id`, bound to `subject`.
91    ///
92    /// Steps (the FS-D core):
93    /// 1. Fork an empty base namespace ([`Namespace::fork`] of `new()` — every
94    ///    sandbox starts from a clean, private mount table).
95    /// 2. Mount the **image filesystem** ([`image_fs_for`]) at `/`. The RAFS image
96    ///    is the shared RO lower; a per-sandbox writable upper under
97    ///    `<sandbox_dir>/rootfs/` is the CoW layer. It is the root purely because
98    ///    it is mounted at `/` — "root" is a mount position, not a type (#633).
99    /// 3. Bind the **injected mounts** — `/stream` (native pipes), `/models`,
100    ///    `/deltas` (synthetic, read-only) — into the namespace.
101    ///
102    /// `models` / `deltas` are passed as synthetic trees so the caller decides
103    /// what to expose (e.g. available model ids / delta metadata for this
104    /// tenant). Pass empty dirs to expose just the mount point.
105    pub fn compose(
106        rafs_store: &RafsStore,
107        image_id: &str,
108        sandbox_dir: &Path,
109        subject: Subject,
110        models: SyntheticNode,
111        deltas: SyntheticNode,
112    ) -> Result<Self> {
113        // 1. Per-sandbox private namespace.
114        let mut namespace = Namespace::new().fork();
115
116        // 2. Image filesystem at "/" (ImageFs: RAFS lower + per-sandbox
117        // writable upper). It is the root purely because the namespace recipe
118        // mounts it here — "root" is a mount position, not a type (#633).
119        let bootstrap = rafs_store.bootstrap_path(image_id);
120        let image_fs = image_fs_for(&bootstrap, rafs_store.blobs_dir(), image_id, sandbox_dir)?;
121        // `FsMount: Mount`, so the image filesystem coerces to the `Arc<dyn Mount>`
122        // the namespace stores; the down-adapter recovers the `FsMount` vtable via
123        // `Mount::as_fsmount` for writes/copy-up.
124        let root_mount: Arc<dyn Mount> = Arc::new(image_fs);
125        // Keep a clone so the root export root can be served over a second
126        // transport (9P-over-vsock, #731) — same Arc, so no second CoW upper.
127        let root = Arc::clone(&root_mount);
128        namespace
129            .mount(ROOTFS_PREFIX, root_mount)
130            .map_err(|e| WorkerError::SandboxCreationFailed(format!("mount root: {e}")))?;
131
132        // 3. Injected mounts — per-sandbox, never shared with another sandbox.
133        let stream_registry = Arc::new(StreamRegistry::new());
134        let stream_mount: Arc<dyn Mount> = Arc::new(StreamMount::new(stream_registry.clone()));
135        namespace
136            .mount(STREAM_PREFIX, stream_mount)
137            .map_err(|e| WorkerError::SandboxCreationFailed(format!("mount /stream: {e}")))?;
138
139        let models_mount: Arc<dyn Mount> = Arc::new(SyntheticMount::new(models));
140        namespace
141            .mount(MODELS_PREFIX, models_mount)
142            .map_err(|e| WorkerError::SandboxCreationFailed(format!("mount /models: {e}")))?;
143
144        let deltas_mount: Arc<dyn Mount> = Arc::new(SyntheticMount::new(deltas));
145        namespace
146            .mount(DELTAS_PREFIX, deltas_mount)
147            .map_err(|e| WorkerError::SandboxCreationFailed(format!("mount /deltas: {e}")))?;
148
149        Ok(Self {
150            namespace,
151            subject,
152            injected: InjectedMounts { stream_registry },
153            root,
154        })
155    }
156
157    /// Wrap an **already-composed** namespace for serving, without doing any
158    /// image composition here (#635).
159    ///
160    /// Used by [`SandboxBackend::deliver_namespace`](super::backend::SandboxBackend::deliver_namespace):
161    /// as the #508/#635 narrowing lands, composition becomes the caller's job
162    /// (the universal filesystem-service Mount, or #634's shared
163    /// namespace-composition builder) and the backend's job narrows to
164    /// serving + attaching the transport. There is no fresh
165    /// [`StreamRegistry`] wired into `namespace` here — the caller is
166    /// responsible for mounting whatever injected registries it needs before
167    /// calling this; the placeholder `InjectedMounts` returned by
168    /// [`Self::injected`] is unused in this path.
169    pub fn from_namespace(namespace: Namespace, subject: Subject) -> Self {
170        Self {
171            namespace,
172            subject,
173            injected: InjectedMounts {
174                stream_registry: Arc::new(StreamRegistry::new()),
175            },
176            // This path composes elsewhere and does not expose a distinct root
177            // export root for a second transport; an empty synthetic dir stands in
178            // (the 9P-over-vsock channel is only wired on the `compose` path).
179            root: Arc::new(SyntheticMount::new(SyntheticNode::dir())),
180        }
181    }
182
183    /// A clone of the tenant **root** `Mount` (the `/` image filesystem).
184    ///
185    /// The same Subject-scoped Mount the virtio-fs namespace roots on — exported
186    /// so a *second* transport (the kata guest's 9P-over-vsock channel, #731) can
187    /// serve it without re-composing or duplicating the writable-upper CoW layer.
188    pub fn root_mount(&self) -> Arc<dyn Mount> {
189        Arc::clone(&self.root)
190    }
191
192    /// The composed namespace (read-only borrow; e.g. for tests / inspection).
193    pub fn namespace(&self) -> &Namespace {
194        &self.namespace
195    }
196
197    /// The Subject this namespace is served under.
198    pub fn subject(&self) -> &Subject {
199        &self.subject
200    }
201
202    /// Runtime handles for the injected mounts.
203    pub fn injected(&self) -> &InjectedMounts {
204        &self.injected
205    }
206
207    /// Build the FS-A [`VfsFileSystem`] down-adapter from this composed namespace,
208    /// consuming `self` (the namespace moves into the adapter).
209    ///
210    /// `rt` is a tokio runtime handle the adapter uses to drive the async VFS ops
211    /// from the server's synchronous (OS-thread) vring loop.
212    pub fn into_filesystem(self, rt: tokio::runtime::Handle) -> (VfsFileSystem, InjectedMounts) {
213        let fs = VfsFileSystem::new(self.namespace, self.subject, rt);
214        (fs, self.injected)
215    }
216
217    /// Compose, then **serve** the namespace over a per-sandbox Unix socket.
218    ///
219    /// Spawns a dedicated OS thread running FS-A's blocking [`serve`] daemon
220    /// (which `VhostUserDaemon::wait()`s until CH disconnects). Returns a
221    /// [`SandboxFsServer`] holding the socket path and the injected handles; the
222    /// caller then attaches the socket to CH via
223    /// [`attach_share_fs`](super::kata_backend::KataBackend) before `start_vm`.
224    ///
225    /// `rt` is captured for the down-adapter's async→sync bridge.
226    pub fn serve_on(
227        self,
228        socket_path: PathBuf,
229        rt: tokio::runtime::Handle,
230    ) -> Result<SandboxFsServer> {
231        let (fs, injected) = self.into_filesystem(rt);
232
233        let sock_str = socket_path
234            .to_str()
235            .ok_or_else(|| {
236                WorkerError::SandboxCreationFailed("VFS socket path contains invalid UTF-8".into())
237            })?
238            .to_owned();
239
240        // FS-A's `serve` blocks until the VMM disconnects; run it on a dedicated
241        // OS thread (the down-adapter's `block_on` uses the captured runtime
242        // handle, not this thread's executor).
243        let thread = std::thread::Builder::new()
244            .name(format!("vfs-serve-{}", socket_filename(&socket_path)))
245            .spawn(move || {
246                if let Err(e) = serve(fs, &sock_str) {
247                    tracing::error!(socket = %sock_str, error = %e, "VFS server exited with error");
248                }
249            })
250            .map_err(|e| {
251                WorkerError::SandboxCreationFailed(format!("spawn VFS server thread: {e}"))
252            })?;
253
254        Ok(SandboxFsServer {
255            socket_path,
256            injected,
257            _thread: thread,
258        })
259    }
260
261    /// FUSE-mount this composed namespace at `mountpoint` as a real host
262    /// directory — the **non-VM sibling of [`serve_on`]** (Model B, #715).
263    ///
264    /// Where [`serve_on`] serves the [`VfsFileSystem`] to a **VM** guest over
265    /// vhost-user-fs (virtio-fs), this serves the *same* down-adapter to the
266    /// **host kernel** over `/dev/fuse` via
267    /// [`hyprstream_vfs_server::serve_local`], so a non-VM sandbox (e.g.
268    /// `systemd-nspawn --directory=<mountpoint>`) can POSIX-root its workload
269    /// directly in the tenant's composed VFS. Any host process that enters the
270    /// sandbox — the inference workload, a shell — then sees the tenant VFS as
271    /// its filesystem, closing the "run a task" gap (#506) for the POSIX case.
272    ///
273    /// ## Subject preservation (MAC-conservative)
274    ///
275    /// The [`Subject`] is preserved **automatically**: [`into_filesystem`] bakes
276    /// it into the [`VfsFileSystem`], so every VFS op driven over the FUSE
277    /// channel is attributed to this sandbox's single Subject at the Mount
278    /// boundary (Subject-per-call). Because the mount is **per-sandbox** and
279    /// served under **one** Subject, there is no uid→Subject mapping to invent
280    /// (and none is invented): every operation over this mount is correctly
281    /// attributed to the sandbox principal, exactly as the virtio-fs path is.
282    ///
283    /// Rootless-userns uid-mapping / `allow_other` nuances (e.g. an unprivileged
284    /// `/dev/fuse` under a podman `--userns=keep-id` remapping) are **out of
285    /// scope** for the nspawn target, which runs closer to the host; they are
286    /// deferred to the OCI/rootless-FUSE feasibility spike (#715 / #617).
287    ///
288    /// [`serve_local`] **blocks** until the mount is torn down, so it runs on a
289    /// dedicated OS thread; the returned [`SandboxFsLocalMount`] unmounts and
290    /// joins that thread on drop (RAII), mirroring [`SandboxFsServer`].
291    ///
292    /// `mountpoint` is created (`create_dir_all`) if absent. `rt` is captured for
293    /// the down-adapter's async→sync bridge.
294    ///
295    /// [`into_filesystem`]: SandboxFs::into_filesystem
296    pub fn serve_local_at(
297        self,
298        rt: tokio::runtime::Handle,
299        mountpoint: PathBuf,
300    ) -> Result<SandboxFsLocalMount> {
301        std::fs::create_dir_all(&mountpoint).map_err(|e| {
302            WorkerError::SandboxCreationFailed(format!(
303                "create FUSE mountpoint {}: {e}",
304                mountpoint.display()
305            ))
306        })?;
307
308        let (fs, injected) = self.into_filesystem(rt);
309
310        let mp = mountpoint.clone();
311        // `serve_local` blocks until the FUSE session ends; run it on a dedicated
312        // OS thread (the down-adapter's `block_on` uses the captured runtime
313        // handle, not this thread's executor).
314        let thread = std::thread::Builder::new()
315            .name(format!("vfs-fuse-{}", socket_filename(&mountpoint)))
316            .spawn(move || {
317                if let Err(e) = serve_local(fs, &mp) {
318                    tracing::error!(
319                        mountpoint = %mp.display(),
320                        error = %e,
321                        "FUSE VFS mount exited with error"
322                    );
323                }
324            })
325            .map_err(|e| {
326                WorkerError::SandboxCreationFailed(format!("spawn FUSE VFS mount thread: {e}"))
327            })?;
328
329        Ok(SandboxFsLocalMount {
330            mountpoint,
331            injected,
332            thread: Some(thread),
333        })
334    }
335}
336
337/// A running per-sandbox VFS server: owns the socket path and injected handles.
338///
339/// Dropping it does not forcibly stop the daemon (the daemon exits when CH
340/// disconnects); the join handle is detached.
341pub struct SandboxFsServer {
342    socket_path: PathBuf,
343    injected: InjectedMounts,
344    _thread: std::thread::JoinHandle<()>,
345}
346
347impl SandboxFsServer {
348    /// The Unix socket the namespace is served on (attach this to CH).
349    pub fn socket_path(&self) -> &Path {
350        &self.socket_path
351    }
352
353    /// Runtime handles for the injected mounts (feed `/stream` here).
354    pub fn injected(&self) -> &InjectedMounts {
355        &self.injected
356    }
357}
358
359/// A running per-sandbox VFS **FUSE mount** at a host directory: the non-VM
360/// sibling of [`SandboxFsServer`] (Model B, #715).
361///
362/// Owns the mountpoint, the injected-mount handles, and the OS thread running
363/// the blocking [`hyprstream_vfs_server::serve_local`] session loop. Dropping it
364/// unmounts the FUSE mount (`fusermount3 -u`, which is unprivileged for a mount
365/// owned by this uid) — which makes the kernel close the `/dev/fuse` session so
366/// the serving thread's loop returns — then joins the thread. This is what makes
367/// the tenant VFS disappear from the host when the sandbox is stopped/destroyed.
368pub struct SandboxFsLocalMount {
369    mountpoint: PathBuf,
370    injected: InjectedMounts,
371    thread: Option<std::thread::JoinHandle<()>>,
372}
373
374impl SandboxFsLocalMount {
375    /// The host directory the tenant VFS is FUSE-mounted at (give this to the
376    /// container as its root, or as a bind-mount source).
377    pub fn mountpoint(&self) -> &Path {
378        &self.mountpoint
379    }
380
381    /// Runtime handles for the injected mounts (feed `/stream` here).
382    pub fn injected(&self) -> &InjectedMounts {
383        &self.injected
384    }
385}
386
387impl std::fmt::Debug for SandboxFsLocalMount {
388    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
389        f.debug_struct("SandboxFsLocalMount")
390            .field("mountpoint", &self.mountpoint)
391            .finish_non_exhaustive()
392    }
393}
394
395impl Drop for SandboxFsLocalMount {
396    fn drop(&mut self) {
397        // `serve_local` blocks until the FUSE session ends; unmount to make it
398        // return. A non-lazy `fusermount3 -u` tears down the kernel mount
399        // unprivileged (for a mount owned by this uid), which the serving thread
400        // observes as the session closing and exits its loop. Best-effort: if
401        // the mount never came up (e.g. no `/dev/fuse`), the unmount is a
402        // harmless no-op and the thread has already exited.
403        //
404        // If the clean unmount fails because the mount is still busy (a workload
405        // process still has it open), fall back to a LAZY unmount (`-uz`): the
406        // kernel detaches the mount as soon as it stops being referenced, which
407        // still closes the FUSE session so the serving thread exits. Without this
408        // fallback a busy mount would leave the session open and the
409        // `thread.join()` below would block forever, hanging the runtime thread
410        // that dropped this handle.
411        if !fusermount_unmount(&self.mountpoint, false) {
412            let _ = fusermount_unmount(&self.mountpoint, true);
413        }
414        if let Some(thread) = self.thread.take() {
415            let _ = thread.join();
416        }
417    }
418}
419
420/// Unmount the FUSE mount at `mountpoint`, trying `fusermount3` then the older
421/// `fusermount`. With `lazy = true` uses `-uz` (detach-when-idle), which
422/// succeeds even when the mount is busy; with `lazy = false` uses a clean `-u`.
423/// Returns whether either tool reported success.
424fn fusermount_unmount(mountpoint: &Path, lazy: bool) -> bool {
425    let flag = if lazy { "-uz" } else { "-u" };
426    for tool in ["fusermount3", "fusermount"] {
427        let ok = std::process::Command::new(tool)
428            .arg(flag)
429            .arg(mountpoint)
430            .status()
431            .map(|s| s.success())
432            .unwrap_or(false);
433        if ok {
434            return true;
435        }
436    }
437    false
438}
439
440fn socket_filename(path: &Path) -> String {
441    path.file_name()
442        .map(|n| n.to_string_lossy().into_owned())
443        .unwrap_or_else(|| "sock".to_owned())
444}
445
446#[cfg(test)]
447#[allow(clippy::unwrap_used, clippy::expect_used)]
448mod tests {
449    use super::*;
450    use flate2::write::GzEncoder;
451    use flate2::Compression;
452
453    use crate::config::ImageConfig;
454    use crate::image::rafs_builder::build_rafs_bootstrap;
455
456    /// Build a `RafsStore` whose image dirs are rooted under `root`.
457    fn make_store(root: &Path) -> RafsStore {
458        let config = ImageConfig {
459            blobs_dir: root.join("blobs"),
460            bootstrap_dir: root.join("bootstrap"),
461            refs_dir: root.join("refs"),
462            cache_dir: root.join("cache"),
463            runtime_dir: root.join("runtime"),
464            ..ImageConfig::default()
465        };
466        for d in [&config.blobs_dir, &config.bootstrap_dir, &config.cache_dir] {
467            std::fs::create_dir_all(d).unwrap();
468        }
469        RafsStore::new(config).expect("create store")
470    }
471
472    /// Build a gzip tar layer of regular files.
473    fn make_gzip_layer(path: &Path, entries: &[(&str, &[u8])]) {
474        let file = std::fs::File::create(path).unwrap();
475        let gz = GzEncoder::new(file, Compression::default());
476        let mut tar = tar::Builder::new(gz);
477        for (name, data) in entries {
478            let mut header = tar::Header::new_gnu();
479            header.set_size(data.len() as u64);
480            header.set_mode(0o644);
481            header.set_mtime(0);
482            header.set_entry_type(tar::EntryType::Regular);
483            tar.append_data(&mut header, name, *data).unwrap();
484        }
485        let gz = tar.into_inner().unwrap();
486        gz.finish().unwrap();
487    }
488
489    /// Build image content (`image_id`) into an existing store's layout.
490    fn add_image(store: &RafsStore, scratch: &Path, image_id: &str, entries: &[(&str, &[u8])]) {
491        let bootstrap = store.bootstrap_path(image_id);
492        if let Some(parent) = bootstrap.parent() {
493            std::fs::create_dir_all(parent).unwrap();
494        }
495        let layer = scratch.join(format!("{}.tar.gz", image_id.replace(':', "_")));
496        make_gzip_layer(&layer, entries);
497        build_rafs_bootstrap(&[layer], store.blobs_dir(), &bootstrap).unwrap();
498    }
499
500    /// A `RafsStore` rooted under `root` carrying one image.
501    fn store_with_image(root: &Path, image_id: &str, entries: &[(&str, &[u8])]) -> RafsStore {
502        let store = make_store(root);
503        add_image(&store, root, image_id, entries);
504        store
505    }
506
507    fn models_tree() -> SyntheticNode {
508        SyntheticNode::dir().with_child(
509            "qwen3",
510            SyntheticNode::dir().with_child("info", SyntheticNode::file(b"qwen3\n".to_vec())),
511        )
512    }
513
514    /// Core: a composed namespace exposes the rootfs + injected mounts, all
515    /// served under the sandbox's Subject.
516    #[tokio::test]
517    async fn composed_namespace_exposes_rootfs_and_injected() {
518        let tmp = tempfile::tempdir().unwrap();
519        let store = store_with_image(tmp.path(), "img:a", &[("etc/hostname", b"worker-a\n")]);
520        let sandbox_dir = tmp.path().join("sandbox-a");
521        let subj = Subject::new("tenant-a");
522
523        let fs = SandboxFs::compose(
524            &store,
525            "img:a",
526            &sandbox_dir,
527            subj.clone(),
528            models_tree(),
529            SyntheticNode::dir(),
530        )
531        .unwrap();
532        let ns = fs.namespace();
533
534        // Rootfs file (from the RAFS lower) is readable via the namespace.
535        let hostname = ns.cat("/etc/hostname", &subj).await.unwrap();
536        assert_eq!(hostname, b"worker-a\n");
537
538        // Injected /models synthetic file is readable.
539        let info = ns.cat("/models/qwen3/info", &subj).await.unwrap();
540        assert_eq!(info, b"qwen3\n");
541
542        // Injected mount points are present at the namespace root.
543        let prefixes = ns.mount_prefixes();
544        assert!(prefixes.contains(&"/"));
545        assert!(prefixes.contains(&"/stream"));
546        assert!(prefixes.contains(&"/models"));
547        assert!(prefixes.contains(&"/deltas"));
548
549        // The /stream registry is reachable and per-sandbox.
550        fs.injected()
551            .stream_registry
552            .register("job-1", Some("tenant-a".into()));
553        fs.injected()
554            .stream_registry
555            .push("job-1", b"chunk\n".to_vec());
556        let chunk = ns.read_one("/stream/job-1/data", &subj).await.unwrap();
557        assert_eq!(chunk, b"chunk\n");
558    }
559
560    /// Isolation: two sandboxes get independent namespaces; A cannot read B's
561    /// rootfs content nor B's injected stream topics, and vice versa.
562    #[tokio::test]
563    async fn two_sandboxes_are_isolated() {
564        let tmp = tempfile::tempdir().unwrap();
565        let store = make_store(tmp.path());
566
567        // Two distinct images, each with a tenant-specific marker file.
568        add_image(
569            &store,
570            tmp.path(),
571            "img:a",
572            &[("etc/secret", b"SECRET-A\n")],
573        );
574        add_image(
575            &store,
576            tmp.path(),
577            "img:b",
578            &[("etc/secret", b"SECRET-B\n")],
579        );
580
581        let subj_a = Subject::new("tenant-a");
582        let subj_b = Subject::new("tenant-b");
583        let fs_a = SandboxFs::compose(
584            &store,
585            "img:a",
586            &tmp.path().join("sandbox-a"),
587            subj_a.clone(),
588            SyntheticNode::dir(),
589            SyntheticNode::dir(),
590        )
591        .unwrap();
592        let fs_b = SandboxFs::compose(
593            &store,
594            "img:b",
595            &tmp.path().join("sandbox-b"),
596            subj_b.clone(),
597            SyntheticNode::dir(),
598            SyntheticNode::dir(),
599        )
600        .unwrap();
601
602        // Each sees only its own rootfs content.
603        assert_eq!(
604            fs_a.namespace().cat("/etc/secret", &subj_a).await.unwrap(),
605            b"SECRET-A\n"
606        );
607        assert_eq!(
608            fs_b.namespace().cat("/etc/secret", &subj_b).await.unwrap(),
609            b"SECRET-B\n"
610        );
611
612        // Injected stream topics are per-sandbox: registering in A is invisible
613        // in B's namespace (separate registries — A's path does not exist in B).
614        fs_a.injected()
615            .stream_registry
616            .register("a-only", Some("tenant-a".into()));
617        fs_a.injected()
618            .stream_registry
619            .push("a-only", b"a-data".to_vec());
620        // A can read it.
621        assert_eq!(
622            fs_a.namespace()
623                .read_one("/stream/a-only/data", &subj_a)
624                .await
625                .unwrap(),
626            b"a-data"
627        );
628        // B's namespace has no such topic: walk fails.
629        assert!(fs_b
630            .namespace()
631            .read_one("/stream/a-only/data", &subj_b)
632            .await
633            .is_err());
634
635        // B's writable rootfs upper is a distinct directory from A's: a write in
636        // A must not appear in B.
637        let _ = std::fs::write(
638            tmp.path()
639                .join("sandbox-a")
640                .join("rootfs/upper/etc/written-by-a"),
641            b"x",
642        );
643        assert!(fs_b
644            .namespace()
645            .cat("/etc/written-by-a", &subj_b)
646            .await
647            .is_err());
648    }
649
650    /// Fail-closed: composing with a missing image bootstrap hard-errors rather
651    /// than serving an empty / partial namespace.
652    #[test]
653    fn compose_missing_image_fails_closed() {
654        let tmp = tempfile::tempdir().unwrap();
655        let store = make_store(tmp.path());
656        let res = SandboxFs::compose(
657            &store,
658            "img:does-not-exist",
659            &tmp.path().join("sandbox"),
660            Subject::new("t"),
661            SyntheticNode::dir(),
662            SyntheticNode::dir(),
663        );
664        assert!(res.is_err(), "missing image must fail-closed");
665    }
666
667    /// The composed namespace can be wrapped as the FS-A down-adapter
668    /// (`VfsFileSystem`), proving the serve path's first step type-checks and the
669    /// namespace is `Send` (moved into the adapter).
670    #[tokio::test]
671    async fn composed_namespace_builds_filesystem() {
672        let tmp = tempfile::tempdir().unwrap();
673        let store = store_with_image(tmp.path(), "img:fs", &[("etc/hostname", b"h\n")]);
674        let fs = SandboxFs::compose(
675            &store,
676            "img:fs",
677            &tmp.path().join("sandbox"),
678            Subject::new("t"),
679            SyntheticNode::dir(),
680            SyntheticNode::dir(),
681        )
682        .unwrap();
683        let (_vfs, injected) = fs.into_filesystem(tokio::runtime::Handle::current());
684        // Injected handles survive the move into the adapter.
685        injected.stream_registry.register("x", None);
686        assert!(injected.stream_registry.exists("x"));
687    }
688
689    /// Model B (#715): `serve_local_at` is the FUSE sibling of `serve_on`. It
690    /// returns an RAII [`SandboxFsLocalMount`] carrying the mountpoint + injected
691    /// handles, and dropping it tears the mount down. A real `/dev/fuse` mount
692    /// can't be exercised in most CI/build sandboxes (no device node or usable
693    /// `fusermount3`), so this asserts the handle *shape* only — the serving
694    /// thread may fail to mount and exit immediately, which drop tolerates. The
695    /// live end-to-end mount is validated on a real host (mirroring the
696    /// `hyprstream-vfs-server` `local_fuse_mount_read_and_readdir` test, which
697    /// self-skips when the environment can't do a real FUSE mount).
698    #[test]
699    fn serve_local_at_returns_raii_mount_handle() {
700        let tmp = tempfile::tempdir().unwrap();
701        let store = store_with_image(tmp.path(), "img:l", &[("etc/hostname", b"h\n")]);
702        let fs = SandboxFs::compose(
703            &store,
704            "img:l",
705            &tmp.path().join("sandbox"),
706            Subject::new("tenant"),
707            SyntheticNode::dir(),
708            SyntheticNode::dir(),
709        )
710        .unwrap();
711
712        // The down-adapter's `block_on` must run on a runtime distinct from the
713        // calling thread, so a multi-thread handle held for the whole test.
714        let rt = tokio::runtime::Builder::new_multi_thread()
715            .worker_threads(1)
716            .enable_all()
717            .build()
718            .unwrap();
719
720        let mountpoint = tmp.path().join("vfs-root");
721        let mount = fs
722            .serve_local_at(rt.handle().clone(), mountpoint.clone())
723            .expect("serve_local_at returns a mount handle even if the mount can't come up");
724
725        // `serve_local_at` created the mountpoint dir and reports it back.
726        assert!(mountpoint.is_dir(), "mountpoint should be created");
727        assert_eq!(mount.mountpoint(), mountpoint);
728
729        // Injected handles survive into the RAII mount handle.
730        mount.injected().stream_registry.register("x", None);
731        assert!(mount.injected().stream_registry.exists("x"));
732
733        // Drop unmounts + joins; tolerates a mount that never came up.
734        drop(mount);
735    }
736
737    /// The FS-A down-adapter serves the composed namespace: driving the
738    /// `FileSystem` trait (the surface the vhost-user-fs server dispatches to)
739    /// resolves both the rootfs and the injected mount points. This is the
740    /// adapter-level proof of "the server serves the composed Namespace"; the
741    /// full vhost-user daemon over a live CH VM is the out-of-scope live
742    /// validation.
743    #[test]
744    fn down_adapter_serves_composed_namespace() {
745        use fuse_backend_rs::api::filesystem::{Context, FileSystem};
746        use std::ffi::CString;
747
748        let tmp = tempfile::tempdir().unwrap();
749        let store = store_with_image(tmp.path(), "img:s", &[("etc/hostname", b"served\n")]);
750        let fs = SandboxFs::compose(
751            &store,
752            "img:s",
753            &tmp.path().join("sandbox"),
754            Subject::new("tenant"),
755            models_tree(),
756            SyntheticNode::dir(),
757        )
758        .unwrap();
759
760        // Build the down-adapter on a dedicated multi-thread runtime (the
761        // adapter's block_on must not run on the calling thread).
762        // Held for the whole test: the adapter's `block_on` must run on a
763        // runtime distinct from this (calling) thread, so a multi-thread handle.
764        let rt = tokio::runtime::Builder::new_multi_thread()
765            .worker_threads(1)
766            .enable_all()
767            .build()
768            .unwrap();
769        let (vfs, _injected) = fs.into_filesystem(rt.handle().clone());
770
771        let ctx = Context::default();
772        let root = 1u64; // FUSE_ROOT_ID
773        let cstr = |s: &str| CString::new(s).unwrap();
774
775        // Rootfs: /etc/hostname resolves through the `/`-rooted overlay mount.
776        let etc = vfs.lookup(&ctx, root, &cstr("etc")).unwrap();
777        assert_eq!(etc.attr.st_mode & libc::S_IFMT, libc::S_IFDIR);
778        let hostname = vfs.lookup(&ctx, etc.inode, &cstr("hostname")).unwrap();
779        assert_eq!(hostname.attr.st_mode & libc::S_IFMT, libc::S_IFREG);
780
781        // Injected mount points are visible as directories at the namespace root.
782        for name in ["stream", "models", "deltas"] {
783            let entry = vfs.lookup(&ctx, root, &cstr(name)).unwrap();
784            assert_eq!(
785                entry.attr.st_mode & libc::S_IFMT,
786                libc::S_IFDIR,
787                "{name} should be a directory"
788            );
789        }
790
791        // Injected synthetic file resolves through the adapter.
792        let models = vfs.lookup(&ctx, root, &cstr("models")).unwrap();
793        let qwen = vfs.lookup(&ctx, models.inode, &cstr("qwen3")).unwrap();
794        let info = vfs.lookup(&ctx, qwen.inode, &cstr("info")).unwrap();
795        assert_eq!(info.attr.st_mode & libc::S_IFMT, libc::S_IFREG);
796    }
797}