hyprstream_workers/runtime/sandbox_fs.rs
1//! Per-sandbox VFS composition + serve (FS-D, #365).
2//!
3//! This is the end-to-end integration that ties FS-A/FS-B/FS-C together for the
4//! Worker GA filesystem: for **each** sandbox it builds a private VFS
5//! [`Namespace`], composes the image filesystem + injected mounts into it, and
6//! serves it over a per-sandbox Unix socket that the Cloud Hypervisor guest
7//! attaches as a virtio-fs ShareFs device.
8//!
9//! ```text
10//! per-sandbox flow (one per sandbox, fully isolated)
11//!
12//! RafsStore (shared, RO) injected mounts (per-sandbox)
13//! │ /stream (StreamMount)
14//! image_fs_for(...) ──┐ /models (SyntheticMount)
15//! (ImageFs: RAFS lower + │ /deltas (SyntheticMount)
16//! per-sandbox writable │ │
17//! upper, OverlayFs) ▼ ▼
18//! Namespace::fork() + bind_mount (FS-C)
19//! │ bound to this sandbox's Subject
20//! ▼
21//! VfsFileSystem (FS-A down-adapter, Subject-per-call)
22//! ▼
23//! serve() on <sandbox>/vfs.sock (FS-A vhost-user-fs)
24//! ▼
25//! attach_share_fs → CH ShareFs device (kata_backend)
26//! ```
27//!
28//! ## Tenant isolation (#365 / #353 / #319 / #328)
29//!
30//! Each sandbox gets its **own** forked `Namespace`, its **own** writable rootfs
31//! upper, its **own** injected-mount registries, its **own** socket, and is
32//! served under its **own** [`Subject`]. Sandbox A's namespace contains only
33//! A's rootfs and injected paths; it has no mount for B's rootfs or B's
34//! injected registries, so A cannot name — let alone read — B's tree. The
35//! `Subject` is threaded into every VFS op as the policy/audit principal at the
36//! Mount boundary.
37//!
38//! ## Fail-closed
39//!
40//! Composition hard-errors if the rootfs cannot be built (missing RAFS
41//! bootstrap, etc.); we never serve a partially-composed namespace. The
42//! injected mounts are read-only except `/stream/{topic}/ctl`.
43
44#![cfg(not(target_arch = "wasm32"))]
45
46use std::path::{Path, PathBuf};
47use std::sync::Arc;
48
49use hyprstream_vfs::injected::{StreamMount, StreamRegistry, SyntheticMount, SyntheticNode};
50use hyprstream_vfs::{Mount, Namespace, Subject};
51use hyprstream_vfs_server::{serve, serve_local, VfsFileSystem};
52
53use crate::error::{Result, WorkerError};
54use crate::image::{image_fs_for, RafsStore};
55
56/// Default guest mount points for the injected mounts.
57const ROOTFS_PREFIX: &str = "/";
58const STREAM_PREFIX: &str = "/stream";
59const MODELS_PREFIX: &str = "/models";
60const DELTAS_PREFIX: &str = "/deltas";
61
62/// File name of the per-sandbox VFS socket under the sandbox dir.
63pub const VFS_SOCKET_NAME: &str = "vfs.sock";
64
65/// The injected mounts composed into a sandbox's namespace, plus the handles
66/// the runtime keeps to feed them at runtime.
67pub struct InjectedMounts {
68 /// `/stream` registry — push blocks / mark complete here at runtime.
69 pub stream_registry: Arc<StreamRegistry>,
70}
71
72/// A fully composed, per-sandbox VFS: the namespace, its Subject, and the
73/// runtime handles for the injected mounts.
74pub struct SandboxFs {
75 /// The composed namespace (rootfs + injected mounts), private to one sandbox.
76 namespace: Namespace,
77 /// The policy/audit principal this namespace is served under.
78 subject: Subject,
79 /// Runtime handles for the injected mounts.
80 injected: InjectedMounts,
81 /// A clone of the tenant **root** `Mount` (the `/` image filesystem), kept so
82 /// the same Subject-scoped Mount can be exported over a *second* transport
83 /// (e.g. the kata guest's 9P-over-vsock channel, #731) without re-composing
84 /// or duplicating the writable-upper CoW layer. The virtio-fs path still
85 /// serves the whole `namespace`; this is only the root export root.
86 root: Arc<dyn Mount>,
87}
88
89impl SandboxFs {
90 /// Compose a per-sandbox VFS namespace for `image_id`, bound to `subject`.
91 ///
92 /// Steps (the FS-D core):
93 /// 1. Fork an empty base namespace ([`Namespace::fork`] of `new()` — every
94 /// sandbox starts from a clean, private mount table).
95 /// 2. Mount the **image filesystem** ([`image_fs_for`]) at `/`. The RAFS image
96 /// is the shared RO lower; a per-sandbox writable upper under
97 /// `<sandbox_dir>/rootfs/` is the CoW layer. It is the root purely because
98 /// it is mounted at `/` — "root" is a mount position, not a type (#633).
99 /// 3. Bind the **injected mounts** — `/stream` (native pipes), `/models`,
100 /// `/deltas` (synthetic, read-only) — into the namespace.
101 ///
102 /// `models` / `deltas` are passed as synthetic trees so the caller decides
103 /// what to expose (e.g. available model ids / delta metadata for this
104 /// tenant). Pass empty dirs to expose just the mount point.
105 pub fn compose(
106 rafs_store: &RafsStore,
107 image_id: &str,
108 sandbox_dir: &Path,
109 subject: Subject,
110 models: SyntheticNode,
111 deltas: SyntheticNode,
112 ) -> Result<Self> {
113 // 1. Per-sandbox private namespace.
114 let mut namespace = Namespace::new().fork();
115
116 // 2. Image filesystem at "/" (ImageFs: RAFS lower + per-sandbox
117 // writable upper). It is the root purely because the namespace recipe
118 // mounts it here — "root" is a mount position, not a type (#633).
119 let bootstrap = rafs_store.bootstrap_path(image_id);
120 let image_fs = image_fs_for(&bootstrap, rafs_store.blobs_dir(), image_id, sandbox_dir)?;
121 // `FsMount: Mount`, so the image filesystem coerces to the `Arc<dyn Mount>`
122 // the namespace stores; the down-adapter recovers the `FsMount` vtable via
123 // `Mount::as_fsmount` for writes/copy-up.
124 let root_mount: Arc<dyn Mount> = Arc::new(image_fs);
125 // Keep a clone so the root export root can be served over a second
126 // transport (9P-over-vsock, #731) — same Arc, so no second CoW upper.
127 let root = Arc::clone(&root_mount);
128 namespace
129 .mount(ROOTFS_PREFIX, root_mount)
130 .map_err(|e| WorkerError::SandboxCreationFailed(format!("mount root: {e}")))?;
131
132 // 3. Injected mounts — per-sandbox, never shared with another sandbox.
133 let stream_registry = Arc::new(StreamRegistry::new());
134 let stream_mount: Arc<dyn Mount> = Arc::new(StreamMount::new(stream_registry.clone()));
135 namespace
136 .mount(STREAM_PREFIX, stream_mount)
137 .map_err(|e| WorkerError::SandboxCreationFailed(format!("mount /stream: {e}")))?;
138
139 let models_mount: Arc<dyn Mount> = Arc::new(SyntheticMount::new(models));
140 namespace
141 .mount(MODELS_PREFIX, models_mount)
142 .map_err(|e| WorkerError::SandboxCreationFailed(format!("mount /models: {e}")))?;
143
144 let deltas_mount: Arc<dyn Mount> = Arc::new(SyntheticMount::new(deltas));
145 namespace
146 .mount(DELTAS_PREFIX, deltas_mount)
147 .map_err(|e| WorkerError::SandboxCreationFailed(format!("mount /deltas: {e}")))?;
148
149 Ok(Self {
150 namespace,
151 subject,
152 injected: InjectedMounts { stream_registry },
153 root,
154 })
155 }
156
157 /// Wrap an **already-composed** namespace for serving, without doing any
158 /// image composition here (#635).
159 ///
160 /// Used by [`SandboxBackend::deliver_namespace`](super::backend::SandboxBackend::deliver_namespace):
161 /// as the #508/#635 narrowing lands, composition becomes the caller's job
162 /// (the universal filesystem-service Mount, or #634's shared
163 /// namespace-composition builder) and the backend's job narrows to
164 /// serving + attaching the transport. There is no fresh
165 /// [`StreamRegistry`] wired into `namespace` here — the caller is
166 /// responsible for mounting whatever injected registries it needs before
167 /// calling this; the placeholder `InjectedMounts` returned by
168 /// [`Self::injected`] is unused in this path.
169 pub fn from_namespace(namespace: Namespace, subject: Subject) -> Self {
170 Self {
171 namespace,
172 subject,
173 injected: InjectedMounts {
174 stream_registry: Arc::new(StreamRegistry::new()),
175 },
176 // This path composes elsewhere and does not expose a distinct root
177 // export root for a second transport; an empty synthetic dir stands in
178 // (the 9P-over-vsock channel is only wired on the `compose` path).
179 root: Arc::new(SyntheticMount::new(SyntheticNode::dir())),
180 }
181 }
182
183 /// A clone of the tenant **root** `Mount` (the `/` image filesystem).
184 ///
185 /// The same Subject-scoped Mount the virtio-fs namespace roots on — exported
186 /// so a *second* transport (the kata guest's 9P-over-vsock channel, #731) can
187 /// serve it without re-composing or duplicating the writable-upper CoW layer.
188 pub fn root_mount(&self) -> Arc<dyn Mount> {
189 Arc::clone(&self.root)
190 }
191
192 /// The composed namespace (read-only borrow; e.g. for tests / inspection).
193 pub fn namespace(&self) -> &Namespace {
194 &self.namespace
195 }
196
197 /// The Subject this namespace is served under.
198 pub fn subject(&self) -> &Subject {
199 &self.subject
200 }
201
202 /// Runtime handles for the injected mounts.
203 pub fn injected(&self) -> &InjectedMounts {
204 &self.injected
205 }
206
207 /// Build the FS-A [`VfsFileSystem`] down-adapter from this composed namespace,
208 /// consuming `self` (the namespace moves into the adapter).
209 ///
210 /// `rt` is a tokio runtime handle the adapter uses to drive the async VFS ops
211 /// from the server's synchronous (OS-thread) vring loop.
212 pub fn into_filesystem(self, rt: tokio::runtime::Handle) -> (VfsFileSystem, InjectedMounts) {
213 let fs = VfsFileSystem::new(self.namespace, self.subject, rt);
214 (fs, self.injected)
215 }
216
217 /// Compose, then **serve** the namespace over a per-sandbox Unix socket.
218 ///
219 /// Spawns a dedicated OS thread running FS-A's blocking [`serve`] daemon
220 /// (which `VhostUserDaemon::wait()`s until CH disconnects). Returns a
221 /// [`SandboxFsServer`] holding the socket path and the injected handles; the
222 /// caller then attaches the socket to CH via
223 /// [`attach_share_fs`](super::kata_backend::KataBackend) before `start_vm`.
224 ///
225 /// `rt` is captured for the down-adapter's async→sync bridge.
226 pub fn serve_on(
227 self,
228 socket_path: PathBuf,
229 rt: tokio::runtime::Handle,
230 ) -> Result<SandboxFsServer> {
231 let (fs, injected) = self.into_filesystem(rt);
232
233 let sock_str = socket_path
234 .to_str()
235 .ok_or_else(|| {
236 WorkerError::SandboxCreationFailed("VFS socket path contains invalid UTF-8".into())
237 })?
238 .to_owned();
239
240 // FS-A's `serve` blocks until the VMM disconnects; run it on a dedicated
241 // OS thread (the down-adapter's `block_on` uses the captured runtime
242 // handle, not this thread's executor).
243 let thread = std::thread::Builder::new()
244 .name(format!("vfs-serve-{}", socket_filename(&socket_path)))
245 .spawn(move || {
246 if let Err(e) = serve(fs, &sock_str) {
247 tracing::error!(socket = %sock_str, error = %e, "VFS server exited with error");
248 }
249 })
250 .map_err(|e| {
251 WorkerError::SandboxCreationFailed(format!("spawn VFS server thread: {e}"))
252 })?;
253
254 Ok(SandboxFsServer {
255 socket_path,
256 injected,
257 _thread: thread,
258 })
259 }
260
261 /// FUSE-mount this composed namespace at `mountpoint` as a real host
262 /// directory — the **non-VM sibling of [`serve_on`]** (Model B, #715).
263 ///
264 /// Where [`serve_on`] serves the [`VfsFileSystem`] to a **VM** guest over
265 /// vhost-user-fs (virtio-fs), this serves the *same* down-adapter to the
266 /// **host kernel** over `/dev/fuse` via
267 /// [`hyprstream_vfs_server::serve_local`], so a non-VM sandbox (e.g.
268 /// `systemd-nspawn --directory=<mountpoint>`) can POSIX-root its workload
269 /// directly in the tenant's composed VFS. Any host process that enters the
270 /// sandbox — the inference workload, a shell — then sees the tenant VFS as
271 /// its filesystem, closing the "run a task" gap (#506) for the POSIX case.
272 ///
273 /// ## Subject preservation (MAC-conservative)
274 ///
275 /// The [`Subject`] is preserved **automatically**: [`into_filesystem`] bakes
276 /// it into the [`VfsFileSystem`], so every VFS op driven over the FUSE
277 /// channel is attributed to this sandbox's single Subject at the Mount
278 /// boundary (Subject-per-call). Because the mount is **per-sandbox** and
279 /// served under **one** Subject, there is no uid→Subject mapping to invent
280 /// (and none is invented): every operation over this mount is correctly
281 /// attributed to the sandbox principal, exactly as the virtio-fs path is.
282 ///
283 /// Rootless-userns uid-mapping / `allow_other` nuances (e.g. an unprivileged
284 /// `/dev/fuse` under a podman `--userns=keep-id` remapping) are **out of
285 /// scope** for the nspawn target, which runs closer to the host; they are
286 /// deferred to the OCI/rootless-FUSE feasibility spike (#715 / #617).
287 ///
288 /// [`serve_local`] **blocks** until the mount is torn down, so it runs on a
289 /// dedicated OS thread; the returned [`SandboxFsLocalMount`] unmounts and
290 /// joins that thread on drop (RAII), mirroring [`SandboxFsServer`].
291 ///
292 /// `mountpoint` is created (`create_dir_all`) if absent. `rt` is captured for
293 /// the down-adapter's async→sync bridge.
294 ///
295 /// [`into_filesystem`]: SandboxFs::into_filesystem
296 pub fn serve_local_at(
297 self,
298 rt: tokio::runtime::Handle,
299 mountpoint: PathBuf,
300 ) -> Result<SandboxFsLocalMount> {
301 std::fs::create_dir_all(&mountpoint).map_err(|e| {
302 WorkerError::SandboxCreationFailed(format!(
303 "create FUSE mountpoint {}: {e}",
304 mountpoint.display()
305 ))
306 })?;
307
308 let (fs, injected) = self.into_filesystem(rt);
309
310 let mp = mountpoint.clone();
311 // `serve_local` blocks until the FUSE session ends; run it on a dedicated
312 // OS thread (the down-adapter's `block_on` uses the captured runtime
313 // handle, not this thread's executor).
314 let thread = std::thread::Builder::new()
315 .name(format!("vfs-fuse-{}", socket_filename(&mountpoint)))
316 .spawn(move || {
317 if let Err(e) = serve_local(fs, &mp) {
318 tracing::error!(
319 mountpoint = %mp.display(),
320 error = %e,
321 "FUSE VFS mount exited with error"
322 );
323 }
324 })
325 .map_err(|e| {
326 WorkerError::SandboxCreationFailed(format!("spawn FUSE VFS mount thread: {e}"))
327 })?;
328
329 Ok(SandboxFsLocalMount {
330 mountpoint,
331 injected,
332 thread: Some(thread),
333 })
334 }
335}
336
337/// A running per-sandbox VFS server: owns the socket path and injected handles.
338///
339/// Dropping it does not forcibly stop the daemon (the daemon exits when CH
340/// disconnects); the join handle is detached.
341pub struct SandboxFsServer {
342 socket_path: PathBuf,
343 injected: InjectedMounts,
344 _thread: std::thread::JoinHandle<()>,
345}
346
347impl SandboxFsServer {
348 /// The Unix socket the namespace is served on (attach this to CH).
349 pub fn socket_path(&self) -> &Path {
350 &self.socket_path
351 }
352
353 /// Runtime handles for the injected mounts (feed `/stream` here).
354 pub fn injected(&self) -> &InjectedMounts {
355 &self.injected
356 }
357}
358
359/// A running per-sandbox VFS **FUSE mount** at a host directory: the non-VM
360/// sibling of [`SandboxFsServer`] (Model B, #715).
361///
362/// Owns the mountpoint, the injected-mount handles, and the OS thread running
363/// the blocking [`hyprstream_vfs_server::serve_local`] session loop. Dropping it
364/// unmounts the FUSE mount (`fusermount3 -u`, which is unprivileged for a mount
365/// owned by this uid) — which makes the kernel close the `/dev/fuse` session so
366/// the serving thread's loop returns — then joins the thread. This is what makes
367/// the tenant VFS disappear from the host when the sandbox is stopped/destroyed.
368pub struct SandboxFsLocalMount {
369 mountpoint: PathBuf,
370 injected: InjectedMounts,
371 thread: Option<std::thread::JoinHandle<()>>,
372}
373
374impl SandboxFsLocalMount {
375 /// The host directory the tenant VFS is FUSE-mounted at (give this to the
376 /// container as its root, or as a bind-mount source).
377 pub fn mountpoint(&self) -> &Path {
378 &self.mountpoint
379 }
380
381 /// Runtime handles for the injected mounts (feed `/stream` here).
382 pub fn injected(&self) -> &InjectedMounts {
383 &self.injected
384 }
385}
386
387impl std::fmt::Debug for SandboxFsLocalMount {
388 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
389 f.debug_struct("SandboxFsLocalMount")
390 .field("mountpoint", &self.mountpoint)
391 .finish_non_exhaustive()
392 }
393}
394
395impl Drop for SandboxFsLocalMount {
396 fn drop(&mut self) {
397 // `serve_local` blocks until the FUSE session ends; unmount to make it
398 // return. A non-lazy `fusermount3 -u` tears down the kernel mount
399 // unprivileged (for a mount owned by this uid), which the serving thread
400 // observes as the session closing and exits its loop. Best-effort: if
401 // the mount never came up (e.g. no `/dev/fuse`), the unmount is a
402 // harmless no-op and the thread has already exited.
403 //
404 // If the clean unmount fails because the mount is still busy (a workload
405 // process still has it open), fall back to a LAZY unmount (`-uz`): the
406 // kernel detaches the mount as soon as it stops being referenced, which
407 // still closes the FUSE session so the serving thread exits. Without this
408 // fallback a busy mount would leave the session open and the
409 // `thread.join()` below would block forever, hanging the runtime thread
410 // that dropped this handle.
411 if !fusermount_unmount(&self.mountpoint, false) {
412 let _ = fusermount_unmount(&self.mountpoint, true);
413 }
414 if let Some(thread) = self.thread.take() {
415 let _ = thread.join();
416 }
417 }
418}
419
420/// Unmount the FUSE mount at `mountpoint`, trying `fusermount3` then the older
421/// `fusermount`. With `lazy = true` uses `-uz` (detach-when-idle), which
422/// succeeds even when the mount is busy; with `lazy = false` uses a clean `-u`.
423/// Returns whether either tool reported success.
424fn fusermount_unmount(mountpoint: &Path, lazy: bool) -> bool {
425 let flag = if lazy { "-uz" } else { "-u" };
426 for tool in ["fusermount3", "fusermount"] {
427 let ok = std::process::Command::new(tool)
428 .arg(flag)
429 .arg(mountpoint)
430 .status()
431 .map(|s| s.success())
432 .unwrap_or(false);
433 if ok {
434 return true;
435 }
436 }
437 false
438}
439
440fn socket_filename(path: &Path) -> String {
441 path.file_name()
442 .map(|n| n.to_string_lossy().into_owned())
443 .unwrap_or_else(|| "sock".to_owned())
444}
445
446#[cfg(test)]
447#[allow(clippy::unwrap_used, clippy::expect_used)]
448mod tests {
449 use super::*;
450 use flate2::write::GzEncoder;
451 use flate2::Compression;
452
453 use crate::config::ImageConfig;
454 use crate::image::rafs_builder::build_rafs_bootstrap;
455
456 /// Build a `RafsStore` whose image dirs are rooted under `root`.
457 fn make_store(root: &Path) -> RafsStore {
458 let config = ImageConfig {
459 blobs_dir: root.join("blobs"),
460 bootstrap_dir: root.join("bootstrap"),
461 refs_dir: root.join("refs"),
462 cache_dir: root.join("cache"),
463 runtime_dir: root.join("runtime"),
464 ..ImageConfig::default()
465 };
466 for d in [&config.blobs_dir, &config.bootstrap_dir, &config.cache_dir] {
467 std::fs::create_dir_all(d).unwrap();
468 }
469 RafsStore::new(config).expect("create store")
470 }
471
472 /// Build a gzip tar layer of regular files.
473 fn make_gzip_layer(path: &Path, entries: &[(&str, &[u8])]) {
474 let file = std::fs::File::create(path).unwrap();
475 let gz = GzEncoder::new(file, Compression::default());
476 let mut tar = tar::Builder::new(gz);
477 for (name, data) in entries {
478 let mut header = tar::Header::new_gnu();
479 header.set_size(data.len() as u64);
480 header.set_mode(0o644);
481 header.set_mtime(0);
482 header.set_entry_type(tar::EntryType::Regular);
483 tar.append_data(&mut header, name, *data).unwrap();
484 }
485 let gz = tar.into_inner().unwrap();
486 gz.finish().unwrap();
487 }
488
489 /// Build image content (`image_id`) into an existing store's layout.
490 fn add_image(store: &RafsStore, scratch: &Path, image_id: &str, entries: &[(&str, &[u8])]) {
491 let bootstrap = store.bootstrap_path(image_id);
492 if let Some(parent) = bootstrap.parent() {
493 std::fs::create_dir_all(parent).unwrap();
494 }
495 let layer = scratch.join(format!("{}.tar.gz", image_id.replace(':', "_")));
496 make_gzip_layer(&layer, entries);
497 build_rafs_bootstrap(&[layer], store.blobs_dir(), &bootstrap).unwrap();
498 }
499
500 /// A `RafsStore` rooted under `root` carrying one image.
501 fn store_with_image(root: &Path, image_id: &str, entries: &[(&str, &[u8])]) -> RafsStore {
502 let store = make_store(root);
503 add_image(&store, root, image_id, entries);
504 store
505 }
506
507 fn models_tree() -> SyntheticNode {
508 SyntheticNode::dir().with_child(
509 "qwen3",
510 SyntheticNode::dir().with_child("info", SyntheticNode::file(b"qwen3\n".to_vec())),
511 )
512 }
513
514 /// Core: a composed namespace exposes the rootfs + injected mounts, all
515 /// served under the sandbox's Subject.
516 #[tokio::test]
517 async fn composed_namespace_exposes_rootfs_and_injected() {
518 let tmp = tempfile::tempdir().unwrap();
519 let store = store_with_image(tmp.path(), "img:a", &[("etc/hostname", b"worker-a\n")]);
520 let sandbox_dir = tmp.path().join("sandbox-a");
521 let subj = Subject::new("tenant-a");
522
523 let fs = SandboxFs::compose(
524 &store,
525 "img:a",
526 &sandbox_dir,
527 subj.clone(),
528 models_tree(),
529 SyntheticNode::dir(),
530 )
531 .unwrap();
532 let ns = fs.namespace();
533
534 // Rootfs file (from the RAFS lower) is readable via the namespace.
535 let hostname = ns.cat("/etc/hostname", &subj).await.unwrap();
536 assert_eq!(hostname, b"worker-a\n");
537
538 // Injected /models synthetic file is readable.
539 let info = ns.cat("/models/qwen3/info", &subj).await.unwrap();
540 assert_eq!(info, b"qwen3\n");
541
542 // Injected mount points are present at the namespace root.
543 let prefixes = ns.mount_prefixes();
544 assert!(prefixes.contains(&"/"));
545 assert!(prefixes.contains(&"/stream"));
546 assert!(prefixes.contains(&"/models"));
547 assert!(prefixes.contains(&"/deltas"));
548
549 // The /stream registry is reachable and per-sandbox.
550 fs.injected()
551 .stream_registry
552 .register("job-1", Some("tenant-a".into()));
553 fs.injected()
554 .stream_registry
555 .push("job-1", b"chunk\n".to_vec());
556 let chunk = ns.read_one("/stream/job-1/data", &subj).await.unwrap();
557 assert_eq!(chunk, b"chunk\n");
558 }
559
560 /// Isolation: two sandboxes get independent namespaces; A cannot read B's
561 /// rootfs content nor B's injected stream topics, and vice versa.
562 #[tokio::test]
563 async fn two_sandboxes_are_isolated() {
564 let tmp = tempfile::tempdir().unwrap();
565 let store = make_store(tmp.path());
566
567 // Two distinct images, each with a tenant-specific marker file.
568 add_image(
569 &store,
570 tmp.path(),
571 "img:a",
572 &[("etc/secret", b"SECRET-A\n")],
573 );
574 add_image(
575 &store,
576 tmp.path(),
577 "img:b",
578 &[("etc/secret", b"SECRET-B\n")],
579 );
580
581 let subj_a = Subject::new("tenant-a");
582 let subj_b = Subject::new("tenant-b");
583 let fs_a = SandboxFs::compose(
584 &store,
585 "img:a",
586 &tmp.path().join("sandbox-a"),
587 subj_a.clone(),
588 SyntheticNode::dir(),
589 SyntheticNode::dir(),
590 )
591 .unwrap();
592 let fs_b = SandboxFs::compose(
593 &store,
594 "img:b",
595 &tmp.path().join("sandbox-b"),
596 subj_b.clone(),
597 SyntheticNode::dir(),
598 SyntheticNode::dir(),
599 )
600 .unwrap();
601
602 // Each sees only its own rootfs content.
603 assert_eq!(
604 fs_a.namespace().cat("/etc/secret", &subj_a).await.unwrap(),
605 b"SECRET-A\n"
606 );
607 assert_eq!(
608 fs_b.namespace().cat("/etc/secret", &subj_b).await.unwrap(),
609 b"SECRET-B\n"
610 );
611
612 // Injected stream topics are per-sandbox: registering in A is invisible
613 // in B's namespace (separate registries — A's path does not exist in B).
614 fs_a.injected()
615 .stream_registry
616 .register("a-only", Some("tenant-a".into()));
617 fs_a.injected()
618 .stream_registry
619 .push("a-only", b"a-data".to_vec());
620 // A can read it.
621 assert_eq!(
622 fs_a.namespace()
623 .read_one("/stream/a-only/data", &subj_a)
624 .await
625 .unwrap(),
626 b"a-data"
627 );
628 // B's namespace has no such topic: walk fails.
629 assert!(fs_b
630 .namespace()
631 .read_one("/stream/a-only/data", &subj_b)
632 .await
633 .is_err());
634
635 // B's writable rootfs upper is a distinct directory from A's: a write in
636 // A must not appear in B.
637 let _ = std::fs::write(
638 tmp.path()
639 .join("sandbox-a")
640 .join("rootfs/upper/etc/written-by-a"),
641 b"x",
642 );
643 assert!(fs_b
644 .namespace()
645 .cat("/etc/written-by-a", &subj_b)
646 .await
647 .is_err());
648 }
649
650 /// Fail-closed: composing with a missing image bootstrap hard-errors rather
651 /// than serving an empty / partial namespace.
652 #[test]
653 fn compose_missing_image_fails_closed() {
654 let tmp = tempfile::tempdir().unwrap();
655 let store = make_store(tmp.path());
656 let res = SandboxFs::compose(
657 &store,
658 "img:does-not-exist",
659 &tmp.path().join("sandbox"),
660 Subject::new("t"),
661 SyntheticNode::dir(),
662 SyntheticNode::dir(),
663 );
664 assert!(res.is_err(), "missing image must fail-closed");
665 }
666
667 /// The composed namespace can be wrapped as the FS-A down-adapter
668 /// (`VfsFileSystem`), proving the serve path's first step type-checks and the
669 /// namespace is `Send` (moved into the adapter).
670 #[tokio::test]
671 async fn composed_namespace_builds_filesystem() {
672 let tmp = tempfile::tempdir().unwrap();
673 let store = store_with_image(tmp.path(), "img:fs", &[("etc/hostname", b"h\n")]);
674 let fs = SandboxFs::compose(
675 &store,
676 "img:fs",
677 &tmp.path().join("sandbox"),
678 Subject::new("t"),
679 SyntheticNode::dir(),
680 SyntheticNode::dir(),
681 )
682 .unwrap();
683 let (_vfs, injected) = fs.into_filesystem(tokio::runtime::Handle::current());
684 // Injected handles survive the move into the adapter.
685 injected.stream_registry.register("x", None);
686 assert!(injected.stream_registry.exists("x"));
687 }
688
689 /// Model B (#715): `serve_local_at` is the FUSE sibling of `serve_on`. It
690 /// returns an RAII [`SandboxFsLocalMount`] carrying the mountpoint + injected
691 /// handles, and dropping it tears the mount down. A real `/dev/fuse` mount
692 /// can't be exercised in most CI/build sandboxes (no device node or usable
693 /// `fusermount3`), so this asserts the handle *shape* only — the serving
694 /// thread may fail to mount and exit immediately, which drop tolerates. The
695 /// live end-to-end mount is validated on a real host (mirroring the
696 /// `hyprstream-vfs-server` `local_fuse_mount_read_and_readdir` test, which
697 /// self-skips when the environment can't do a real FUSE mount).
698 #[test]
699 fn serve_local_at_returns_raii_mount_handle() {
700 let tmp = tempfile::tempdir().unwrap();
701 let store = store_with_image(tmp.path(), "img:l", &[("etc/hostname", b"h\n")]);
702 let fs = SandboxFs::compose(
703 &store,
704 "img:l",
705 &tmp.path().join("sandbox"),
706 Subject::new("tenant"),
707 SyntheticNode::dir(),
708 SyntheticNode::dir(),
709 )
710 .unwrap();
711
712 // The down-adapter's `block_on` must run on a runtime distinct from the
713 // calling thread, so a multi-thread handle held for the whole test.
714 let rt = tokio::runtime::Builder::new_multi_thread()
715 .worker_threads(1)
716 .enable_all()
717 .build()
718 .unwrap();
719
720 let mountpoint = tmp.path().join("vfs-root");
721 let mount = fs
722 .serve_local_at(rt.handle().clone(), mountpoint.clone())
723 .expect("serve_local_at returns a mount handle even if the mount can't come up");
724
725 // `serve_local_at` created the mountpoint dir and reports it back.
726 assert!(mountpoint.is_dir(), "mountpoint should be created");
727 assert_eq!(mount.mountpoint(), mountpoint);
728
729 // Injected handles survive into the RAII mount handle.
730 mount.injected().stream_registry.register("x", None);
731 assert!(mount.injected().stream_registry.exists("x"));
732
733 // Drop unmounts + joins; tolerates a mount that never came up.
734 drop(mount);
735 }
736
737 /// The FS-A down-adapter serves the composed namespace: driving the
738 /// `FileSystem` trait (the surface the vhost-user-fs server dispatches to)
739 /// resolves both the rootfs and the injected mount points. This is the
740 /// adapter-level proof of "the server serves the composed Namespace"; the
741 /// full vhost-user daemon over a live CH VM is the out-of-scope live
742 /// validation.
743 #[test]
744 fn down_adapter_serves_composed_namespace() {
745 use fuse_backend_rs::api::filesystem::{Context, FileSystem};
746 use std::ffi::CString;
747
748 let tmp = tempfile::tempdir().unwrap();
749 let store = store_with_image(tmp.path(), "img:s", &[("etc/hostname", b"served\n")]);
750 let fs = SandboxFs::compose(
751 &store,
752 "img:s",
753 &tmp.path().join("sandbox"),
754 Subject::new("tenant"),
755 models_tree(),
756 SyntheticNode::dir(),
757 )
758 .unwrap();
759
760 // Build the down-adapter on a dedicated multi-thread runtime (the
761 // adapter's block_on must not run on the calling thread).
762 // Held for the whole test: the adapter's `block_on` must run on a
763 // runtime distinct from this (calling) thread, so a multi-thread handle.
764 let rt = tokio::runtime::Builder::new_multi_thread()
765 .worker_threads(1)
766 .enable_all()
767 .build()
768 .unwrap();
769 let (vfs, _injected) = fs.into_filesystem(rt.handle().clone());
770
771 let ctx = Context::default();
772 let root = 1u64; // FUSE_ROOT_ID
773 let cstr = |s: &str| CString::new(s).unwrap();
774
775 // Rootfs: /etc/hostname resolves through the `/`-rooted overlay mount.
776 let etc = vfs.lookup(&ctx, root, &cstr("etc")).unwrap();
777 assert_eq!(etc.attr.st_mode & libc::S_IFMT, libc::S_IFDIR);
778 let hostname = vfs.lookup(&ctx, etc.inode, &cstr("hostname")).unwrap();
779 assert_eq!(hostname.attr.st_mode & libc::S_IFMT, libc::S_IFREG);
780
781 // Injected mount points are visible as directories at the namespace root.
782 for name in ["stream", "models", "deltas"] {
783 let entry = vfs.lookup(&ctx, root, &cstr(name)).unwrap();
784 assert_eq!(
785 entry.attr.st_mode & libc::S_IFMT,
786 libc::S_IFDIR,
787 "{name} should be a directory"
788 );
789 }
790
791 // Injected synthetic file resolves through the adapter.
792 let models = vfs.lookup(&ctx, root, &cstr("models")).unwrap();
793 let qwen = vfs.lookup(&ctx, models.inode, &cstr("qwen3")).unwrap();
794 let info = vfs.lookup(&ctx, qwen.inode, &cstr("info")).unwrap();
795 assert_eq!(info.attr.st_mode & libc::S_IFMT, libc::S_IFREG);
796 }
797}