hyprstream_workers/image/image_fs.rs
1//! **ImageFs** — a universal mountable OCI/RAFS image filesystem service (#633,
2//! spine of epic #508).
3//!
4//! An `ImageFs` is an [`FsMount`](hyprstream_vfs::FsMount): an `OverlayFs` whose
5//!
6//! - **lower** is the image's RAFS loaded **in-process** as a
7//! `fuse_backend_rs::FileSystem` (via `nydus-rafs`'s [`Rafs`], which also
8//! implements the overlay [`Layer`](fuse_backend_rs::api::filesystem::Layer)
9//! trait) — read-only, lazily fetching chunks from the CAS / Dragonfly P2P, so
10//! nothing is materialised on disk up front; and
11//! - **upper** is a per-sandbox writable directory (the copy-up / whiteout
12//! target), so writes never touch the shared image.
13//!
14//! "Root" is a mount position, not a type: an `ImageFs` mounted at `/` is the
15//! root purely because the namespace recipe mounted it there. Any backend can
16//! compose one — kata serves it to its guest via virtio-fs today, and
17//! podman/nspawn/wasm wiring is a follow-on (TODO, #633). This module compiles
18//! under the `oci-image` feature alone, with no VM toolchain required.
19//!
20//! Composition is delegated to FS-C's
21//! [`rootfs_overlay`](hyprstream_vfs::overlay::rootfs_overlay) — copy-on-write,
22//! whiteout-on-delete and opaque dirs are performed by `OverlayFs` (Kata's
23//! production overlay). We do **not** hand-roll CoW here; the native hyprstream
24//! CoW engine (per-tenant writable layers over shared RO model weights, delta /
25//! sandbox snapshots) is the backlog ticket and will implement the same
26//! `FsMount` trait with zero churn here.
27//!
28//! ```text
29//! ImageFs (FsMount)
30//! │
31//! OverlayFs (FS-C rootfs_overlay)
32//! ┌──────┴──────┐
33//! upper lower
34//! <sandbox>/rootfs/ Rafs (in-process FileSystem)
35//! {upper,work}/ bootstrap .meta + data blobs (lazy chunk CAS)
36//! ```
37//!
38//! ## In-process RAFS — no external `nydusd`
39//!
40//! [`Rafs::new`] loads the bootstrap super-block and constructs a `BlobDevice`
41//! over a **localfs** backend + **filecache** rooted at the image store's blobs
42//! directory (where FS-B0 wrote the content-addressed RAFS data blobs). RAFS v6
43//! (what FS-B0 emits) *requires* a local blob cache, which the localfs config
44//! supplies. The whole thing runs in this process; there is no `nydusd`, no
45//! FUSE mount, no shell-out.
46
47use std::path::{Path, PathBuf};
48use std::sync::Arc;
49
50use fuse_backend_rs::overlayfs::OverlayFs;
51use nydus_api::ConfigV2;
52use nydus_rafs::fs::Rafs;
53
54use hyprstream_vfs::overlay::{layer_from_fs, rootfs_overlay};
55use hyprstream_vfs::{FsMount, FuseFileSystemMount, ZeroOpenLayer};
56
57use crate::error::{Result, WorkerError};
58
59/// The concrete type of a mountable OCI/RAFS image filesystem service.
60///
61/// An `OverlayFs` (FS-C's production overlay) over an in-process RAFS lower and
62/// a per-sandbox writable upper. This is what [`image_fs_for`] /
63/// [`RafsStore::image_fs`](crate::image::RafsStore::image_fs) build; it
64/// implements [`FsMount`], so a backend composes it into a namespace at whatever
65/// mount position it chooses (at `/` it is the root purely by virtue of the
66/// mount). "Root" is a mount position, not a type.
67pub type ImageFs = FuseFileSystemMount<OverlayFs>;
68
69/// Subdirectory under a sandbox dir holding the writable rootfs overlay state.
70const ROOTFS_SUBDIR: &str = "rootfs";
71/// The copy-up / whiteout target (overlay "upper").
72const UPPER_SUBDIR: &str = "upper";
73/// The overlay work directory (scratch for atomic operations).
74const WORK_SUBDIR: &str = "work";
75
76/// Load an image's RAFS bootstrap as an **in-process** read-only `FileSystem`.
77///
78/// * `bootstrap_path` — the RAFS bootstrap (`.meta`) produced by FS-B0
79/// ([`build_rafs_bootstrap`](crate::image::rafs_builder)), resolved via
80/// [`RafsStore::bootstrap_path`](crate::image::RafsStore::bootstrap_path).
81/// * `blobs_dir` — directory holding the content-addressed RAFS data blobs; used
82/// as both the localfs backend dir and the filecache work dir.
83/// * `id` — an identifier for the RAFS instance (metrics / logging only).
84///
85/// The returned [`Rafs`] is `import`ed and ready to serve. Chunks are fetched
86/// lazily on read from `blobs_dir`, preserving the lazy chunk-CAS property.
87fn load_rafs_lower(bootstrap_path: &Path, blobs_dir: &Path, id: &str) -> Result<Rafs> {
88 if !bootstrap_path.exists() {
89 return Err(WorkerError::RafsError(format!(
90 "RAFS bootstrap {} does not exist (image not pulled / built?)",
91 bootstrap_path.display()
92 )));
93 }
94
95 // localfs backend + filecache, both rooted at the blobs dir (RAFS v6
96 // mandates a local blob cache), plus the `rafs` section `Rafs::new`
97 // requires (`get_rafs_config`). `mode = "direct"` is the standard v6
98 // metadata mode; xattrs are enabled so overlay/whiteout xattrs resolve.
99 let blobs = blobs_dir.to_string_lossy();
100 let toml = format!(
101 r#"
102version = 2
103id = "{id}"
104backend.type = "localfs"
105backend.localfs.dir = "{blobs}"
106cache.type = "filecache"
107cache.compressed = false
108cache.validate = false
109cache.filecache.work_dir = "{blobs}"
110rafs.mode = "direct"
111rafs.enable_xattr = true
112"#
113 );
114 let config: ConfigV2 = toml.parse().map_err(|e| {
115 WorkerError::RafsError(format!(
116 "failed to build RAFS config for {}: {e}",
117 blobs_dir.display()
118 ))
119 })?;
120 let config = Arc::new(config);
121
122 let (mut rafs, reader) = Rafs::new(&config, id, bootstrap_path).map_err(|e| {
123 WorkerError::RafsError(format!(
124 "failed to load in-process RAFS from {}: {e}",
125 bootstrap_path.display()
126 ))
127 })?;
128 rafs.import(reader, None).map_err(|e| {
129 WorkerError::RafsError(format!(
130 "failed to import RAFS bootstrap {}: {e}",
131 bootstrap_path.display()
132 ))
133 })?;
134
135 Ok(rafs)
136}
137
138/// Lay out (and create) the per-sandbox writable overlay directories under
139/// `sandbox_dir`, returning `(upper, work)`.
140///
141/// Both live under `<sandbox_dir>/rootfs/`: `upper/` is the copy-up target,
142/// `work/` is `OverlayFs`'s scratch space. They must be on the same filesystem
143/// (they are — siblings) so `OverlayFs` can rename atomically between them.
144fn prepare_upper(sandbox_dir: &Path) -> Result<(PathBuf, PathBuf)> {
145 let rootfs = sandbox_dir.join(ROOTFS_SUBDIR);
146 let upper = rootfs.join(UPPER_SUBDIR);
147 let work = rootfs.join(WORK_SUBDIR);
148 for dir in [&upper, &work] {
149 std::fs::create_dir_all(dir)
150 .map_err(|e| WorkerError::IoError(format!("create {}: {e}", dir.display())))?;
151 }
152 Ok((upper, work))
153}
154
155/// Build an [`ImageFs`] from an already-loaded RAFS lower and explicit
156/// upper/work directories.
157///
158/// The lowest-level constructor: callers that hold a [`Rafs`] (e.g. tests, or a
159/// future caller that loaded RAFS itself) compose it here. Wraps the RAFS as the
160/// single read-only overlay lower and `upper` as the writable layer via FS-C's
161/// [`rootfs_overlay`].
162pub fn image_fs_from_rafs(
163 rafs: Rafs,
164 upper: &Path,
165 work: &Path,
166) -> Result<FuseFileSystemMount<OverlayFs>> {
167 let upper_layer = hyprstream_vfs::overlay::passthrough_layer(upper)
168 .map_err(|e| WorkerError::RafsError(format!("image-fs upper layer: {e}")))?;
169 // The RAFS `Rafs` implements `Layer`, but it is *handleless* (`open` yields
170 // no handle), which OverlayFs rejects as ENOENT for a lower. Wrap it in
171 // `ZeroOpenLayer` so it presents a constant handle; reads still resolve by
172 // inode and writes copy-up to the (handle-based) upper.
173 let lower_layer = layer_from_fs(ZeroOpenLayer::new(rafs));
174 rootfs_overlay(upper_layer, vec![lower_layer], work)
175 .map_err(|e| WorkerError::RafsError(format!("image-fs overlay: {e}")))
176}
177
178/// Build an [`ImageFs`] for `image_id`.
179///
180/// * `bootstrap_path` — the image's RAFS bootstrap (`.meta`).
181/// * `blobs_dir` — the image store's blobs directory (RAFS data blobs).
182/// * `image_id` — the image identifier (used as the RAFS instance id).
183/// * `sandbox_dir` — the per-sandbox directory; the writable upper + work dirs
184/// are created under `<sandbox_dir>/rootfs/`.
185///
186/// Returns the image [`FsMount`]: the RAFS image is the read-only lower, a
187/// per-sandbox directory is the writable upper, composed via `OverlayFs`. The
188/// shared image (RAFS bootstrap + blobs) is never written to — copy-ups and
189/// whiteouts land in the upper. Mount it wherever the namespace recipe chooses;
190/// at `/` it is the root purely by virtue of the mount position.
191pub fn image_fs_for(
192 bootstrap_path: &Path,
193 blobs_dir: &Path,
194 image_id: &str,
195 sandbox_dir: &Path,
196) -> Result<impl FsMount> {
197 let rafs = load_rafs_lower(bootstrap_path, blobs_dir, image_id)?;
198 let (upper, work) = prepare_upper(sandbox_dir)?;
199 image_fs_from_rafs(rafs, &upper, &work)
200}
201
202impl crate::image::RafsStore {
203 /// Build the image [`FsMount`] for an image into a per-sandbox directory.
204 ///
205 /// Resolves the image's RAFS bootstrap and blobs from this store, loads the
206 /// RAFS in-process as the read-only lower, and overlays a per-sandbox
207 /// writable upper under `<sandbox_dir>/rootfs/`. See [`image_fs_for`].
208 pub fn image_fs(&self, image_id: &str, sandbox_dir: &Path) -> Result<impl FsMount> {
209 let bootstrap = self.bootstrap_path(image_id);
210 image_fs_for(&bootstrap, self.blobs_dir(), image_id, sandbox_dir)
211 }
212}
213
214#[cfg(test)]
215#[allow(clippy::unwrap_used, clippy::expect_used)]
216mod tests {
217 use super::*;
218 use flate2::write::GzEncoder;
219 use flate2::Compression;
220 use hyprstream_rpc::Subject;
221 use hyprstream_vfs::{FsMount, Mount, ORDWR, OREAD};
222
223 use crate::image::rafs_builder::build_rafs_bootstrap;
224
225 /// Build a small gzip tar layer (OCI default) of regular files.
226 fn make_gzip_layer(path: &Path, entries: &[(&str, &[u8])]) {
227 let file = std::fs::File::create(path).unwrap();
228 let gz = GzEncoder::new(file, Compression::default());
229 let mut tar = tar::Builder::new(gz);
230 for (name, data) in entries {
231 let mut header = tar::Header::new_gnu();
232 header.set_size(data.len() as u64);
233 header.set_mode(0o644);
234 header.set_mtime(0);
235 header.set_entry_type(tar::EntryType::Regular);
236 tar.append_data(&mut header, name, *data).unwrap();
237 }
238 let gz = tar.into_inner().unwrap();
239 gz.finish().unwrap();
240 }
241
242 /// Build a synthetic RAFS image (bootstrap + data blobs) under `dir`,
243 /// returning `(bootstrap_path, blobs_dir)`.
244 fn build_image(dir: &Path, layers: &[&[(&str, &[u8])]]) -> (PathBuf, PathBuf) {
245 let blobs_dir = dir.join("blobs");
246 let bootstrap_dir = dir.join("bootstrap");
247 std::fs::create_dir_all(&blobs_dir).unwrap();
248 std::fs::create_dir_all(&bootstrap_dir).unwrap();
249
250 let mut layer_paths = Vec::new();
251 for (i, entries) in layers.iter().enumerate() {
252 let p = dir.join(format!("layer{i}.tar.gz"));
253 make_gzip_layer(&p, entries);
254 layer_paths.push(p);
255 }
256
257 let bootstrap = bootstrap_dir.join("img.meta");
258 build_rafs_bootstrap(&layer_paths, &blobs_dir, &bootstrap).unwrap();
259 (bootstrap, blobs_dir)
260 }
261
262 /// Snapshot of the blobs dir (name -> len) to assert the image is untouched.
263 fn blobs_snapshot(blobs_dir: &Path) -> Vec<(String, u64)> {
264 let mut v: Vec<(String, u64)> = std::fs::read_dir(blobs_dir)
265 .unwrap()
266 .map(|e| {
267 let e = e.unwrap();
268 (
269 e.file_name().to_string_lossy().into_owned(),
270 e.metadata().unwrap().len(),
271 )
272 })
273 .collect();
274 v.sort();
275 v
276 }
277
278 async fn read_file(mount: &impl FsMount, path: &[&str], subj: &Subject) -> Vec<u8> {
279 let mut fid = mount.walk(path, subj).await.unwrap();
280 mount.open(&mut fid, OREAD, subj).await.unwrap();
281 let data = mount.read(&fid, 0, 1 << 20, subj).await.unwrap();
282 mount.clunk(fid, subj).await;
283 data
284 }
285
286 #[tokio::test]
287 async fn reads_from_rafs_lower_and_copies_up_on_write() {
288 let tmp = tempfile::tempdir().unwrap();
289 let (bootstrap, blobs_dir) = build_image(
290 tmp.path(),
291 &[&[
292 ("etc/hostname", b"worker\n"),
293 ("etc/config", b"v1\n"),
294 ("usr/bin/app", b"#!/bin/sh\necho hi\n"),
295 ]],
296 );
297
298 // Filecache writes into the blobs dir on first chunk read; snapshot the
299 // RAFS *data blobs* (the immutable image content) by name+len. The
300 // filecache materialises sidecar files (different names), so we assert
301 // the original blobs are byte-for-byte unchanged rather than that the
302 // directory is frozen.
303 let before = blobs_snapshot(&blobs_dir);
304
305 let sandbox = tmp.path().join("sandbox-1");
306 let mount = image_fs_for(&bootstrap, &blobs_dir, "img:test", &sandbox).unwrap();
307 let subj = Subject::new("test");
308
309 // 1. Read a file present only in the RAFS lower.
310 let hostname = read_file(&mount, &["etc", "hostname"], &subj).await;
311 assert_eq!(hostname, b"worker\n", "must read through to RAFS lower");
312
313 // 2. Write a brand-new file -> lands in the writable upper.
314 let new_path = ["etc", "greeting"];
315 FsMount::create(&mount, &new_path, 0o644, &subj)
316 .await
317 .unwrap();
318 let mut fid = mount.walk(&new_path, &subj).await.unwrap();
319 mount.open(&mut fid, ORDWR, &subj).await.unwrap();
320 let n = mount.write(&fid, 0, b"hello\n", &subj).await.unwrap();
321 assert_eq!(n, 6);
322 mount.clunk(fid, &subj).await;
323
324 // The new file is visible through the overlay...
325 let greeting = read_file(&mount, &new_path, &subj).await;
326 assert_eq!(greeting, b"hello\n");
327 // ...and physically present in the sandbox upper.
328 let upper_file = sandbox
329 .join(ROOTFS_SUBDIR)
330 .join(UPPER_SUBDIR)
331 .join("etc")
332 .join("greeting");
333 assert!(upper_file.exists(), "new file must copy-up into the upper");
334 assert_eq!(std::fs::read(&upper_file).unwrap(), b"hello\n");
335
336 // 3. Overwrite a file that exists in the lower -> copy-up, lower intact.
337 let cfg_path = ["etc", "config"];
338 let mut fid = mount.walk(&cfg_path, &subj).await.unwrap();
339 mount.open(&mut fid, ORDWR, &subj).await.unwrap();
340 mount.write(&fid, 0, b"v2\n", &subj).await.unwrap();
341 mount.clunk(fid, &subj).await;
342 let cfg = read_file(&mount, &cfg_path, &subj).await;
343 assert_eq!(&cfg[..3], b"v2\n", "overwrite visible via overlay");
344 let upper_cfg = sandbox
345 .join(ROOTFS_SUBDIR)
346 .join(UPPER_SUBDIR)
347 .join("etc")
348 .join("config");
349 assert!(upper_cfg.exists(), "modified lower file must copy-up");
350
351 // 4. The RAFS image (data blobs) is untouched: every original blob is
352 // still present with its original length.
353 let after = blobs_snapshot(&blobs_dir);
354 for (name, len) in &before {
355 let found = after.iter().find(|(n, _)| n == name);
356 assert_eq!(
357 found,
358 Some(&(name.clone(), *len)),
359 "RAFS data blob {name} must be unchanged"
360 );
361 }
362 }
363
364 #[tokio::test]
365 async fn missing_bootstrap_is_fail_closed() {
366 let tmp = tempfile::tempdir().unwrap();
367 let blobs = tmp.path().join("blobs");
368 std::fs::create_dir_all(&blobs).unwrap();
369 let bootstrap = tmp.path().join("nope.meta");
370 let sandbox = tmp.path().join("sandbox");
371 let err = image_fs_for(&bootstrap, &blobs, "img:missing", &sandbox);
372 assert!(err.is_err(), "missing RAFS bootstrap must hard-error");
373 }
374}