Skip to main content

hyprstream_workers/image/
image_fs.rs

1//! **ImageFs** — a universal mountable OCI/RAFS image filesystem service (#633,
2//! spine of epic #508).
3//!
4//! An `ImageFs` is an [`FsMount`](hyprstream_vfs::FsMount): an `OverlayFs` whose
5//!
6//! - **lower** is the image's RAFS loaded **in-process** as a
7//!   `fuse_backend_rs::FileSystem` (via `nydus-rafs`'s [`Rafs`], which also
8//!   implements the overlay [`Layer`](fuse_backend_rs::api::filesystem::Layer)
9//!   trait) — read-only, lazily fetching chunks from the CAS / Dragonfly P2P, so
10//!   nothing is materialised on disk up front; and
11//! - **upper** is a per-sandbox writable directory (the copy-up / whiteout
12//!   target), so writes never touch the shared image.
13//!
14//! "Root" is a mount position, not a type: an `ImageFs` mounted at `/` is the
15//! root purely because the namespace recipe mounted it there. Any backend can
16//! compose one — kata serves it to its guest via virtio-fs today, and
17//! podman/nspawn/wasm wiring is a follow-on (TODO, #633). This module compiles
18//! under the `oci-image` feature alone, with no VM toolchain required.
19//!
20//! Composition is delegated to FS-C's
21//! [`rootfs_overlay`](hyprstream_vfs::overlay::rootfs_overlay) — copy-on-write,
22//! whiteout-on-delete and opaque dirs are performed by `OverlayFs` (Kata's
23//! production overlay). We do **not** hand-roll CoW here; the native hyprstream
24//! CoW engine (per-tenant writable layers over shared RO model weights, delta /
25//! sandbox snapshots) is the backlog ticket and will implement the same
26//! `FsMount` trait with zero churn here.
27//!
28//! ```text
29//!            ImageFs  (FsMount)
30//!                 │
31//!         OverlayFs (FS-C rootfs_overlay)
32//!          ┌──────┴──────┐
33//!       upper           lower
34//!   <sandbox>/rootfs/   Rafs (in-process FileSystem)
35//!   {upper,work}/       bootstrap .meta + data blobs (lazy chunk CAS)
36//! ```
37//!
38//! ## In-process RAFS — no external `nydusd`
39//!
40//! [`Rafs::new`] loads the bootstrap super-block and constructs a `BlobDevice`
41//! over a **localfs** backend + **filecache** rooted at the image store's blobs
42//! directory (where FS-B0 wrote the content-addressed RAFS data blobs). RAFS v6
43//! (what FS-B0 emits) *requires* a local blob cache, which the localfs config
44//! supplies. The whole thing runs in this process; there is no `nydusd`, no
45//! FUSE mount, no shell-out.
46
47use std::path::{Path, PathBuf};
48use std::sync::Arc;
49
50use fuse_backend_rs::overlayfs::OverlayFs;
51use nydus_api::ConfigV2;
52use nydus_rafs::fs::Rafs;
53
54use hyprstream_vfs::overlay::{layer_from_fs, rootfs_overlay};
55use hyprstream_vfs::{FsMount, FuseFileSystemMount, ZeroOpenLayer};
56
57use crate::error::{Result, WorkerError};
58
59/// The concrete type of a mountable OCI/RAFS image filesystem service.
60///
61/// An `OverlayFs` (FS-C's production overlay) over an in-process RAFS lower and
62/// a per-sandbox writable upper. This is what [`image_fs_for`] /
63/// [`RafsStore::image_fs`](crate::image::RafsStore::image_fs) build; it
64/// implements [`FsMount`], so a backend composes it into a namespace at whatever
65/// mount position it chooses (at `/` it is the root purely by virtue of the
66/// mount). "Root" is a mount position, not a type.
67pub type ImageFs = FuseFileSystemMount<OverlayFs>;
68
69/// Subdirectory under a sandbox dir holding the writable rootfs overlay state.
70const ROOTFS_SUBDIR: &str = "rootfs";
71/// The copy-up / whiteout target (overlay "upper").
72const UPPER_SUBDIR: &str = "upper";
73/// The overlay work directory (scratch for atomic operations).
74const WORK_SUBDIR: &str = "work";
75
76/// Load an image's RAFS bootstrap as an **in-process** read-only `FileSystem`.
77///
78/// * `bootstrap_path` — the RAFS bootstrap (`.meta`) produced by FS-B0
79///   ([`build_rafs_bootstrap`](crate::image::rafs_builder)), resolved via
80///   [`RafsStore::bootstrap_path`](crate::image::RafsStore::bootstrap_path).
81/// * `blobs_dir` — directory holding the content-addressed RAFS data blobs; used
82///   as both the localfs backend dir and the filecache work dir.
83/// * `id` — an identifier for the RAFS instance (metrics / logging only).
84///
85/// The returned [`Rafs`] is `import`ed and ready to serve. Chunks are fetched
86/// lazily on read from `blobs_dir`, preserving the lazy chunk-CAS property.
87fn load_rafs_lower(bootstrap_path: &Path, blobs_dir: &Path, id: &str) -> Result<Rafs> {
88    if !bootstrap_path.exists() {
89        return Err(WorkerError::RafsError(format!(
90            "RAFS bootstrap {} does not exist (image not pulled / built?)",
91            bootstrap_path.display()
92        )));
93    }
94
95    // localfs backend + filecache, both rooted at the blobs dir (RAFS v6
96    // mandates a local blob cache), plus the `rafs` section `Rafs::new`
97    // requires (`get_rafs_config`). `mode = "direct"` is the standard v6
98    // metadata mode; xattrs are enabled so overlay/whiteout xattrs resolve.
99    let blobs = blobs_dir.to_string_lossy();
100    let toml = format!(
101        r#"
102version = 2
103id = "{id}"
104backend.type = "localfs"
105backend.localfs.dir = "{blobs}"
106cache.type = "filecache"
107cache.compressed = false
108cache.validate = false
109cache.filecache.work_dir = "{blobs}"
110rafs.mode = "direct"
111rafs.enable_xattr = true
112"#
113    );
114    let config: ConfigV2 = toml.parse().map_err(|e| {
115        WorkerError::RafsError(format!(
116            "failed to build RAFS config for {}: {e}",
117            blobs_dir.display()
118        ))
119    })?;
120    let config = Arc::new(config);
121
122    let (mut rafs, reader) = Rafs::new(&config, id, bootstrap_path).map_err(|e| {
123        WorkerError::RafsError(format!(
124            "failed to load in-process RAFS from {}: {e}",
125            bootstrap_path.display()
126        ))
127    })?;
128    rafs.import(reader, None).map_err(|e| {
129        WorkerError::RafsError(format!(
130            "failed to import RAFS bootstrap {}: {e}",
131            bootstrap_path.display()
132        ))
133    })?;
134
135    Ok(rafs)
136}
137
138/// Lay out (and create) the per-sandbox writable overlay directories under
139/// `sandbox_dir`, returning `(upper, work)`.
140///
141/// Both live under `<sandbox_dir>/rootfs/`: `upper/` is the copy-up target,
142/// `work/` is `OverlayFs`'s scratch space. They must be on the same filesystem
143/// (they are — siblings) so `OverlayFs` can rename atomically between them.
144fn prepare_upper(sandbox_dir: &Path) -> Result<(PathBuf, PathBuf)> {
145    let rootfs = sandbox_dir.join(ROOTFS_SUBDIR);
146    let upper = rootfs.join(UPPER_SUBDIR);
147    let work = rootfs.join(WORK_SUBDIR);
148    for dir in [&upper, &work] {
149        std::fs::create_dir_all(dir)
150            .map_err(|e| WorkerError::IoError(format!("create {}: {e}", dir.display())))?;
151    }
152    Ok((upper, work))
153}
154
155/// Build an [`ImageFs`] from an already-loaded RAFS lower and explicit
156/// upper/work directories.
157///
158/// The lowest-level constructor: callers that hold a [`Rafs`] (e.g. tests, or a
159/// future caller that loaded RAFS itself) compose it here. Wraps the RAFS as the
160/// single read-only overlay lower and `upper` as the writable layer via FS-C's
161/// [`rootfs_overlay`].
162pub fn image_fs_from_rafs(
163    rafs: Rafs,
164    upper: &Path,
165    work: &Path,
166) -> Result<FuseFileSystemMount<OverlayFs>> {
167    let upper_layer = hyprstream_vfs::overlay::passthrough_layer(upper)
168        .map_err(|e| WorkerError::RafsError(format!("image-fs upper layer: {e}")))?;
169    // The RAFS `Rafs` implements `Layer`, but it is *handleless* (`open` yields
170    // no handle), which OverlayFs rejects as ENOENT for a lower. Wrap it in
171    // `ZeroOpenLayer` so it presents a constant handle; reads still resolve by
172    // inode and writes copy-up to the (handle-based) upper.
173    let lower_layer = layer_from_fs(ZeroOpenLayer::new(rafs));
174    rootfs_overlay(upper_layer, vec![lower_layer], work)
175        .map_err(|e| WorkerError::RafsError(format!("image-fs overlay: {e}")))
176}
177
178/// Build an [`ImageFs`] for `image_id`.
179///
180/// * `bootstrap_path` — the image's RAFS bootstrap (`.meta`).
181/// * `blobs_dir` — the image store's blobs directory (RAFS data blobs).
182/// * `image_id` — the image identifier (used as the RAFS instance id).
183/// * `sandbox_dir` — the per-sandbox directory; the writable upper + work dirs
184///   are created under `<sandbox_dir>/rootfs/`.
185///
186/// Returns the image [`FsMount`]: the RAFS image is the read-only lower, a
187/// per-sandbox directory is the writable upper, composed via `OverlayFs`. The
188/// shared image (RAFS bootstrap + blobs) is never written to — copy-ups and
189/// whiteouts land in the upper. Mount it wherever the namespace recipe chooses;
190/// at `/` it is the root purely by virtue of the mount position.
191pub fn image_fs_for(
192    bootstrap_path: &Path,
193    blobs_dir: &Path,
194    image_id: &str,
195    sandbox_dir: &Path,
196) -> Result<impl FsMount> {
197    let rafs = load_rafs_lower(bootstrap_path, blobs_dir, image_id)?;
198    let (upper, work) = prepare_upper(sandbox_dir)?;
199    image_fs_from_rafs(rafs, &upper, &work)
200}
201
202impl crate::image::RafsStore {
203    /// Build the image [`FsMount`] for an image into a per-sandbox directory.
204    ///
205    /// Resolves the image's RAFS bootstrap and blobs from this store, loads the
206    /// RAFS in-process as the read-only lower, and overlays a per-sandbox
207    /// writable upper under `<sandbox_dir>/rootfs/`. See [`image_fs_for`].
208    pub fn image_fs(&self, image_id: &str, sandbox_dir: &Path) -> Result<impl FsMount> {
209        let bootstrap = self.bootstrap_path(image_id);
210        image_fs_for(&bootstrap, self.blobs_dir(), image_id, sandbox_dir)
211    }
212}
213
214#[cfg(test)]
215#[allow(clippy::unwrap_used, clippy::expect_used)]
216mod tests {
217    use super::*;
218    use flate2::write::GzEncoder;
219    use flate2::Compression;
220    use hyprstream_rpc::Subject;
221    use hyprstream_vfs::{FsMount, Mount, ORDWR, OREAD};
222
223    use crate::image::rafs_builder::build_rafs_bootstrap;
224
225    /// Build a small gzip tar layer (OCI default) of regular files.
226    fn make_gzip_layer(path: &Path, entries: &[(&str, &[u8])]) {
227        let file = std::fs::File::create(path).unwrap();
228        let gz = GzEncoder::new(file, Compression::default());
229        let mut tar = tar::Builder::new(gz);
230        for (name, data) in entries {
231            let mut header = tar::Header::new_gnu();
232            header.set_size(data.len() as u64);
233            header.set_mode(0o644);
234            header.set_mtime(0);
235            header.set_entry_type(tar::EntryType::Regular);
236            tar.append_data(&mut header, name, *data).unwrap();
237        }
238        let gz = tar.into_inner().unwrap();
239        gz.finish().unwrap();
240    }
241
242    /// Build a synthetic RAFS image (bootstrap + data blobs) under `dir`,
243    /// returning `(bootstrap_path, blobs_dir)`.
244    fn build_image(dir: &Path, layers: &[&[(&str, &[u8])]]) -> (PathBuf, PathBuf) {
245        let blobs_dir = dir.join("blobs");
246        let bootstrap_dir = dir.join("bootstrap");
247        std::fs::create_dir_all(&blobs_dir).unwrap();
248        std::fs::create_dir_all(&bootstrap_dir).unwrap();
249
250        let mut layer_paths = Vec::new();
251        for (i, entries) in layers.iter().enumerate() {
252            let p = dir.join(format!("layer{i}.tar.gz"));
253            make_gzip_layer(&p, entries);
254            layer_paths.push(p);
255        }
256
257        let bootstrap = bootstrap_dir.join("img.meta");
258        build_rafs_bootstrap(&layer_paths, &blobs_dir, &bootstrap).unwrap();
259        (bootstrap, blobs_dir)
260    }
261
262    /// Snapshot of the blobs dir (name -> len) to assert the image is untouched.
263    fn blobs_snapshot(blobs_dir: &Path) -> Vec<(String, u64)> {
264        let mut v: Vec<(String, u64)> = std::fs::read_dir(blobs_dir)
265            .unwrap()
266            .map(|e| {
267                let e = e.unwrap();
268                (
269                    e.file_name().to_string_lossy().into_owned(),
270                    e.metadata().unwrap().len(),
271                )
272            })
273            .collect();
274        v.sort();
275        v
276    }
277
278    async fn read_file(mount: &impl FsMount, path: &[&str], subj: &Subject) -> Vec<u8> {
279        let mut fid = mount.walk(path, subj).await.unwrap();
280        mount.open(&mut fid, OREAD, subj).await.unwrap();
281        let data = mount.read(&fid, 0, 1 << 20, subj).await.unwrap();
282        mount.clunk(fid, subj).await;
283        data
284    }
285
286    #[tokio::test]
287    async fn reads_from_rafs_lower_and_copies_up_on_write() {
288        let tmp = tempfile::tempdir().unwrap();
289        let (bootstrap, blobs_dir) = build_image(
290            tmp.path(),
291            &[&[
292                ("etc/hostname", b"worker\n"),
293                ("etc/config", b"v1\n"),
294                ("usr/bin/app", b"#!/bin/sh\necho hi\n"),
295            ]],
296        );
297
298        // Filecache writes into the blobs dir on first chunk read; snapshot the
299        // RAFS *data blobs* (the immutable image content) by name+len. The
300        // filecache materialises sidecar files (different names), so we assert
301        // the original blobs are byte-for-byte unchanged rather than that the
302        // directory is frozen.
303        let before = blobs_snapshot(&blobs_dir);
304
305        let sandbox = tmp.path().join("sandbox-1");
306        let mount = image_fs_for(&bootstrap, &blobs_dir, "img:test", &sandbox).unwrap();
307        let subj = Subject::new("test");
308
309        // 1. Read a file present only in the RAFS lower.
310        let hostname = read_file(&mount, &["etc", "hostname"], &subj).await;
311        assert_eq!(hostname, b"worker\n", "must read through to RAFS lower");
312
313        // 2. Write a brand-new file -> lands in the writable upper.
314        let new_path = ["etc", "greeting"];
315        FsMount::create(&mount, &new_path, 0o644, &subj)
316            .await
317            .unwrap();
318        let mut fid = mount.walk(&new_path, &subj).await.unwrap();
319        mount.open(&mut fid, ORDWR, &subj).await.unwrap();
320        let n = mount.write(&fid, 0, b"hello\n", &subj).await.unwrap();
321        assert_eq!(n, 6);
322        mount.clunk(fid, &subj).await;
323
324        // The new file is visible through the overlay...
325        let greeting = read_file(&mount, &new_path, &subj).await;
326        assert_eq!(greeting, b"hello\n");
327        // ...and physically present in the sandbox upper.
328        let upper_file = sandbox
329            .join(ROOTFS_SUBDIR)
330            .join(UPPER_SUBDIR)
331            .join("etc")
332            .join("greeting");
333        assert!(upper_file.exists(), "new file must copy-up into the upper");
334        assert_eq!(std::fs::read(&upper_file).unwrap(), b"hello\n");
335
336        // 3. Overwrite a file that exists in the lower -> copy-up, lower intact.
337        let cfg_path = ["etc", "config"];
338        let mut fid = mount.walk(&cfg_path, &subj).await.unwrap();
339        mount.open(&mut fid, ORDWR, &subj).await.unwrap();
340        mount.write(&fid, 0, b"v2\n", &subj).await.unwrap();
341        mount.clunk(fid, &subj).await;
342        let cfg = read_file(&mount, &cfg_path, &subj).await;
343        assert_eq!(&cfg[..3], b"v2\n", "overwrite visible via overlay");
344        let upper_cfg = sandbox
345            .join(ROOTFS_SUBDIR)
346            .join(UPPER_SUBDIR)
347            .join("etc")
348            .join("config");
349        assert!(upper_cfg.exists(), "modified lower file must copy-up");
350
351        // 4. The RAFS image (data blobs) is untouched: every original blob is
352        // still present with its original length.
353        let after = blobs_snapshot(&blobs_dir);
354        for (name, len) in &before {
355            let found = after.iter().find(|(n, _)| n == name);
356            assert_eq!(
357                found,
358                Some(&(name.clone(), *len)),
359                "RAFS data blob {name} must be unchanged"
360            );
361        }
362    }
363
364    #[tokio::test]
365    async fn missing_bootstrap_is_fail_closed() {
366        let tmp = tempfile::tempdir().unwrap();
367        let blobs = tmp.path().join("blobs");
368        std::fs::create_dir_all(&blobs).unwrap();
369        let bootstrap = tmp.path().join("nope.meta");
370        let sandbox = tmp.path().join("sandbox");
371        let err = image_fs_for(&bootstrap, &blobs, "img:missing", &sandbox);
372        assert!(err.is_err(), "missing RAFS bootstrap must hard-error");
373    }
374}