Skip to main content

hyprstream_workers/image/
rafs_builder.rs

1//! In-process RAFS bootstrap builder (FS-B0).
2//!
3//! Converts the OCI layer tarballs pulled into the CAS (`blobs/`) into a single
4//! RAFS bootstrap (`.meta`) that can be consumed by an in-process RAFS
5//! `FileSystem` (FS-B, #363) or by `nydusd`.
6//!
7//! # Why in-process
8//!
9//! The `nydus-builder` crate (the same code that backs the `nydus-image create`
10//! binary) is reachable from the dependency tree, so the whole conversion runs
11//! in-process in pure Rust — no external `nydus-image` binary, no shell-out, no
12//! silent fallback. If the conversion fails the caller hard-errors.
13//!
14//! # Approach: layered build
15//!
16//! OCI layers are ordered lower → upper in the manifest. We build them in that
17//! order with [`TarballBuilder`] using [`ConversionType::TargzToRafs`], chaining
18//! each layer's build to the previous layer's bootstrap via
19//! [`BootstrapManager`]'s parent path. This is the same overlay-aware path that
20//! `nydus-image create` uses for multi-layer images:
21//!
22//! - OCI whiteouts (`.wh.*`) from upper layers correctly delete lower entries.
23//! - The builder's `layered_chunk_dict` deduplicates chunks across layers, so a
24//!   chunk that already exists in a lower layer is not written again.
25//!
26//! Data blobs are written content-addressed into `blobs_dir` (named by their
27//! RAFS blob hash). The final layer's bootstrap is written to `bootstrap_path`
28//! (a `SingleFile`), so `RafsStore::bootstrap_path()` resolves to a real RAFS
29//! bootstrap.
30
31use std::path::{Path, PathBuf};
32use std::sync::Arc;
33
34use nydus_api::ConfigV2;
35use nydus_builder::attributes::Attributes;
36use nydus_builder::{
37    ArtifactStorage, BlobManager, BootstrapManager, BuildContext, Builder, ConversionType,
38    Features, Prefetch, TarballBuilder, WhiteoutSpec,
39};
40use nydus_rafs::metadata::RafsVersion;
41use nydus_utils::{compress, digest};
42
43use crate::error::{Result, WorkerError};
44
45/// RAFS metadata format version emitted by the builder.
46///
47/// V6 is the EROFS-compatible format used by modern nydusd / Kata virtiofs and
48/// is what FS-B / the in-kernel EROFS path expects.
49const RAFS_FS_VERSION: RafsVersion = RafsVersion::V6;
50
51/// Build a RAFS bootstrap from an ordered set of pulled OCI layer tarballs.
52///
53/// * `layer_blob_paths` — absolute paths to the layer blobs already present in
54///   the CAS, ordered **lower → upper** (i.e. manifest layer order). The blobs
55///   may be gzip-compressed (`*.tar.gz`, the OCI default) or plain tar; the
56///   builder auto-detects.
57/// * `blobs_dir` — directory the generated RAFS data blobs are written into,
58///   content-addressed by their RAFS blob hash.
59/// * `bootstrap_path` — destination for the final RAFS bootstrap (`.meta`).
60///
61/// Returns the path to the produced bootstrap (== `bootstrap_path`). Hard-errors
62/// if no layers are supplied or any layer fails to convert; never leaves a
63/// partial/missing bootstrap silently.
64pub fn build_rafs_bootstrap(
65    layer_blob_paths: &[PathBuf],
66    blobs_dir: &Path,
67    bootstrap_path: &Path,
68) -> Result<PathBuf> {
69    if layer_blob_paths.is_empty() {
70        return Err(WorkerError::RafsError(
71            "cannot build RAFS bootstrap: image has no layers".to_owned(),
72        ));
73    }
74
75    // Shared RAFS configuration. Layered builds require these to be identical
76    // across layers (see `RafsSuperConfig::check_compatibility`).
77    let config = Arc::new(ConfigV2::default());
78
79    // Each intermediate (and the final) bootstrap is written here. The layered
80    // build reads the previous layer's bootstrap as its parent, so we keep a
81    // single rolling bootstrap file and chain to it. The final write lands at
82    // `bootstrap_path`.
83    let mut parent_bootstrap: Option<PathBuf> = None;
84
85    // Intermediate bootstraps for non-final layers live next to the final
86    // bootstrap so the parent reader can find them; the final layer overwrites
87    // `bootstrap_path` itself.
88    let bootstrap_dir = bootstrap_path.parent().ok_or_else(|| {
89        WorkerError::RafsError(format!(
90            "bootstrap path {} has no parent directory",
91            bootstrap_path.display()
92        ))
93    })?;
94
95    let layer_count = layer_blob_paths.len();
96    for (idx, layer_path) in layer_blob_paths.iter().enumerate() {
97        let is_final = idx + 1 == layer_count;
98
99        // Final layer writes directly to the requested bootstrap path; earlier
100        // layers write to a deterministic intermediate path so the next layer
101        // can read it as its parent.
102        let out_bootstrap = if is_final {
103            bootstrap_path.to_path_buf()
104        } else {
105            bootstrap_dir.join(format!(
106                ".rafs-build-{}-layer{}.meta",
107                bootstrap_basename(bootstrap_path),
108                idx
109            ))
110        };
111
112        build_one_layer(
113            layer_path,
114            blobs_dir,
115            &out_bootstrap,
116            parent_bootstrap.as_deref(),
117            idx,
118            config.clone(),
119        )?;
120
121        // Clean up the previous intermediate bootstrap (it has been folded into
122        // the current one) before advancing.
123        if let Some(prev) = parent_bootstrap.take() {
124            if prev != out_bootstrap {
125                let _ = std::fs::remove_file(&prev);
126            }
127        }
128        parent_bootstrap = Some(out_bootstrap);
129    }
130
131    if !bootstrap_path.exists() {
132        return Err(WorkerError::RafsError(format!(
133            "RAFS build completed but bootstrap {} is missing",
134            bootstrap_path.display()
135        )));
136    }
137
138    Ok(bootstrap_path.to_path_buf())
139}
140
141/// Build a single layer into `out_bootstrap`, optionally overlaid on `parent`.
142fn build_one_layer(
143    layer_path: &Path,
144    blobs_dir: &Path,
145    out_bootstrap: &Path,
146    parent: Option<&Path>,
147    layer_idx: usize,
148    config: Arc<ConfigV2>,
149) -> Result<()> {
150    // Data blobs are written into the shared CAS directory, named by their RAFS
151    // blob hash (the empty suffix keeps the bare hash as the filename).
152    let blob_storage = ArtifactStorage::FileDir((blobs_dir.to_path_buf(), String::new()));
153    // The bootstrap is a single explicit file so the caller controls its path.
154    let bootstrap_storage = ArtifactStorage::SingleFile(out_bootstrap.to_path_buf());
155
156    let mut ctx = BuildContext::new(
157        String::new(), // blob_id: derive from blob hash
158        // RAFS v6 (EROFS) requires 4K-aligned chunks; `nydus-image` forces this
159        // on for any v6 build that is not TarToTarfs.
160        true,                        // aligned_chunk
161        0,                           // blob_offset
162        compress::Algorithm::Zstd,   // compressor
163        digest::Algorithm::Sha256,   // digester
164        false,                       // explicit_uidgid (OCI: normalize to 0)
165        WhiteoutSpec::Oci,           // OCI .wh.* whiteout semantics
166        ConversionType::TargzToRafs, // gzip/tar -> RAFS (+ data blob)
167        layer_path.to_path_buf(),    // source_path
168        Prefetch::default(),
169        Some(blob_storage),
170        None,  // external_blob_storage
171        false, // blob_inline_meta: keep bootstrap as a standalone .meta file
172        Features::new(),
173        false, // encrypt
174        Attributes::default(),
175    );
176    ctx.set_fs_version(RAFS_FS_VERSION);
177    ctx.set_configuration(config);
178
179    let mut bootstrap_mgr = BootstrapManager::new(
180        Some(bootstrap_storage),
181        parent.map(|p| p.to_string_lossy().into_owned()),
182    );
183    let mut blob_mgr = BlobManager::new(digest::Algorithm::Sha256, false);
184
185    let mut builder = TarballBuilder::new(ConversionType::TargzToRafs);
186    builder
187        .build(&mut ctx, &mut bootstrap_mgr, &mut blob_mgr)
188        .map_err(|e| {
189            WorkerError::RafsError(format!(
190                "failed to build RAFS layer {} from {}: {e:#}",
191                layer_idx,
192                layer_path.display()
193            ))
194        })?;
195
196    Ok(())
197}
198
199/// List the RAFS data-blob IDs referenced by a bootstrap.
200///
201/// The IDs are the bare blob-hash filenames the data blobs are stored under in
202/// `blobs_dir`. Used by garbage collection so it does not delete the data blobs
203/// a live bootstrap depends on (those blobs are not OCI layer digests and so
204/// are invisible to the OCI-layer-based GC pass).
205pub fn bootstrap_blob_ids(bootstrap_path: &Path) -> Result<Vec<String>> {
206    let config = Arc::new(ConfigV2::default());
207    let (rs, _reader) =
208        nydus_rafs::metadata::RafsSuper::load_from_file(bootstrap_path, config, false).map_err(
209            |e| {
210                WorkerError::RafsError(format!(
211                    "failed to load RAFS bootstrap {}: {e:#}",
212                    bootstrap_path.display()
213                ))
214            },
215        )?;
216    Ok(rs
217        .superblock
218        .get_blob_infos()
219        .iter()
220        .map(|b| b.blob_id())
221        .collect())
222}
223
224/// Filesystem-safe base name (no extension) used to disambiguate intermediate
225/// bootstraps for concurrent pulls of different images.
226fn bootstrap_basename(path: &Path) -> String {
227    path.file_stem()
228        .map(|s| s.to_string_lossy().into_owned())
229        .unwrap_or_else(|| "image".to_owned())
230}
231
232#[cfg(test)]
233#[allow(clippy::unwrap_used, clippy::expect_used)]
234mod tests {
235    use super::*;
236    use flate2::write::GzEncoder;
237    use flate2::Compression;
238    use nydus_rafs::metadata::RafsSuper;
239
240    /// Build a small gzip-compressed tar layer (the OCI default media type)
241    /// with the given (path, contents) regular-file entries.
242    fn make_gzip_layer(path: &Path, entries: &[(&str, &[u8])]) {
243        let file = std::fs::File::create(path).unwrap();
244        let gz = GzEncoder::new(file, Compression::default());
245        let mut tar = tar::Builder::new(gz);
246        for (name, data) in entries {
247            let mut header = tar::Header::new_gnu();
248            header.set_size(data.len() as u64);
249            header.set_mode(0o644);
250            header.set_mtime(0);
251            header.set_entry_type(tar::EntryType::Regular);
252            tar.append_data(&mut header, name, *data).unwrap();
253        }
254        let gz = tar.into_inner().unwrap();
255        gz.finish().unwrap();
256    }
257
258    #[test]
259    fn builds_bootstrap_from_single_layer() {
260        let tmp = tempfile::tempdir().unwrap();
261        let blobs_dir = tmp.path().join("blobs");
262        let bootstrap_dir = tmp.path().join("bootstrap");
263        std::fs::create_dir_all(&blobs_dir).unwrap();
264        std::fs::create_dir_all(&bootstrap_dir).unwrap();
265
266        let layer = tmp.path().join("layer0.tar.gz");
267        make_gzip_layer(
268            &layer,
269            &[
270                ("etc/hostname", b"worker\n"),
271                ("usr/bin/app", b"#!/bin/sh\necho hi\n"),
272            ],
273        );
274
275        let bootstrap = bootstrap_dir.join("img.meta");
276        let out = build_rafs_bootstrap(&[layer], &blobs_dir, &bootstrap).unwrap();
277        assert_eq!(out, bootstrap);
278
279        let meta = std::fs::metadata(&bootstrap).unwrap();
280        assert!(meta.len() > 0, "bootstrap must be non-empty");
281
282        // It must parse as a real RAFS bootstrap.
283        let config = Arc::new(ConfigV2::default());
284        let (rs, _reader) =
285            RafsSuper::load_from_file(&bootstrap, config, false).expect("bootstrap must load");
286        assert!(rs.meta.inodes_count > 0, "RAFS must have inodes");
287    }
288
289    #[test]
290    fn builds_bootstrap_from_layered_image() {
291        let tmp = tempfile::tempdir().unwrap();
292        let blobs_dir = tmp.path().join("blobs");
293        let bootstrap_dir = tmp.path().join("bootstrap");
294        std::fs::create_dir_all(&blobs_dir).unwrap();
295        std::fs::create_dir_all(&bootstrap_dir).unwrap();
296
297        let lower = tmp.path().join("layer0.tar.gz");
298        make_gzip_layer(
299            &lower,
300            &[("etc/hostname", b"base\n"), ("etc/config", b"v1\n")],
301        );
302        let upper = tmp.path().join("layer1.tar.gz");
303        // Upper overwrites etc/config and adds a new file.
304        make_gzip_layer(
305            &upper,
306            &[("etc/config", b"v2\n"), ("etc/extra", b"added\n")],
307        );
308
309        let bootstrap = bootstrap_dir.join("img.meta");
310        let out = build_rafs_bootstrap(&[lower, upper], &blobs_dir, &bootstrap).unwrap();
311        assert_eq!(out, bootstrap);
312        assert!(std::fs::metadata(&bootstrap).unwrap().len() > 0);
313
314        let config = Arc::new(ConfigV2::default());
315        let (rs, _reader) =
316            RafsSuper::load_from_file(&bootstrap, config, false).expect("bootstrap must load");
317        assert!(rs.meta.inodes_count > 0);
318
319        // No intermediate bootstraps should be left behind.
320        for entry in std::fs::read_dir(&bootstrap_dir).unwrap() {
321            let name = entry.unwrap().file_name();
322            let name = name.to_string_lossy();
323            assert!(
324                !name.starts_with(".rafs-build-"),
325                "intermediate bootstrap leaked: {name}"
326            );
327        }
328    }
329}