Skip to main content

git2db/
pinned_tree.rs

1//! Owned, exact-commit input capture for consumers that cannot trust a worktree.
2//!
3//! Callers may consume [`PinnedTree::file`] bytes directly or retain a Linux
4//! [`SealedTreeProjection`] for legacy path readers. Copying these bytes back
5//! into a writable worktree does not preserve the pin. Acquisition currently
6//! buffers the full resolved tree before sealing, so memory use is proportional
7//! to the model size until projection completes.
8
9use crate::{Git2DBError, Git2DBResult, GitManager, Oid};
10use git2::{ObjectType, Repository, Tree};
11use std::collections::BTreeMap;
12use std::path::{Component, Path, PathBuf};
13use std::sync::Arc;
14
15#[cfg(target_os = "linux")]
16use std::fs::{self, File};
17#[cfg(target_os = "linux")]
18use std::io::Write;
19#[cfg(target_os = "linux")]
20use std::os::fd::{AsRawFd, FromRawFd};
21
22/// A complete commit tree captured independently of a mutable checkout.
23///
24/// `Arc<[u8]>` owns every resolved file for the lifetime of this value. No
25/// borrowed worktree paths or mutable file descriptors escape acquisition.
26#[derive(Debug)]
27pub struct PinnedTree {
28    commit: Oid,
29    files: BTreeMap<PathBuf, PinnedFile>,
30}
31
32/// One resolved input with its source Git blob identity.
33#[derive(Debug)]
34pub struct PinnedFile {
35    source_blob: Oid,
36    bytes: Arc<[u8]>,
37}
38
39/// A private directory whose entries point to sealed, owned Linux memfds.
40///
41/// All file content is kernel-sealed against writes and size changes. The
42/// directory exists only for path-based readers; callers must retain this
43/// value through every read, including lazy reads after model startup.
44#[derive(Debug)]
45pub struct SealedTreeProjection {
46    commit: Oid,
47    root: tempfile::TempDir,
48    files: Vec<std::fs::File>,
49}
50
51impl SealedTreeProjection {
52    pub fn commit(&self) -> Oid {
53        self.commit
54    }
55
56    pub fn root(&self) -> &Path {
57        self.root.path()
58    }
59
60    pub fn file_count(&self) -> usize {
61        self.files.len()
62    }
63}
64
65impl PinnedFile {
66    pub fn source_blob(&self) -> Oid {
67        self.source_blob
68    }
69
70    pub fn bytes(&self) -> &[u8] {
71        &self.bytes
72    }
73}
74
75#[derive(Debug)]
76struct TreeBlob {
77    path: PathBuf,
78    source_blob: Oid,
79    bytes: Vec<u8>,
80}
81
82impl PinnedTree {
83    /// Capture the full commit tree, including resolved LFS/XET content.
84    /// Missing commits, symlinks, submodules, and unresolved pointers fail.
85    /// Worktree cleanliness is a caller policy; this method never reads the
86    /// checkout and does not report whether a selected checkout is dirty.
87    pub async fn acquire(repo_path: &Path, commit: Oid) -> Git2DBResult<Self> {
88        let blobs = capture_blobs(repo_path, commit)?;
89        Self::from_blobs(commit, blobs).await
90    }
91
92    async fn from_blobs(commit: Oid, blobs: Vec<TreeBlob>) -> Git2DBResult<Self> {
93        let mut files = BTreeMap::new();
94        #[cfg(feature = "xet-storage")]
95        let mut storage = None;
96        for blob in blobs {
97            #[cfg(feature = "xet-storage")]
98            let bytes = resolve_pointer(blob.bytes, &mut storage).await?;
99            #[cfg(not(feature = "xet-storage"))]
100            let bytes = reject_unresolved_pointer(blob.bytes)?;
101            files.insert(
102                blob.path,
103                PinnedFile {
104                    source_blob: blob.source_blob,
105                    bytes: Arc::<[u8]>::from(bytes),
106                },
107            );
108        }
109        Ok(Self { commit, files })
110    }
111
112    pub fn commit(&self) -> Oid {
113        self.commit
114    }
115
116    pub fn file(&self, path: &Path) -> Option<&[u8]> {
117        self.files.get(path).map(PinnedFile::bytes)
118    }
119
120    pub fn files(&self) -> impl Iterator<Item = (&Path, &PinnedFile)> {
121        self.files.iter().map(|(path, file)| (path.as_path(), file))
122    }
123
124    /// Move the captured bytes into kernel-sealed files for existing path
125    /// loaders. The temporary directory is private and owns no writable file
126    /// content; its lifetime must cover every later path access.
127    #[cfg(target_os = "linux")]
128    pub fn into_sealed_projection(self) -> Git2DBResult<SealedTreeProjection> {
129        let root = tempfile::Builder::new()
130            .prefix("git2db-pinned-tree-")
131            .tempdir()
132            .map_err(|e| Git2DBError::internal(format!("create pinned tree directory: {e}")))?;
133        let mut files = Vec::with_capacity(self.files.len());
134        for (relative_path, pinned) in self.files {
135            let target = root.path().join(&relative_path);
136            let parent = target.parent().ok_or_else(|| {
137                Git2DBError::internal(format!("pinned file has no parent: {}", target.display()))
138            })?;
139            fs::create_dir_all(parent).map_err(|e| {
140                Git2DBError::internal(format!("create pinned directory {}: {e}", parent.display()))
141            })?;
142            let mut file = create_sealable_file()?;
143            file.write_all(pinned.bytes()).map_err(|e| {
144                Git2DBError::internal(format!(
145                    "write pinned file {}: {e}",
146                    relative_path.display()
147                ))
148            })?;
149            seal_file(&file)?;
150            std::os::unix::fs::symlink(format!("/proc/self/fd/{}", file.as_raw_fd()), &target)
151                .map_err(|e| {
152                    Git2DBError::internal(format!("project pinned file {}: {e}", target.display()))
153                })?;
154            files.push(file);
155        }
156        Ok(SealedTreeProjection {
157            commit: self.commit,
158            root,
159            files,
160        })
161    }
162
163    #[cfg(not(target_os = "linux"))]
164    pub fn into_sealed_projection(self) -> Git2DBResult<SealedTreeProjection> {
165        Err(Git2DBError::internal(
166            "sealed pinned model projection requires Linux memfd seals",
167        ))
168    }
169}
170
171#[cfg(target_os = "linux")]
172fn create_sealable_file() -> Git2DBResult<File> {
173    const NAME: &[u8] = b"git2db-pinned-input\0";
174    // SAFETY: NAME is a NUL-terminated static byte string. A successful
175    // memfd_create returns a new owned descriptor, transferred into File.
176    let fd = unsafe {
177        libc::memfd_create(
178            NAME.as_ptr().cast(),
179            libc::MFD_CLOEXEC | libc::MFD_ALLOW_SEALING,
180        )
181    };
182    if fd < 0 {
183        return Err(Git2DBError::internal(format!(
184            "create sealable pinned file: {}",
185            std::io::Error::last_os_error()
186        )));
187    }
188    // SAFETY: fd is newly created and owned exclusively by this function.
189    Ok(unsafe { File::from_raw_fd(fd) })
190}
191
192#[cfg(target_os = "linux")]
193fn seal_file(file: &File) -> Git2DBResult<()> {
194    let seals = libc::F_SEAL_WRITE | libc::F_SEAL_GROW | libc::F_SEAL_SHRINK | libc::F_SEAL_SEAL;
195    // SAFETY: fcntl operates on a valid owned descriptor; the constant-sized
196    // integer third argument is the documented F_ADD_SEALS ABI.
197    let result = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_ADD_SEALS, seals) };
198    if result < 0 {
199        return Err(Git2DBError::internal(format!(
200            "seal pinned file: {}",
201            std::io::Error::last_os_error()
202        )));
203    }
204    Ok(())
205}
206
207fn capture_blobs(repo_path: &Path, commit: Oid) -> Git2DBResult<Vec<TreeBlob>> {
208    let repo = GitManager::global().get_repository(repo_path)?.open()?;
209    let _ = verified_git_object_bytes(&repo, commit, ObjectType::Commit)?;
210    // find_commit, rather than revparse/peel, requires exactly the requested
211    // object to be a commit. It cannot silently substitute HEAD or a tag.
212    let tree = repo
213        .find_commit(commit)
214        .map_err(|e| Git2DBError::internal(format!("pinned commit {commit} unavailable: {e}")))?
215        .tree()
216        .map_err(|e| Git2DBError::internal(format!("pinned tree: {e}")))?;
217    let mut blobs = Vec::new();
218    capture_tree(&repo, &tree, Path::new(""), &mut blobs)?;
219    Ok(blobs)
220}
221
222fn capture_tree(
223    repo: &Repository,
224    tree: &Tree<'_>,
225    prefix: &Path,
226    blobs: &mut Vec<TreeBlob>,
227) -> Git2DBResult<()> {
228    let _ = verified_git_object_bytes(repo, tree.id(), ObjectType::Tree)?;
229    for entry in tree {
230        let name = entry
231            .name()
232            .ok_or_else(|| Git2DBError::internal("non-UTF-8 tree entry"))?;
233        let component = Path::new(name);
234        if name.contains('/')
235            || name.contains('\\')
236            || component.components().count() != 1
237            || !matches!(component.components().next(), Some(Component::Normal(_)))
238        {
239            return Err(Git2DBError::internal(format!(
240                "unsafe tree entry: {name:?}"
241            )));
242        }
243        let path = prefix.join(component);
244        match (entry.kind(), entry.filemode()) {
245            (Some(ObjectType::Tree), 0o040000) => {
246                let child = repo
247                    .find_tree(entry.id())
248                    .map_err(|e| Git2DBError::internal(format!("tree {}: {e}", path.display())))?;
249                capture_tree(repo, &child, &path, blobs)?;
250            }
251            (Some(ObjectType::Blob), 0o100644 | 0o100755) => {
252                blobs.push(TreeBlob {
253                    path,
254                    source_blob: entry.id(),
255                    bytes: verified_git_object_bytes(repo, entry.id(), ObjectType::Blob)?,
256                });
257            }
258            _ => {
259                return Err(Git2DBError::internal(format!(
260                    "unsupported tree entry {} (mode {:o})",
261                    path.display(),
262                    entry.filemode()
263                )))
264            }
265        }
266    }
267    Ok(())
268}
269
270fn verified_git_object_bytes(
271    repo: &Repository,
272    oid: Oid,
273    expected_kind: ObjectType,
274) -> Git2DBResult<Vec<u8>> {
275    let odb = repo
276        .odb()
277        .map_err(|e| Git2DBError::internal(format!("open Git object database: {e}")))?;
278    let object = odb
279        .read(oid)
280        .map_err(|e| Git2DBError::internal(format!("read Git object {oid}: {e}")))?;
281    if object.kind() != expected_kind {
282        return Err(Git2DBError::internal(format!(
283            "Git object {oid} has type {:?}, expected {expected_kind:?}",
284            object.kind()
285        )));
286    }
287    let computed = Oid::hash_object(expected_kind, object.data())
288        .map_err(|e| Git2DBError::internal(format!("hash Git object {oid}: {e}")))?;
289    if computed != oid {
290        return Err(Git2DBError::internal(format!(
291            "Git object mismatch: requested {oid}, got {computed}"
292        )));
293    }
294    Ok(object.data().to_vec())
295}
296
297pub(crate) fn is_pointer(bytes: &[u8]) -> bool {
298    let trimmed = bytes.trim_ascii_start();
299    bytes.starts_with(b"version https://git-lfs")
300        || bytes.starts_with(b"version https://hawser")
301        || bytes.starts_with(b"# xet version")
302        || (trimmed.starts_with(b"{")
303            && serde_json::from_slice::<serde_json::Value>(trimmed).is_ok_and(|v| {
304                (v.get("hash").is_some() && v.get("file_size").is_some())
305                    || v.get("xet").and_then(serde_json::Value::as_str) == Some("gittorrent")
306            }))
307}
308
309#[cfg(not(feature = "xet-storage"))]
310fn reject_unresolved_pointer(bytes: Vec<u8>) -> Git2DBResult<Vec<u8>> {
311    if is_pointer(&bytes) {
312        return Err(Git2DBError::internal(
313            "pinned tree contains a pointer but xet-storage is disabled",
314        ));
315    }
316    Ok(bytes)
317}
318
319#[cfg(feature = "xet-storage")]
320async fn resolve_pointer(
321    bytes: Vec<u8>,
322    storage: &mut Option<crate::LfsStorage>,
323) -> Git2DBResult<Vec<u8>> {
324    if !is_pointer(&bytes) {
325        return Ok(bytes);
326    }
327    let text = std::str::from_utf8(&bytes)
328        .map_err(|e| Git2DBError::internal(format!("pointer is not UTF-8: {e}")))?;
329    if text.starts_with("# xet version") {
330        return Err(Git2DBError::internal("unsupported legacy XET pointer"));
331    }
332    if storage.is_none() {
333        *storage = Some(crate::LfsStorage::new(&crate::XetConfig::default()).await?);
334    }
335    let storage = storage
336        .as_ref()
337        .ok_or_else(|| Git2DBError::internal("LFS storage unavailable"))?;
338    if crate::is_lfs_pointer(text) {
339        let pointer = crate::LfsPointer::parse(text)?;
340        let resolved = storage.smudge_lfs_pointer(&pointer).await?;
341        validate_lfs_content(&pointer, &resolved)?;
342        return Ok(resolved);
343    }
344    let info: data::XetFileInfo = serde_json::from_str(text)
345        .map_err(|e| Git2DBError::internal(format!("invalid XET pointer: {e}")))?;
346    info.merkle_hash()
347        .map_err(|e| Git2DBError::internal(format!("invalid XET hash: {e}")))?;
348    let resolved = storage.smudge_bytes(text).await?;
349    validate_xet_content(&info, &resolved)?;
350    Ok(resolved)
351}
352
353#[cfg(feature = "xet-storage")]
354fn validate_xet_content(info: &data::XetFileInfo, content: &[u8]) -> Git2DBResult<()> {
355    if content.len() as u64 != info.file_size() {
356        return Err(Git2DBError::internal(format!(
357            "XET size mismatch: expected {}, got {}",
358            info.file_size(),
359            content.len()
360        )));
361    }
362    // XET's file hash is the Merkle aggregation of content-defined chunks.
363    // Recompute it from the returned bytes: a CAS response keyed by the
364    // captured hash is not itself proof that its payload matches that hash.
365    let mut chunker = deduplication::Chunker::default();
366    let chunks = chunker.next_block_bytes(&bytes::Bytes::copy_from_slice(content), true);
367    let chunk_hashes: Vec<_> = chunks
368        .iter()
369        .map(|chunk| (chunk.hash, chunk.data.len() as u64))
370        .collect();
371    let computed = merklehash::file_hash(&chunk_hashes);
372    let expected = info
373        .merkle_hash()
374        .map_err(|e| Git2DBError::internal(format!("invalid XET hash: {e}")))?;
375    if computed != expected {
376        return Err(Git2DBError::internal(format!(
377            "XET Merkle mismatch: expected {}, got {}",
378            expected.hex(),
379            computed.hex()
380        )));
381    }
382    Ok(())
383}
384
385#[cfg(feature = "xet-storage")]
386fn validate_lfs_content(pointer: &crate::LfsPointer, content: &[u8]) -> Git2DBResult<()> {
387    use sha2::{Digest, Sha256};
388    if content.len() as u64 != pointer.size() {
389        return Err(Git2DBError::internal(format!(
390            "LFS size mismatch: expected {}, got {}",
391            pointer.size(),
392            content.len()
393        )));
394    }
395    let computed = format!("{:x}", Sha256::digest(content));
396    if computed != pointer.oid() {
397        return Err(Git2DBError::internal(format!(
398            "LFS SHA-256 mismatch: expected {}, got {computed}",
399            pointer.oid()
400        )));
401    }
402    Ok(())
403}
404
405#[cfg(test)]
406mod tests {
407    use super::*;
408
409    #[tokio::test]
410    async fn captures_requested_tree_without_reopening_worktree() -> Git2DBResult<()> {
411        let dir = tempfile::tempdir().map_err(|e| Git2DBError::internal(e.to_string()))?;
412        let repo =
413            Repository::init(dir.path()).map_err(|e| Git2DBError::internal(e.to_string()))?;
414        std::fs::write(dir.path().join("config.json"), b"reviewed")
415            .map_err(|e| Git2DBError::internal(e.to_string()))?;
416        let mut index = repo
417            .index()
418            .map_err(|e| Git2DBError::internal(e.to_string()))?;
419        index
420            .add_path(Path::new("config.json"))
421            .map_err(|e| Git2DBError::internal(e.to_string()))?;
422        let tree_oid = index
423            .write_tree()
424            .map_err(|e| Git2DBError::internal(e.to_string()))?;
425        let tree = repo
426            .find_tree(tree_oid)
427            .map_err(|e| Git2DBError::internal(e.to_string()))?;
428        let sig = git2::Signature::now("test", "test@example.invalid")
429            .map_err(|e| Git2DBError::internal(e.to_string()))?;
430        let oid = repo
431            .commit(Some("HEAD"), &sig, &sig, "reviewed", &tree, &[])
432            .map_err(|e| Git2DBError::internal(e.to_string()))?;
433        drop(tree);
434        drop(repo);
435
436        // The worktree is already dirty when acquisition begins. The exact
437        // commit object, rather than the checkout, supplies the captured file.
438        std::fs::write(dir.path().join("config.json"), b"changed")
439            .map_err(|e| Git2DBError::internal(e.to_string()))?;
440        std::fs::write(dir.path().join("untracked.safetensors"), b"not reviewed")
441            .map_err(|e| Git2DBError::internal(e.to_string()))?;
442        let pinned = PinnedTree::acquire(dir.path(), oid).await?;
443        assert_eq!(
444            pinned.file(Path::new("config.json")),
445            Some(&b"reviewed"[..])
446        );
447        assert!(pinned.file(Path::new("untracked.safetensors")).is_none());
448        // Simulate mutation after the Git tree has been captured but before
449        // pointer resolution and ownership transfer complete.
450        let blobs = capture_blobs(dir.path(), oid)?;
451        std::fs::write(dir.path().join("config.json"), b"changed again")
452            .map_err(|e| Git2DBError::internal(e.to_string()))?;
453        let during_acquire = PinnedTree::from_blobs(oid, blobs).await?;
454        assert_eq!(
455            during_acquire.file(Path::new("config.json")),
456            Some(&b"reviewed"[..])
457        );
458        assert_eq!(
459            pinned.file(Path::new("config.json")),
460            Some(&b"reviewed"[..])
461        );
462        assert!(PinnedTree::acquire(dir.path(), Oid::zero()).await.is_err());
463        Ok(())
464    }
465
466    #[cfg(target_os = "linux")]
467    #[tokio::test]
468    async fn sealed_projection_survives_worktree_mutation_and_rejects_writes() -> Git2DBResult<()> {
469        let dir = tempfile::tempdir().map_err(|e| Git2DBError::internal(e.to_string()))?;
470        let repo =
471            Repository::init(dir.path()).map_err(|e| Git2DBError::internal(e.to_string()))?;
472        std::fs::write(dir.path().join("tokenizer.json"), b"reviewed")
473            .map_err(|e| Git2DBError::internal(e.to_string()))?;
474        let mut index = repo
475            .index()
476            .map_err(|e| Git2DBError::internal(e.to_string()))?;
477        index
478            .add_path(Path::new("tokenizer.json"))
479            .map_err(|e| Git2DBError::internal(e.to_string()))?;
480        let tree_oid = index
481            .write_tree()
482            .map_err(|e| Git2DBError::internal(e.to_string()))?;
483        let tree = repo
484            .find_tree(tree_oid)
485            .map_err(|e| Git2DBError::internal(e.to_string()))?;
486        let sig = git2::Signature::now("test", "test@example.invalid")
487            .map_err(|e| Git2DBError::internal(e.to_string()))?;
488        let oid = repo
489            .commit(Some("HEAD"), &sig, &sig, "reviewed", &tree, &[])
490            .map_err(|e| Git2DBError::internal(e.to_string()))?;
491        drop(tree);
492        drop(repo);
493
494        let projected = PinnedTree::acquire(dir.path(), oid)
495            .await?
496            .into_sealed_projection()?;
497        let input = projected.root().join("tokenizer.json");
498        std::fs::write(dir.path().join("tokenizer.json"), b"replaced")
499            .map_err(|e| Git2DBError::internal(e.to_string()))?;
500        assert_eq!(
501            std::fs::read(&input).map_err(|e| Git2DBError::internal(e.to_string()))?,
502            b"reviewed"
503        );
504        assert!(std::fs::write(&input, b"tampered").is_err());
505        assert_eq!(
506            std::fs::read(&input).map_err(|e| Git2DBError::internal(e.to_string()))?,
507            b"reviewed"
508        );
509        Ok(())
510    }
511
512    #[cfg(unix)]
513    #[tokio::test]
514    async fn rejects_symlink_in_requested_tree() -> Git2DBResult<()> {
515        let dir = tempfile::tempdir().map_err(|e| Git2DBError::internal(e.to_string()))?;
516        let repo =
517            Repository::init(dir.path()).map_err(|e| Git2DBError::internal(e.to_string()))?;
518        std::os::unix::fs::symlink("/etc/passwd", dir.path().join("config.json"))
519            .map_err(|e| Git2DBError::internal(e.to_string()))?;
520        let mut index = repo
521            .index()
522            .map_err(|e| Git2DBError::internal(e.to_string()))?;
523        index
524            .add_path(Path::new("config.json"))
525            .map_err(|e| Git2DBError::internal(e.to_string()))?;
526        let tree_oid = index
527            .write_tree()
528            .map_err(|e| Git2DBError::internal(e.to_string()))?;
529        let tree = repo
530            .find_tree(tree_oid)
531            .map_err(|e| Git2DBError::internal(e.to_string()))?;
532        let sig = git2::Signature::now("test", "test@example.invalid")
533            .map_err(|e| Git2DBError::internal(e.to_string()))?;
534        let oid = repo
535            .commit(Some("HEAD"), &sig, &sig, "symlink", &tree, &[])
536            .map_err(|e| Git2DBError::internal(e.to_string()))?;
537        drop(tree);
538        drop(repo);
539        assert!(PinnedTree::acquire(dir.path(), oid).await.is_err());
540        Ok(())
541    }
542
543    #[cfg(feature = "xet-storage")]
544    #[test]
545    fn rejects_resolved_pointer_mismatch() -> Git2DBResult<()> {
546        use sha2::{Digest, Sha256};
547        let expected = format!("{:x}", Sha256::digest(b"reviewed"));
548        let pointer = crate::LfsPointer::parse(&format!(
549            "version https://git-lfs.github.com/spec/v1\noid sha256:{expected}\nsize 8\n"
550        ))?;
551        validate_lfs_content(&pointer, b"reviewed")?;
552        assert!(validate_lfs_content(&pointer, b"replaced").is_err());
553        assert!(validate_lfs_content(&pointer, b"short").is_err());
554        Ok(())
555    }
556
557    #[cfg(feature = "xet-storage")]
558    #[test]
559    fn rejects_xet_payload_with_matching_size_but_wrong_bytes() -> Git2DBResult<()> {
560        let expected = merklehash::file_hash(&[(merklehash::compute_data_hash(b"reviewed"), 8)]);
561        let info = data::XetFileInfo::new(expected.hex(), 8);
562        validate_xet_content(&info, b"reviewed")?;
563        assert!(validate_xet_content(&info, b"replaced").is_err());
564        assert!(validate_xet_content(&info, b"short").is_err());
565        Ok(())
566    }
567}