1use crate::{Git2DBError, Git2DBResult, GitManager, Oid};
11use git2::{ObjectType, Repository, Tree};
12use std::collections::BTreeMap;
13use std::fs::{self, File, OpenOptions};
14use std::io::{BufRead, BufReader, Read, Write};
15#[cfg(target_os = "linux")]
16use std::os::fd::AsRawFd;
17#[cfg(target_os = "linux")]
18use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt};
19use std::path::{Component, Path, PathBuf};
20use std::process::{Command, Stdio};
21use std::sync::{Arc, OnceLock};
22use tokio::sync::{OwnedSemaphorePermit, Semaphore};
23
24const MAX_FILES: usize = 512;
25const MAX_TREE_ENTRIES: usize = 4096;
26const MAX_TREE_DEPTH: usize = 64;
27const MAX_EXPANDED_BYTES: u64 = 2 * 1024 * 1024 * 1024;
28const FREE_SPACE_RESERVE: u64 = 512 * 1024 * 1024;
29const PROJECTION_PREFIX: &str = "git2db-pinned-disk-";
30const OWNER_FILE: &str = ".git2db-pinned-disk-owner-v1";
31const OWNER_MARKER: &[u8] = b"git2db-pinned-disk-owner-v1\n";
32static PROJECTION_CAPACITY: OnceLock<Arc<Semaphore>> = OnceLock::new();
33
34#[derive(Clone, Debug, PartialEq, Eq, Hash)]
37pub struct ProjectionPolicy {
38 max_expanded_bytes: u64,
39 local_lfs: BTreeMap<String, PathBuf>,
40}
41
42impl Default for ProjectionPolicy {
43 fn default() -> Self {
44 Self {
45 max_expanded_bytes: MAX_EXPANDED_BYTES,
46 local_lfs: BTreeMap::new(),
47 }
48 }
49}
50
51impl ProjectionPolicy {
52 pub fn from_trusted_contract(
56 max_expanded_bytes: u64,
57 local_lfs: BTreeMap<String, PathBuf>,
58 ) -> Git2DBResult<Self> {
59 if max_expanded_bytes == 0 || max_expanded_bytes.checked_add(FREE_SPACE_RESERVE).is_none() {
60 return Err(internal("invalid projection byte ceiling"));
61 }
62 for (oid, path) in &local_lfs {
63 if oid.len() != 64
64 || !oid
65 .bytes()
66 .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
67 || !path.is_absolute()
68 {
69 return Err(internal(
70 "local LFS map requires lowercase SHA256 and absolute paths",
71 ));
72 }
73 }
74 Ok(Self {
75 max_expanded_bytes,
76 local_lfs,
77 })
78 }
79}
80
81const GIT_CLI: &str = "/usr/bin/git";
84const TMPFS_MAGIC: libc::c_long = 0x0102_1994;
85const RAMFS_MAGIC: libc::c_long = 0x8584_58f6;
86
87#[derive(Debug)]
88struct BlobEntry {
89 path: PathBuf,
90 oid: Oid,
91 source_size: u64,
92}
93
94#[cfg_attr(not(test), allow(dead_code))]
98struct StreamHooks {
99 fail_after_streams: usize,
100 observe: Box<dyn FnOnce(&Path, usize) + Send>,
101}
102
103#[derive(Debug)]
113pub struct DiskPinnedTreeProjection {
114 commit: Oid,
115 owner_pid: u32,
116 input_root: PathBuf,
117 files: Vec<File>,
120 _root: OwnedProjection,
121}
122
123#[cfg(target_os = "linux")]
126#[derive(Debug, Clone)]
127struct OwnedProjection(Arc<OwnedProjectionInner>);
128
129#[cfg(target_os = "linux")]
130#[derive(Debug)]
131struct OwnedProjectionInner {
132 path: PathBuf,
133 _owner_lock: File,
134 _capacity: Option<OwnedSemaphorePermit>,
137 #[cfg(test)]
138 observed_reader_fd: Option<std::os::fd::RawFd>,
139}
140
141#[cfg(not(target_os = "linux"))]
142#[derive(Debug)]
143struct OwnedProjection;
144
145#[cfg(target_os = "linux")]
146impl OwnedProjection {
147 fn create(parent: &Path, capacity: Option<OwnedSemaphorePermit>) -> Git2DBResult<Self> {
148 let root = tempfile::Builder::new()
149 .prefix(PROJECTION_PREFIX)
150 .tempdir_in(parent)
151 .map_err(|e| internal(format!("create private projection: {e}")))?;
152 let owner_lock = create_projection_owner(root.path())?;
154 Ok(Self(Arc::new(OwnedProjectionInner {
155 path: root.keep(),
156 _owner_lock: owner_lock,
157 _capacity: capacity,
158 #[cfg(test)]
159 observed_reader_fd: None,
160 })))
161 }
162
163 fn path(&self) -> &Path {
164 &self.0.path
165 }
166}
167
168#[cfg(target_os = "linux")]
169impl Drop for OwnedProjectionInner {
170 fn drop(&mut self) {
171 #[cfg(test)]
172 if let Some(fd) = self.observed_reader_fd {
173 if let Ok(target) = fs::read_link(format!("/proc/self/fd/{fd}")) {
176 assert!(
177 !target.starts_with(&self.path),
178 "projection reader is still open when its owner drops"
179 );
180 }
181 }
182 if let Err(error) = remove_owned_projection(&self.path) {
185 tracing::warn!(path = %self.path.display(), %error, "projection cleanup deferred");
186 }
187 }
188}
189
190impl DiskPinnedTreeProjection {
191 pub fn commit(&self) -> Oid {
192 self.commit
193 }
194
195 pub fn root(&self) -> &Path {
196 &self.input_root
197 }
198
199 pub fn file_count(&self) -> usize {
200 self.files.len()
201 }
202
203 pub fn ensure_current_process(&self) -> Git2DBResult<()> {
208 if self.owner_pid != std::process::id() {
209 return Err(internal(
210 "disk-backed pinned projection cannot cross a process boundary",
211 ));
212 }
213 Ok(())
214 }
215
216 #[cfg(target_os = "linux")]
219 pub async fn acquire(
220 repo_path: &Path,
221 commit: Oid,
222 private_disk_parent: &Path,
223 ) -> Git2DBResult<Self> {
224 Self::acquire_with_capacity(
225 repo_path,
226 commit,
227 private_disk_parent,
228 None,
229 None,
230 ProjectionPolicy::default(),
231 )
232 .await
233 }
234
235 #[cfg(target_os = "linux")]
238 pub async fn acquire_bounded(
239 repo_path: &Path,
240 commit: Oid,
241 private_disk_parent: &Path,
242 ) -> Git2DBResult<Self> {
243 Self::acquire_bounded_hooked(repo_path, commit, private_disk_parent, None).await
244 }
245
246 #[cfg(target_os = "linux")]
249 async fn acquire_bounded_hooked(
250 repo_path: &Path,
251 commit: Oid,
252 private_disk_parent: &Path,
253 hooks: Option<StreamHooks>,
254 ) -> Git2DBResult<Self> {
255 Self::acquire_bounded_policy_hooked(
256 repo_path,
257 commit,
258 private_disk_parent,
259 hooks,
260 ProjectionPolicy::default(),
261 )
262 .await
263 }
264
265 #[cfg(target_os = "linux")]
267 pub async fn acquire_bounded_with_policy(
268 repo_path: &Path,
269 commit: Oid,
270 private_disk_parent: &Path,
271 policy: ProjectionPolicy,
272 ) -> Git2DBResult<Self> {
273 Self::acquire_bounded_policy_hooked(repo_path, commit, private_disk_parent, None, policy)
274 .await
275 }
276
277 #[cfg(target_os = "linux")]
278 async fn acquire_bounded_policy_hooked(
279 repo_path: &Path,
280 commit: Oid,
281 private_disk_parent: &Path,
282 hooks: Option<StreamHooks>,
283 policy: ProjectionPolicy,
284 ) -> Git2DBResult<Self> {
285 let capacity = Arc::clone(PROJECTION_CAPACITY.get_or_init(|| Arc::new(Semaphore::new(1))))
286 .try_acquire_owned()
287 .map_err(|_| internal("bounded model projection capacity is occupied"))?;
288 let repo_path = repo_path.to_path_buf();
289 let private_disk_parent = private_disk_parent.to_path_buf();
290 run_on_projection_worker(move || {
295 let runtime = tokio::runtime::Builder::new_multi_thread()
296 .worker_threads(2)
297 .enable_all()
298 .build()
299 .map_err(|e| internal(format!("build projection I/O runtime: {e}")))?;
300 runtime.block_on(Self::acquire_with_capacity(
301 &repo_path,
302 commit,
303 &private_disk_parent,
304 Some(capacity),
305 hooks,
306 policy,
307 ))
308 })
309 .await
310 }
311
312 #[cfg(target_os = "linux")]
313 async fn acquire_with_capacity(
314 repo_path: &Path,
315 commit: Oid,
316 private_disk_parent: &Path,
317 capacity: Option<OwnedSemaphorePermit>,
318 mut hooks: Option<StreamHooks>,
319 policy: ProjectionPolicy,
320 ) -> Git2DBResult<Self> {
321 verify_private_disk_parent(private_disk_parent)?;
322 scavenge_abandoned_projections(private_disk_parent)?;
323 let repo = GitManager::global().get_repository(repo_path)?.open()?;
324 verify_object(&repo, commit, ObjectType::Commit)?;
325 let tree = repo
326 .find_commit(commit)
327 .map_err(|e| internal(format!("pinned commit {commit} unavailable: {e}")))?
328 .tree()
329 .map_err(|e| internal(format!("pinned tree: {e}")))?;
330 let mut entries = Vec::new();
331 let mut visited_entries = 0;
332 collect_entries(
333 &repo,
334 &tree,
335 Path::new(""),
336 &mut entries,
337 &mut visited_entries,
338 )?;
339 let projected_bytes =
340 preflight_projection(&repo, &mut entries, private_disk_parent, &policy)?;
341 if entries.len() > MAX_FILES {
342 return Err(internal(format!(
343 "model tree exceeds {MAX_FILES} file limit"
344 )));
345 }
346 if projected_bytes > policy.max_expanded_bytes {
347 return Err(internal(format!(
348 "model projection exceeds {}-byte limit",
349 policy.max_expanded_bytes
350 )));
351 }
352 let git_dir = repo.path().to_path_buf();
353 drop(tree);
354 drop(repo);
355
356 let root = OwnedProjection::create(private_disk_parent, capacity)?;
357 let payload_dir = root.path().join("payloads");
358 let staged_inputs = root.path().join("staged-inputs");
359 fs::create_dir(&payload_dir)
360 .map_err(|e| internal(format!("create payload directory: {e}")))?;
361 fs::create_dir(&staged_inputs)
362 .map_err(|e| internal(format!("create staged inputs: {e}")))?;
363 let mut files = Vec::with_capacity(entries.len());
364 #[cfg(feature = "xet-storage")]
365 let mut storage = None;
366
367 for (index, entry) in entries.into_iter().enumerate() {
368 let source_path = payload_dir.join(format!("{index}.git"));
369 let source_git_dir = git_dir.clone();
370 let source_oid = entry.oid;
371 let source_size = entry.source_size;
372 let source_copy = source_path.clone();
373 let task_owner = root.clone();
376 let size = tokio::task::spawn_blocking(move || {
377 let _task_owner = task_owner;
378 stream_verified_blob(&source_git_dir, source_oid, source_size, &source_copy)
379 })
380 .await
381 .map_err(|e| internal(format!("Git stream task failed: {e}")))??;
382
383 #[cfg(feature = "xet-storage")]
384 let mut final_path = source_path;
385 #[cfg(not(feature = "xet-storage"))]
386 let final_path = source_path;
387 if size <= 4096 {
388 let bytes = fs::read(&final_path)
389 .map_err(|e| internal(format!("read pointer candidate: {e}")))?;
390 if crate::pinned_tree::is_pointer(&bytes) {
391 #[cfg(feature = "xet-storage")]
392 {
393 let resolved = payload_dir.join(format!("{index}.resolved"));
394 resolve_pointer_to_file(&bytes, &resolved, &mut storage, &root, &policy)
395 .await?;
396 final_path = resolved;
397 }
398 #[cfg(not(feature = "xet-storage"))]
399 return Err(internal("pinned tree contains unresolved pointer"));
400 }
401 } else {
402 let mut prefix = [0u8; 128];
404 let count = File::open(&final_path)
405 .and_then(|mut file| file.read(&mut prefix))
406 .map_err(|e| internal(format!("read payload prefix: {e}")))?;
407 if prefix[..count].starts_with(b"version https://git-lfs")
408 || prefix[..count].starts_with(b"version https://hawser")
409 || prefix[..count].starts_with(b"# xet version")
410 {
411 return Err(internal("oversized model pointer"));
412 }
413 }
414
415 use std::os::unix::fs::PermissionsExt;
419 fs::set_permissions(&final_path, fs::Permissions::from_mode(0o400))
420 .map_err(|e| internal(format!("make private payload read-only: {e}")))?;
421 let reader = File::open(&final_path)
422 .map_err(|e| internal(format!("open verified payload: {e}")))?;
423 fs::remove_file(&final_path)
424 .map_err(|e| internal(format!("unlink private payload: {e}")))?;
425 let projected_path = staged_inputs.join(&entry.path);
426 let parent = projected_path
427 .parent()
428 .ok_or_else(|| internal("invalid projected path"))?;
429 fs::create_dir_all(parent)
430 .map_err(|e| internal(format!("create projected directories: {e}")))?;
431 std::os::unix::fs::symlink(
432 format!("/proc/self/fd/{}", reader.as_raw_fd()),
433 &projected_path,
434 )
435 .map_err(|e| internal(format!("project verified input: {e}")))?;
436 files.push(reader);
437 if let Some(hook) = hooks.take_if(|hook| hook.fail_after_streams == index + 1) {
438 (hook.observe)(root.path(), index + 1);
439 return Err(internal(format!(
440 "injected stream failure after {} streamed model payloads",
441 index + 1
442 )));
443 }
444 }
445
446 let input_root = root.path().join("inputs");
447 fs::rename(&staged_inputs, &input_root)
448 .map_err(|e| internal(format!("publish verified inputs: {e}")))?;
449 Ok(Self {
450 commit,
451 owner_pid: std::process::id(),
452 input_root,
453 files,
454 _root: root,
455 })
456 }
457
458 #[cfg(not(target_os = "linux"))]
459 async fn acquire_with_capacity(
460 _repo_path: &Path,
461 _commit: Oid,
462 _private_disk_parent: &Path,
463 _capacity: Option<OwnedSemaphorePermit>,
464 _hooks: Option<StreamHooks>,
465 _policy: ProjectionPolicy,
466 ) -> Git2DBResult<Self> {
467 Err(internal(
468 "disk-backed pinned projection requires Linux /proc/self/fd",
469 ))
470 }
471
472 #[cfg(not(target_os = "linux"))]
473 pub async fn acquire_bounded_with_policy(
474 _repo_path: &Path,
475 _commit: Oid,
476 _private_disk_parent: &Path,
477 _policy: ProjectionPolicy,
478 ) -> Git2DBResult<Self> {
479 Err(internal(
480 "bounded pinned projection requires Linux /proc/self/fd",
481 ))
482 }
483
484 #[cfg(not(target_os = "linux"))]
485 pub async fn acquire_bounded(
486 _repo_path: &Path,
487 _commit: Oid,
488 _private_disk_parent: &Path,
489 ) -> Git2DBResult<Self> {
490 Err(internal(
491 "bounded pinned projection requires Linux /proc/self/fd",
492 ))
493 }
494
495 #[cfg(not(target_os = "linux"))]
496 pub async fn acquire(
497 _repo_path: &Path,
498 _commit: Oid,
499 _private_disk_parent: &Path,
500 ) -> Git2DBResult<Self> {
501 Err(internal(
502 "disk-backed pinned projection requires Linux /proc/self/fd",
503 ))
504 }
505}
506
507fn internal(message: impl std::fmt::Display) -> Git2DBError {
508 Git2DBError::internal(message.to_string())
509}
510
511#[cfg(target_os = "linux")]
512async fn run_on_projection_worker<T: Send + 'static>(
513 work: impl FnOnce() -> Git2DBResult<T> + Send + 'static,
514) -> Git2DBResult<T> {
515 tokio::task::spawn_blocking(work)
516 .await
517 .map_err(|e| internal(format!("projection I/O worker failed: {e}")))?
518}
519
520#[cfg(target_os = "linux")]
521fn try_lock_projection(file: &File) -> Git2DBResult<bool> {
522 if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 {
525 return Ok(true);
526 }
527 let error = std::io::Error::last_os_error();
528 if error.raw_os_error() == Some(libc::EWOULDBLOCK) {
529 Ok(false)
530 } else {
531 Err(internal(format!("lock private projection: {error}")))
532 }
533}
534
535#[cfg(target_os = "linux")]
536fn create_projection_owner(root: &Path) -> Git2DBResult<File> {
537 let mut file = OpenOptions::new()
538 .read(true)
539 .write(true)
540 .create_new(true)
541 .mode(0o600)
542 .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
543 .open(root.join(OWNER_FILE))
544 .map_err(|e| internal(format!("create projection owner: {e}")))?;
545 if !try_lock_projection(&file)? {
546 return Err(internal("new projection owner lock is occupied"));
547 }
548 file.write_all(OWNER_MARKER)
549 .and_then(|_| file.sync_all())
550 .map_err(|e| internal(format!("write projection owner: {e}")))?;
551 Ok(file)
552}
553
554#[cfg(target_os = "linux")]
555fn remove_owned_projection(path: &Path) -> Git2DBResult<()> {
556 for entry in fs::read_dir(path).map_err(|e| internal(format!("read projection: {e}")))? {
560 let entry = entry.map_err(|e| internal(format!("read projection child: {e}")))?;
561 if entry.file_name() == OWNER_FILE {
562 continue;
563 }
564 let child = entry.path();
565 let kind = entry
566 .file_type()
567 .map_err(|e| internal(format!("inspect projection child: {e}")))?;
568 if kind.is_dir() {
569 fs::remove_dir_all(&child)
570 .map_err(|e| internal(format!("remove projection child: {e}")))?;
571 } else {
572 fs::remove_file(&child)
574 .map_err(|e| internal(format!("remove projection child: {e}")))?;
575 }
576 }
577 fs::remove_file(path.join(OWNER_FILE))
578 .map_err(|e| internal(format!("remove projection owner: {e}")))?;
579 fs::remove_dir(path).map_err(|e| internal(format!("remove empty projection: {e}")))?;
580 Ok(())
581}
582
583#[cfg(target_os = "linux")]
587fn scavenge_abandoned_projections(parent: &Path) -> Git2DBResult<()> {
588 for entry in fs::read_dir(parent).map_err(|e| internal(format!("scan projections: {e}")))? {
589 let entry = entry.map_err(|e| internal(format!("scan projection entry: {e}")))?;
590 let name = entry.file_name();
591 let Some(name) = name.to_str() else { continue };
592 let Some(suffix) = name.strip_prefix(PROJECTION_PREFIX) else {
593 continue;
594 };
595 if suffix.is_empty() || !suffix.bytes().all(|b| b.is_ascii_alphanumeric()) {
596 continue;
597 }
598 let path = entry.path();
599 let Ok(metadata) = fs::symlink_metadata(&path) else {
600 continue;
601 };
602 if !metadata.is_dir()
603 || metadata.file_type().is_symlink()
604 || metadata.uid() != unsafe { libc::geteuid() }
605 || metadata.permissions().mode() & 0o077 != 0
606 {
607 continue;
608 }
609 let Ok(mut file) = OpenOptions::new()
610 .read(true)
611 .write(true)
612 .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
613 .open(path.join(OWNER_FILE))
614 else {
615 continue;
616 };
617 let Ok(owner) = file.metadata() else { continue };
618 if !owner.is_file()
619 || owner.uid() != unsafe { libc::geteuid() }
620 || owner.permissions().mode() & 0o077 != 0
621 || owner.len() != OWNER_MARKER.len() as u64
622 {
623 continue;
624 }
625 let mut marker = vec![0; OWNER_MARKER.len()];
626 if file.read_exact(&mut marker).is_err() || marker != OWNER_MARKER {
627 continue;
628 }
629 if !try_lock_projection(&file)? {
630 continue;
631 }
632 remove_owned_projection(&path)?;
635 }
636 Ok(())
637}
638
639#[cfg(target_os = "linux")]
640fn verify_private_disk_parent(path: &Path) -> Git2DBResult<()> {
641 let metadata =
642 fs::symlink_metadata(path).map_err(|e| internal(format!("private parent: {e}")))?;
643 if !metadata.is_dir() || metadata.file_type().is_symlink() {
644 return Err(internal("projection parent must be a real directory"));
645 }
646 if metadata.uid() != unsafe { libc::geteuid() } || metadata.permissions().mode() & 0o077 != 0 {
647 return Err(internal(
648 "projection parent must be owned by loader and mode 0700",
649 ));
650 }
651 let c_path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes())
652 .map_err(|e| internal(format!("private parent path: {e}")))?;
653 let mut stat = std::mem::MaybeUninit::<libc::statfs>::uninit();
654 if unsafe { libc::statfs(c_path.as_ptr(), stat.as_mut_ptr()) } != 0 {
656 return Err(internal(format!(
657 "stat private parent: {}",
658 std::io::Error::last_os_error()
659 )));
660 }
661 let stat = unsafe { stat.assume_init() };
663 if stat.f_type == TMPFS_MAGIC || stat.f_type == RAMFS_MAGIC {
664 return Err(internal("private projection parent is memory-backed"));
665 }
666 if !Path::new(GIT_CLI).is_file() {
667 return Err(internal("packaged /usr/bin/git is unavailable"));
668 }
669 Ok(())
670}
671
672fn verify_object(repo: &Repository, oid: Oid, expected: ObjectType) -> Git2DBResult<()> {
673 let odb = repo
674 .odb()
675 .map_err(|e| internal(format!("open Git ODB: {e}")))?;
676 let object = odb
677 .read(oid)
678 .map_err(|e| internal(format!("read Git object {oid}: {e}")))?;
679 if object.kind() != expected
680 || Oid::hash_object(expected, object.data())
681 .map_err(|e| internal(format!("hash Git object {oid}: {e}")))?
682 != oid
683 {
684 return Err(internal(format!("Git object type/hash mismatch at {oid}")));
685 }
686 Ok(())
687}
688
689fn preflight_projection(
692 repo: &Repository,
693 entries: &mut [BlobEntry],
694 parent: &Path,
695 policy: &ProjectionPolicy,
696) -> Git2DBResult<u64> {
697 let odb = repo
698 .odb()
699 .map_err(|e| internal(format!("open projection ODB: {e}")))?;
700 let mut total = 0u64;
701 for entry in entries.iter_mut() {
702 let (blob_size, kind) = odb
703 .read_header(entry.oid)
704 .map_err(|e| internal(format!("read model blob size {}: {e}", entry.oid)))?;
705 if kind != ObjectType::Blob {
706 return Err(internal("model tree object ceased to be a blob"));
707 }
708 entry.source_size = blob_size as u64;
709 let mut size = blob_size as u64;
710 if blob_size <= 4096 {
711 let blob = odb
712 .read(entry.oid)
713 .map_err(|e| internal(format!("read pointer candidate: {e}")))?;
714 let bytes = blob.data();
715 if crate::pinned_tree::is_pointer(bytes) {
716 #[cfg(feature = "xet-storage")]
717 {
718 let text = std::str::from_utf8(bytes)
719 .map_err(|e| internal(format!("pointer UTF-8: {e}")))?;
720 if crate::is_lfs_pointer(text) {
721 size = crate::LfsPointer::parse(text)?.size();
722 } else if text.starts_with("# xet version") {
723 return Err(internal("unsupported legacy XET pointer"));
724 } else {
725 let info: data::XetFileInfo = serde_json::from_str(text)
726 .map_err(|e| internal(format!("invalid XET pointer: {e}")))?;
727 size = info.file_size();
728 }
729 }
730 }
731 } else {
732 total = total
733 .checked_add(size)
734 .ok_or_else(|| internal("model size overflow"))?;
735 }
736 total = total
737 .checked_add(if blob_size <= 4096 { size } else { 0 })
738 .ok_or_else(|| internal("model size overflow"))?;
739 if total > policy.max_expanded_bytes {
740 return Err(internal(format!(
741 "model projection exceeds {}-byte limit",
742 policy.max_expanded_bytes
743 )));
744 }
745 }
746 let allocation_bound = total.saturating_add((entries.len() as u64).saturating_mul(4096));
749 ensure_disk_space(parent, allocation_bound)?;
750 Ok(total)
751}
752
753#[cfg(target_os = "linux")]
754fn ensure_disk_space(parent: &Path, required: u64) -> Git2DBResult<()> {
755 let c_path = std::ffi::CString::new(parent.as_os_str().as_encoded_bytes())
756 .map_err(|e| internal(format!("projection parent path: {e}")))?;
757 let mut stats = std::mem::MaybeUninit::<libc::statvfs>::uninit();
758 if unsafe { libc::statvfs(c_path.as_ptr(), stats.as_mut_ptr()) } != 0 {
760 return Err(internal(format!(
761 "stat projection free space: {}",
762 std::io::Error::last_os_error()
763 )));
764 }
765 let stats = unsafe { stats.assume_init() };
767 let available = stats.f_bavail.saturating_mul(stats.f_frsize);
768 if available < required.saturating_add(FREE_SPACE_RESERVE) {
769 return Err(internal(format!(
770 "insufficient disk space for bounded model projection: need {required} bytes plus reserve"
771 )));
772 }
773 Ok(())
774}
775
776#[cfg(not(target_os = "linux"))]
777fn ensure_disk_space(_parent: &Path, _required: u64) -> Git2DBResult<()> {
778 Ok(())
779}
780
781fn collect_entries(
782 repo: &Repository,
783 tree: &Tree<'_>,
784 prefix: &Path,
785 entries: &mut Vec<BlobEntry>,
786 visited_entries: &mut usize,
787) -> Git2DBResult<()> {
788 if prefix.components().count() > MAX_TREE_DEPTH {
789 return Err(internal(format!(
790 "model tree exceeds {MAX_TREE_DEPTH}-level depth limit"
791 )));
792 }
793 verify_object(repo, tree.id(), ObjectType::Tree)?;
794 for entry in tree {
795 *visited_entries = visited_entries
796 .checked_add(1)
797 .ok_or_else(|| internal("model tree entry count overflow"))?;
798 if *visited_entries > MAX_TREE_ENTRIES {
799 return Err(internal(format!(
800 "model tree exceeds {MAX_TREE_ENTRIES}-entry traversal limit"
801 )));
802 }
803 let name = entry
804 .name()
805 .ok_or_else(|| internal("non-UTF-8 tree entry"))?;
806 let component = Path::new(name);
807 if name.contains('/')
808 || name.contains('\\')
809 || component.components().count() != 1
810 || !matches!(component.components().next(), Some(Component::Normal(_)))
811 {
812 return Err(internal(format!("unsafe tree entry: {name:?}")));
813 }
814 let path = prefix.join(component);
815 match (entry.kind(), entry.filemode()) {
816 (Some(ObjectType::Tree), 0o040000) => {
817 let child = repo
818 .find_tree(entry.id())
819 .map_err(|e| internal(format!("tree {}: {e}", path.display())))?;
820 collect_entries(repo, &child, &path, entries, visited_entries)?;
821 }
822 (Some(ObjectType::Blob), 0o100644 | 0o100755) => {
823 if entries.len() == MAX_FILES {
824 return Err(internal(format!(
825 "model tree exceeds {MAX_FILES} file limit"
826 )));
827 }
828 entries.push(BlobEntry {
829 path,
830 oid: entry.id(),
831 source_size: 0,
832 });
833 }
834 _ => {
835 return Err(internal(format!(
836 "unsupported tree entry {}",
837 path.display()
838 )));
839 }
840 }
841 }
842 Ok(())
843}
844
845fn stream_verified_blob(
846 git_dir: &Path,
847 oid: Oid,
848 preflight_size: u64,
849 destination: &Path,
850) -> Git2DBResult<u64> {
851 let mut child = Command::new(GIT_CLI)
852 .arg("cat-file")
853 .arg("--batch")
854 .env_clear()
855 .env("GIT_DIR", git_dir)
856 .env("GIT_NO_REPLACE_OBJECTS", "1")
857 .env("GIT_CONFIG_NOSYSTEM", "1")
858 .env("GIT_OPTIONAL_LOCKS", "0")
859 .stdin(Stdio::piped())
860 .stdout(Stdio::piped())
861 .stderr(Stdio::null())
862 .spawn()
863 .map_err(|e| internal(format!("spawn packaged Git: {e}")))?;
864 let result = (|| {
865 let mut stdin = child
866 .stdin
867 .take()
868 .ok_or_else(|| internal("Git stdin unavailable"))?;
869 writeln!(stdin, "{oid}").map_err(|e| internal(format!("request Git blob: {e}")))?;
870 drop(stdin);
871 let stdout = child
872 .stdout
873 .take()
874 .ok_or_else(|| internal("Git stdout unavailable"))?;
875 let mut output = BufReader::new(stdout);
876 let mut header = String::new();
877 output
878 .read_line(&mut header)
879 .map_err(|e| internal(format!("read Git blob header: {e}")))?;
880 let mut fields = header.split_whitespace();
881 let expected_oid = oid.to_string();
882 if fields.next() != Some(expected_oid.as_str()) || fields.next() != Some("blob") {
883 return Err(internal(format!("Git returned wrong object for {oid}")));
884 }
885 let size = fields
886 .next()
887 .ok_or_else(|| internal("Git blob size missing"))?
888 .parse::<u64>()
889 .map_err(|e| internal(format!("Git blob size invalid: {e}")))?;
890 if fields.next().is_some() {
891 return Err(internal("Git blob header has extra fields"));
892 }
893 if size != preflight_size {
897 return Err(internal(format!(
898 "Git blob size changed after preflight: expected {preflight_size}, got {size}"
899 )));
900 }
901 let mut file = OpenOptions::new()
902 .write(true)
903 .create_new(true)
904 .open(destination)
905 .map_err(|e| internal(format!("create private payload: {e}")))?;
906 let copied = std::io::copy(&mut output.by_ref().take(preflight_size), &mut file)
907 .map_err(|e| internal(format!("stream Git blob: {e}")))?;
908 if copied != size {
909 return Err(internal(format!(
910 "short Git blob: expected {size}, got {copied}"
911 )));
912 }
913 file.sync_all()
914 .map_err(|e| internal(format!("sync Git payload: {e}")))?;
915 drop(file);
916 let mut terminator = [0u8; 1];
917 output
918 .read_exact(&mut terminator)
919 .map_err(|e| internal(format!("Git blob terminator: {e}")))?;
920 if terminator != *b"\n" {
921 return Err(internal("Git blob terminator mismatch"));
922 }
923 Ok(size)
924 })();
925 if result.is_err() {
926 let _ = child.kill();
927 }
928 let status = child
929 .wait()
930 .map_err(|e| internal(format!("wait for Git blob: {e}")))?;
931 let size = result?;
932 if !status.success() {
933 return Err(internal("Git cat-file failed"));
934 }
935 verify_blob_file(oid, destination)?;
936 Ok(size)
937}
938
939fn verify_blob_file(oid: Oid, path: &Path) -> Git2DBResult<()> {
940 let computed = Oid::hash_file(ObjectType::Blob, path)
941 .map_err(|e| internal(format!("hash streamed Git blob: {e}")))?;
942 if computed != oid {
943 return Err(internal(format!(
944 "streamed Git blob hash mismatch: expected {oid}, got {computed}"
945 )));
946 }
947 Ok(())
948}
949
950#[cfg(all(feature = "xet-storage", target_os = "linux"))]
951async fn resolve_pointer_to_file(
952 bytes: &[u8],
953 path: &Path,
954 storage: &mut Option<crate::LfsStorage>,
955 owner: &OwnedProjection,
956 policy: &ProjectionPolicy,
957) -> Git2DBResult<()> {
958 let text = std::str::from_utf8(bytes).map_err(|e| internal(format!("pointer UTF-8: {e}")))?;
959 if text.starts_with("# xet version") {
960 return Err(internal("unsupported legacy XET pointer"));
961 }
962 if crate::is_lfs_pointer(text) && !policy.local_lfs.is_empty() {
963 let pointer = crate::LfsPointer::parse(text)?;
964 let source = policy
965 .local_lfs
966 .get(pointer.oid())
967 .ok_or_else(|| internal("captured LFS oid missing from local hydration map"))?
968 .clone();
969 let destination = path.to_path_buf();
970 let expected = pointer.oid().to_owned();
971 let size = pointer.size();
972 spawn_owned_verifier(owner, move || {
973 hydrate_local_lfs(&source, &destination, &expected, size)
974 })
975 .await
976 .map_err(|e| internal(format!("local LFS hydration task failed: {e}")))??;
977 return Ok(());
978 }
979 if storage.is_none() {
980 *storage = Some(crate::LfsStorage::new(&crate::XetConfig::default()).await?);
981 }
982 let storage = storage
983 .as_ref()
984 .ok_or_else(|| internal("LFS storage unavailable"))?;
985 if crate::is_lfs_pointer(text) {
986 let pointer = crate::LfsPointer::parse(text)?;
987 let written = storage
988 .smudge_lfs_pointer_to_file_bounded(&pointer, path)
989 .await?;
990 let expected = pointer.oid().to_owned();
991 let expected_size = pointer.size();
992 if written != expected_size {
993 return Err(internal(
994 "LFS downloader reported a size different from its pointer",
995 ));
996 }
997 let path = path.to_path_buf();
998 spawn_owned_verifier(owner, move || {
999 verify_lfs_file(&path, &expected, expected_size)
1000 })
1001 .await
1002 .map_err(|e| internal(format!("LFS verify task failed: {e}")))??;
1003 return Ok(());
1004 }
1005 let info: data::XetFileInfo =
1006 serde_json::from_str(text).map_err(|e| internal(format!("invalid XET pointer: {e}")))?;
1007 let written = storage
1008 .smudge_xet_pointer_to_file_bounded(
1009 text,
1010 path,
1011 info.file_size().min(policy.max_expanded_bytes),
1012 )
1013 .await?;
1014 if written != info.file_size() {
1015 return Err(internal(
1016 "XET downloader reported a size different from its pointer",
1017 ));
1018 }
1019 let path = path.to_path_buf();
1020 spawn_owned_verifier(owner, move || verify_xet_file(&path, &info))
1021 .await
1022 .map_err(|e| internal(format!("XET verify task failed: {e}")))??;
1023 Ok(())
1024}
1025
1026#[cfg(all(target_os = "linux", any(feature = "xet-storage", test)))]
1027fn spawn_owned_verifier(
1028 owner: &OwnedProjection,
1029 verify: impl FnOnce() -> Git2DBResult<()> + Send + 'static,
1030) -> tokio::task::JoinHandle<Git2DBResult<()>> {
1031 let task_owner = owner.clone();
1032 tokio::task::spawn_blocking(move || {
1033 let _task_owner = task_owner;
1034 verify()
1035 })
1036}
1037
1038#[cfg(all(feature = "xet-storage", target_os = "linux"))]
1041fn hydrate_local_lfs(
1042 source: &Path,
1043 destination: &Path,
1044 expected_sha: &str,
1045 expected_size: u64,
1046) -> Git2DBResult<()> {
1047 use std::io::{Read as _, Write as _};
1048 if source.canonicalize().map_err(internal)? != source {
1049 return Err(internal(
1050 "local LFS hydration source contains a symlink or noncanonical path",
1051 ));
1052 }
1053 let mut input = OpenOptions::new()
1054 .read(true)
1055 .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK)
1056 .open(source)
1057 .map_err(internal)?;
1058 let metadata = input.metadata().map_err(internal)?;
1059 if !metadata.is_file() || metadata.len() != expected_size {
1060 return Err(internal(
1061 "local LFS hydration source is not a regular file of captured size",
1062 ));
1063 }
1064 let mut output = OpenOptions::new()
1065 .write(true)
1066 .create_new(true)
1067 .mode(0o600)
1068 .open(destination)
1069 .map_err(internal)?;
1070 let mut remaining = expected_size;
1071 let mut buffer = [0u8; 64 * 1024];
1072 while remaining > 0 {
1073 let cap = remaining.min(buffer.len() as u64) as usize;
1074 let count = input.read(&mut buffer[..cap]).map_err(internal)?;
1075 if count == 0 {
1076 return Err(internal("local LFS hydration source truncated"));
1077 }
1078 output.write_all(&buffer[..count]).map_err(internal)?;
1079 remaining -= count as u64;
1080 }
1081 if input.read(&mut buffer[..1]).map_err(internal)? != 0 {
1082 return Err(internal("local LFS hydration source grew"));
1083 }
1084 drop(output);
1085 verify_lfs_file(destination, expected_sha, expected_size)
1086}
1087
1088#[cfg(feature = "xet-storage")]
1089fn verify_lfs_file(path: &Path, expected_sha: &str, expected_size: u64) -> Git2DBResult<()> {
1090 use sha2::{Digest, Sha256};
1091 let mut file = File::open(path).map_err(|e| internal(format!("open LFS payload: {e}")))?;
1092 let mut hash = Sha256::new();
1093 let mut size = 0u64;
1094 let mut buf = [0u8; 64 * 1024];
1095 loop {
1096 let n = file
1097 .read(&mut buf)
1098 .map_err(|e| internal(format!("hash LFS payload: {e}")))?;
1099 if n == 0 {
1100 break;
1101 }
1102 hash.update(&buf[..n]);
1103 size += n as u64;
1104 }
1105 let actual = format!("{:x}", hash.finalize());
1106 if size != expected_size || actual != expected_sha {
1107 return Err(internal("LFS payload SHA-256/size mismatch"));
1108 }
1109 Ok(())
1110}
1111
1112#[cfg(feature = "xet-storage")]
1113fn verify_xet_file(path: &Path, info: &data::XetFileInfo) -> Git2DBResult<()> {
1114 let mut file = File::open(path).map_err(|e| internal(format!("open XET payload: {e}")))?;
1115 let mut chunker = deduplication::Chunker::default();
1116 let mut chunks = Vec::new();
1117 let mut total = 0u64;
1118 let mut buf = [0u8; 64 * 1024];
1119 loop {
1120 let n = file
1121 .read(&mut buf)
1122 .map_err(|e| internal(format!("read XET payload: {e}")))?;
1123 if n == 0 {
1124 break;
1125 }
1126 total += n as u64;
1127 let block = bytes::Bytes::copy_from_slice(&buf[..n]);
1128 chunks.extend(
1129 chunker
1130 .next_block_bytes(&block, false)
1131 .into_iter()
1132 .map(|chunk| (chunk.hash, chunk.data.len() as u64)),
1133 );
1134 }
1135 chunks.extend(
1136 chunker
1137 .next_block_bytes(&bytes::Bytes::new(), true)
1138 .into_iter()
1139 .map(|chunk| (chunk.hash, chunk.data.len() as u64)),
1140 );
1141 let expected = info
1142 .merkle_hash()
1143 .map_err(|e| internal(format!("XET hash invalid: {e}")))?;
1144 let actual = merklehash::file_hash(&chunks);
1145 if total != info.file_size() || actual != expected {
1146 return Err(internal("XET payload Merkle/size mismatch"));
1147 }
1148 Ok(())
1149}
1150
1151#[cfg(all(test, target_os = "linux"))]
1152mod tests {
1153 use super::*;
1154 use std::os::unix::fs::PermissionsExt;
1155
1156 static BOUNDED_PROJECTION_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
1160
1161 fn disk_backed_private_parent() -> Git2DBResult<tempfile::TempDir> {
1162 let parent = tempfile::tempdir_in(
1163 std::env::current_exe()
1164 .map_err(internal)?
1165 .parent()
1166 .ok_or_else(|| internal("test binary has no parent directory"))?,
1167 )
1168 .map_err(internal)?;
1169 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1170 Ok(parent)
1171 }
1172
1173 #[cfg(feature = "xet-storage")]
1174 fn lfs_pointer(content: &[u8]) -> String {
1175 use sha2::{Digest, Sha256};
1176 format!(
1177 "version https://git-lfs.github.com/spec/v1\noid sha256:{:x}\nsize {}\n",
1178 Sha256::digest(content),
1179 content.len()
1180 )
1181 }
1182
1183 #[tokio::test]
1184 async fn projection_policy_small_ceiling_is_causal_and_default_unchanged() -> Git2DBResult<()> {
1185 let _guard = BOUNDED_PROJECTION_TEST_LOCK.lock().await;
1186 let source = tempfile::tempdir().map_err(internal)?;
1187 let commit = commit_files(source.path(), &[("a", b"12345678")])?;
1188 let parent = disk_backed_private_parent()?;
1189 assert_eq!(
1190 ProjectionPolicy::default().max_expanded_bytes,
1191 2 * 1024 * 1024 * 1024
1192 );
1193 let too_small = ProjectionPolicy::from_trusted_contract(7, BTreeMap::new())?;
1194 assert!(DiskPinnedTreeProjection::acquire_bounded_with_policy(
1195 source.path(),
1196 commit,
1197 parent.path(),
1198 too_small
1199 )
1200 .await
1201 .is_err());
1202 assert!(fs::read_dir(parent.path())
1203 .map_err(internal)?
1204 .next()
1205 .is_none());
1206 let admitted = ProjectionPolicy::from_trusted_contract(8, BTreeMap::new())?;
1207 let artifact = DiskPinnedTreeProjection::acquire_bounded_with_policy(
1208 source.path(),
1209 commit,
1210 parent.path(),
1211 admitted,
1212 )
1213 .await?;
1214 assert_eq!(
1215 fs::read(artifact.root().join("a")).map_err(internal)?,
1216 b"12345678"
1217 );
1218 drop(artifact);
1219 assert!(
1220 DiskPinnedTreeProjection::acquire_bounded(source.path(), commit, parent.path())
1221 .await
1222 .is_ok()
1223 );
1224 Ok(())
1225 }
1226
1227 #[cfg(feature = "xet-storage")]
1228 #[tokio::test]
1229 async fn projection_policy_local_lfs_without_xet_is_immutable_and_hash_verified(
1230 ) -> Git2DBResult<()> {
1231 use sha2::{Digest, Sha256};
1232 let _guard = BOUNDED_PROJECTION_TEST_LOCK.lock().await;
1233 let source = tempfile::tempdir().map_err(internal)?;
1234 let hydrated = tempfile::tempdir().map_err(internal)?;
1235 let bytes = b"authentic original tensor bytes";
1236 let pointer = lfs_pointer(bytes);
1237 let oid = format!("{:x}", Sha256::digest(bytes));
1238 let blob = hydrated.path().join("weights");
1239 fs::write(&blob, bytes).map_err(internal)?;
1240 let commit = commit_files(source.path(), &[("weights", pointer.as_bytes())])?;
1241 let parent = disk_backed_private_parent()?;
1242 let policy = ProjectionPolicy::from_trusted_contract(
1243 bytes.len() as u64,
1244 BTreeMap::from([(oid, blob.clone())]),
1245 )?;
1246 let artifact = DiskPinnedTreeProjection::acquire_bounded_with_policy(
1247 source.path(),
1248 commit,
1249 parent.path(),
1250 policy.clone(),
1251 )
1252 .await?;
1253 fs::write(&blob, vec![b'x'; bytes.len()]).map_err(internal)?;
1255 assert_eq!(
1256 fs::read(artifact.root().join("weights")).map_err(internal)?,
1257 bytes
1258 );
1259 drop(artifact);
1260 let error = DiskPinnedTreeProjection::acquire_bounded_with_policy(
1261 source.path(),
1262 commit,
1263 parent.path(),
1264 policy,
1265 )
1266 .await
1267 .err()
1268 .ok_or_else(|| internal("malformed input unexpectedly admitted"))?;
1269 assert!(error.to_string().contains("SHA-256/size mismatch"));
1270 assert!(fs::read_dir(parent.path())
1271 .map_err(internal)?
1272 .next()
1273 .is_none());
1274 Ok(())
1275 }
1276
1277 #[cfg(feature = "xet-storage")]
1278 #[tokio::test]
1279 async fn projection_policy_giant_declared_lfs_stops_before_payload_allocation(
1280 ) -> Git2DBResult<()> {
1281 let _guard = BOUNDED_PROJECTION_TEST_LOCK.lock().await;
1282 let source = tempfile::tempdir().map_err(internal)?;
1283 let hydrated = tempfile::tempdir().map_err(internal)?;
1284 let oid = "0".repeat(64);
1285 let declared = 3 * 1024 * 1024 * 1024;
1286 let pointer = format!(
1287 "version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {declared}\n"
1288 );
1289 let commit = commit_files(source.path(), &[("weights", pointer.as_bytes())])?;
1290 let parent = disk_backed_private_parent()?;
1291 let error = DiskPinnedTreeProjection::acquire_bounded(source.path(), commit, parent.path())
1292 .await
1293 .err()
1294 .ok_or_else(|| internal("malformed input unexpectedly admitted"))?;
1295 assert!(error.to_string().contains("2147483648-byte limit"));
1296 assert!(fs::read_dir(parent.path())
1297 .map_err(internal)?
1298 .next()
1299 .is_none());
1300 let small = hydrated.path().join("small");
1301 fs::write(&small, b"tiny").map_err(internal)?;
1302 let policy =
1303 ProjectionPolicy::from_trusted_contract(declared, BTreeMap::from([(oid, small)]))?;
1304 let error = DiskPinnedTreeProjection::acquire_bounded_with_policy(
1305 source.path(),
1306 commit,
1307 parent.path(),
1308 policy,
1309 )
1310 .await
1311 .err()
1312 .ok_or_else(|| internal("malformed input unexpectedly admitted"))?;
1313 assert!(error.to_string().contains("regular file of captured size"));
1314 assert!(fs::read_dir(parent.path())
1315 .map_err(internal)?
1316 .next()
1317 .is_none());
1318 Ok(())
1319 }
1320
1321 #[cfg(feature = "xet-storage")]
1322 #[test]
1323 fn projection_policy_local_source_rejects_alias_size_and_oversize_without_large_allocation(
1324 ) -> Git2DBResult<()> {
1325 use sha2::{Digest, Sha256};
1326 let directory = tempfile::tempdir().map_err(internal)?;
1327 let source = directory.path().join("source");
1328 fs::write(&source, b"good").map_err(internal)?;
1329 let oid = format!("{:x}", Sha256::digest(b"good"));
1330 let link = directory.path().join("link");
1331 std::os::unix::fs::symlink(&source, &link).map_err(internal)?;
1332 assert!(hydrate_local_lfs(&link, &directory.path().join("copy"), &oid, 4).is_err());
1333 assert!(hydrate_local_lfs(&source, &directory.path().join("short"), &oid, 3).is_err());
1334 let sparse = directory.path().join("sparse");
1335 File::create(&sparse)
1336 .map_err(internal)?
1337 .set_len(3 * 1024 * 1024 * 1024)
1338 .map_err(internal)?;
1339 assert!(hydrate_local_lfs(&sparse, &directory.path().join("large"), &oid, 4).is_err());
1340 assert!(!directory.path().join("large").exists());
1341 assert!(ProjectionPolicy::from_trusted_contract(0, BTreeMap::new()).is_err());
1342 assert!(ProjectionPolicy::from_trusted_contract(u64::MAX, BTreeMap::new()).is_err());
1343 Ok(())
1344 }
1345
1346 #[test]
1347 fn scavenger_preserves_live_projection_then_reclaims_abandoned_bytes() -> Git2DBResult<()> {
1348 let parent = tempfile::tempdir().map_err(internal)?;
1349 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1350 let owned = parent.path().join("git2db-pinned-disk-owned123");
1351 fs::create_dir(&owned).map_err(internal)?;
1352 fs::set_permissions(&owned, fs::Permissions::from_mode(0o700)).map_err(internal)?;
1353 let owner_lock = create_projection_owner(&owned)?;
1354 fs::write(owned.join("stale-payload"), b"disk bytes").map_err(internal)?;
1355 let foreign_target = parent.path().join("foreign-target");
1356 fs::create_dir(&foreign_target).map_err(internal)?;
1357 fs::write(foreign_target.join("keep"), b"foreign").map_err(internal)?;
1358 std::os::unix::fs::symlink(&foreign_target, owned.join("foreign-link"))
1359 .map_err(internal)?;
1360
1361 scavenge_abandoned_projections(parent.path())?;
1362 assert!(
1363 owned.join("stale-payload").exists(),
1364 "live owner must not be scavenged"
1365 );
1366 drop(owner_lock); scavenge_abandoned_projections(parent.path())?;
1368 assert!(!owned.exists(), "abandoned payload must be reclaimed");
1369 assert_eq!(
1370 fs::read(foreign_target.join("keep")).map_err(internal)?,
1371 b"foreign",
1372 "scavenging must not follow a symlink inside an owned projection"
1373 );
1374 Ok(())
1375 }
1376
1377 #[test]
1378 fn interrupted_cleanup_keeps_marker_until_remaining_payload_is_reclaimed() -> Git2DBResult<()> {
1379 let parent = tempfile::tempdir().map_err(internal)?;
1380 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1381 let owned = parent.path().join("git2db-pinned-disk-interrupted123");
1382 fs::create_dir(&owned).map_err(internal)?;
1383 fs::set_permissions(&owned, fs::Permissions::from_mode(0o700)).map_err(internal)?;
1384 let owner_lock = create_projection_owner(&owned)?;
1385 fs::write(owned.join("already-removed"), b"first").map_err(internal)?;
1386 fs::write(owned.join("remaining-payload"), b"second").map_err(internal)?;
1387
1388 fs::remove_file(owned.join("already-removed")).map_err(internal)?;
1391 assert!(owned.join(OWNER_FILE).exists());
1392 scavenge_abandoned_projections(parent.path())?;
1393 assert!(owned.join("remaining-payload").exists(), "live lock wins");
1394 drop(owner_lock);
1395 scavenge_abandoned_projections(parent.path())?;
1396 assert!(!owned.exists(), "interrupted cleanup must be resumable");
1397 Ok(())
1398 }
1399
1400 #[test]
1401 fn failed_acquire_guard_keeps_lock_until_cleanup_completes() -> Git2DBResult<()> {
1402 let parent = tempfile::tempdir().map_err(internal)?;
1403 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1404 let root = OwnedProjection::create(parent.path(), None)?;
1405 let path = root.path().to_path_buf();
1406 fs::write(path.join("partial-payload"), b"not published").map_err(internal)?;
1407 scavenge_abandoned_projections(parent.path())?;
1408 assert!(path.join("partial-payload").exists());
1409 drop(root); assert!(!path.exists());
1411 assert_eq!(fs::read_dir(parent.path()).map_err(internal)?.count(), 0);
1412 Ok(())
1413 }
1414
1415 #[test]
1416 fn cancelled_acquire_cannot_cleanup_before_blocking_writer_releases_owner() -> Git2DBResult<()>
1417 {
1418 let parent = tempfile::tempdir().map_err(internal)?;
1419 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1420 let slots = Arc::new(Semaphore::new(1));
1421 let permit = Arc::clone(&slots).try_acquire_owned().map_err(internal)?;
1422 let root = OwnedProjection::create(parent.path(), Some(permit))?;
1423 let path = root.path().to_path_buf();
1424 let writer_owner = root.clone();
1425 drop(root); fs::write(path.join("late-payload"), b"writer still owns root").map_err(internal)?;
1427 scavenge_abandoned_projections(parent.path())?;
1428 assert!(path.join("late-payload").exists());
1429 assert!(
1430 Arc::clone(&slots).try_acquire_owned().is_err(),
1431 "late writer must retain the disk-capacity permit"
1432 );
1433 drop(writer_owner);
1434 assert!(!path.exists());
1435 assert!(Arc::clone(&slots).try_acquire_owned().is_ok());
1436 Ok(())
1437 }
1438
1439 #[tokio::test]
1440 async fn cancelled_verifier_keeps_unlinked_payload_and_lease_until_reader_finishes(
1441 ) -> Git2DBResult<()> {
1442 use std::sync::mpsc;
1443 use std::time::Duration;
1444
1445 let parent = tempfile::tempdir().map_err(internal)?;
1446 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1447 let slots = Arc::new(Semaphore::new(1));
1448 let permit = Arc::clone(&slots).try_acquire_owned().map_err(internal)?;
1449 let root = OwnedProjection::create(parent.path(), Some(permit))?;
1450 let path = root.path().to_path_buf();
1451 let payload = path.join("resolved-payload");
1452 fs::write(&payload, b"verified bytes").map_err(internal)?;
1453 let (started_tx, started_rx) = mpsc::channel();
1454 let (finish_tx, finish_rx) = mpsc::channel();
1455 let task = spawn_owned_verifier(&root, move || {
1456 let mut reader = File::open(&payload).map_err(internal)?;
1457 let mut prefix = [0u8; 4];
1458 reader.read_exact(&mut prefix).map_err(internal)?;
1459 started_tx.send(()).map_err(internal)?;
1460 finish_rx
1461 .recv_timeout(Duration::from_secs(5))
1462 .map_err(internal)?;
1463 let mut bytes = prefix.to_vec();
1464 reader.read_to_end(&mut bytes).map_err(internal)?;
1465 if bytes != b"verified bytes" {
1466 return Err(internal("detached verifier read changed bytes"));
1467 }
1468 Ok(())
1469 });
1470 started_rx
1471 .recv_timeout(Duration::from_secs(5))
1472 .map_err(internal)?;
1473 drop(task); drop(root);
1475 scavenge_abandoned_projections(parent.path())?;
1476 assert!(path.exists(), "detached verifier must retain its root");
1477 assert!(Arc::clone(&slots).try_acquire_owned().is_err());
1478 finish_tx.send(()).map_err(internal)?;
1479 for _ in 0..100 {
1480 if !path.exists() && Arc::clone(&slots).try_acquire_owned().is_ok() {
1481 return Ok(());
1482 }
1483 tokio::time::sleep(Duration::from_millis(10)).await;
1484 }
1485 Err(internal(
1486 "detached verifier did not release the projection lease",
1487 ))
1488 }
1489
1490 #[test]
1491 fn ordinary_drop_closes_unlinked_reader_before_releasing_capacity() -> Git2DBResult<()> {
1492 let parent = tempfile::tempdir().map_err(internal)?;
1493 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1494 let slots = Arc::new(Semaphore::new(1));
1495 let permit = Arc::clone(&slots).try_acquire_owned().map_err(internal)?;
1496 let mut root = OwnedProjection::create(parent.path(), Some(permit))?;
1497 let payload = root.path().join("payload");
1498 fs::write(&payload, b"owned bytes").map_err(internal)?;
1499 let reader = File::open(&payload).map_err(internal)?;
1500 fs::remove_file(&payload).map_err(internal)?;
1501 let reader_fd = reader.as_raw_fd();
1502 let owner = Arc::get_mut(&mut root.0)
1503 .ok_or_else(|| internal("unexpected retained projection owner"))?;
1504 owner.observed_reader_fd = Some(reader_fd);
1505 let projection = DiskPinnedTreeProjection {
1506 commit: Oid::zero(),
1507 owner_pid: std::process::id(),
1508 input_root: root.path().to_path_buf(),
1509 files: vec![reader],
1510 _root: root,
1511 };
1512 assert!(Arc::clone(&slots).try_acquire_owned().is_err());
1513 drop(projection); assert!(Arc::clone(&slots).try_acquire_owned().is_ok());
1515 assert!(fs::read_dir(parent.path())
1516 .map_err(internal)?
1517 .next()
1518 .is_none());
1519 Ok(())
1520 }
1521
1522 #[tokio::test(flavor = "current_thread")]
1523 async fn blocked_projection_write_keeps_service_reactor_live_and_lease_owned(
1524 ) -> Git2DBResult<()> {
1525 use std::os::fd::FromRawFd;
1526 use std::sync::mpsc;
1527 use std::time::{Duration, Instant};
1528
1529 let parent = tempfile::tempdir().map_err(internal)?;
1530 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1531 let slots = Arc::new(Semaphore::new(1));
1532 let permit = Arc::clone(&slots).try_acquire_owned().map_err(internal)?;
1533 let mut pipe = [-1; 2];
1534 if unsafe { libc::pipe2(pipe.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
1535 return Err(internal(std::io::Error::last_os_error()));
1536 }
1537 let mut reader = unsafe { File::from_raw_fd(pipe[0]) };
1539 let mut writer = unsafe { File::from_raw_fd(pipe[1]) };
1540 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
1541 let parent_path = parent.path().to_path_buf();
1542 let task = tokio::spawn(run_on_projection_worker(move || {
1543 let root = OwnedProjection::create(&parent_path, Some(permit))?;
1544 let _ = started_tx.send(root.path().to_path_buf());
1545 writer
1548 .write_all(&vec![0u8; 8 * 1024 * 1024])
1549 .map_err(internal)?;
1550 Ok(())
1551 }));
1552 let root_path = started_rx
1553 .await
1554 .map_err(|e| internal(format!("projection worker did not start: {e}")))?;
1555 let (release_tx, release_rx) = mpsc::channel();
1556 let drainer = std::thread::spawn(move || {
1557 let _ = release_rx.recv_timeout(Duration::from_secs(5));
1560 std::io::copy(&mut reader, &mut std::io::sink())
1561 });
1562 let start = Instant::now();
1563 tokio::time::sleep(Duration::from_millis(25)).await;
1564 assert!(
1565 start.elapsed() < Duration::from_secs(2),
1566 "blocking disk backpressure stalled the current-thread service reactor"
1567 );
1568 assert!(!task.is_finished(), "the blocked write lost backpressure");
1569 task.abort();
1570 match task.await {
1571 Err(error) if error.is_cancelled() => {}
1572 outcome => return Err(internal(format!("caller was not cancelled: {outcome:?}"))),
1573 }
1574 assert!(Arc::clone(&slots).try_acquire_owned().is_err());
1576 assert!(root_path.exists());
1577 release_tx.send(()).map_err(internal)?;
1578 tokio::task::spawn_blocking(move || drainer.join())
1579 .await
1580 .map_err(internal)?
1581 .map_err(|_| internal("pipe drainer panicked"))?
1582 .map_err(internal)?;
1583 for _ in 0..100 {
1584 if !root_path.exists() && Arc::clone(&slots).try_acquire_owned().is_ok() {
1585 return Ok(());
1586 }
1587 tokio::time::sleep(Duration::from_millis(10)).await;
1588 }
1589 Err(internal(
1590 "blocked projection worker did not release its lease",
1591 ))
1592 }
1593
1594 #[test]
1595 fn scavenger_never_follows_symlinks_or_removes_unmarked_foreign_dirs() -> Git2DBResult<()> {
1596 let parent = tempfile::tempdir().map_err(internal)?;
1597 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1598 let foreign = parent.path().join("git2db-pinned-disk-foreign123");
1599 fs::create_dir(&foreign).map_err(internal)?;
1600 fs::write(foreign.join("keep"), b"foreign").map_err(internal)?;
1601 let target = parent.path().join("outside-projection");
1602 fs::create_dir(&target).map_err(internal)?;
1603 fs::write(target.join("keep"), b"target").map_err(internal)?;
1604 let link = parent.path().join("git2db-pinned-disk-symlink123");
1605 std::os::unix::fs::symlink(&target, &link).map_err(internal)?;
1606
1607 scavenge_abandoned_projections(parent.path())?;
1608 assert_eq!(
1609 fs::read(foreign.join("keep")).map_err(internal)?,
1610 b"foreign"
1611 );
1612 assert_eq!(fs::read(target.join("keep")).map_err(internal)?, b"target");
1613 assert!(link.is_symlink(), "lookalike symlink must not be removed");
1614 Ok(())
1615 }
1616
1617 fn commit_files(root: &Path, files: &[(&str, &[u8])]) -> Git2DBResult<Oid> {
1618 let repo = Repository::init(root).map_err(internal)?;
1619 let mut index = repo.index().map_err(internal)?;
1620 for &(name, content) in files {
1621 fs::write(root.join(name), content).map_err(internal)?;
1622 index.add_path(Path::new(name)).map_err(internal)?;
1623 }
1624 index.write().map_err(internal)?;
1625 let tree = repo
1626 .find_tree(index.write_tree().map_err(internal)?)
1627 .map_err(internal)?;
1628 let signature = git2::Signature::now("test", "test@example.invalid").map_err(internal)?;
1629 repo.commit(Some("HEAD"), &signature, &signature, "pinned", &tree, &[])
1630 .map_err(internal)
1631 }
1632
1633 #[tokio::test]
1634 async fn bounded_projection_rejects_distinct_ref_burst_and_reclaims_lease() -> Git2DBResult<()>
1635 {
1636 let _guard = BOUNDED_PROJECTION_TEST_LOCK.lock().await;
1637 let parent = tempfile::tempdir().map_err(internal)?;
1638 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1639 let first_repo = tempfile::tempdir().map_err(internal)?;
1640 let first_oid = commit_files(first_repo.path(), &[("weights.bin", b"first")])?;
1641 let second_repo = tempfile::tempdir().map_err(internal)?;
1642 let second_oid = commit_files(second_repo.path(), &[("weights.bin", b"second")])?;
1643
1644 let first =
1645 DiskPinnedTreeProjection::acquire_bounded(first_repo.path(), first_oid, parent.path())
1646 .await?;
1647 assert!(
1648 DiskPinnedTreeProjection::acquire_bounded(
1649 second_repo.path(),
1650 second_oid,
1651 parent.path()
1652 )
1653 .await
1654 .is_err(),
1655 "a different ref must not acquire a second private projection while the lease is held"
1656 );
1657 assert_eq!(
1658 fs::read_dir(parent.path()).map_err(internal)?.count(),
1659 1,
1660 "rejected burst must not create a partial projection"
1661 );
1662 drop(first);
1663 assert_eq!(
1664 fs::read_dir(parent.path()).map_err(internal)?.count(),
1665 0,
1666 "dropping the artifact must reclaim projection bytes and lease"
1667 );
1668 let second = DiskPinnedTreeProjection::acquire_bounded(
1669 second_repo.path(),
1670 second_oid,
1671 parent.path(),
1672 )
1673 .await?;
1674 assert_eq!(second.commit(), second_oid);
1675 drop(second);
1676 Ok(())
1677 }
1678
1679 #[tokio::test]
1680 async fn bounded_projection_rejects_file_count_before_temp_copy() -> Git2DBResult<()> {
1681 let _guard = BOUNDED_PROJECTION_TEST_LOCK.lock().await;
1685 let parent = disk_backed_private_parent()?;
1686 let source = tempfile::tempdir().map_err(internal)?;
1687 let files = (0..=MAX_FILES)
1688 .map(|index| (format!("model-{index}.bin"), b"small".as_slice()))
1689 .collect::<Vec<_>>();
1690 let borrowed = files
1691 .iter()
1692 .map(|(name, bytes)| (name.as_str(), *bytes))
1693 .collect::<Vec<_>>();
1694 let commit = commit_files(source.path(), &borrowed)?;
1695 let error =
1696 match DiskPinnedTreeProjection::acquire_bounded(source.path(), commit, parent.path())
1697 .await
1698 {
1699 Ok(projection) => {
1700 return Err(internal(format!(
1701 "oversized model tree must fail bounded admission: {projection:?}"
1702 )))
1703 }
1704 Err(error) => error,
1705 };
1706 assert!(
1707 error
1708 .to_string()
1709 .contains("model tree exceeds 512 file limit"),
1710 "unexpected bounded admission failure: {error}"
1711 );
1712 assert_eq!(
1713 fs::read_dir(parent.path()).map_err(internal)?.count(),
1714 0,
1715 "file-cap rejection must precede projection directory creation"
1716 );
1717 Ok(())
1718 }
1719
1720 #[tokio::test]
1721 async fn packed_blob_stays_exact_during_checkout_mutation_and_lazy_thread_read(
1722 ) -> Git2DBResult<()> {
1723 let source = tempfile::tempdir().map_err(internal)?;
1724 let parent = tempfile::tempdir().map_err(internal)?;
1725 fs::set_permissions(parent.path(), fs::Permissions::from_mode(0o700)).map_err(internal)?;
1726 let reviewed = b"reviewed packed model bytes";
1727 let commit = commit_files(source.path(), &[("weights.safetensors", reviewed)])?;
1728 let repo = Repository::open(source.path()).map_err(internal)?;
1729 let blob = repo
1730 .find_commit(commit)
1731 .map_err(internal)?
1732 .tree()
1733 .map_err(internal)?
1734 .get_name("weights.safetensors")
1735 .ok_or_else(|| internal("missing test blob"))?
1736 .id();
1737 drop(repo);
1738 let status = Command::new(GIT_CLI)
1739 .arg("-C")
1740 .arg(source.path())
1741 .args(["gc", "--prune=now"])
1742 .status()
1743 .map_err(internal)?;
1744 assert!(status.success());
1745 let loose = source
1746 .path()
1747 .join(".git/objects")
1748 .join(&blob.to_string()[..2])
1749 .join(&blob.to_string()[2..]);
1750 assert!(!loose.exists(), "test blob must be packed");
1751 let linked_parent = tempfile::tempdir().map_err(internal)?;
1752 let linked = linked_parent.path().join("linked-model");
1753 let status = Command::new(GIT_CLI)
1754 .arg("-C")
1755 .arg(source.path())
1756 .args(["worktree", "add", "--detach"])
1757 .arg(&linked)
1758 .arg(commit.to_string())
1759 .status()
1760 .map_err(internal)?;
1761 assert!(status.success());
1762 let mut projection =
1763 DiskPinnedTreeProjection::acquire(&linked, commit, parent.path()).await?;
1764 projection.ensure_current_process()?;
1765 assert_eq!(projection.file_count(), 1);
1766 let input = projection.root().join("weights.safetensors");
1767 assert!(
1768 input.is_file(),
1769 "loader file-type checks must follow the fd alias"
1770 );
1771 scavenge_abandoned_projections(parent.path())?;
1772 assert!(input.is_file(), "active projection must survive scavenging");
1773 fs::write(linked.join("weights.safetensors"), b"unreviewed mutation").map_err(internal)?;
1774 assert_eq!(fs::read(&input).map_err(internal)?, reviewed);
1775 assert_eq!(
1776 std::thread::spawn({
1777 let input = input.clone();
1778 move || fs::read(input)
1779 })
1780 .join()
1781 .map_err(|_| internal("reader thread panicked"))?
1782 .map_err(internal)?,
1783 reviewed
1784 );
1785 assert!(OpenOptions::new().write(true).open(&input).is_err());
1786 let child = Command::new("/usr/bin/cat")
1787 .arg(&input)
1788 .output()
1789 .map_err(internal)?;
1790 assert!(
1791 !child.status.success(),
1792 "subprocess must not inherit projection descriptors"
1793 );
1794 assert_ne!(child.stdout, reviewed);
1795 projection.owner_pid = projection.owner_pid.wrapping_add(1);
1796 assert!(projection.ensure_current_process().is_err());
1797 projection.owner_pid = std::process::id();
1798 assert_eq!(
1799 fs::read_dir(projection._root.path().join("payloads"))
1800 .map_err(internal)?
1801 .count(),
1802 0,
1803 "no payload pathname may remain"
1804 );
1805 drop(projection);
1806 assert!(
1807 !input.exists(),
1808 "dropping the projection removes lazy aliases"
1809 );
1810 assert_eq!(fs::read_dir(parent.path()).map_err(internal)?.count(), 0);
1811 Ok(())
1812 }
1813
1814 #[tokio::test]
1815 async fn failed_acquire_removes_partially_streamed_private_tree() -> Git2DBResult<()> {
1816 let _guard = BOUNDED_PROJECTION_TEST_LOCK.lock().await;
1817 let source = tempfile::tempdir().map_err(internal)?;
1818 let commit = commit_files(
1819 source.path(),
1820 &[
1821 ("a-config.json", b"verified first file" as &[u8]),
1822 ("z-weights.bin", b"never published"),
1823 ],
1824 )?;
1825 let parent = disk_backed_private_parent()?;
1826 let error = match DiskPinnedTreeProjection::acquire_bounded_hooked(
1830 source.path(),
1831 commit,
1832 parent.path(),
1833 Some(StreamHooks {
1834 fail_after_streams: 1,
1835 observe: Box::new(|root: &Path, streamed: usize| {
1836 assert!(streamed >= 1, "failure must follow a streamed file");
1837 assert!(
1838 root.join(OWNER_FILE).exists(),
1839 "projection owner must already exist"
1840 );
1841 let projected = root.join("staged-inputs").join("a-config.json");
1842 let bytes = fs::read(&projected).unwrap_or_else(|error| {
1843 panic!("projected input must read through its live descriptor: {error}")
1844 });
1845 assert_eq!(bytes, b"verified first file");
1846 let capacity_occupied = PROJECTION_CAPACITY
1847 .get()
1848 .map(|capacity| Arc::clone(capacity).try_acquire_owned().is_err());
1849 assert_eq!(
1850 capacity_occupied,
1851 Some(true),
1852 "capacity must remain leased while owner and reader live"
1853 );
1854 }),
1855 }),
1856 )
1857 .await
1858 {
1859 Ok(projection) => {
1860 return Err(internal(format!(
1861 "injected post-stream failure must fail the acquisition: {projection:?}"
1862 )))
1863 }
1864 Err(error) => error,
1865 };
1866 assert!(
1867 error
1868 .to_string()
1869 .contains("injected stream failure after 1 streamed model payloads"),
1870 "unexpected acquire failure: {error}"
1871 );
1872 assert_eq!(
1873 fs::read_dir(parent.path()).map_err(internal)?.count(),
1874 0,
1875 "partially streamed projection root must be removed"
1876 );
1877 let follow = tempfile::tempdir().map_err(internal)?;
1880 let follow_commit = commit_files(follow.path(), &[("weights.bin", b"fresh")])?;
1881 let reclaimed =
1882 DiskPinnedTreeProjection::acquire_bounded(follow.path(), follow_commit, parent.path())
1883 .await?;
1884 drop(reclaimed);
1885 Ok(())
1886 }
1887
1888 #[test]
1889 fn streamed_blob_tamper_and_oid_mismatch_fail_closed() -> Git2DBResult<()> {
1890 let dir = tempfile::tempdir().map_err(internal)?;
1891 let original = dir.path().join("blob");
1892 fs::write(&original, b"reviewed").map_err(internal)?;
1893 let oid = Oid::hash_file(ObjectType::Blob, &original).map_err(internal)?;
1894 verify_blob_file(oid, &original)?;
1895 fs::write(&original, b"replaced").map_err(internal)?;
1896 assert!(verify_blob_file(oid, &original).is_err());
1897 let repo = Repository::init(dir.path().join("repo")).map_err(internal)?;
1898 let other = repo.blob(b"other").map_err(internal)?;
1899 let destination = dir.path().join("wrong-oid");
1900 assert!(stream_verified_blob(repo.path(), other, 5, &destination).is_ok());
1901 assert!(
1902 stream_verified_blob(repo.path(), oid, 8, &dir.path().join("missing-oid")).is_err()
1903 );
1904 Ok(())
1905 }
1906
1907 #[test]
1908 fn stream_rejects_larger_blob_than_preflight_before_writing() -> Git2DBResult<()> {
1909 let dir = tempfile::tempdir().map_err(internal)?;
1910 let repo = Repository::init(dir.path().join("repo")).map_err(internal)?;
1911 let oid = repo
1912 .blob(b"larger-than-the-preflighted-blob")
1913 .map_err(internal)?;
1914 let destination = dir.path().join("must-not-exist");
1915 assert!(stream_verified_blob(repo.path(), oid, 1, &destination).is_err());
1916 assert!(
1917 !destination.exists(),
1918 "oversized Git blob must be rejected before file creation"
1919 );
1920 Ok(())
1921 }
1922
1923 #[cfg(feature = "xet-storage")]
1924 #[test]
1925 fn streamed_lfs_and_xet_digest_checks_reject_matching_size_tamper() -> Git2DBResult<()> {
1926 use sha2::{Digest, Sha256};
1927 let dir = tempfile::tempdir().map_err(internal)?;
1928 let path = dir.path().join("resolved");
1929 let bytes = vec![b'R'; 256 * 1024 + 17];
1930 fs::write(&path, &bytes).map_err(internal)?;
1931 let sha = format!("{:x}", Sha256::digest(&bytes));
1932 verify_lfs_file(&path, &sha, bytes.len() as u64)?;
1933 let mut chunker = deduplication::Chunker::default();
1934 let chunks = chunker.next_block_bytes(&bytes::Bytes::copy_from_slice(&bytes), true);
1935 let expected = merklehash::file_hash(
1936 &chunks
1937 .iter()
1938 .map(|chunk| (chunk.hash, chunk.data.len() as u64))
1939 .collect::<Vec<_>>(),
1940 );
1941 let info = data::XetFileInfo::new(expected.hex(), bytes.len() as u64);
1942 verify_xet_file(&path, &info)?;
1943 fs::write(&path, vec![b'X'; bytes.len()]).map_err(internal)?;
1944 assert!(verify_lfs_file(&path, &sha, bytes.len() as u64).is_err());
1945 assert!(verify_xet_file(&path, &info).is_err());
1946 Ok(())
1947 }
1948}