1use crate::errors::{Git2DBError, Git2DBResult, LfsErrorKind};
36use async_trait::async_trait;
37use sha2::{Digest, Sha256};
38use std::collections::HashMap;
39use std::path::{Path, PathBuf};
40use std::sync::Arc;
41use tokio::sync::Semaphore;
42use tracing::{debug, info, warn};
43
44#[cfg(feature = "xet-storage")]
45use crate::xet::{StorageBackend, XetConfig, XetStorage};
46
47pub const XET_MERKLE_EXT_KEY: &str = "xet-merkle";
53
54pub const XET_SHARD_EXT_KEY: &str = "xet-shard";
58
59#[derive(Debug, Clone)]
69pub struct LfsPointer {
70 version: String,
72 hash_method: String,
74 oid: String,
76 size: u64,
78 extensions: HashMap<String, String>,
80}
81
82impl LfsPointer {
83 pub fn new(version: String, oid: String, size: u64) -> Git2DBResult<Self> {
87 Self::new_with_hash_method(version, "sha256".to_owned(), oid, size)
89 }
90
91 pub fn new_with_hash_method(
93 version: String,
94 hash_method: String,
95 oid: String,
96 size: u64,
97 ) -> Git2DBResult<Self> {
98 if hash_method == "sha256"
100 && (oid.len() != 64 || !oid.chars().all(|c| c.is_ascii_hexdigit()))
101 {
102 return Err(Git2DBError::lfs(
103 LfsErrorKind::InvalidPointer,
104 format!(
105 "Invalid OID format: expected 64 hex characters, got {}",
106 oid.len()
107 ),
108 ));
109 }
110
111 Ok(LfsPointer {
112 version,
113 hash_method,
114 oid,
115 size,
116 extensions: HashMap::new(),
117 })
118 }
119
120 pub fn parse(content: &str) -> Git2DBResult<Self> {
134 let mut version = None;
135 let mut hash_method = None;
136 let mut oid = None;
137 let mut size = None;
138 let mut extensions = HashMap::new();
139
140 for line in content.lines() {
141 let line = line.trim();
142 if line.is_empty() {
143 continue;
144 }
145
146 if let Some(v) = line.strip_prefix("version ") {
147 version = Some(v.to_owned());
148 } else if let Some(o) = line.strip_prefix("oid sha256:") {
149 hash_method = Some("sha256".to_owned());
150 oid = Some(o.to_owned());
151 } else if let Some(s) = line.strip_prefix("size ") {
152 size = Some(s.parse::<u64>().map_err(|_| {
153 Git2DBError::lfs(LfsErrorKind::ParseError, "Invalid size value")
154 })?);
155 } else if let Some((key, value)) = line.split_once(' ') {
156 extensions.insert(key.to_owned(), value.to_owned());
158 }
159 }
160
161 let version = version.ok_or_else(|| {
162 Git2DBError::lfs(LfsErrorKind::InvalidPointer, "Missing version field")
163 })?;
164 let hash_method = hash_method.ok_or_else(|| {
165 Git2DBError::lfs(
166 LfsErrorKind::InvalidPointer,
167 "Missing or unsupported oid hash method",
168 )
169 })?;
170 let oid =
171 oid.ok_or_else(|| Git2DBError::lfs(LfsErrorKind::InvalidPointer, "Missing oid field"))?;
172 let size = size
173 .ok_or_else(|| Git2DBError::lfs(LfsErrorKind::InvalidPointer, "Missing size field"))?;
174
175 if oid.len() != 64 || !oid.chars().all(|c| c.is_ascii_hexdigit()) {
177 return Err(Git2DBError::lfs(
178 LfsErrorKind::InvalidPointer,
179 format!(
180 "Invalid OID format: expected 64 hex characters, got {}",
181 oid.len()
182 ),
183 ));
184 }
185
186 Ok(LfsPointer {
187 version,
188 hash_method,
189 oid,
190 size,
191 extensions,
192 })
193 }
194
195 pub fn version(&self) -> &str {
197 &self.version
198 }
199
200 pub fn hash_method(&self) -> &str {
202 &self.hash_method
203 }
204
205 pub fn oid(&self) -> &str {
207 &self.oid
208 }
209
210 pub fn size(&self) -> u64 {
212 self.size
213 }
214
215 pub fn extensions(&self) -> &HashMap<String, String> {
217 &self.extensions
218 }
219
220 #[cfg(feature = "xet-storage")]
234 #[deprecated(
235 since = "0.1.0",
236 note = "unsound: reinterprets LFS SHA-256 as a XET MerkleHash; use the xet-merkle extension via xet_merkle_hash() instead — see #386"
237 )]
238 pub fn to_merkle_hash(&self) -> Git2DBResult<merklehash::MerkleHash> {
239 warn!(
240 oid = %self.oid,
241 "LfsPointer::to_merkle_hash() is unsound (SHA-256 != XET MerkleHash); use the xet-merkle extension instead — see #386"
242 );
243 merklehash::MerkleHash::from_hex(&self.oid).map_err(|e| {
244 Git2DBError::lfs(
245 LfsErrorKind::HashConversion,
246 format!("Failed to convert SHA256 to MerkleHash: {e}"),
247 )
248 })
249 }
250
251 #[cfg(feature = "xet-storage")]
256 pub fn xet_merkle_hex(&self) -> Option<&str> {
257 self.extensions.get(XET_MERKLE_EXT_KEY).map(String::as_str)
258 }
259
260 pub fn xet_shard(&self) -> Option<&str> {
262 self.extensions.get(XET_SHARD_EXT_KEY).map(String::as_str)
263 }
264
265 #[cfg(feature = "xet-storage")]
272 pub fn xet_merkle_hash(&self) -> Git2DBResult<merklehash::MerkleHash> {
273 match self.xet_merkle_hex() {
274 Some(hex) => merklehash::MerkleHash::from_hex(hex).map_err(|e| {
275 Git2DBError::lfs(
276 LfsErrorKind::HashConversion,
277 format!("Invalid xet-merkle extension (expected 64 hex chars): {e}"),
278 )
279 }),
280 None => Err(Git2DBError::lfs(
281 LfsErrorKind::MissingXetMerkle,
282 "missing xet-merkle; to_merkle_hash() is unsound — see #386",
283 )),
284 }
285 }
286
287 #[cfg(feature = "xet-storage")]
294 pub fn set_xet_extensions(&mut self, merkle: &merklehash::MerkleHash, shard: Option<&str>) {
295 self.extensions
296 .insert(XET_MERKLE_EXT_KEY.to_owned(), merkle.to_string());
297 if let Some(s) = shard {
298 self.extensions
299 .insert(XET_SHARD_EXT_KEY.to_owned(), s.to_owned());
300 }
301 }
302}
303
304impl std::fmt::Display for LfsPointer {
305 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
306 write!(
311 f,
312 "version {}\noid {}:{}\nsize {}\n",
313 self.version, self.hash_method, self.oid, self.size
314 )?;
315 let mut keys: Vec<&String> = self.extensions.keys().collect();
316 keys.sort_unstable();
317 for key in keys {
318 writeln!(f, "{} {}", key, self.extensions[key])?;
319 }
320 Ok(())
321 }
322}
323
324pub fn is_lfs_pointer(content: &str) -> bool {
336 content.starts_with("version https://git-lfs.github.com/spec/v1")
337 || content.starts_with("version https://hawser.github.com/spec/v1")
338}
339
340#[cfg(feature = "xet-storage")]
344#[async_trait]
345pub trait LfsSmudge {
346 async fn smudge_lfs(&self, pointer: &LfsPointer) -> Git2DBResult<Vec<u8>>;
348
349 async fn smudge_lfs_to_file(&self, pointer: &LfsPointer, path: &Path) -> Git2DBResult<()>;
351
352 async fn smudge_lfs_validated(&self, pointer: &LfsPointer) -> Git2DBResult<Vec<u8>>;
356}
357
358#[cfg(feature = "xet-storage")]
359#[async_trait]
360impl LfsSmudge for XetStorage {
361 async fn smudge_lfs(&self, pointer: &LfsPointer) -> Git2DBResult<Vec<u8>> {
362 let hash = pointer.xet_merkle_hash()?;
363 self.smudge_from_hash(&hash).await.map_err(|e| {
364 Git2DBError::lfs(
365 LfsErrorKind::SmudgeFailed,
366 format!("XET smudge failed: {e}"),
367 )
368 })
369 }
370
371 async fn smudge_lfs_to_file(&self, pointer: &LfsPointer, path: &Path) -> Git2DBResult<()> {
372 let hash = pointer.xet_merkle_hash()?;
373 self.smudge_from_hash_to_file(&hash, path)
374 .await
375 .map_err(|e| {
376 Git2DBError::lfs(
377 LfsErrorKind::SmudgeFailed,
378 format!("XET smudge to file failed: {e}"),
379 )
380 })
381 }
382
383 async fn smudge_lfs_validated(&self, pointer: &LfsPointer) -> Git2DBResult<Vec<u8>> {
384 let content = self.smudge_lfs(pointer).await?;
385
386 if content.len() as u64 != pointer.size() {
388 return Err(Git2DBError::lfs(
389 LfsErrorKind::SizeMismatch,
390 format!(
391 "Size mismatch: expected {} bytes, got {} bytes",
392 pointer.size(),
393 content.len()
394 ),
395 ));
396 }
397
398 let mut hasher = Sha256::new();
400 hasher.update(&content);
401 let computed_hash = format!("{:x}", hasher.finalize());
402
403 if computed_hash != pointer.oid() {
404 return Err(Git2DBError::lfs(
405 LfsErrorKind::HashMismatch,
406 format!(
407 "SHA256 mismatch: expected {}, got {}",
408 pointer.oid(),
409 computed_hash
410 ),
411 ));
412 }
413
414 debug!(
415 "Validated LFS content: {} bytes, SHA256: {}...",
416 pointer.size(),
417 &pointer.oid()[..8]
418 );
419
420 Ok(content)
421 }
422}
423
424#[derive(Debug, Default, Clone)]
426pub struct ProcessingStats {
427 pub total_files: usize,
429 pub processed: usize,
431 pub skipped: usize,
433 pub failed: usize,
435 pub bytes_processed: u64,
437}
438
439#[cfg(feature = "xet-storage")]
444#[derive(Clone)]
445pub struct LfsStorage {
446 xet: Arc<XetStorage>,
447}
448
449#[cfg(feature = "xet-storage")]
450impl LfsStorage {
451 pub async fn new(config: &XetConfig) -> Git2DBResult<Self> {
453 let xet = XetStorage::new(config).await.map_err(|e| {
454 Git2DBError::lfs(
455 LfsErrorKind::IoError,
456 format!("Failed to create XET storage: {e}"),
457 )
458 })?;
459 Ok(Self { xet: Arc::new(xet) })
460 }
461
462 pub fn is_lfs_pointer(&self, content: &str) -> bool {
466 is_lfs_pointer(content)
467 }
468
469 pub fn is_xet_pointer(&self, content: &str) -> bool {
471 self.xet.is_pointer(content)
472 }
473
474 pub fn is_pointer(&self, content: &str) -> bool {
476 self.is_lfs_pointer(content) || self.is_xet_pointer(content)
477 }
478
479 pub fn parse_lfs_pointer(&self, content: &str) -> Git2DBResult<LfsPointer> {
481 LfsPointer::parse(content)
482 }
483
484 pub async fn smudge_lfs_pointer(&self, pointer: &LfsPointer) -> Git2DBResult<Vec<u8>> {
488 self.xet.smudge_lfs(pointer).await
489 }
490
491 pub async fn smudge_lfs_pointer_to_file_bounded(
494 &self,
495 pointer: &LfsPointer,
496 path: &Path,
497 ) -> Git2DBResult<u64> {
498 let hash = pointer.xet_merkle_hash()?;
499 self.xet
500 .smudge_hash_to_file_bounded(&hash, path, pointer.size())
501 .await
502 .map_err(|e| Git2DBError::lfs(LfsErrorKind::SmudgeFailed, e.to_string()))
503 }
504
505 pub async fn smudge_xet_pointer_to_file_bounded(
508 &self,
509 pointer: &str,
510 path: &Path,
511 max_output_bytes: u64,
512 ) -> Git2DBResult<u64> {
513 self.xet
514 .smudge_file_bounded(pointer, path, max_output_bytes)
515 .await
516 .map_err(|e| Git2DBError::lfs(LfsErrorKind::SmudgeFailed, e.to_string()))
517 }
518
519 pub async fn smudge_lfs_pointer_to_file(
522 &self,
523 pointer: &LfsPointer,
524 path: &Path,
525 ) -> Git2DBResult<()> {
526 self.xet.smudge_lfs_to_file(pointer, path).await
527 }
528
529 pub async fn smudge_lfs_pointer_validated(
531 &self,
532 pointer: &LfsPointer,
533 ) -> Git2DBResult<Vec<u8>> {
534 self.xet.smudge_lfs_validated(pointer).await
535 }
536
537 pub async fn clean_file(&self, path: &Path) -> Git2DBResult<String> {
539 self.xet.clean_file(path).await.map_err(|e| {
540 Git2DBError::lfs(LfsErrorKind::IoError, format!("XET clean_file failed: {e}"))
541 })
542 }
543
544 pub async fn clean_bytes(&self, data: &[u8]) -> Git2DBResult<String> {
546 self.xet.clean_bytes(data).await.map_err(|e| {
547 Git2DBError::lfs(
548 LfsErrorKind::IoError,
549 format!("XET clean_bytes failed: {e}"),
550 )
551 })
552 }
553
554 pub async fn smudge_file(&self, pointer: &str, output_path: &Path) -> Git2DBResult<()> {
556 self.xet
557 .smudge_file(pointer, output_path)
558 .await
559 .map_err(|e| {
560 Git2DBError::lfs(
561 LfsErrorKind::SmudgeFailed,
562 format!("XET smudge_file failed: {e}"),
563 )
564 })
565 }
566
567 pub async fn smudge_bytes(&self, pointer: &str) -> Git2DBResult<Vec<u8>> {
569 self.xet.smudge_bytes(pointer).await.map_err(|e| {
570 Git2DBError::lfs(
571 LfsErrorKind::SmudgeFailed,
572 format!("XET smudge_bytes failed: {e}"),
573 )
574 })
575 }
576
577 pub async fn load_file(&self, file_path: &Path) -> Git2DBResult<Vec<u8>> {
589 use tokio::io::{AsyncReadExt, AsyncSeekExt};
590
591 let metadata = tokio::fs::metadata(file_path).await.map_err(|e| {
594 Git2DBError::lfs(
595 LfsErrorKind::IoError,
596 format!("Failed to get file metadata: {e}"),
597 )
598 })?;
599 let file_size = metadata.len();
600
601 if file_size >= 1024 {
603 return tokio::fs::read(file_path).await.map_err(|e| {
604 Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to read file: {e}"))
605 });
606 }
607
608 let mut file = tokio::fs::File::open(file_path).await.map_err(|e| {
611 Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to open file: {e}"))
612 })?;
613
614 let mut header = vec![0u8; 100];
615 let n = file.read(&mut header).await.map_err(|e| {
616 Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to read header: {e}"))
617 })?;
618 header.truncate(n);
619
620 if let Ok(text) = String::from_utf8(header.clone()) {
622 if text.starts_with("version https://git-lfs")
623 || text.starts_with("version https://hawser")
624 || text.starts_with("# xet version")
625 {
626 let mut rest = Vec::new();
628 file.read_to_end(&mut rest).await.map_err(|e| {
629 Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to read rest: {e}"))
630 })?;
631
632 let full_content = [&header[..n], &rest[..]].concat();
633 let full_text = String::from_utf8(full_content).map_err(|_| {
634 Git2DBError::lfs(
635 LfsErrorKind::InvalidPointer,
636 "Pointer file is not valid UTF-8",
637 )
638 })?;
639
640 if self.is_lfs_pointer(&full_text) {
642 let lfs_pointer = self.parse_lfs_pointer(&full_text)?;
643 return self.smudge_lfs_pointer_validated(&lfs_pointer).await;
644 } else if self.is_xet_pointer(&full_text) {
645 return self.smudge_bytes(&full_text).await;
646 }
647 }
648 }
649
650 file.seek(std::io::SeekFrom::Start(0))
652 .await
653 .map_err(|e| Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to seek: {e}")))?;
654 let mut contents = Vec::new();
655 file.read_to_end(&mut contents).await.map_err(|e| {
656 Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to read file: {e}"))
657 })?;
658
659 Ok(contents)
660 }
661
662 pub async fn scan_lfs_files(&self, directory: &Path) -> Git2DBResult<Vec<PathBuf>> {
666 let directory = directory.to_path_buf();
667 let this = self.clone();
668
669 tokio::task::spawn_blocking(move || this.scan_lfs_files_sync(&directory))
670 .await
671 .map_err(|e| {
672 Git2DBError::lfs(LfsErrorKind::IoError, format!("Task join failed: {e}"))
673 })?
674 }
675
676 fn scan_lfs_files_sync(&self, directory: &Path) -> Git2DBResult<Vec<PathBuf>> {
677 use git2::{ObjectType, TreeWalkMode, TreeWalkResult};
678
679 let repo = git2::Repository::discover(directory).map_err(|e| {
681 Git2DBError::lfs(
682 LfsErrorKind::NotInRepository,
683 format!("Directory is not part of a git repository: {e}"),
684 )
685 })?;
686
687 let repo_workdir = repo.workdir().ok_or_else(|| {
688 Git2DBError::lfs(
689 LfsErrorKind::NotInRepository,
690 "Repository has no working directory",
691 )
692 })?;
693
694 let head = repo.head().map_err(|e| {
696 Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to get HEAD: {e}"))
697 })?;
698 let commit = head.peel_to_commit().map_err(|e| {
699 Git2DBError::lfs(
700 LfsErrorKind::IoError,
701 format!("Failed to get HEAD commit: {e}"),
702 )
703 })?;
704 let tree = commit.tree().map_err(|e| {
705 Git2DBError::lfs(
706 LfsErrorKind::IoError,
707 format!("Failed to get commit tree: {e}"),
708 )
709 })?;
710
711 let scan_prefix = if directory.starts_with(repo_workdir) {
713 directory
714 .strip_prefix(repo_workdir)
715 .map_err(|_| {
716 Git2DBError::lfs(LfsErrorKind::IoError, "Failed to get relative path")
717 })?
718 .to_string_lossy()
719 .into_owned()
720 } else {
721 String::new() };
723
724 let mut lfs_candidates = Vec::new();
726 let this = self.clone();
727
728 tree.walk(TreeWalkMode::PreOrder, |root, entry| {
729 let entry_path = if root.is_empty() {
731 entry.name().unwrap_or("").to_owned()
732 } else {
733 format!(
734 "{}/{}",
735 root.trim_end_matches('/'),
736 entry.name().unwrap_or("")
737 )
738 };
739
740 if !scan_prefix.is_empty() && !entry_path.starts_with(&scan_prefix) {
742 return TreeWalkResult::Skip;
743 }
744
745 if let Some(ObjectType::Blob) = entry.kind() {
747 if let Ok(blob) = repo.find_blob(entry.id()) {
749 if let Ok(content) = std::str::from_utf8(blob.content()) {
750 if this.is_lfs_pointer(content) {
751 let full_path = repo_workdir.join(&entry_path);
752 lfs_candidates.push(full_path);
753 }
754 }
755 }
756 }
757
758 TreeWalkResult::Ok
759 })
760 .map_err(|e| Git2DBError::lfs(LfsErrorKind::IoError, format!("Tree walk failed: {e}")))?;
761
762 let filtered = self.filter_by_git_status(&repo, lfs_candidates)?;
764
765 Ok(filtered)
766 }
767
768 fn filter_by_git_status(
769 &self,
770 repo: &git2::Repository,
771 candidates: Vec<PathBuf>,
772 ) -> Git2DBResult<Vec<PathBuf>> {
773 let repo_workdir = repo.workdir().ok_or_else(|| {
774 Git2DBError::lfs(
775 LfsErrorKind::NotInRepository,
776 "Repository has no working directory",
777 )
778 })?;
779
780 let mut filtered = Vec::new();
781 let total = candidates.len();
782
783 for candidate in candidates {
784 if let Ok(relative_path) = candidate.strip_prefix(repo_workdir) {
786 if candidate.exists() {
788 let is_ignored = repo
790 .status_file(relative_path)
791 .map(|status| status.contains(git2::Status::IGNORED))
792 .unwrap_or(false);
793
794 if !is_ignored {
795 debug!("Found LFS file: {}", candidate.display());
796 filtered.push(candidate);
797 }
798 }
799 }
800 }
801
802 debug!(
803 "Filtered {} LFS candidates to {} files",
804 total,
805 filtered.len()
806 );
807 Ok(filtered)
808 }
809
810 pub async fn process_worktree(&self, worktree_path: &Path) -> Git2DBResult<ProcessingStats> {
812 if !worktree_path.is_dir() {
813 return Err(Git2DBError::lfs(
814 LfsErrorKind::IoError,
815 format!("Path is not a directory: {}", worktree_path.display()),
816 ));
817 }
818
819 info!(
820 "Scanning worktree for LFS files: {}",
821 worktree_path.display()
822 );
823 let lfs_files = self.scan_lfs_files(worktree_path).await?;
824
825 if lfs_files.is_empty() {
826 debug!("No LFS files found in worktree");
827 return Ok(ProcessingStats::default());
828 }
829
830 info!("Found {} LFS files to process", lfs_files.len());
831 let mut stats = ProcessingStats {
832 total_files: lfs_files.len(),
833 ..Default::default()
834 };
835
836 for lfs_file in &lfs_files {
837 match self.smudge_file_in_place(lfs_file).await {
838 Ok(bytes) => {
839 stats.processed += 1;
840 stats.bytes_processed += bytes;
841 debug!("Processed: {}", lfs_file.display());
842 }
843 Err(e) => {
844 stats.failed += 1;
845 warn!("Failed to process {}: {}", lfs_file.display(), e);
846 }
847 }
848 }
849
850 info!(
851 "Processed {}/{} LFS files ({} bytes)",
852 stats.processed, stats.total_files, stats.bytes_processed
853 );
854 Ok(stats)
855 }
856
857 pub async fn process_worktree_with_progress(
859 &self,
860 worktree_path: &Path,
861 progress: Option<&Arc<dyn crate::callback_config::ProgressReporter>>,
862 ) -> Git2DBResult<ProcessingStats> {
863 if !worktree_path.is_dir() {
864 return Err(Git2DBError::lfs(
865 LfsErrorKind::IoError,
866 format!("Path is not a directory: {}", worktree_path.display()),
867 ));
868 }
869
870 info!(
871 "Scanning worktree for LFS files: {}",
872 worktree_path.display()
873 );
874 let lfs_files = self.scan_lfs_files(worktree_path).await?;
875
876 if lfs_files.is_empty() {
877 debug!("No LFS files found in worktree");
878 return Ok(ProcessingStats::default());
879 }
880
881 info!("Found {} LFS files to process", lfs_files.len());
882 let mut stats = ProcessingStats {
883 total_files: lfs_files.len(),
884 ..Default::default()
885 };
886
887 for (idx, lfs_file) in lfs_files.iter().enumerate() {
888 if let Some(reporter) = &progress {
889 reporter.report("lfs", idx + 1, lfs_files.len());
890 }
891 match self.smudge_file_in_place(lfs_file).await {
892 Ok(bytes) => {
893 stats.processed += 1;
894 stats.bytes_processed += bytes;
895 debug!("Processed: {}", lfs_file.display());
896 }
897 Err(e) => {
898 stats.failed += 1;
899 warn!("Failed to process {}: {}", lfs_file.display(), e);
900 }
901 }
902 }
903
904 info!(
905 "Processed {}/{} LFS files ({} bytes)",
906 stats.processed, stats.total_files, stats.bytes_processed
907 );
908 Ok(stats)
909 }
910
911 pub async fn batch_process(
913 &self,
914 worktree_path: &Path,
915 concurrency: usize,
916 ) -> Git2DBResult<ProcessingStats> {
917 let lfs_files = self.scan_lfs_files(worktree_path).await?;
918
919 if lfs_files.is_empty() {
920 debug!("No LFS files found in worktree");
921 return Ok(ProcessingStats::default());
922 }
923
924 info!(
925 "Found {} LFS files to process with concurrency {}",
926 lfs_files.len(),
927 concurrency
928 );
929
930 let semaphore = Arc::new(Semaphore::new(concurrency));
931 let mut tasks = Vec::new();
932
933 for lfs_file in lfs_files {
934 let semaphore = Arc::clone(&semaphore);
935 let storage = self.clone();
936
937 let task = tokio::spawn(async move {
938 let _permit = semaphore.acquire().await.map_err(|_| {
939 Git2DBError::internal("Semaphore closed during batch processing")
940 })?;
941 storage
942 .smudge_file_in_place(&lfs_file)
943 .await
944 .map(|bytes| (lfs_file, bytes))
945 });
946 tasks.push(task);
947 }
948
949 let mut stats = ProcessingStats {
950 total_files: tasks.len(),
951 ..Default::default()
952 };
953
954 for task in tasks {
955 match task.await {
956 Ok(Ok((path, bytes))) => {
957 stats.processed += 1;
958 stats.bytes_processed += bytes;
959 debug!("Processed: {}", path.display());
960 }
961 Ok(Err(e)) => {
962 stats.failed += 1;
963 warn!("Processing failed: {}", e);
964 }
965 Err(e) => {
966 stats.failed += 1;
967 warn!("Task failed: {}", e);
968 }
969 }
970 }
971
972 info!(
973 "Batch processed {}/{} LFS files ({} bytes)",
974 stats.processed, stats.total_files, stats.bytes_processed
975 );
976 Ok(stats)
977 }
978
979 async fn smudge_file_in_place(&self, file_path: &Path) -> Git2DBResult<u64> {
982 if self.is_already_smudged(file_path).await? {
984 debug!("Skipping already smudged file: {}", file_path.display());
985 return Ok(0);
986 }
987
988 let pointer_content = tokio::fs::read_to_string(file_path).await.map_err(|e| {
990 Git2DBError::lfs(
991 LfsErrorKind::IoError,
992 format!("Failed to read LFS pointer: {e}"),
993 )
994 })?;
995
996 if !self.is_lfs_pointer(&pointer_content) {
997 return Err(Git2DBError::lfs(
998 LfsErrorKind::InvalidPointer,
999 format!("File is not an LFS pointer: {}", file_path.display()),
1000 ));
1001 }
1002
1003 let lfs_pointer = self.parse_lfs_pointer(&pointer_content)?;
1005
1006 debug!(
1007 "Smudging LFS file: {} ({} bytes, SHA256: {}...)",
1008 file_path.display(),
1009 lfs_pointer.size(),
1010 &lfs_pointer.oid()[..8]
1011 );
1012
1013 let actual_content = self.smudge_lfs_pointer_validated(&lfs_pointer).await?;
1015 let bytes_written = actual_content.len() as u64;
1016
1017 let temp_file =
1019 tempfile::NamedTempFile::new_in(file_path.parent().unwrap_or_else(|| Path::new(".")))
1020 .map_err(|e| {
1021 Git2DBError::lfs(
1022 LfsErrorKind::IoError,
1023 format!("Failed to create temp file: {e}"),
1024 )
1025 })?;
1026
1027 tokio::fs::write(temp_file.path(), &actual_content)
1028 .await
1029 .map_err(|e| {
1030 Git2DBError::lfs(
1031 LfsErrorKind::IoError,
1032 format!("Failed to write temp file: {e}"),
1033 )
1034 })?;
1035
1036 let temp_path = temp_file.into_temp_path();
1038 temp_path.persist(file_path).map_err(|e| {
1039 Git2DBError::lfs(
1040 LfsErrorKind::IoError,
1041 format!("Failed to replace file atomically: {e}"),
1042 )
1043 })?;
1044
1045 Ok(bytes_written)
1046 }
1047
1048 async fn is_already_smudged(&self, file_path: &Path) -> Git2DBResult<bool> {
1049 use tokio::io::AsyncReadExt;
1050
1051 let mut file = match tokio::fs::File::open(file_path).await {
1053 Ok(f) => f,
1054 Err(_) => return Ok(false), };
1056
1057 let mut buffer = vec![0u8; 200];
1058 let n = file.read(&mut buffer).await.map_err(|e| {
1059 Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to read header: {e}"))
1060 })?;
1061
1062 if n == 0 {
1063 return Ok(false); }
1065
1066 if let Ok(content) = String::from_utf8(buffer[..n].to_vec()) {
1068 if self.is_lfs_pointer(&content) || self.is_xet_pointer(&content) {
1069 return Ok(false); }
1071 }
1072
1073 Ok(true)
1075 }
1076}
1077
1078#[cfg(test)]
1079mod tests {
1080 use super::*;
1081
1082 #[test]
1083 fn test_lfs_pointer_detection() {
1084 assert!(is_lfs_pointer(
1086 "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 100\n"
1087 ));
1088
1089 assert!(is_lfs_pointer(
1091 "version https://hawser.github.com/spec/v1\noid sha256:abc\nsize 100\n"
1092 ));
1093
1094 assert!(!is_lfs_pointer("This is regular content"));
1096 assert!(!is_lfs_pointer("{\"hash\": \"abc123\"}")); }
1098
1099 #[test]
1100 fn test_lfs_pointer_parsing() -> crate::Git2DBResult<()> {
1101 let content = r#"version https://git-lfs.github.com/spec/v1
1102oid sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890
1103size 1024
1104"#;
1105
1106 let pointer = LfsPointer::parse(content)?;
1107
1108 assert_eq!(pointer.version(), "https://git-lfs.github.com/spec/v1");
1109 assert_eq!(pointer.hash_method(), "sha256");
1110 assert_eq!(
1111 pointer.oid(),
1112 "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
1113 );
1114 assert_eq!(pointer.size(), 1024);
1115 Ok(())
1116 }
1117
1118 #[test]
1119 fn test_lfs_pointer_extensions() -> crate::Git2DBResult<()> {
1120 let content = r#"version https://git-lfs.github.com/spec/v1
1121oid sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890
1122size 1024
1123ext-0-custom value1
1124ext-1-another value2
1125"#;
1126
1127 let pointer = LfsPointer::parse(content)?;
1128
1129 assert_eq!(
1130 pointer.extensions().get("ext-0-custom"),
1131 Some(&"value1".to_owned())
1132 );
1133 assert_eq!(
1134 pointer.extensions().get("ext-1-another"),
1135 Some(&"value2".to_owned())
1136 );
1137 Ok(())
1138 }
1139
1140 #[test]
1141 fn test_lfs_pointer_invalid_oid() {
1142 let short_oid = r#"version https://git-lfs.github.com/spec/v1
1144oid sha256:abc123
1145size 1024
1146"#;
1147 assert!(LfsPointer::parse(short_oid).is_err());
1148
1149 let invalid_chars = r#"version https://git-lfs.github.com/spec/v1
1151oid sha256:gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg
1152size 1024
1153"#;
1154 assert!(LfsPointer::parse(invalid_chars).is_err());
1155 }
1156
1157 #[test]
1158 fn test_lfs_pointer_missing_fields() {
1159 assert!(LfsPointer::parse("oid sha256:abc123\nsize 100\n").is_err());
1161
1162 assert!(
1164 LfsPointer::parse("version https://git-lfs.github.com/spec/v1\nsize 100\n").is_err()
1165 );
1166
1167 let valid_oid = "a".repeat(64);
1169 assert!(LfsPointer::parse(&format!(
1170 "version https://git-lfs.github.com/spec/v1\noid sha256:{valid_oid}\n"
1171 ))
1172 .is_err());
1173 }
1174
1175 #[test]
1176 fn test_lfs_pointer_display() -> crate::Git2DBResult<()> {
1177 let content = r#"version https://git-lfs.github.com/spec/v1
1178oid sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890
1179size 1024
1180"#;
1181
1182 let pointer = LfsPointer::parse(content)?;
1183 let display = pointer.to_string();
1184
1185 assert!(display.contains("version https://git-lfs.github.com/spec/v1"));
1186 assert!(display.contains("oid sha256:"));
1187 assert!(display.contains("size 1024"));
1188 Ok(())
1189 }
1190
1191 #[test]
1196 fn test_lfs_pointer_vanilla_no_xet_merkle() -> crate::Git2DBResult<()> {
1197 let content = r#"version https://git-lfs.github.com/spec/v1
1198oid sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890
1199size 1024
1200"#;
1201 let pointer = LfsPointer::parse(content)?;
1202 assert_eq!(pointer.xet_merkle_hex(), None);
1203 assert_eq!(pointer.xet_shard(), None);
1204 Ok(())
1205 }
1206
1207 #[test]
1211 fn test_lfs_pointer_xet_extensions_round_trip() -> crate::Git2DBResult<()> {
1212 let merkle_hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
1215 let shard = "shard-7";
1216 let content = format!(
1217 "version https://git-lfs.github.com/spec/v1\n\
1218 oid sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890\n\
1219 size 1024\n\
1220 {XET_MERKLE_EXT_KEY} {merkle_hex}\n\
1221 {XET_SHARD_EXT_KEY} {shard}\n"
1222 );
1223
1224 let pointer = LfsPointer::parse(&content)?;
1225 assert_eq!(pointer.xet_merkle_hex(), Some(merkle_hex));
1226 assert_eq!(pointer.xet_shard(), Some(shard));
1227
1228 let redisplayed = pointer.to_string();
1230 let reparsed = LfsPointer::parse(&redisplayed)?;
1231 assert_eq!(reparsed.xet_merkle_hex(), Some(merkle_hex));
1232 assert_eq!(reparsed.xet_shard(), Some(shard));
1233 assert_eq!(reparsed.oid(), pointer.oid());
1235 assert_eq!(reparsed.size(), pointer.size());
1236 Ok(())
1237 }
1238
1239 #[cfg(feature = "xet-storage")]
1240 #[test]
1241 fn test_lfs_pointer_set_xet_extensions() -> crate::Git2DBResult<()> {
1242 let oid = "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890";
1243 let mut pointer = LfsPointer::new(
1244 "https://git-lfs.github.com/spec/v1".to_owned(),
1245 oid.to_owned(),
1246 2048,
1247 )?;
1248
1249 let merkle = merklehash::MerkleHash::from_hex(
1251 "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210",
1252 )
1253 .map_err(|e| {
1254 Git2DBError::lfs(
1255 LfsErrorKind::HashConversion,
1256 format!("invalid test merkle hex: {e}"),
1257 )
1258 })?;
1259 pointer.set_xet_extensions(&merkle, Some("shard-3"));
1260
1261 let expected_hex = merkle.to_string();
1263 assert_eq!(pointer.xet_merkle_hex(), Some(expected_hex.as_str()));
1264 assert_eq!(pointer.xet_merkle_hash()?, merkle);
1265 assert_eq!(pointer.xet_shard(), Some("shard-3"));
1266
1267 let reparsed = LfsPointer::parse(&pointer.to_string())?;
1269 assert_eq!(reparsed.xet_merkle_hash()?, merkle);
1270 Ok(())
1271 }
1272
1273 #[cfg(feature = "xet-storage")]
1274 #[test]
1275 fn test_lfs_pointer_xet_merkle_hash_missing_errors() -> crate::Git2DBResult<()> {
1276 let oid = "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890";
1279 let pointer = LfsPointer::new(
1280 "https://git-lfs.github.com/spec/v1".to_owned(),
1281 oid.to_owned(),
1282 1024,
1283 )?;
1284
1285 match pointer.xet_merkle_hash() {
1286 Err(e) => {
1287 assert_eq!(e.lfs_kind(), Some(LfsErrorKind::MissingXetMerkle));
1288 let msg = e.to_string();
1289 assert!(
1290 msg.contains("xet-merkle"),
1291 "error must name the missing extension: {msg}"
1292 );
1293 assert!(
1294 msg.contains("#386"),
1295 "error must reference issue #386: {msg}"
1296 );
1297 }
1298 Ok(h) => panic!("vanilla pointer unexpectedly resolved a MerkleHash: {h}"),
1299 }
1300 Ok(())
1301 }
1302}