Skip to main content

git2db/
lfs.rs

1//! Git LFS pointer support with XET storage backend
2//!
3//! This module provides complete LFS handling for git2db:
4//! - Pointer parsing (spec-compliant, supports legacy hawser version)
5//! - File loading with automatic pointer detection
6//! - Worktree scanning and batch processing
7//! - SHA256 validation (defense in depth against XET CAS bugs)
8//!
9//! # Architecture
10//!
11//! ```text
12//! Application → git2db::lfs::LfsStorage → git2db::xet::XetStorage
13//!                                       → git-xet-filter (XET only)
14//! ```
15//!
16//! # Example
17//!
18//! ```rust,ignore
19//! use git2db::lfs::{LfsStorage, is_lfs_pointer};
20//!
21//! // Initialize LFS storage
22//! let config = git2db::XetConfig::default();
23//! let storage = LfsStorage::new(&config).await?;
24//!
25//! // Load file with automatic pointer detection
26//! let data = storage.load_file(&path).await?;
27//!
28//! // Or parse and smudge manually
29//! if is_lfs_pointer(&content) {
30//!     let pointer = storage.parse_lfs_pointer(&content)?;
31//!     let data = storage.smudge_lfs_pointer(&pointer).await?;
32//! }
33//! ```
34
35use crate::errors::{Git2DBError, Git2DBResult, LfsErrorKind};
36use async_trait::async_trait;
37use sha2::{Digest, Sha256};
38use std::collections::HashMap;
39use std::path::{Path, PathBuf};
40use std::sync::Arc;
41use tokio::sync::Semaphore;
42use tracing::{debug, info, warn};
43
44#[cfg(feature = "xet-storage")]
45use crate::xet::{StorageBackend, XetConfig, XetStorage};
46
47/// Well-known LFS pointer extension key carrying the XET `MerkleHash` (BLAKE3 keyed
48/// with `DATA_KEY` over the content-defined chunk DAG) for the blob, as a 64-char
49/// lowercase hex string. Per the LFS spec, unknown extension lines are ignored by
50/// vanilla git-lfs clients, so this rides the existing `extensions` map without
51/// breaking back-compat. See issue #386.
52pub const XET_MERKLE_EXT_KEY: &str = "xet-merkle";
53
54/// Well-known LFS pointer extension key carrying an opaque shard/cas-location hint
55/// (e.g. a shard index or replication-group identifier) for the blob. Optional;
56/// consumed by smudge code that needs a locality hint beyond the Merkle hash.
57pub const XET_SHARD_EXT_KEY: &str = "xet-shard";
58
59/// Parsed Git LFS pointer with spec-compliant fields
60///
61/// Per the Git LFS specification, pointer files contain:
62/// - `version` - URL of the LFS spec
63/// - `oid sha256:` - SHA256 hash of the content
64/// - `size` - Size in bytes
65/// - Additional extension fields (preserved for spec compliance)
66///
67/// Fields are private to allow future API evolution without breaking changes.
68#[derive(Debug, Clone)]
69pub struct LfsPointer {
70    /// LFS version URL (git-lfs or legacy hawser)
71    version: String,
72    /// Hash method used (currently always "sha256", future-proof)
73    hash_method: String,
74    /// Object ID (SHA256 hash in hex)
75    oid: String,
76    /// File size in bytes
77    size: u64,
78    /// Extension fields (preserves unknown keys per LFS spec)
79    extensions: HashMap<String, String>,
80}
81
82impl LfsPointer {
83    /// Create a new LFS pointer with the given fields
84    ///
85    /// This constructor validates the OID format.
86    pub fn new(version: String, oid: String, size: u64) -> Git2DBResult<Self> {
87        // Default to sha256 hash method
88        Self::new_with_hash_method(version, "sha256".to_owned(), oid, size)
89    }
90
91    /// Create a new LFS pointer with explicit hash method
92    pub fn new_with_hash_method(
93        version: String,
94        hash_method: String,
95        oid: String,
96        size: u64,
97    ) -> Git2DBResult<Self> {
98        // Validate OID format for SHA256 (should be 64 character hex)
99        if hash_method == "sha256"
100            && (oid.len() != 64 || !oid.chars().all(|c| c.is_ascii_hexdigit()))
101        {
102            return Err(Git2DBError::lfs(
103                LfsErrorKind::InvalidPointer,
104                format!(
105                    "Invalid OID format: expected 64 hex characters, got {}",
106                    oid.len()
107                ),
108            ));
109        }
110
111        Ok(LfsPointer {
112            version,
113            hash_method,
114            oid,
115            size,
116            extensions: HashMap::new(),
117        })
118    }
119
120    /// Parse LFS pointer from text content
121    ///
122    /// Supports both standard git-lfs and legacy hawser version URLs.
123    ///
124    /// # Example
125    ///
126    /// ```rust,ignore
127    /// let content = r#"version https://git-lfs.github.com/spec/v1
128    /// oid sha256:abc123...
129    /// size 1024
130    /// "#;
131    /// let pointer = LfsPointer::parse(content)?;
132    /// ```
133    pub fn parse(content: &str) -> Git2DBResult<Self> {
134        let mut version = None;
135        let mut hash_method = None;
136        let mut oid = None;
137        let mut size = None;
138        let mut extensions = HashMap::new();
139
140        for line in content.lines() {
141            let line = line.trim();
142            if line.is_empty() {
143                continue;
144            }
145
146            if let Some(v) = line.strip_prefix("version ") {
147                version = Some(v.to_owned());
148            } else if let Some(o) = line.strip_prefix("oid sha256:") {
149                hash_method = Some("sha256".to_owned());
150                oid = Some(o.to_owned());
151            } else if let Some(s) = line.strip_prefix("size ") {
152                size = Some(s.parse::<u64>().map_err(|_| {
153                    Git2DBError::lfs(LfsErrorKind::ParseError, "Invalid size value")
154                })?);
155            } else if let Some((key, value)) = line.split_once(' ') {
156                // Preserve extension fields per LFS spec
157                extensions.insert(key.to_owned(), value.to_owned());
158            }
159        }
160
161        let version = version.ok_or_else(|| {
162            Git2DBError::lfs(LfsErrorKind::InvalidPointer, "Missing version field")
163        })?;
164        let hash_method = hash_method.ok_or_else(|| {
165            Git2DBError::lfs(
166                LfsErrorKind::InvalidPointer,
167                "Missing or unsupported oid hash method",
168            )
169        })?;
170        let oid =
171            oid.ok_or_else(|| Git2DBError::lfs(LfsErrorKind::InvalidPointer, "Missing oid field"))?;
172        let size = size
173            .ok_or_else(|| Git2DBError::lfs(LfsErrorKind::InvalidPointer, "Missing size field"))?;
174
175        // Validate OID format (should be 64 character hex for SHA256)
176        if oid.len() != 64 || !oid.chars().all(|c| c.is_ascii_hexdigit()) {
177            return Err(Git2DBError::lfs(
178                LfsErrorKind::InvalidPointer,
179                format!(
180                    "Invalid OID format: expected 64 hex characters, got {}",
181                    oid.len()
182                ),
183            ));
184        }
185
186        Ok(LfsPointer {
187            version,
188            hash_method,
189            oid,
190            size,
191            extensions,
192        })
193    }
194
195    /// Get the LFS version URL
196    pub fn version(&self) -> &str {
197        &self.version
198    }
199
200    /// Get the hash method (e.g., "sha256")
201    pub fn hash_method(&self) -> &str {
202        &self.hash_method
203    }
204
205    /// Get the object ID (SHA256 hash in hex)
206    pub fn oid(&self) -> &str {
207        &self.oid
208    }
209
210    /// Get the file size in bytes
211    pub fn size(&self) -> u64 {
212        self.size
213    }
214
215    /// Get extension fields (spec compliance)
216    pub fn extensions(&self) -> &HashMap<String, String> {
217        &self.extensions
218    }
219
220    /// Convert the LFS SHA-256 OID to a XET `MerkleHash` by hex re-interpretation.
221    ///
222    /// **Deprecated and unsound.** A XET `MerkleHash` is BLAKE3 keyed with the global
223    /// `DATA_KEY` over the content-defined chunk DAG, NOT the SHA-256 of the raw
224    /// bytes. The two hash families are unrelated; this conversion only succeeds by
225    /// coincidence when the upload side also (incorrectly) used the SHA-256 as a
226    /// MerkleHash, which real XET never does. Feeding this into `smudge_from_hash`
227    /// therefore cannot resolve real XET content.
228    ///
229    /// New code must populate the `xet-merkle` extension at promote time and read it
230    /// back via [`LfsPointer::xet_merkle_hash`]. The smudge path no longer calls this.
231    /// Kept temporarily for back-compat callers; emits a `tracing::warn!` so misuse is
232    /// observable in logs. See issue #386.
233    #[cfg(feature = "xet-storage")]
234    #[deprecated(
235        since = "0.1.0",
236        note = "unsound: reinterprets LFS SHA-256 as a XET MerkleHash; use the xet-merkle extension via xet_merkle_hash() instead — see #386"
237    )]
238    pub fn to_merkle_hash(&self) -> Git2DBResult<merklehash::MerkleHash> {
239        warn!(
240            oid = %self.oid,
241            "LfsPointer::to_merkle_hash() is unsound (SHA-256 != XET MerkleHash); use the xet-merkle extension instead — see #386"
242        );
243        merklehash::MerkleHash::from_hex(&self.oid).map_err(|e| {
244            Git2DBError::lfs(
245                LfsErrorKind::HashConversion,
246                format!("Failed to convert SHA256 to MerkleHash: {e}"),
247            )
248        })
249    }
250
251    /// Get the `xet-merkle` extension value (64-char hex `MerkleHash`), if present.
252    ///
253    /// Returns `None` for vanilla LFS pointers that carry no XET metadata. Presence of
254    /// this field indicates the pointer is self-describing for Hub-free XET smudge.
255    #[cfg(feature = "xet-storage")]
256    pub fn xet_merkle_hex(&self) -> Option<&str> {
257        self.extensions.get(XET_MERKLE_EXT_KEY).map(String::as_str)
258    }
259
260    /// Get the `xet-shard` extension value (opaque locality hint), if present.
261    pub fn xet_shard(&self) -> Option<&str> {
262        self.extensions.get(XET_SHARD_EXT_KEY).map(String::as_str)
263    }
264
265    /// Resolve the XET `MerkleHash` for this pointer from the `xet-merkle` extension.
266    ///
267    /// This is the correct, Hub-free derivation: the hash is read verbatim from the
268    /// pointer rather than mis-converted from the LFS SHA-256 OID. Returns a
269    /// [`LfsErrorKind::MissingXetMerkle`] error if the extension is absent, so callers
270    /// cannot silently fall through to the unsound legacy path.
271    #[cfg(feature = "xet-storage")]
272    pub fn xet_merkle_hash(&self) -> Git2DBResult<merklehash::MerkleHash> {
273        match self.xet_merkle_hex() {
274            Some(hex) => merklehash::MerkleHash::from_hex(hex).map_err(|e| {
275                Git2DBError::lfs(
276                    LfsErrorKind::HashConversion,
277                    format!("Invalid xet-merkle extension (expected 64 hex chars): {e}"),
278                )
279            }),
280            None => Err(Git2DBError::lfs(
281                LfsErrorKind::MissingXetMerkle,
282                "missing xet-merkle; to_merkle_hash() is unsound — see #386",
283            )),
284        }
285    }
286
287    /// Set the `xet-merkle` and (optionally) `xet-shard` extensions in place.
288    ///
289    /// Intended for promote-time code that has just computed the real XET `MerkleHash`
290    /// for the blob's content-defined chunk DAG and wants to record it on the pointer
291    /// so downstream smudge is Hub-free and self-describing. Overwrites any prior
292    /// value for these keys; other extensions are preserved.
293    #[cfg(feature = "xet-storage")]
294    pub fn set_xet_extensions(&mut self, merkle: &merklehash::MerkleHash, shard: Option<&str>) {
295        self.extensions
296            .insert(XET_MERKLE_EXT_KEY.to_owned(), merkle.to_string());
297        if let Some(s) = shard {
298            self.extensions
299                .insert(XET_SHARD_EXT_KEY.to_owned(), s.to_owned());
300        }
301    }
302}
303
304impl std::fmt::Display for LfsPointer {
305    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
306        // Mandatory fields first, then extension lines (sorted for deterministic output).
307        // Per the LFS spec, unknown extension keys are allowed and ignored by vanilla
308        // git-lfs clients, so the well-known `xet-merkle` / `xet-shard` keys round-trip
309        // transparently through unmodified tooling.
310        write!(
311            f,
312            "version {}\noid {}:{}\nsize {}\n",
313            self.version, self.hash_method, self.oid, self.size
314        )?;
315        let mut keys: Vec<&String> = self.extensions.keys().collect();
316        keys.sort_unstable();
317        for key in keys {
318            writeln!(f, "{} {}", key, self.extensions[key])?;
319        }
320        Ok(())
321    }
322}
323
324/// Check if content is a Git LFS pointer file
325///
326/// Supports both standard git-lfs and legacy hawser version URLs.
327///
328/// # Example
329///
330/// ```rust,ignore
331/// if git2db::lfs::is_lfs_pointer(&content) {
332///     // Handle as LFS pointer
333/// }
334/// ```
335pub fn is_lfs_pointer(content: &str) -> bool {
336    content.starts_with("version https://git-lfs.github.com/spec/v1")
337        || content.starts_with("version https://hawser.github.com/spec/v1")
338}
339
340/// Extension trait for XetStorage - adds LFS capabilities
341///
342/// This allows using `xet.smudge_lfs(&pointer)` syntax for LFS operations.
343#[cfg(feature = "xet-storage")]
344#[async_trait]
345pub trait LfsSmudge {
346    /// Smudge LFS pointer to memory
347    async fn smudge_lfs(&self, pointer: &LfsPointer) -> Git2DBResult<Vec<u8>>;
348
349    /// Smudge LFS pointer directly to file
350    async fn smudge_lfs_to_file(&self, pointer: &LfsPointer, path: &Path) -> Git2DBResult<()>;
351
352    /// Smudge LFS pointer with size and SHA256 validation
353    ///
354    /// This provides defense-in-depth validation against potential XET CAS bugs.
355    async fn smudge_lfs_validated(&self, pointer: &LfsPointer) -> Git2DBResult<Vec<u8>>;
356}
357
358#[cfg(feature = "xet-storage")]
359#[async_trait]
360impl LfsSmudge for XetStorage {
361    async fn smudge_lfs(&self, pointer: &LfsPointer) -> Git2DBResult<Vec<u8>> {
362        let hash = pointer.xet_merkle_hash()?;
363        self.smudge_from_hash(&hash).await.map_err(|e| {
364            Git2DBError::lfs(
365                LfsErrorKind::SmudgeFailed,
366                format!("XET smudge failed: {e}"),
367            )
368        })
369    }
370
371    async fn smudge_lfs_to_file(&self, pointer: &LfsPointer, path: &Path) -> Git2DBResult<()> {
372        let hash = pointer.xet_merkle_hash()?;
373        self.smudge_from_hash_to_file(&hash, path)
374            .await
375            .map_err(|e| {
376                Git2DBError::lfs(
377                    LfsErrorKind::SmudgeFailed,
378                    format!("XET smudge to file failed: {e}"),
379                )
380            })
381    }
382
383    async fn smudge_lfs_validated(&self, pointer: &LfsPointer) -> Git2DBResult<Vec<u8>> {
384        let content = self.smudge_lfs(pointer).await?;
385
386        // Validate size
387        if content.len() as u64 != pointer.size() {
388            return Err(Git2DBError::lfs(
389                LfsErrorKind::SizeMismatch,
390                format!(
391                    "Size mismatch: expected {} bytes, got {} bytes",
392                    pointer.size(),
393                    content.len()
394                ),
395            ));
396        }
397
398        // Validate SHA256
399        let mut hasher = Sha256::new();
400        hasher.update(&content);
401        let computed_hash = format!("{:x}", hasher.finalize());
402
403        if computed_hash != pointer.oid() {
404            return Err(Git2DBError::lfs(
405                LfsErrorKind::HashMismatch,
406                format!(
407                    "SHA256 mismatch: expected {}, got {}",
408                    pointer.oid(),
409                    computed_hash
410                ),
411            ));
412        }
413
414        debug!(
415            "Validated LFS content: {} bytes, SHA256: {}...",
416            pointer.size(),
417            &pointer.oid()[..8]
418        );
419
420        Ok(content)
421    }
422}
423
424/// Processing statistics for batch operations
425#[derive(Debug, Default, Clone)]
426pub struct ProcessingStats {
427    /// Total files found
428    pub total_files: usize,
429    /// Files successfully processed
430    pub processed: usize,
431    /// Files skipped (already smudged)
432    pub skipped: usize,
433    /// Files that failed to process
434    pub failed: usize,
435    /// Total bytes processed
436    pub bytes_processed: u64,
437}
438
439/// Complete LFS storage operations
440///
441/// This struct provides all LFS functionality in one place, delegating
442/// to XetStorage for the actual smudge/clean operations.
443#[cfg(feature = "xet-storage")]
444#[derive(Clone)]
445pub struct LfsStorage {
446    xet: Arc<XetStorage>,
447}
448
449#[cfg(feature = "xet-storage")]
450impl LfsStorage {
451    /// Create new LFS storage with XET backend
452    pub async fn new(config: &XetConfig) -> Git2DBResult<Self> {
453        let xet = XetStorage::new(config).await.map_err(|e| {
454            Git2DBError::lfs(
455                LfsErrorKind::IoError,
456                format!("Failed to create XET storage: {e}"),
457            )
458        })?;
459        Ok(Self { xet: Arc::new(xet) })
460    }
461
462    // === Pointer Detection ===
463
464    /// Check if content is an LFS pointer
465    pub fn is_lfs_pointer(&self, content: &str) -> bool {
466        is_lfs_pointer(content)
467    }
468
469    /// Check if content is an XET pointer
470    pub fn is_xet_pointer(&self, content: &str) -> bool {
471        self.xet.is_pointer(content)
472    }
473
474    /// Check if content is any type of pointer (LFS or XET)
475    pub fn is_pointer(&self, content: &str) -> bool {
476        self.is_lfs_pointer(content) || self.is_xet_pointer(content)
477    }
478
479    /// Parse LFS pointer from content
480    pub fn parse_lfs_pointer(&self, content: &str) -> Git2DBResult<LfsPointer> {
481        LfsPointer::parse(content)
482    }
483
484    // === Core Operations ===
485
486    /// Smudge LFS pointer to memory
487    pub async fn smudge_lfs_pointer(&self, pointer: &LfsPointer) -> Git2DBResult<Vec<u8>> {
488        self.xet.smudge_lfs(pointer).await
489    }
490
491    /// Stream a captured LFS pointer through a writer that rejects any byte
492    /// beyond its declared size before it reaches the projection filesystem.
493    pub async fn smudge_lfs_pointer_to_file_bounded(
494        &self,
495        pointer: &LfsPointer,
496        path: &Path,
497    ) -> Git2DBResult<u64> {
498        let hash = pointer.xet_merkle_hash()?;
499        self.xet
500            .smudge_hash_to_file_bounded(&hash, path, pointer.size())
501            .await
502            .map_err(|e| Git2DBError::lfs(LfsErrorKind::SmudgeFailed, e.to_string()))
503    }
504
505    /// Stream an XET JSON pointer through a writer that rejects bytes beyond
506    /// its authenticated pointer size before they reach the projection disk.
507    pub async fn smudge_xet_pointer_to_file_bounded(
508        &self,
509        pointer: &str,
510        path: &Path,
511        max_output_bytes: u64,
512    ) -> Git2DBResult<u64> {
513        self.xet
514            .smudge_file_bounded(pointer, path, max_output_bytes)
515            .await
516            .map_err(|e| Git2DBError::lfs(LfsErrorKind::SmudgeFailed, e.to_string()))
517    }
518
519    /// Stream a captured LFS pointer's payload into a private file. Callers
520    /// must independently verify SHA-256 and size before publishing it.
521    pub async fn smudge_lfs_pointer_to_file(
522        &self,
523        pointer: &LfsPointer,
524        path: &Path,
525    ) -> Git2DBResult<()> {
526        self.xet.smudge_lfs_to_file(pointer, path).await
527    }
528
529    /// Smudge LFS pointer with validation
530    pub async fn smudge_lfs_pointer_validated(
531        &self,
532        pointer: &LfsPointer,
533    ) -> Git2DBResult<Vec<u8>> {
534        self.xet.smudge_lfs_validated(pointer).await
535    }
536
537    /// Upload file and return XET pointer
538    pub async fn clean_file(&self, path: &Path) -> Git2DBResult<String> {
539        self.xet.clean_file(path).await.map_err(|e| {
540            Git2DBError::lfs(LfsErrorKind::IoError, format!("XET clean_file failed: {e}"))
541        })
542    }
543
544    /// Upload data from memory and return XET pointer
545    pub async fn clean_bytes(&self, data: &[u8]) -> Git2DBResult<String> {
546        self.xet.clean_bytes(data).await.map_err(|e| {
547            Git2DBError::lfs(
548                LfsErrorKind::IoError,
549                format!("XET clean_bytes failed: {e}"),
550            )
551        })
552    }
553
554    /// Download XET pointer to file
555    pub async fn smudge_file(&self, pointer: &str, output_path: &Path) -> Git2DBResult<()> {
556        self.xet
557            .smudge_file(pointer, output_path)
558            .await
559            .map_err(|e| {
560                Git2DBError::lfs(
561                    LfsErrorKind::SmudgeFailed,
562                    format!("XET smudge_file failed: {e}"),
563                )
564            })
565    }
566
567    /// Download XET pointer to memory
568    pub async fn smudge_bytes(&self, pointer: &str) -> Git2DBResult<Vec<u8>> {
569        self.xet.smudge_bytes(pointer).await.map_err(|e| {
570            Git2DBError::lfs(
571                LfsErrorKind::SmudgeFailed,
572                format!("XET smudge_bytes failed: {e}"),
573            )
574        })
575    }
576
577    // === Universal File Loading ===
578
579    /// Load file with automatic LFS/XET pointer detection and smudging
580    ///
581    /// This implements LFS spec-compliant pointer detection:
582    /// 1. Checks file size via metadata (pointers MUST be < 1024 bytes)
583    /// 2. Reads first 100 bytes for pointer detection (per LFS spec)
584    /// 3. If pointer detected, reads rest and smudges
585    /// 4. If not pointer, reads full file efficiently
586    ///
587    /// Use this for files outside git operations (git-xet-filter handles files in git repos).
588    pub async fn load_file(&self, file_path: &Path) -> Git2DBResult<Vec<u8>> {
589        use tokio::io::{AsyncReadExt, AsyncSeekExt};
590
591        // 1. Check metadata FIRST (instant, no file content I/O)
592        // LFS spec: "Pointer files must be less than 1024 bytes in size"
593        let metadata = tokio::fs::metadata(file_path).await.map_err(|e| {
594            Git2DBError::lfs(
595                LfsErrorKind::IoError,
596                format!("Failed to get file metadata: {e}"),
597            )
598        })?;
599        let file_size = metadata.len();
600
601        // 2. Large files (>= 1024 bytes) cannot be pointers - read directly
602        if file_size >= 1024 {
603            return tokio::fs::read(file_path).await.map_err(|e| {
604                Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to read file: {e}"))
605            });
606        }
607
608        // 3. Small file - might be a pointer
609        // LFS spec: "Read 100 bytes. If the content is ASCII and matches the pointer file format"
610        let mut file = tokio::fs::File::open(file_path).await.map_err(|e| {
611            Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to open file: {e}"))
612        })?;
613
614        let mut header = vec![0u8; 100];
615        let n = file.read(&mut header).await.map_err(|e| {
616            Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to read header: {e}"))
617        })?;
618        header.truncate(n);
619
620        // 4. Check for pointer markers (LFS and XET pointers are UTF-8 text)
621        if let Ok(text) = String::from_utf8(header.clone()) {
622            if text.starts_with("version https://git-lfs")
623                || text.starts_with("version https://hawser")
624                || text.starts_with("# xet version")
625            {
626                // It's a pointer - read the rest (file is < 1024 bytes total)
627                let mut rest = Vec::new();
628                file.read_to_end(&mut rest).await.map_err(|e| {
629                    Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to read rest: {e}"))
630                })?;
631
632                let full_content = [&header[..n], &rest[..]].concat();
633                let full_text = String::from_utf8(full_content).map_err(|_| {
634                    Git2DBError::lfs(
635                        LfsErrorKind::InvalidPointer,
636                        "Pointer file is not valid UTF-8",
637                    )
638                })?;
639
640                // Parse and smudge the appropriate pointer type
641                if self.is_lfs_pointer(&full_text) {
642                    let lfs_pointer = self.parse_lfs_pointer(&full_text)?;
643                    return self.smudge_lfs_pointer_validated(&lfs_pointer).await;
644                } else if self.is_xet_pointer(&full_text) {
645                    return self.smudge_bytes(&full_text).await;
646                }
647            }
648        }
649
650        // 5. Not a pointer - seek back to start and read full file
651        file.seek(std::io::SeekFrom::Start(0))
652            .await
653            .map_err(|e| Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to seek: {e}")))?;
654        let mut contents = Vec::new();
655        file.read_to_end(&mut contents).await.map_err(|e| {
656            Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to read file: {e}"))
657        })?;
658
659        Ok(contents)
660    }
661
662    // === Worktree Operations ===
663
664    /// Scan directory for LFS files using git2 tree traversal
665    pub async fn scan_lfs_files(&self, directory: &Path) -> Git2DBResult<Vec<PathBuf>> {
666        let directory = directory.to_path_buf();
667        let this = self.clone();
668
669        tokio::task::spawn_blocking(move || this.scan_lfs_files_sync(&directory))
670            .await
671            .map_err(|e| {
672                Git2DBError::lfs(LfsErrorKind::IoError, format!("Task join failed: {e}"))
673            })?
674    }
675
676    fn scan_lfs_files_sync(&self, directory: &Path) -> Git2DBResult<Vec<PathBuf>> {
677        use git2::{ObjectType, TreeWalkMode, TreeWalkResult};
678
679        // Find the git repository for this directory
680        let repo = git2::Repository::discover(directory).map_err(|e| {
681            Git2DBError::lfs(
682                LfsErrorKind::NotInRepository,
683                format!("Directory is not part of a git repository: {e}"),
684            )
685        })?;
686
687        let repo_workdir = repo.workdir().ok_or_else(|| {
688            Git2DBError::lfs(
689                LfsErrorKind::NotInRepository,
690                "Repository has no working directory",
691            )
692        })?;
693
694        // Get the current HEAD commit and its tree
695        let head = repo.head().map_err(|e| {
696            Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to get HEAD: {e}"))
697        })?;
698        let commit = head.peel_to_commit().map_err(|e| {
699            Git2DBError::lfs(
700                LfsErrorKind::IoError,
701                format!("Failed to get HEAD commit: {e}"),
702            )
703        })?;
704        let tree = commit.tree().map_err(|e| {
705            Git2DBError::lfs(
706                LfsErrorKind::IoError,
707                format!("Failed to get commit tree: {e}"),
708            )
709        })?;
710
711        // Convert directory to relative path if needed
712        let scan_prefix = if directory.starts_with(repo_workdir) {
713            directory
714                .strip_prefix(repo_workdir)
715                .map_err(|_| {
716                    Git2DBError::lfs(LfsErrorKind::IoError, "Failed to get relative path")
717                })?
718                .to_string_lossy()
719                .into_owned()
720        } else {
721            String::new() // Scan entire repo
722        };
723
724        // Collect LFS candidates using git2's tree walk
725        let mut lfs_candidates = Vec::new();
726        let this = self.clone();
727
728        tree.walk(TreeWalkMode::PreOrder, |root, entry| {
729            // Build full path
730            let entry_path = if root.is_empty() {
731                entry.name().unwrap_or("").to_owned()
732            } else {
733                format!(
734                    "{}/{}",
735                    root.trim_end_matches('/'),
736                    entry.name().unwrap_or("")
737                )
738            };
739
740            // Check if this path is within our scan prefix
741            if !scan_prefix.is_empty() && !entry_path.starts_with(&scan_prefix) {
742                return TreeWalkResult::Skip;
743            }
744
745            // Only process regular files (blobs)
746            if let Some(ObjectType::Blob) = entry.kind() {
747                // Try to read the blob content to check for LFS pointer
748                if let Ok(blob) = repo.find_blob(entry.id()) {
749                    if let Ok(content) = std::str::from_utf8(blob.content()) {
750                        if this.is_lfs_pointer(content) {
751                            let full_path = repo_workdir.join(&entry_path);
752                            lfs_candidates.push(full_path);
753                        }
754                    }
755                }
756            }
757
758            TreeWalkResult::Ok
759        })
760        .map_err(|e| Git2DBError::lfs(LfsErrorKind::IoError, format!("Tree walk failed: {e}")))?;
761
762        // Filter candidates using git status to respect .gitignore
763        let filtered = self.filter_by_git_status(&repo, lfs_candidates)?;
764
765        Ok(filtered)
766    }
767
768    fn filter_by_git_status(
769        &self,
770        repo: &git2::Repository,
771        candidates: Vec<PathBuf>,
772    ) -> Git2DBResult<Vec<PathBuf>> {
773        let repo_workdir = repo.workdir().ok_or_else(|| {
774            Git2DBError::lfs(
775                LfsErrorKind::NotInRepository,
776                "Repository has no working directory",
777            )
778        })?;
779
780        let mut filtered = Vec::new();
781        let total = candidates.len();
782
783        for candidate in candidates {
784            // Convert to relative path for git operations
785            if let Ok(relative_path) = candidate.strip_prefix(repo_workdir) {
786                // Check if file exists in worktree
787                if candidate.exists() {
788                    // Check if it's ignored using git's ignore rules
789                    let is_ignored = repo
790                        .status_file(relative_path)
791                        .map(|status| status.contains(git2::Status::IGNORED))
792                        .unwrap_or(false);
793
794                    if !is_ignored {
795                        debug!("Found LFS file: {}", candidate.display());
796                        filtered.push(candidate);
797                    }
798                }
799            }
800        }
801
802        debug!(
803            "Filtered {} LFS candidates to {} files",
804            total,
805            filtered.len()
806        );
807        Ok(filtered)
808    }
809
810    /// Process all LFS files in a worktree by smudging them in place
811    pub async fn process_worktree(&self, worktree_path: &Path) -> Git2DBResult<ProcessingStats> {
812        if !worktree_path.is_dir() {
813            return Err(Git2DBError::lfs(
814                LfsErrorKind::IoError,
815                format!("Path is not a directory: {}", worktree_path.display()),
816            ));
817        }
818
819        info!(
820            "Scanning worktree for LFS files: {}",
821            worktree_path.display()
822        );
823        let lfs_files = self.scan_lfs_files(worktree_path).await?;
824
825        if lfs_files.is_empty() {
826            debug!("No LFS files found in worktree");
827            return Ok(ProcessingStats::default());
828        }
829
830        info!("Found {} LFS files to process", lfs_files.len());
831        let mut stats = ProcessingStats {
832            total_files: lfs_files.len(),
833            ..Default::default()
834        };
835
836        for lfs_file in &lfs_files {
837            match self.smudge_file_in_place(lfs_file).await {
838                Ok(bytes) => {
839                    stats.processed += 1;
840                    stats.bytes_processed += bytes;
841                    debug!("Processed: {}", lfs_file.display());
842                }
843                Err(e) => {
844                    stats.failed += 1;
845                    warn!("Failed to process {}: {}", lfs_file.display(), e);
846                }
847            }
848        }
849
850        info!(
851            "Processed {}/{} LFS files ({} bytes)",
852            stats.processed, stats.total_files, stats.bytes_processed
853        );
854        Ok(stats)
855    }
856
857    /// Process all LFS files in a worktree with optional progress reporting
858    pub async fn process_worktree_with_progress(
859        &self,
860        worktree_path: &Path,
861        progress: Option<&Arc<dyn crate::callback_config::ProgressReporter>>,
862    ) -> Git2DBResult<ProcessingStats> {
863        if !worktree_path.is_dir() {
864            return Err(Git2DBError::lfs(
865                LfsErrorKind::IoError,
866                format!("Path is not a directory: {}", worktree_path.display()),
867            ));
868        }
869
870        info!(
871            "Scanning worktree for LFS files: {}",
872            worktree_path.display()
873        );
874        let lfs_files = self.scan_lfs_files(worktree_path).await?;
875
876        if lfs_files.is_empty() {
877            debug!("No LFS files found in worktree");
878            return Ok(ProcessingStats::default());
879        }
880
881        info!("Found {} LFS files to process", lfs_files.len());
882        let mut stats = ProcessingStats {
883            total_files: lfs_files.len(),
884            ..Default::default()
885        };
886
887        for (idx, lfs_file) in lfs_files.iter().enumerate() {
888            if let Some(reporter) = &progress {
889                reporter.report("lfs", idx + 1, lfs_files.len());
890            }
891            match self.smudge_file_in_place(lfs_file).await {
892                Ok(bytes) => {
893                    stats.processed += 1;
894                    stats.bytes_processed += bytes;
895                    debug!("Processed: {}", lfs_file.display());
896                }
897                Err(e) => {
898                    stats.failed += 1;
899                    warn!("Failed to process {}: {}", lfs_file.display(), e);
900                }
901            }
902        }
903
904        info!(
905            "Processed {}/{} LFS files ({} bytes)",
906            stats.processed, stats.total_files, stats.bytes_processed
907        );
908        Ok(stats)
909    }
910
911    /// Batch process LFS files with concurrency control
912    pub async fn batch_process(
913        &self,
914        worktree_path: &Path,
915        concurrency: usize,
916    ) -> Git2DBResult<ProcessingStats> {
917        let lfs_files = self.scan_lfs_files(worktree_path).await?;
918
919        if lfs_files.is_empty() {
920            debug!("No LFS files found in worktree");
921            return Ok(ProcessingStats::default());
922        }
923
924        info!(
925            "Found {} LFS files to process with concurrency {}",
926            lfs_files.len(),
927            concurrency
928        );
929
930        let semaphore = Arc::new(Semaphore::new(concurrency));
931        let mut tasks = Vec::new();
932
933        for lfs_file in lfs_files {
934            let semaphore = Arc::clone(&semaphore);
935            let storage = self.clone();
936
937            let task = tokio::spawn(async move {
938                let _permit = semaphore.acquire().await.map_err(|_| {
939                    Git2DBError::internal("Semaphore closed during batch processing")
940                })?;
941                storage
942                    .smudge_file_in_place(&lfs_file)
943                    .await
944                    .map(|bytes| (lfs_file, bytes))
945            });
946            tasks.push(task);
947        }
948
949        let mut stats = ProcessingStats {
950            total_files: tasks.len(),
951            ..Default::default()
952        };
953
954        for task in tasks {
955            match task.await {
956                Ok(Ok((path, bytes))) => {
957                    stats.processed += 1;
958                    stats.bytes_processed += bytes;
959                    debug!("Processed: {}", path.display());
960                }
961                Ok(Err(e)) => {
962                    stats.failed += 1;
963                    warn!("Processing failed: {}", e);
964                }
965                Err(e) => {
966                    stats.failed += 1;
967                    warn!("Task failed: {}", e);
968                }
969            }
970        }
971
972        info!(
973            "Batch processed {}/{} LFS files ({} bytes)",
974            stats.processed, stats.total_files, stats.bytes_processed
975        );
976        Ok(stats)
977    }
978
979    // === Internal Helpers ===
980
981    async fn smudge_file_in_place(&self, file_path: &Path) -> Git2DBResult<u64> {
982        // Check if already smudged
983        if self.is_already_smudged(file_path).await? {
984            debug!("Skipping already smudged file: {}", file_path.display());
985            return Ok(0);
986        }
987
988        // Read the LFS pointer content
989        let pointer_content = tokio::fs::read_to_string(file_path).await.map_err(|e| {
990            Git2DBError::lfs(
991                LfsErrorKind::IoError,
992                format!("Failed to read LFS pointer: {e}"),
993            )
994        })?;
995
996        if !self.is_lfs_pointer(&pointer_content) {
997            return Err(Git2DBError::lfs(
998                LfsErrorKind::InvalidPointer,
999                format!("File is not an LFS pointer: {}", file_path.display()),
1000            ));
1001        }
1002
1003        // Parse the LFS pointer
1004        let lfs_pointer = self.parse_lfs_pointer(&pointer_content)?;
1005
1006        debug!(
1007            "Smudging LFS file: {} ({} bytes, SHA256: {}...)",
1008            file_path.display(),
1009            lfs_pointer.size(),
1010            &lfs_pointer.oid()[..8]
1011        );
1012
1013        // Retrieve and validate content via XET
1014        let actual_content = self.smudge_lfs_pointer_validated(&lfs_pointer).await?;
1015        let bytes_written = actual_content.len() as u64;
1016
1017        // Write to temporary file first
1018        let temp_file =
1019            tempfile::NamedTempFile::new_in(file_path.parent().unwrap_or_else(|| Path::new(".")))
1020                .map_err(|e| {
1021                Git2DBError::lfs(
1022                    LfsErrorKind::IoError,
1023                    format!("Failed to create temp file: {e}"),
1024                )
1025            })?;
1026
1027        tokio::fs::write(temp_file.path(), &actual_content)
1028            .await
1029            .map_err(|e| {
1030                Git2DBError::lfs(
1031                    LfsErrorKind::IoError,
1032                    format!("Failed to write temp file: {e}"),
1033                )
1034            })?;
1035
1036        // Atomically replace the original file
1037        let temp_path = temp_file.into_temp_path();
1038        temp_path.persist(file_path).map_err(|e| {
1039            Git2DBError::lfs(
1040                LfsErrorKind::IoError,
1041                format!("Failed to replace file atomically: {e}"),
1042            )
1043        })?;
1044
1045        Ok(bytes_written)
1046    }
1047
1048    async fn is_already_smudged(&self, file_path: &Path) -> Git2DBResult<bool> {
1049        use tokio::io::AsyncReadExt;
1050
1051        // Read first 200 bytes to check for pointer markers
1052        let mut file = match tokio::fs::File::open(file_path).await {
1053            Ok(f) => f,
1054            Err(_) => return Ok(false), // File doesn't exist = not smudged
1055        };
1056
1057        let mut buffer = vec![0u8; 200];
1058        let n = file.read(&mut buffer).await.map_err(|e| {
1059            Git2DBError::lfs(LfsErrorKind::IoError, format!("Failed to read header: {e}"))
1060        })?;
1061
1062        if n == 0 {
1063            return Ok(false); // Empty file = not smudged
1064        }
1065
1066        // Try to parse as text for pointer detection
1067        if let Ok(content) = String::from_utf8(buffer[..n].to_vec()) {
1068            if self.is_lfs_pointer(&content) || self.is_xet_pointer(&content) {
1069                return Ok(false); // Still a pointer, needs smudging
1070            }
1071        }
1072
1073        // Not a pointer = already smudged
1074        Ok(true)
1075    }
1076}
1077
1078#[cfg(test)]
1079mod tests {
1080    use super::*;
1081
1082    #[test]
1083    fn test_lfs_pointer_detection() {
1084        // Standard git-lfs pointer
1085        assert!(is_lfs_pointer(
1086            "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 100\n"
1087        ));
1088
1089        // Legacy hawser pointer
1090        assert!(is_lfs_pointer(
1091            "version https://hawser.github.com/spec/v1\noid sha256:abc\nsize 100\n"
1092        ));
1093
1094        // Not an LFS pointer
1095        assert!(!is_lfs_pointer("This is regular content"));
1096        assert!(!is_lfs_pointer("{\"hash\": \"abc123\"}")); // XET pointer
1097    }
1098
1099    #[test]
1100    fn test_lfs_pointer_parsing() -> crate::Git2DBResult<()> {
1101        let content = r#"version https://git-lfs.github.com/spec/v1
1102oid sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890
1103size 1024
1104"#;
1105
1106        let pointer = LfsPointer::parse(content)?;
1107
1108        assert_eq!(pointer.version(), "https://git-lfs.github.com/spec/v1");
1109        assert_eq!(pointer.hash_method(), "sha256");
1110        assert_eq!(
1111            pointer.oid(),
1112            "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
1113        );
1114        assert_eq!(pointer.size(), 1024);
1115        Ok(())
1116    }
1117
1118    #[test]
1119    fn test_lfs_pointer_extensions() -> crate::Git2DBResult<()> {
1120        let content = r#"version https://git-lfs.github.com/spec/v1
1121oid sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890
1122size 1024
1123ext-0-custom value1
1124ext-1-another value2
1125"#;
1126
1127        let pointer = LfsPointer::parse(content)?;
1128
1129        assert_eq!(
1130            pointer.extensions().get("ext-0-custom"),
1131            Some(&"value1".to_owned())
1132        );
1133        assert_eq!(
1134            pointer.extensions().get("ext-1-another"),
1135            Some(&"value2".to_owned())
1136        );
1137        Ok(())
1138    }
1139
1140    #[test]
1141    fn test_lfs_pointer_invalid_oid() {
1142        // Too short
1143        let short_oid = r#"version https://git-lfs.github.com/spec/v1
1144oid sha256:abc123
1145size 1024
1146"#;
1147        assert!(LfsPointer::parse(short_oid).is_err());
1148
1149        // Invalid characters
1150        let invalid_chars = r#"version https://git-lfs.github.com/spec/v1
1151oid sha256:gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg
1152size 1024
1153"#;
1154        assert!(LfsPointer::parse(invalid_chars).is_err());
1155    }
1156
1157    #[test]
1158    fn test_lfs_pointer_missing_fields() {
1159        // Missing version
1160        assert!(LfsPointer::parse("oid sha256:abc123\nsize 100\n").is_err());
1161
1162        // Missing oid
1163        assert!(
1164            LfsPointer::parse("version https://git-lfs.github.com/spec/v1\nsize 100\n").is_err()
1165        );
1166
1167        // Missing size
1168        let valid_oid = "a".repeat(64);
1169        assert!(LfsPointer::parse(&format!(
1170            "version https://git-lfs.github.com/spec/v1\noid sha256:{valid_oid}\n"
1171        ))
1172        .is_err());
1173    }
1174
1175    #[test]
1176    fn test_lfs_pointer_display() -> crate::Git2DBResult<()> {
1177        let content = r#"version https://git-lfs.github.com/spec/v1
1178oid sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890
1179size 1024
1180"#;
1181
1182        let pointer = LfsPointer::parse(content)?;
1183        let display = pointer.to_string();
1184
1185        assert!(display.contains("version https://git-lfs.github.com/spec/v1"));
1186        assert!(display.contains("oid sha256:"));
1187        assert!(display.contains("size 1024"));
1188        Ok(())
1189    }
1190
1191    // ---- #386: xet-merkle / xet-shard extension round-trip ----
1192
1193    /// A vanilla LFS pointer (no xet-* extensions) must still parse and carry no
1194    /// xet-merkle.
1195    #[test]
1196    fn test_lfs_pointer_vanilla_no_xet_merkle() -> crate::Git2DBResult<()> {
1197        let content = r#"version https://git-lfs.github.com/spec/v1
1198oid sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890
1199size 1024
1200"#;
1201        let pointer = LfsPointer::parse(content)?;
1202        assert_eq!(pointer.xet_merkle_hex(), None);
1203        assert_eq!(pointer.xet_shard(), None);
1204        Ok(())
1205    }
1206
1207    /// Round-trip: an extended pointer parses, the xet-merkle/xet-shard values are
1208    /// recoverable, and re-displaying yields a pointer that parses back to the same
1209    /// values.
1210    #[test]
1211    fn test_lfs_pointer_xet_extensions_round_trip() -> crate::Git2DBResult<()> {
1212        // An unrelated XET MerkleHash (64 hex chars) — distinct from the LFS SHA-256 OID,
1213        // which is exactly the case the old to_merkle_hash() got wrong.
1214        let merkle_hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
1215        let shard = "shard-7";
1216        let content = format!(
1217            "version https://git-lfs.github.com/spec/v1\n\
1218             oid sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890\n\
1219             size 1024\n\
1220             {XET_MERKLE_EXT_KEY} {merkle_hex}\n\
1221             {XET_SHARD_EXT_KEY} {shard}\n"
1222        );
1223
1224        let pointer = LfsPointer::parse(&content)?;
1225        assert_eq!(pointer.xet_merkle_hex(), Some(merkle_hex));
1226        assert_eq!(pointer.xet_shard(), Some(shard));
1227
1228        // Re-display and re-parse; values must survive a round-trip (Display sorts keys).
1229        let redisplayed = pointer.to_string();
1230        let reparsed = LfsPointer::parse(&redisplayed)?;
1231        assert_eq!(reparsed.xet_merkle_hex(), Some(merkle_hex));
1232        assert_eq!(reparsed.xet_shard(), Some(shard));
1233        // Mandatory fields are unchanged.
1234        assert_eq!(reparsed.oid(), pointer.oid());
1235        assert_eq!(reparsed.size(), pointer.size());
1236        Ok(())
1237    }
1238
1239    #[cfg(feature = "xet-storage")]
1240    #[test]
1241    fn test_lfs_pointer_set_xet_extensions() -> crate::Git2DBResult<()> {
1242        let oid = "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890";
1243        let mut pointer = LfsPointer::new(
1244            "https://git-lfs.github.com/spec/v1".to_owned(),
1245            oid.to_owned(),
1246            2048,
1247        )?;
1248
1249        // A real XET MerkleHash computed by the uploader over the chunk DAG.
1250        let merkle = merklehash::MerkleHash::from_hex(
1251            "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210",
1252        )
1253        .map_err(|e| {
1254            Git2DBError::lfs(
1255                LfsErrorKind::HashConversion,
1256                format!("invalid test merkle hex: {e}"),
1257            )
1258        })?;
1259        pointer.set_xet_extensions(&merkle, Some("shard-3"));
1260
1261        // Round-trips through hex verbatim.
1262        let expected_hex = merkle.to_string();
1263        assert_eq!(pointer.xet_merkle_hex(), Some(expected_hex.as_str()));
1264        assert_eq!(pointer.xet_merkle_hash()?, merkle);
1265        assert_eq!(pointer.xet_shard(), Some("shard-3"));
1266
1267        // Display serializes the extension, and parsing recovers it.
1268        let reparsed = LfsPointer::parse(&pointer.to_string())?;
1269        assert_eq!(reparsed.xet_merkle_hash()?, merkle);
1270        Ok(())
1271    }
1272
1273    #[cfg(feature = "xet-storage")]
1274    #[test]
1275    fn test_lfs_pointer_xet_merkle_hash_missing_errors() -> crate::Git2DBResult<()> {
1276        // Vanilla pointer, no xet-merkle: the smudge path must error clearly rather
1277        // than silently mis-converting the SHA-256 OID into a MerkleHash.
1278        let oid = "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890";
1279        let pointer = LfsPointer::new(
1280            "https://git-lfs.github.com/spec/v1".to_owned(),
1281            oid.to_owned(),
1282            1024,
1283        )?;
1284
1285        match pointer.xet_merkle_hash() {
1286            Err(e) => {
1287                assert_eq!(e.lfs_kind(), Some(LfsErrorKind::MissingXetMerkle));
1288                let msg = e.to_string();
1289                assert!(
1290                    msg.contains("xet-merkle"),
1291                    "error must name the missing extension: {msg}"
1292                );
1293                assert!(
1294                    msg.contains("#386"),
1295                    "error must reference issue #386: {msg}"
1296                );
1297            }
1298            Ok(h) => panic!("vanilla pointer unexpectedly resolved a MerkleHash: {h}"),
1299        }
1300        Ok(())
1301    }
1302}