Skip to main content

git_xet_filter/
storage.rs

1//! Storage backend for XET content-addressed storage operations.
2//!
3//! This module provides the [`StorageBackend`] trait for abstracting XET CAS
4//! (Content-Addressed Storage) operations, and the [`XetStorage`] implementation
5//! that integrates with XET's data client.
6//!
7//! # Overview
8//!
9//! The storage backend handles two main operations:
10//!
11//! - **Clean**: Upload file content to XET CAS and return a JSON pointer
12//! - **Smudge**: Download content from XET CAS using a JSON pointer or merkle hash
13//!
14//! # Example
15//!
16//! ```ignore
17//! use git_xet_filter::storage::{StorageBackend, XetStorage};
18//! use git_xet_filter::config::XetConfig;
19//!
20//! let config = XetConfig::huggingface();
21//! let storage = XetStorage::new(&config).await?;
22//!
23//! // Upload a file
24//! let pointer = storage.clean_file(Path::new("model.safetensors")).await?;
25//!
26//! // Download to bytes
27//! let data = storage.smudge_bytes(&pointer).await?;
28//! ```
29
30#[cfg(feature = "xet-storage")]
31use async_trait::async_trait;
32use std::io::Write;
33use std::path::Path;
34use std::sync::Arc;
35use std::{
36    io,
37    pin::Pin,
38    task::{Context, Poll},
39};
40
41#[cfg(feature = "xet-storage")]
42use data::{FileDownloader, FileUploadSession, XetFileInfo};
43#[cfg(feature = "xet-storage")]
44use tokio::io::AsyncWrite;
45
46use crate::error::{Result, XetError, XetErrorKind};
47
48/// Storage backend trait for XET filter operations.
49///
50/// This trait abstracts the XET content-addressed storage operations,
51/// allowing for clean (upload) and smudge (download) of file content.
52///
53/// All methods are async and designed for use with the XET data client.
54/// Implementations must be `Send + Sync` for use across threads.
55#[cfg(feature = "xet-storage")]
56#[async_trait]
57pub trait StorageBackend: Send + Sync {
58    /// Upload a file to XET CAS and return a JSON pointer string.
59    ///
60    /// The returned pointer contains metadata (size, merkle hash) needed
61    /// to retrieve the content later via [`smudge_bytes`] or [`smudge_file`].
62    ///
63    /// [`smudge_bytes`]: StorageBackend::smudge_bytes
64    /// [`smudge_file`]: StorageBackend::smudge_file
65    async fn clean_file(&self, path: &Path) -> Result<String>;
66
67    /// Check if a string is a valid XET pointer.
68    ///
69    /// Returns `true` if the content can be parsed as a valid XET pointer JSON.
70    fn is_pointer(&self, content: &str) -> bool;
71
72    /// Upload data from memory to XET CAS and return a JSON pointer string.
73    ///
74    /// This is more efficient than [`clean_file`] when data is already in memory.
75    ///
76    /// [`clean_file`]: StorageBackend::clean_file
77    async fn clean_bytes(&self, data: &[u8]) -> Result<String>;
78
79    /// Download content from a pointer directly to a file.
80    ///
81    /// This is more efficient than [`smudge_bytes`] when the destination is a file,
82    /// as it avoids intermediate memory allocation.
83    ///
84    /// [`smudge_bytes`]: StorageBackend::smudge_bytes
85    async fn smudge_file(&self, pointer: &str, output_path: &Path) -> Result<()>;
86
87    /// Download content from a pointer to an in-memory buffer.
88    ///
89    /// Use this for small files or when the content needs processing.
90    /// For large files, prefer [`smudge_file`] to write directly to disk.
91    ///
92    /// [`smudge_file`]: StorageBackend::smudge_file
93    async fn smudge_bytes(&self, pointer: &str) -> Result<Vec<u8>>;
94
95    /// Download content by merkle hash to an in-memory buffer.
96    ///
97    /// This is useful for LFS pointer resolution where you have a SHA256 hash
98    /// instead of a full XET pointer JSON string.
99    async fn smudge_from_hash(&self, hash: &merklehash::MerkleHash) -> Result<Vec<u8>>;
100
101    /// Download content by merkle hash directly to a file.
102    ///
103    /// Combines the efficiency of direct file writes with hash-based lookups
104    /// for LFS pointer resolution.
105    async fn smudge_from_hash_to_file(
106        &self,
107        hash: &merklehash::MerkleHash,
108        output_path: &Path,
109    ) -> Result<()>;
110}
111
112/// XET storage backend implementation.
113///
114/// Wraps the XET data client to provide clean/smudge operations for
115/// git filter integration. Uses async I/O for all network operations.
116///
117/// # Example
118///
119/// ```ignore
120/// use git_xet_filter::storage::XetStorage;
121/// use git_xet_filter::config::XetConfig;
122///
123/// let config = XetConfig::huggingface();
124/// let storage = XetStorage::new(&config).await?;
125///
126/// // Clean (upload) a file
127/// let pointer = storage.clean_file(Path::new("weights.safetensors")).await?;
128///
129/// // Smudge (download) to memory
130/// let data = storage.smudge_bytes(&pointer).await?;
131/// ```
132#[cfg(feature = "xet-storage")]
133pub struct XetStorage {
134    upload_session: Arc<FileUploadSession>,
135    downloader: Arc<FileDownloader>,
136}
137
138/// Sequential CAS output that refuses writes beyond a caller-provided byte
139/// ceiling. Unlike a post-download metadata check, this bounds bytes reaching
140/// the private projection even when a CAS object disagrees with its pointer.
141#[cfg(feature = "xet-storage")]
142struct BoundedFileWriter {
143    file: std::fs::File,
144    limit: u64,
145    written: u64,
146}
147
148#[cfg(feature = "xet-storage")]
149impl BoundedFileWriter {
150    fn create(path: &Path, limit: u64) -> io::Result<Self> {
151        let file = std::fs::OpenOptions::new()
152            .write(true)
153            .create_new(true)
154            .open(path)?;
155        Ok(Self {
156            file,
157            limit,
158            written: 0,
159        })
160    }
161}
162
163#[cfg(feature = "xet-storage")]
164impl AsyncWrite for BoundedFileWriter {
165    fn poll_write(
166        mut self: Pin<&mut Self>,
167        _cx: &mut Context<'_>,
168        buf: &[u8],
169    ) -> Poll<io::Result<usize>> {
170        let Some(end) = self.written.checked_add(buf.len() as u64) else {
171            return Poll::Ready(Err(io::Error::new(
172                io::ErrorKind::InvalidData,
173                "CAS output size overflow",
174            )));
175        };
176        if end > self.limit {
177            return Poll::Ready(Err(io::Error::new(
178                io::ErrorKind::InvalidData,
179                format!("CAS output exceeds {0}-byte limit", self.limit),
180            )));
181        }
182        loop {
183            match self.file.write(buf) {
184                Ok(written) => {
185                    self.written += written as u64;
186                    return Poll::Ready(Ok(written));
187                }
188                Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
189                Err(error) => return Poll::Ready(Err(error)),
190            }
191        }
192    }
193
194    fn poll_flush(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
195        Poll::Ready(self.file.flush())
196    }
197
198    fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
199        Poll::Ready(self.file.flush())
200    }
201}
202
203#[cfg(all(test, feature = "xet-storage"))]
204mod bounded_output_tests {
205    use super::BoundedFileWriter;
206    use std::io;
207    use tokio::io::AsyncWriteExt;
208
209    #[tokio::test]
210    async fn oversized_cas_chunk_is_rejected_before_file_growth() -> io::Result<()> {
211        let temp = tempfile::tempdir()?;
212        let output = temp.path().join("bounded-output");
213        let mut writer = BoundedFileWriter::create(&output, 4)?;
214
215        assert!(writer.write_all(b"oversized").await.is_err());
216        writer.flush().await?;
217        drop(writer);
218        assert_eq!(std::fs::metadata(&output)?.len(), 0);
219        assert_eq!(std::fs::read(&output)?, b"");
220        Ok(())
221    }
222
223    #[tokio::test]
224    async fn understated_pointer_cannot_write_past_declared_size() -> io::Result<()> {
225        let temp = tempfile::tempdir()?;
226        let output = temp.path().join("bounded-output");
227        let mut writer = BoundedFileWriter::create(&output, 4)?;
228
229        writer.write_all(b"1234").await?;
230        assert!(writer.write_all(b"5").await.is_err());
231        writer.flush().await?;
232        drop(writer);
233        assert_eq!(std::fs::read(&output)?, b"1234");
234        assert_eq!(std::fs::metadata(&output)?.len(), 4);
235        Ok(())
236    }
237}
238
239#[cfg(feature = "xet-storage")]
240impl XetStorage {
241    /// Create a new XET storage backend with the given configuration.
242    ///
243    /// Initializes the upload session and downloader with the endpoint
244    /// and authentication from the config.
245    ///
246    /// # Errors
247    ///
248    /// Returns an error if the XET data client fails to initialize,
249    /// typically due to network issues or invalid credentials.
250    pub async fn new(config: &crate::config::XetConfig) -> Result<Self> {
251        let translator_config = Arc::new(
252            data::data_client::default_config(
253                config.endpoint.clone(),
254                config.compression,
255                config.token.as_ref().map(|t| (t.clone(), u64::MAX)),
256                None,
257                "hyprstream".to_owned(),
258            )
259            .map_err(|e| {
260                XetError::new(
261                    XetErrorKind::RuntimeError,
262                    format!("Failed to create XET config: {e}"),
263                )
264            })?,
265        );
266
267        let upload_session = FileUploadSession::new(translator_config.clone(), None)
268            .await
269            .map_err(|e| {
270                XetError::new(
271                    XetErrorKind::UploadFailed,
272                    format!("Failed to create upload session: {e}"),
273                )
274            })?;
275
276        let downloader = FileDownloader::new(translator_config).await.map_err(|e| {
277            XetError::new(
278                XetErrorKind::DownloadFailed,
279                format!("Failed to create downloader: {e}"),
280            )
281        })?;
282
283        Ok(Self {
284            upload_session,
285            downloader: Arc::new(downloader),
286        })
287    }
288
289    /// Upload data from memory and return XET pointer JSON
290    ///
291    /// This is more efficient than writing to a temp file first when the data
292    /// is already in memory (e.g., from tensor serialization).
293    pub async fn clean_bytes(&self, data: &[u8]) -> Result<String> {
294        let (xet_info, _metrics) =
295            data::data_client::clean_bytes(self.upload_session.clone(), data.to_vec())
296                .await
297                .map_err(|e| {
298                    XetError::new(
299                        XetErrorKind::UploadFailed,
300                        format!("Clean bytes failed: {e}"),
301                    )
302                })?;
303
304        xet_info.as_pointer_file().map_err(|e| {
305            XetError::new(
306                XetErrorKind::InvalidPointer,
307                format!("Failed to serialize pointer: {e}"),
308            )
309        })
310    }
311
312    /// Download from XET pointer directly to a file path
313    ///
314    /// This is more efficient than downloading to memory first when the final
315    /// destination is a file (e.g., model weights).
316    pub async fn smudge_file(&self, pointer: &str, output_path: &Path) -> Result<()> {
317        let xet_info: XetFileInfo = serde_json::from_str(pointer).map_err(|e| {
318            XetError::new(
319                XetErrorKind::InvalidPointer,
320                format!("Invalid pointer JSON: {e}"),
321            )
322        })?;
323
324        let merkle_hash = xet_info.merkle_hash().map_err(|e| {
325            XetError::new(
326                XetErrorKind::InvalidPointer,
327                format!("Invalid merkle hash: {e}"),
328            )
329        })?;
330
331        use cas_client::{FileProvider, SeekingOutputProvider};
332        self.downloader
333            .smudge_file_from_hash(
334                &merkle_hash,
335                output_path.to_string_lossy().into(),
336                SeekingOutputProvider::File(FileProvider::new(output_path.to_path_buf())),
337                None,
338                None,
339            )
340            .await
341            .map_err(|e| {
342                XetError::new(
343                    XetErrorKind::DownloadFailed,
344                    format!("Smudge to file failed: {e}"),
345                )
346            })?;
347
348        Ok(())
349    }
350
351    /// Download a pointer into a new file while enforcing `max_output_bytes`
352    /// at each sequential write. This is intended for callers that must bound
353    /// untrusted or stale pointer metadata before the payload reaches disk.
354    pub async fn smudge_file_bounded(
355        &self,
356        pointer: &str,
357        output_path: &Path,
358        max_output_bytes: u64,
359    ) -> Result<u64> {
360        let info: XetFileInfo = serde_json::from_str(pointer).map_err(|e| {
361            XetError::new(
362                XetErrorKind::InvalidPointer,
363                format!("Invalid pointer JSON: {e}"),
364            )
365        })?;
366        if info.file_size() > max_output_bytes {
367            return Err(XetError::new(
368                XetErrorKind::InvalidPointer,
369                format!("Pointer size exceeds {max_output_bytes}-byte output limit"),
370            ));
371        }
372        let hash = info.merkle_hash().map_err(|e| {
373            XetError::new(
374                XetErrorKind::InvalidPointer,
375                format!("Invalid merkle hash: {e}"),
376            )
377        })?;
378        self.smudge_hash_to_file_bounded(&hash, output_path, max_output_bytes)
379            .await
380    }
381
382    /// Download content by Merkle hash using a strict write-time byte ceiling.
383    pub async fn smudge_hash_to_file_bounded(
384        &self,
385        hash: &merklehash::MerkleHash,
386        output_path: &Path,
387        max_output_bytes: u64,
388    ) -> Result<u64> {
389        let writer = BoundedFileWriter::create(output_path, max_output_bytes).map_err(|e| {
390            XetError::new(
391                XetErrorKind::IoError,
392                format!("Create bounded CAS output: {e}"),
393            )
394        })?;
395        let output: cas_client::SequentialOutput = Box::new(writer);
396        let written = self
397            .downloader
398            .smudge_file_from_hash_sequential(
399                hash,
400                output_path.to_string_lossy().into(),
401                output,
402                None,
403                None,
404            )
405            .await
406            .map_err(|e| {
407                XetError::new(
408                    XetErrorKind::DownloadFailed,
409                    format!("Bounded smudge to file failed: {e}"),
410                )
411            })?;
412        if written > max_output_bytes {
413            return Err(XetError::new(
414                XetErrorKind::DownloadFailed,
415                "CAS downloader reported bytes beyond its enforced output limit",
416            ));
417        }
418        Ok(written)
419    }
420
421    /// Download from XET pointer to memory
422    ///
423    /// This is a better-named version of `smudge_pointer()` and uses async I/O.
424    pub async fn smudge_bytes(&self, pointer: &str) -> Result<Vec<u8>> {
425        let xet_info: XetFileInfo = serde_json::from_str(pointer).map_err(|e| {
426            XetError::new(
427                XetErrorKind::InvalidPointer,
428                format!("Invalid pointer JSON: {e}"),
429            )
430        })?;
431
432        let merkle_hash = xet_info.merkle_hash().map_err(|e| {
433            XetError::new(
434                XetErrorKind::InvalidPointer,
435                format!("Invalid merkle hash: {e}"),
436            )
437        })?;
438
439        // Download to temp file
440        let temp_file = tempfile::NamedTempFile::new().map_err(|e| {
441            XetError::new(
442                XetErrorKind::IoError,
443                format!("Failed to create temp file: {e}"),
444            )
445        })?;
446
447        use cas_client::{FileProvider, SeekingOutputProvider};
448        self.downloader
449            .smudge_file_from_hash(
450                &merkle_hash,
451                temp_file.path().to_string_lossy().into(),
452                SeekingOutputProvider::File(FileProvider::new(temp_file.path().to_path_buf())),
453                None,
454                None,
455            )
456            .await
457            .map_err(|e| {
458                XetError::new(
459                    XetErrorKind::DownloadFailed,
460                    format!("Smudge to bytes failed: {e}"),
461                )
462            })?;
463
464        // Read back with async I/O
465        tokio::fs::read(temp_file.path()).await.map_err(|e| {
466            XetError::new(
467                XetErrorKind::IoError,
468                format!("Failed to read smudged file: {e}"),
469            )
470        })
471    }
472
473    /// Download from XET by merkle hash to memory
474    ///
475    /// This is useful for LFS pointer resolution where you have a SHA256 hash
476    /// and need to retrieve the content directly without a XET pointer.
477    pub async fn smudge_from_hash(&self, hash: &merklehash::MerkleHash) -> Result<Vec<u8>> {
478        // Download to temp file
479        let temp_file = tempfile::NamedTempFile::new().map_err(|e| {
480            XetError::new(
481                XetErrorKind::IoError,
482                format!("Failed to create temp file: {e}"),
483            )
484        })?;
485
486        use cas_client::{FileProvider, SeekingOutputProvider};
487        self.downloader
488            .smudge_file_from_hash(
489                hash,
490                temp_file.path().to_string_lossy().into(),
491                SeekingOutputProvider::File(FileProvider::new(temp_file.path().to_path_buf())),
492                None,
493                None,
494            )
495            .await
496            .map_err(|e| {
497                XetError::new(
498                    XetErrorKind::DownloadFailed,
499                    format!("Smudge from hash failed: {e}"),
500                )
501            })?;
502
503        // Read back with async I/O
504        tokio::fs::read(temp_file.path()).await.map_err(|e| {
505            XetError::new(
506                XetErrorKind::IoError,
507                format!("Failed to read smudged file: {e}"),
508            )
509        })
510    }
511
512    /// Download from XET by merkle hash directly to a file path
513    ///
514    /// This is useful for LFS pointer resolution where you have a SHA256 hash
515    /// and want to write directly to disk.
516    pub async fn smudge_from_hash_to_file(
517        &self,
518        hash: &merklehash::MerkleHash,
519        output_path: &Path,
520    ) -> Result<()> {
521        use cas_client::{FileProvider, SeekingOutputProvider};
522        self.downloader
523            .smudge_file_from_hash(
524                hash,
525                output_path.to_string_lossy().into(),
526                SeekingOutputProvider::File(FileProvider::new(output_path.to_path_buf())),
527                None,
528                None,
529            )
530            .await
531            .map_err(|e| {
532                XetError::new(
533                    XetErrorKind::DownloadFailed,
534                    format!("Smudge from hash to file failed: {e}"),
535                )
536            })?;
537
538        Ok(())
539    }
540}
541
542#[cfg(feature = "xet-storage")]
543#[async_trait]
544impl StorageBackend for XetStorage {
545    async fn clean_file(&self, path: &Path) -> Result<String> {
546        let (xet_info, _metrics) =
547            data::data_client::clean_file(self.upload_session.clone(), path, "")
548                .await
549                .map_err(|e| {
550                    XetError::new(XetErrorKind::UploadFailed, format!("Clean failed: {e}"))
551                })?;
552
553        xet_info.as_pointer_file().map_err(|e| {
554            XetError::new(
555                XetErrorKind::InvalidPointer,
556                format!("Failed to serialize pointer: {e}"),
557            )
558        })
559    }
560
561    fn is_pointer(&self, content: &str) -> bool {
562        serde_json::from_str::<XetFileInfo>(content).is_ok()
563    }
564
565    async fn clean_bytes(&self, data: &[u8]) -> Result<String> {
566        XetStorage::clean_bytes(self, data).await
567    }
568
569    async fn smudge_file(&self, pointer: &str, output_path: &Path) -> Result<()> {
570        XetStorage::smudge_file(self, pointer, output_path).await
571    }
572
573    async fn smudge_bytes(&self, pointer: &str) -> Result<Vec<u8>> {
574        XetStorage::smudge_bytes(self, pointer).await
575    }
576
577    async fn smudge_from_hash(&self, hash: &merklehash::MerkleHash) -> Result<Vec<u8>> {
578        XetStorage::smudge_from_hash(self, hash).await
579    }
580
581    async fn smudge_from_hash_to_file(
582        &self,
583        hash: &merklehash::MerkleHash,
584        output_path: &Path,
585    ) -> Result<()> {
586        XetStorage::smudge_from_hash_to_file(self, hash, output_path).await
587    }
588}