Skip to main content

git_xet_filter/
gittorrent_storage.rs

1//! GittorrentStorage — StorageBackend implementation for P2P distribution.
2//!
3//! Fetches XET objects via gittorrent's DHT network with fallback to HTTPS origin.
4//! XET `MerkleHash` values cross into hyprstream-p2p as self-describing CIDs;
5//! they are never reinterpreted as SHA-256.
6
7#[cfg(feature = "xet-storage")]
8use async_trait::async_trait;
9use std::path::Path;
10use std::sync::Arc;
11
12use crate::error::{Result, XetError, XetErrorKind};
13
14const XET_MERKLE_FIELD: &str = "xet-merkle";
15
16#[derive(Debug, serde::Serialize, serde::Deserialize)]
17struct GittorrentPointer {
18    xet: String,
19    #[serde(rename = "xet-merkle")]
20    xet_merkle: String,
21    sha256: String,
22    size: u64,
23}
24
25impl GittorrentPointer {
26    fn parse(pointer: &str) -> Result<Self> {
27        let parsed: Self = serde_json::from_str(pointer).map_err(|e| {
28            XetError::new(
29                XetErrorKind::InvalidPointer,
30                format!("Invalid gittorrent pointer: {e}"),
31            )
32        })?;
33        if parsed.xet != "gittorrent" {
34            return Err(XetError::new(
35                XetErrorKind::InvalidPointer,
36                "Pointer is not a gittorrent XET pointer",
37            ));
38        }
39        parsed.merkle_hash()?;
40        hyprstream_p2p::Sha256Hash::new(&parsed.sha256).map_err(|e| {
41            XetError::new(
42                XetErrorKind::InvalidPointer,
43                format!("Invalid sha256 field: {e}"),
44            )
45        })?;
46        Ok(parsed)
47    }
48
49    fn merkle_hash(&self) -> Result<merklehash::MerkleHash> {
50        merklehash::MerkleHash::from_hex(&self.xet_merkle).map_err(|e| {
51            XetError::new(
52                XetErrorKind::InvalidPointer,
53                format!("Invalid {XET_MERKLE_FIELD} field: {e}"),
54            )
55        })
56    }
57}
58
59/// P2P storage backend using gittorrent DHT.
60///
61/// Attempts to fetch objects from DHT peers first, falling back to an optional
62/// HTTPS origin (typically XetStorage).
63pub struct GittorrentStorage {
64    service: Arc<hyprstream_p2p::service::GitTorrentService>,
65    fallback: Option<Box<dyn super::storage::StorageBackend>>,
66}
67
68impl GittorrentStorage {
69    /// Create a new gittorrent storage backend.
70    ///
71    /// # Arguments
72    /// * `service` - The gittorrent service instance (DHT + object cache)
73    /// * `fallback` - Optional fallback storage (e.g., XetStorage for HTTPS origin)
74    pub fn new(
75        service: Arc<hyprstream_p2p::service::GitTorrentService>,
76        fallback: Option<Box<dyn super::storage::StorageBackend>>,
77    ) -> Self {
78        Self { service, fallback }
79    }
80
81    /// Fetch reconstruction bytes for a Merkle hash without committing any
82    /// CID binding. The returned flag is `true` when the bytes did not come
83    /// from the already-validated local p2p store and should be persisted by
84    /// the caller only after end-to-end validation — i.e. when they arrived
85    /// via the HTTPS fallback OR via an uncommitted provider hit on a
86    /// non-self-verifying (XET Merkle) CID. The flag is `false` only when
87    /// the service reports the bytes as committed (local store hit or a
88    /// self-verifying fetch that the service committed itself).
89    ///
90    /// Provenance is taken structurally from
91    /// [`GitTorrentService::get_object_by_cid_with_provenance`]; it is never
92    /// assumed from the mere fact of a P2P hit, because the service returns
93    /// provider-fetched XET Merkle bytes without committing them.
94    ///
95    /// This is an internal helper rather than a [`StorageBackend`] method:
96    /// the trait exposes only the `smudge_*` surface, and the extra
97    /// `uncommitted` flag is meaningful solely to this backend's callers.
98    async fn fetch_from_hash(&self, hash: &merklehash::MerkleHash) -> Result<(Vec<u8>, bool)> {
99        let cid = hyprstream_p2p::ContentCid::xet_merkle(hash.as_bytes()).map_err(|e| {
100            XetError::new(
101                XetErrorKind::DownloadFailed,
102                format!("CID encoding failed: {e}"),
103            )
104        })?;
105
106        // Try P2P first. Propagate commit provenance rather than assuming
107        // every P2P hit is already-validated: a non-self-verifying CID
108        // (XET Merkle addresses the reconstruction DAG, not the raw file)
109        // is returned uncommitted by the service, and only the caller's
110        // end-to-end size + SHA-256 check can license persisting it.
111        match self.service.get_object_by_cid_with_provenance(&cid).await {
112            Ok(Some(fetch)) => return Ok((fetch.data, !fetch.committed)),
113            Ok(None) => {
114                tracing::debug!("Object {} not found in DHT, trying fallback", hash);
115            }
116            Err(e) => {
117                tracing::warn!("DHT fetch failed for {}: {e}", hash);
118            }
119        }
120
121        // Fallback to HTTPS origin; caching is deferred to the caller until
122        // the pointer's size and SHA-256 checks pass.
123        if let Some(ref fb) = self.fallback {
124            let data = fb.smudge_from_hash(hash).await?;
125            return Ok((data, true));
126        }
127
128        Err(XetError::new(
129            XetErrorKind::DownloadFailed,
130            format!("Object {hash} not found in DHT or fallback"),
131        ))
132    }
133}
134
135#[cfg(feature = "xet-storage")]
136#[async_trait]
137impl super::storage::StorageBackend for GittorrentStorage {
138    async fn clean_file(&self, path: &Path) -> Result<String> {
139        // Read file and delegate to clean_bytes
140        let data = tokio::fs::read(path).await.map_err(|e| {
141            XetError::new(
142                XetErrorKind::IoError,
143                format!("Failed to read {}: {e}", path.display()),
144            )
145        })?;
146        self.clean_bytes(&data).await
147    }
148
149    fn is_pointer(&self, content: &str) -> bool {
150        GittorrentPointer::parse(content).is_ok()
151    }
152
153    async fn clean_bytes(&self, data: &[u8]) -> Result<String> {
154        // XET's keyed-BLAKE3 Merkle root cannot be derived from the raw-file
155        // SHA-256. Let the XET origin perform the real clean operation, then
156        // carry that root into the P2P key and pointer extension verbatim.
157        let fallback = self.fallback.as_ref().ok_or_else(|| {
158            XetError::new(
159                XetErrorKind::UploadFailed,
160                "gittorrent clean requires an XET origin to compute xet-merkle",
161            )
162        })?;
163        let xet_pointer = fallback.clean_bytes(data).await?;
164        let xet_info: data::XetFileInfo = serde_json::from_str(&xet_pointer).map_err(|e| {
165            XetError::new(
166                XetErrorKind::InvalidPointer,
167                format!("XET origin returned an invalid pointer: {e}"),
168            )
169        })?;
170        let merkle = xet_info.merkle_hash().map_err(|e| {
171            XetError::new(
172                XetErrorKind::InvalidPointer,
173                format!("XET origin returned an invalid Merkle hash: {e}"),
174            )
175        })?;
176        let cid = hyprstream_p2p::ContentCid::xet_merkle(merkle.as_bytes()).map_err(|e| {
177            XetError::new(
178                XetErrorKind::UploadFailed,
179                format!("CID encoding failed: {e}"),
180            )
181        })?;
182
183        self.service
184            .put_object_by_cid(cid, data.to_vec())
185            .await
186            .map_err(|e| XetError::new(XetErrorKind::UploadFailed, e.to_string()))?;
187
188        let sha256 = hyprstream_p2p::crypto::hash::sha256_git(data)
189            .map_err(|e| XetError::new(XetErrorKind::UploadFailed, e.to_string()))?;
190        serde_json::to_string(&GittorrentPointer {
191            xet: "gittorrent".to_owned(),
192            xet_merkle: merkle.to_string(),
193            sha256: sha256.to_string(),
194            size: data.len() as u64,
195        })
196        .map_err(|e| {
197            XetError::new(
198                XetErrorKind::InvalidPointer,
199                format!("Failed to serialize gittorrent pointer: {e}"),
200            )
201        })
202    }
203
204    async fn smudge_file(&self, pointer: &str, output_path: &Path) -> Result<()> {
205        let data = self.smudge_bytes(pointer).await?;
206        tokio::fs::write(output_path, &data).await.map_err(|e| {
207            XetError::new(
208                XetErrorKind::IoError,
209                format!("Failed to write {}: {e}", output_path.display()),
210            )
211        })
212    }
213
214    async fn smudge_bytes(&self, pointer: &str) -> Result<Vec<u8>> {
215        let pointer = GittorrentPointer::parse(pointer)?;
216        let merkle = pointer.merkle_hash()?;
217        let (data, uncommitted) = self.fetch_from_hash(&merkle).await?;
218        if data.len() as u64 != pointer.size {
219            return Err(XetError::new(
220                XetErrorKind::DownloadFailed,
221                format!(
222                    "Size mismatch: expected {}, got {}",
223                    pointer.size,
224                    data.len()
225                ),
226            ));
227        }
228        let expected = hyprstream_p2p::Sha256Hash::new(pointer.sha256).map_err(|e| {
229            XetError::new(
230                XetErrorKind::InvalidPointer,
231                format!("Invalid sha256 field: {e}"),
232            )
233        })?;
234        if !hyprstream_p2p::crypto::hash::verify_sha256(&data, &expected)
235            .map_err(|e| XetError::new(XetErrorKind::DownloadFailed, e.to_string()))?
236        {
237            return Err(XetError::new(
238                XetErrorKind::DownloadFailed,
239                "SHA-256 validation failed for gittorrent object",
240            ));
241        }
242        if uncommitted {
243            // The bytes passed the pointer's end-to-end size + SHA-256 checks;
244            // only now is it safe to bind them to the Merkle CID in the p2p
245            // object plane (a Merkle CID cannot verify raw bytes on its own).
246            let cid = hyprstream_p2p::ContentCid::xet_merkle(merkle.as_bytes()).map_err(|e| {
247                XetError::new(
248                    XetErrorKind::DownloadFailed,
249                    format!("CID encoding failed: {e}"),
250                )
251            })?;
252            if let Err(e) = self.service.put_object_by_cid(cid, data.clone()).await {
253                tracing::warn!("Failed to cache XET object in gittorrent: {e}");
254            }
255        }
256        Ok(data)
257    }
258
259    async fn smudge_from_hash(&self, hash: &merklehash::MerkleHash) -> Result<Vec<u8>> {
260        // Hash-only callers carry no pointer metadata (size, SHA-256) to
261        // validate against, so fetched bytes are never committed to the p2p
262        // plane on this path — only `smudge_bytes` caches, after validation.
263        Ok(self.fetch_from_hash(hash).await?.0)
264    }
265
266    async fn smudge_from_hash_to_file(
267        &self,
268        hash: &merklehash::MerkleHash,
269        output_path: &Path,
270    ) -> Result<()> {
271        let data = self.smudge_from_hash(hash).await?;
272        tokio::fs::write(output_path, &data).await.map_err(|e| {
273            XetError::new(
274                XetErrorKind::IoError,
275                format!("Failed to write {}: {e}", output_path.display()),
276            )
277        })
278    }
279}
280
281#[cfg(test)]
282mod tests {
283    use super::*;
284
285    #[test]
286    fn test_parse_pointer_carries_distinct_hash_domains() -> Result<()> {
287        let pointer = r#"{"xet":"gittorrent","xet-merkle":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","sha256":"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789","size":1024}"#;
288        let parsed = GittorrentPointer::parse(pointer)?;
289        assert_eq!(
290            parsed.xet_merkle,
291            "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
292        );
293        assert_eq!(
294            parsed.sha256,
295            "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"
296        );
297        Ok(())
298    }
299
300    #[test]
301    fn test_parse_invalid_pointer() {
302        assert!(GittorrentPointer::parse("not a pointer").is_err());
303        let legacy = r#"{"xet":"gittorrent","sha256":"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789","size":1}"#;
304        assert!(GittorrentPointer::parse(legacy).is_err());
305    }
306}
307
308/// Provider-side regression coverage (#1115). These tests drive a real
309/// `GitTorrentService` through its public `testing` mocks so the XET smudge
310/// path — provenance propagation, end-to-end validation, and the persist
311/// gate — flows through production code.
312#[cfg(all(test, feature = "gittorrent-transport"))]
313mod provider_fetch_tests {
314    use super::*;
315    use crate::storage::StorageBackend;
316    use hyprstream_p2p::locator::PeerContact;
317    use hyprstream_p2p::service::testing::{
318        new_service_with_mocks, test_config, MockBlobFetcher, MockObjectLocator,
319    };
320    use hyprstream_p2p::ContentCid;
321    use std::net::{Ipv4Addr, SocketAddrV4};
322    use std::sync::Arc;
323    use tempfile::TempDir;
324
325    fn p2p_err(e: hyprstream_p2p::Error) -> XetError {
326        XetError::new(XetErrorKind::DownloadFailed, e.to_string())
327    }
328
329    fn io_err(e: std::io::Error) -> XetError {
330        XetError::new(XetErrorKind::IoError, e.to_string())
331    }
332
333    /// Build a storage backed by a mock object plane that advertises `data`
334    /// under `merkle_bytes` from a single local provider.
335    async fn storage_with_provider(
336        temp: &TempDir,
337        merkle_bytes: [u8; 32],
338        data: Vec<u8>,
339    ) -> Result<(
340        GittorrentStorage,
341        Arc<MockObjectLocator>,
342        Arc<MockBlobFetcher>,
343    )> {
344        let locator = Arc::new(MockObjectLocator::default());
345        let fetcher = Arc::new(MockBlobFetcher::default());
346        let service =
347            new_service_with_mocks(test_config(temp.path()), locator.clone(), fetcher.clone())
348                .await
349                .map_err(p2p_err)?;
350        let cid = ContentCid::xet_merkle(&merkle_bytes).map_err(p2p_err)?;
351        locator
352            .add_provider_for_content(
353                &cid,
354                PeerContact::untrusted(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 6881)),
355            )
356            .await;
357        fetcher.insert_raw(cid, data).await;
358        Ok((
359            GittorrentStorage::new(Arc::new(service), None),
360            locator,
361            fetcher,
362        ))
363    }
364
365    fn pointer_json(merkle_hex: &str, sha256_hex: &str, size: u64) -> String {
366        serde_json::json!({
367            "xet": "gittorrent",
368            "xet-merkle": merkle_hex,
369            "sha256": sha256_hex,
370            "size": size,
371        })
372        .to_string()
373    }
374
375    /// Regression for #1115: provider-fetched XET Merkle bytes that pass the
376    /// pointer's end-to-end size + SHA-256 check MUST be persisted to the
377    /// p2p object plane. Before the provenance fix, `fetch_from_hash` mapped
378    /// every P2P hit to `uncommitted = false`, so `smudge_bytes` skipped the
379    /// persist and the next fetch re-hit the provider.
380    #[tokio::test]
381    async fn provider_fetched_xet_bytes_are_persisted_after_validation() -> Result<()> {
382        let temp = TempDir::new().map_err(io_err)?;
383        let data = b"xet payload from provider".to_vec();
384        let merkle_bytes = [0x7f_u8; 32];
385        let merkle_hex: String = merkle_bytes.iter().map(|b| format!("{b:02x}")).collect();
386        let sha256 = hyprstream_p2p::crypto::hash::sha256_git(&data).map_err(p2p_err)?;
387        let (storage, _locator, fetcher) =
388            storage_with_provider(&temp, merkle_bytes, data.clone()).await?;
389        let pointer = pointer_json(&merkle_hex, &sha256.to_string(), data.len() as u64);
390
391        // First smudge: provider fetch + end-to-end validation + persist.
392        let smudged = storage.smudge_bytes(&pointer).await?;
393        assert_eq!(smudged, data);
394        assert_eq!(fetcher.calls().await, 1, "first smudge hits the provider");
395
396        // Second smudge: bytes are now committed locally, so the provider is
397        // NOT consulted again. Before the fix this re-fetched (calls == 2)
398        // because the unvalidated provider bytes were never persisted.
399        let smudged_again = storage.smudge_bytes(&pointer).await?;
400        assert_eq!(smudged_again, data);
401        assert_eq!(
402            fetcher.calls().await,
403            1,
404            "validated provider bytes are served from the local store on retry"
405        );
406        Ok(())
407    }
408
409    /// The persistence gate must NOT fire when validation fails: a provider
410    /// returning the wrong bytes is rejected, not persisted.
411    #[tokio::test]
412    async fn provider_fetched_xet_bytes_with_wrong_sha256_are_rejected() -> Result<()> {
413        let temp = TempDir::new().map_err(io_err)?;
414        // Same length as the declared real payload so the size check passes
415        // and the SHA-256 check is the gate actually exercised here.
416        let bogus = b"not real payload".to_vec();
417        let real_sha =
418            hyprstream_p2p::crypto::hash::sha256_git(b"the real payload").map_err(p2p_err)?;
419        let merkle_bytes = [0x33_u8; 32];
420        let merkle_hex: String = merkle_bytes.iter().map(|b| format!("{b:02x}")).collect();
421        let (storage, _locator, _fetcher) =
422            storage_with_provider(&temp, merkle_bytes, bogus).await?;
423        // Pointer describes the real payload; provider serves something else.
424        let pointer = pointer_json(&merkle_hex, &real_sha.to_string(), 16);
425
426        match storage.smudge_bytes(&pointer).await {
427            Err(err) => assert!(
428                matches!(err.kind(), XetErrorKind::DownloadFailed),
429                "validation failure must surface as DownloadFailed, got {:?}",
430                err.kind()
431            ),
432            Ok(bytes) => {
433                panic!("validation must reject provider bytes with wrong SHA-256; got {bytes:?}")
434            }
435        }
436        Ok(())
437    }
438}