Skip to main content

hyprstream_p2p/
blobs.rs

1//! iroh-blobs-backed content-addressed object store (F1, #899).
2//!
3//! Evaluation + first-integration spike for Track F of the gittorrent →
4//! `hyprstream-p2p` convergence (epic #880 §2). Provides a LOCAL (in-process,
5//! no network) put/get pair preserving the consumer contract used by
6//! git-xet-filter and git2db:
7//!
8//! ```text
9//! put_object(Vec<u8>)              -> Sha256Hash
10//! get_object(&Sha256Hash)          -> Option<Vec<u8>>
11//! put_object_by_cid(ContentCid, ..) -> ()
12//! get_object_by_cid(&ContentCid)    -> Option<Vec<u8>>
13//! ```
14//!
15//! # CID → BLAKE3 locator index
16//!
17//! iroh-blobs addresses raw bytes by BLAKE3. Consumers address several domains:
18//! raw Git objects use SHA-256, while XET file reconstruction DAGs use a keyed
19//! BLAKE3 `MerkleHash`. Those are carried in a self-describing [`ContentCid`]
20//! and indexed to the raw iroh-blobs hash without collapsing algorithm or
21//! content-domain identity. SHA-256 methods remain compatibility wrappers that
22//! construct a `git-raw + sha2-256` CID.
23//!
24//! For the persistent (`FsStore`) backend the index is an append-only sidecar
25//! file (`sha256-blake3.index`, one `cid blake3hex` line per object) loaded at
26//! open. Legacy `sha256hex blake3hex` entries are upgraded in memory to a Git
27//! object CID. Malformed entries are skipped with a warning.
28//!
29//! # Scope
30//!
31//! Local store only. Networked fetch (provider discovery via the at9p
32//! mainline locator + remote transfer) is F2 (#900). Wiring this store under
33//! `GitTorrentService::{put,get}_object` in place of the libp2p DHT is also
34//! F2; retiring libp2p is F3 (#901).
35
36use std::collections::HashMap;
37use std::path::{Path, PathBuf};
38
39use iroh_blobs::store::fs::FsStore;
40use iroh_blobs::store::mem::MemStore;
41use iroh_blobs::Hash as Blake3Hash;
42use tokio::io::AsyncWriteExt;
43use tokio::sync::Mutex;
44
45use crate::crypto::hash::{sha256_git, verify_sha256};
46use crate::{ContentCid, Error, Result, Sha256Hash};
47
48/// Sidecar index file name (relative to the `FsStore` root).
49const INDEX_FILE: &str = "sha256-blake3.index";
50
51enum Backend {
52    Mem(MemStore),
53    Fs(FsStore),
54}
55
56impl Backend {
57    fn store(&self) -> &iroh_blobs::api::Store {
58        match self {
59            Backend::Mem(s) => s,
60            Backend::Fs(s) => s,
61        }
62    }
63}
64
65/// Content-addressed object store backed by iroh-blobs, keyed by canonical CIDs.
66///
67/// See the module docs for the CID → BLAKE3 locator model.
68pub struct IrohBlobStore {
69    backend: Backend,
70    /// canonical content CID → raw-byte BLAKE3 mapping (locator only).
71    index: Mutex<HashMap<ContentCid, Blake3Hash>>,
72    /// Append-only index persistence (fs backend only).
73    index_path: Option<PathBuf>,
74}
75
76impl IrohBlobStore {
77    /// Create an ephemeral, in-memory store (tests, throwaway caches).
78    pub fn new_memory() -> Self {
79        Self {
80            backend: Backend::Mem(MemStore::new()),
81            index: Mutex::new(HashMap::new()),
82            index_path: None,
83        }
84    }
85
86    /// Open (or create) a persistent store rooted at `root`.
87    ///
88    /// Blob bytes live in the iroh-blobs `FsStore` under `root`; the
89    /// CID→blake3 index is an append-only sidecar file next to it.
90    pub async fn open_fs(root: impl AsRef<Path>) -> Result<Self> {
91        let root = root.as_ref();
92        tokio::fs::create_dir_all(root).await?;
93        let store = FsStore::load(root)
94            .await
95            .map_err(|e| Error::other(format!("failed to open iroh-blobs store: {e}")))?;
96
97        let index_path = root.join(INDEX_FILE);
98        let index = load_index(&index_path).await?;
99
100        Ok(Self {
101            backend: Backend::Fs(store),
102            index: Mutex::new(index),
103            index_path: Some(index_path),
104        })
105    }
106
107    /// Store an object, returning its SHA256 hash (the consumer-facing key).
108    ///
109    /// The bytes are added to the iroh-blobs store (BLAKE3-addressed, tagged
110    /// so garbage collection cannot reclaim them) and the Git CID→blake3
111    /// mapping is recorded.
112    pub async fn put_object(&self, data: Vec<u8>) -> Result<Sha256Hash> {
113        let sha256 = sha256_git(&data)?;
114        self.put_verified_object(sha256.clone(), data).await?;
115        Ok(sha256)
116    }
117
118    /// Store bytes under an expected SHA256 key after re-verifying them.
119    ///
120    /// F2 uses this after a remote iroh-blobs transfer: provider hints are
121    /// untrusted, so fetched bytes are accepted only when they match the
122    /// consumer-facing SHA256 requested by the facade.
123    pub async fn put_verified_object(
124        &self,
125        expected_sha256: Sha256Hash,
126        data: Vec<u8>,
127    ) -> Result<Sha256Hash> {
128        if !verify_sha256(&data, &expected_sha256)? {
129            return Err(Error::other(format!(
130                "sha256 verification failed for fetched object {expected_sha256}"
131            )));
132        }
133
134        let cid = ContentCid::git_sha256(&expected_sha256)?;
135        self.put_object_by_cid(cid, data).await?;
136        Ok(expected_sha256)
137    }
138
139    /// Store bytes under an algorithm-preserving content CID.
140    ///
141    /// Raw Git SHA-256 CIDs are re-verified against the bytes here. An XET
142    /// Merkle CID addresses the reconstruction DAG rather than the raw file,
143    /// so its binding is established by the XET clean operation and the LFS
144    /// SHA-256 remains the end-to-end raw-file integrity check.
145    pub async fn put_object_by_cid(&self, cid: ContentCid, data: Vec<u8>) -> Result<()> {
146        verify_raw_cid_when_applicable(&cid, &data)?;
147        let raw_hash = Blake3Hash::new(&data);
148        if let Some(existing) = self.index.lock().await.get(&cid).copied() {
149            if existing != raw_hash {
150                return Err(Error::other(format!(
151                    "content CID {cid} is already bound to a different raw blob"
152                )));
153            }
154        }
155
156        let tag = self
157            .backend
158            .store()
159            .blobs()
160            .add_bytes(data)
161            .await
162            .map_err(|e| Error::other(format!("iroh-blobs add failed: {e}")))?;
163
164        let mut index = self.index.lock().await;
165        match index.get(&cid) {
166            Some(existing) if *existing != tag.hash => {
167                return Err(Error::other(format!(
168                    "content CID {cid} is already bound to a different raw blob"
169                )));
170            }
171            Some(_) => {}
172            None => {
173                // Persist the sidecar entry before publishing the in-memory
174                // binding: if the append fails after an insert, a retry would
175                // hit the `Some(_)` arm above and never re-attempt the append,
176                // losing the mapping across restarts.
177                if let Some(path) = &self.index_path {
178                    append_index_entry(path, &cid, &tag.hash).await?;
179                }
180                index.insert(cid.clone(), tag.hash);
181            }
182        }
183        drop(index);
184
185        tracing::debug!(content_cid = %cid, blake3 = %tag.hash, "stored object");
186        Ok(())
187    }
188
189    /// Fetch an object by its SHA256 hash.
190    ///
191    /// Returns `Ok(None)` when the hash is unknown. Returns an error if the
192    /// index resolves but the retrieved bytes do not hash back to the
193    /// requested SHA256 (poisoned/corrupt mapping — never served).
194    pub async fn get_object(&self, hash: &Sha256Hash) -> Result<Option<Vec<u8>>> {
195        let cid = ContentCid::git_sha256(hash)?;
196        self.get_object_by_cid(&cid).await
197    }
198
199    /// Fetch an object by its self-describing content CID.
200    pub async fn get_object_by_cid(&self, cid: &ContentCid) -> Result<Option<Vec<u8>>> {
201        let blake3 = {
202            let index = self.index.lock().await;
203            match index.get(cid) {
204                Some(b3) => *b3,
205                None => return Ok(None),
206            }
207        };
208
209        let bytes = match self.backend.store().blobs().get_bytes(blake3).await {
210            Ok(bytes) => bytes,
211            Err(e) => {
212                // Index entry without a backing blob: unreachable object, not
213                // a hard error for a cache-shaped lookup API.
214                tracing::warn!("indexed blob {blake3} missing from store for CID {cid}: {e}");
215                return Ok(None);
216            }
217        };
218
219        let data = bytes.to_vec();
220        verify_raw_cid_when_applicable(cid, &data)?;
221        Ok(Some(data))
222    }
223
224    /// Resolve the BLAKE3 hash backing a SHA256 key, if known.
225    ///
226    /// This is the seam F2 (#900) uses to announce/locate providers: the
227    /// locator plane finds peers, iroh-blobs transfers by BLAKE3.
228    pub async fn blake3_of(&self, hash: &Sha256Hash) -> Option<Blake3Hash> {
229        let cid = ContentCid::git_sha256(hash).ok()?;
230        self.blake3_of_cid(&cid).await
231    }
232
233    /// Resolve the raw-byte BLAKE3 hash backing a content CID, if known.
234    pub async fn blake3_of_cid(&self, cid: &ContentCid) -> Option<Blake3Hash> {
235        self.index.lock().await.get(cid).copied()
236    }
237
238    /// Number of CID-addressable objects.
239    pub async fn len(&self) -> usize {
240        self.index.lock().await.len()
241    }
242
243    /// Whether the store has no CID-addressable objects.
244    pub async fn is_empty(&self) -> bool {
245        self.index.lock().await.is_empty()
246    }
247
248    /// Shut down the underlying iroh-blobs store, flushing pending writes.
249    ///
250    /// Required before the same `FsStore` root can be reopened (the fs
251    /// backend holds a database lock until its actor shuts down); dropping
252    /// the store alone does not release it promptly.
253    pub async fn shutdown(&self) -> Result<()> {
254        self.backend
255            .store()
256            .shutdown()
257            .await
258            .map_err(|e| Error::other(format!("iroh-blobs shutdown failed: {e}")))
259    }
260}
261
262/// Load the sidecar index, skipping (with a warning) any malformed lines.
263async fn load_index(path: &Path) -> Result<HashMap<ContentCid, Blake3Hash>> {
264    let mut index = HashMap::new();
265    let contents = match tokio::fs::read_to_string(path).await {
266        Ok(c) => c,
267        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(index),
268        Err(e) => return Err(e.into()),
269    };
270
271    for (lineno, line) in contents.lines().enumerate() {
272        let line = line.trim();
273        if line.is_empty() {
274            continue;
275        }
276        let parsed = line.split_once(' ').and_then(|(key, b3_hex)| {
277            let cid = ContentCid::new(key)
278                .or_else(|_| {
279                    // Backward compatibility for the original SHA256-only index.
280                    let sha = Sha256Hash::new(key)?;
281                    ContentCid::git_sha256(&sha)
282                })
283                .ok()?;
284            let b3 = b3_hex.parse::<Blake3Hash>().ok()?;
285            Some((cid, b3))
286        });
287        match parsed {
288            Some((cid, b3)) => {
289                index.insert(cid, b3);
290            }
291            None => {
292                tracing::warn!(
293                    "skipping malformed index line {} in {}",
294                    lineno + 1,
295                    path.display()
296                );
297            }
298        }
299    }
300    Ok(index)
301}
302
303/// Append one `cid blake3hex` line to the sidecar index.
304async fn append_index_entry(path: &Path, cid: &ContentCid, blake3: &Blake3Hash) -> Result<()> {
305    let mut file = tokio::fs::OpenOptions::new()
306        .create(true)
307        .append(true)
308        .open(path)
309        .await?;
310    file.write_all(format!("{cid} {blake3}\n").as_bytes())
311        .await?;
312    file.flush().await?;
313    Ok(())
314}
315
316fn verify_raw_cid_when_applicable(cid: &ContentCid, data: &[u8]) -> Result<()> {
317    let decoded = cid.decoded()?;
318    use hyprstream_rpc::cid::{Codec, HashAlgo};
319
320    match (decoded.codec, decoded.multihash.algo) {
321        (Codec::GitRaw, HashAlgo::Sha2_256) => {
322            let expected = Sha256Hash::from_bytes(&decoded.multihash.digest)?;
323            if !verify_sha256(data, &expected)? {
324                return Err(Error::other(format!(
325                    "sha256 verification failed for Git object CID {cid}"
326                )));
327            }
328            Ok(())
329        }
330        (Codec::XetShard, HashAlgo::Blake3) if decoded.multihash.digest.len() == 32 => Ok(()),
331        (codec, algo) => Err(Error::other(format!(
332            "unsupported object CID domain: codec={codec:?}, algorithm={algo:?}"
333        ))),
334    }
335}
336
337#[cfg(test)]
338mod tests {
339    use super::*;
340    use tempfile::TempDir;
341
342    #[tokio::test]
343    async fn test_memory_roundtrip() -> Result<()> {
344        let store = IrohBlobStore::new_memory();
345        let data = b"hello iroh-blobs".to_vec();
346
347        let hash = store.put_object(data.clone()).await?;
348        // The consumer-facing key is plain SHA256 of the bytes.
349        assert_eq!(hash, sha256_git(&data)?);
350
351        let fetched = store.get_object(&hash).await?;
352        assert_eq!(fetched.as_deref(), Some(data.as_slice()));
353        assert_eq!(store.len().await, 1);
354        Ok(())
355    }
356
357    #[tokio::test]
358    async fn test_unknown_hash_is_none() -> Result<()> {
359        let store = IrohBlobStore::new_memory();
360        let unknown = sha256_git(b"never stored")?;
361        assert!(store.get_object(&unknown).await?.is_none());
362        Ok(())
363    }
364
365    #[tokio::test]
366    async fn test_put_is_idempotent() -> Result<()> {
367        let store = IrohBlobStore::new_memory();
368        let data = b"same bytes twice".to_vec();
369        let h1 = store.put_object(data.clone()).await?;
370        let h2 = store.put_object(data.clone()).await?;
371        assert_eq!(h1, h2);
372        assert_eq!(store.len().await, 1);
373        Ok(())
374    }
375
376    #[tokio::test]
377    async fn test_blake3_mapping_exposed() -> Result<()> {
378        let store = IrohBlobStore::new_memory();
379        let data = b"mapped".to_vec();
380        let sha = store.put_object(data.clone()).await?;
381
382        let b3 = store
383            .blake3_of(&sha)
384            .await
385            .ok_or_else(|| Error::not_found("missing mapping"))?;
386        // Hash::new computes BLAKE3 of the bytes — the mapping must point at
387        // the BLAKE3 of the stored content.
388        assert_eq!(b3, Blake3Hash::new(data.as_slice()));
389        Ok(())
390    }
391
392    #[tokio::test]
393    async fn test_put_verified_object_rejects_wrong_sha256() -> Result<()> {
394        let store = IrohBlobStore::new_memory();
395        let expected = sha256_git(b"expected")?;
396
397        assert!(store
398            .put_verified_object(expected.clone(), b"different".to_vec())
399            .await
400            .is_err());
401        assert!(store.get_object(&expected).await?.is_none());
402        Ok(())
403    }
404
405    #[tokio::test]
406    async fn test_xet_cid_cannot_be_rebound_to_different_bytes() -> Result<()> {
407        let store = IrohBlobStore::new_memory();
408        let cid = ContentCid::xet_merkle(&[0x7b; 32])?;
409
410        store
411            .put_object_by_cid(cid.clone(), b"first reconstruction".to_vec())
412            .await?;
413        assert!(store
414            .put_object_by_cid(cid.clone(), b"different reconstruction".to_vec())
415            .await
416            .is_err());
417        assert_eq!(
418            store.get_object_by_cid(&cid).await?.as_deref(),
419            Some(b"first reconstruction".as_slice())
420        );
421        Ok(())
422    }
423
424    #[tokio::test]
425    async fn test_fs_persistence_across_reopen() -> Result<()> {
426        let dir = TempDir::new()?;
427        let data = b"persisted across reopen".to_vec();
428
429        let sha = {
430            let store = IrohBlobStore::open_fs(dir.path()).await?;
431            let sha = store.put_object(data.clone()).await?;
432            store.shutdown().await?;
433            sha
434        };
435
436        let store = IrohBlobStore::open_fs(dir.path()).await?;
437        let fetched = store.get_object(&sha).await?;
438        assert_eq!(fetched.as_deref(), Some(data.as_slice()));
439        store.shutdown().await?;
440        Ok(())
441    }
442
443    #[tokio::test]
444    async fn test_poisoned_index_never_serves_wrong_bytes() -> Result<()> {
445        let dir = TempDir::new()?;
446        let a = b"object a".to_vec();
447        let b = b"object b".to_vec();
448
449        let (sha_a, sha_b) = {
450            let store = IrohBlobStore::open_fs(dir.path()).await?;
451            let pair = (
452                store.put_object(a.clone()).await?,
453                store.put_object(b.clone()).await?,
454            );
455            store.shutdown().await?;
456            pair
457        };
458
459        // Poison the sidecar: cross-wire a's Git CID to b's blake3.
460        let index_path = dir.path().join(INDEX_FILE);
461        let contents = tokio::fs::read_to_string(&index_path).await?;
462        let mut b3 = HashMap::new();
463        for line in contents.lines() {
464            if let Some((cid, blake)) = line.split_once(' ') {
465                b3.insert(cid.to_owned(), blake.to_owned());
466            }
467        }
468        let cid_a = ContentCid::git_sha256(&sha_a)?;
469        let cid_b = ContentCid::git_sha256(&sha_b)?;
470        let poisoned = format!(
471            "{} {}\n{} {}\n",
472            cid_a,
473            b3[cid_b.as_str()],
474            cid_b,
475            b3[cid_a.as_str()]
476        );
477        tokio::fs::write(&index_path, poisoned).await?;
478
479        let store = IrohBlobStore::open_fs(dir.path()).await?;
480        // Both lookups resolve to real blobs, but the bytes fail SHA256
481        // re-verification — the store must error, never serve wrong bytes.
482        assert!(store.get_object(&sha_a).await.is_err());
483        assert!(store.get_object(&sha_b).await.is_err());
484        store.shutdown().await?;
485        Ok(())
486    }
487
488    #[tokio::test]
489    async fn test_malformed_index_lines_skipped() -> Result<()> {
490        let dir = TempDir::new()?;
491        let data = b"good object".to_vec();
492
493        let sha = {
494            let store = IrohBlobStore::open_fs(dir.path()).await?;
495            let sha = store.put_object(data.clone()).await?;
496            store.shutdown().await?;
497            sha
498        };
499
500        // Prepend garbage lines to the sidecar.
501        let index_path = dir.path().join(INDEX_FILE);
502        let contents = tokio::fs::read_to_string(&index_path).await?;
503        tokio::fs::write(
504            &index_path,
505            format!("not an index line\ndeadbeef tooshort\n{contents}"),
506        )
507        .await?;
508
509        let store = IrohBlobStore::open_fs(dir.path()).await?;
510        let fetched = store.get_object(&sha).await?;
511        assert_eq!(fetched.as_deref(), Some(data.as_slice()));
512        assert_eq!(store.len().await, 1);
513        store.shutdown().await?;
514        Ok(())
515    }
516}