1use std::collections::HashMap;
37use std::path::{Path, PathBuf};
38
39use iroh_blobs::store::fs::FsStore;
40use iroh_blobs::store::mem::MemStore;
41use iroh_blobs::Hash as Blake3Hash;
42use tokio::io::AsyncWriteExt;
43use tokio::sync::Mutex;
44
45use crate::crypto::hash::{sha256_git, verify_sha256};
46use crate::{ContentCid, Error, Result, Sha256Hash};
47
48const INDEX_FILE: &str = "sha256-blake3.index";
50
51enum Backend {
52 Mem(MemStore),
53 Fs(FsStore),
54}
55
56impl Backend {
57 fn store(&self) -> &iroh_blobs::api::Store {
58 match self {
59 Backend::Mem(s) => s,
60 Backend::Fs(s) => s,
61 }
62 }
63}
64
65pub struct IrohBlobStore {
69 backend: Backend,
70 index: Mutex<HashMap<ContentCid, Blake3Hash>>,
72 index_path: Option<PathBuf>,
74}
75
76impl IrohBlobStore {
77 pub fn new_memory() -> Self {
79 Self {
80 backend: Backend::Mem(MemStore::new()),
81 index: Mutex::new(HashMap::new()),
82 index_path: None,
83 }
84 }
85
86 pub async fn open_fs(root: impl AsRef<Path>) -> Result<Self> {
91 let root = root.as_ref();
92 tokio::fs::create_dir_all(root).await?;
93 let store = FsStore::load(root)
94 .await
95 .map_err(|e| Error::other(format!("failed to open iroh-blobs store: {e}")))?;
96
97 let index_path = root.join(INDEX_FILE);
98 let index = load_index(&index_path).await?;
99
100 Ok(Self {
101 backend: Backend::Fs(store),
102 index: Mutex::new(index),
103 index_path: Some(index_path),
104 })
105 }
106
107 pub async fn put_object(&self, data: Vec<u8>) -> Result<Sha256Hash> {
113 let sha256 = sha256_git(&data)?;
114 self.put_verified_object(sha256.clone(), data).await?;
115 Ok(sha256)
116 }
117
118 pub async fn put_verified_object(
124 &self,
125 expected_sha256: Sha256Hash,
126 data: Vec<u8>,
127 ) -> Result<Sha256Hash> {
128 if !verify_sha256(&data, &expected_sha256)? {
129 return Err(Error::other(format!(
130 "sha256 verification failed for fetched object {expected_sha256}"
131 )));
132 }
133
134 let cid = ContentCid::git_sha256(&expected_sha256)?;
135 self.put_object_by_cid(cid, data).await?;
136 Ok(expected_sha256)
137 }
138
139 pub async fn put_object_by_cid(&self, cid: ContentCid, data: Vec<u8>) -> Result<()> {
146 verify_raw_cid_when_applicable(&cid, &data)?;
147 let raw_hash = Blake3Hash::new(&data);
148 if let Some(existing) = self.index.lock().await.get(&cid).copied() {
149 if existing != raw_hash {
150 return Err(Error::other(format!(
151 "content CID {cid} is already bound to a different raw blob"
152 )));
153 }
154 }
155
156 let tag = self
157 .backend
158 .store()
159 .blobs()
160 .add_bytes(data)
161 .await
162 .map_err(|e| Error::other(format!("iroh-blobs add failed: {e}")))?;
163
164 let mut index = self.index.lock().await;
165 match index.get(&cid) {
166 Some(existing) if *existing != tag.hash => {
167 return Err(Error::other(format!(
168 "content CID {cid} is already bound to a different raw blob"
169 )));
170 }
171 Some(_) => {}
172 None => {
173 if let Some(path) = &self.index_path {
178 append_index_entry(path, &cid, &tag.hash).await?;
179 }
180 index.insert(cid.clone(), tag.hash);
181 }
182 }
183 drop(index);
184
185 tracing::debug!(content_cid = %cid, blake3 = %tag.hash, "stored object");
186 Ok(())
187 }
188
189 pub async fn get_object(&self, hash: &Sha256Hash) -> Result<Option<Vec<u8>>> {
195 let cid = ContentCid::git_sha256(hash)?;
196 self.get_object_by_cid(&cid).await
197 }
198
199 pub async fn get_object_by_cid(&self, cid: &ContentCid) -> Result<Option<Vec<u8>>> {
201 let blake3 = {
202 let index = self.index.lock().await;
203 match index.get(cid) {
204 Some(b3) => *b3,
205 None => return Ok(None),
206 }
207 };
208
209 let bytes = match self.backend.store().blobs().get_bytes(blake3).await {
210 Ok(bytes) => bytes,
211 Err(e) => {
212 tracing::warn!("indexed blob {blake3} missing from store for CID {cid}: {e}");
215 return Ok(None);
216 }
217 };
218
219 let data = bytes.to_vec();
220 verify_raw_cid_when_applicable(cid, &data)?;
221 Ok(Some(data))
222 }
223
224 pub async fn blake3_of(&self, hash: &Sha256Hash) -> Option<Blake3Hash> {
229 let cid = ContentCid::git_sha256(hash).ok()?;
230 self.blake3_of_cid(&cid).await
231 }
232
233 pub async fn blake3_of_cid(&self, cid: &ContentCid) -> Option<Blake3Hash> {
235 self.index.lock().await.get(cid).copied()
236 }
237
238 pub async fn len(&self) -> usize {
240 self.index.lock().await.len()
241 }
242
243 pub async fn is_empty(&self) -> bool {
245 self.index.lock().await.is_empty()
246 }
247
248 pub async fn shutdown(&self) -> Result<()> {
254 self.backend
255 .store()
256 .shutdown()
257 .await
258 .map_err(|e| Error::other(format!("iroh-blobs shutdown failed: {e}")))
259 }
260}
261
262async fn load_index(path: &Path) -> Result<HashMap<ContentCid, Blake3Hash>> {
264 let mut index = HashMap::new();
265 let contents = match tokio::fs::read_to_string(path).await {
266 Ok(c) => c,
267 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(index),
268 Err(e) => return Err(e.into()),
269 };
270
271 for (lineno, line) in contents.lines().enumerate() {
272 let line = line.trim();
273 if line.is_empty() {
274 continue;
275 }
276 let parsed = line.split_once(' ').and_then(|(key, b3_hex)| {
277 let cid = ContentCid::new(key)
278 .or_else(|_| {
279 let sha = Sha256Hash::new(key)?;
281 ContentCid::git_sha256(&sha)
282 })
283 .ok()?;
284 let b3 = b3_hex.parse::<Blake3Hash>().ok()?;
285 Some((cid, b3))
286 });
287 match parsed {
288 Some((cid, b3)) => {
289 index.insert(cid, b3);
290 }
291 None => {
292 tracing::warn!(
293 "skipping malformed index line {} in {}",
294 lineno + 1,
295 path.display()
296 );
297 }
298 }
299 }
300 Ok(index)
301}
302
303async fn append_index_entry(path: &Path, cid: &ContentCid, blake3: &Blake3Hash) -> Result<()> {
305 let mut file = tokio::fs::OpenOptions::new()
306 .create(true)
307 .append(true)
308 .open(path)
309 .await?;
310 file.write_all(format!("{cid} {blake3}\n").as_bytes())
311 .await?;
312 file.flush().await?;
313 Ok(())
314}
315
316fn verify_raw_cid_when_applicable(cid: &ContentCid, data: &[u8]) -> Result<()> {
317 let decoded = cid.decoded()?;
318 use hyprstream_rpc::cid::{Codec, HashAlgo};
319
320 match (decoded.codec, decoded.multihash.algo) {
321 (Codec::GitRaw, HashAlgo::Sha2_256) => {
322 let expected = Sha256Hash::from_bytes(&decoded.multihash.digest)?;
323 if !verify_sha256(data, &expected)? {
324 return Err(Error::other(format!(
325 "sha256 verification failed for Git object CID {cid}"
326 )));
327 }
328 Ok(())
329 }
330 (Codec::XetShard, HashAlgo::Blake3) if decoded.multihash.digest.len() == 32 => Ok(()),
331 (codec, algo) => Err(Error::other(format!(
332 "unsupported object CID domain: codec={codec:?}, algorithm={algo:?}"
333 ))),
334 }
335}
336
337#[cfg(test)]
338mod tests {
339 use super::*;
340 use tempfile::TempDir;
341
342 #[tokio::test]
343 async fn test_memory_roundtrip() -> Result<()> {
344 let store = IrohBlobStore::new_memory();
345 let data = b"hello iroh-blobs".to_vec();
346
347 let hash = store.put_object(data.clone()).await?;
348 assert_eq!(hash, sha256_git(&data)?);
350
351 let fetched = store.get_object(&hash).await?;
352 assert_eq!(fetched.as_deref(), Some(data.as_slice()));
353 assert_eq!(store.len().await, 1);
354 Ok(())
355 }
356
357 #[tokio::test]
358 async fn test_unknown_hash_is_none() -> Result<()> {
359 let store = IrohBlobStore::new_memory();
360 let unknown = sha256_git(b"never stored")?;
361 assert!(store.get_object(&unknown).await?.is_none());
362 Ok(())
363 }
364
365 #[tokio::test]
366 async fn test_put_is_idempotent() -> Result<()> {
367 let store = IrohBlobStore::new_memory();
368 let data = b"same bytes twice".to_vec();
369 let h1 = store.put_object(data.clone()).await?;
370 let h2 = store.put_object(data.clone()).await?;
371 assert_eq!(h1, h2);
372 assert_eq!(store.len().await, 1);
373 Ok(())
374 }
375
376 #[tokio::test]
377 async fn test_blake3_mapping_exposed() -> Result<()> {
378 let store = IrohBlobStore::new_memory();
379 let data = b"mapped".to_vec();
380 let sha = store.put_object(data.clone()).await?;
381
382 let b3 = store
383 .blake3_of(&sha)
384 .await
385 .ok_or_else(|| Error::not_found("missing mapping"))?;
386 assert_eq!(b3, Blake3Hash::new(data.as_slice()));
389 Ok(())
390 }
391
392 #[tokio::test]
393 async fn test_put_verified_object_rejects_wrong_sha256() -> Result<()> {
394 let store = IrohBlobStore::new_memory();
395 let expected = sha256_git(b"expected")?;
396
397 assert!(store
398 .put_verified_object(expected.clone(), b"different".to_vec())
399 .await
400 .is_err());
401 assert!(store.get_object(&expected).await?.is_none());
402 Ok(())
403 }
404
405 #[tokio::test]
406 async fn test_xet_cid_cannot_be_rebound_to_different_bytes() -> Result<()> {
407 let store = IrohBlobStore::new_memory();
408 let cid = ContentCid::xet_merkle(&[0x7b; 32])?;
409
410 store
411 .put_object_by_cid(cid.clone(), b"first reconstruction".to_vec())
412 .await?;
413 assert!(store
414 .put_object_by_cid(cid.clone(), b"different reconstruction".to_vec())
415 .await
416 .is_err());
417 assert_eq!(
418 store.get_object_by_cid(&cid).await?.as_deref(),
419 Some(b"first reconstruction".as_slice())
420 );
421 Ok(())
422 }
423
424 #[tokio::test]
425 async fn test_fs_persistence_across_reopen() -> Result<()> {
426 let dir = TempDir::new()?;
427 let data = b"persisted across reopen".to_vec();
428
429 let sha = {
430 let store = IrohBlobStore::open_fs(dir.path()).await?;
431 let sha = store.put_object(data.clone()).await?;
432 store.shutdown().await?;
433 sha
434 };
435
436 let store = IrohBlobStore::open_fs(dir.path()).await?;
437 let fetched = store.get_object(&sha).await?;
438 assert_eq!(fetched.as_deref(), Some(data.as_slice()));
439 store.shutdown().await?;
440 Ok(())
441 }
442
443 #[tokio::test]
444 async fn test_poisoned_index_never_serves_wrong_bytes() -> Result<()> {
445 let dir = TempDir::new()?;
446 let a = b"object a".to_vec();
447 let b = b"object b".to_vec();
448
449 let (sha_a, sha_b) = {
450 let store = IrohBlobStore::open_fs(dir.path()).await?;
451 let pair = (
452 store.put_object(a.clone()).await?,
453 store.put_object(b.clone()).await?,
454 );
455 store.shutdown().await?;
456 pair
457 };
458
459 let index_path = dir.path().join(INDEX_FILE);
461 let contents = tokio::fs::read_to_string(&index_path).await?;
462 let mut b3 = HashMap::new();
463 for line in contents.lines() {
464 if let Some((cid, blake)) = line.split_once(' ') {
465 b3.insert(cid.to_owned(), blake.to_owned());
466 }
467 }
468 let cid_a = ContentCid::git_sha256(&sha_a)?;
469 let cid_b = ContentCid::git_sha256(&sha_b)?;
470 let poisoned = format!(
471 "{} {}\n{} {}\n",
472 cid_a,
473 b3[cid_b.as_str()],
474 cid_b,
475 b3[cid_a.as_str()]
476 );
477 tokio::fs::write(&index_path, poisoned).await?;
478
479 let store = IrohBlobStore::open_fs(dir.path()).await?;
480 assert!(store.get_object(&sha_a).await.is_err());
483 assert!(store.get_object(&sha_b).await.is_err());
484 store.shutdown().await?;
485 Ok(())
486 }
487
488 #[tokio::test]
489 async fn test_malformed_index_lines_skipped() -> Result<()> {
490 let dir = TempDir::new()?;
491 let data = b"good object".to_vec();
492
493 let sha = {
494 let store = IrohBlobStore::open_fs(dir.path()).await?;
495 let sha = store.put_object(data.clone()).await?;
496 store.shutdown().await?;
497 sha
498 };
499
500 let index_path = dir.path().join(INDEX_FILE);
502 let contents = tokio::fs::read_to_string(&index_path).await?;
503 tokio::fs::write(
504 &index_path,
505 format!("not an index line\ndeadbeef tooshort\n{contents}"),
506 )
507 .await?;
508
509 let store = IrohBlobStore::open_fs(dir.path()).await?;
510 let fetched = store.get_object(&sha).await?;
511 assert_eq!(fetched.as_deref(), Some(data.as_slice()));
512 assert_eq!(store.len().await, 1);
513 store.shutdown().await?;
514 Ok(())
515 }
516}