Skip to main content

cas_serve/
store.rs

1//! Content-addressed store: filesystem-backed XET reconstruction.
2//!
3//! This is the read/write core extracted from the `cas-serve` binary so it can
4//! be reused **in-process** by embedders (e.g. the registry service's `getBlob`)
5//! without spawning a subprocess or reimplementing reconstruction.
6//!
7//! Reconstruction walks the file's `.mdb` reconstruction shard, fetches each
8//! referenced xorb, and concatenates the segment bytes — the same algorithm the
9//! binary serves over its NDJSON `GetFile` protocol.
10
11use crate::protocol::Request;
12use crate::shard::Shard;
13use std::path::{Path, PathBuf};
14
15/// Errors from store reconstruction.
16#[derive(Debug, thiserror::Error)]
17pub enum StoreError {
18    #[error("invalid hash: {0}")]
19    InvalidHash(String),
20    #[error("not found: {0}")]
21    NotFound(String),
22    #[error("corrupt shard: {0}")]
23    CorruptShard(String),
24    #[error("io error: {0}")]
25    Io(String),
26}
27
28/// Result of ingesting bytes via [`CasStore::put_file_bytes`].
29#[derive(Debug, Clone)]
30pub struct PutResult {
31    /// Server-computed XET merkle root (hex) of the ingested bytes. Authoritative
32    /// — derived from the actual content, never supplied by the caller.
33    pub merkle: String,
34    /// Hex xorb hashes forming the reconstruction set for this content.
35    pub xorb_hashes: Vec<String>,
36    /// Bytes newly written to the store after content-addressed dedup
37    /// (0 if every xorb was already present — fully deduplicated).
38    pub bytes_stored: u64,
39}
40
41/// Storage path resolution order (matches the `cas-serve` binary):
42/// 1. `CAS_STORAGE` environment variable
43/// 2. `XET_CACHE_DIR` environment variable
44/// 3. `~/.cache/xet/` (default)
45pub fn resolve_storage_path() -> PathBuf {
46    if let Ok(path) = std::env::var("CAS_STORAGE") {
47        return PathBuf::from(shellexpand::tilde(&path).into_owned());
48    }
49    if let Ok(path) = std::env::var("XET_CACHE_DIR") {
50        return PathBuf::from(shellexpand::tilde(&path).into_owned());
51    }
52    dirs::cache_dir()
53        .unwrap_or_else(|| PathBuf::from("."))
54        .join("xet")
55}
56
57/// Filesystem-backed content-addressed store.
58///
59/// Storage layout (identical to the `cas-serve` binary):
60/// - `xorbs/default.{xorb_hash}` — raw xorb bytes (concatenated chunks).
61/// - `shards/{file_hash}` — reconstruction shard (`.mdb` binary).
62#[derive(Debug, Clone)]
63pub struct CasStore {
64    storage_path: PathBuf,
65}
66
67impl CasStore {
68    /// Open a store rooted at `storage_path`.
69    pub fn new(storage_path: impl AsRef<Path>) -> Self {
70        Self {
71            storage_path: storage_path.as_ref().to_path_buf(),
72        }
73    }
74
75    /// Open a store using the standard env/default path resolution.
76    pub fn from_env() -> Self {
77        Self::new(resolve_storage_path())
78    }
79
80    /// Path for a raw xorb, keyed by its hex xorb hash.
81    fn xorb_path(&self, xorb_hash_hex: &str) -> PathBuf {
82        self.storage_path
83            .join("xorbs")
84            .join(format!("default.{xorb_hash_hex}"))
85    }
86
87    /// Path for a reconstruction shard, keyed by the file's hex merkle hash.
88    fn shard_path(&self, file_hash_hex: &str) -> PathBuf {
89        self.storage_path.join("shards").join(file_hash_hex)
90    }
91
92    /// True if the store can serve this content address (shard or raw blob).
93    pub fn exists(&self, hash: &str) -> bool {
94        self.shard_path(hash).exists() || self.xorb_path(hash).exists()
95    }
96
97    /// Reassemble a file from its reconstruction shard (or raw-blob fallback) and
98    /// return the original bytes. This is the in-process equivalent of the
99    /// binary's `GetFile` handler.
100    pub async fn get_file_bytes(&self, hash: &str) -> Result<Vec<u8>, StoreError> {
101        let _file_hash = Request::parse_hash(hash).map_err(StoreError::InvalidHash)?;
102
103        // 1. Reconstruction shard (chunked / multi-xorb path).
104        let shard_path = self.shard_path(hash);
105        if shard_path.exists() {
106            return self.reassemble_from_shard(hash, &shard_path).await;
107        }
108
109        // 2. Fall back to a directly-stored raw blob (single-chunk file uploaded
110        //    via the legacy whole-blob path, where the xorb hash == file hash).
111        let xorb_path = self.xorb_path(hash);
112        match tokio::fs::read(&xorb_path).await {
113            Ok(data) => Ok(data),
114            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
115                Err(StoreError::NotFound(format!("File not found: {hash}")))
116            }
117            Err(e) => Err(StoreError::Io(format!("Failed to read file: {e}"))),
118        }
119    }
120
121    /// Ingest raw bytes: chunk (Gearhash CDC), aggregate into xorbs, build the
122    /// reconstruction shard, and store both content-addressed. The XET merkle
123    /// root is computed HERE from the actual bytes and returned in
124    /// [`PutResult::merkle`] — the caller never supplies it. Existing xorbs are
125    /// skipped (global content-addressed dedup), so [`PutResult::bytes_stored`]
126    /// counts only newly-written xorb payload.
127    ///
128    /// This is the in-process, library form of the binary's `UploadFile`
129    /// handler; both go through this single implementation.
130    pub async fn put_file_bytes(&self, data: &[u8]) -> Result<PutResult, StoreError> {
131        use crate::chunker;
132        use crate::shard::Shard;
133
134        // Server-side chunk + merkle: the file_hash is derived from the bytes,
135        // not asserted by the caller (this is what closes the OID-planting vector
136        // for the authenticated write path — see hyprstream registry putBlob).
137        let chunks = chunker::chunk_all(data);
138        let (shard, xorbs) = Shard::from_chunks(&chunks);
139
140        let xorbs_dir = self.storage_path.join("xorbs");
141        let shards_dir = self.storage_path.join("shards");
142        tokio::fs::create_dir_all(&xorbs_dir)
143            .await
144            .map_err(|e| StoreError::Io(format!("create xorbs dir: {e}")))?;
145        tokio::fs::create_dir_all(&shards_dir)
146            .await
147            .map_err(|e| StoreError::Io(format!("create shards dir: {e}")))?;
148
149        // Store each xorb content-addressed. An identical xorb already present is
150        // not an error (dedup): skip the write and don't count it.
151        let mut xorb_hashes = Vec::with_capacity(xorbs.len());
152        let mut bytes_stored: u64 = 0;
153        for (xorb_hash, bytes) in &xorbs {
154            let hex = xorb_hash.hex();
155            let path = self.xorb_path(&hex);
156            if !path.exists() {
157                tokio::fs::write(&path, bytes)
158                    .await
159                    .map_err(|e| StoreError::Io(format!("write xorb {hex}: {e}")))?;
160                bytes_stored += bytes.len() as u64;
161            }
162            xorb_hashes.push(hex);
163        }
164
165        // Reconstruction shard (`.mdb`), keyed by the file merkle. Small metadata,
166        // identical for identical content, so an unconditional (re)write is fine.
167        let shard_path = self.shard_path(&shard.file_hash);
168        tokio::fs::write(&shard_path, shard.to_bytes())
169            .await
170            .map_err(|e| StoreError::Io(format!("write shard {}: {e}", shard.file_hash)))?;
171
172        Ok(PutResult {
173            merkle: shard.file_hash,
174            xorb_hashes,
175            bytes_stored,
176        })
177    }
178
179    /// Read the raw bytes of a single stored xorb, keyed by its hex xorb hash.
180    ///
181    /// Unlike [`CasStore::get_file_bytes`] this does NOT reconstruct a file: it
182    /// returns the exact on-disk `xorbs/default.{hash}` payload. This backs the
183    /// HuggingFace-XET `GET /get_xorb/{hash}/` wire route, which fetches an
184    /// individual xorb (not a reconstructed file). Callers wanting a byte
185    /// sub-range should slice the returned buffer (the store keeps whole xorbs).
186    pub async fn read_xorb(&self, xorb_hash_hex: &str) -> Result<Vec<u8>, StoreError> {
187        let _ = Request::parse_hash(xorb_hash_hex).map_err(StoreError::InvalidHash)?;
188        let path = self.xorb_path(xorb_hash_hex);
189        match tokio::fs::read(&path).await {
190            Ok(data) => Ok(data),
191            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(StoreError::NotFound(
192                format!("Xorb not found: {xorb_hash_hex}"),
193            )),
194            Err(e) => Err(StoreError::Io(format!("Failed to read xorb: {e}"))),
195        }
196    }
197
198    /// Load the `.mdb` shard, fetch each referenced xorb, and concatenate the
199    /// segment bytes to reconstruct the file.
200    async fn reassemble_from_shard(
201        &self,
202        file_hash: &str,
203        shard_path: &Path,
204    ) -> Result<Vec<u8>, StoreError> {
205        let mdb_bytes = match tokio::fs::read(shard_path).await {
206            Ok(b) => b,
207            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
208                return Err(StoreError::NotFound(format!(
209                    "Shard not found: {file_hash}"
210                )))
211            }
212            Err(e) => return Err(StoreError::Io(format!("Failed to read shard: {e}"))),
213        };
214
215        let file_hash_parsed = Request::parse_hash(file_hash).map_err(StoreError::InvalidHash)?;
216
217        let segments = Shard::segments(&mdb_bytes, &file_hash_parsed)
218            .map_err(|e| StoreError::CorruptShard(e.to_string()))?;
219
220        let expected_len: u64 = segments.iter().map(|s| s.byte_len).sum();
221        let mut out = Vec::with_capacity(expected_len as usize);
222        for seg in &segments {
223            let xorb_hex = seg.xorb_hash.hex();
224            let xorb_path = self.xorb_path(&xorb_hex);
225            let xorb_bytes = match tokio::fs::read(&xorb_path).await {
226                Ok(b) => b,
227                Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
228                    return Err(StoreError::NotFound(format!(
229                        "Missing xorb {xorb_hex} for file {file_hash}"
230                    )))
231                }
232                Err(e) => {
233                    return Err(StoreError::Io(format!(
234                        "Failed to read xorb {xorb_hex}: {e}"
235                    )))
236                }
237            };
238            // Raw concatenated xorbs: every segment spans the whole xorb
239            // (chunk_start=0). Slice defensively for partial spans.
240            let len = (seg.byte_len as usize).min(xorb_bytes.len());
241            out.extend_from_slice(&xorb_bytes[..len]);
242        }
243
244        if out.len() as u64 != expected_len {
245            return Err(StoreError::CorruptShard(format!(
246                "Reconstructed {} bytes but shard expected {expected_len} for {file_hash}",
247                out.len()
248            )));
249        }
250
251        Ok(out)
252    }
253}
254
255#[cfg(test)]
256#[allow(clippy::unwrap_used, clippy::expect_used)] // panicking is correct in unit tests
257mod tests {
258    use super::*;
259
260    // A payload large enough to exercise CDC into multiple chunks/xorbs.
261    fn payload(seed: u8, len: usize) -> Vec<u8> {
262        (0..len)
263            .map(|i| seed.wrapping_add((i as u8).wrapping_mul(31)))
264            .collect()
265    }
266
267    #[tokio::test]
268    async fn put_then_get_round_trips_and_computes_merkle() {
269        let dir = tempfile::tempdir().unwrap();
270        let store = CasStore::new(dir.path());
271        let original = payload(7, 512 * 1024);
272
273        let put = store.put_file_bytes(&original).await.unwrap();
274        // Merkle is server-computed and non-empty; content is content-addressable.
275        assert!(!put.merkle.is_empty(), "server must compute a merkle");
276        assert!(!put.xorb_hashes.is_empty());
277        assert!(put.bytes_stored > 0, "first upload writes new bytes");
278
279        // The store can now reconstruct the exact original bytes by that merkle.
280        let got = store.get_file_bytes(&put.merkle).await.unwrap();
281        assert_eq!(got, original);
282    }
283
284    #[tokio::test]
285    async fn merkle_is_deterministic_and_reupload_dedupes() {
286        let dir = tempfile::tempdir().unwrap();
287        let store = CasStore::new(dir.path());
288        let original = payload(42, 300 * 1024);
289
290        let first = store.put_file_bytes(&original).await.unwrap();
291        let second = store.put_file_bytes(&original).await.unwrap();
292
293        // Same bytes ⇒ same merkle (content-addressed, caller-independent).
294        assert_eq!(first.merkle, second.merkle);
295        assert_eq!(first.xorb_hashes, second.xorb_hashes);
296        // Re-upload of identical content writes nothing new (global dedup).
297        assert_eq!(second.bytes_stored, 0, "re-upload must fully dedup");
298    }
299
300    #[tokio::test]
301    async fn distinct_content_yields_distinct_merkle() {
302        let dir = tempfile::tempdir().unwrap();
303        let store = CasStore::new(dir.path());
304
305        let a = store.put_file_bytes(&payload(1, 200 * 1024)).await.unwrap();
306        let b = store.put_file_bytes(&payload(2, 200 * 1024)).await.unwrap();
307        assert_ne!(a.merkle, b.merkle);
308    }
309}