hyprstream_core/storage/cas/
substrate.rs1use std::fs;
10use std::path::{Path, PathBuf};
11
12use cas_serve::CasStore;
13use hyprstream_rpc::auth::mac::SecurityLabel;
14use hyprstream_rpc::cid::{decode_cid, encode_cid, Codec, HashAlgo};
15
16use super::domain::DedupDomain;
17use super::manifest::{looks_like_legacy_hex, merkle_from_address, BlobManifest};
18use super::CasError;
19
20const fn hex_digest_len(digest_length: u16) -> usize {
22 digest_length as usize * 2
23}
24
25#[derive(Debug, Clone)]
30pub struct CasSubstrate {
31 root: PathBuf,
32}
33
34impl CasSubstrate {
35 pub fn new(root: impl AsRef<Path>) -> Self {
37 Self {
38 root: root.as_ref().to_path_buf(),
39 }
40 }
41
42 pub fn from_env() -> Self {
46 Self::new(cas_serve::store::resolve_storage_path())
47 }
48
49 fn store_for(&self, domain: &DedupDomain) -> CasStore {
51 CasStore::new(self.root.join(domain.relative_path()))
52 }
53
54 fn manifest_dir(&self, domain: &DedupDomain) -> PathBuf {
55 self.root.join(domain.relative_path()).join("manifests")
56 }
57
58 fn manifest_path(&self, domain: &DedupDomain, cid: &str) -> PathBuf {
59 self.manifest_dir(domain).join(format!("{cid}.json"))
60 }
61
62 fn canonical_cid(address: &str) -> Result<String, CasError> {
63 let decoded = decode_cid(address).map_err(|e| CasError::Cid(e.to_string()))?;
64 encode_cid(
65 decoded.codec,
66 decoded.multihash.algo,
67 &decoded.multihash.digest,
68 )
69 .map_err(|e| CasError::Cid(e.to_string()))
70 }
71
72 fn persist_manifest(
73 &self,
74 domain: &DedupDomain,
75 manifest: &BlobManifest,
76 ) -> Result<(), CasError> {
77 let dir = self.manifest_dir(domain);
78 fs::create_dir_all(&dir)
79 .map_err(|e| CasError::Manifest(format!("create {}: {e}", dir.display())))?;
80 let bytes = serde_json::to_vec(manifest)
81 .map_err(|e| CasError::Manifest(format!("serialize manifest: {e}")))?;
82 let path = self.manifest_path(domain, &manifest.cid);
83 fs::write(&path, bytes)
84 .map_err(|e| CasError::Manifest(format!("write {}: {e}", path.display())))
85 }
86
87 fn load_manifest(&self, domain: &DedupDomain, address: &str) -> Result<BlobManifest, CasError> {
88 let cid = Self::canonical_cid(address)?;
89 let decoded = decode_cid(&cid).map_err(|e| CasError::Cid(e.to_string()))?;
90 if decoded.codec != Codec::XetManifest {
91 return Err(CasError::Cid(
92 "CID does not address a labeled CAS manifest".into(),
93 ));
94 }
95 let path = self.manifest_path(domain, &cid);
96 let bytes = fs::read(&path)
97 .map_err(|e| CasError::Manifest(format!("read {}: {e}", path.display())))?;
98 let manifest: BlobManifest = serde_json::from_slice(&bytes)
99 .map_err(|e| CasError::Manifest(format!("decode {}: {e}", path.display())))?;
100 if manifest.cid != cid
101 || !hyprstream_rpc::auth::mac::ContentBoundLabel::verify_binding(&manifest)
102 {
103 return Err(CasError::Manifest(format!(
104 "CID binding verification failed for {cid}"
105 )));
106 }
107 Ok(manifest)
108 }
109
110 pub async fn put(
134 &self,
135 domain: &DedupDomain,
136 data: &[u8],
137 hint_label: Option<SecurityLabel>,
138 ) -> Result<BlobManifest, CasError> {
139 if domain.algorithm != HashAlgo::Blake3 || domain.digest_length != 32 {
140 return Err(CasError::UnsupportedIngestAlgorithm(domain.algorithm));
141 }
142 let store = self.store_for(domain);
143 let put = store.put_file_bytes(data).await?;
144 debug_assert_eq!(
147 put.merkle.len(),
148 hex_digest_len(domain.digest_length),
149 "store merkle must be the declared {}-byte digest",
150 domain.digest_length
151 );
152 let security_label = crate::mac::cas_pep::seal_label(domain, hint_label);
155 let manifest = BlobManifest::new(
156 put.merkle,
157 put.xorb_hashes,
158 put.bytes_stored,
159 data.len() as u64,
160 security_label,
161 )?;
162 self.persist_manifest(domain, &manifest)?;
163 Ok(manifest)
164 }
165
166 pub async fn get(&self, domain: &DedupDomain, address: &str) -> Result<Vec<u8>, CasError> {
171 let merkle = if looks_like_legacy_hex(address) {
172 merkle_from_address(address)?
173 } else {
174 self.load_manifest(domain, address)?.merkle
175 };
176 let store = self.store_for(domain);
177 Ok(store.get_file_bytes(&merkle).await?)
178 }
179
180 pub fn manifest(&self, domain: &DedupDomain, cid: &str) -> Result<BlobManifest, CasError> {
182 self.load_manifest(domain, cid)
183 }
184
185 pub async fn read_xorb(
188 &self,
189 domain: &DedupDomain,
190 xorb_hash_hex: &str,
191 ) -> Result<Vec<u8>, CasError> {
192 let store = self.store_for(domain);
193 Ok(store.read_xorb(xorb_hash_hex).await?)
194 }
195
196 pub fn exists(&self, domain: &DedupDomain, address: &str) -> bool {
198 let merkle = if looks_like_legacy_hex(address) {
199 merkle_from_address(address)
200 } else {
201 self.load_manifest(domain, address)
202 .map(|manifest| manifest.merkle)
203 };
204 merkle.is_ok_and(|merkle| self.store_for(domain).exists(&merkle))
205 }
206}
207
208#[cfg(test)]
209#[allow(clippy::unwrap_used, clippy::expect_used)]
210mod tests {
211 use super::*;
212 use hyprstream_rpc::auth::mac::{Assurance, Compartment, CompartmentSet, Level};
213
214 fn label() -> SecurityLabel {
215 SecurityLabel::new(Level::Internal, Assurance::Classical, CompartmentSet::EMPTY)
216 }
217
218 fn payload(seed: u8, len: usize) -> Vec<u8> {
219 (0..len)
220 .map(|i| seed.wrapping_add((i as u8).wrapping_mul(31)))
221 .collect()
222 }
223
224 #[tokio::test]
225 async fn put_get_round_trips_via_cid_and_legacy_hex() {
226 let dir = tempfile::tempdir().unwrap();
227 let sub = CasSubstrate::new(dir.path());
228 let domain = DedupDomain::local_default();
229 let original = payload(7, 256 * 1024);
230
231 let m = sub.put(&domain, &original, Some(label())).await.unwrap();
232 assert!(m.cid.starts_with('b'));
233 assert_eq!(m.byte_len, original.len() as u64);
234 assert!(m.bytes_stored > 0);
235
236 assert_eq!(sub.get(&domain, &m.cid).await.unwrap(), original);
238 assert_eq!(sub.get(&domain, &m.merkle).await.unwrap(), original);
240 assert!(sub.exists(&domain, &m.cid));
241 }
242
243 #[tokio::test]
244 async fn default_domain_matches_legacy_casstore_layout() {
245 let dir = tempfile::tempdir().unwrap();
248 let sub = CasSubstrate::new(dir.path());
249 let original = payload(3, 200 * 1024);
250
251 let m = sub
252 .put(&DedupDomain::local_default(), &original, Some(label()))
253 .await
254 .unwrap();
255
256 let legacy = CasStore::new(dir.path());
257 assert_eq!(legacy.get_file_bytes(&m.merkle).await.unwrap(), original);
258 }
259
260 #[tokio::test]
261 async fn distinct_domains_do_not_dedup_together() {
262 let dir = tempfile::tempdir().unwrap();
265 let sub = CasSubstrate::new(dir.path());
266 let original = payload(9, 200 * 1024);
267
268 let default_domain = DedupDomain::local_default();
269 let tenant_domain = DedupDomain::local_tenant(Compartment::new("tenant:acme"));
270
271 let a = sub
272 .put(&default_domain, &original, Some(label()))
273 .await
274 .unwrap();
275 let b = sub
276 .put(&tenant_domain, &original, Some(label()))
277 .await
278 .unwrap();
279
280 assert_eq!(a.merkle, b.merkle);
282 assert!(
284 b.bytes_stored > 0,
285 "cross-domain content must not deduplicate: existence/content leak"
286 );
287
288 assert!(sub.exists(&tenant_domain, &b.merkle));
291 }
292
293 #[tokio::test]
294 async fn same_domain_reupload_dedups() {
295 let dir = tempfile::tempdir().unwrap();
296 let sub = CasSubstrate::new(dir.path());
297 let domain = DedupDomain::local_default();
298 let original = payload(11, 200 * 1024);
299
300 let first = sub.put(&domain, &original, Some(label())).await.unwrap();
301 let second = sub.put(&domain, &original, Some(label())).await.unwrap();
302 assert_eq!(first.merkle, second.merkle);
303 assert_eq!(second.bytes_stored, 0, "same-domain re-upload fully dedups");
304 }
305
306 #[tokio::test]
307 async fn non_blake3_ingest_rejected() {
308 let dir = tempfile::tempdir().unwrap();
309 let sub = CasSubstrate::new(dir.path());
310 let domain = DedupDomain {
311 algorithm: HashAlgo::Sha2_256,
312 digest_length: 32,
313 ..DedupDomain::local_default()
314 };
315 let err = sub.put(&domain, b"hello", Some(label())).await.unwrap_err();
316 assert!(matches!(err, CasError::UnsupportedIngestAlgorithm(_)));
317 }
318
319 #[tokio::test]
320 async fn blake3_512_ingest_not_yet_supported() {
321 let dir = tempfile::tempdir().unwrap();
326 let sub = CasSubstrate::new(dir.path());
327 let domain = DedupDomain {
328 digest_length: 64,
329 ..DedupDomain::local_default()
330 };
331 let err = sub.put(&domain, b"hello", Some(label())).await.unwrap_err();
332 assert!(
333 matches!(err, CasError::UnsupportedIngestAlgorithm(_)),
334 "BLAKE3-512 ingest must be rejected until #881 lands"
335 );
336 }
337
338 #[tokio::test]
339 async fn seal_label_is_derived_from_domain() {
340 use hyprstream_rpc::auth::mac::{Assurance, CompartmentSet, Level};
341 let dir = tempfile::tempdir().unwrap();
342 let sub = CasSubstrate::new(dir.path());
343 let m = sub
345 .put(&DedupDomain::local_default(), b"payload-bytes", None)
346 .await
347 .unwrap();
348 let expected =
349 SecurityLabel::new(Level::Internal, Assurance::Classical, CompartmentSet::EMPTY);
350 assert_eq!(
351 m.security_label, expected,
352 "seal label must be derived from the domain, not caller-asserted"
353 );
354 }
355
356 #[tokio::test]
357 async fn manifest_cid_remains_resolvable_after_reopening_the_substrate() {
358 let dir = tempfile::tempdir().unwrap();
359 let domain = DedupDomain::local_default();
360 let manifest = CasSubstrate::new(dir.path())
361 .put(&domain, b"durable-manifest", Some(label()))
362 .await
363 .unwrap();
364 let reopened = CasSubstrate::new(dir.path());
365 assert_eq!(
366 reopened.get(&domain, &manifest.cid).await.unwrap(),
367 b"durable-manifest"
368 );
369 }
370
371 #[tokio::test]
372 async fn seal_label_hint_joins_restrict_only() {
373 use hyprstream_rpc::auth::mac::{Assurance, CompartmentSet, Level};
374 let dir = tempfile::tempdir().unwrap();
375 let sub = CasSubstrate::new(dir.path());
376 let hint = SecurityLabel::new(Level::Secret, Assurance::Classical, CompartmentSet::EMPTY);
378 let m = sub
379 .put(&DedupDomain::local_default(), b"payload-bytes", Some(hint))
380 .await
381 .unwrap();
382 assert_eq!(
383 m.security_label.level,
384 Level::Secret,
385 "hint label must join restrict-only with the domain floor"
386 );
387 }
388}