cas_serve/shard.rs
1//! Reconstruction shard — the XET `mdb_shard` binary reconstruction format.
2//!
3//! A *shard* (the `.mdb` file) carries the File-Info + CAS-Info sections that
4//! map a file to the ordered sequence of (xorb hash, chunk-index range, byte
5//! length) segments needed to reassemble it. This module is the cas-serve
6//! integration layer over [`crate::mdb_shard`]: it owns the chunk → xorb →
7//! shard pipeline and exposes the helpers the binary's request handlers use.
8//!
9//! The shard is content-addressed by the file's merkle hash and stored at
10//! `shards/{file_hash}` as the raw `.mdb` bytes. Reconstruction (`GetFile`)
11//! loads the shard, walks its File-Info segments, fetches each referenced
12//! xorb, slices out the chunk range, and concatenates the bytes in order.
13//!
14//! This is the critical-path BYTES layer for multi-xorb (i.e. >64 MiB) file
15//! transfer described in #390. Unlike the earlier bespoke JSON shard, this
16//! format is byte-compatible with upstream xet-core (`MDBShardFile` /
17//! `FileReconstructor`), so a stock xet-core client can consume the shard
18//! returned by `GetReconstructionInfo`.
19
20use merklehash::MerkleHash;
21
22use crate::chunker::Chunk;
23use crate::mdb_shard;
24
25/// One reconstruction segment: the xorb that holds the bytes, the chunk-index
26/// range within that xorb, and the total byte length of those chunks. This is
27/// a deserialized view of one mdb_shard `FileDataSequenceEntry`, exposed for
28/// callers (e.g. `GetFile`) that reassemble without keeping a full xorb chunk
29/// table in memory.
30#[derive(Debug, Clone)]
31pub struct Segment {
32 /// Xorb hash (Merkle root over the xorb's chunk hashes).
33 pub xorb_hash: MerkleHash,
34 /// Inclusive start chunk index within the xorb (local to the xorb).
35 pub chunk_start: u32,
36 /// Exclusive end chunk index within the xorb.
37 pub chunk_end: u32,
38 /// Total byte length of the chunk range `[chunk_start, chunk_end)`.
39 pub byte_len: u64,
40}
41
42/// The reconstruction manifest for a single file: the `.mdb` bytes plus the
43/// derived metadata the protocol response carries (file hash, length, xorb
44/// hashes).
45#[derive(Debug, Clone)]
46pub struct Shard {
47 /// Hex-encoded file merkle hash this shard reconstructs.
48 pub file_hash: String,
49 /// Total file length in bytes (sum of all segment byte lengths).
50 pub file_len: u64,
51 /// Hex-encoded xorb hashes the file was split across, in order.
52 pub xorb_hashes: Vec<String>,
53 /// The raw `.mdb` shard bytes (the XET wire format).
54 pub mdb_bytes: Vec<u8>,
55}
56
57impl Shard {
58 /// Build a shard from the file's chunk list, grouping chunks into xorbs
59 /// that respect the XET `MAX_XORB_BYTES` (64 MiB) and `MAX_XORB_CHUNKS`
60 /// (8192) limits. Returns the shard plus, for each xorb in order, the
61 /// xorb hash and the concatenated xorb bytes.
62 ///
63 /// Chunk → xorb aggregation, xorb hashing (`merklehash::xorb_hash`), and
64 /// file hashing (`merklehash::file_hash`) all match xet-core's
65 /// `RawXorbData::from_chunks`, so the hashes embedded in the produced
66 /// shard are identical to an upstream client's.
67 pub fn from_chunks(chunks: &[Chunk]) -> (Shard, Vec<(MerkleHash, Vec<u8>)>) {
68 Self::from_chunks_with_cap(chunks, crate::chunker::MAX_XORB_BYTES)
69 }
70
71 /// Like [`from_chunks`], but with a custom per-xorb byte cap. Used by tests
72 /// to force multi-xorb aggregation without allocating 64 MiB. The chunk
73 /// count cap (`MAX_XORB_CHUNKS` = 8192) still applies.
74 pub fn from_chunks_with_cap(
75 chunks: &[Chunk],
76 xorb_byte_cap: usize,
77 ) -> (Shard, Vec<(MerkleHash, Vec<u8>)>) {
78 let (builder, file_hash, xorbs) = mdb_shard::build_shard_with_cap(chunks, xorb_byte_cap);
79 // `serialize` writes into a `Vec<u8>`; the only failure mode is OOM,
80 // which is already a process abort — so this is effectively infallible.
81 let mdb_bytes = builder
82 .serialize()
83 .unwrap_or_else(|e| panic!("mdb_shard serialize failed: {e}"));
84
85 // Derive file_len + xorb_hashes directly from the inputs (they are
86 // exactly what the shard encodes, and this avoids a self-parse round
87 // trip that would need `expect`).
88 let file_len: u64 = chunks.iter().map(|c| c.data.len() as u64).sum();
89 let xorb_hashes: Vec<String> = xorbs.iter().map(|(h, _)| h.hex()).collect();
90 let shard = Shard {
91 file_hash: file_hash.hex(),
92 file_len,
93 xorb_hashes,
94 mdb_bytes,
95 };
96 (shard, xorbs)
97 }
98
99 /// Serialize the shard to its raw `.mdb` bytes for storage/transmission.
100 /// This is the XET wire format — a stock xet-core `MDBShardFile` reader can
101 /// parse it.
102 pub fn to_bytes(&self) -> &[u8] {
103 &self.mdb_bytes
104 }
105
106 /// Parse the reconstruction segments for this file out of a `.mdb` byte
107 /// buffer. Each segment tells the reassembler which xorb to fetch and which
108 /// local chunk range to slice out of it.
109 pub fn segments(mdb_bytes: &[u8], file_hash: &MerkleHash) -> std::io::Result<Vec<Segment>> {
110 let shard = mdb_shard::MdbShard::parse(mdb_bytes)?;
111 let info = shard.file_info(mdb_bytes, file_hash)?.ok_or_else(|| {
112 std::io::Error::new(std::io::ErrorKind::NotFound, "file hash not in shard")
113 })?;
114 Ok(info
115 .segments
116 .into_iter()
117 .map(|e| Segment {
118 xorb_hash: e.cas_hash,
119 chunk_start: e.chunk_index_start,
120 chunk_end: e.chunk_index_end,
121 byte_len: e.unpacked_segment_bytes as u64,
122 })
123 .collect())
124 }
125}
126
127#[cfg(test)]
128#[allow(clippy::unwrap_used, clippy::expect_used)]
129mod tests {
130 use super::*;
131 use crate::chunker::chunk_all;
132
133 #[test]
134 fn single_xorb_shard() {
135 let data = vec![0x42u8; 200_000];
136 let chunks = chunk_all(&data);
137 let (shard, xorbs) = Shard::from_chunks(&chunks);
138 assert_eq!(xorbs.len(), 1, "200KB fits in a single 64MiB xorb");
139 assert_eq!(shard.file_len, data.len() as u64);
140 assert_eq!(shard.xorb_hashes.len(), 1);
141 assert_eq!(shard.xorb_hashes[0], xorbs[0].0.hex());
142
143 // The shard must parse as an mdb_shard file and yield one segment
144 // spanning the whole xorb.
145 let file_hash = MerkleHash::from_hex(&shard.file_hash).unwrap();
146 let segs = Shard::segments(&shard.mdb_bytes, &file_hash).unwrap();
147 assert_eq!(segs.len(), 1);
148 assert_eq!(segs[0].byte_len, data.len() as u64);
149 assert_eq!(segs[0].chunk_start, 0);
150 assert_eq!(segs[0].chunk_end as usize, chunks.len());
151 }
152
153 #[test]
154 fn round_trip_reassembly_matches_original() {
155 // Pseudo-random data so CDC cuts multiple chunks.
156 let mut data = Vec::with_capacity(300_000);
157 let mut s: u64 = 42;
158 for _ in 0..37500 {
159 s = s.wrapping_mul(0x94d049bb133111eb) ^ (s >> 31);
160 data.extend_from_slice(&s.to_le_bytes());
161 }
162 let chunks = chunk_all(&data);
163 assert!(chunks.len() > 1, "expected multiple chunks");
164 let (shard, xorbs) = Shard::from_chunks(&chunks);
165
166 // Reassemble from xorbs using the shard's segments.
167 let file_hash = MerkleHash::from_hex(&shard.file_hash).unwrap();
168 let segs = Shard::segments(&shard.mdb_bytes, &file_hash).unwrap();
169 let mut rebuilt = Vec::new();
170 for seg in &segs {
171 let xorb = xorbs
172 .iter()
173 .find(|(h, _)| *h == seg.xorb_hash)
174 .expect("segment references a present xorb");
175 // cas-serve stores raw concatenated xorbs; the segment spans the
176 // whole xorb (chunk_start=0, chunk_end=xorb chunk count), so
177 // byte_len is the full xorb length.
178 rebuilt.extend_from_slice(&xorb.1[..seg.byte_len as usize]);
179 }
180 assert_eq!(rebuilt, data);
181 }
182}