Skip to main content

cas_serve/
shard.rs

1//! Reconstruction shard — the XET `mdb_shard` binary reconstruction format.
2//!
3//! A *shard* (the `.mdb` file) carries the File-Info + CAS-Info sections that
4//! map a file to the ordered sequence of (xorb hash, chunk-index range, byte
5//! length) segments needed to reassemble it. This module is the cas-serve
6//! integration layer over [`crate::mdb_shard`]: it owns the chunk → xorb →
7//! shard pipeline and exposes the helpers the binary's request handlers use.
8//!
9//! The shard is content-addressed by the file's merkle hash and stored at
10//! `shards/{file_hash}` as the raw `.mdb` bytes. Reconstruction (`GetFile`)
11//! loads the shard, walks its File-Info segments, fetches each referenced
12//! xorb, slices out the chunk range, and concatenates the bytes in order.
13//!
14//! This is the critical-path BYTES layer for multi-xorb (i.e. >64 MiB) file
15//! transfer described in #390. Unlike the earlier bespoke JSON shard, this
16//! format is byte-compatible with upstream xet-core (`MDBShardFile` /
17//! `FileReconstructor`), so a stock xet-core client can consume the shard
18//! returned by `GetReconstructionInfo`.
19
20use merklehash::MerkleHash;
21
22use crate::chunker::Chunk;
23use crate::mdb_shard;
24
25/// One reconstruction segment: the xorb that holds the bytes, the chunk-index
26/// range within that xorb, and the total byte length of those chunks. This is
27/// a deserialized view of one mdb_shard `FileDataSequenceEntry`, exposed for
28/// callers (e.g. `GetFile`) that reassemble without keeping a full xorb chunk
29/// table in memory.
30#[derive(Debug, Clone)]
31pub struct Segment {
32    /// Xorb hash (Merkle root over the xorb's chunk hashes).
33    pub xorb_hash: MerkleHash,
34    /// Inclusive start chunk index within the xorb (local to the xorb).
35    pub chunk_start: u32,
36    /// Exclusive end chunk index within the xorb.
37    pub chunk_end: u32,
38    /// Total byte length of the chunk range `[chunk_start, chunk_end)`.
39    pub byte_len: u64,
40}
41
42/// The reconstruction manifest for a single file: the `.mdb` bytes plus the
43/// derived metadata the protocol response carries (file hash, length, xorb
44/// hashes).
45#[derive(Debug, Clone)]
46pub struct Shard {
47    /// Hex-encoded file merkle hash this shard reconstructs.
48    pub file_hash: String,
49    /// Total file length in bytes (sum of all segment byte lengths).
50    pub file_len: u64,
51    /// Hex-encoded xorb hashes the file was split across, in order.
52    pub xorb_hashes: Vec<String>,
53    /// The raw `.mdb` shard bytes (the XET wire format).
54    pub mdb_bytes: Vec<u8>,
55}
56
57impl Shard {
58    /// Build a shard from the file's chunk list, grouping chunks into xorbs
59    /// that respect the XET `MAX_XORB_BYTES` (64 MiB) and `MAX_XORB_CHUNKS`
60    /// (8192) limits. Returns the shard plus, for each xorb in order, the
61    /// xorb hash and the concatenated xorb bytes.
62    ///
63    /// Chunk → xorb aggregation, xorb hashing (`merklehash::xorb_hash`), and
64    /// file hashing (`merklehash::file_hash`) all match xet-core's
65    /// `RawXorbData::from_chunks`, so the hashes embedded in the produced
66    /// shard are identical to an upstream client's.
67    pub fn from_chunks(chunks: &[Chunk]) -> (Shard, Vec<(MerkleHash, Vec<u8>)>) {
68        Self::from_chunks_with_cap(chunks, crate::chunker::MAX_XORB_BYTES)
69    }
70
71    /// Like [`from_chunks`], but with a custom per-xorb byte cap. Used by tests
72    /// to force multi-xorb aggregation without allocating 64 MiB. The chunk
73    /// count cap (`MAX_XORB_CHUNKS` = 8192) still applies.
74    pub fn from_chunks_with_cap(
75        chunks: &[Chunk],
76        xorb_byte_cap: usize,
77    ) -> (Shard, Vec<(MerkleHash, Vec<u8>)>) {
78        let (builder, file_hash, xorbs) = mdb_shard::build_shard_with_cap(chunks, xorb_byte_cap);
79        // `serialize` writes into a `Vec<u8>`; the only failure mode is OOM,
80        // which is already a process abort — so this is effectively infallible.
81        let mdb_bytes = builder
82            .serialize()
83            .unwrap_or_else(|e| panic!("mdb_shard serialize failed: {e}"));
84
85        // Derive file_len + xorb_hashes directly from the inputs (they are
86        // exactly what the shard encodes, and this avoids a self-parse round
87        // trip that would need `expect`).
88        let file_len: u64 = chunks.iter().map(|c| c.data.len() as u64).sum();
89        let xorb_hashes: Vec<String> = xorbs.iter().map(|(h, _)| h.hex()).collect();
90        let shard = Shard {
91            file_hash: file_hash.hex(),
92            file_len,
93            xorb_hashes,
94            mdb_bytes,
95        };
96        (shard, xorbs)
97    }
98
99    /// Serialize the shard to its raw `.mdb` bytes for storage/transmission.
100    /// This is the XET wire format — a stock xet-core `MDBShardFile` reader can
101    /// parse it.
102    pub fn to_bytes(&self) -> &[u8] {
103        &self.mdb_bytes
104    }
105
106    /// Parse the reconstruction segments for this file out of a `.mdb` byte
107    /// buffer. Each segment tells the reassembler which xorb to fetch and which
108    /// local chunk range to slice out of it.
109    pub fn segments(mdb_bytes: &[u8], file_hash: &MerkleHash) -> std::io::Result<Vec<Segment>> {
110        let shard = mdb_shard::MdbShard::parse(mdb_bytes)?;
111        let info = shard.file_info(mdb_bytes, file_hash)?.ok_or_else(|| {
112            std::io::Error::new(std::io::ErrorKind::NotFound, "file hash not in shard")
113        })?;
114        Ok(info
115            .segments
116            .into_iter()
117            .map(|e| Segment {
118                xorb_hash: e.cas_hash,
119                chunk_start: e.chunk_index_start,
120                chunk_end: e.chunk_index_end,
121                byte_len: e.unpacked_segment_bytes as u64,
122            })
123            .collect())
124    }
125}
126
127#[cfg(test)]
128#[allow(clippy::unwrap_used, clippy::expect_used)]
129mod tests {
130    use super::*;
131    use crate::chunker::chunk_all;
132
133    #[test]
134    fn single_xorb_shard() {
135        let data = vec![0x42u8; 200_000];
136        let chunks = chunk_all(&data);
137        let (shard, xorbs) = Shard::from_chunks(&chunks);
138        assert_eq!(xorbs.len(), 1, "200KB fits in a single 64MiB xorb");
139        assert_eq!(shard.file_len, data.len() as u64);
140        assert_eq!(shard.xorb_hashes.len(), 1);
141        assert_eq!(shard.xorb_hashes[0], xorbs[0].0.hex());
142
143        // The shard must parse as an mdb_shard file and yield one segment
144        // spanning the whole xorb.
145        let file_hash = MerkleHash::from_hex(&shard.file_hash).unwrap();
146        let segs = Shard::segments(&shard.mdb_bytes, &file_hash).unwrap();
147        assert_eq!(segs.len(), 1);
148        assert_eq!(segs[0].byte_len, data.len() as u64);
149        assert_eq!(segs[0].chunk_start, 0);
150        assert_eq!(segs[0].chunk_end as usize, chunks.len());
151    }
152
153    #[test]
154    fn round_trip_reassembly_matches_original() {
155        // Pseudo-random data so CDC cuts multiple chunks.
156        let mut data = Vec::with_capacity(300_000);
157        let mut s: u64 = 42;
158        for _ in 0..37500 {
159            s = s.wrapping_mul(0x94d049bb133111eb) ^ (s >> 31);
160            data.extend_from_slice(&s.to_le_bytes());
161        }
162        let chunks = chunk_all(&data);
163        assert!(chunks.len() > 1, "expected multiple chunks");
164        let (shard, xorbs) = Shard::from_chunks(&chunks);
165
166        // Reassemble from xorbs using the shard's segments.
167        let file_hash = MerkleHash::from_hex(&shard.file_hash).unwrap();
168        let segs = Shard::segments(&shard.mdb_bytes, &file_hash).unwrap();
169        let mut rebuilt = Vec::new();
170        for seg in &segs {
171            let xorb = xorbs
172                .iter()
173                .find(|(h, _)| *h == seg.xorb_hash)
174                .expect("segment references a present xorb");
175            // cas-serve stores raw concatenated xorbs; the segment spans the
176            // whole xorb (chunk_start=0, chunk_end=xorb chunk count), so
177            // byte_len is the full xorb length.
178            rebuilt.extend_from_slice(&xorb.1[..seg.byte_len as usize]);
179        }
180        assert_eq!(rebuilt, data);
181    }
182}