1#[cfg(feature = "xet-storage")]
31use async_trait::async_trait;
32use std::io::Write;
33use std::path::Path;
34use std::sync::Arc;
35use std::{
36 io,
37 pin::Pin,
38 task::{Context, Poll},
39};
40
41#[cfg(feature = "xet-storage")]
42use data::{FileDownloader, FileUploadSession, XetFileInfo};
43#[cfg(feature = "xet-storage")]
44use tokio::io::AsyncWrite;
45
46use crate::error::{Result, XetError, XetErrorKind};
47
48#[cfg(feature = "xet-storage")]
56#[async_trait]
57pub trait StorageBackend: Send + Sync {
58 async fn clean_file(&self, path: &Path) -> Result<String>;
66
67 fn is_pointer(&self, content: &str) -> bool;
71
72 async fn clean_bytes(&self, data: &[u8]) -> Result<String>;
78
79 async fn smudge_file(&self, pointer: &str, output_path: &Path) -> Result<()>;
86
87 async fn smudge_bytes(&self, pointer: &str) -> Result<Vec<u8>>;
94
95 async fn smudge_from_hash(&self, hash: &merklehash::MerkleHash) -> Result<Vec<u8>>;
100
101 async fn smudge_from_hash_to_file(
106 &self,
107 hash: &merklehash::MerkleHash,
108 output_path: &Path,
109 ) -> Result<()>;
110}
111
112#[cfg(feature = "xet-storage")]
133pub struct XetStorage {
134 upload_session: Arc<FileUploadSession>,
135 downloader: Arc<FileDownloader>,
136}
137
138#[cfg(feature = "xet-storage")]
142struct BoundedFileWriter {
143 file: std::fs::File,
144 limit: u64,
145 written: u64,
146}
147
148#[cfg(feature = "xet-storage")]
149impl BoundedFileWriter {
150 fn create(path: &Path, limit: u64) -> io::Result<Self> {
151 let file = std::fs::OpenOptions::new()
152 .write(true)
153 .create_new(true)
154 .open(path)?;
155 Ok(Self {
156 file,
157 limit,
158 written: 0,
159 })
160 }
161}
162
163#[cfg(feature = "xet-storage")]
164impl AsyncWrite for BoundedFileWriter {
165 fn poll_write(
166 mut self: Pin<&mut Self>,
167 _cx: &mut Context<'_>,
168 buf: &[u8],
169 ) -> Poll<io::Result<usize>> {
170 let Some(end) = self.written.checked_add(buf.len() as u64) else {
171 return Poll::Ready(Err(io::Error::new(
172 io::ErrorKind::InvalidData,
173 "CAS output size overflow",
174 )));
175 };
176 if end > self.limit {
177 return Poll::Ready(Err(io::Error::new(
178 io::ErrorKind::InvalidData,
179 format!("CAS output exceeds {0}-byte limit", self.limit),
180 )));
181 }
182 loop {
183 match self.file.write(buf) {
184 Ok(written) => {
185 self.written += written as u64;
186 return Poll::Ready(Ok(written));
187 }
188 Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
189 Err(error) => return Poll::Ready(Err(error)),
190 }
191 }
192 }
193
194 fn poll_flush(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
195 Poll::Ready(self.file.flush())
196 }
197
198 fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
199 Poll::Ready(self.file.flush())
200 }
201}
202
203#[cfg(all(test, feature = "xet-storage"))]
204mod bounded_output_tests {
205 use super::BoundedFileWriter;
206 use std::io;
207 use tokio::io::AsyncWriteExt;
208
209 #[tokio::test]
210 async fn oversized_cas_chunk_is_rejected_before_file_growth() -> io::Result<()> {
211 let temp = tempfile::tempdir()?;
212 let output = temp.path().join("bounded-output");
213 let mut writer = BoundedFileWriter::create(&output, 4)?;
214
215 assert!(writer.write_all(b"oversized").await.is_err());
216 writer.flush().await?;
217 drop(writer);
218 assert_eq!(std::fs::metadata(&output)?.len(), 0);
219 assert_eq!(std::fs::read(&output)?, b"");
220 Ok(())
221 }
222
223 #[tokio::test]
224 async fn understated_pointer_cannot_write_past_declared_size() -> io::Result<()> {
225 let temp = tempfile::tempdir()?;
226 let output = temp.path().join("bounded-output");
227 let mut writer = BoundedFileWriter::create(&output, 4)?;
228
229 writer.write_all(b"1234").await?;
230 assert!(writer.write_all(b"5").await.is_err());
231 writer.flush().await?;
232 drop(writer);
233 assert_eq!(std::fs::read(&output)?, b"1234");
234 assert_eq!(std::fs::metadata(&output)?.len(), 4);
235 Ok(())
236 }
237}
238
239#[cfg(feature = "xet-storage")]
240impl XetStorage {
241 pub async fn new(config: &crate::config::XetConfig) -> Result<Self> {
251 let translator_config = Arc::new(
252 data::data_client::default_config(
253 config.endpoint.clone(),
254 config.compression,
255 config.token.as_ref().map(|t| (t.clone(), u64::MAX)),
256 None,
257 "hyprstream".to_owned(),
258 )
259 .map_err(|e| {
260 XetError::new(
261 XetErrorKind::RuntimeError,
262 format!("Failed to create XET config: {e}"),
263 )
264 })?,
265 );
266
267 let upload_session = FileUploadSession::new(translator_config.clone(), None)
268 .await
269 .map_err(|e| {
270 XetError::new(
271 XetErrorKind::UploadFailed,
272 format!("Failed to create upload session: {e}"),
273 )
274 })?;
275
276 let downloader = FileDownloader::new(translator_config).await.map_err(|e| {
277 XetError::new(
278 XetErrorKind::DownloadFailed,
279 format!("Failed to create downloader: {e}"),
280 )
281 })?;
282
283 Ok(Self {
284 upload_session,
285 downloader: Arc::new(downloader),
286 })
287 }
288
289 pub async fn clean_bytes(&self, data: &[u8]) -> Result<String> {
294 let (xet_info, _metrics) =
295 data::data_client::clean_bytes(self.upload_session.clone(), data.to_vec())
296 .await
297 .map_err(|e| {
298 XetError::new(
299 XetErrorKind::UploadFailed,
300 format!("Clean bytes failed: {e}"),
301 )
302 })?;
303
304 xet_info.as_pointer_file().map_err(|e| {
305 XetError::new(
306 XetErrorKind::InvalidPointer,
307 format!("Failed to serialize pointer: {e}"),
308 )
309 })
310 }
311
312 pub async fn smudge_file(&self, pointer: &str, output_path: &Path) -> Result<()> {
317 let xet_info: XetFileInfo = serde_json::from_str(pointer).map_err(|e| {
318 XetError::new(
319 XetErrorKind::InvalidPointer,
320 format!("Invalid pointer JSON: {e}"),
321 )
322 })?;
323
324 let merkle_hash = xet_info.merkle_hash().map_err(|e| {
325 XetError::new(
326 XetErrorKind::InvalidPointer,
327 format!("Invalid merkle hash: {e}"),
328 )
329 })?;
330
331 use cas_client::{FileProvider, SeekingOutputProvider};
332 self.downloader
333 .smudge_file_from_hash(
334 &merkle_hash,
335 output_path.to_string_lossy().into(),
336 SeekingOutputProvider::File(FileProvider::new(output_path.to_path_buf())),
337 None,
338 None,
339 )
340 .await
341 .map_err(|e| {
342 XetError::new(
343 XetErrorKind::DownloadFailed,
344 format!("Smudge to file failed: {e}"),
345 )
346 })?;
347
348 Ok(())
349 }
350
351 pub async fn smudge_file_bounded(
355 &self,
356 pointer: &str,
357 output_path: &Path,
358 max_output_bytes: u64,
359 ) -> Result<u64> {
360 let info: XetFileInfo = serde_json::from_str(pointer).map_err(|e| {
361 XetError::new(
362 XetErrorKind::InvalidPointer,
363 format!("Invalid pointer JSON: {e}"),
364 )
365 })?;
366 if info.file_size() > max_output_bytes {
367 return Err(XetError::new(
368 XetErrorKind::InvalidPointer,
369 format!("Pointer size exceeds {max_output_bytes}-byte output limit"),
370 ));
371 }
372 let hash = info.merkle_hash().map_err(|e| {
373 XetError::new(
374 XetErrorKind::InvalidPointer,
375 format!("Invalid merkle hash: {e}"),
376 )
377 })?;
378 self.smudge_hash_to_file_bounded(&hash, output_path, max_output_bytes)
379 .await
380 }
381
382 pub async fn smudge_hash_to_file_bounded(
384 &self,
385 hash: &merklehash::MerkleHash,
386 output_path: &Path,
387 max_output_bytes: u64,
388 ) -> Result<u64> {
389 let writer = BoundedFileWriter::create(output_path, max_output_bytes).map_err(|e| {
390 XetError::new(
391 XetErrorKind::IoError,
392 format!("Create bounded CAS output: {e}"),
393 )
394 })?;
395 let output: cas_client::SequentialOutput = Box::new(writer);
396 let written = self
397 .downloader
398 .smudge_file_from_hash_sequential(
399 hash,
400 output_path.to_string_lossy().into(),
401 output,
402 None,
403 None,
404 )
405 .await
406 .map_err(|e| {
407 XetError::new(
408 XetErrorKind::DownloadFailed,
409 format!("Bounded smudge to file failed: {e}"),
410 )
411 })?;
412 if written > max_output_bytes {
413 return Err(XetError::new(
414 XetErrorKind::DownloadFailed,
415 "CAS downloader reported bytes beyond its enforced output limit",
416 ));
417 }
418 Ok(written)
419 }
420
421 pub async fn smudge_bytes(&self, pointer: &str) -> Result<Vec<u8>> {
425 let xet_info: XetFileInfo = serde_json::from_str(pointer).map_err(|e| {
426 XetError::new(
427 XetErrorKind::InvalidPointer,
428 format!("Invalid pointer JSON: {e}"),
429 )
430 })?;
431
432 let merkle_hash = xet_info.merkle_hash().map_err(|e| {
433 XetError::new(
434 XetErrorKind::InvalidPointer,
435 format!("Invalid merkle hash: {e}"),
436 )
437 })?;
438
439 let temp_file = tempfile::NamedTempFile::new().map_err(|e| {
441 XetError::new(
442 XetErrorKind::IoError,
443 format!("Failed to create temp file: {e}"),
444 )
445 })?;
446
447 use cas_client::{FileProvider, SeekingOutputProvider};
448 self.downloader
449 .smudge_file_from_hash(
450 &merkle_hash,
451 temp_file.path().to_string_lossy().into(),
452 SeekingOutputProvider::File(FileProvider::new(temp_file.path().to_path_buf())),
453 None,
454 None,
455 )
456 .await
457 .map_err(|e| {
458 XetError::new(
459 XetErrorKind::DownloadFailed,
460 format!("Smudge to bytes failed: {e}"),
461 )
462 })?;
463
464 tokio::fs::read(temp_file.path()).await.map_err(|e| {
466 XetError::new(
467 XetErrorKind::IoError,
468 format!("Failed to read smudged file: {e}"),
469 )
470 })
471 }
472
473 pub async fn smudge_from_hash(&self, hash: &merklehash::MerkleHash) -> Result<Vec<u8>> {
478 let temp_file = tempfile::NamedTempFile::new().map_err(|e| {
480 XetError::new(
481 XetErrorKind::IoError,
482 format!("Failed to create temp file: {e}"),
483 )
484 })?;
485
486 use cas_client::{FileProvider, SeekingOutputProvider};
487 self.downloader
488 .smudge_file_from_hash(
489 hash,
490 temp_file.path().to_string_lossy().into(),
491 SeekingOutputProvider::File(FileProvider::new(temp_file.path().to_path_buf())),
492 None,
493 None,
494 )
495 .await
496 .map_err(|e| {
497 XetError::new(
498 XetErrorKind::DownloadFailed,
499 format!("Smudge from hash failed: {e}"),
500 )
501 })?;
502
503 tokio::fs::read(temp_file.path()).await.map_err(|e| {
505 XetError::new(
506 XetErrorKind::IoError,
507 format!("Failed to read smudged file: {e}"),
508 )
509 })
510 }
511
512 pub async fn smudge_from_hash_to_file(
517 &self,
518 hash: &merklehash::MerkleHash,
519 output_path: &Path,
520 ) -> Result<()> {
521 use cas_client::{FileProvider, SeekingOutputProvider};
522 self.downloader
523 .smudge_file_from_hash(
524 hash,
525 output_path.to_string_lossy().into(),
526 SeekingOutputProvider::File(FileProvider::new(output_path.to_path_buf())),
527 None,
528 None,
529 )
530 .await
531 .map_err(|e| {
532 XetError::new(
533 XetErrorKind::DownloadFailed,
534 format!("Smudge from hash to file failed: {e}"),
535 )
536 })?;
537
538 Ok(())
539 }
540}
541
542#[cfg(feature = "xet-storage")]
543#[async_trait]
544impl StorageBackend for XetStorage {
545 async fn clean_file(&self, path: &Path) -> Result<String> {
546 let (xet_info, _metrics) =
547 data::data_client::clean_file(self.upload_session.clone(), path, "")
548 .await
549 .map_err(|e| {
550 XetError::new(XetErrorKind::UploadFailed, format!("Clean failed: {e}"))
551 })?;
552
553 xet_info.as_pointer_file().map_err(|e| {
554 XetError::new(
555 XetErrorKind::InvalidPointer,
556 format!("Failed to serialize pointer: {e}"),
557 )
558 })
559 }
560
561 fn is_pointer(&self, content: &str) -> bool {
562 serde_json::from_str::<XetFileInfo>(content).is_ok()
563 }
564
565 async fn clean_bytes(&self, data: &[u8]) -> Result<String> {
566 XetStorage::clean_bytes(self, data).await
567 }
568
569 async fn smudge_file(&self, pointer: &str, output_path: &Path) -> Result<()> {
570 XetStorage::smudge_file(self, pointer, output_path).await
571 }
572
573 async fn smudge_bytes(&self, pointer: &str) -> Result<Vec<u8>> {
574 XetStorage::smudge_bytes(self, pointer).await
575 }
576
577 async fn smudge_from_hash(&self, hash: &merklehash::MerkleHash) -> Result<Vec<u8>> {
578 XetStorage::smudge_from_hash(self, hash).await
579 }
580
581 async fn smudge_from_hash_to_file(
582 &self,
583 hash: &merklehash::MerkleHash,
584 output_path: &Path,
585 ) -> Result<()> {
586 XetStorage::smudge_from_hash_to_file(self, hash, output_path).await
587 }
588}