1use anyhow::{anyhow, Context, Result};
11use chrono::{DateTime, Utc};
12use serde::{Deserialize, Serialize};
13use std::path::{Path, PathBuf};
14use tokio::fs;
15use tokio::sync::mpsc;
16use tokio::task::JoinHandle;
17
18use crate::worktree_ext::WorktreeClientExt;
19use hyprstream_rpc_std::registry_client::{RepositoryClient, WorktreeClient};
20
21#[derive(Debug, Clone)]
23pub struct CheckpointRequest {
24 pub step: usize,
25 pub epoch: Option<usize>,
26 pub weights: WeightSnapshot,
27 pub metrics: Option<TrainingMetrics>,
28 pub timestamp: DateTime<Utc>,
29 pub commit_to_git: bool,
30}
31
32#[derive(Debug, Clone)]
34pub enum WeightSnapshot {
35 Memory { data: Vec<u8>, format: WeightFormat },
37
38 FilePath {
40 source: PathBuf,
41 format: WeightFormat,
42 },
43
44 Diff { base_step: usize, changes: Vec<u8> },
46}
47
48#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
49pub enum WeightFormat {
50 SafeTensors,
51 PyTorch,
52 AdapterBin,
53}
54
55#[derive(Debug, Clone, Serialize, Deserialize)]
57pub struct TrainingMetrics {
58 pub loss: f32,
59 pub learning_rate: f32,
60 pub gradient_norm: Option<f32>,
61 pub validation_loss: Option<f32>,
62 pub validation_accuracy: Option<f32>,
63 pub duration_seconds: f64,
64}
65
66#[derive(Debug, Clone, Serialize, Deserialize)]
68pub struct CheckpointMetadata {
69 pub step: usize,
70 pub epoch: Option<usize>,
71 pub timestamp: DateTime<Utc>,
72 pub metrics: Option<TrainingMetrics>,
73 pub format: WeightFormat,
74 pub parent_checkpoint: Option<String>,
75}
76
77pub struct CheckpointManager {
79 model_path: PathBuf,
80 checkpoint_dir: PathBuf,
81 checkpoint_tx: mpsc::Sender<CheckpointRequest>,
82 worker_handle: Option<JoinHandle<()>>,
83 #[allow(dead_code)]
84 max_checkpoints: usize,
85 repo_client: Option<RepositoryClient>,
87 #[allow(dead_code)]
88 branch_name: Option<String>, target_adapter: Option<String>,
92 fs: Option<WorktreeClient>,
94}
95
96impl CheckpointManager {
97 pub fn new(model_path: PathBuf) -> Result<Self> {
99 Self::with_config(model_path, CheckpointConfig::default(), None, None)
100 }
101
102 pub fn with_repo_client(model_path: PathBuf, repo_client: RepositoryClient) -> Result<Self> {
104 Self::with_config(
105 model_path,
106 CheckpointConfig::default(),
107 None,
108 Some(repo_client),
109 )
110 }
111
112 pub fn for_adapter(
114 model_path: PathBuf,
115 branch_name: String,
116 repo_client: Option<RepositoryClient>,
117 ) -> Result<Self> {
118 Self::with_config(
119 model_path,
120 CheckpointConfig::default(),
121 Some(branch_name),
122 repo_client,
123 )
124 }
125
126 pub fn with_config(
128 model_path: PathBuf,
129 config: CheckpointConfig,
130 branch_name: Option<String>,
131 repo_client: Option<RepositoryClient>,
132 ) -> Result<Self> {
133 let checkpoint_dir = model_path.join(".checkpoints");
134 std::fs::create_dir_all(&checkpoint_dir)?;
135
136 let fs: Option<WorktreeClient> = repo_client.as_ref().map(|client| {
138 let branch = branch_name.as_deref().unwrap_or("main");
139 client.worktree(branch)
140 });
141
142 let (tx, rx) = mpsc::channel(config.queue_size);
143
144 let worker_handle = {
146 let max_checkpoints = config.max_checkpoints;
147 let git_interval = config.git_commit_interval;
148 let branch = branch_name.clone();
149 let client = repo_client.clone();
150 let worker_fs = fs.clone();
151 let worker_checkpoint_dir = checkpoint_dir.clone();
152
153 tokio::spawn(async move {
154 Self::checkpoint_worker(
155 rx,
156 worker_checkpoint_dir,
157 max_checkpoints,
158 git_interval,
159 client,
160 branch,
161 worker_fs,
162 )
163 .await;
164 })
165 };
166
167 Ok(Self {
168 model_path,
169 checkpoint_dir,
170 checkpoint_tx: tx,
171 worker_handle: Some(worker_handle),
172 max_checkpoints: config.max_checkpoints,
173 repo_client,
174 branch_name,
175 target_adapter: None,
176 fs,
177 })
178 }
179
180 pub fn with_target_adapter(mut self, adapter_name: String) -> Self {
185 self.target_adapter = Some(adapter_name);
186 self
187 }
188
189 pub fn target_adapter(&self) -> Option<&str> {
191 self.target_adapter.as_deref()
192 }
193
194 pub async fn checkpoint(&self, request: CheckpointRequest) -> Result<()> {
196 self.checkpoint_tx
197 .send(request)
198 .await
199 .context("Failed to send checkpoint request")?;
200 Ok(())
201 }
202
203 pub fn create_request(
205 step: usize,
206 weights: WeightSnapshot,
207 metrics: Option<TrainingMetrics>,
208 ) -> CheckpointRequest {
209 CheckpointRequest {
210 step,
211 epoch: None,
212 weights,
213 metrics,
214 timestamp: Utc::now(),
215 commit_to_git: step.is_multiple_of(10000), }
217 }
218
219 pub async fn write_checkpoint(
222 &self,
223 weights: WeightSnapshot,
224 step: usize,
225 metadata: Option<TrainingMetrics>,
226 ) -> Result<PathBuf> {
227 let checkpoint_path = self.checkpoint_dir.join("checkpoint.safetensors");
229
230 if let Some(fs) = &self.fs {
231 match weights {
233 WeightSnapshot::Memory {
234 ref data,
235 format: _,
236 } => {
237 fs.write_file_chunked(".checkpoints/checkpoint.safetensors", data)
238 .await?;
239 }
240 WeightSnapshot::FilePath {
241 ref source,
242 format: _,
243 } => {
244 if let Ok(rel) = source.strip_prefix(&self.model_path) {
248 let rel_str = rel.to_string_lossy();
249 fs.copy_path(&rel_str, ".checkpoints/checkpoint.safetensors")
250 .await?;
251 } else {
252 let data = tokio::fs::read(source).await?;
254 fs.write_file_chunked(".checkpoints/checkpoint.safetensors", &data)
255 .await?;
256 }
257 }
258 WeightSnapshot::Diff {
259 base_step,
260 ref changes,
261 } => {
262 let rel_diff = format!(".checkpoints/diff_from_{base_step}_to_{step}.bin");
263 fs.write_file_chunked(&rel_diff, changes).await?;
264 return Ok(self
265 .checkpoint_dir
266 .join(format!("diff_from_{base_step}_to_{step}.bin")));
267 }
268 }
269
270 if let Some(metrics) = metadata {
272 let ckpt_meta = CheckpointMetadata {
273 step,
274 epoch: None,
275 timestamp: Utc::now(),
276 metrics: Some(metrics),
277 format: WeightFormat::SafeTensors,
278 parent_checkpoint: None,
279 };
280 let metadata_json = serde_json::to_string_pretty(&ckpt_meta)?;
281 fs.write_file_chunked(".checkpoints/checkpoint.json", metadata_json.as_bytes())
282 .await?;
283 }
284
285 if let Some(ref adapter_name) = self.target_adapter {
287 fs.mkdir_p("adapters").await?;
288 let rel_adapter = format!("adapters/{adapter_name}.safetensors");
289 fs.copy_path(".checkpoints/checkpoint.safetensors", &rel_adapter)
290 .await?;
291 tracing::info!("Updated target adapter via FsOps: {}", rel_adapter);
292 }
293 } else {
294 match weights {
296 WeightSnapshot::Memory {
297 ref data,
298 format: _,
299 } => {
300 fs::write(&checkpoint_path, data).await?;
301 }
302 WeightSnapshot::FilePath {
303 ref source,
304 format: _,
305 } => {
306 let metadata = std::fs::symlink_metadata(source)
308 .map_err(|e| anyhow!("Cannot stat source file {:?}: {}", source, e))?;
309 if metadata.file_type().is_symlink() {
310 return Err(anyhow!(
311 "Source path {:?} is a symlink; refusing to copy",
312 source
313 ));
314 }
315 crate::worktree_ext::cow_copy(source, &checkpoint_path)?;
316 }
317 WeightSnapshot::Diff {
318 base_step,
319 ref changes,
320 } => {
321 let diff_path = self
322 .checkpoint_dir
323 .join(format!("diff_from_{base_step}_to_{step}.bin"));
324 fs::write(&diff_path, changes).await?;
325 return Ok(diff_path);
326 }
327 }
328
329 if let Some(metrics) = metadata {
331 let ckpt_meta = CheckpointMetadata {
332 step,
333 epoch: None,
334 timestamp: Utc::now(),
335 metrics: Some(metrics),
336 format: WeightFormat::SafeTensors,
337 parent_checkpoint: None,
338 };
339 let metadata_path = checkpoint_path.with_extension("json");
340 let metadata_json = serde_json::to_string_pretty(&ckpt_meta)?;
341 fs::write(&metadata_path, metadata_json).await?;
342 }
343
344 if let Some(ref adapter_name) = self.target_adapter {
346 self.update_target_adapter(&checkpoint_path, adapter_name)
347 .await?;
348 }
349 }
350
351 tracing::info!(
352 "Checkpoint written to filesystem: {}",
353 checkpoint_path.display()
354 );
355 Ok(checkpoint_path)
356 }
357
358 async fn update_target_adapter(
360 &self,
361 checkpoint_path: &Path,
362 adapter_name: &str,
363 ) -> Result<()> {
364 let adapters_dir = self.model_path.join("adapters");
365 fs::create_dir_all(&adapters_dir).await?;
366
367 let adapter_path = adapters_dir.join(format!("{adapter_name}.safetensors"));
368 crate::worktree_ext::cow_copy(checkpoint_path, &adapter_path)?;
369
370 tracing::info!("Updated target adapter: {}", adapter_path.display());
371 Ok(())
372 }
373
374 pub async fn commit_checkpoint(
376 &self,
377 checkpoint_path: &Path,
378 message: Option<String>,
379 branch: Option<String>,
380 ) -> Result<String> {
381 let repo_client = self
382 .repo_client
383 .as_ref()
384 .ok_or_else(|| anyhow!("Git is not enabled (no RepositoryClient configured)"))?;
385
386 let wt_branch = if let Some(ref branch_name) = branch {
388 let _ = repo_client
390 .create_branch(&hyprstream_rpc_std::registry_client::BranchRequest {
391 branch_name: branch_name.clone(),
392 start_point: String::new(),
393 })
394 .await;
395 repo_client
397 .worktree(branch_name)
398 .checkout(&hyprstream_rpc_std::registry_client::CheckoutRequest {
399 ref_name: branch_name.clone(),
400 create_branch: false,
401 })
402 .await
403 .map_err(|e| anyhow!("Failed to checkout branch '{}': {}", branch_name, e))?;
404 branch_name.clone()
405 } else {
406 repo_client
408 .get_head()
409 .await
410 .unwrap_or_else(|_| "main".to_owned())
411 };
412 let wt = repo_client.worktree(&wt_branch);
413
414 let relative_path = checkpoint_path
416 .strip_prefix(&self.model_path)
417 .unwrap_or(checkpoint_path);
418 let mut files_to_stage: Vec<&str> = vec![relative_path.to_str().unwrap_or("")];
419
420 let metadata_path = checkpoint_path.with_extension("json");
421 let relative_metadata;
422
423 let meta_exists = if let Some(fs) = &self.fs {
424 let rel = metadata_path
425 .strip_prefix(&self.model_path)
426 .unwrap_or(&metadata_path)
427 .to_string_lossy()
428 .to_string();
429 fs.stat_path(&rel).await.map(|s| s.exists).unwrap_or(false)
430 } else {
431 metadata_path.exists()
432 };
433
434 if meta_exists {
435 relative_metadata = metadata_path
436 .strip_prefix(&self.model_path)
437 .unwrap_or(&metadata_path)
438 .to_string_lossy()
439 .to_string();
440 files_to_stage.push(&relative_metadata);
441 }
442
443 let files_owned: Vec<String> = files_to_stage.iter().map(|s| (*s).to_owned()).collect();
444 wt.stage_files(&hyprstream_rpc_std::registry_client::StageFilesRequest {
445 files: files_owned,
446 })
447 .await
448 .map_err(|e| anyhow!("Failed to stage files: {}", e))?;
449
450 let commit_message = message.unwrap_or_else(|| {
451 format!(
452 "Training checkpoint: {}",
453 checkpoint_path
454 .file_name()
455 .and_then(|n| n.to_str())
456 .unwrap_or("unknown")
457 )
458 });
459
460 let oid = wt
461 .commit(&hyprstream_rpc_std::registry_client::CommitRequest {
462 message: commit_message,
463 author: String::new(),
464 email: String::new(),
465 })
466 .await
467 .map_err(|e| anyhow!("Failed to commit: {}", e))?;
468
469 tracing::info!("Checkpoint committed to Git: {}", oid);
470 Ok(oid)
471 }
472
473 pub async fn flush(&self) -> Result<()> {
475 let sentinel = CheckpointRequest {
477 step: usize::MAX,
478 epoch: None,
479 weights: WeightSnapshot::Memory {
480 data: vec![],
481 format: WeightFormat::SafeTensors,
482 },
483 metrics: None,
484 timestamp: Utc::now(),
485 commit_to_git: false,
486 };
487
488 self.checkpoint(sentinel).await?;
489
490 tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
492
493 Ok(())
494 }
495
496 pub async fn shutdown(mut self) -> Result<()> {
498 drop(std::mem::replace(
500 &mut self.checkpoint_tx,
501 mpsc::channel(1).0,
502 ));
503
504 if let Some(handle) = self.worker_handle.take() {
506 match tokio::time::timeout(tokio::time::Duration::from_secs(30), handle).await {
507 Ok(Ok(())) => {
508 tracing::debug!("CheckpointManager worker shut down cleanly");
509 }
510 Ok(Err(e)) => {
511 tracing::warn!("CheckpointManager worker panicked during shutdown: {}", e);
512 }
513 Err(_) => {
514 tracing::error!(
515 "CheckpointManager worker timed out during shutdown - \
516 checkpoint may be incomplete"
517 );
518 }
519 }
520 }
521
522 Ok(())
523 }
524
525 pub fn has_pending(&self) -> bool {
527 !self.checkpoint_tx.is_closed()
528 }
529
530 pub async fn get_checkpoint(&self) -> Result<Option<CheckpointInfo>> {
532 if let Some(fs) = &self.fs {
533 if !fs
534 .stat_path(".checkpoints/checkpoint.json")
535 .await
536 .map(|s| s.exists)
537 .unwrap_or(false)
538 {
539 return Ok(None);
540 }
541 let json =
542 String::from_utf8(fs.read_file_chunked(".checkpoints/checkpoint.json").await?)?;
543 let metadata: CheckpointMetadata = serde_json::from_str(&json)?;
544 Ok(Some(CheckpointInfo {
545 step: metadata.step,
546 path: self.checkpoint_dir.clone(),
547 metadata: Some(metadata),
548 }))
549 } else {
550 let metadata_path = self.checkpoint_dir.join("checkpoint.json");
551 if !metadata_path.exists() {
552 return Ok(None);
553 }
554 let json = fs::read_to_string(&metadata_path).await?;
555 let metadata: CheckpointMetadata = serde_json::from_str(&json)?;
556 Ok(Some(CheckpointInfo {
557 step: metadata.step,
558 path: self.checkpoint_dir.clone(),
559 metadata: Some(metadata),
560 }))
561 }
562 }
563
564 pub async fn load_checkpoint(&self) -> Result<PathBuf> {
566 let checkpoint_path = self.checkpoint_dir.join("checkpoint.safetensors");
567
568 if let Some(fs) = &self.fs {
569 if !fs
570 .stat_path(".checkpoints/checkpoint.safetensors")
571 .await
572 .map(|s| s.exists)
573 .unwrap_or(false)
574 {
575 anyhow::bail!("No checkpoint found at {:?}", checkpoint_path);
576 }
577 } else if !checkpoint_path.exists() {
578 anyhow::bail!("No checkpoint found at {:?}", checkpoint_path);
579 }
580
581 Ok(checkpoint_path)
582 }
583
584 async fn checkpoint_worker(
586 mut rx: mpsc::Receiver<CheckpointRequest>,
587 checkpoint_dir: PathBuf,
588 _max_checkpoints: usize,
589 _git_interval: usize,
590 repo_client: Option<RepositoryClient>,
591 branch_name: Option<String>,
592 fs: Option<WorktreeClient>,
593 ) {
594 while let Some(request) = rx.recv().await {
595 if request.step == usize::MAX {
597 continue;
598 }
599
600 if let Err(e) = Self::process_checkpoint(
601 &checkpoint_dir,
602 request,
603 repo_client.as_ref(),
604 &branch_name,
605 fs.as_ref(),
606 )
607 .await
608 {
609 tracing::error!("Failed to process checkpoint: {}", e);
610 }
611 }
612 }
613
614 async fn process_checkpoint(
616 checkpoint_dir: &Path,
617 request: CheckpointRequest,
618 repo_client: Option<&RepositoryClient>,
619 branch_name: &Option<String>,
620 fs: Option<&WorktreeClient>,
621 ) -> Result<()> {
622 if let Some(fs) = fs {
623 fs.mkdir_p(".checkpoints").await?;
625
626 match request.weights {
628 WeightSnapshot::Memory { ref data, format } => {
629 let filename = match format {
630 WeightFormat::SafeTensors => "checkpoint.safetensors",
631 WeightFormat::PyTorch => "checkpoint.bin",
632 WeightFormat::AdapterBin => "checkpoint_adapter.bin",
633 };
634 let rel_path = format!(".checkpoints/{filename}");
635 fs.write_file_chunked(&rel_path, data).await?;
636 }
637 WeightSnapshot::FilePath { ref source, format } => {
638 let filename = match format {
639 WeightFormat::SafeTensors => "checkpoint.safetensors",
640 WeightFormat::PyTorch => "checkpoint.bin",
641 WeightFormat::AdapterBin => "checkpoint_adapter.bin",
642 };
643 let rel_dest = format!(".checkpoints/{filename}");
644 let data = tokio::fs::read(source).await?;
646 fs.write_file_chunked(&rel_dest, &data).await?;
647 }
648 WeightSnapshot::Diff {
649 base_step: _,
650 ref changes,
651 } => {
652 fs.write_file_chunked(".checkpoints/checkpoint_diff.bin", changes)
653 .await?;
654 }
655 }
656
657 let metadata = CheckpointMetadata {
659 step: request.step,
660 epoch: request.epoch,
661 timestamp: request.timestamp,
662 metrics: request.metrics,
663 format: match &request.weights {
664 WeightSnapshot::Memory { format, .. }
665 | WeightSnapshot::FilePath { format, .. } => *format,
666 WeightSnapshot::Diff { .. } => WeightFormat::SafeTensors,
667 },
668 parent_checkpoint: None,
669 };
670 let metadata_json = serde_json::to_string_pretty(&metadata)?;
671 fs.write_file_chunked(".checkpoints/checkpoint.json", metadata_json.as_bytes())
672 .await?;
673 } else {
674 tokio::fs::create_dir_all(checkpoint_dir).await?;
676
677 let _weights_file = match request.weights {
678 WeightSnapshot::Memory { ref data, format } => {
679 let filename = match format {
680 WeightFormat::SafeTensors => "checkpoint.safetensors",
681 WeightFormat::PyTorch => "checkpoint.bin",
682 WeightFormat::AdapterBin => "checkpoint_adapter.bin",
683 };
684 let path = checkpoint_dir.join(filename);
685 tokio::fs::write(&path, data).await?;
686 path
687 }
688 WeightSnapshot::FilePath { ref source, format } => {
689 let filename = match format {
690 WeightFormat::SafeTensors => "checkpoint.safetensors",
691 WeightFormat::PyTorch => "checkpoint.bin",
692 WeightFormat::AdapterBin => "checkpoint_adapter.bin",
693 };
694 let dest = checkpoint_dir.join(filename);
695 crate::worktree_ext::cow_copy(source, &dest)?;
696 dest
697 }
698 WeightSnapshot::Diff {
699 base_step: _,
700 ref changes,
701 } => {
702 let diff_path = checkpoint_dir.join("checkpoint_diff.bin");
703 tokio::fs::write(&diff_path, changes).await?;
704 diff_path
705 }
706 };
707
708 let metadata = CheckpointMetadata {
710 step: request.step,
711 epoch: request.epoch,
712 timestamp: request.timestamp,
713 metrics: request.metrics,
714 format: match &request.weights {
715 WeightSnapshot::Memory { format, .. }
716 | WeightSnapshot::FilePath { format, .. } => *format,
717 WeightSnapshot::Diff { .. } => WeightFormat::SafeTensors,
718 },
719 parent_checkpoint: None,
720 };
721 let metadata_json = serde_json::to_string_pretty(&metadata)?;
722 tokio::fs::write(checkpoint_dir.join("checkpoint.json"), metadata_json).await?;
723 }
724
725 if let Some(client) = repo_client {
727 if request.commit_to_git {
728 if let Err(e) = Self::git_commit_checkpoint(client, request.step, branch_name).await
729 {
730 tracing::warn!("Failed to commit checkpoint to Git: {}", e);
731 }
732 }
733 }
734
735 tracing::info!(
736 "Checkpoint saved: step {} at {}",
737 request.step,
738 checkpoint_dir.display()
739 );
740
741 Ok(())
742 }
743
744 async fn git_commit_checkpoint(
746 repo_client: &RepositoryClient,
747 step: usize,
748 branch_name: &Option<String>,
749 ) -> Result<()> {
750 let wt_branch = if let Some(branch) = branch_name {
752 let _ = repo_client
754 .create_branch(&hyprstream_rpc_std::registry_client::BranchRequest {
755 branch_name: branch.clone(),
756 start_point: String::new(),
757 })
758 .await;
759 repo_client
760 .worktree(branch)
761 .checkout(&hyprstream_rpc_std::registry_client::CheckoutRequest {
762 ref_name: branch.clone(),
763 create_branch: false,
764 })
765 .await
766 .map_err(|e| anyhow!("Failed to checkout branch '{}': {}", branch, e))?;
767 branch.clone()
768 } else {
769 repo_client
770 .get_head()
771 .await
772 .unwrap_or_else(|_| "main".to_owned())
773 };
774 let wt = repo_client.worktree(&wt_branch);
775
776 wt.stage_files(&hyprstream_rpc_std::registry_client::StageFilesRequest {
778 files: vec![
779 ".checkpoints/checkpoint.safetensors".to_owned(),
780 ".checkpoints/checkpoint.json".to_owned(),
781 ],
782 })
783 .await
784 .map_err(|e| anyhow!("Failed to stage checkpoint files: {}", e))?;
785
786 let message = if let Some(name) = branch_name.as_ref() {
788 format!("Training checkpoint step {step} (branch: {name})")
789 } else {
790 format!("Training checkpoint step {step}")
791 };
792
793 wt.commit(&hyprstream_rpc_std::registry_client::CommitRequest {
794 message,
795 author: String::new(),
796 email: String::new(),
797 })
798 .await
799 .map_err(|e| anyhow!("Failed to commit: {}", e))?;
800
801 if step.is_multiple_of(50000) {
803 let tag_name = if let Some(branch) = branch_name {
804 format!("checkpoint-{branch}-step-{step}")
805 } else {
806 format!("checkpoint-step-{step}")
807 };
808
809 let _ = repo_client
810 .create_tag(&hyprstream_rpc_std::registry_client::CreateTagRequest {
811 name: tag_name,
812 target: String::new(),
813 })
814 .await;
815 }
816
817 Ok(())
818 }
819}
820
821#[derive(Debug, Clone)]
823pub struct CheckpointConfig {
824 pub max_checkpoints: usize,
825 pub git_commit_interval: usize,
826 pub queue_size: usize,
827}
828
829impl Default for CheckpointConfig {
830 fn default() -> Self {
831 Self {
832 max_checkpoints: 5,
833 git_commit_interval: 10000,
834 queue_size: 10,
835 }
836 }
837}
838
839#[derive(Debug, Clone)]
841pub struct CheckpointInfo {
842 pub step: usize,
843 pub path: PathBuf,
844 pub metadata: Option<CheckpointMetadata>,
845}
846
847impl Drop for CheckpointManager {
848 fn drop(&mut self) {
849 if let Some(handle) = self.worker_handle.take() {
851 tracing::warn!(
852 "CheckpointManager dropped without calling shutdown()! \
853 This may cause checkpoint corruption if writes were in progress. \
854 Always call `manager.shutdown().await` before dropping."
855 );
856 handle.abort();
857 }
858 }
859}