Skip to main content

hyprstream_inference/training/
checkpoint.rs

1//! Async checkpoint management for training
2//!
3//! Provides non-blocking checkpoint saving during training,
4//! with commits to model branches for version control.
5//!
6//! Two modes of operation:
7//! - **Direct**: Uses `PathBuf` and `tokio::fs` (when `fs` field is None).
8//! - **FsOps**: Uses `WorktreeClient` for worktree-scoped, path-contained access.
9
10use anyhow::{anyhow, Context, Result};
11use chrono::{DateTime, Utc};
12use serde::{Deserialize, Serialize};
13use std::path::{Path, PathBuf};
14use tokio::fs;
15use tokio::sync::mpsc;
16use tokio::task::JoinHandle;
17
18use crate::worktree_ext::WorktreeClientExt;
19use hyprstream_rpc_std::registry_client::{RepositoryClient, WorktreeClient};
20
21/// Checkpoint request sent to background worker
22#[derive(Debug, Clone)]
23pub struct CheckpointRequest {
24    pub step: usize,
25    pub epoch: Option<usize>,
26    pub weights: WeightSnapshot,
27    pub metrics: Option<TrainingMetrics>,
28    pub timestamp: DateTime<Utc>,
29    pub commit_to_git: bool,
30}
31
32/// Snapshot of model weights for checkpointing
33#[derive(Debug, Clone)]
34pub enum WeightSnapshot {
35    /// Weights copied to memory (for small models/adapters)
36    Memory { data: Vec<u8>, format: WeightFormat },
37
38    /// Path to file that should be copied (for large models)
39    FilePath {
40        source: PathBuf,
41        format: WeightFormat,
42    },
43
44    /// Incremental diff from previous checkpoint
45    Diff { base_step: usize, changes: Vec<u8> },
46}
47
48#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
49pub enum WeightFormat {
50    SafeTensors,
51    PyTorch,
52    AdapterBin,
53}
54
55/// Training metrics to save with checkpoint
56#[derive(Debug, Clone, Serialize, Deserialize)]
57pub struct TrainingMetrics {
58    pub loss: f32,
59    pub learning_rate: f32,
60    pub gradient_norm: Option<f32>,
61    pub validation_loss: Option<f32>,
62    pub validation_accuracy: Option<f32>,
63    pub duration_seconds: f64,
64}
65
66/// Checkpoint metadata
67#[derive(Debug, Clone, Serialize, Deserialize)]
68pub struct CheckpointMetadata {
69    pub step: usize,
70    pub epoch: Option<usize>,
71    pub timestamp: DateTime<Utc>,
72    pub metrics: Option<TrainingMetrics>,
73    pub format: WeightFormat,
74    pub parent_checkpoint: Option<String>,
75}
76
77/// Manages asynchronous checkpointing during training
78pub struct CheckpointManager {
79    model_path: PathBuf,
80    checkpoint_dir: PathBuf,
81    checkpoint_tx: mpsc::Sender<CheckpointRequest>,
82    worker_handle: Option<JoinHandle<()>>,
83    #[allow(dead_code)]
84    max_checkpoints: usize,
85    /// Repository client for git operations (None = git disabled)
86    repo_client: Option<RepositoryClient>,
87    #[allow(dead_code)]
88    branch_name: Option<String>, // Track which branch we're on
89    /// Target adapter to update on each checkpoint (e.g., "01_coding")
90    /// When set, checkpoints will also update adapters/{target_adapter}.safetensors
91    target_adapter: Option<String>,
92    /// WorktreeClient for worktree-scoped file operations (None = direct access)
93    fs: Option<WorktreeClient>,
94}
95
96impl CheckpointManager {
97    /// Create new checkpoint manager (no git operations)
98    pub fn new(model_path: PathBuf) -> Result<Self> {
99        Self::with_config(model_path, CheckpointConfig::default(), None, None)
100    }
101
102    /// Create checkpoint manager with a repository client for git operations
103    pub fn with_repo_client(model_path: PathBuf, repo_client: RepositoryClient) -> Result<Self> {
104        Self::with_config(
105            model_path,
106            CheckpointConfig::default(),
107            None,
108            Some(repo_client),
109        )
110    }
111
112    /// Create checkpoint manager for an adapter branch
113    pub fn for_adapter(
114        model_path: PathBuf,
115        branch_name: String,
116        repo_client: Option<RepositoryClient>,
117    ) -> Result<Self> {
118        Self::with_config(
119            model_path,
120            CheckpointConfig::default(),
121            Some(branch_name),
122            repo_client,
123        )
124    }
125
126    /// Create with custom configuration
127    pub fn with_config(
128        model_path: PathBuf,
129        config: CheckpointConfig,
130        branch_name: Option<String>,
131        repo_client: Option<RepositoryClient>,
132    ) -> Result<Self> {
133        let checkpoint_dir = model_path.join(".checkpoints");
134        std::fs::create_dir_all(&checkpoint_dir)?;
135
136        // Derive FsOps from repo_client if available
137        let fs: Option<WorktreeClient> = repo_client.as_ref().map(|client| {
138            let branch = branch_name.as_deref().unwrap_or("main");
139            client.worktree(branch)
140        });
141
142        let (tx, rx) = mpsc::channel(config.queue_size);
143
144        // Spawn checkpoint worker thread
145        let worker_handle = {
146            let max_checkpoints = config.max_checkpoints;
147            let git_interval = config.git_commit_interval;
148            let branch = branch_name.clone();
149            let client = repo_client.clone();
150            let worker_fs = fs.clone();
151            let worker_checkpoint_dir = checkpoint_dir.clone();
152
153            tokio::spawn(async move {
154                Self::checkpoint_worker(
155                    rx,
156                    worker_checkpoint_dir,
157                    max_checkpoints,
158                    git_interval,
159                    client,
160                    branch,
161                    worker_fs,
162                )
163                .await;
164            })
165        };
166
167        Ok(Self {
168            model_path,
169            checkpoint_dir,
170            checkpoint_tx: tx,
171            worker_handle: Some(worker_handle),
172            max_checkpoints: config.max_checkpoints,
173            repo_client,
174            branch_name,
175            target_adapter: None,
176            fs,
177        })
178    }
179
180    /// Set target adapter to update on each checkpoint
181    ///
182    /// When set, each checkpoint will also copy weights to
183    /// `adapters/{target_adapter}.safetensors` for live inference.
184    pub fn with_target_adapter(mut self, adapter_name: String) -> Self {
185        self.target_adapter = Some(adapter_name);
186        self
187    }
188
189    /// Get the target adapter name if set
190    pub fn target_adapter(&self) -> Option<&str> {
191        self.target_adapter.as_deref()
192    }
193
194    /// Request asynchronous checkpoint
195    pub async fn checkpoint(&self, request: CheckpointRequest) -> Result<()> {
196        self.checkpoint_tx
197            .send(request)
198            .await
199            .context("Failed to send checkpoint request")?;
200        Ok(())
201    }
202
203    /// Create checkpoint request for current training state
204    pub fn create_request(
205        step: usize,
206        weights: WeightSnapshot,
207        metrics: Option<TrainingMetrics>,
208    ) -> CheckpointRequest {
209        CheckpointRequest {
210            step,
211            epoch: None,
212            weights,
213            metrics,
214            timestamp: Utc::now(),
215            commit_to_git: step.is_multiple_of(10000), // Commit every 10k steps by default
216        }
217    }
218
219    /// Write checkpoint to filesystem without Git commit
220    /// Note: Uses single checkpoint.safetensors file - git handles versioning
221    pub async fn write_checkpoint(
222        &self,
223        weights: WeightSnapshot,
224        step: usize,
225        metadata: Option<TrainingMetrics>,
226    ) -> Result<PathBuf> {
227        // Single checkpoint file - git tracks version history
228        let checkpoint_path = self.checkpoint_dir.join("checkpoint.safetensors");
229
230        if let Some(fs) = &self.fs {
231            // FsOps path: use relative paths within worktree
232            match weights {
233                WeightSnapshot::Memory {
234                    ref data,
235                    format: _,
236                } => {
237                    fs.write_file_chunked(".checkpoints/checkpoint.safetensors", data)
238                        .await?;
239                }
240                WeightSnapshot::FilePath {
241                    ref source,
242                    format: _,
243                } => {
244                    // source is an absolute path, need to handle this:
245                    // If source is within worktree, compute relative path for copy.
246                    // Otherwise fall back to reading the source and writing via FsOps.
247                    if let Ok(rel) = source.strip_prefix(&self.model_path) {
248                        let rel_str = rel.to_string_lossy();
249                        fs.copy_path(&rel_str, ".checkpoints/checkpoint.safetensors")
250                            .await?;
251                    } else {
252                        // Source outside worktree - read it directly and write via FsOps
253                        let data = tokio::fs::read(source).await?;
254                        fs.write_file_chunked(".checkpoints/checkpoint.safetensors", &data)
255                            .await?;
256                    }
257                }
258                WeightSnapshot::Diff {
259                    base_step,
260                    ref changes,
261                } => {
262                    let rel_diff = format!(".checkpoints/diff_from_{base_step}_to_{step}.bin");
263                    fs.write_file_chunked(&rel_diff, changes).await?;
264                    return Ok(self
265                        .checkpoint_dir
266                        .join(format!("diff_from_{base_step}_to_{step}.bin")));
267                }
268            }
269
270            // Save metadata if provided
271            if let Some(metrics) = metadata {
272                let ckpt_meta = CheckpointMetadata {
273                    step,
274                    epoch: None,
275                    timestamp: Utc::now(),
276                    metrics: Some(metrics),
277                    format: WeightFormat::SafeTensors,
278                    parent_checkpoint: None,
279                };
280                let metadata_json = serde_json::to_string_pretty(&ckpt_meta)?;
281                fs.write_file_chunked(".checkpoints/checkpoint.json", metadata_json.as_bytes())
282                    .await?;
283            }
284
285            // Update target adapter if configured
286            if let Some(ref adapter_name) = self.target_adapter {
287                fs.mkdir_p("adapters").await?;
288                let rel_adapter = format!("adapters/{adapter_name}.safetensors");
289                fs.copy_path(".checkpoints/checkpoint.safetensors", &rel_adapter)
290                    .await?;
291                tracing::info!("Updated target adapter via FsOps: {}", rel_adapter);
292            }
293        } else {
294            // Direct path (original behavior)
295            match weights {
296                WeightSnapshot::Memory {
297                    ref data,
298                    format: _,
299                } => {
300                    fs::write(&checkpoint_path, data).await?;
301                }
302                WeightSnapshot::FilePath {
303                    ref source,
304                    format: _,
305                } => {
306                    // Validate source is a regular file (not a symlink to an external location)
307                    let metadata = std::fs::symlink_metadata(source)
308                        .map_err(|e| anyhow!("Cannot stat source file {:?}: {}", source, e))?;
309                    if metadata.file_type().is_symlink() {
310                        return Err(anyhow!(
311                            "Source path {:?} is a symlink; refusing to copy",
312                            source
313                        ));
314                    }
315                    crate::worktree_ext::cow_copy(source, &checkpoint_path)?;
316                }
317                WeightSnapshot::Diff {
318                    base_step,
319                    ref changes,
320                } => {
321                    let diff_path = self
322                        .checkpoint_dir
323                        .join(format!("diff_from_{base_step}_to_{step}.bin"));
324                    fs::write(&diff_path, changes).await?;
325                    return Ok(diff_path);
326                }
327            }
328
329            // Save metadata if provided
330            if let Some(metrics) = metadata {
331                let ckpt_meta = CheckpointMetadata {
332                    step,
333                    epoch: None,
334                    timestamp: Utc::now(),
335                    metrics: Some(metrics),
336                    format: WeightFormat::SafeTensors,
337                    parent_checkpoint: None,
338                };
339                let metadata_path = checkpoint_path.with_extension("json");
340                let metadata_json = serde_json::to_string_pretty(&ckpt_meta)?;
341                fs::write(&metadata_path, metadata_json).await?;
342            }
343
344            // Update target adapter if configured
345            if let Some(ref adapter_name) = self.target_adapter {
346                self.update_target_adapter(&checkpoint_path, adapter_name)
347                    .await?;
348            }
349        }
350
351        tracing::info!(
352            "Checkpoint written to filesystem: {}",
353            checkpoint_path.display()
354        );
355        Ok(checkpoint_path)
356    }
357
358    /// Update the target adapter file with checkpoint weights (direct path only)
359    async fn update_target_adapter(
360        &self,
361        checkpoint_path: &Path,
362        adapter_name: &str,
363    ) -> Result<()> {
364        let adapters_dir = self.model_path.join("adapters");
365        fs::create_dir_all(&adapters_dir).await?;
366
367        let adapter_path = adapters_dir.join(format!("{adapter_name}.safetensors"));
368        crate::worktree_ext::cow_copy(checkpoint_path, &adapter_path)?;
369
370        tracing::info!("Updated target adapter: {}", adapter_path.display());
371        Ok(())
372    }
373
374    /// Commit existing checkpoint to Git (separate from write)
375    pub async fn commit_checkpoint(
376        &self,
377        checkpoint_path: &Path,
378        message: Option<String>,
379        branch: Option<String>,
380    ) -> Result<String> {
381        let repo_client = self
382            .repo_client
383            .as_ref()
384            .ok_or_else(|| anyhow!("Git is not enabled (no RepositoryClient configured)"))?;
385
386        // Determine worktree branch name
387        let wt_branch = if let Some(ref branch_name) = branch {
388            // create_branch may fail if it already exists — that's fine
389            let _ = repo_client
390                .create_branch(&hyprstream_rpc_std::registry_client::BranchRequest {
391                    branch_name: branch_name.clone(),
392                    start_point: String::new(),
393                })
394                .await;
395            // Checkout within the worktree
396            repo_client
397                .worktree(branch_name)
398                .checkout(&hyprstream_rpc_std::registry_client::CheckoutRequest {
399                    ref_name: branch_name.clone(),
400                    create_branch: false,
401                })
402                .await
403                .map_err(|e| anyhow!("Failed to checkout branch '{}': {}", branch_name, e))?;
404            branch_name.clone()
405        } else {
406            // Use default branch (HEAD)
407            repo_client
408                .get_head()
409                .await
410                .unwrap_or_else(|_| "main".to_owned())
411        };
412        let wt = repo_client.worktree(&wt_branch);
413
414        // Stage checkpoint files
415        let relative_path = checkpoint_path
416            .strip_prefix(&self.model_path)
417            .unwrap_or(checkpoint_path);
418        let mut files_to_stage: Vec<&str> = vec![relative_path.to_str().unwrap_or("")];
419
420        let metadata_path = checkpoint_path.with_extension("json");
421        let relative_metadata;
422
423        let meta_exists = if let Some(fs) = &self.fs {
424            let rel = metadata_path
425                .strip_prefix(&self.model_path)
426                .unwrap_or(&metadata_path)
427                .to_string_lossy()
428                .to_string();
429            fs.stat_path(&rel).await.map(|s| s.exists).unwrap_or(false)
430        } else {
431            metadata_path.exists()
432        };
433
434        if meta_exists {
435            relative_metadata = metadata_path
436                .strip_prefix(&self.model_path)
437                .unwrap_or(&metadata_path)
438                .to_string_lossy()
439                .to_string();
440            files_to_stage.push(&relative_metadata);
441        }
442
443        let files_owned: Vec<String> = files_to_stage.iter().map(|s| (*s).to_owned()).collect();
444        wt.stage_files(&hyprstream_rpc_std::registry_client::StageFilesRequest {
445            files: files_owned,
446        })
447        .await
448        .map_err(|e| anyhow!("Failed to stage files: {}", e))?;
449
450        let commit_message = message.unwrap_or_else(|| {
451            format!(
452                "Training checkpoint: {}",
453                checkpoint_path
454                    .file_name()
455                    .and_then(|n| n.to_str())
456                    .unwrap_or("unknown")
457            )
458        });
459
460        let oid = wt
461            .commit(&hyprstream_rpc_std::registry_client::CommitRequest {
462                message: commit_message,
463                author: String::new(),
464                email: String::new(),
465            })
466            .await
467            .map_err(|e| anyhow!("Failed to commit: {}", e))?;
468
469        tracing::info!("Checkpoint committed to Git: {}", oid);
470        Ok(oid)
471    }
472
473    /// Wait for all pending checkpoints to complete
474    pub async fn flush(&self) -> Result<()> {
475        // Send a sentinel value to ensure all previous requests are processed
476        let sentinel = CheckpointRequest {
477            step: usize::MAX,
478            epoch: None,
479            weights: WeightSnapshot::Memory {
480                data: vec![],
481                format: WeightFormat::SafeTensors,
482            },
483            metrics: None,
484            timestamp: Utc::now(),
485            commit_to_git: false,
486        };
487
488        self.checkpoint(sentinel).await?;
489
490        // Give worker time to process
491        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
492
493        Ok(())
494    }
495
496    /// Gracefully shutdown the checkpoint manager
497    pub async fn shutdown(mut self) -> Result<()> {
498        // Drop the sender first to prevent new requests, then let worker drain
499        drop(std::mem::replace(
500            &mut self.checkpoint_tx,
501            mpsc::channel(1).0,
502        ));
503
504        // Wait for worker to complete
505        if let Some(handle) = self.worker_handle.take() {
506            match tokio::time::timeout(tokio::time::Duration::from_secs(30), handle).await {
507                Ok(Ok(())) => {
508                    tracing::debug!("CheckpointManager worker shut down cleanly");
509                }
510                Ok(Err(e)) => {
511                    tracing::warn!("CheckpointManager worker panicked during shutdown: {}", e);
512                }
513                Err(_) => {
514                    tracing::error!(
515                        "CheckpointManager worker timed out during shutdown - \
516                         checkpoint may be incomplete"
517                    );
518                }
519            }
520        }
521
522        Ok(())
523    }
524
525    /// Check if there are pending checkpoints in the queue
526    pub fn has_pending(&self) -> bool {
527        !self.checkpoint_tx.is_closed()
528    }
529
530    /// Get current checkpoint info (single file, git handles history)
531    pub async fn get_checkpoint(&self) -> Result<Option<CheckpointInfo>> {
532        if let Some(fs) = &self.fs {
533            if !fs
534                .stat_path(".checkpoints/checkpoint.json")
535                .await
536                .map(|s| s.exists)
537                .unwrap_or(false)
538            {
539                return Ok(None);
540            }
541            let json =
542                String::from_utf8(fs.read_file_chunked(".checkpoints/checkpoint.json").await?)?;
543            let metadata: CheckpointMetadata = serde_json::from_str(&json)?;
544            Ok(Some(CheckpointInfo {
545                step: metadata.step,
546                path: self.checkpoint_dir.clone(),
547                metadata: Some(metadata),
548            }))
549        } else {
550            let metadata_path = self.checkpoint_dir.join("checkpoint.json");
551            if !metadata_path.exists() {
552                return Ok(None);
553            }
554            let json = fs::read_to_string(&metadata_path).await?;
555            let metadata: CheckpointMetadata = serde_json::from_str(&json)?;
556            Ok(Some(CheckpointInfo {
557                step: metadata.step,
558                path: self.checkpoint_dir.clone(),
559                metadata: Some(metadata),
560            }))
561        }
562    }
563
564    /// Load current checkpoint weights path
565    pub async fn load_checkpoint(&self) -> Result<PathBuf> {
566        let checkpoint_path = self.checkpoint_dir.join("checkpoint.safetensors");
567
568        if let Some(fs) = &self.fs {
569            if !fs
570                .stat_path(".checkpoints/checkpoint.safetensors")
571                .await
572                .map(|s| s.exists)
573                .unwrap_or(false)
574            {
575                anyhow::bail!("No checkpoint found at {:?}", checkpoint_path);
576            }
577        } else if !checkpoint_path.exists() {
578            anyhow::bail!("No checkpoint found at {:?}", checkpoint_path);
579        }
580
581        Ok(checkpoint_path)
582    }
583
584    /// Worker that processes checkpoint requests in background
585    async fn checkpoint_worker(
586        mut rx: mpsc::Receiver<CheckpointRequest>,
587        checkpoint_dir: PathBuf,
588        _max_checkpoints: usize,
589        _git_interval: usize,
590        repo_client: Option<RepositoryClient>,
591        branch_name: Option<String>,
592        fs: Option<WorktreeClient>,
593    ) {
594        while let Some(request) = rx.recv().await {
595            // Skip sentinel values
596            if request.step == usize::MAX {
597                continue;
598            }
599
600            if let Err(e) = Self::process_checkpoint(
601                &checkpoint_dir,
602                request,
603                repo_client.as_ref(),
604                &branch_name,
605                fs.as_ref(),
606            )
607            .await
608            {
609                tracing::error!("Failed to process checkpoint: {}", e);
610            }
611        }
612    }
613
614    /// Process a single checkpoint request
615    async fn process_checkpoint(
616        checkpoint_dir: &Path,
617        request: CheckpointRequest,
618        repo_client: Option<&RepositoryClient>,
619        branch_name: &Option<String>,
620        fs: Option<&WorktreeClient>,
621    ) -> Result<()> {
622        if let Some(fs) = fs {
623            // FsOps path
624            fs.mkdir_p(".checkpoints").await?;
625
626            // Save weights
627            match request.weights {
628                WeightSnapshot::Memory { ref data, format } => {
629                    let filename = match format {
630                        WeightFormat::SafeTensors => "checkpoint.safetensors",
631                        WeightFormat::PyTorch => "checkpoint.bin",
632                        WeightFormat::AdapterBin => "checkpoint_adapter.bin",
633                    };
634                    let rel_path = format!(".checkpoints/{filename}");
635                    fs.write_file_chunked(&rel_path, data).await?;
636                }
637                WeightSnapshot::FilePath { ref source, format } => {
638                    let filename = match format {
639                        WeightFormat::SafeTensors => "checkpoint.safetensors",
640                        WeightFormat::PyTorch => "checkpoint.bin",
641                        WeightFormat::AdapterBin => "checkpoint_adapter.bin",
642                    };
643                    let rel_dest = format!(".checkpoints/{filename}");
644                    // Source may be outside worktree; read and write if so
645                    let data = tokio::fs::read(source).await?;
646                    fs.write_file_chunked(&rel_dest, &data).await?;
647                }
648                WeightSnapshot::Diff {
649                    base_step: _,
650                    ref changes,
651                } => {
652                    fs.write_file_chunked(".checkpoints/checkpoint_diff.bin", changes)
653                        .await?;
654                }
655            }
656
657            // Save metadata
658            let metadata = CheckpointMetadata {
659                step: request.step,
660                epoch: request.epoch,
661                timestamp: request.timestamp,
662                metrics: request.metrics,
663                format: match &request.weights {
664                    WeightSnapshot::Memory { format, .. }
665                    | WeightSnapshot::FilePath { format, .. } => *format,
666                    WeightSnapshot::Diff { .. } => WeightFormat::SafeTensors,
667                },
668                parent_checkpoint: None,
669            };
670            let metadata_json = serde_json::to_string_pretty(&metadata)?;
671            fs.write_file_chunked(".checkpoints/checkpoint.json", metadata_json.as_bytes())
672                .await?;
673        } else {
674            // Direct path (original behavior)
675            tokio::fs::create_dir_all(checkpoint_dir).await?;
676
677            let _weights_file = match request.weights {
678                WeightSnapshot::Memory { ref data, format } => {
679                    let filename = match format {
680                        WeightFormat::SafeTensors => "checkpoint.safetensors",
681                        WeightFormat::PyTorch => "checkpoint.bin",
682                        WeightFormat::AdapterBin => "checkpoint_adapter.bin",
683                    };
684                    let path = checkpoint_dir.join(filename);
685                    tokio::fs::write(&path, data).await?;
686                    path
687                }
688                WeightSnapshot::FilePath { ref source, format } => {
689                    let filename = match format {
690                        WeightFormat::SafeTensors => "checkpoint.safetensors",
691                        WeightFormat::PyTorch => "checkpoint.bin",
692                        WeightFormat::AdapterBin => "checkpoint_adapter.bin",
693                    };
694                    let dest = checkpoint_dir.join(filename);
695                    crate::worktree_ext::cow_copy(source, &dest)?;
696                    dest
697                }
698                WeightSnapshot::Diff {
699                    base_step: _,
700                    ref changes,
701                } => {
702                    let diff_path = checkpoint_dir.join("checkpoint_diff.bin");
703                    tokio::fs::write(&diff_path, changes).await?;
704                    diff_path
705                }
706            };
707
708            // Save metadata
709            let metadata = CheckpointMetadata {
710                step: request.step,
711                epoch: request.epoch,
712                timestamp: request.timestamp,
713                metrics: request.metrics,
714                format: match &request.weights {
715                    WeightSnapshot::Memory { format, .. }
716                    | WeightSnapshot::FilePath { format, .. } => *format,
717                    WeightSnapshot::Diff { .. } => WeightFormat::SafeTensors,
718                },
719                parent_checkpoint: None,
720            };
721            let metadata_json = serde_json::to_string_pretty(&metadata)?;
722            tokio::fs::write(checkpoint_dir.join("checkpoint.json"), metadata_json).await?;
723        }
724
725        // Optionally commit to Git via RepositoryClient
726        if let Some(client) = repo_client {
727            if request.commit_to_git {
728                if let Err(e) = Self::git_commit_checkpoint(client, request.step, branch_name).await
729                {
730                    tracing::warn!("Failed to commit checkpoint to Git: {}", e);
731                }
732            }
733        }
734
735        tracing::info!(
736            "Checkpoint saved: step {} at {}",
737            request.step,
738            checkpoint_dir.display()
739        );
740
741        Ok(())
742    }
743
744    /// Commit checkpoint to Git branch via RepositoryClient
745    async fn git_commit_checkpoint(
746        repo_client: &RepositoryClient,
747        step: usize,
748        branch_name: &Option<String>,
749    ) -> Result<()> {
750        // Determine worktree branch name
751        let wt_branch = if let Some(branch) = branch_name {
752            // create_branch may fail if it already exists — that's fine
753            let _ = repo_client
754                .create_branch(&hyprstream_rpc_std::registry_client::BranchRequest {
755                    branch_name: branch.clone(),
756                    start_point: String::new(),
757                })
758                .await;
759            repo_client
760                .worktree(branch)
761                .checkout(&hyprstream_rpc_std::registry_client::CheckoutRequest {
762                    ref_name: branch.clone(),
763                    create_branch: false,
764                })
765                .await
766                .map_err(|e| anyhow!("Failed to checkout branch '{}': {}", branch, e))?;
767            branch.clone()
768        } else {
769            repo_client
770                .get_head()
771                .await
772                .unwrap_or_else(|_| "main".to_owned())
773        };
774        let wt = repo_client.worktree(&wt_branch);
775
776        // Stage checkpoint files
777        wt.stage_files(&hyprstream_rpc_std::registry_client::StageFilesRequest {
778            files: vec![
779                ".checkpoints/checkpoint.safetensors".to_owned(),
780                ".checkpoints/checkpoint.json".to_owned(),
781            ],
782        })
783        .await
784        .map_err(|e| anyhow!("Failed to stage checkpoint files: {}", e))?;
785
786        // Commit
787        let message = if let Some(name) = branch_name.as_ref() {
788            format!("Training checkpoint step {step} (branch: {name})")
789        } else {
790            format!("Training checkpoint step {step}")
791        };
792
793        wt.commit(&hyprstream_rpc_std::registry_client::CommitRequest {
794            message,
795            author: String::new(),
796            email: String::new(),
797        })
798        .await
799        .map_err(|e| anyhow!("Failed to commit: {}", e))?;
800
801        // For major checkpoints, create human-friendly tags
802        if step.is_multiple_of(50000) {
803            let tag_name = if let Some(branch) = branch_name {
804                format!("checkpoint-{branch}-step-{step}")
805            } else {
806                format!("checkpoint-step-{step}")
807            };
808
809            let _ = repo_client
810                .create_tag(&hyprstream_rpc_std::registry_client::CreateTagRequest {
811                    name: tag_name,
812                    target: String::new(),
813                })
814                .await;
815        }
816
817        Ok(())
818    }
819}
820
821/// Checkpoint configuration
822#[derive(Debug, Clone)]
823pub struct CheckpointConfig {
824    pub max_checkpoints: usize,
825    pub git_commit_interval: usize,
826    pub queue_size: usize,
827}
828
829impl Default for CheckpointConfig {
830    fn default() -> Self {
831        Self {
832            max_checkpoints: 5,
833            git_commit_interval: 10000,
834            queue_size: 10,
835        }
836    }
837}
838
839/// Information about a checkpoint
840#[derive(Debug, Clone)]
841pub struct CheckpointInfo {
842    pub step: usize,
843    pub path: PathBuf,
844    pub metadata: Option<CheckpointMetadata>,
845}
846
847impl Drop for CheckpointManager {
848    fn drop(&mut self) {
849        // Check if shutdown was called properly
850        if let Some(handle) = self.worker_handle.take() {
851            tracing::warn!(
852                "CheckpointManager dropped without calling shutdown()! \
853                 This may cause checkpoint corruption if writes were in progress. \
854                 Always call `manager.shutdown().await` before dropping."
855            );
856            handle.abort();
857        }
858    }
859}