Skip to main content

hyprstream_metrics/checkpoint/
manager.rs

1//! Checkpoint manager for git2db-based versioned storage.
2
3use crate::checkpoint::state::{Checkpoint, CheckpointMetadata, TableCheckpoint};
4use crate::storage::duckdb::DuckDbBackend;
5use crate::storage::StorageBackend;
6use async_trait::async_trait;
7use chrono::Utc;
8use git2db::{GitManager, RepoId, TrackedRepository};
9
10/// Minimal registry client trait for checkpoint integration.
11///
12/// This is a subset of the full `RegistryClient` trait from hyprstream_core,
13/// containing only the methods needed for checkpoint management.
14#[async_trait]
15pub trait RegistryClient: Send + Sync {
16    /// Get repository by name
17    async fn get_by_name(&self, name: &str) -> Result<Option<TrackedRepository>, RegistryError>;
18
19    /// Register an existing repository
20    async fn register(
21        &self,
22        id: &RepoId,
23        name: Option<&str>,
24        path: &std::path::Path,
25    ) -> Result<(), RegistryError>;
26}
27
28/// Error type for registry operations in checkpoint context
29#[derive(Debug, thiserror::Error)]
30pub enum RegistryError {
31    #[error("Registry operation failed: {0}")]
32    Operation(String),
33}
34use std::path::PathBuf;
35use std::sync::Arc;
36use std::time::Duration;
37use tokio::sync::RwLock;
38use tokio::task::JoinHandle;
39use tonic::Status;
40use tracing::{debug, error, info, instrument};
41
42/// Configuration for checkpoint behavior
43#[derive(Debug, Clone)]
44pub struct CheckpointConfig {
45    /// Interval between automatic checkpoints
46    pub interval: Duration,
47    /// Number of checkpoints to retain (0 = unlimited)
48    pub retention_count: usize,
49    /// Whether to create a checkpoint on shutdown
50    pub checkpoint_on_shutdown: bool,
51    /// Base directory for checkpoint storage
52    pub base_dir: PathBuf,
53}
54
55impl Default for CheckpointConfig {
56    fn default() -> Self {
57        Self {
58            interval: Duration::from_secs(5 * 60), // 5 minutes
59            retention_count: 10,
60            checkpoint_on_shutdown: true,
61            base_dir: PathBuf::from(".checkpoints"),
62        }
63    }
64}
65
66impl CheckpointConfig {
67    /// Create a new config with specified interval
68    pub fn with_interval(mut self, interval: Duration) -> Self {
69        self.interval = interval;
70        self
71    }
72
73    /// Set retention count
74    pub fn with_retention(mut self, count: usize) -> Self {
75        self.retention_count = count;
76        self
77    }
78
79    /// Set base directory
80    pub fn with_base_dir(mut self, dir: impl Into<PathBuf>) -> Self {
81        self.base_dir = dir.into();
82        self
83    }
84}
85
86/// Manages checkpoints for metrics storage
87pub struct CheckpointManager {
88    /// Configuration
89    config: CheckpointConfig,
90    /// Shared registry client (if using shared registry)
91    client: Option<Arc<dyn RegistryClient>>,
92    /// Repository name in registry (if using shared registry)
93    repo_name: Option<String>,
94    /// Path to the checkpoint repository
95    repo_path: PathBuf,
96    /// List of recent checkpoints (cached)
97    checkpoints: RwLock<Vec<Checkpoint>>,
98}
99
100impl CheckpointManager {
101    /// Create a new checkpoint manager (standalone mode)
102    ///
103    /// This creates a local git repository that is not tracked in any registry.
104    /// For shared registry integration, use `new_with_client()` instead.
105    #[instrument(skip(config), fields(base_dir = ?config.base_dir))]
106    pub async fn new(config: CheckpointConfig) -> Result<Self, Status> {
107        Self::create_internal(config, None, None).await
108    }
109
110    /// Create a new checkpoint manager with shared registry client
111    ///
112    /// The checkpoint repository will be registered in the shared registry,
113    /// allowing it to be discovered and managed alongside other repositories.
114    ///
115    /// # Arguments
116    /// - `client` - Shared registry client
117    /// - `repo_name` - Name for the checkpoint repository (e.g., "metrics-checkpoints")
118    /// - `config` - Checkpoint configuration
119    #[instrument(skip(client, config), fields(repo_name = %repo_name, base_dir = ?config.base_dir))]
120    pub async fn new_with_client(
121        client: Arc<dyn RegistryClient>,
122        repo_name: &str,
123        config: CheckpointConfig,
124    ) -> Result<Self, Status> {
125        info!("Initializing CheckpointManager with shared registry");
126
127        // Check if repo already exists in registry
128        let existing = client
129            .get_by_name(repo_name)
130            .await
131            .map_err(|e| Status::internal(format!("Failed to query registry: {e}")))?;
132
133        if let Some(tracked) = existing {
134            // Repository already registered - use its path
135            let repo_path = PathBuf::from(&tracked.worktree_path);
136            info!(
137                "Using existing registered checkpoint repository at {:?}",
138                repo_path
139            );
140
141            return Ok(Self {
142                config,
143                client: Some(client),
144                repo_name: Some(repo_name.to_owned()),
145                repo_path,
146                checkpoints: RwLock::new(Vec::new()),
147            });
148        }
149
150        // Create the repository and register it
151        let manager =
152            Self::create_internal(config, Some(client.clone()), Some(repo_name.to_owned())).await?;
153
154        // Register with the registry
155        let repo_id = RepoId::new();
156        client
157            .register(&repo_id, Some(repo_name), &manager.repo_path)
158            .await
159            .map_err(|e| Status::internal(format!("Failed to register checkpoint repo: {e}")))?;
160
161        info!(
162            "Registered checkpoint repository '{}' in registry",
163            repo_name
164        );
165
166        Ok(manager)
167    }
168
169    /// Internal helper to create the checkpoint manager
170    async fn create_internal(
171        config: CheckpointConfig,
172        client: Option<Arc<dyn RegistryClient>>,
173        repo_name: Option<String>,
174    ) -> Result<Self, Status> {
175        info!("Initializing CheckpointManager");
176
177        // Ensure base directory exists
178        std::fs::create_dir_all(&config.base_dir)
179            .map_err(|e| Status::internal(format!("Failed to create checkpoint directory: {e}")))?;
180
181        // Create or get the checkpoint repository
182        let repo_path = config.base_dir.join("repo");
183
184        // Initialize git repository if it doesn't exist
185        if !repo_path.join(".git").exists() {
186            std::fs::create_dir_all(&repo_path).map_err(|e| {
187                Status::internal(format!("Failed to create repository directory: {e}"))
188            })?;
189
190            git2::Repository::init(&repo_path).map_err(|e| {
191                Status::internal(format!("Failed to initialize git repository: {e}"))
192            })?;
193
194            info!("Created new checkpoint repository at {:?}", repo_path);
195        } else {
196            info!("Using existing checkpoint repository at {:?}", repo_path);
197        }
198
199        Ok(Self {
200            config,
201            client,
202            repo_name,
203            repo_path,
204            checkpoints: RwLock::new(Vec::new()),
205        })
206    }
207
208    /// Create a checkpoint from the current storage state
209    #[instrument(skip(self, storage), fields(checkpoint_name = ?name))]
210    pub async fn create_checkpoint(
211        &self,
212        storage: &DuckDbBackend,
213        name: Option<String>,
214    ) -> Result<Checkpoint, Status> {
215        info!("Creating checkpoint");
216
217        // Get list of tables
218        let tables = storage.list_tables().await?;
219        debug!(table_count = tables.len(), "Found tables to checkpoint");
220
221        // Ensure tables directory exists
222        let tables_dir = self.repo_path.join("tables");
223        std::fs::create_dir_all(&tables_dir)
224            .map_err(|e| Status::internal(format!("Failed to create tables directory: {e}")))?;
225
226        // Export each table to Parquet
227        let mut table_checkpoints = Vec::new();
228        for table_name in &tables {
229            // Skip internal tables
230            if table_name.starts_with("sqlite_") || table_name == "view_metadata" {
231                continue;
232            }
233
234            let parquet_path = tables_dir.join(format!("{table_name}.parquet"));
235            debug!(table = %table_name, path = ?parquet_path, "Exporting table");
236
237            storage.export_to_parquet(table_name, &parquet_path).await?;
238
239            // Get file size
240            let file_size = std::fs::metadata(&parquet_path)
241                .map(|m| m.len())
242                .unwrap_or(0);
243
244            // Get row count (estimate from file size for now)
245            // TODO: Query actual row count from DuckDB
246            let row_count = 0u64; // Placeholder
247
248            table_checkpoints.push(TableCheckpoint::new(
249                table_name.clone(),
250                row_count,
251                file_size,
252                format!("tables/{table_name}.parquet"),
253            ));
254        }
255
256        // Create checkpoint metadata
257        let mut metadata = CheckpointMetadata::new(table_checkpoints);
258        if let Some(n) = name.clone() {
259            metadata = metadata.with_name(n);
260        }
261        metadata = metadata.with_metadata("created_by", "checkpoint_manager");
262
263        // Save metadata file
264        let metadata_path = self.repo_path.join("checkpoint.json");
265        let metadata_json = serde_json::to_string_pretty(&metadata).map_err(|e| {
266            Status::internal(format!("Failed to serialize checkpoint metadata: {e}"))
267        })?;
268        std::fs::write(&metadata_path, metadata_json)
269            .map_err(|e| Status::internal(format!("Failed to write checkpoint metadata: {e}")))?;
270
271        // Commit the checkpoint
272        let commit_id = self.commit_checkpoint(&metadata, name.as_deref()).await?;
273
274        // Update metadata with commit ID
275        let mut final_metadata = metadata;
276        final_metadata.id = commit_id.clone();
277
278        let checkpoint = Checkpoint::new(commit_id, final_metadata);
279
280        // Update cache
281        {
282            let mut checkpoints = self.checkpoints.write().await;
283            checkpoints.push(checkpoint.clone());
284
285            // Enforce retention policy
286            if self.config.retention_count > 0 && checkpoints.len() > self.config.retention_count {
287                checkpoints.remove(0);
288            }
289        }
290
291        info!(checkpoint_id = %checkpoint.id(), "Checkpoint created successfully");
292        Ok(checkpoint)
293    }
294
295    /// Commit the checkpoint to git
296    async fn commit_checkpoint(
297        &self,
298        metadata: &CheckpointMetadata,
299        name: Option<&str>,
300    ) -> Result<String, Status> {
301        // Use GitManager for repository access
302        let repo_cache = GitManager::global()
303            .get_repository(&self.repo_path)
304            .map_err(|e| Status::internal(format!("Failed to get repository: {e}")))?;
305
306        let repo = repo_cache
307            .open()
308            .map_err(|e| Status::internal(format!("Failed to open repository: {e}")))?;
309
310        // Stage all files using git2 directly
311        let mut index = repo
312            .index()
313            .map_err(|e| Status::internal(format!("Failed to get index: {e}")))?;
314        index
315            .add_all(["*"].iter(), git2::IndexAddOption::DEFAULT, None)
316            .map_err(|e| Status::internal(format!("Failed to stage files: {e}")))?;
317        index
318            .write()
319            .map_err(|e| Status::internal(format!("Failed to write index: {e}")))?;
320
321        let sig = GitManager::global()
322            .create_signature(None, None)
323            .map_err(|e| Status::internal(format!("Failed to create signature: {e}")))?;
324
325        let message = format!(
326            "Checkpoint: {}\n\nTables: {}\nTotal size: {} bytes",
327            name.unwrap_or(&metadata.created_at.to_rfc3339()),
328            metadata.tables.len(),
329            metadata.total_size
330        );
331
332        // Get tree from index
333        let mut index = repo
334            .index()
335            .map_err(|e| Status::internal(format!("Failed to get index: {e}")))?;
336        let tree_id = index
337            .write_tree()
338            .map_err(|e| Status::internal(format!("Failed to write tree: {e}")))?;
339        let tree = repo
340            .find_tree(tree_id)
341            .map_err(|e| Status::internal(format!("Failed to find tree: {e}")))?;
342
343        // Get parent commit if exists
344        let parent = repo.head().ok().and_then(|h| h.peel_to_commit().ok());
345
346        // Create commit
347        let commit_id = if let Some(parent) = parent {
348            repo.commit(Some("HEAD"), &sig, &sig, &message, &tree, &[&parent])
349        } else {
350            repo.commit(Some("HEAD"), &sig, &sig, &message, &tree, &[])
351        }
352        .map_err(|e| Status::internal(format!("Failed to create commit: {e}")))?;
353
354        // Create tag if name provided
355        if let Some(tag_name) = name {
356            if let Ok(obj) = repo.find_object(commit_id, None) {
357                let _ = repo.tag_lightweight(tag_name, &obj, true);
358            }
359        }
360
361        Ok(commit_id.to_string())
362    }
363
364    /// Restore from a checkpoint
365    #[instrument(skip(self, storage), fields(checkpoint_id = %checkpoint_id))]
366    pub async fn restore(
367        &self,
368        checkpoint_id: &str,
369        storage: &DuckDbBackend,
370    ) -> Result<(), Status> {
371        info!("Restoring from checkpoint");
372
373        // Use GitManager for repository access
374        let repo_cache = GitManager::global()
375            .get_repository(&self.repo_path)
376            .map_err(|e| Status::internal(format!("Failed to get repository: {e}")))?;
377
378        let repo = repo_cache
379            .open()
380            .map_err(|e| Status::internal(format!("Failed to open repository: {e}")))?;
381
382        let oid = git2::Oid::from_str(checkpoint_id)
383            .map_err(|e| Status::invalid_argument(format!("Invalid checkpoint ID: {e}")))?;
384
385        let commit = repo
386            .find_commit(oid)
387            .map_err(|e| Status::not_found(format!("Checkpoint not found: {e}")))?;
388
389        // Reset to the checkpoint
390        repo.reset(commit.as_object(), git2::ResetType::Hard, None)
391            .map_err(|e| Status::internal(format!("Failed to reset to checkpoint: {e}")))?;
392
393        // Read checkpoint metadata
394        let metadata_path = self.repo_path.join("checkpoint.json");
395        let metadata_json = std::fs::read_to_string(&metadata_path)
396            .map_err(|e| Status::internal(format!("Failed to read checkpoint metadata: {e}")))?;
397        let metadata: CheckpointMetadata = serde_json::from_str(&metadata_json)
398            .map_err(|e| Status::internal(format!("Failed to parse checkpoint metadata: {e}")))?;
399
400        // Import each table from Parquet
401        for table in &metadata.tables {
402            let parquet_path = self.repo_path.join(&table.file_path);
403            debug!(table = %table.name, path = ?parquet_path, "Importing table");
404
405            // Drop existing table if it exists
406            let _ = storage.drop_table(&table.name).await;
407
408            // Import from Parquet
409            storage
410                .import_from_parquet(&table.name, &parquet_path)
411                .await?;
412        }
413
414        info!("Checkpoint restored successfully");
415        Ok(())
416    }
417
418    /// Get the latest checkpoint
419    pub async fn get_latest_checkpoint(&self) -> Option<Checkpoint> {
420        let checkpoints = self.checkpoints.read().await;
421        checkpoints.last().cloned()
422    }
423
424    /// List all checkpoints
425    pub async fn list_checkpoints(&self) -> Vec<Checkpoint> {
426        self.checkpoints.read().await.clone()
427    }
428
429    /// Start background checkpoint task
430    pub fn start_background(self: Arc<Self>, storage: Arc<DuckDbBackend>) -> JoinHandle<()> {
431        let interval = self.config.interval;
432
433        tokio::spawn(async move {
434            let mut interval_timer = tokio::time::interval(interval);
435            loop {
436                interval_timer.tick().await;
437
438                let checkpoint_name = format!("auto-{}", Utc::now().format("%Y%m%d-%H%M%S"));
439                match self
440                    .create_checkpoint(&storage, Some(checkpoint_name))
441                    .await
442                {
443                    Ok(checkpoint) => {
444                        info!(
445                            checkpoint_id = %checkpoint.id(),
446                            "Background checkpoint created"
447                        );
448                    }
449                    Err(e) => {
450                        error!(error = %e, "Failed to create background checkpoint");
451                    }
452                }
453            }
454        })
455    }
456
457    /// Get the configuration
458    pub fn config(&self) -> &CheckpointConfig {
459        &self.config
460    }
461
462    /// Get the registry client if using shared registry
463    pub fn registry_client(&self) -> Option<&Arc<dyn RegistryClient>> {
464        self.client.as_ref()
465    }
466
467    /// Get the repository name in registry if using shared registry
468    pub fn repo_name(&self) -> Option<&str> {
469        self.repo_name.as_deref()
470    }
471
472    /// Get the repository path
473    pub fn repo_path(&self) -> &PathBuf {
474        &self.repo_path
475    }
476}
477
478impl std::fmt::Debug for CheckpointManager {
479    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
480        f.debug_struct("CheckpointManager")
481            .field("config", &self.config)
482            .field("repo_path", &self.repo_path)
483            .finish()
484    }
485}
486
487#[cfg(test)]
488mod tests {
489    use super::*;
490
491    #[test]
492    fn test_checkpoint_config_defaults() {
493        let config = CheckpointConfig::default();
494        assert_eq!(config.interval, Duration::from_secs(300));
495        assert_eq!(config.retention_count, 10);
496        assert!(config.checkpoint_on_shutdown);
497    }
498
499    #[test]
500    fn test_checkpoint_config_builder() {
501        let config = CheckpointConfig::default()
502            .with_interval(Duration::from_secs(60))
503            .with_retention(5)
504            .with_base_dir("/tmp/checkpoints");
505
506        assert_eq!(config.interval, Duration::from_secs(60));
507        assert_eq!(config.retention_count, 5);
508        assert_eq!(config.base_dir, PathBuf::from("/tmp/checkpoints"));
509    }
510}