Skip to main content

hyprstream_metrics/checkpoint/
recovery.rs

1//! Recovery manager for restoring from checkpoints on startup.
2
3use crate::checkpoint::{Checkpoint, CheckpointManager};
4use crate::storage::duckdb::DuckDbBackend;
5use crate::storage::StorageBackend;
6use std::sync::Arc;
7use tonic::Status;
8use tracing::{info, instrument, warn};
9
10/// Status of recovery operation
11#[derive(Debug, Clone)]
12pub enum RecoveryStatus {
13    /// Successfully recovered from a checkpoint
14    Recovered {
15        /// The checkpoint that was restored
16        checkpoint_id: String,
17        /// Number of tables restored
18        tables_restored: usize,
19    },
20    /// No checkpoint found to recover from
21    NoCheckpointFound,
22    /// Recovery was skipped (e.g., fresh database)
23    Skipped {
24        /// Reason for skipping
25        reason: String,
26    },
27    /// Recovery failed
28    Failed {
29        /// Error message
30        error: String,
31    },
32}
33
34impl RecoveryStatus {
35    /// Check if recovery was successful
36    pub fn is_success(&self) -> bool {
37        matches!(
38            self,
39            RecoveryStatus::Recovered { .. }
40                | RecoveryStatus::NoCheckpointFound
41                | RecoveryStatus::Skipped { .. }
42        )
43    }
44}
45
46/// Recovery manager for startup checkpoint restoration
47pub struct RecoveryManager {
48    /// Reference to the checkpoint manager
49    checkpoint_mgr: Arc<CheckpointManager>,
50}
51
52impl RecoveryManager {
53    /// Create a new recovery manager
54    pub fn new(checkpoint_mgr: Arc<CheckpointManager>) -> Self {
55        Self { checkpoint_mgr }
56    }
57
58    /// Attempt to recover from the latest checkpoint
59    #[instrument(skip(self, storage))]
60    pub async fn recover_latest(&self, storage: &DuckDbBackend) -> RecoveryStatus {
61        info!("Attempting recovery from latest checkpoint");
62
63        // Get the latest checkpoint
64        let latest = self.checkpoint_mgr.get_latest_checkpoint().await;
65
66        match latest {
67            Some(checkpoint) => {
68                info!(checkpoint_id = %checkpoint.id(), "Found checkpoint to restore");
69                self.restore_from(&checkpoint, storage).await
70            }
71            None => {
72                info!("No checkpoint found for recovery");
73                RecoveryStatus::NoCheckpointFound
74            }
75        }
76    }
77
78    /// Restore from a specific checkpoint
79    #[instrument(skip(self, checkpoint, storage), fields(checkpoint_id = %checkpoint.id()))]
80    pub async fn restore_from(
81        &self,
82        checkpoint: &Checkpoint,
83        storage: &DuckDbBackend,
84    ) -> RecoveryStatus {
85        info!("Restoring from checkpoint");
86
87        match self.checkpoint_mgr.restore(checkpoint.id(), storage).await {
88            Ok(()) => {
89                let tables_restored = checkpoint.tables().len();
90                info!(
91                    tables_restored = tables_restored,
92                    "Recovery completed successfully"
93                );
94                RecoveryStatus::Recovered {
95                    checkpoint_id: checkpoint.id().to_owned(),
96                    tables_restored,
97                }
98            }
99            Err(e) => {
100                warn!(error = %e, "Recovery failed");
101                RecoveryStatus::Failed {
102                    error: e.message().to_owned(),
103                }
104            }
105        }
106    }
107
108    /// Recover from a checkpoint by ID
109    #[instrument(skip(self, storage))]
110    pub async fn recover_by_id(
111        &self,
112        checkpoint_id: &str,
113        storage: &DuckDbBackend,
114    ) -> Result<RecoveryStatus, Status> {
115        info!(checkpoint_id = %checkpoint_id, "Recovering from specific checkpoint");
116
117        // Find the checkpoint in the manager's list
118        let checkpoints = self.checkpoint_mgr.list_checkpoints().await;
119        let checkpoint = checkpoints
120            .iter()
121            .find(|c| c.id() == checkpoint_id)
122            .cloned();
123
124        match checkpoint {
125            Some(cp) => Ok(self.restore_from(&cp, storage).await),
126            None => Err(Status::not_found(format!(
127                "Checkpoint {checkpoint_id} not found"
128            ))),
129        }
130    }
131
132    /// Check if recovery is needed
133    ///
134    /// Returns true if:
135    /// - A checkpoint exists
136    /// - The database appears to be empty or in an inconsistent state
137    #[instrument(skip(self, storage))]
138    pub async fn needs_recovery(&self, storage: &DuckDbBackend) -> bool {
139        // Check if there are any checkpoints
140        let has_checkpoints = self.checkpoint_mgr.get_latest_checkpoint().await.is_some();
141
142        if !has_checkpoints {
143            return false;
144        }
145
146        // Check if database has tables (simple heuristic)
147        match storage.list_tables().await {
148            Ok(tables) => {
149                // Filter out internal tables
150                let user_tables: Vec<_> = tables
151                    .into_iter()
152                    .filter(|t| !t.starts_with("sqlite_") && t != "view_metadata")
153                    .collect();
154
155                // If no user tables, recovery might be needed
156                user_tables.is_empty()
157            }
158            Err(_) => {
159                // If we can't list tables, assume recovery is needed
160                true
161            }
162        }
163    }
164
165    /// Perform automatic recovery if needed
166    #[instrument(skip(self, storage))]
167    pub async fn auto_recover(&self, storage: &DuckDbBackend) -> RecoveryStatus {
168        if self.needs_recovery(storage).await {
169            info!("Automatic recovery triggered");
170            self.recover_latest(storage).await
171        } else {
172            info!("No recovery needed");
173            RecoveryStatus::Skipped {
174                reason: "Database appears to be in valid state".to_owned(),
175            }
176        }
177    }
178}
179
180impl std::fmt::Debug for RecoveryManager {
181    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
182        f.debug_struct("RecoveryManager")
183            .field("checkpoint_mgr", &"<CheckpointManager>")
184            .finish()
185    }
186}
187
188#[cfg(test)]
189mod tests {
190    use super::*;
191
192    #[test]
193    fn test_recovery_status_success_check() {
194        assert!(RecoveryStatus::Recovered {
195            checkpoint_id: "abc".to_owned(),
196            tables_restored: 5
197        }
198        .is_success());
199
200        assert!(RecoveryStatus::NoCheckpointFound.is_success());
201
202        assert!(RecoveryStatus::Skipped {
203            reason: "test".to_owned()
204        }
205        .is_success());
206
207        assert!(!RecoveryStatus::Failed {
208            error: "error".to_owned()
209        }
210        .is_success());
211    }
212}