Skip to main content

hyprstream_metrics/storage/
cached.rs

1//! Two-tier cached storage backend implementation.
2//!
3//! This module provides a caching layer on top of any storage backend:
4//! - Fast access to frequently used data
5//! - Write-through caching for consistency
6//! - Support for any StorageBackend as cache or store
7//!
8//! # Configuration
9//!
10//! The cached backend can be configured using the following options:
11//!
12//! ```toml
13//! # Primary storage configuration
14//! [engine]
15//! engine = "adbc"
16//! connection = "postgresql://localhost:5432"
17//! options = {
18//!     driver_path = "/usr/local/lib/libadbc_driver_postgresql.so",
19//!     username = "postgres",
20//!     database = "metrics"
21//! }
22//!
23//! # Cache configuration
24//! [cache]
25//! enabled = true
26//! engine = "duckdb"
27//! connection = ":memory:"
28//! options = {
29//!     threads = "2"
30//! }
31//! ```
32//!
33//! Or via command line:
34//!
35//! ```bash
36//! hyprstream \
37//!   --engine adbc \
38//!   --engine-connection "postgresql://localhost:5432" \
39//!   --engine-options driver_path=/usr/local/lib/libadbc_driver_postgresql.so \
40//!   --engine-options username=postgres \
41//!   --enable-cache \
42//!   --cache-engine duckdb \
43//!   --cache-connection ":memory:" \
44//!   --cache-options threads=2
45//! ```
46//!
47//! The implementation follows standard caching patterns while ensuring
48//! data consistency between cache and backing store.
49
50#[cfg(feature = "adbc")]
51use crate::storage::adbc::AdbcBackend;
52use crate::storage::view::ViewDefinition;
53use crate::storage::{Credentials, StorageBackend};
54use duckdb::arrow::array::RecordBatch;
55use duckdb::arrow::datatypes::Schema;
56use std::collections::HashMap;
57use std::sync::Arc;
58use tonic::Status;
59
60/// Two-tier storage backend with caching support.
61///
62/// This backend provides:
63/// - Fast access to recent data through caching
64/// - Write-through caching for data consistency
65/// - Configurable cache duration
66/// - Support for any StorageBackend implementation
67///
68/// The implementation uses two storage backends:
69/// 1. A fast cache (e.g., in-memory DuckDB)
70/// 2. A persistent store (e.g., PostgreSQL via ADBC)
71#[derive(Clone)]
72pub struct CachedStorageBackend {
73    /// Fast storage backend for caching
74    cache: Arc<dyn StorageBackend>,
75    /// Persistent storage backend for data
76    store: Arc<dyn StorageBackend>,
77}
78
79impl CachedStorageBackend {
80    /// Creates a new cached storage backend.
81    ///
82    /// This method sets up a two-tier storage system with:
83    /// - A fast cache layer for frequent access
84    /// - A persistent backing store
85    ///
86    /// # Arguments
87    ///
88    /// * `cache` - Fast storage backend for caching
89    /// * `store` - Persistent storage backend
90    pub fn new(cache: Arc<dyn StorageBackend>, store: Arc<dyn StorageBackend>) -> Self {
91        Self { cache, store }
92    }
93}
94
95#[async_trait::async_trait]
96impl StorageBackend for CachedStorageBackend {
97    /// Initializes both cache and backing store.
98    ///
99    /// This method ensures both storage layers are properly
100    /// initialized and ready for use.
101    async fn init(&self) -> Result<(), Status> {
102        // Initialize both cache and backing store
103        self.cache.init().await?;
104        self.store.init().await?;
105        Ok(())
106    }
107
108    /// Prepares a SQL statement on the backing store.
109    ///
110    /// This method bypasses the cache and prepares statements
111    /// directly on the backing store, as prepared statements
112    /// are typically used for complex queries.
113    ///
114    /// # Arguments
115    ///
116    /// * `query` - SQL query to prepare
117    async fn prepare_sql(&self, query: &str) -> Result<Vec<u8>, Status> {
118        // Prepare on backing store only
119        self.store.prepare_sql(query).await
120    }
121
122    /// Executes a prepared SQL statement on the backing store.
123    ///
124    /// This method bypasses the cache and executes statements
125    /// directly on the backing store, ensuring consistent results
126    /// for complex queries.
127    ///
128    /// # Arguments
129    ///
130    /// * `statement_handle` - Handle of the prepared statement
131    async fn query_sql(&self, statement_handle: &[u8]) -> Result<RecordBatch, Status> {
132        // Execute on backing store only
133        self.store.query_sql(statement_handle).await
134    }
135
136    fn new_with_options(
137        _connection_string: &str,
138        _options: &HashMap<String, String>,
139        _credentials: Option<&Credentials>,
140    ) -> Result<Self, Status> {
141        Err(Status::invalid_argument(
142            "Use CachedStorageBackend::new() instead",
143        ))
144    }
145
146    async fn create_table(&self, table_name: &str, schema: &Schema) -> Result<(), Status> {
147        // Create table in both cache and store
148        self.cache.create_table(table_name, schema).await?;
149        self.store.create_table(table_name, schema).await?;
150        Ok(())
151    }
152
153    async fn execute_ddl(&self, sql: &str) -> Result<(), Status> {
154        // DDL applies to both tiers, like create_table
155        self.cache.execute_ddl(sql).await?;
156        self.store.execute_ddl(sql).await?;
157        Ok(())
158    }
159
160    async fn insert_into_table(&self, table_name: &str, batch: RecordBatch) -> Result<(), Status> {
161        // Insert into both cache and store
162        self.cache
163            .insert_into_table(table_name, batch.clone())
164            .await?;
165        self.store.insert_into_table(table_name, batch).await?;
166        Ok(())
167    }
168
169    async fn create_view(&self, name: &str, definition: ViewDefinition) -> Result<(), Status> {
170        // Create view in both cache and store
171        self.cache.create_view(name, definition.clone()).await?;
172        self.store.create_view(name, definition).await?;
173        Ok(())
174    }
175
176    async fn get_view(&self, name: &str) -> Result<crate::storage::view::ViewMetadata, Status> {
177        // Try cache first
178        match self.cache.get_view(name).await {
179            Ok(view) => Ok(view),
180            Err(_) => {
181                // On cache miss, get from store and update cache
182                let view = self.store.get_view(name).await?;
183                self.cache
184                    .create_view(name, view.definition.clone())
185                    .await?;
186                Ok(view)
187            }
188        }
189    }
190
191    async fn list_views(&self) -> Result<Vec<String>, Status> {
192        // List views from store (source of truth)
193        self.store.list_views().await
194    }
195
196    async fn drop_view(&self, name: &str) -> Result<(), Status> {
197        // Drop from both cache and store
198        self.cache.drop_view(name).await?;
199        self.store.drop_view(name).await?;
200        Ok(())
201    }
202
203    async fn drop_table(&self, table_name: &str) -> Result<(), Status> {
204        // Drop from both cache and store
205        self.cache.drop_table(table_name).await?;
206        self.store.drop_table(table_name).await?;
207        Ok(())
208    }
209
210    async fn list_tables(&self) -> Result<Vec<String>, Status> {
211        // List tables from store (source of truth)
212        self.store.list_tables().await
213    }
214
215    async fn get_table_schema(&self, table_name: &str) -> Result<Arc<Schema>, Status> {
216        // Get schema from store (source of truth)
217        self.store.get_table_schema(table_name).await
218    }
219
220    async fn export_to_parquet(
221        &self,
222        table_name: &str,
223        path: &std::path::Path,
224    ) -> Result<(), Status> {
225        // Export from store (source of truth)
226        self.store.export_to_parquet(table_name, path).await
227    }
228
229    async fn import_from_parquet(
230        &self,
231        table_name: &str,
232        path: &std::path::Path,
233    ) -> Result<(), Status> {
234        // Import into both cache and store
235        self.cache.import_from_parquet(table_name, path).await?;
236        self.store.import_from_parquet(table_name, path).await?;
237        Ok(())
238    }
239}