Skip to main content

hyprstream_metrics/storage/
mod.rs

1//! Storage backends for SQL database operations.
2//!
3//! This module provides multiple storage backend implementations:
4//! - `duckdb`: High-performance embedded SQL database
5//! - `adbc`: Arrow Database Connectivity for external database integration (optional)
6//! - `cached`: Two-tier storage with configurable caching layer
7//! - `datafusion_provider`: DataFusion TableProvider bridge for DuckDB
8//! - `context`: Embedding storage for RAG/CAG functionality
9//!
10//! The `StorageBackend` trait itself, the view types, and the SQL string
11//! utilities live in the Apache-2.0 `hyprstream-metrics-api` crate and are
12//! re-exported here; each backend here implements that trait.
13
14// tonic::Status is the idiomatic gRPC error type - boxing would break API
15#![allow(clippy::result_large_err)]
16
17pub mod cache;
18pub mod cached;
19pub mod context;
20pub mod datafusion_provider;
21pub mod duckdb;
22pub mod table_manager;
23
24#[cfg(feature = "adbc")]
25pub mod adbc;
26
27pub use hyprstream_metrics_api::storage::view;
28pub use hyprstream_metrics_api::storage::{Credentials, StorageBackend, StorageUtils};
29
30#[cfg(feature = "adbc")]
31use self::adbc::AdbcBackend;
32use self::{
33    cached::CachedStorageBackend,
34    duckdb::DuckDbBackend,
35    view::{ViewDefinition, ViewMetadata},
36};
37
38pub use datafusion_provider::{DuckDBExec, DuckDBTableProvider};
39
40use ::duckdb::arrow::array::RecordBatch;
41use ::duckdb::arrow::datatypes::Schema;
42use std::collections::HashMap;
43use std::sync::Arc;
44use tonic::Status;
45
46#[derive(Clone)]
47pub enum StorageBackendType {
48    DuckDb(DuckDbBackend),
49    #[cfg(feature = "adbc")]
50    Adbc(AdbcBackend),
51    Cached(CachedStorageBackend),
52}
53
54impl AsRef<dyn StorageBackend> for StorageBackendType {
55    fn as_ref(&self) -> &(dyn StorageBackend + 'static) {
56        match self {
57            #[cfg(feature = "adbc")]
58            StorageBackendType::Adbc(backend) => backend,
59            StorageBackendType::DuckDb(backend) => backend,
60            StorageBackendType::Cached(backend) => backend,
61        }
62    }
63}
64
65#[async_trait::async_trait]
66impl StorageBackend for StorageBackendType {
67    async fn init(&self) -> Result<(), Status> {
68        self.as_ref().init().await
69    }
70
71    async fn prepare_sql(&self, query: &str) -> Result<Vec<u8>, Status> {
72        self.as_ref().prepare_sql(query).await
73    }
74
75    async fn query_sql(&self, statement_handle: &[u8]) -> Result<RecordBatch, Status> {
76        self.as_ref().query_sql(statement_handle).await
77    }
78
79    fn new_with_options(
80        connection_string: &str,
81        options: &HashMap<String, String>,
82        credentials: Option<&Credentials>,
83    ) -> Result<Self, Status>
84    where
85        Self: Sized,
86    {
87        let engine_type = options
88            .get("engine")
89            .ok_or_else(|| Status::invalid_argument("Missing engine type"))?;
90
91        match engine_type.as_str() {
92            #[cfg(feature = "adbc")]
93            "adbc" => Ok(StorageBackendType::Adbc(
94                adbc::AdbcBackend::new_with_options(connection_string, options, credentials)?,
95            )),
96            "duckdb" => Ok(StorageBackendType::DuckDb(
97                duckdb::DuckDbBackend::new_with_options(connection_string, options, credentials)?,
98            )),
99            "cached" => {
100                // Create cache backend (in-memory DuckDB)
101                let cache = Arc::new(DuckDbBackend::new_in_memory()?);
102
103                // Create store backend based on store_engine option
104                let store_engine = options.get("store_engine").ok_or_else(|| {
105                    Status::invalid_argument("Missing store_engine for cached backend")
106                })?;
107
108                let store: Arc<dyn StorageBackend> = match store_engine.as_str() {
109                    #[cfg(feature = "adbc")]
110                    "adbc" => Arc::new(AdbcBackend::new_with_options(
111                        connection_string,
112                        options,
113                        credentials,
114                    )?),
115                    "duckdb" => Arc::new(DuckDbBackend::new_with_options(
116                        connection_string,
117                        options,
118                        credentials,
119                    )?),
120                    _ => return Err(Status::invalid_argument("Invalid store_engine type")),
121                };
122
123                Ok(StorageBackendType::Cached(CachedStorageBackend::new(
124                    cache, store,
125                )))
126            }
127            _ => Err(Status::invalid_argument("Invalid engine type")),
128        }
129    }
130
131    async fn create_table(&self, table_name: &str, schema: &Schema) -> Result<(), Status> {
132        self.as_ref().create_table(table_name, schema).await
133    }
134
135    async fn execute_ddl(&self, sql: &str) -> Result<(), Status> {
136        self.as_ref().execute_ddl(sql).await
137    }
138
139    async fn insert_into_table(&self, table_name: &str, batch: RecordBatch) -> Result<(), Status> {
140        self.as_ref().insert_into_table(table_name, batch).await
141    }
142
143    async fn create_view(&self, name: &str, definition: ViewDefinition) -> Result<(), Status> {
144        self.as_ref().create_view(name, definition).await
145    }
146
147    async fn get_view(&self, name: &str) -> Result<ViewMetadata, Status> {
148        self.as_ref().get_view(name).await
149    }
150
151    async fn list_views(&self) -> Result<Vec<String>, Status> {
152        self.as_ref().list_views().await
153    }
154
155    async fn drop_view(&self, name: &str) -> Result<(), Status> {
156        self.as_ref().drop_view(name).await
157    }
158
159    async fn drop_table(&self, table_name: &str) -> Result<(), Status> {
160        self.as_ref().drop_table(table_name).await
161    }
162
163    async fn list_tables(&self) -> Result<Vec<String>, Status> {
164        self.as_ref().list_tables().await
165    }
166
167    async fn get_table_schema(&self, table_name: &str) -> Result<Arc<Schema>, Status> {
168        self.as_ref().get_table_schema(table_name).await
169    }
170
171    async fn export_to_parquet(
172        &self,
173        table_name: &str,
174        path: &std::path::Path,
175    ) -> Result<(), Status> {
176        self.as_ref().export_to_parquet(table_name, path).await
177    }
178
179    async fn import_from_parquet(
180        &self,
181        table_name: &str,
182        path: &std::path::Path,
183    ) -> Result<(), Status> {
184        self.as_ref().import_from_parquet(table_name, path).await
185    }
186}