Skip to main content

hyprstream_metrics/query/
orchestrator.rs

1//! Query orchestrator that coordinates DataFusion planning and execution.
2//!
3//! The orchestrator provides a unified interface for:
4//! - Preparing SQL queries into physical plans
5//! - Caching prepared statements for reuse
6//! - Executing queries and streaming results
7
8use crate::query::{
9    DataFusionExecutor, DataFusionPlanner, ExecutorConfig, Query, QueryExecutor, QueryPlanner,
10};
11use crate::storage::StorageBackend;
12use datafusion::arrow::array::RecordBatch;
13use datafusion::arrow::datatypes::SchemaRef;
14use datafusion::error::Result;
15use datafusion::physical_plan::ExecutionPlan;
16use futures::Stream;
17use std::collections::HashMap;
18use std::pin::Pin;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::Arc;
21use tokio::sync::RwLock;
22use tracing::{debug, info, instrument};
23
24/// A cached prepared statement with its physical plan
25#[derive(Clone)]
26pub struct CachedStatement {
27    /// Unique handle for this statement
28    pub handle: u64,
29    /// Original SQL query
30    pub sql: String,
31    /// Compiled physical execution plan
32    pub physical_plan: Arc<dyn ExecutionPlan>,
33    /// Schema of the result set
34    pub schema: SchemaRef,
35}
36
37impl std::fmt::Debug for CachedStatement {
38    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39        f.debug_struct("CachedStatement")
40            .field("handle", &self.handle)
41            .field("sql", &self.sql)
42            .field("schema", &self.schema)
43            .finish()
44    }
45}
46
47/// Cache for prepared statements
48struct StatementCache {
49    /// Map from handle to cached statement
50    by_handle: HashMap<u64, CachedStatement>,
51    /// Map from SQL to handle for deduplication
52    by_sql: HashMap<String, u64>,
53    /// Maximum number of cached statements
54    max_size: usize,
55}
56
57impl StatementCache {
58    fn new(max_size: usize) -> Self {
59        Self {
60            by_handle: HashMap::new(),
61            by_sql: HashMap::new(),
62            max_size,
63        }
64    }
65
66    fn get_by_handle(&self, handle: u64) -> Option<&CachedStatement> {
67        self.by_handle.get(&handle)
68    }
69
70    fn get_by_sql(&self, sql: &str) -> Option<&CachedStatement> {
71        self.by_sql.get(sql).and_then(|h| self.by_handle.get(h))
72    }
73
74    fn insert(&mut self, stmt: CachedStatement) {
75        // Evict oldest entries if at capacity
76        if self.by_handle.len() >= self.max_size {
77            // Simple eviction: remove first entry (could be improved with LRU)
78            if let Some((&handle, _)) = self.by_handle.iter().next() {
79                if let Some(stmt) = self.by_handle.remove(&handle) {
80                    self.by_sql.remove(&stmt.sql);
81                }
82            }
83        }
84
85        let handle = stmt.handle;
86        let sql = stmt.sql.clone();
87        self.by_handle.insert(handle, stmt);
88        self.by_sql.insert(sql, handle);
89    }
90
91    fn remove(&mut self, handle: u64) -> Option<CachedStatement> {
92        if let Some(stmt) = self.by_handle.remove(&handle) {
93            self.by_sql.remove(&stmt.sql);
94            Some(stmt)
95        } else {
96            None
97        }
98    }
99}
100
101/// Query orchestrator that coordinates planning and execution
102pub struct QueryOrchestrator {
103    /// DataFusion query planner
104    planner: Arc<RwLock<DataFusionPlanner>>,
105    /// Query executor
106    executor: Arc<DataFusionExecutor>,
107    /// Statement cache
108    cache: RwLock<StatementCache>,
109    /// Handle counter
110    next_handle: AtomicU64,
111    /// Storage backend reference
112    storage: Arc<dyn StorageBackend>,
113}
114
115impl QueryOrchestrator {
116    /// Create a new query orchestrator
117    #[instrument(skip(storage))]
118    pub async fn new(storage: Arc<dyn StorageBackend>) -> Result<Self> {
119        info!("Creating QueryOrchestrator");
120
121        // Create planner with storage backend
122        let planner = DataFusionPlanner::new(Arc::clone(&storage)).await?;
123        let executor = DataFusionExecutor::new(ExecutorConfig::default());
124
125        debug!("QueryOrchestrator initialized with default executor config");
126
127        Ok(Self {
128            planner: Arc::new(RwLock::new(planner)),
129            executor: Arc::new(executor),
130            cache: RwLock::new(StatementCache::new(1000)),
131            next_handle: AtomicU64::new(1),
132            storage,
133        })
134    }
135
136    /// Create a new query orchestrator with custom executor config
137    pub async fn with_executor_config(
138        storage: Arc<dyn StorageBackend>,
139        config: ExecutorConfig,
140    ) -> Result<Self> {
141        let planner = DataFusionPlanner::new(Arc::clone(&storage)).await?;
142        let executor = DataFusionExecutor::new(config);
143
144        Ok(Self {
145            planner: Arc::new(RwLock::new(planner)),
146            executor: Arc::new(executor),
147            cache: RwLock::new(StatementCache::new(1000)),
148            next_handle: AtomicU64::new(1),
149            storage,
150        })
151    }
152
153    /// Prepare a SQL query and return a cached statement
154    #[instrument(skip(self), fields(sql_len = sql.len()))]
155    pub async fn prepare(&self, sql: &str) -> Result<CachedStatement> {
156        // Check cache first
157        {
158            let cache = self.cache.read().await;
159            if let Some(stmt) = cache.get_by_sql(sql) {
160                debug!(handle = stmt.handle, "Cache hit for prepared statement");
161                return Ok(stmt.clone());
162            }
163        }
164
165        debug!("Cache miss, planning query");
166
167        // Create query and plan it
168        let query = Query {
169            sql: sql.to_owned(),
170            schema_hint: None,
171            hints: vec![],
172        };
173
174        let planner = self.planner.read().await;
175        let physical_plan = planner.plan_query(&query).await?;
176        let schema = physical_plan.schema();
177
178        // Generate handle and create cached statement
179        let handle = self.next_handle.fetch_add(1, Ordering::SeqCst);
180        let stmt = CachedStatement {
181            handle,
182            sql: sql.to_owned(),
183            physical_plan,
184            schema,
185        };
186
187        // Insert into cache
188        {
189            let mut cache = self.cache.write().await;
190            cache.insert(stmt.clone());
191        }
192
193        info!(handle = handle, "Prepared and cached new statement");
194        Ok(stmt)
195    }
196
197    /// Get a prepared statement by handle
198    pub async fn get_statement(&self, handle: u64) -> Option<CachedStatement> {
199        let cache = self.cache.read().await;
200        cache.get_by_handle(handle).cloned()
201    }
202
203    /// Execute a prepared statement and return results as a stream
204    #[instrument(skip(self, stmt), fields(handle = stmt.handle))]
205    pub async fn execute(
206        &self,
207        stmt: &CachedStatement,
208    ) -> Result<Pin<Box<dyn Stream<Item = Result<RecordBatch>> + Send>>> {
209        debug!("Executing prepared statement");
210        self.executor
211            .execute_stream(Arc::clone(&stmt.physical_plan))
212            .await
213    }
214
215    /// Execute a prepared statement and collect all results
216    #[instrument(skip(self, stmt), fields(handle = stmt.handle))]
217    pub async fn execute_collect(&self, stmt: &CachedStatement) -> Result<Vec<RecordBatch>> {
218        debug!("Executing and collecting prepared statement results");
219        self.executor
220            .execute_collect(Arc::clone(&stmt.physical_plan))
221            .await
222    }
223
224    /// Convenience method: prepare and execute in one call
225    #[instrument(skip(self), fields(sql_len = sql.len()))]
226    pub async fn query(
227        &self,
228        sql: &str,
229    ) -> Result<Pin<Box<dyn Stream<Item = Result<RecordBatch>> + Send>>> {
230        let stmt = self.prepare(sql).await?;
231        self.execute(&stmt).await
232    }
233
234    /// Convenience method: prepare, execute, and collect in one call
235    #[instrument(skip(self), fields(sql_len = sql.len()))]
236    pub async fn query_collect(&self, sql: &str) -> Result<Vec<RecordBatch>> {
237        let stmt = self.prepare(sql).await?;
238        self.execute_collect(&stmt).await
239    }
240
241    /// Get the schema for a SQL query without executing it
242    #[instrument(skip(self), fields(sql_len = sql.len()))]
243    pub async fn get_schema(&self, sql: &str) -> Result<SchemaRef> {
244        // Check cache first
245        {
246            let cache = self.cache.read().await;
247            if let Some(stmt) = cache.get_by_sql(sql) {
248                debug!(handle = stmt.handle, "Schema from cached statement");
249                return Ok(Arc::clone(&stmt.schema));
250            }
251        }
252
253        // Prepare to get schema
254        let stmt = self.prepare(sql).await?;
255        Ok(stmt.schema)
256    }
257
258    /// Close a prepared statement and remove from cache
259    pub async fn close_statement(&self, handle: u64) -> bool {
260        let mut cache = self.cache.write().await;
261        cache.remove(handle).is_some()
262    }
263
264    /// Get the underlying storage backend
265    pub fn storage(&self) -> &Arc<dyn StorageBackend> {
266        &self.storage
267    }
268
269    /// Refresh table registrations (call after schema changes)
270    #[instrument(skip(self))]
271    pub async fn refresh_tables(&self) -> Result<()> {
272        info!("Refreshing table registrations");
273
274        // Re-create planner with updated storage
275        let new_planner = DataFusionPlanner::new(Arc::clone(&self.storage)).await?;
276
277        // Replace planner
278        let mut planner = self.planner.write().await;
279        *planner = new_planner;
280
281        // Clear statement cache since plans may be invalid
282        let mut cache = self.cache.write().await;
283        cache.by_handle.clear();
284        cache.by_sql.clear();
285
286        info!("Table registrations refreshed, statement cache cleared");
287        Ok(())
288    }
289}
290
291impl std::fmt::Debug for QueryOrchestrator {
292    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
293        f.debug_struct("QueryOrchestrator")
294            .field("next_handle", &self.next_handle)
295            .finish()
296    }
297}
298
299#[cfg(test)]
300mod tests {
301    use super::*;
302    use crate::storage::duckdb::DuckDbBackend;
303    use datafusion::arrow::datatypes::{DataType, Field, Schema};
304    use datafusion::common::DataFusionError;
305
306    fn status_to_df_error(s: tonic::Status) -> DataFusionError {
307        DataFusionError::Execution(s.message().to_owned())
308    }
309
310    #[tokio::test]
311    async fn test_orchestrator_prepare_and_execute() -> Result<()> {
312        // Create test backend
313        let backend = Arc::new(DuckDbBackend::new_in_memory().map_err(status_to_df_error)?);
314        backend.init().await.map_err(status_to_df_error)?;
315
316        // Create test table
317        let schema = Arc::new(Schema::new(vec![
318            Field::new("id", DataType::Int64, false),
319            Field::new("value", DataType::Float64, false),
320        ]));
321        backend
322            .create_table("test_table", &schema)
323            .await
324            .map_err(status_to_df_error)?;
325
326        // Create orchestrator
327        let orchestrator = QueryOrchestrator::new(backend).await?;
328
329        // Prepare query
330        let stmt = orchestrator.prepare("SELECT * FROM test_table").await?;
331        assert!(stmt.handle > 0);
332        assert_eq!(stmt.schema.fields().len(), 2);
333
334        // Execute (should return empty since no data)
335        let results = orchestrator.execute_collect(&stmt).await?;
336        assert!(results.is_empty() || results[0].num_rows() == 0);
337
338        Ok(())
339    }
340
341    #[tokio::test]
342    async fn test_orchestrator_statement_caching() -> Result<()> {
343        let backend = Arc::new(DuckDbBackend::new_in_memory().map_err(status_to_df_error)?);
344        backend.init().await.map_err(status_to_df_error)?;
345
346        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
347        backend
348            .create_table("cache_test", &schema)
349            .await
350            .map_err(status_to_df_error)?;
351
352        let orchestrator = QueryOrchestrator::new(backend).await?;
353
354        // First prepare
355        let stmt1 = orchestrator.prepare("SELECT * FROM cache_test").await?;
356        let handle1 = stmt1.handle;
357
358        // Second prepare should return same handle (cache hit)
359        let stmt2 = orchestrator.prepare("SELECT * FROM cache_test").await?;
360        assert_eq!(stmt2.handle, handle1);
361
362        // Different query should get different handle
363        let stmt3 = orchestrator.prepare("SELECT id FROM cache_test").await?;
364        assert_ne!(stmt3.handle, handle1);
365
366        Ok(())
367    }
368
369    #[tokio::test]
370    async fn test_orchestrator_close_statement() -> Result<()> {
371        let backend = Arc::new(DuckDbBackend::new_in_memory().map_err(status_to_df_error)?);
372        backend.init().await.map_err(status_to_df_error)?;
373
374        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
375        backend
376            .create_table("close_test", &schema)
377            .await
378            .map_err(status_to_df_error)?;
379
380        let orchestrator = QueryOrchestrator::new(backend).await?;
381
382        // Prepare
383        let stmt = orchestrator.prepare("SELECT * FROM close_test").await?;
384        let handle = stmt.handle;
385
386        // Statement should be in cache
387        assert!(orchestrator.get_statement(handle).await.is_some());
388
389        // Close
390        assert!(orchestrator.close_statement(handle).await);
391
392        // Statement should be gone
393        assert!(orchestrator.get_statement(handle).await.is_none());
394
395        // Closing again should return false
396        assert!(!orchestrator.close_statement(handle).await);
397
398        Ok(())
399    }
400}