Skip to main content

hyprstream_metrics/query/
planner.rs

1use crate::error::StatusWrapper;
2use crate::query::rules::view::ViewOptimizationRule;
3use crate::storage::StorageBackend;
4use datafusion::arrow::datatypes::Schema;
5use datafusion::datasource::memory::MemTable;
6use datafusion::error::{DataFusionError, Result};
7use datafusion::execution::context::SessionContext;
8use datafusion::logical_expr::LogicalPlan;
9use datafusion::optimizer::optimizer::OptimizerRule;
10use datafusion::physical_plan::ExecutionPlan;
11use std::sync::Arc;
12
13/// Represents a query that can be planned and executed
14#[derive(Debug, Clone)]
15pub struct Query {
16    /// SQL query string
17    pub sql: String,
18    /// Optional schema hint for better planning
19    pub schema_hint: Option<Schema>,
20    /// Query optimization hints
21    pub hints: Vec<OptimizationHint>,
22}
23
24/// Optimization hints that can be passed to the query planner
25#[derive(Debug, Clone)]
26pub enum OptimizationHint {
27    /// Prefer pushing predicates to storage
28    PreferPredicatePushdown,
29    /// Prefer parallel execution where possible
30    PreferParallelExecution,
31    /// Optimize for streaming execution
32    OptimizeForStreaming,
33    /// Optimize for vector operations
34    OptimizeForVectorOps,
35    /// Use views when possible
36    PreferViews,
37}
38
39/// Core trait for query planning and optimization
40#[async_trait::async_trait]
41pub trait QueryPlanner: Send + Sync {
42    /// Create a logical plan from a query
43    async fn create_logical_plan(&self, query: &Query) -> Result<LogicalPlan>;
44
45    /// Optimize a logical plan into a physical plan
46    async fn optimize(&self, plan: LogicalPlan) -> Result<Arc<dyn ExecutionPlan>>;
47
48    /// Create and optimize a physical plan directly from a query
49    async fn plan_query(&self, query: &Query) -> Result<Arc<dyn ExecutionPlan>> {
50        let logical_plan = self.create_logical_plan(query).await?;
51        self.optimize(logical_plan).await
52    }
53}
54
55/// Default query planner implementation using DataFusion
56pub struct DataFusionPlanner {
57    /// DataFusion context for planning and optimization
58    ctx: SessionContext,
59    /// Custom optimization rules
60    optimizations: Vec<Box<dyn OptimizerRule + Send + Sync>>,
61    /// Storage backend for view lookups
62    storage: Arc<dyn StorageBackend>,
63}
64
65impl DataFusionPlanner {
66    pub async fn new(storage: Arc<dyn StorageBackend>) -> Result<Self> {
67        let ctx = SessionContext::new();
68        let mut planner = Self {
69            ctx,
70            optimizations: Vec::new(),
71            storage: storage.clone(),
72        };
73
74        // Add default optimization rules
75        planner.add_default_rules();
76
77        // Register tables from storage
78        let tables = storage
79            .list_tables()
80            .await
81            .map_err(|e| Into::<DataFusionError>::into(StatusWrapper(e)))?;
82        for table_name in tables {
83            let schema = storage
84                .get_table_schema(&table_name)
85                .await
86                .map_err(|e| Into::<DataFusionError>::into(StatusWrapper(e)))?;
87            // Use vec![vec![]] to provide one empty partition (required by MemTable)
88            planner.ctx.register_table(
89                &table_name,
90                Arc::new(MemTable::try_new(schema, vec![vec![]])?),
91            )?;
92        }
93
94        Ok(planner)
95    }
96
97    /// Add a custom optimization rule
98    pub fn with_optimization(mut self, rule: Box<dyn OptimizerRule + Send + Sync>) -> Self {
99        self.optimizations.push(rule);
100        self
101    }
102
103    /// Add default optimization rules
104    fn add_default_rules(&mut self) {
105        // Add view optimization rule
106        self.optimizations
107            .push(Box::new(ViewOptimizationRule::new(self.storage.clone())));
108    }
109}
110
111#[async_trait::async_trait]
112impl QueryPlanner for DataFusionPlanner {
113    async fn create_logical_plan(&self, query: &Query) -> Result<LogicalPlan> {
114        // Parse SQL into logical plan
115        let logical_plan = self.ctx.sql(&query.sql).await?.into_optimized_plan()?;
116
117        // Apply schema hints if available
118        if let Some(_schema) = &query.schema_hint {
119            // TODO: Implement schema hint application
120        }
121
122        Ok(logical_plan)
123    }
124
125    async fn optimize(&self, plan: LogicalPlan) -> Result<Arc<dyn ExecutionPlan>> {
126        // Get session state
127        let state = self.ctx.state();
128
129        // Apply custom optimization rules
130        let mut optimized_plan = plan;
131        for rule in &self.optimizations {
132            optimized_plan = rule.rewrite(optimized_plan, &state)?.data;
133        }
134
135        // Convert to physical plan using session state
136        state.create_physical_plan(&optimized_plan).await
137    }
138}
139
140/// Context for optimization rules
141#[derive(Debug)]
142pub struct OptimizerContext {
143    /// Statistics about the data
144    pub statistics: Statistics,
145    /// Available optimizations
146    pub available_optimizations: Vec<OptimizationHint>,
147}
148
149impl Default for OptimizerContext {
150    fn default() -> Self {
151        Self::new()
152    }
153}
154
155impl OptimizerContext {
156    pub fn new() -> Self {
157        Self {
158            statistics: Statistics::default(),
159            available_optimizations: Vec::new(),
160        }
161    }
162}
163
164/// Statistics used for optimization decisions
165#[derive(Debug, Default)]
166pub struct Statistics {
167    /// Estimated row count
168    pub row_count: Option<usize>,
169    /// Estimated total bytes
170    pub total_bytes: Option<usize>,
171    /// Column statistics
172    pub column_statistics: HashMap<String, ColumnStatistics>,
173}
174
175/// Statistics for a single column
176#[derive(Debug)]
177pub struct ColumnStatistics {
178    /// Number of distinct values
179    pub distinct_count: Option<usize>,
180    /// Number of null values
181    pub null_count: Option<usize>,
182    /// Minimum value
183    pub min_value: Option<ScalarValue>,
184    /// Maximum value
185    pub max_value: Option<ScalarValue>,
186}
187
188// Add missing imports
189use datafusion::scalar::ScalarValue;
190use std::collections::HashMap;
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195    use crate::storage::duckdb::DuckDbBackend;
196    use datafusion::error::DataFusionError;
197    use duckdb::arrow::datatypes::{DataType, Field, Schema};
198
199    fn status_to_df_error(s: tonic::Status) -> DataFusionError {
200        DataFusionError::Execution(s.message().to_owned())
201    }
202
203    #[tokio::test]
204    async fn test_query_planning_with_views() -> Result<()> {
205        // Create test backend
206        let backend = Arc::new(DuckDbBackend::new_in_memory().map_err(status_to_df_error)?);
207        backend.init().await.map_err(status_to_df_error)?;
208
209        // Create test table
210        let schema = Arc::new(Schema::new(vec![
211            Field::new("id", DataType::Int64, false),
212            Field::new("value", DataType::Float64, false),
213        ]));
214        backend
215            .create_table("test_table", &schema)
216            .await
217            .map_err(status_to_df_error)?;
218
219        // Create planner with backend
220        let planner = DataFusionPlanner::new(backend).await?;
221
222        // Create test query
223        let query = Query {
224            sql: "SELECT * FROM test_table".to_owned(),
225            schema_hint: None,
226            hints: vec![OptimizationHint::PreferViews],
227        };
228
229        // Plan should succeed
230        let plan = planner.create_logical_plan(&query).await?;
231        assert!(!plan.schema().fields().is_empty());
232
233        Ok(())
234    }
235}