hyprstream_metrics/query/
planner.rs1use crate::error::StatusWrapper;
2use crate::query::rules::view::ViewOptimizationRule;
3use crate::storage::StorageBackend;
4use datafusion::arrow::datatypes::Schema;
5use datafusion::datasource::memory::MemTable;
6use datafusion::error::{DataFusionError, Result};
7use datafusion::execution::context::SessionContext;
8use datafusion::logical_expr::LogicalPlan;
9use datafusion::optimizer::optimizer::OptimizerRule;
10use datafusion::physical_plan::ExecutionPlan;
11use std::sync::Arc;
12
13#[derive(Debug, Clone)]
15pub struct Query {
16 pub sql: String,
18 pub schema_hint: Option<Schema>,
20 pub hints: Vec<OptimizationHint>,
22}
23
24#[derive(Debug, Clone)]
26pub enum OptimizationHint {
27 PreferPredicatePushdown,
29 PreferParallelExecution,
31 OptimizeForStreaming,
33 OptimizeForVectorOps,
35 PreferViews,
37}
38
39#[async_trait::async_trait]
41pub trait QueryPlanner: Send + Sync {
42 async fn create_logical_plan(&self, query: &Query) -> Result<LogicalPlan>;
44
45 async fn optimize(&self, plan: LogicalPlan) -> Result<Arc<dyn ExecutionPlan>>;
47
48 async fn plan_query(&self, query: &Query) -> Result<Arc<dyn ExecutionPlan>> {
50 let logical_plan = self.create_logical_plan(query).await?;
51 self.optimize(logical_plan).await
52 }
53}
54
55pub struct DataFusionPlanner {
57 ctx: SessionContext,
59 optimizations: Vec<Box<dyn OptimizerRule + Send + Sync>>,
61 storage: Arc<dyn StorageBackend>,
63}
64
65impl DataFusionPlanner {
66 pub async fn new(storage: Arc<dyn StorageBackend>) -> Result<Self> {
67 let ctx = SessionContext::new();
68 let mut planner = Self {
69 ctx,
70 optimizations: Vec::new(),
71 storage: storage.clone(),
72 };
73
74 planner.add_default_rules();
76
77 let tables = storage
79 .list_tables()
80 .await
81 .map_err(|e| Into::<DataFusionError>::into(StatusWrapper(e)))?;
82 for table_name in tables {
83 let schema = storage
84 .get_table_schema(&table_name)
85 .await
86 .map_err(|e| Into::<DataFusionError>::into(StatusWrapper(e)))?;
87 planner.ctx.register_table(
89 &table_name,
90 Arc::new(MemTable::try_new(schema, vec![vec![]])?),
91 )?;
92 }
93
94 Ok(planner)
95 }
96
97 pub fn with_optimization(mut self, rule: Box<dyn OptimizerRule + Send + Sync>) -> Self {
99 self.optimizations.push(rule);
100 self
101 }
102
103 fn add_default_rules(&mut self) {
105 self.optimizations
107 .push(Box::new(ViewOptimizationRule::new(self.storage.clone())));
108 }
109}
110
111#[async_trait::async_trait]
112impl QueryPlanner for DataFusionPlanner {
113 async fn create_logical_plan(&self, query: &Query) -> Result<LogicalPlan> {
114 let logical_plan = self.ctx.sql(&query.sql).await?.into_optimized_plan()?;
116
117 if let Some(_schema) = &query.schema_hint {
119 }
121
122 Ok(logical_plan)
123 }
124
125 async fn optimize(&self, plan: LogicalPlan) -> Result<Arc<dyn ExecutionPlan>> {
126 let state = self.ctx.state();
128
129 let mut optimized_plan = plan;
131 for rule in &self.optimizations {
132 optimized_plan = rule.rewrite(optimized_plan, &state)?.data;
133 }
134
135 state.create_physical_plan(&optimized_plan).await
137 }
138}
139
140#[derive(Debug)]
142pub struct OptimizerContext {
143 pub statistics: Statistics,
145 pub available_optimizations: Vec<OptimizationHint>,
147}
148
149impl Default for OptimizerContext {
150 fn default() -> Self {
151 Self::new()
152 }
153}
154
155impl OptimizerContext {
156 pub fn new() -> Self {
157 Self {
158 statistics: Statistics::default(),
159 available_optimizations: Vec::new(),
160 }
161 }
162}
163
164#[derive(Debug, Default)]
166pub struct Statistics {
167 pub row_count: Option<usize>,
169 pub total_bytes: Option<usize>,
171 pub column_statistics: HashMap<String, ColumnStatistics>,
173}
174
175#[derive(Debug)]
177pub struct ColumnStatistics {
178 pub distinct_count: Option<usize>,
180 pub null_count: Option<usize>,
182 pub min_value: Option<ScalarValue>,
184 pub max_value: Option<ScalarValue>,
186}
187
188use datafusion::scalar::ScalarValue;
190use std::collections::HashMap;
191
192#[cfg(test)]
193mod tests {
194 use super::*;
195 use crate::storage::duckdb::DuckDbBackend;
196 use datafusion::error::DataFusionError;
197 use duckdb::arrow::datatypes::{DataType, Field, Schema};
198
199 fn status_to_df_error(s: tonic::Status) -> DataFusionError {
200 DataFusionError::Execution(s.message().to_owned())
201 }
202
203 #[tokio::test]
204 async fn test_query_planning_with_views() -> Result<()> {
205 let backend = Arc::new(DuckDbBackend::new_in_memory().map_err(status_to_df_error)?);
207 backend.init().await.map_err(status_to_df_error)?;
208
209 let schema = Arc::new(Schema::new(vec![
211 Field::new("id", DataType::Int64, false),
212 Field::new("value", DataType::Float64, false),
213 ]));
214 backend
215 .create_table("test_table", &schema)
216 .await
217 .map_err(status_to_df_error)?;
218
219 let planner = DataFusionPlanner::new(backend).await?;
221
222 let query = Query {
224 sql: "SELECT * FROM test_table".to_owned(),
225 schema_hint: None,
226 hints: vec![OptimizationHint::PreferViews],
227 };
228
229 let plan = planner.create_logical_plan(&query).await?;
231 assert!(!plan.schema().fields().is_empty());
232
233 Ok(())
234 }
235}