hyprstream_metrics/query/
executor.rs1use datafusion::arrow::record_batch::RecordBatch;
2use datafusion::error::Result;
3use datafusion::execution::context::{SessionConfig, SessionContext, TaskContext};
4use datafusion::execution::runtime_env::RuntimeEnv;
5use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties};
6use futures::Stream;
7use std::pin::Pin;
8use std::sync::Arc;
9use tracing::{debug, info, instrument, warn};
10
11#[async_trait::async_trait]
13pub trait QueryExecutor: Send + Sync {
14 async fn execute_stream(
16 &self,
17 plan: Arc<dyn ExecutionPlan>,
18 ) -> Result<Pin<Box<dyn Stream<Item = Result<RecordBatch>> + Send>>>;
19
20 #[instrument(skip(self, plan))]
22 async fn execute_collect(&self, plan: Arc<dyn ExecutionPlan>) -> Result<Vec<RecordBatch>> {
23 debug!("Starting batch collection");
24 let mut stream = self.execute_stream(plan).await?;
25 let mut results = Vec::new();
26 let mut total_rows = 0;
27
28 while let Some(batch) = stream.next().await {
29 let batch = batch?;
30 total_rows += batch.num_rows();
31 debug!(
32 batch_rows = batch.num_rows(),
33 total_rows = total_rows,
34 "Collected batch"
35 );
36 results.push(batch);
37 }
38
39 info!(
40 total_batches = results.len(),
41 total_rows = total_rows,
42 "Query execution completed"
43 );
44 Ok(results)
45 }
46}
47
48pub struct DataFusionExecutor {
50 config: ExecutorConfig,
52 #[allow(dead_code)]
54 ctx: SessionContext,
55}
56
57#[derive(Debug, Clone)]
59pub struct ExecutorConfig {
60 pub max_concurrent_tasks: usize,
62 pub batch_size: usize,
64 pub memory_limit: usize,
66}
67
68impl Default for ExecutorConfig {
69 fn default() -> Self {
70 Self {
71 max_concurrent_tasks: num_cpus::get(),
72 batch_size: 8192,
73 memory_limit: 1024 * 1024 * 1024, }
75 }
76}
77
78impl DataFusionExecutor {
79 #[instrument(skip(config))]
80 pub fn new(config: ExecutorConfig) -> Self {
81 debug!(
82 max_tasks = config.max_concurrent_tasks,
83 batch_size = config.batch_size,
84 memory_limit = config.memory_limit,
85 "Creating new DataFusionExecutor"
86 );
87 let ctx = SessionContext::new();
88 info!("Created new DataFusionExecutor");
89 Self { config, ctx }
90 }
91
92 #[instrument(skip(self, config))]
93 pub fn with_config(mut self, config: ExecutorConfig) -> Self {
94 debug!(
95 old_max_tasks = self.config.max_concurrent_tasks,
96 new_max_tasks = config.max_concurrent_tasks,
97 old_batch_size = self.config.batch_size,
98 new_batch_size = config.batch_size,
99 old_memory_limit = self.config.memory_limit,
100 new_memory_limit = config.memory_limit,
101 "Updating executor configuration"
102 );
103 self.config = config;
104 info!("Updated executor configuration");
105 self
106 }
107
108 #[instrument(skip(self))]
109 fn create_runtime_env(&self) -> Arc<RuntimeEnv> {
110 debug!("Creating runtime environment");
111 let env = self.ctx.runtime_env();
113 debug!("Created runtime environment");
114 env
115 }
116}
117
118#[async_trait::async_trait]
119impl QueryExecutor for DataFusionExecutor {
120 #[instrument(skip(self, plan), fields(plan_type = ?plan.as_ref()))]
121 async fn execute_stream(
122 &self,
123 plan: Arc<dyn ExecutionPlan>,
124 ) -> Result<Pin<Box<dyn Stream<Item = Result<RecordBatch>> + Send>>> {
125 use std::collections::HashMap;
126
127 debug!(
128 schema = ?plan.schema(),
129 partitions = ?plan.as_ref().output_partitioning(),
130 "Starting query execution"
131 );
132
133 let runtime = self.create_runtime_env();
135 debug!(
136 max_tasks = self.config.max_concurrent_tasks,
137 batch_size = self.config.batch_size,
138 memory_limit = self.config.memory_limit,
139 "Created runtime environment"
140 );
141
142 let session_config = SessionConfig::new().with_batch_size(self.config.batch_size);
144
145 let task_ctx = Arc::new(TaskContext::new(
147 Some("query_execution".to_owned()),
148 "task_1".to_owned(),
149 session_config,
150 HashMap::new(), HashMap::new(), HashMap::new(), runtime,
154 ));
155 debug!("Created task context");
156
157 let stream = plan.execute(0, task_ctx)?;
159 info!("Query execution started successfully");
160
161 let logged_stream = Box::pin(stream.inspect(|result| match result {
163 Ok(batch) => debug!(
164 rows = batch.num_rows(),
165 columns = batch.num_columns(),
166 "Processed record batch"
167 ),
168 Err(e) => warn!(error = ?e, "Error processing record batch"),
169 }));
170
171 Ok(logged_stream)
172 }
173}
174
175#[derive(Debug, Default)]
177pub struct ExecutionMetrics {
178 pub records_processed: usize,
180 pub batches_processed: usize,
182 pub execution_time_ms: u64,
184 pub memory_usage: usize,
186}
187
188use futures::StreamExt;