Skip to main content

hyprstream_metrics/query/
executor.rs

1use datafusion::arrow::record_batch::RecordBatch;
2use datafusion::error::Result;
3use datafusion::execution::context::{SessionConfig, SessionContext, TaskContext};
4use datafusion::execution::runtime_env::RuntimeEnv;
5use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties};
6use futures::Stream;
7use std::pin::Pin;
8use std::sync::Arc;
9use tracing::{debug, info, instrument, warn};
10
11/// Trait for executing physical query plans
12#[async_trait::async_trait]
13pub trait QueryExecutor: Send + Sync {
14    /// Execute a physical plan and return the results as a stream of record batches
15    async fn execute_stream(
16        &self,
17        plan: Arc<dyn ExecutionPlan>,
18    ) -> Result<Pin<Box<dyn Stream<Item = Result<RecordBatch>> + Send>>>;
19
20    /// Execute a physical plan and collect all results into a vector
21    #[instrument(skip(self, plan))]
22    async fn execute_collect(&self, plan: Arc<dyn ExecutionPlan>) -> Result<Vec<RecordBatch>> {
23        debug!("Starting batch collection");
24        let mut stream = self.execute_stream(plan).await?;
25        let mut results = Vec::new();
26        let mut total_rows = 0;
27
28        while let Some(batch) = stream.next().await {
29            let batch = batch?;
30            total_rows += batch.num_rows();
31            debug!(
32                batch_rows = batch.num_rows(),
33                total_rows = total_rows,
34                "Collected batch"
35            );
36            results.push(batch);
37        }
38
39        info!(
40            total_batches = results.len(),
41            total_rows = total_rows,
42            "Query execution completed"
43        );
44        Ok(results)
45    }
46}
47
48/// Default executor implementation using DataFusion
49pub struct DataFusionExecutor {
50    /// Runtime configuration
51    config: ExecutorConfig,
52    /// Session context - used indirectly through task context in execute_stream
53    #[allow(dead_code)]
54    ctx: SessionContext,
55}
56
57/// Configuration for the executor
58#[derive(Debug, Clone)]
59pub struct ExecutorConfig {
60    /// Maximum number of concurrent tasks
61    pub max_concurrent_tasks: usize,
62    /// Batch size for processing
63    pub batch_size: usize,
64    /// Memory limit per query in bytes
65    pub memory_limit: usize,
66}
67
68impl Default for ExecutorConfig {
69    fn default() -> Self {
70        Self {
71            max_concurrent_tasks: num_cpus::get(),
72            batch_size: 8192,
73            memory_limit: 1024 * 1024 * 1024, // 1GB
74        }
75    }
76}
77
78impl DataFusionExecutor {
79    #[instrument(skip(config))]
80    pub fn new(config: ExecutorConfig) -> Self {
81        debug!(
82            max_tasks = config.max_concurrent_tasks,
83            batch_size = config.batch_size,
84            memory_limit = config.memory_limit,
85            "Creating new DataFusionExecutor"
86        );
87        let ctx = SessionContext::new();
88        info!("Created new DataFusionExecutor");
89        Self { config, ctx }
90    }
91
92    #[instrument(skip(self, config))]
93    pub fn with_config(mut self, config: ExecutorConfig) -> Self {
94        debug!(
95            old_max_tasks = self.config.max_concurrent_tasks,
96            new_max_tasks = config.max_concurrent_tasks,
97            old_batch_size = self.config.batch_size,
98            new_batch_size = config.batch_size,
99            old_memory_limit = self.config.memory_limit,
100            new_memory_limit = config.memory_limit,
101            "Updating executor configuration"
102        );
103        self.config = config;
104        info!("Updated executor configuration");
105        self
106    }
107
108    #[instrument(skip(self))]
109    fn create_runtime_env(&self) -> Arc<RuntimeEnv> {
110        debug!("Creating runtime environment");
111        // In DataFusion 50, get RuntimeEnv from the SessionContext
112        let env = self.ctx.runtime_env();
113        debug!("Created runtime environment");
114        env
115    }
116}
117
118#[async_trait::async_trait]
119impl QueryExecutor for DataFusionExecutor {
120    #[instrument(skip(self, plan), fields(plan_type = ?plan.as_ref()))]
121    async fn execute_stream(
122        &self,
123        plan: Arc<dyn ExecutionPlan>,
124    ) -> Result<Pin<Box<dyn Stream<Item = Result<RecordBatch>> + Send>>> {
125        use std::collections::HashMap;
126
127        debug!(
128            schema = ?plan.schema(),
129            partitions = ?plan.as_ref().output_partitioning(),
130            "Starting query execution"
131        );
132
133        // Create runtime environment
134        let runtime = self.create_runtime_env();
135        debug!(
136            max_tasks = self.config.max_concurrent_tasks,
137            batch_size = self.config.batch_size,
138            memory_limit = self.config.memory_limit,
139            "Created runtime environment"
140        );
141
142        // Create session config
143        let session_config = SessionConfig::new().with_batch_size(self.config.batch_size);
144
145        // Create task context with minimal configuration
146        let task_ctx = Arc::new(TaskContext::new(
147            Some("query_execution".to_owned()),
148            "task_1".to_owned(),
149            session_config,
150            HashMap::new(), // scalar functions
151            HashMap::new(), // aggregate functions
152            HashMap::new(), // window functions
153            runtime,
154        ));
155        debug!("Created task context");
156
157        // Execute the plan
158        let stream = plan.execute(0, task_ctx)?;
159        info!("Query execution started successfully");
160
161        // Wrap stream with logging
162        let logged_stream = Box::pin(stream.inspect(|result| match result {
163            Ok(batch) => debug!(
164                rows = batch.num_rows(),
165                columns = batch.num_columns(),
166                "Processed record batch"
167            ),
168            Err(e) => warn!(error = ?e, "Error processing record batch"),
169        }));
170
171        Ok(logged_stream)
172    }
173}
174
175/// Execution metrics for monitoring and optimization
176#[derive(Debug, Default)]
177pub struct ExecutionMetrics {
178    /// Number of records processed
179    pub records_processed: usize,
180    /// Number of batches processed
181    pub batches_processed: usize,
182    /// Execution time in milliseconds
183    pub execution_time_ms: u64,
184    /// Memory usage in bytes
185    pub memory_usage: usize,
186}
187
188// Add missing imports
189use futures::StreamExt;