Skip to main content

hyprstream_metrics/storage/
datafusion_provider.rs

1//! DataFusion TableProvider implementation for DuckDB storage.
2//!
3//! This module provides a bridge between DuckDB tables and DataFusion's
4//! query planning system, allowing DataFusion to query DuckDB tables directly.
5
6use crate::storage::duckdb::DuckDbBackend;
7use crate::storage::StorageBackend;
8use async_trait::async_trait;
9use datafusion::arrow::datatypes::SchemaRef;
10use datafusion::catalog::Session;
11use datafusion::datasource::TableProvider;
12use datafusion::datasource::TableType;
13use datafusion::error::Result;
14use datafusion::logical_expr::Expr;
15use datafusion::physical_plan::ExecutionPlan;
16use std::any::Any;
17use std::sync::Arc;
18use tracing::{debug, instrument};
19
20/// A DataFusion TableProvider that reads from a DuckDB table.
21///
22/// This provider enables DataFusion to plan and execute queries against
23/// DuckDB tables using its standard query planning infrastructure.
24pub struct DuckDBTableProvider {
25    /// Reference to the DuckDB backend
26    backend: Arc<DuckDbBackend>,
27    /// Name of the table in DuckDB
28    table_name: String,
29    /// Arrow schema for the table
30    schema: SchemaRef,
31}
32
33impl DuckDBTableProvider {
34    /// Create a new DuckDBTableProvider for the specified table.
35    #[instrument(skip(backend), fields(table = %table_name))]
36    pub async fn new(
37        backend: Arc<DuckDbBackend>,
38        table_name: String,
39    ) -> std::result::Result<Self, tonic::Status> {
40        debug!("Creating DuckDBTableProvider");
41
42        // Get schema from DuckDB
43        let schema = backend.get_table_schema(&table_name).await?;
44
45        debug!(fields = schema.fields().len(), "Retrieved schema for table");
46
47        Ok(Self {
48            backend,
49            table_name,
50            schema,
51        })
52    }
53
54    /// Create a provider with a known schema (avoids async schema lookup).
55    pub fn with_schema(backend: Arc<DuckDbBackend>, table_name: String, schema: SchemaRef) -> Self {
56        Self {
57            backend,
58            table_name,
59            schema,
60        }
61    }
62}
63
64impl std::fmt::Debug for DuckDBTableProvider {
65    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66        f.debug_struct("DuckDBTableProvider")
67            .field("table_name", &self.table_name)
68            .field("schema", &self.schema)
69            .finish()
70    }
71}
72
73#[async_trait]
74impl TableProvider for DuckDBTableProvider {
75    fn as_any(&self) -> &dyn Any {
76        self
77    }
78
79    fn schema(&self) -> SchemaRef {
80        Arc::clone(&self.schema)
81    }
82
83    fn table_type(&self) -> TableType {
84        TableType::Base
85    }
86
87    #[instrument(skip(self, _state, projection, filters, limit), fields(table = %self.table_name))]
88    async fn scan(
89        &self,
90        _state: &dyn Session,
91        projection: Option<&Vec<usize>>,
92        filters: &[Expr],
93        limit: Option<usize>,
94    ) -> Result<Arc<dyn ExecutionPlan>> {
95        debug!(
96            projection = ?projection,
97            filters_count = filters.len(),
98            limit = ?limit,
99            "Creating scan plan for DuckDB table"
100        );
101
102        // Create a DuckDBExec plan that will execute the query
103        let plan = DuckDBExec::new(
104            Arc::clone(&self.backend),
105            self.table_name.clone(),
106            Arc::clone(&self.schema),
107            projection.cloned(),
108            filters.to_vec(),
109            limit,
110        );
111
112        Ok(Arc::new(plan))
113    }
114}
115
116/// Physical execution plan for scanning a DuckDB table.
117pub struct DuckDBExec {
118    backend: Arc<DuckDbBackend>,
119    table_name: String,
120    schema: SchemaRef,
121    projected_schema: SchemaRef,
122    projection: Option<Vec<usize>>,
123    filters: Vec<Expr>,
124    limit: Option<usize>,
125    /// Cached plan properties
126    properties: datafusion::physical_plan::PlanProperties,
127}
128
129impl std::fmt::Debug for DuckDBExec {
130    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
131        f.debug_struct("DuckDBExec")
132            .field("table_name", &self.table_name)
133            .field("schema", &self.schema)
134            .field("projected_schema", &self.projected_schema)
135            .field("projection", &self.projection)
136            .field("filters", &self.filters)
137            .field("limit", &self.limit)
138            .finish()
139    }
140}
141
142impl DuckDBExec {
143    fn new(
144        backend: Arc<DuckDbBackend>,
145        table_name: String,
146        schema: SchemaRef,
147        projection: Option<Vec<usize>>,
148        filters: Vec<Expr>,
149        limit: Option<usize>,
150    ) -> Self {
151        use datafusion::physical_expr::EquivalenceProperties;
152        use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
153        use datafusion::physical_plan::Partitioning;
154
155        // Compute projected schema
156        let projected_schema = if let Some(ref proj) = projection {
157            let fields: Vec<_> = proj
158                .iter()
159                .filter_map(|i| schema.fields().get(*i).cloned())
160                .collect();
161            Arc::new(datafusion::arrow::datatypes::Schema::new(fields))
162        } else {
163            Arc::clone(&schema)
164        };
165
166        // Create properties with the correct schema
167        let properties = datafusion::physical_plan::PlanProperties::new(
168            EquivalenceProperties::new(Arc::clone(&projected_schema)),
169            Partitioning::UnknownPartitioning(1),
170            EmissionType::Incremental,
171            Boundedness::Bounded,
172        );
173
174        Self {
175            backend,
176            table_name,
177            schema,
178            projected_schema,
179            projection,
180            filters,
181            limit,
182            properties,
183        }
184    }
185
186    /// Build the SQL query for this scan
187    fn build_sql(&self) -> String {
188        // Build column list
189        let columns = if let Some(ref proj) = self.projection {
190            proj.iter()
191                .filter_map(|i| self.schema.fields().get(*i))
192                .map(|f| f.name().clone())
193                .collect::<Vec<_>>()
194                .join(", ")
195        } else {
196            "*".to_owned()
197        };
198
199        let mut sql = format!("SELECT {} FROM {}", columns, self.table_name);
200
201        // Add filters (simplified - full implementation would convert Expr to SQL)
202        // For now, we'll push down simple filters
203        if !self.filters.is_empty() {
204            // TODO: Implement proper filter pushdown
205            // This would require converting DataFusion Expr to SQL WHERE clauses
206        }
207
208        // Add limit
209        if let Some(limit) = self.limit {
210            sql.push_str(&format!(" LIMIT {limit}"));
211        }
212
213        sql
214    }
215}
216
217impl datafusion::physical_plan::DisplayAs for DuckDBExec {
218    fn fmt_as(
219        &self,
220        t: datafusion::physical_plan::DisplayFormatType,
221        f: &mut std::fmt::Formatter,
222    ) -> std::fmt::Result {
223        match t {
224            datafusion::physical_plan::DisplayFormatType::Default
225            | datafusion::physical_plan::DisplayFormatType::Verbose
226            | datafusion::physical_plan::DisplayFormatType::TreeRender => {
227                write!(
228                    f,
229                    "DuckDBExec: table={}, projection={:?}, limit={:?}",
230                    self.table_name, self.projection, self.limit
231                )
232            }
233        }
234    }
235}
236
237impl datafusion::physical_plan::ExecutionPlan for DuckDBExec {
238    fn name(&self) -> &str {
239        "DuckDBExec"
240    }
241
242    fn as_any(&self) -> &dyn Any {
243        self
244    }
245
246    fn schema(&self) -> SchemaRef {
247        Arc::clone(&self.projected_schema)
248    }
249
250    fn properties(&self) -> &datafusion::physical_plan::PlanProperties {
251        &self.properties
252    }
253
254    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
255        vec![]
256    }
257
258    fn with_new_children(
259        self: Arc<Self>,
260        _children: Vec<Arc<dyn ExecutionPlan>>,
261    ) -> Result<Arc<dyn ExecutionPlan>> {
262        Ok(self)
263    }
264
265    fn execute(
266        &self,
267        _partition: usize,
268        _context: Arc<datafusion::execution::TaskContext>,
269    ) -> Result<datafusion::physical_plan::SendableRecordBatchStream> {
270        use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
271        use futures::stream;
272        use futures::StreamExt;
273
274        let sql = self.build_sql();
275        let backend = Arc::clone(&self.backend);
276        let schema = Arc::clone(&self.projected_schema);
277
278        // Create a stream that executes the query
279        let stream = stream::once(async move {
280            // Prepare and execute the query
281            let handle = backend
282                .prepare_sql(&sql)
283                .await
284                .map_err(|e| datafusion::error::DataFusionError::External(Box::new(e)))?;
285
286            let batch = backend
287                .query_sql(&handle)
288                .await
289                .map_err(|e| datafusion::error::DataFusionError::External(Box::new(e)))?;
290
291            Ok(batch)
292        });
293
294        Ok(Box::pin(RecordBatchStreamAdapter::new(
295            schema,
296            stream.boxed(),
297        )))
298    }
299}
300
301#[cfg(test)]
302mod tests {
303    use super::*;
304    use datafusion::arrow::datatypes::{DataType, Field, Schema};
305    use datafusion::prelude::SessionContext;
306
307    #[tokio::test]
308    async fn test_duckdb_table_provider() -> std::result::Result<(), Box<dyn std::error::Error>> {
309        // Create test backend
310        let backend = Arc::new(DuckDbBackend::new_in_memory()?);
311        backend.init().await?;
312
313        // Create test table
314        let schema = Arc::new(Schema::new(vec![
315            Field::new("id", DataType::Int64, false),
316            Field::new("name", DataType::Utf8, true),
317        ]));
318        backend.create_table("test_provider", &schema).await?;
319
320        // Create provider
321        let provider = DuckDBTableProvider::new(backend, "test_provider".to_owned()).await?;
322
323        // Verify schema
324        assert_eq!(provider.schema().fields().len(), 2);
325        assert_eq!(provider.table_type(), TableType::Base);
326
327        Ok(())
328    }
329
330    #[tokio::test]
331    async fn test_duckdb_provider_with_datafusion(
332    ) -> std::result::Result<(), Box<dyn std::error::Error>> {
333        // Create test backend
334        let backend = Arc::new(DuckDbBackend::new_in_memory()?);
335        backend.init().await?;
336
337        // Create test table
338        let schema = Arc::new(Schema::new(vec![
339            Field::new("id", DataType::Int64, false),
340            Field::new("value", DataType::Float64, true),
341        ]));
342        backend.create_table("metrics", &schema).await?;
343
344        // Create provider and register with DataFusion
345        let provider = DuckDBTableProvider::new(backend, "metrics".to_owned()).await?;
346        let ctx = SessionContext::new();
347        ctx.register_table("metrics", Arc::new(provider))?;
348
349        // Query through DataFusion (should work even with empty table)
350        let df = ctx.sql("SELECT * FROM metrics").await?;
351        let results = df.collect().await?;
352
353        // Empty table returns 0-1 batches
354        assert!(results.is_empty() || results[0].num_rows() == 0);
355
356        Ok(())
357    }
358}