hyprstream_metrics/storage/
datafusion_provider.rs1use crate::storage::duckdb::DuckDbBackend;
7use crate::storage::StorageBackend;
8use async_trait::async_trait;
9use datafusion::arrow::datatypes::SchemaRef;
10use datafusion::catalog::Session;
11use datafusion::datasource::TableProvider;
12use datafusion::datasource::TableType;
13use datafusion::error::Result;
14use datafusion::logical_expr::Expr;
15use datafusion::physical_plan::ExecutionPlan;
16use std::any::Any;
17use std::sync::Arc;
18use tracing::{debug, instrument};
19
20pub struct DuckDBTableProvider {
25 backend: Arc<DuckDbBackend>,
27 table_name: String,
29 schema: SchemaRef,
31}
32
33impl DuckDBTableProvider {
34 #[instrument(skip(backend), fields(table = %table_name))]
36 pub async fn new(
37 backend: Arc<DuckDbBackend>,
38 table_name: String,
39 ) -> std::result::Result<Self, tonic::Status> {
40 debug!("Creating DuckDBTableProvider");
41
42 let schema = backend.get_table_schema(&table_name).await?;
44
45 debug!(fields = schema.fields().len(), "Retrieved schema for table");
46
47 Ok(Self {
48 backend,
49 table_name,
50 schema,
51 })
52 }
53
54 pub fn with_schema(backend: Arc<DuckDbBackend>, table_name: String, schema: SchemaRef) -> Self {
56 Self {
57 backend,
58 table_name,
59 schema,
60 }
61 }
62}
63
64impl std::fmt::Debug for DuckDBTableProvider {
65 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66 f.debug_struct("DuckDBTableProvider")
67 .field("table_name", &self.table_name)
68 .field("schema", &self.schema)
69 .finish()
70 }
71}
72
73#[async_trait]
74impl TableProvider for DuckDBTableProvider {
75 fn as_any(&self) -> &dyn Any {
76 self
77 }
78
79 fn schema(&self) -> SchemaRef {
80 Arc::clone(&self.schema)
81 }
82
83 fn table_type(&self) -> TableType {
84 TableType::Base
85 }
86
87 #[instrument(skip(self, _state, projection, filters, limit), fields(table = %self.table_name))]
88 async fn scan(
89 &self,
90 _state: &dyn Session,
91 projection: Option<&Vec<usize>>,
92 filters: &[Expr],
93 limit: Option<usize>,
94 ) -> Result<Arc<dyn ExecutionPlan>> {
95 debug!(
96 projection = ?projection,
97 filters_count = filters.len(),
98 limit = ?limit,
99 "Creating scan plan for DuckDB table"
100 );
101
102 let plan = DuckDBExec::new(
104 Arc::clone(&self.backend),
105 self.table_name.clone(),
106 Arc::clone(&self.schema),
107 projection.cloned(),
108 filters.to_vec(),
109 limit,
110 );
111
112 Ok(Arc::new(plan))
113 }
114}
115
116pub struct DuckDBExec {
118 backend: Arc<DuckDbBackend>,
119 table_name: String,
120 schema: SchemaRef,
121 projected_schema: SchemaRef,
122 projection: Option<Vec<usize>>,
123 filters: Vec<Expr>,
124 limit: Option<usize>,
125 properties: datafusion::physical_plan::PlanProperties,
127}
128
129impl std::fmt::Debug for DuckDBExec {
130 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
131 f.debug_struct("DuckDBExec")
132 .field("table_name", &self.table_name)
133 .field("schema", &self.schema)
134 .field("projected_schema", &self.projected_schema)
135 .field("projection", &self.projection)
136 .field("filters", &self.filters)
137 .field("limit", &self.limit)
138 .finish()
139 }
140}
141
142impl DuckDBExec {
143 fn new(
144 backend: Arc<DuckDbBackend>,
145 table_name: String,
146 schema: SchemaRef,
147 projection: Option<Vec<usize>>,
148 filters: Vec<Expr>,
149 limit: Option<usize>,
150 ) -> Self {
151 use datafusion::physical_expr::EquivalenceProperties;
152 use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
153 use datafusion::physical_plan::Partitioning;
154
155 let projected_schema = if let Some(ref proj) = projection {
157 let fields: Vec<_> = proj
158 .iter()
159 .filter_map(|i| schema.fields().get(*i).cloned())
160 .collect();
161 Arc::new(datafusion::arrow::datatypes::Schema::new(fields))
162 } else {
163 Arc::clone(&schema)
164 };
165
166 let properties = datafusion::physical_plan::PlanProperties::new(
168 EquivalenceProperties::new(Arc::clone(&projected_schema)),
169 Partitioning::UnknownPartitioning(1),
170 EmissionType::Incremental,
171 Boundedness::Bounded,
172 );
173
174 Self {
175 backend,
176 table_name,
177 schema,
178 projected_schema,
179 projection,
180 filters,
181 limit,
182 properties,
183 }
184 }
185
186 fn build_sql(&self) -> String {
188 let columns = if let Some(ref proj) = self.projection {
190 proj.iter()
191 .filter_map(|i| self.schema.fields().get(*i))
192 .map(|f| f.name().clone())
193 .collect::<Vec<_>>()
194 .join(", ")
195 } else {
196 "*".to_owned()
197 };
198
199 let mut sql = format!("SELECT {} FROM {}", columns, self.table_name);
200
201 if !self.filters.is_empty() {
204 }
207
208 if let Some(limit) = self.limit {
210 sql.push_str(&format!(" LIMIT {limit}"));
211 }
212
213 sql
214 }
215}
216
217impl datafusion::physical_plan::DisplayAs for DuckDBExec {
218 fn fmt_as(
219 &self,
220 t: datafusion::physical_plan::DisplayFormatType,
221 f: &mut std::fmt::Formatter,
222 ) -> std::fmt::Result {
223 match t {
224 datafusion::physical_plan::DisplayFormatType::Default
225 | datafusion::physical_plan::DisplayFormatType::Verbose
226 | datafusion::physical_plan::DisplayFormatType::TreeRender => {
227 write!(
228 f,
229 "DuckDBExec: table={}, projection={:?}, limit={:?}",
230 self.table_name, self.projection, self.limit
231 )
232 }
233 }
234 }
235}
236
237impl datafusion::physical_plan::ExecutionPlan for DuckDBExec {
238 fn name(&self) -> &str {
239 "DuckDBExec"
240 }
241
242 fn as_any(&self) -> &dyn Any {
243 self
244 }
245
246 fn schema(&self) -> SchemaRef {
247 Arc::clone(&self.projected_schema)
248 }
249
250 fn properties(&self) -> &datafusion::physical_plan::PlanProperties {
251 &self.properties
252 }
253
254 fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
255 vec![]
256 }
257
258 fn with_new_children(
259 self: Arc<Self>,
260 _children: Vec<Arc<dyn ExecutionPlan>>,
261 ) -> Result<Arc<dyn ExecutionPlan>> {
262 Ok(self)
263 }
264
265 fn execute(
266 &self,
267 _partition: usize,
268 _context: Arc<datafusion::execution::TaskContext>,
269 ) -> Result<datafusion::physical_plan::SendableRecordBatchStream> {
270 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
271 use futures::stream;
272 use futures::StreamExt;
273
274 let sql = self.build_sql();
275 let backend = Arc::clone(&self.backend);
276 let schema = Arc::clone(&self.projected_schema);
277
278 let stream = stream::once(async move {
280 let handle = backend
282 .prepare_sql(&sql)
283 .await
284 .map_err(|e| datafusion::error::DataFusionError::External(Box::new(e)))?;
285
286 let batch = backend
287 .query_sql(&handle)
288 .await
289 .map_err(|e| datafusion::error::DataFusionError::External(Box::new(e)))?;
290
291 Ok(batch)
292 });
293
294 Ok(Box::pin(RecordBatchStreamAdapter::new(
295 schema,
296 stream.boxed(),
297 )))
298 }
299}
300
301#[cfg(test)]
302mod tests {
303 use super::*;
304 use datafusion::arrow::datatypes::{DataType, Field, Schema};
305 use datafusion::prelude::SessionContext;
306
307 #[tokio::test]
308 async fn test_duckdb_table_provider() -> std::result::Result<(), Box<dyn std::error::Error>> {
309 let backend = Arc::new(DuckDbBackend::new_in_memory()?);
311 backend.init().await?;
312
313 let schema = Arc::new(Schema::new(vec![
315 Field::new("id", DataType::Int64, false),
316 Field::new("name", DataType::Utf8, true),
317 ]));
318 backend.create_table("test_provider", &schema).await?;
319
320 let provider = DuckDBTableProvider::new(backend, "test_provider".to_owned()).await?;
322
323 assert_eq!(provider.schema().fields().len(), 2);
325 assert_eq!(provider.table_type(), TableType::Base);
326
327 Ok(())
328 }
329
330 #[tokio::test]
331 async fn test_duckdb_provider_with_datafusion(
332 ) -> std::result::Result<(), Box<dyn std::error::Error>> {
333 let backend = Arc::new(DuckDbBackend::new_in_memory()?);
335 backend.init().await?;
336
337 let schema = Arc::new(Schema::new(vec![
339 Field::new("id", DataType::Int64, false),
340 Field::new("value", DataType::Float64, true),
341 ]));
342 backend.create_table("metrics", &schema).await?;
343
344 let provider = DuckDBTableProvider::new(backend, "metrics".to_owned()).await?;
346 let ctx = SessionContext::new();
347 ctx.register_table("metrics", Arc::new(provider))?;
348
349 let df = ctx.sql("SELECT * FROM metrics").await?;
351 let results = df.collect().await?;
352
353 assert!(results.is_empty() || results[0].num_rows() == 0);
355
356 Ok(())
357 }
358}