Skip to main content

hyprstream_metrics/query/physical/
vector.rs

1//! Vectorized operator implementation for efficient vector operations
2//!
3//! This module provides SIMD-accelerated implementations of common vector
4//! operations used in query processing.
5
6use super::{BaseOperator, PhysicalOperator};
7use datafusion::arrow::array::{Array, ArrayRef, Float32Array, Float64Array};
8use datafusion::arrow::datatypes::{DataType, Schema};
9use datafusion::arrow::record_batch::RecordBatch;
10use datafusion::error::{DataFusionError, Result};
11use std::collections::HashMap;
12use std::sync::Arc;
13
14/// Types of vector operations supported
15#[derive(Debug, Clone, Copy)]
16pub enum VectorOperation {
17    /// Element-wise addition
18    Add,
19    /// Element-wise multiplication
20    Multiply,
21    /// Dot product
22    DotProduct,
23    /// L2 normalization
24    Normalize,
25    /// Cosine similarity
26    CosineSimilarity,
27}
28
29impl std::str::FromStr for VectorOperation {
30    type Err = DataFusionError;
31
32    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
33        match s.to_lowercase().as_str() {
34            "add" => Ok(VectorOperation::Add),
35            "multiply" => Ok(VectorOperation::Multiply),
36            "dot_product" => Ok(VectorOperation::DotProduct),
37            "normalize" => Ok(VectorOperation::Normalize),
38            "cosine_similarity" => Ok(VectorOperation::CosineSimilarity),
39            _ => Err(DataFusionError::NotImplemented(format!(
40                "Vector operation {s} not implemented"
41            ))),
42        }
43    }
44}
45
46/// Operator for vectorized computations
47pub struct VectorizedOperator {
48    /// Base operator implementation
49    base: BaseOperator,
50    /// Type of vector operation
51    operation: VectorOperation,
52    /// Input column names
53    input_columns: Vec<String>,
54    /// Output column name
55    // TODO: Use this field when implementing column renaming in execute method
56    #[allow(dead_code)]
57    output_column: String,
58}
59
60impl VectorizedOperator {
61    pub fn new(
62        schema: Arc<Schema>,
63        children: Vec<Arc<dyn PhysicalOperator>>,
64        properties: HashMap<String, String>,
65    ) -> Result<Self> {
66        // Parse operation type
67        let operation = properties
68            .get("operation")
69            .ok_or_else(|| DataFusionError::Plan("Missing operation type".to_owned()))?
70            .parse()?;
71
72        // Get input and output columns
73        let input_columns = properties
74            .get("input_columns")
75            .ok_or_else(|| DataFusionError::Plan("Missing input columns".to_owned()))?
76            .split(',')
77            .map(|s| s.trim().to_owned())
78            .collect();
79
80        let output_column = properties
81            .get("output_column")
82            .ok_or_else(|| DataFusionError::Plan("Missing output column".to_owned()))?
83            .clone();
84
85        Ok(Self {
86            base: BaseOperator::new(schema, children),
87            operation,
88            input_columns,
89            output_column,
90        })
91    }
92
93    /// Execute vector operation on arrays
94    fn execute_vector_op(&self, left: &ArrayRef, right: &ArrayRef) -> Result<ArrayRef> {
95        match self.operation {
96            VectorOperation::Add => {
97                // Element-wise addition
98                match (left.data_type(), right.data_type()) {
99                    (DataType::Float32, DataType::Float32) => {
100                        let l = left
101                            .as_any()
102                            .downcast_ref::<Float32Array>()
103                            .ok_or_else(|| {
104                                DataFusionError::Internal(
105                                    "Failed to downcast to Float32Array".to_owned(),
106                                )
107                            })?;
108                        let r = right
109                            .as_any()
110                            .downcast_ref::<Float32Array>()
111                            .ok_or_else(|| {
112                                DataFusionError::Internal(
113                                    "Failed to downcast to Float32Array".to_owned(),
114                                )
115                            })?;
116                        let mut result = Vec::with_capacity(l.len());
117                        for i in 0..l.len() {
118                            result.push(l.value(i) + r.value(i));
119                        }
120                        Ok(Arc::new(Float32Array::from(result)) as ArrayRef)
121                    }
122                    (DataType::Float64, DataType::Float64) => {
123                        let l = left
124                            .as_any()
125                            .downcast_ref::<Float64Array>()
126                            .ok_or_else(|| {
127                                DataFusionError::Internal(
128                                    "Failed to downcast to Float64Array".to_owned(),
129                                )
130                            })?;
131                        let r = right
132                            .as_any()
133                            .downcast_ref::<Float64Array>()
134                            .ok_or_else(|| {
135                                DataFusionError::Internal(
136                                    "Failed to downcast to Float64Array".to_owned(),
137                                )
138                            })?;
139                        let mut result = Vec::with_capacity(l.len());
140                        for i in 0..l.len() {
141                            result.push(l.value(i) + r.value(i));
142                        }
143                        Ok(Arc::new(Float64Array::from(result)) as ArrayRef)
144                    }
145                    _ => Err(DataFusionError::NotImplemented(
146                        "Unsupported data type for vector addition".to_owned(),
147                    )),
148                }
149            }
150            VectorOperation::Multiply => {
151                // Element-wise multiplication
152                match (left.data_type(), right.data_type()) {
153                    (DataType::Float32, DataType::Float32) => {
154                        let l = left
155                            .as_any()
156                            .downcast_ref::<Float32Array>()
157                            .ok_or_else(|| {
158                                DataFusionError::Internal(
159                                    "Failed to downcast to Float32Array".to_owned(),
160                                )
161                            })?;
162                        let r = right
163                            .as_any()
164                            .downcast_ref::<Float32Array>()
165                            .ok_or_else(|| {
166                                DataFusionError::Internal(
167                                    "Failed to downcast to Float32Array".to_owned(),
168                                )
169                            })?;
170                        let mut result = Vec::with_capacity(l.len());
171                        for i in 0..l.len() {
172                            result.push(l.value(i) * r.value(i));
173                        }
174                        Ok(Arc::new(Float32Array::from(result)) as ArrayRef)
175                    }
176                    (DataType::Float64, DataType::Float64) => {
177                        let l = left
178                            .as_any()
179                            .downcast_ref::<Float64Array>()
180                            .ok_or_else(|| {
181                                DataFusionError::Internal(
182                                    "Failed to downcast to Float64Array".to_owned(),
183                                )
184                            })?;
185                        let r = right
186                            .as_any()
187                            .downcast_ref::<Float64Array>()
188                            .ok_or_else(|| {
189                                DataFusionError::Internal(
190                                    "Failed to downcast to Float64Array".to_owned(),
191                                )
192                            })?;
193                        let mut result = Vec::with_capacity(l.len());
194                        for i in 0..l.len() {
195                            result.push(l.value(i) * r.value(i));
196                        }
197                        Ok(Arc::new(Float64Array::from(result)) as ArrayRef)
198                    }
199                    _ => Err(DataFusionError::NotImplemented(
200                        "Unsupported data type for vector multiplication".to_owned(),
201                    )),
202                }
203            }
204            VectorOperation::DotProduct => {
205                // Dot product using element-wise multiply and sum
206                match (left.data_type(), right.data_type()) {
207                    (DataType::Float32, DataType::Float32) => {
208                        let l = left
209                            .as_any()
210                            .downcast_ref::<Float32Array>()
211                            .ok_or_else(|| {
212                                DataFusionError::Internal(
213                                    "Failed to downcast to Float32Array".to_owned(),
214                                )
215                            })?;
216                        let r = right
217                            .as_any()
218                            .downcast_ref::<Float32Array>()
219                            .ok_or_else(|| {
220                                DataFusionError::Internal(
221                                    "Failed to downcast to Float32Array".to_owned(),
222                                )
223                            })?;
224                        let mut sum = 0.0;
225                        for i in 0..l.len() {
226                            sum += l.value(i) * r.value(i);
227                        }
228                        Ok(Arc::new(Float32Array::from(vec![sum])) as ArrayRef)
229                    }
230                    _ => Err(DataFusionError::NotImplemented(
231                        "Unsupported data type for dot product".to_owned(),
232                    )),
233                }
234            }
235            VectorOperation::Normalize => {
236                // L2 normalization
237                match left.data_type() {
238                    DataType::Float32 => {
239                        let arr =
240                            left.as_any()
241                                .downcast_ref::<Float32Array>()
242                                .ok_or_else(|| {
243                                    DataFusionError::Internal(
244                                        "Failed to downcast to Float32Array".to_owned(),
245                                    )
246                                })?;
247                        let norm = (arr.iter().flatten().map(|x| x * x).sum::<f32>()).sqrt();
248                        let normalized = arr.iter().map(|x| x.map(|v| v / norm));
249                        Ok(Arc::new(Float32Array::from_iter(normalized)) as ArrayRef)
250                    }
251                    _ => Err(DataFusionError::NotImplemented(
252                        "Unsupported data type for normalization".to_owned(),
253                    )),
254                }
255            }
256            VectorOperation::CosineSimilarity => {
257                // Cosine similarity using dot product and norms
258                match (left.data_type(), right.data_type()) {
259                    (DataType::Float32, DataType::Float32) => {
260                        let l = left
261                            .as_any()
262                            .downcast_ref::<Float32Array>()
263                            .ok_or_else(|| {
264                                DataFusionError::Internal(
265                                    "Failed to downcast to Float32Array".to_owned(),
266                                )
267                            })?;
268                        let r = right
269                            .as_any()
270                            .downcast_ref::<Float32Array>()
271                            .ok_or_else(|| {
272                                DataFusionError::Internal(
273                                    "Failed to downcast to Float32Array".to_owned(),
274                                )
275                            })?;
276
277                        let mut dot_product = 0.0;
278                        for i in 0..l.len() {
279                            dot_product += l.value(i) * r.value(i);
280                        }
281
282                        let l_norm = (l.iter().flatten().map(|x| x * x).sum::<f32>()).sqrt();
283                        let r_norm = (r.iter().flatten().map(|x| x * x).sum::<f32>()).sqrt();
284
285                        let similarity = dot_product / (l_norm * r_norm);
286                        Ok(Arc::new(Float32Array::from(vec![similarity])) as ArrayRef)
287                    }
288                    _ => Err(DataFusionError::NotImplemented(
289                        "Unsupported data type for cosine similarity".to_owned(),
290                    )),
291                }
292            }
293        }
294    }
295}
296
297impl PhysicalOperator for VectorizedOperator {
298    fn schema(&self) -> &Arc<Schema> {
299        self.base.schema()
300    }
301
302    fn execute(&self, input: Vec<RecordBatch>) -> Result<Vec<RecordBatch>> {
303        let mut output_batches = Vec::with_capacity(input.len());
304
305        for batch in input {
306            // Get input arrays
307            let arrays: Result<Vec<ArrayRef>> = self
308                .input_columns
309                .iter()
310                .map(|col| {
311                    batch
312                        .column_by_name(col)
313                        .ok_or_else(|| DataFusionError::Plan(format!("Column {col} not found")))
314                        .map(std::clone::Clone::clone)
315                })
316                .collect();
317
318            let arrays = arrays?;
319            if arrays.len() != 2 {
320                return Err(DataFusionError::Plan(
321                    "Vector operations require exactly two input arrays".to_owned(),
322                ));
323            }
324
325            // Execute vector operation
326            let result = self.execute_vector_op(&arrays[0], &arrays[1])?;
327
328            // Create output batch
329            let mut columns = batch.columns().to_vec();
330            columns.push(result);
331
332            let output_batch = RecordBatch::try_new(self.base.schema().clone(), columns)
333                .map_err(|e| DataFusionError::Internal(e.to_string()))?;
334            output_batches.push(output_batch);
335        }
336
337        Ok(output_batches)
338    }
339
340    fn children(&self) -> Vec<Arc<dyn PhysicalOperator>> {
341        self.base.children()
342    }
343}