hyprstream_metrics/query/physical/
vector.rs1use super::{BaseOperator, PhysicalOperator};
7use datafusion::arrow::array::{Array, ArrayRef, Float32Array, Float64Array};
8use datafusion::arrow::datatypes::{DataType, Schema};
9use datafusion::arrow::record_batch::RecordBatch;
10use datafusion::error::{DataFusionError, Result};
11use std::collections::HashMap;
12use std::sync::Arc;
13
14#[derive(Debug, Clone, Copy)]
16pub enum VectorOperation {
17 Add,
19 Multiply,
21 DotProduct,
23 Normalize,
25 CosineSimilarity,
27}
28
29impl std::str::FromStr for VectorOperation {
30 type Err = DataFusionError;
31
32 fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
33 match s.to_lowercase().as_str() {
34 "add" => Ok(VectorOperation::Add),
35 "multiply" => Ok(VectorOperation::Multiply),
36 "dot_product" => Ok(VectorOperation::DotProduct),
37 "normalize" => Ok(VectorOperation::Normalize),
38 "cosine_similarity" => Ok(VectorOperation::CosineSimilarity),
39 _ => Err(DataFusionError::NotImplemented(format!(
40 "Vector operation {s} not implemented"
41 ))),
42 }
43 }
44}
45
46pub struct VectorizedOperator {
48 base: BaseOperator,
50 operation: VectorOperation,
52 input_columns: Vec<String>,
54 #[allow(dead_code)]
57 output_column: String,
58}
59
60impl VectorizedOperator {
61 pub fn new(
62 schema: Arc<Schema>,
63 children: Vec<Arc<dyn PhysicalOperator>>,
64 properties: HashMap<String, String>,
65 ) -> Result<Self> {
66 let operation = properties
68 .get("operation")
69 .ok_or_else(|| DataFusionError::Plan("Missing operation type".to_owned()))?
70 .parse()?;
71
72 let input_columns = properties
74 .get("input_columns")
75 .ok_or_else(|| DataFusionError::Plan("Missing input columns".to_owned()))?
76 .split(',')
77 .map(|s| s.trim().to_owned())
78 .collect();
79
80 let output_column = properties
81 .get("output_column")
82 .ok_or_else(|| DataFusionError::Plan("Missing output column".to_owned()))?
83 .clone();
84
85 Ok(Self {
86 base: BaseOperator::new(schema, children),
87 operation,
88 input_columns,
89 output_column,
90 })
91 }
92
93 fn execute_vector_op(&self, left: &ArrayRef, right: &ArrayRef) -> Result<ArrayRef> {
95 match self.operation {
96 VectorOperation::Add => {
97 match (left.data_type(), right.data_type()) {
99 (DataType::Float32, DataType::Float32) => {
100 let l = left
101 .as_any()
102 .downcast_ref::<Float32Array>()
103 .ok_or_else(|| {
104 DataFusionError::Internal(
105 "Failed to downcast to Float32Array".to_owned(),
106 )
107 })?;
108 let r = right
109 .as_any()
110 .downcast_ref::<Float32Array>()
111 .ok_or_else(|| {
112 DataFusionError::Internal(
113 "Failed to downcast to Float32Array".to_owned(),
114 )
115 })?;
116 let mut result = Vec::with_capacity(l.len());
117 for i in 0..l.len() {
118 result.push(l.value(i) + r.value(i));
119 }
120 Ok(Arc::new(Float32Array::from(result)) as ArrayRef)
121 }
122 (DataType::Float64, DataType::Float64) => {
123 let l = left
124 .as_any()
125 .downcast_ref::<Float64Array>()
126 .ok_or_else(|| {
127 DataFusionError::Internal(
128 "Failed to downcast to Float64Array".to_owned(),
129 )
130 })?;
131 let r = right
132 .as_any()
133 .downcast_ref::<Float64Array>()
134 .ok_or_else(|| {
135 DataFusionError::Internal(
136 "Failed to downcast to Float64Array".to_owned(),
137 )
138 })?;
139 let mut result = Vec::with_capacity(l.len());
140 for i in 0..l.len() {
141 result.push(l.value(i) + r.value(i));
142 }
143 Ok(Arc::new(Float64Array::from(result)) as ArrayRef)
144 }
145 _ => Err(DataFusionError::NotImplemented(
146 "Unsupported data type for vector addition".to_owned(),
147 )),
148 }
149 }
150 VectorOperation::Multiply => {
151 match (left.data_type(), right.data_type()) {
153 (DataType::Float32, DataType::Float32) => {
154 let l = left
155 .as_any()
156 .downcast_ref::<Float32Array>()
157 .ok_or_else(|| {
158 DataFusionError::Internal(
159 "Failed to downcast to Float32Array".to_owned(),
160 )
161 })?;
162 let r = right
163 .as_any()
164 .downcast_ref::<Float32Array>()
165 .ok_or_else(|| {
166 DataFusionError::Internal(
167 "Failed to downcast to Float32Array".to_owned(),
168 )
169 })?;
170 let mut result = Vec::with_capacity(l.len());
171 for i in 0..l.len() {
172 result.push(l.value(i) * r.value(i));
173 }
174 Ok(Arc::new(Float32Array::from(result)) as ArrayRef)
175 }
176 (DataType::Float64, DataType::Float64) => {
177 let l = left
178 .as_any()
179 .downcast_ref::<Float64Array>()
180 .ok_or_else(|| {
181 DataFusionError::Internal(
182 "Failed to downcast to Float64Array".to_owned(),
183 )
184 })?;
185 let r = right
186 .as_any()
187 .downcast_ref::<Float64Array>()
188 .ok_or_else(|| {
189 DataFusionError::Internal(
190 "Failed to downcast to Float64Array".to_owned(),
191 )
192 })?;
193 let mut result = Vec::with_capacity(l.len());
194 for i in 0..l.len() {
195 result.push(l.value(i) * r.value(i));
196 }
197 Ok(Arc::new(Float64Array::from(result)) as ArrayRef)
198 }
199 _ => Err(DataFusionError::NotImplemented(
200 "Unsupported data type for vector multiplication".to_owned(),
201 )),
202 }
203 }
204 VectorOperation::DotProduct => {
205 match (left.data_type(), right.data_type()) {
207 (DataType::Float32, DataType::Float32) => {
208 let l = left
209 .as_any()
210 .downcast_ref::<Float32Array>()
211 .ok_or_else(|| {
212 DataFusionError::Internal(
213 "Failed to downcast to Float32Array".to_owned(),
214 )
215 })?;
216 let r = right
217 .as_any()
218 .downcast_ref::<Float32Array>()
219 .ok_or_else(|| {
220 DataFusionError::Internal(
221 "Failed to downcast to Float32Array".to_owned(),
222 )
223 })?;
224 let mut sum = 0.0;
225 for i in 0..l.len() {
226 sum += l.value(i) * r.value(i);
227 }
228 Ok(Arc::new(Float32Array::from(vec![sum])) as ArrayRef)
229 }
230 _ => Err(DataFusionError::NotImplemented(
231 "Unsupported data type for dot product".to_owned(),
232 )),
233 }
234 }
235 VectorOperation::Normalize => {
236 match left.data_type() {
238 DataType::Float32 => {
239 let arr =
240 left.as_any()
241 .downcast_ref::<Float32Array>()
242 .ok_or_else(|| {
243 DataFusionError::Internal(
244 "Failed to downcast to Float32Array".to_owned(),
245 )
246 })?;
247 let norm = (arr.iter().flatten().map(|x| x * x).sum::<f32>()).sqrt();
248 let normalized = arr.iter().map(|x| x.map(|v| v / norm));
249 Ok(Arc::new(Float32Array::from_iter(normalized)) as ArrayRef)
250 }
251 _ => Err(DataFusionError::NotImplemented(
252 "Unsupported data type for normalization".to_owned(),
253 )),
254 }
255 }
256 VectorOperation::CosineSimilarity => {
257 match (left.data_type(), right.data_type()) {
259 (DataType::Float32, DataType::Float32) => {
260 let l = left
261 .as_any()
262 .downcast_ref::<Float32Array>()
263 .ok_or_else(|| {
264 DataFusionError::Internal(
265 "Failed to downcast to Float32Array".to_owned(),
266 )
267 })?;
268 let r = right
269 .as_any()
270 .downcast_ref::<Float32Array>()
271 .ok_or_else(|| {
272 DataFusionError::Internal(
273 "Failed to downcast to Float32Array".to_owned(),
274 )
275 })?;
276
277 let mut dot_product = 0.0;
278 for i in 0..l.len() {
279 dot_product += l.value(i) * r.value(i);
280 }
281
282 let l_norm = (l.iter().flatten().map(|x| x * x).sum::<f32>()).sqrt();
283 let r_norm = (r.iter().flatten().map(|x| x * x).sum::<f32>()).sqrt();
284
285 let similarity = dot_product / (l_norm * r_norm);
286 Ok(Arc::new(Float32Array::from(vec![similarity])) as ArrayRef)
287 }
288 _ => Err(DataFusionError::NotImplemented(
289 "Unsupported data type for cosine similarity".to_owned(),
290 )),
291 }
292 }
293 }
294 }
295}
296
297impl PhysicalOperator for VectorizedOperator {
298 fn schema(&self) -> &Arc<Schema> {
299 self.base.schema()
300 }
301
302 fn execute(&self, input: Vec<RecordBatch>) -> Result<Vec<RecordBatch>> {
303 let mut output_batches = Vec::with_capacity(input.len());
304
305 for batch in input {
306 let arrays: Result<Vec<ArrayRef>> = self
308 .input_columns
309 .iter()
310 .map(|col| {
311 batch
312 .column_by_name(col)
313 .ok_or_else(|| DataFusionError::Plan(format!("Column {col} not found")))
314 .map(std::clone::Clone::clone)
315 })
316 .collect();
317
318 let arrays = arrays?;
319 if arrays.len() != 2 {
320 return Err(DataFusionError::Plan(
321 "Vector operations require exactly two input arrays".to_owned(),
322 ));
323 }
324
325 let result = self.execute_vector_op(&arrays[0], &arrays[1])?;
327
328 let mut columns = batch.columns().to_vec();
330 columns.push(result);
331
332 let output_batch = RecordBatch::try_new(self.base.schema().clone(), columns)
333 .map_err(|e| DataFusionError::Internal(e.to_string()))?;
334 output_batches.push(output_batch);
335 }
336
337 Ok(output_batches)
338 }
339
340 fn children(&self) -> Vec<Arc<dyn PhysicalOperator>> {
341 self.base.children()
342 }
343}