Skip to main content

hyprstream_metrics/metrics/
storage.rs

1use crate::aggregation::{AggregateFunction, AggregateResult, GroupBy, TimeWindow};
2use crate::metrics::{create_record_batch, encode_record_batch, MetricRecord};
3use crate::storage::StorageBackend;
4use duckdb::arrow::array::{Float64Array, Int64Array, StringArray};
5use duckdb::arrow::datatypes::{DataType, Field, Schema};
6use std::collections::HashMap;
7use std::sync::Arc;
8use tonic::Status;
9
10/// Batch-level aggregation state for efficient updates
11#[derive(Debug, Clone)]
12pub struct BatchAggregation {
13    /// The metric ID this aggregation belongs to
14    pub metric_id: String,
15    /// Start of the time window
16    pub window_start: i64,
17    /// End of the time window
18    pub window_end: i64,
19    /// Running sum within the window
20    pub running_sum: f64,
21    /// Running count within the window
22    pub running_count: i64,
23    /// Minimum value in the window
24    pub min_value: f64,
25    /// Maximum value in the window
26    pub max_value: f64,
27    /// Schema for the aggregation
28    pub schema: Arc<Schema>,
29    /// Column to aggregate
30    pub value_column: String,
31    /// Grouping specification
32    pub group_by: GroupBy,
33    /// Time window specification
34    pub window: Option<TimeWindow>,
35}
36
37impl BatchAggregation {
38    pub fn new_window(
39        metric_id: String,
40        window_start: i64,
41        window_end: i64,
42        schema: Arc<Schema>,
43        value_column: String,
44        group_by: GroupBy,
45        window: Option<TimeWindow>,
46    ) -> Self {
47        Self {
48            metric_id,
49            window_start,
50            window_end,
51            running_sum: 0.0,
52            running_count: 0,
53            min_value: f64::INFINITY,
54            max_value: f64::NEG_INFINITY,
55            schema,
56            value_column,
57            group_by,
58            window,
59        }
60    }
61
62    pub fn new_from_metric(
63        metric_id: String,
64        window_start: i64,
65        window_end: i64,
66        window: TimeWindow,
67    ) -> Self {
68        let schema = Arc::new(Schema::new(vec![
69            Field::new("metric", DataType::Utf8, false),
70            Field::new("value", DataType::Float64, false),
71            Field::new("timestamp", DataType::Int64, false),
72        ]));
73        let group_by = GroupBy {
74            columns: vec!["metric".to_owned()],
75            time_column: Some("timestamp".to_owned()),
76        };
77        Self {
78            metric_id,
79            window_start,
80            window_end,
81            running_sum: 0.0,
82            running_count: 0,
83            min_value: f64::INFINITY,
84            max_value: f64::NEG_INFINITY,
85            schema,
86            value_column: "value".to_owned(),
87            group_by,
88            window: Some(window),
89        }
90    }
91
92    pub fn new(
93        schema: Arc<Schema>,
94        value_column: String,
95        group_by: GroupBy,
96        window: Option<TimeWindow>,
97    ) -> Self {
98        Self {
99            metric_id: String::new(),
100            window_start: 0,
101            window_end: 0,
102            running_sum: 0.0,
103            running_count: 0,
104            min_value: f64::INFINITY,
105            max_value: f64::NEG_INFINITY,
106            schema,
107            value_column,
108            group_by,
109            window,
110        }
111    }
112}
113
114/// Storage operations specific to metrics data.
115#[async_trait::async_trait]
116pub trait MetricsStorage: Send + Sync + 'static {
117    /// Get the underlying storage backend
118    fn backend(&self) -> &dyn StorageBackend;
119
120    /// Initialize metrics storage (create schema-v2 tables, migrate pre-v2 `metrics`).
121    async fn init(&self) -> Result<(), Status> {
122        crate::metrics::create_schema_v2_tables(self.backend()).await
123    }
124
125    /// Insert metrics into storage
126    async fn insert_metrics(&self, metrics: Vec<MetricRecord>) -> Result<(), Status> {
127        let batch = create_record_batch(&metrics)?;
128        self.backend().insert_into_table("metrics", batch).await
129    }
130
131    /// Query metrics from storage
132    async fn query_metrics(&self, from_timestamp: i64) -> Result<Vec<MetricRecord>, Status> {
133        let sql = Self::generate_metric_query_sql(from_timestamp);
134        let handle = self.backend().prepare_sql(&sql).await?;
135        let batch = self.backend().query_sql(&handle).await?;
136        encode_record_batch(&batch)
137    }
138
139    /// Aggregate metrics using the specified function and grouping
140    async fn aggregate_metrics(
141        &self,
142        function: AggregateFunction,
143        group_by: &GroupBy,
144        from_timestamp: i64,
145        to_timestamp: Option<i64>,
146    ) -> Result<Vec<AggregateResult>, Status> {
147        let sql =
148            Self::generate_metric_aggregation_sql(function, group_by, from_timestamp, to_timestamp);
149        let handle = self.backend().prepare_sql(&sql).await?;
150        let batch = self.backend().query_sql(&handle).await?;
151
152        let mut results = Vec::with_capacity(batch.num_rows());
153
154        // Extract column arrays
155        let value_col = batch
156            .column_by_name("value")
157            .and_then(|col| col.as_any().downcast_ref::<Float64Array>())
158            .ok_or_else(|| Status::internal("Invalid value column"))?;
159
160        // Get group by columns
161        let mut group_cols = Vec::new();
162        for col_name in &group_by.columns {
163            let col = batch
164                .column_by_name(col_name)
165                .ok_or_else(|| Status::internal(format!("Missing group by column: {col_name}")))?;
166            group_cols.push(col);
167        }
168
169        // Build results
170        for row in 0..batch.num_rows() {
171            let mut group_values = HashMap::new();
172            for (col_name, col) in group_by.columns.iter().zip(&group_cols) {
173                let value = if let Some(str_col) = col.as_any().downcast_ref::<StringArray>() {
174                    str_col.value(row).to_owned()
175                } else if let Some(int_col) = col.as_any().downcast_ref::<Int64Array>() {
176                    int_col.value(row).to_string()
177                } else if let Some(float_col) = col.as_any().downcast_ref::<Float64Array>() {
178                    float_col.value(row).to_string()
179                } else {
180                    return Err(Status::internal(format!(
181                        "Unsupported group by column type: {col_name}"
182                    )));
183                };
184                group_values.insert(col_name.clone(), value);
185            }
186
187            // Extract timestamp if it's part of the group by
188            let timestamp = if let Some(time_col) = &group_by.time_column {
189                if let Some(col) = batch.column_by_name(time_col) {
190                    col.as_any()
191                        .downcast_ref::<Int64Array>()
192                        .map(|int_col| int_col.value(row))
193                } else {
194                    None
195                }
196            } else {
197                None
198            };
199
200            results.push(AggregateResult {
201                value: value_col.value(row),
202                group_values,
203                timestamp,
204            });
205        }
206
207        Ok(results)
208    }
209
210    /// Get the standard schema for metric records
211    fn get_metrics_schema() -> Schema {
212        crate::metrics::get_metrics_schema()
213    }
214
215    /// Generate SQL for inserting metric records
216    fn generate_metric_insert_sql() -> String {
217        "INSERT INTO metrics (metric_id, timestamp, value_running_window_sum, value_running_window_avg, value_running_window_count, labels, tenant_id) VALUES (?, ?, ?, ?, ?, ?, ?)".to_owned()
218    }
219
220    /// Generate SQL for querying metrics
221    fn generate_metric_query_sql(from_timestamp: i64) -> String {
222        format!("SELECT * FROM metrics WHERE timestamp >= {from_timestamp} ORDER BY timestamp ASC")
223    }
224
225    /// Generate SQL for aggregating metrics
226    fn generate_metric_aggregation_sql(
227        function: AggregateFunction,
228        group_by: &GroupBy,
229        from_timestamp: i64,
230        to_timestamp: Option<i64>,
231    ) -> String {
232        let mut sql = format!(
233            "SELECT {}, CAST({}({}) AS DOUBLE) as value",
234            group_by.columns.join(", "),
235            function,
236            "value_running_window_avg" // Use avg for now
237        );
238
239        sql.push_str(" FROM metrics");
240        sql.push_str(&format!(" WHERE timestamp >= {from_timestamp}"));
241
242        if let Some(to) = to_timestamp {
243            sql.push_str(&format!(" AND timestamp <= {to}"));
244        }
245
246        sql.push_str(&format!(" GROUP BY {}", group_by.columns.join(", ")));
247        sql
248    }
249}
250
251/// A wrapper around StorageBackend that implements MetricsStorage
252pub struct MetricsStorageImpl<B: StorageBackend> {
253    backend: B,
254}
255
256impl<B: StorageBackend> MetricsStorageImpl<B> {
257    pub fn new(backend: B) -> Self {
258        Self { backend }
259    }
260}
261
262#[async_trait::async_trait]
263impl<B: StorageBackend> MetricsStorage for MetricsStorageImpl<B> {
264    fn backend(&self) -> &dyn StorageBackend {
265        &self.backend
266    }
267}