hyprstream_metrics/metrics/
storage.rs1use crate::aggregation::{AggregateFunction, AggregateResult, GroupBy, TimeWindow};
2use crate::metrics::{create_record_batch, encode_record_batch, MetricRecord};
3use crate::storage::StorageBackend;
4use duckdb::arrow::array::{Float64Array, Int64Array, StringArray};
5use duckdb::arrow::datatypes::{DataType, Field, Schema};
6use std::collections::HashMap;
7use std::sync::Arc;
8use tonic::Status;
9
10#[derive(Debug, Clone)]
12pub struct BatchAggregation {
13 pub metric_id: String,
15 pub window_start: i64,
17 pub window_end: i64,
19 pub running_sum: f64,
21 pub running_count: i64,
23 pub min_value: f64,
25 pub max_value: f64,
27 pub schema: Arc<Schema>,
29 pub value_column: String,
31 pub group_by: GroupBy,
33 pub window: Option<TimeWindow>,
35}
36
37impl BatchAggregation {
38 pub fn new_window(
39 metric_id: String,
40 window_start: i64,
41 window_end: i64,
42 schema: Arc<Schema>,
43 value_column: String,
44 group_by: GroupBy,
45 window: Option<TimeWindow>,
46 ) -> Self {
47 Self {
48 metric_id,
49 window_start,
50 window_end,
51 running_sum: 0.0,
52 running_count: 0,
53 min_value: f64::INFINITY,
54 max_value: f64::NEG_INFINITY,
55 schema,
56 value_column,
57 group_by,
58 window,
59 }
60 }
61
62 pub fn new_from_metric(
63 metric_id: String,
64 window_start: i64,
65 window_end: i64,
66 window: TimeWindow,
67 ) -> Self {
68 let schema = Arc::new(Schema::new(vec![
69 Field::new("metric", DataType::Utf8, false),
70 Field::new("value", DataType::Float64, false),
71 Field::new("timestamp", DataType::Int64, false),
72 ]));
73 let group_by = GroupBy {
74 columns: vec!["metric".to_owned()],
75 time_column: Some("timestamp".to_owned()),
76 };
77 Self {
78 metric_id,
79 window_start,
80 window_end,
81 running_sum: 0.0,
82 running_count: 0,
83 min_value: f64::INFINITY,
84 max_value: f64::NEG_INFINITY,
85 schema,
86 value_column: "value".to_owned(),
87 group_by,
88 window: Some(window),
89 }
90 }
91
92 pub fn new(
93 schema: Arc<Schema>,
94 value_column: String,
95 group_by: GroupBy,
96 window: Option<TimeWindow>,
97 ) -> Self {
98 Self {
99 metric_id: String::new(),
100 window_start: 0,
101 window_end: 0,
102 running_sum: 0.0,
103 running_count: 0,
104 min_value: f64::INFINITY,
105 max_value: f64::NEG_INFINITY,
106 schema,
107 value_column,
108 group_by,
109 window,
110 }
111 }
112}
113
114#[async_trait::async_trait]
116pub trait MetricsStorage: Send + Sync + 'static {
117 fn backend(&self) -> &dyn StorageBackend;
119
120 async fn init(&self) -> Result<(), Status> {
122 crate::metrics::create_schema_v2_tables(self.backend()).await
123 }
124
125 async fn insert_metrics(&self, metrics: Vec<MetricRecord>) -> Result<(), Status> {
127 let batch = create_record_batch(&metrics)?;
128 self.backend().insert_into_table("metrics", batch).await
129 }
130
131 async fn query_metrics(&self, from_timestamp: i64) -> Result<Vec<MetricRecord>, Status> {
133 let sql = Self::generate_metric_query_sql(from_timestamp);
134 let handle = self.backend().prepare_sql(&sql).await?;
135 let batch = self.backend().query_sql(&handle).await?;
136 encode_record_batch(&batch)
137 }
138
139 async fn aggregate_metrics(
141 &self,
142 function: AggregateFunction,
143 group_by: &GroupBy,
144 from_timestamp: i64,
145 to_timestamp: Option<i64>,
146 ) -> Result<Vec<AggregateResult>, Status> {
147 let sql =
148 Self::generate_metric_aggregation_sql(function, group_by, from_timestamp, to_timestamp);
149 let handle = self.backend().prepare_sql(&sql).await?;
150 let batch = self.backend().query_sql(&handle).await?;
151
152 let mut results = Vec::with_capacity(batch.num_rows());
153
154 let value_col = batch
156 .column_by_name("value")
157 .and_then(|col| col.as_any().downcast_ref::<Float64Array>())
158 .ok_or_else(|| Status::internal("Invalid value column"))?;
159
160 let mut group_cols = Vec::new();
162 for col_name in &group_by.columns {
163 let col = batch
164 .column_by_name(col_name)
165 .ok_or_else(|| Status::internal(format!("Missing group by column: {col_name}")))?;
166 group_cols.push(col);
167 }
168
169 for row in 0..batch.num_rows() {
171 let mut group_values = HashMap::new();
172 for (col_name, col) in group_by.columns.iter().zip(&group_cols) {
173 let value = if let Some(str_col) = col.as_any().downcast_ref::<StringArray>() {
174 str_col.value(row).to_owned()
175 } else if let Some(int_col) = col.as_any().downcast_ref::<Int64Array>() {
176 int_col.value(row).to_string()
177 } else if let Some(float_col) = col.as_any().downcast_ref::<Float64Array>() {
178 float_col.value(row).to_string()
179 } else {
180 return Err(Status::internal(format!(
181 "Unsupported group by column type: {col_name}"
182 )));
183 };
184 group_values.insert(col_name.clone(), value);
185 }
186
187 let timestamp = if let Some(time_col) = &group_by.time_column {
189 if let Some(col) = batch.column_by_name(time_col) {
190 col.as_any()
191 .downcast_ref::<Int64Array>()
192 .map(|int_col| int_col.value(row))
193 } else {
194 None
195 }
196 } else {
197 None
198 };
199
200 results.push(AggregateResult {
201 value: value_col.value(row),
202 group_values,
203 timestamp,
204 });
205 }
206
207 Ok(results)
208 }
209
210 fn get_metrics_schema() -> Schema {
212 crate::metrics::get_metrics_schema()
213 }
214
215 fn generate_metric_insert_sql() -> String {
217 "INSERT INTO metrics (metric_id, timestamp, value_running_window_sum, value_running_window_avg, value_running_window_count, labels, tenant_id) VALUES (?, ?, ?, ?, ?, ?, ?)".to_owned()
218 }
219
220 fn generate_metric_query_sql(from_timestamp: i64) -> String {
222 format!("SELECT * FROM metrics WHERE timestamp >= {from_timestamp} ORDER BY timestamp ASC")
223 }
224
225 fn generate_metric_aggregation_sql(
227 function: AggregateFunction,
228 group_by: &GroupBy,
229 from_timestamp: i64,
230 to_timestamp: Option<i64>,
231 ) -> String {
232 let mut sql = format!(
233 "SELECT {}, CAST({}({}) AS DOUBLE) as value",
234 group_by.columns.join(", "),
235 function,
236 "value_running_window_avg" );
238
239 sql.push_str(" FROM metrics");
240 sql.push_str(&format!(" WHERE timestamp >= {from_timestamp}"));
241
242 if let Some(to) = to_timestamp {
243 sql.push_str(&format!(" AND timestamp <= {to}"));
244 }
245
246 sql.push_str(&format!(" GROUP BY {}", group_by.columns.join(", ")));
247 sql
248 }
249}
250
251pub struct MetricsStorageImpl<B: StorageBackend> {
253 backend: B,
254}
255
256impl<B: StorageBackend> MetricsStorageImpl<B> {
257 pub fn new(backend: B) -> Self {
258 Self { backend }
259 }
260}
261
262#[async_trait::async_trait]
263impl<B: StorageBackend> MetricsStorage for MetricsStorageImpl<B> {
264 fn backend(&self) -> &dyn StorageBackend {
265 &self.backend
266 }
267}