Skip to main content

hyprstream_metrics_api/metrics/
mod.rs

1// tonic::Status is the idiomatic gRPC error type - boxing would break API
2#![allow(clippy::result_large_err)]
3
4pub mod calibration;
5pub mod decision_batch;
6pub mod producer;
7
8use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, RecordBatch, StringArray};
9use arrow::datatypes::{DataType, Field, Schema};
10use serde::{Deserialize, Serialize};
11use std::collections::BTreeMap;
12use std::sync::Arc;
13use tonic::Status;
14
15/// Primary metrics table name.
16pub const METRICS_TABLE: &str = "metrics";
17
18/// Version of the metrics table schema emitted by [`get_metrics_schema`].
19/// v1: five running-window columns; v2: adds `labels` and `tenant_id`.
20pub const METRICS_SCHEMA_VERSION: u32 = 2;
21
22/// A single metric record with running window calculations.
23///
24/// Schema v2 adds `labels` and `tenant_id`; both default to empty/`None` so
25/// v1 producers and serialized records keep working unchanged.
26#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
27pub struct MetricRecord {
28    /// Unique identifier for the metric
29    pub metric_id: String,
30    /// Unix timestamp in milliseconds
31    pub timestamp: i64,
32    /// Running sum within the window
33    pub value_running_window_sum: f64,
34    /// Running average within the window
35    pub value_running_window_avg: f64,
36    /// Running count within the window
37    pub value_running_window_count: i64,
38    /// Metric labels; stored as a JSON object in the nullable `labels` column
39    #[serde(default)]
40    pub labels: BTreeMap<String, String>,
41    /// Tenant the row belongs to; `None` = node-global (no tenant)
42    #[serde(default)]
43    pub tenant_id: Option<String>,
44}
45
46/// Serialize labels to their canonical storage form: a JSON object with
47/// sorted keys. Empty label sets map to `None` (SQL NULL on write).
48pub fn encode_labels(labels: &BTreeMap<String, String>) -> Option<String> {
49    if labels.is_empty() {
50        None
51    } else {
52        serde_json::to_string(labels).ok()
53    }
54}
55
56/// Parse the JSON storage form of a label map.
57pub fn decode_labels(json: &str) -> Result<BTreeMap<String, String>, Status> {
58    serde_json::from_str(json).map_err(|e| Status::internal(format!("Invalid labels JSON: {e}")))
59}
60
61impl MetricRecord {
62    pub fn try_from_record_batch(batch: &RecordBatch) -> Result<Vec<Self>, Status> {
63        let metric_ids = batch
64            .column_by_name("metric_id")
65            .and_then(|col| col.as_any().downcast_ref::<StringArray>())
66            .ok_or_else(|| Status::internal("Invalid metric_id column"))?;
67
68        let timestamps = batch
69            .column_by_name("timestamp")
70            .and_then(|col| col.as_any().downcast_ref::<Int64Array>())
71            .ok_or_else(|| Status::internal("Invalid timestamp column"))?;
72
73        let sums = batch
74            .column_by_name("value_running_window_sum")
75            .and_then(|col| col.as_any().downcast_ref::<Float64Array>())
76            .ok_or_else(|| Status::internal("Invalid value_running_window_sum column"))?;
77
78        let avgs = batch
79            .column_by_name("value_running_window_avg")
80            .and_then(|col| col.as_any().downcast_ref::<Float64Array>())
81            .ok_or_else(|| Status::internal("Invalid value_running_window_avg column"))?;
82
83        let counts = batch
84            .column_by_name("value_running_window_count")
85            .and_then(|col| col.as_any().downcast_ref::<Int64Array>())
86            .ok_or_else(|| Status::internal("Invalid value_running_window_count column"))?;
87
88        // Schema v2 columns are optional so pre-v2 batches still decode.
89        let labels_col = batch
90            .column_by_name("labels")
91            .and_then(|col| col.as_any().downcast_ref::<StringArray>());
92        let tenant_col = batch
93            .column_by_name("tenant_id")
94            .and_then(|col| col.as_any().downcast_ref::<StringArray>());
95
96        let mut metrics = Vec::with_capacity(batch.num_rows());
97        for i in 0..batch.num_rows() {
98            let labels = match labels_col {
99                Some(col) if !col.is_null(i) && !col.value(i).is_empty() => {
100                    decode_labels(col.value(i))?
101                }
102                _ => BTreeMap::new(),
103            };
104            let tenant_id = tenant_col.and_then(|col| {
105                if col.is_null(i) || col.value(i).is_empty() {
106                    None
107                } else {
108                    Some(col.value(i).to_owned())
109                }
110            });
111            metrics.push(MetricRecord {
112                metric_id: metric_ids.value(i).to_owned(),
113                timestamp: timestamps.value(i),
114                value_running_window_sum: sums.value(i),
115                value_running_window_avg: avgs.value(i),
116                value_running_window_count: counts.value(i),
117                labels,
118                tenant_id,
119            });
120        }
121
122        Ok(metrics)
123    }
124}
125
126/// Gets the schema for metric records in Arrow format (schema v2).
127pub fn get_metrics_schema() -> Schema {
128    Schema::new(vec![
129        Field::new("metric_id", DataType::Utf8, false),
130        Field::new("timestamp", DataType::Int64, false),
131        Field::new("value_running_window_sum", DataType::Float64, false),
132        Field::new("value_running_window_avg", DataType::Float64, false),
133        Field::new("value_running_window_count", DataType::Int64, false),
134        // v2: label map as canonical JSON (see encode_labels) and tenant key.
135        // Nullable so pre-v2 rows and writers stay valid; DuckDB stores both
136        // as VARCHAR, queryable with the built-in JSON functions.
137        Field::new("labels", DataType::Utf8, true),
138        Field::new("tenant_id", DataType::Utf8, true),
139    ])
140}
141
142/// Creates a RecordBatch from a vector of MetricRecords.
143pub fn create_record_batch(metrics: &[MetricRecord]) -> Result<RecordBatch, Status> {
144    let schema = get_metrics_schema();
145
146    let metric_ids = StringArray::from_iter_values(metrics.iter().map(|m| m.metric_id.as_str()));
147    let timestamps = Int64Array::from_iter_values(metrics.iter().map(|m| m.timestamp));
148    let sums = Float64Array::from_iter_values(metrics.iter().map(|m| m.value_running_window_sum));
149    let avgs = Float64Array::from_iter_values(metrics.iter().map(|m| m.value_running_window_avg));
150    let counts = Int64Array::from_iter_values(metrics.iter().map(|m| m.value_running_window_count));
151    let labels = StringArray::from_iter(metrics.iter().map(|m| encode_labels(&m.labels)));
152    let tenant_ids = StringArray::from_iter(metrics.iter().map(|m| m.tenant_id.as_deref()));
153
154    let arrays: Vec<ArrayRef> = vec![
155        Arc::new(metric_ids),
156        Arc::new(timestamps),
157        Arc::new(sums),
158        Arc::new(avgs),
159        Arc::new(counts),
160        Arc::new(labels),
161        Arc::new(tenant_ids),
162    ];
163
164    RecordBatch::try_new(Arc::new(schema), arrays)
165        .map_err(|e| Status::internal(format!("Failed to create record batch: {e}")))
166}
167
168/// Encodes a RecordBatch into a vector of MetricRecords.
169pub fn encode_record_batch(batch: &RecordBatch) -> Result<Vec<MetricRecord>, Status> {
170    MetricRecord::try_from_record_batch(batch)
171}
172
173/// Creates all schema-v2 tables and migrates pre-v2 `metrics` tables in place.
174///
175/// `metrics` is created with the v2 schema; `calibration_params` and
176/// `decision_batches` are created fresh. On databases initialized before v2,
177/// the new `metrics` columns are added with `ALTER TABLE ... IF NOT EXISTS`
178/// (a no-op on fresh v2 tables); existing rows read back with empty
179/// labels/no tenant.
180pub async fn create_schema_v2_tables(
181    backend: &dyn crate::storage::StorageBackend,
182) -> Result<(), Status> {
183    backend
184        .create_table(METRICS_TABLE, &get_metrics_schema())
185        .await?;
186    backend
187        .create_table(
188            calibration::CALIBRATION_TABLE,
189            &calibration::calibration_params_schema(),
190        )
191        .await?;
192    backend
193        .create_table(
194            decision_batch::DECISION_BATCH_TABLE,
195            &decision_batch::decision_batch_schema(),
196        )
197        .await?;
198    for column in ["labels", "tenant_id"] {
199        backend
200            .execute_ddl(&format!(
201                "ALTER TABLE {METRICS_TABLE} ADD COLUMN IF NOT EXISTS {column} VARCHAR"
202            ))
203            .await?;
204    }
205    Ok(())
206}
207
208#[cfg(test)]
209mod tests {
210    use super::*;
211
212    fn labeled_record() -> MetricRecord {
213        MetricRecord {
214            metric_id: "decision.latency_ms".to_owned(),
215            timestamp: 1_758_000_000_000,
216            value_running_window_sum: 12.5,
217            value_running_window_avg: 12.5,
218            value_running_window_count: 1,
219            labels: BTreeMap::from([
220                ("head".to_owned(), "routing".to_owned()),
221                ("primitive".to_owned(), "choice".to_owned()),
222            ]),
223            tenant_id: Some("tenant-a".to_owned()),
224        }
225    }
226
227    #[test]
228    fn test_metrics_schema_v2_columns() -> Result<(), Status> {
229        let schema = get_metrics_schema();
230        assert_eq!(schema.fields().len(), 7);
231        let labels = schema
232            .field_with_name("labels")
233            .map_err(|e| Status::internal(e.to_string()))?;
234        assert!(labels.is_nullable());
235        assert_eq!(labels.data_type(), &DataType::Utf8);
236        let tenant = schema
237            .field_with_name("tenant_id")
238            .map_err(|e| Status::internal(e.to_string()))?;
239        assert!(tenant.is_nullable());
240        assert_eq!(tenant.data_type(), &DataType::Utf8);
241        Ok(())
242    }
243
244    #[test]
245    fn test_record_batch_round_trip_with_labels_and_tenant() -> Result<(), Status> {
246        let records = vec![
247            labeled_record(),
248            MetricRecord {
249                metric_id: "requests.total".to_owned(),
250                timestamp: 1_758_000_001_000,
251                value_running_window_sum: 3.0,
252                value_running_window_avg: 1.5,
253                value_running_window_count: 2,
254                labels: BTreeMap::new(),
255                tenant_id: None,
256            },
257        ];
258        let batch = create_record_batch(&records)?;
259        assert_eq!(batch.num_rows(), 2);
260        assert_eq!(batch.num_columns(), 7);
261
262        // Labels land as NULL (not empty string) when unset.
263        let labels_col = batch
264            .column_by_name("labels")
265            .and_then(|c| c.as_any().downcast_ref::<StringArray>())
266            .ok_or_else(|| Status::internal("labels column missing"))?;
267        assert!(labels_col.is_null(1));
268        assert!(!labels_col.is_null(0));
269        assert_eq!(
270            labels_col.value(0),
271            r#"{"head":"routing","primitive":"choice"}"#
272        );
273
274        let decoded = MetricRecord::try_from_record_batch(&batch)?;
275        assert_eq!(decoded, records);
276
277        let decoded_via_encode = encode_record_batch(&batch)?;
278        assert_eq!(decoded_via_encode, records);
279        Ok(())
280    }
281
282    #[test]
283    fn test_pre_v2_batch_decodes_with_defaults() -> Result<(), Status> {
284        let v1_schema = Arc::new(Schema::new(vec![
285            Field::new("metric_id", DataType::Utf8, false),
286            Field::new("timestamp", DataType::Int64, false),
287            Field::new("value_running_window_sum", DataType::Float64, false),
288            Field::new("value_running_window_avg", DataType::Float64, false),
289            Field::new("value_running_window_count", DataType::Int64, false),
290        ]));
291        let v1_batch = RecordBatch::try_new(
292            v1_schema,
293            vec![
294                Arc::new(StringArray::from(vec!["cpu.usage"])) as ArrayRef,
295                Arc::new(Int64Array::from(vec![42])) as ArrayRef,
296                Arc::new(Float64Array::from(vec![1.0])) as ArrayRef,
297                Arc::new(Float64Array::from(vec![1.0])) as ArrayRef,
298                Arc::new(Int64Array::from(vec![1])) as ArrayRef,
299            ],
300        )
301        .map_err(|e| Status::internal(e.to_string()))?;
302
303        let decoded = MetricRecord::try_from_record_batch(&v1_batch)?;
304        assert_eq!(decoded.len(), 1);
305        assert!(decoded[0].labels.is_empty());
306        assert_eq!(decoded[0].tenant_id, None);
307        assert_eq!(decoded[0].metric_id, "cpu.usage");
308        Ok(())
309    }
310
311    #[test]
312    fn test_labels_json_canonical_order() {
313        let labels = BTreeMap::from([
314            ("zeta".to_owned(), "1".to_owned()),
315            ("alpha".to_owned(), "2".to_owned()),
316        ]);
317        assert_eq!(
318            encode_labels(&labels).as_deref(),
319            Some(r#"{"alpha":"2","zeta":"1"}"#)
320        );
321        assert_eq!(encode_labels(&BTreeMap::new()), None);
322    }
323
324    #[test]
325    fn test_default_record_is_unlabeled_and_untenanted() {
326        let record = MetricRecord::default();
327        assert!(record.labels.is_empty());
328        assert!(record.tenant_id.is_none());
329    }
330}