hyprstream_metrics_api/metrics/
mod.rs1#![allow(clippy::result_large_err)]
3
4pub mod calibration;
5pub mod decision_batch;
6pub mod producer;
7
8use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, RecordBatch, StringArray};
9use arrow::datatypes::{DataType, Field, Schema};
10use serde::{Deserialize, Serialize};
11use std::collections::BTreeMap;
12use std::sync::Arc;
13use tonic::Status;
14
15pub const METRICS_TABLE: &str = "metrics";
17
18pub const METRICS_SCHEMA_VERSION: u32 = 2;
21
22#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
27pub struct MetricRecord {
28 pub metric_id: String,
30 pub timestamp: i64,
32 pub value_running_window_sum: f64,
34 pub value_running_window_avg: f64,
36 pub value_running_window_count: i64,
38 #[serde(default)]
40 pub labels: BTreeMap<String, String>,
41 #[serde(default)]
43 pub tenant_id: Option<String>,
44}
45
46pub fn encode_labels(labels: &BTreeMap<String, String>) -> Option<String> {
49 if labels.is_empty() {
50 None
51 } else {
52 serde_json::to_string(labels).ok()
53 }
54}
55
56pub fn decode_labels(json: &str) -> Result<BTreeMap<String, String>, Status> {
58 serde_json::from_str(json).map_err(|e| Status::internal(format!("Invalid labels JSON: {e}")))
59}
60
61impl MetricRecord {
62 pub fn try_from_record_batch(batch: &RecordBatch) -> Result<Vec<Self>, Status> {
63 let metric_ids = batch
64 .column_by_name("metric_id")
65 .and_then(|col| col.as_any().downcast_ref::<StringArray>())
66 .ok_or_else(|| Status::internal("Invalid metric_id column"))?;
67
68 let timestamps = batch
69 .column_by_name("timestamp")
70 .and_then(|col| col.as_any().downcast_ref::<Int64Array>())
71 .ok_or_else(|| Status::internal("Invalid timestamp column"))?;
72
73 let sums = batch
74 .column_by_name("value_running_window_sum")
75 .and_then(|col| col.as_any().downcast_ref::<Float64Array>())
76 .ok_or_else(|| Status::internal("Invalid value_running_window_sum column"))?;
77
78 let avgs = batch
79 .column_by_name("value_running_window_avg")
80 .and_then(|col| col.as_any().downcast_ref::<Float64Array>())
81 .ok_or_else(|| Status::internal("Invalid value_running_window_avg column"))?;
82
83 let counts = batch
84 .column_by_name("value_running_window_count")
85 .and_then(|col| col.as_any().downcast_ref::<Int64Array>())
86 .ok_or_else(|| Status::internal("Invalid value_running_window_count column"))?;
87
88 let labels_col = batch
90 .column_by_name("labels")
91 .and_then(|col| col.as_any().downcast_ref::<StringArray>());
92 let tenant_col = batch
93 .column_by_name("tenant_id")
94 .and_then(|col| col.as_any().downcast_ref::<StringArray>());
95
96 let mut metrics = Vec::with_capacity(batch.num_rows());
97 for i in 0..batch.num_rows() {
98 let labels = match labels_col {
99 Some(col) if !col.is_null(i) && !col.value(i).is_empty() => {
100 decode_labels(col.value(i))?
101 }
102 _ => BTreeMap::new(),
103 };
104 let tenant_id = tenant_col.and_then(|col| {
105 if col.is_null(i) || col.value(i).is_empty() {
106 None
107 } else {
108 Some(col.value(i).to_owned())
109 }
110 });
111 metrics.push(MetricRecord {
112 metric_id: metric_ids.value(i).to_owned(),
113 timestamp: timestamps.value(i),
114 value_running_window_sum: sums.value(i),
115 value_running_window_avg: avgs.value(i),
116 value_running_window_count: counts.value(i),
117 labels,
118 tenant_id,
119 });
120 }
121
122 Ok(metrics)
123 }
124}
125
126pub fn get_metrics_schema() -> Schema {
128 Schema::new(vec![
129 Field::new("metric_id", DataType::Utf8, false),
130 Field::new("timestamp", DataType::Int64, false),
131 Field::new("value_running_window_sum", DataType::Float64, false),
132 Field::new("value_running_window_avg", DataType::Float64, false),
133 Field::new("value_running_window_count", DataType::Int64, false),
134 Field::new("labels", DataType::Utf8, true),
138 Field::new("tenant_id", DataType::Utf8, true),
139 ])
140}
141
142pub fn create_record_batch(metrics: &[MetricRecord]) -> Result<RecordBatch, Status> {
144 let schema = get_metrics_schema();
145
146 let metric_ids = StringArray::from_iter_values(metrics.iter().map(|m| m.metric_id.as_str()));
147 let timestamps = Int64Array::from_iter_values(metrics.iter().map(|m| m.timestamp));
148 let sums = Float64Array::from_iter_values(metrics.iter().map(|m| m.value_running_window_sum));
149 let avgs = Float64Array::from_iter_values(metrics.iter().map(|m| m.value_running_window_avg));
150 let counts = Int64Array::from_iter_values(metrics.iter().map(|m| m.value_running_window_count));
151 let labels = StringArray::from_iter(metrics.iter().map(|m| encode_labels(&m.labels)));
152 let tenant_ids = StringArray::from_iter(metrics.iter().map(|m| m.tenant_id.as_deref()));
153
154 let arrays: Vec<ArrayRef> = vec![
155 Arc::new(metric_ids),
156 Arc::new(timestamps),
157 Arc::new(sums),
158 Arc::new(avgs),
159 Arc::new(counts),
160 Arc::new(labels),
161 Arc::new(tenant_ids),
162 ];
163
164 RecordBatch::try_new(Arc::new(schema), arrays)
165 .map_err(|e| Status::internal(format!("Failed to create record batch: {e}")))
166}
167
168pub fn encode_record_batch(batch: &RecordBatch) -> Result<Vec<MetricRecord>, Status> {
170 MetricRecord::try_from_record_batch(batch)
171}
172
173pub async fn create_schema_v2_tables(
181 backend: &dyn crate::storage::StorageBackend,
182) -> Result<(), Status> {
183 backend
184 .create_table(METRICS_TABLE, &get_metrics_schema())
185 .await?;
186 backend
187 .create_table(
188 calibration::CALIBRATION_TABLE,
189 &calibration::calibration_params_schema(),
190 )
191 .await?;
192 backend
193 .create_table(
194 decision_batch::DECISION_BATCH_TABLE,
195 &decision_batch::decision_batch_schema(),
196 )
197 .await?;
198 for column in ["labels", "tenant_id"] {
199 backend
200 .execute_ddl(&format!(
201 "ALTER TABLE {METRICS_TABLE} ADD COLUMN IF NOT EXISTS {column} VARCHAR"
202 ))
203 .await?;
204 }
205 Ok(())
206}
207
208#[cfg(test)]
209mod tests {
210 use super::*;
211
212 fn labeled_record() -> MetricRecord {
213 MetricRecord {
214 metric_id: "decision.latency_ms".to_owned(),
215 timestamp: 1_758_000_000_000,
216 value_running_window_sum: 12.5,
217 value_running_window_avg: 12.5,
218 value_running_window_count: 1,
219 labels: BTreeMap::from([
220 ("head".to_owned(), "routing".to_owned()),
221 ("primitive".to_owned(), "choice".to_owned()),
222 ]),
223 tenant_id: Some("tenant-a".to_owned()),
224 }
225 }
226
227 #[test]
228 fn test_metrics_schema_v2_columns() -> Result<(), Status> {
229 let schema = get_metrics_schema();
230 assert_eq!(schema.fields().len(), 7);
231 let labels = schema
232 .field_with_name("labels")
233 .map_err(|e| Status::internal(e.to_string()))?;
234 assert!(labels.is_nullable());
235 assert_eq!(labels.data_type(), &DataType::Utf8);
236 let tenant = schema
237 .field_with_name("tenant_id")
238 .map_err(|e| Status::internal(e.to_string()))?;
239 assert!(tenant.is_nullable());
240 assert_eq!(tenant.data_type(), &DataType::Utf8);
241 Ok(())
242 }
243
244 #[test]
245 fn test_record_batch_round_trip_with_labels_and_tenant() -> Result<(), Status> {
246 let records = vec![
247 labeled_record(),
248 MetricRecord {
249 metric_id: "requests.total".to_owned(),
250 timestamp: 1_758_000_001_000,
251 value_running_window_sum: 3.0,
252 value_running_window_avg: 1.5,
253 value_running_window_count: 2,
254 labels: BTreeMap::new(),
255 tenant_id: None,
256 },
257 ];
258 let batch = create_record_batch(&records)?;
259 assert_eq!(batch.num_rows(), 2);
260 assert_eq!(batch.num_columns(), 7);
261
262 let labels_col = batch
264 .column_by_name("labels")
265 .and_then(|c| c.as_any().downcast_ref::<StringArray>())
266 .ok_or_else(|| Status::internal("labels column missing"))?;
267 assert!(labels_col.is_null(1));
268 assert!(!labels_col.is_null(0));
269 assert_eq!(
270 labels_col.value(0),
271 r#"{"head":"routing","primitive":"choice"}"#
272 );
273
274 let decoded = MetricRecord::try_from_record_batch(&batch)?;
275 assert_eq!(decoded, records);
276
277 let decoded_via_encode = encode_record_batch(&batch)?;
278 assert_eq!(decoded_via_encode, records);
279 Ok(())
280 }
281
282 #[test]
283 fn test_pre_v2_batch_decodes_with_defaults() -> Result<(), Status> {
284 let v1_schema = Arc::new(Schema::new(vec![
285 Field::new("metric_id", DataType::Utf8, false),
286 Field::new("timestamp", DataType::Int64, false),
287 Field::new("value_running_window_sum", DataType::Float64, false),
288 Field::new("value_running_window_avg", DataType::Float64, false),
289 Field::new("value_running_window_count", DataType::Int64, false),
290 ]));
291 let v1_batch = RecordBatch::try_new(
292 v1_schema,
293 vec![
294 Arc::new(StringArray::from(vec!["cpu.usage"])) as ArrayRef,
295 Arc::new(Int64Array::from(vec![42])) as ArrayRef,
296 Arc::new(Float64Array::from(vec![1.0])) as ArrayRef,
297 Arc::new(Float64Array::from(vec![1.0])) as ArrayRef,
298 Arc::new(Int64Array::from(vec![1])) as ArrayRef,
299 ],
300 )
301 .map_err(|e| Status::internal(e.to_string()))?;
302
303 let decoded = MetricRecord::try_from_record_batch(&v1_batch)?;
304 assert_eq!(decoded.len(), 1);
305 assert!(decoded[0].labels.is_empty());
306 assert_eq!(decoded[0].tenant_id, None);
307 assert_eq!(decoded[0].metric_id, "cpu.usage");
308 Ok(())
309 }
310
311 #[test]
312 fn test_labels_json_canonical_order() {
313 let labels = BTreeMap::from([
314 ("zeta".to_owned(), "1".to_owned()),
315 ("alpha".to_owned(), "2".to_owned()),
316 ]);
317 assert_eq!(
318 encode_labels(&labels).as_deref(),
319 Some(r#"{"alpha":"2","zeta":"1"}"#)
320 );
321 assert_eq!(encode_labels(&BTreeMap::new()), None);
322 }
323
324 #[test]
325 fn test_default_record_is_unlabeled_and_untenanted() {
326 let record = MetricRecord::default();
327 assert!(record.labels.is_empty());
328 assert!(record.tenant_id.is_none());
329 }
330}