Skip to main content

hyprstream_metrics_api/metrics/
decision_batch.rs

1//! Arrow-native decision-batch storage (System One P0.3).
2//!
3//! Decision model outputs are stored as opaque Arrow IPC stream payloads keyed
4//! by the producing decision-schema fingerprint, so eval/training reads back
5//! the exact `RecordBatch` the model emitted — no row-wise SQL materialization
6//! and no lossy re-typing. Each row also carries the `(schema, model, calib)`
7//! version triple for joins against `calibration_params`.
8
9// tonic::Status is the idiomatic gRPC error type - boxing would break API
10#![allow(clippy::result_large_err)]
11
12use crate::storage::StorageBackend;
13use arrow::array::{
14    Array, ArrayRef, BinaryArray, Int64Array, LargeBinaryArray, RecordBatch, StringArray,
15};
16use arrow::datatypes::{DataType, Field, Schema};
17use arrow::ipc::reader::StreamReader;
18use arrow::ipc::writer::StreamWriter;
19use std::sync::Arc;
20use std::time::SystemTime;
21use tonic::Status;
22
23/// Decision batch table name.
24pub const DECISION_BATCH_TABLE: &str = "decision_batches";
25
26/// Payload format tag for Arrow IPC stream bytes.
27pub const PAYLOAD_FORMAT_IPC_STREAM: &str = "arrow-ipc-stream";
28
29/// Mint a new batch id.
30pub fn mint_batch_id() -> String {
31    format!("batch-{}", uuid::Uuid::new_v4().simple())
32}
33
34fn now_unix_ms() -> i64 {
35    let ms = SystemTime::now()
36        .duration_since(SystemTime::UNIX_EPOCH)
37        .map(|d| d.as_millis())
38        .unwrap_or(0);
39    i64::try_from(ms).unwrap_or(i64::MAX)
40}
41
42/// Row metadata for one stored decision batch (everything but the payload).
43#[derive(Debug, Clone, PartialEq, Eq)]
44pub struct DecisionBatchMeta {
45    /// Unique batch id (see [`mint_batch_id`]).
46    pub batch_id: String,
47    /// Decision-schema fingerprint of the payload schema (P0.1a IR identity).
48    pub schema_fingerprint: String,
49    /// Model id/version that produced the batch.
50    pub model_id: String,
51    /// Calibration version applied at emission; `None` for uncalibrated/raw output.
52    pub calib_version: Option<String>,
53    /// Creation time, Unix epoch milliseconds.
54    pub created_at: i64,
55    /// Number of rows in the payload batch.
56    pub row_count: i64,
57}
58
59/// Arrow schema of the `decision_batches` table.
60pub fn decision_batch_schema() -> Schema {
61    Schema::new(vec![
62        Field::new("batch_id", DataType::Utf8, false),
63        Field::new("schema_fingerprint", DataType::Utf8, false),
64        Field::new("model_id", DataType::Utf8, false),
65        Field::new("calib_version", DataType::Utf8, true),
66        Field::new("created_at", DataType::Int64, false),
67        Field::new("row_count", DataType::Int64, false),
68        Field::new("payload_format", DataType::Utf8, false),
69        Field::new("payload", DataType::Binary, false),
70    ])
71}
72
73/// Serialize a batch as a self-describing Arrow IPC stream.
74pub fn encode_batch_ipc(batch: &RecordBatch) -> Result<Vec<u8>, Status> {
75    let mut buf = Vec::new();
76    {
77        let mut writer = StreamWriter::try_new(&mut buf, &batch.schema())
78            .map_err(|e| Status::internal(format!("IPC writer init: {e}")))?;
79        writer
80            .write(batch)
81            .map_err(|e| Status::internal(format!("IPC write: {e}")))?;
82        writer
83            .finish()
84            .map_err(|e| Status::internal(format!("IPC finish: {e}")))?;
85    }
86    Ok(buf)
87}
88
89/// Decode an Arrow IPC stream payload back into a single batch
90/// (multi-batch streams are concatenated).
91pub fn decode_batch_ipc(bytes: &[u8]) -> Result<RecordBatch, Status> {
92    let reader = StreamReader::try_new(std::io::Cursor::new(bytes), None)
93        .map_err(|e| Status::internal(format!("IPC reader init: {e}")))?;
94    let mut batches = Vec::new();
95    for batch in reader {
96        batches.push(batch.map_err(|e| Status::internal(format!("IPC read: {e}")))?);
97    }
98    match batches.len() {
99        0 => Err(Status::internal("IPC payload contained no batches")),
100        1 => Ok(batches.remove(0)),
101        _ => {
102            let schema = batches[0].schema();
103            arrow::compute::concat_batches(&schema, &batches)
104                .map_err(|e| Status::internal(format!("IPC concat: {e}")))
105        }
106    }
107}
108
109fn meta_from_row(batch: &RecordBatch, row: usize) -> Result<DecisionBatchMeta, Status> {
110    let string_at = |name: &str| -> Result<String, Status> {
111        batch
112            .column_by_name(name)
113            .and_then(|c| c.as_any().downcast_ref::<StringArray>())
114            .map(|arr| arr.value(row).to_owned())
115            .ok_or_else(|| Status::internal(format!("invalid {name} column")))
116    };
117    let i64_at = |name: &str| -> Result<i64, Status> {
118        batch
119            .column_by_name(name)
120            .and_then(|c| c.as_any().downcast_ref::<Int64Array>())
121            .map(|arr| arr.value(row))
122            .ok_or_else(|| Status::internal(format!("invalid {name} column")))
123    };
124    let calib_version = batch
125        .column_by_name("calib_version")
126        .and_then(|c| c.as_any().downcast_ref::<StringArray>())
127        .and_then(|arr| {
128            if arr.is_null(row) || arr.value(row).is_empty() {
129                None
130            } else {
131                Some(arr.value(row).to_owned())
132            }
133        });
134    Ok(DecisionBatchMeta {
135        batch_id: string_at("batch_id")?,
136        schema_fingerprint: string_at("schema_fingerprint")?,
137        model_id: string_at("model_id")?,
138        calib_version,
139        created_at: i64_at("created_at")?,
140        row_count: i64_at("row_count")?,
141    })
142}
143
144fn payload_from_row(batch: &RecordBatch, row: usize) -> Result<RecordBatch, Status> {
145    let col = batch
146        .column_by_name("payload")
147        .ok_or_else(|| Status::internal("missing payload column"))?;
148    let bytes: Vec<u8> = if let Some(arr) = col.as_any().downcast_ref::<BinaryArray>() {
149        arr.value(row).to_vec()
150    } else if let Some(arr) = col.as_any().downcast_ref::<LargeBinaryArray>() {
151        arr.value(row).to_vec()
152    } else {
153        return Err(Status::internal(format!(
154            "unexpected payload column type: {:?}",
155            col.data_type()
156        )));
157    };
158    decode_batch_ipc(&bytes)
159}
160
161/// Build the one-row RecordBatch for a decision-batch insert.
162fn meta_to_record_batch(
163    meta: &DecisionBatchMeta,
164    payload_format: &str,
165    payload: &[u8],
166) -> Result<RecordBatch, Status> {
167    let schema = Arc::new(decision_batch_schema());
168    let arrays: Vec<ArrayRef> = vec![
169        Arc::new(StringArray::from(vec![meta.batch_id.as_str()])),
170        Arc::new(StringArray::from(vec![meta.schema_fingerprint.as_str()])),
171        Arc::new(StringArray::from(vec![meta.model_id.as_str()])),
172        Arc::new(StringArray::from(vec![meta.calib_version.as_deref()])),
173        Arc::new(Int64Array::from(vec![meta.created_at])),
174        Arc::new(Int64Array::from(vec![meta.row_count])),
175        Arc::new(StringArray::from(vec![payload_format])),
176        Arc::new(BinaryArray::from(vec![payload])),
177    ];
178    RecordBatch::try_new(schema, arrays)
179        .map_err(|e| Status::internal(format!("Failed to create decision batch row: {e}")))
180}
181
182fn escape_sql(value: &str) -> String {
183    value.replace('\'', "''")
184}
185
186/// Decision-batch store over any [`StorageBackend`]: the write/read API for
187/// Arrow-native model output (eval harness, distillation corpora, re-fit
188/// input).
189pub struct DecisionBatchStore<B: StorageBackend> {
190    backend: Arc<B>,
191}
192
193impl<B: StorageBackend> DecisionBatchStore<B> {
194    pub fn new(backend: Arc<B>) -> Self {
195        Self { backend }
196    }
197
198    /// Create the `decision_batches` table.
199    pub async fn init(&self) -> Result<(), Status> {
200        self.backend
201            .create_table(DECISION_BATCH_TABLE, &decision_batch_schema())
202            .await
203    }
204
205    /// Store one model-emitted batch. Returns the minted batch id.
206    pub async fn put(
207        &self,
208        schema_fingerprint: &str,
209        model_id: &str,
210        calib_version: Option<&str>,
211        batch: &RecordBatch,
212    ) -> Result<String, Status> {
213        if schema_fingerprint.is_empty() {
214            return Err(Status::invalid_argument(
215                "schema_fingerprint must not be empty",
216            ));
217        }
218        if model_id.is_empty() {
219            return Err(Status::invalid_argument("model_id must not be empty"));
220        }
221        let meta = DecisionBatchMeta {
222            batch_id: mint_batch_id(),
223            schema_fingerprint: schema_fingerprint.to_owned(),
224            model_id: model_id.to_owned(),
225            calib_version: calib_version.map(str::to_owned),
226            created_at: now_unix_ms(),
227            row_count: i64::try_from(batch.num_rows())
228                .map_err(|_| Status::invalid_argument("batch row count overflows i64"))?,
229        };
230        let payload = encode_batch_ipc(batch)?;
231        let row = meta_to_record_batch(&meta, PAYLOAD_FORMAT_IPC_STREAM, &payload)?;
232        self.backend
233            .insert_into_table(DECISION_BATCH_TABLE, row)
234            .await?;
235        Ok(meta.batch_id)
236    }
237
238    /// Fetch one batch by id, decoding the payload back to its RecordBatch.
239    pub async fn get(
240        &self,
241        batch_id: &str,
242    ) -> Result<Option<(DecisionBatchMeta, RecordBatch)>, Status> {
243        let sql = format!(
244            "SELECT * FROM {DECISION_BATCH_TABLE} WHERE batch_id = '{}'",
245            escape_sql(batch_id)
246        );
247        let handle = self.backend.prepare_sql(&sql).await?;
248        let batch = self.backend.query_sql(&handle).await?;
249        if batch.num_rows() == 0 {
250            return Ok(None);
251        }
252        let meta = meta_from_row(&batch, 0)?;
253        let payload = payload_from_row(&batch, 0)?;
254        Ok(Some((meta, payload)))
255    }
256
257    /// All batches for a decision-schema fingerprint, oldest first.
258    pub async fn fetch_by_schema(
259        &self,
260        schema_fingerprint: &str,
261    ) -> Result<Vec<(DecisionBatchMeta, RecordBatch)>, Status> {
262        let sql = format!(
263            "SELECT * FROM {DECISION_BATCH_TABLE} WHERE schema_fingerprint = '{}' \
264             ORDER BY created_at ASC",
265            escape_sql(schema_fingerprint)
266        );
267        let handle = self.backend.prepare_sql(&sql).await?;
268        let batch = self.backend.query_sql(&handle).await?;
269        let mut out = Vec::with_capacity(batch.num_rows());
270        for row in 0..batch.num_rows() {
271            out.push((meta_from_row(&batch, row)?, payload_from_row(&batch, row)?));
272        }
273        Ok(out)
274    }
275
276    /// List batch metadata, optionally filtered by schema fingerprint.
277    pub async fn list(
278        &self,
279        schema_fingerprint: Option<&str>,
280    ) -> Result<Vec<DecisionBatchMeta>, Status> {
281        let mut sql = format!(
282            "SELECT batch_id, schema_fingerprint, model_id, calib_version, created_at, row_count \
283             FROM {DECISION_BATCH_TABLE}"
284        );
285        if let Some(fingerprint) = schema_fingerprint {
286            sql.push_str(&format!(
287                " WHERE schema_fingerprint = '{}'",
288                escape_sql(fingerprint)
289            ));
290        }
291        sql.push_str(" ORDER BY created_at ASC");
292        let handle = self.backend.prepare_sql(&sql).await?;
293        let batch = self.backend.query_sql(&handle).await?;
294        (0..batch.num_rows())
295            .map(|row| meta_from_row(&batch, row))
296            .collect()
297    }
298}