Skip to main content

hyprstream_metrics_api/metrics/
calibration.rs

1//! Calibration parameter tables for decision models (System One P0.3/P2.2).
2//!
3//! One row holds the post-hoc calibration parameters for a single decision
4//! primitive (`noul`/`choice`/`score`) of one decision schema on one model,
5//! keyed by the `(schema_id, model_id, calib_version)` version triple:
6//!
7//! - per-primitive temperature,
8//! - null-state (content-free) bias vector,
9//! - optional Dirichlet-calibration params for high-cardinality choice,
10//! - conformal quantiles per primitive — score uses ordinal
11//!   contiguous-interval params, noul/choice use APS/RAPS-style quantiles.
12//!
13//! Rows are append-only: a calibration re-fit mints a NEW `calib_version` row
14//! and never updates in place, so every past fit stays queryable and serving
15//! can pin an exact triple.
16
17// tonic::Status is the idiomatic gRPC error type - boxing would break API
18#![allow(clippy::result_large_err)]
19
20use crate::storage::StorageBackend;
21use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, RecordBatch, StringArray};
22use arrow::datatypes::{DataType, Field, Schema};
23use serde::{Deserialize, Serialize};
24use std::sync::Arc;
25use tonic::Status;
26
27/// Calibration parameter table name.
28pub const CALIBRATION_TABLE: &str = "calibration_params";
29
30/// Mint a new calibration version id. Re-fits call this (or supply their own)
31/// and insert a new row; existing rows are never mutated.
32pub fn mint_calib_version() -> String {
33    format!("cal-{}", uuid::Uuid::new_v4().simple())
34}
35
36/// Decision primitive a calibration row applies to (jev-1 profile superset).
37#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
38#[serde(rename_all = "snake_case")]
39pub enum DecisionPrimitive {
40    /// Statement → P(true).
41    Noul,
42    /// One of ≤255 enumerated options.
43    Choice,
44    /// Ordered levels → distribution + probability-weighted value.
45    Score,
46}
47
48impl DecisionPrimitive {
49    pub fn as_str(&self) -> &'static str {
50        match self {
51            Self::Noul => "noul",
52            Self::Choice => "choice",
53            Self::Score => "score",
54        }
55    }
56
57    pub fn parse(s: &str) -> Result<Self, Status> {
58        match s {
59            "noul" => Ok(Self::Noul),
60            "choice" => Ok(Self::Choice),
61            "score" => Ok(Self::Score),
62            other => Err(Status::invalid_argument(format!(
63                "unknown decision primitive: {other:?}"
64            ))),
65        }
66    }
67}
68
69impl std::fmt::Display for DecisionPrimitive {
70    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
71        f.write_str(self.as_str())
72    }
73}
74
75/// The (schema, model, calib) version triple keying every calibration row.
76#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
77pub struct VersionTriple {
78    /// Decision-schema fingerprint (P0.1a IR identity).
79    pub schema_id: String,
80    /// Model id/version the parameters were fit for.
81    pub model_id: String,
82    /// Calibration version; minted fresh by every re-fit (see
83    /// [`mint_calib_version`]).
84    pub calib_version: String,
85}
86
87/// Dirichlet-calibration parameters (Kull et al. 2019) for high-cardinality
88/// choice questions, where scalar temperature underfits: calibrated log-probs
89/// are `W · ln(p) + b` with `weights` row-major K×K and `bias` of length K.
90#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
91pub struct DirichletParams {
92    pub weights: Vec<Vec<f64>>,
93    pub bias: Vec<f64>,
94}
95
96/// One coverage level → nonconformity quantile (APS/RAPS-style).
97#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
98pub struct ConformalQuantile {
99    /// Target coverage level in (0, 1).
100    pub level: f64,
101    /// Nonconformity-score quantile at that level.
102    pub quantile: f64,
103}
104
105/// One coverage level → ordinal contiguous interval half-widths, in level
106/// units, applied around the predicted level (min-CPS / RPS-based
107/// nonconformity for the score primitive).
108#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
109pub struct OrdinalInterval {
110    /// Target coverage level in (0, 1).
111    pub level: f64,
112    /// Levels below the prediction included at this coverage.
113    pub lower: f64,
114    /// Levels above the prediction included at this coverage.
115    pub upper: f64,
116}
117
118/// Conformal parameters for one primitive. Score questions are ordinal, so
119/// their prediction sets are contiguous intervals; noul/choice use
120/// APS/RAPS-style quantiles.
121#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
122#[serde(tag = "kind", rename_all = "snake_case")]
123pub enum ConformalParams {
124    /// APS/RAPS: nonconformity quantile per coverage level (noul/choice).
125    Quantiles { quantiles: Vec<ConformalQuantile> },
126    /// Ordinal contiguous intervals per coverage level (score).
127    OrdinalIntervals { intervals: Vec<OrdinalInterval> },
128}
129
130/// One calibration parameter row.
131#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
132pub struct CalibrationParams {
133    /// (schema, model, calib) version triple — the primary key.
134    pub version: VersionTriple,
135    /// Primitive these parameters apply to.
136    pub primitive: DecisionPrimitive,
137    /// Per-primitive temperature (softmax scale). `None` = identity.
138    pub temperature: Option<f64>,
139    /// Null-state (content-free input) bias vector, one entry per option/level.
140    pub null_bias: Option<Vec<f64>>,
141    /// Dirichlet-calibration params (choice only, high-cardinality).
142    pub dirichlet: Option<DirichletParams>,
143    /// Conformal parameters; the variant must match `primitive`.
144    pub conformal: Option<ConformalParams>,
145    /// Option/level cardinality K at fit time.
146    pub cardinality: Option<i64>,
147    /// Id of the calibration set the parameters were fit on.
148    pub calib_set_id: String,
149    /// Number of examples in the calibration set.
150    pub calib_size: Option<i64>,
151    /// Fitting objective, free-form (e.g. "nll", "rps", "brier").
152    pub objective: Option<String>,
153    /// Fit time, Unix epoch milliseconds.
154    pub fitted_at: i64,
155}
156
157impl CalibrationParams {
158    /// Structural validation applied on every write.
159    pub fn validate(&self) -> Result<(), Status> {
160        let invalid = |msg: &str| Status::invalid_argument(msg.to_owned());
161        if self.version.schema_id.is_empty() {
162            return Err(invalid("schema_id must not be empty"));
163        }
164        if self.version.model_id.is_empty() {
165            return Err(invalid("model_id must not be empty"));
166        }
167        if self.version.calib_version.is_empty() {
168            return Err(invalid("calib_version must not be empty"));
169        }
170        if self.calib_set_id.is_empty() {
171            return Err(invalid("calib_set_id must not be empty"));
172        }
173        if let Some(t) = self.temperature {
174            if !t.is_finite() || t <= 0.0 {
175                return Err(invalid("temperature must be finite and > 0"));
176            }
177        }
178        if let Some(bias) = &self.null_bias {
179            if bias.iter().any(|v| !v.is_finite()) {
180                return Err(invalid("null_bias entries must be finite"));
181            }
182        }
183        if let Some(dirichlet) = &self.dirichlet {
184            if self.primitive != DecisionPrimitive::Choice {
185                return Err(invalid(
186                    "dirichlet params are only valid for the choice primitive",
187                ));
188            }
189            let k = dirichlet.weights.len();
190            if k == 0
191                || dirichlet.weights.iter().any(|row| row.len() != k)
192                || dirichlet.bias.len() != k
193            {
194                return Err(invalid(
195                    "dirichlet weights must be square KxK with bias of length K",
196                ));
197            }
198            let finite = dirichlet
199                .weights
200                .iter()
201                .flatten()
202                .chain(dirichlet.bias.iter())
203                .all(|v| v.is_finite());
204            if !finite {
205                return Err(invalid("dirichlet params must be finite"));
206            }
207            if let Some(card) = self.cardinality {
208                if card >= 0 && k as i64 != card {
209                    return Err(invalid("dirichlet dimension must equal cardinality"));
210                }
211            }
212        }
213        match (&self.conformal, self.primitive) {
214            (Some(ConformalParams::Quantiles { .. }), DecisionPrimitive::Score) => {
215                return Err(invalid(
216                    "score conformal params must be ordinal intervals, not quantiles",
217                ));
218            }
219            (Some(ConformalParams::OrdinalIntervals { intervals }), DecisionPrimitive::Score) => {
220                for e in intervals {
221                    if !(0.0..1.0).contains(&e.level) {
222                        return Err(invalid("conformal levels must be in (0, 1)"));
223                    }
224                    if !e.lower.is_finite()
225                        || !e.upper.is_finite()
226                        || e.lower < 0.0
227                        || e.upper < 0.0
228                    {
229                        return Err(invalid("ordinal interval offsets must be finite and >= 0"));
230                    }
231                }
232            }
233            (Some(ConformalParams::OrdinalIntervals { .. }), _) => {
234                return Err(invalid(
235                    "ordinal interval conformal params are only valid for the score primitive",
236                ));
237            }
238            (Some(ConformalParams::Quantiles { quantiles }), _) => {
239                for e in quantiles {
240                    if !(0.0..1.0).contains(&e.level) || !e.quantile.is_finite() {
241                        return Err(invalid(
242                            "conformal levels must be in (0, 1) with finite quantiles",
243                        ));
244                    }
245                }
246            }
247            (None, _) => {}
248        }
249        if let Some(card) = self.cardinality {
250            if card <= 0 {
251                return Err(invalid("cardinality must be > 0"));
252            }
253            if let Some(bias) = &self.null_bias {
254                if bias.len() as i64 != card {
255                    return Err(invalid("null_bias length must equal cardinality"));
256                }
257            }
258        }
259        if let Some(size) = self.calib_size {
260            if size < 0 {
261                return Err(invalid("calib_size must be >= 0"));
262            }
263        }
264        Ok(())
265    }
266}
267
268/// Arrow schema of the `calibration_params` table. Vector/composite params
269/// are stored as JSON in nullable VARCHAR columns (the crate's existing
270/// convention for composite values, queryable via DuckDB JSON functions).
271pub fn calibration_params_schema() -> Schema {
272    Schema::new(vec![
273        Field::new("schema_id", DataType::Utf8, false),
274        Field::new("model_id", DataType::Utf8, false),
275        Field::new("calib_version", DataType::Utf8, false),
276        Field::new("primitive", DataType::Utf8, false),
277        Field::new("temperature", DataType::Float64, true),
278        Field::new("null_bias", DataType::Utf8, true),
279        Field::new("dirichlet_params", DataType::Utf8, true),
280        Field::new("conformal_params", DataType::Utf8, true),
281        Field::new("cardinality", DataType::Int64, true),
282        Field::new("calib_set_id", DataType::Utf8, false),
283        Field::new("calib_size", DataType::Int64, true),
284        Field::new("objective", DataType::Utf8, true),
285        Field::new("fitted_at", DataType::Int64, false),
286    ])
287}
288
289fn json_or_none<T: Serialize>(value: &Option<T>) -> Result<Option<String>, Status> {
290    value
291        .as_ref()
292        .map(|v| {
293            serde_json::to_string(v)
294                .map_err(|e| Status::internal(format!("calibration params JSON encode: {e}")))
295        })
296        .transpose()
297}
298
299/// Build the one-row RecordBatch for a calibration insert.
300pub fn params_to_record_batch(params: &CalibrationParams) -> Result<RecordBatch, Status> {
301    params.validate()?;
302    let schema = Arc::new(calibration_params_schema());
303    let null_bias = json_or_none(&params.null_bias)?;
304    let dirichlet = json_or_none(&params.dirichlet)?;
305    let conformal = json_or_none(&params.conformal)?;
306
307    let arrays: Vec<ArrayRef> = vec![
308        Arc::new(StringArray::from(vec![params.version.schema_id.as_str()])),
309        Arc::new(StringArray::from(vec![params.version.model_id.as_str()])),
310        Arc::new(StringArray::from(vec![params
311            .version
312            .calib_version
313            .as_str()])),
314        Arc::new(StringArray::from(vec![params.primitive.as_str()])),
315        Arc::new(Float64Array::from(vec![params.temperature])),
316        Arc::new(StringArray::from(vec![null_bias.as_deref()])),
317        Arc::new(StringArray::from(vec![dirichlet.as_deref()])),
318        Arc::new(StringArray::from(vec![conformal.as_deref()])),
319        Arc::new(Int64Array::from(vec![params.cardinality])),
320        Arc::new(StringArray::from(vec![params.calib_set_id.as_str()])),
321        Arc::new(Int64Array::from(vec![params.calib_size])),
322        Arc::new(StringArray::from(vec![params.objective.as_deref()])),
323        Arc::new(Int64Array::from(vec![params.fitted_at])),
324    ];
325    RecordBatch::try_new(schema, arrays)
326        .map_err(|e| Status::internal(format!("Failed to create calibration batch: {e}")))
327}
328
329fn opt_string(col: Option<&ArrayRef>, row: usize) -> Result<Option<String>, Status> {
330    let col = col.ok_or_else(|| Status::internal("missing column in calibration row"))?;
331    let arr = col
332        .as_any()
333        .downcast_ref::<StringArray>()
334        .ok_or_else(|| Status::internal("invalid Utf8 column in calibration row"))?;
335    Ok(if arr.is_null(row) {
336        None
337    } else {
338        Some(arr.value(row).to_owned())
339    })
340}
341
342fn req_string(col: Option<&ArrayRef>, row: usize, name: &str) -> Result<String, Status> {
343    opt_string(col, row)?.ok_or_else(|| Status::internal(format!("NULL {name} in calibration row")))
344}
345
346fn opt_f64(col: Option<&ArrayRef>, row: usize) -> Result<Option<f64>, Status> {
347    let col = col.ok_or_else(|| Status::internal("missing column in calibration row"))?;
348    let arr = col
349        .as_any()
350        .downcast_ref::<Float64Array>()
351        .ok_or_else(|| Status::internal("invalid Float64 column in calibration row"))?;
352    Ok(if arr.is_null(row) {
353        None
354    } else {
355        Some(arr.value(row))
356    })
357}
358
359fn opt_i64(col: Option<&ArrayRef>, row: usize) -> Result<Option<i64>, Status> {
360    let col = col.ok_or_else(|| Status::internal("missing column in calibration row"))?;
361    let arr = col
362        .as_any()
363        .downcast_ref::<Int64Array>()
364        .ok_or_else(|| Status::internal("invalid Int64 column in calibration row"))?;
365    Ok(if arr.is_null(row) {
366        None
367    } else {
368        Some(arr.value(row))
369    })
370}
371
372fn decode_json<T: serde::de::DeserializeOwned>(
373    raw: Option<String>,
374    column: &str,
375) -> Result<Option<T>, Status> {
376    raw.map(|s| {
377        serde_json::from_str(&s)
378            .map_err(|e| Status::internal(format!("invalid {column} JSON: {e}")))
379    })
380    .transpose()
381}
382
383/// Decode one calibration row from a query result batch.
384pub fn params_from_row(batch: &RecordBatch, row: usize) -> Result<CalibrationParams, Status> {
385    let col = |name: &str| batch.column_by_name(name);
386    let primitive = DecisionPrimitive::parse(&req_string(col("primitive"), row, "primitive")?)?;
387    let null_bias: Option<Vec<f64>> = decode_json(opt_string(col("null_bias"), row)?, "null_bias")?;
388    let dirichlet: Option<DirichletParams> = decode_json(
389        opt_string(col("dirichlet_params"), row)?,
390        "dirichlet_params",
391    )?;
392    let conformal: Option<ConformalParams> = decode_json(
393        opt_string(col("conformal_params"), row)?,
394        "conformal_params",
395    )?;
396    Ok(CalibrationParams {
397        version: VersionTriple {
398            schema_id: req_string(col("schema_id"), row, "schema_id")?,
399            model_id: req_string(col("model_id"), row, "model_id")?,
400            calib_version: req_string(col("calib_version"), row, "calib_version")?,
401        },
402        primitive,
403        temperature: opt_f64(col("temperature"), row)?,
404        null_bias,
405        dirichlet,
406        conformal,
407        cardinality: opt_i64(col("cardinality"), row)?,
408        calib_set_id: req_string(col("calib_set_id"), row, "calib_set_id")?,
409        calib_size: opt_i64(col("calib_size"), row)?,
410        objective: opt_string(col("objective"), row)?,
411        fitted_at: opt_i64(col("fitted_at"), row)?
412            .ok_or_else(|| Status::internal("NULL fitted_at in calibration row"))?,
413    })
414}
415
416fn escape_sql(value: &str) -> String {
417    value.replace('\'', "''")
418}
419
420/// Calibration parameter store over any [`StorageBackend`]: the writer API for
421/// the P2.2 re-fit loop and the reader API for serving.
422pub struct CalibrationStore<B: StorageBackend> {
423    backend: Arc<B>,
424}
425
426impl<B: StorageBackend> CalibrationStore<B> {
427    pub fn new(backend: Arc<B>) -> Self {
428        Self { backend }
429    }
430
431    /// Create the `calibration_params` table.
432    pub async fn init(&self) -> Result<(), Status> {
433        self.backend
434            .create_table(CALIBRATION_TABLE, &calibration_params_schema())
435            .await
436    }
437
438    /// Insert one validated parameter row. The (primitive, version triple) key
439    /// must be new — re-fits mint a fresh `calib_version`; rows are never
440    /// updated in place.
441    pub async fn insert(&self, params: &CalibrationParams) -> Result<(), Status> {
442        params.validate()?;
443        let v = &params.version;
444        let count_sql = format!(
445            "SELECT COUNT(*) AS cnt FROM {CALIBRATION_TABLE} \
446             WHERE schema_id = '{}' AND model_id = '{}' AND calib_version = '{}' AND primitive = '{}'",
447            escape_sql(&v.schema_id),
448            escape_sql(&v.model_id),
449            escape_sql(&v.calib_version),
450            params.primitive.as_str(),
451        );
452        let handle = self.backend.prepare_sql(&count_sql).await?;
453        let batch = self.backend.query_sql(&handle).await?;
454        if count_rows(&batch)? > 0 {
455            return Err(Status::already_exists(format!(
456                "calibration row for ({}, {}, {}, {}) already exists; re-fit must mint a new calib_version",
457                params.primitive.as_str(), v.schema_id, v.model_id, v.calib_version,
458            )));
459        }
460        let batch = params_to_record_batch(params)?;
461        self.backend
462            .insert_into_table(CALIBRATION_TABLE, batch)
463            .await
464    }
465
466    /// The newest parameter row for a (schema, model, primitive), by fit time.
467    pub async fn latest(
468        &self,
469        schema_id: &str,
470        model_id: &str,
471        primitive: DecisionPrimitive,
472    ) -> Result<Option<CalibrationParams>, Status> {
473        let history = self.history(schema_id, model_id, primitive).await?;
474        Ok(history.into_iter().next())
475    }
476
477    /// All parameter versions for a (schema, model, primitive), newest first.
478    pub async fn history(
479        &self,
480        schema_id: &str,
481        model_id: &str,
482        primitive: DecisionPrimitive,
483    ) -> Result<Vec<CalibrationParams>, Status> {
484        let sql = format!(
485            "SELECT * FROM {CALIBRATION_TABLE} \
486             WHERE schema_id = '{}' AND model_id = '{}' AND primitive = '{}' \
487             ORDER BY fitted_at DESC",
488            escape_sql(schema_id),
489            escape_sql(model_id),
490            primitive.as_str(),
491        );
492        let handle = self.backend.prepare_sql(&sql).await?;
493        let batch = self.backend.query_sql(&handle).await?;
494        (0..batch.num_rows())
495            .map(|row| params_from_row(&batch, row))
496            .collect()
497    }
498}
499
500/// Extract a COUNT(*) result that DuckDB may type as Int64 or UInt64.
501fn count_rows(batch: &RecordBatch) -> Result<u64, Status> {
502    use arrow::array::UInt64Array;
503    let col = batch
504        .column_by_name("cnt")
505        .ok_or_else(|| Status::internal("COUNT(*) result has no cnt column"))?;
506    if let Some(arr) = col.as_any().downcast_ref::<Int64Array>() {
507        return Ok(arr.value(0).max(0) as u64);
508    }
509    if let Some(arr) = col.as_any().downcast_ref::<UInt64Array>() {
510        return Ok(arr.value(0));
511    }
512    Err(Status::internal(format!(
513        "unexpected COUNT(*) column type: {:?}",
514        col.data_type()
515    )))
516}
517
518#[cfg(test)]
519mod tests {
520    use super::*;
521
522    fn params(
523        primitive: DecisionPrimitive,
524        calib_version: &str,
525        fitted_at: i64,
526    ) -> CalibrationParams {
527        CalibrationParams {
528            version: VersionTriple {
529                schema_id: "jev1:q1".to_owned(),
530                model_id: "qwen3.5-0.8b@main".to_owned(),
531                calib_version: calib_version.to_owned(),
532            },
533            primitive,
534            temperature: Some(1.25),
535            null_bias: Some(vec![0.1, -0.2, 0.05]),
536            dirichlet: None,
537            conformal: None,
538            cardinality: Some(3),
539            calib_set_id: "calibset-2026-09".to_owned(),
540            calib_size: Some(10_000),
541            objective: Some("nll".to_owned()),
542            fitted_at,
543        }
544    }
545
546    #[test]
547    fn test_validate_rejects_bad_rows() {
548        let mut p = params(DecisionPrimitive::Choice, "cal-v1", 1);
549        p.temperature = Some(0.0);
550        assert!(p.validate().is_err());
551
552        let mut p = params(DecisionPrimitive::Noul, "cal-v1", 1);
553        p.dirichlet = Some(DirichletParams {
554            weights: vec![vec![1.0]],
555            bias: vec![0.0],
556        });
557        assert!(p.validate().is_err(), "dirichlet is choice-only");
558
559        let mut p = params(DecisionPrimitive::Score, "cal-v1", 1);
560        p.conformal = Some(ConformalParams::Quantiles {
561            quantiles: vec![ConformalQuantile {
562                level: 0.9,
563                quantile: 0.5,
564            }],
565        });
566        assert!(p.validate().is_err(), "score needs ordinal intervals");
567
568        let mut p = params(DecisionPrimitive::Choice, "cal-v1", 1);
569        p.conformal = Some(ConformalParams::OrdinalIntervals {
570            intervals: vec![OrdinalInterval {
571                level: 0.9,
572                lower: 1.0,
573                upper: 1.0,
574            }],
575        });
576        assert!(p.validate().is_err(), "choice needs quantiles");
577
578        let mut p = params(DecisionPrimitive::Choice, "cal-v1", 1);
579        p.version.calib_version = String::new();
580        assert!(p.validate().is_err());
581    }
582
583    #[test]
584    fn test_minted_versions_are_unique() {
585        assert_ne!(mint_calib_version(), mint_calib_version());
586    }
587}