Skip to main content

hyprstream_calibration/
protocol.rs

1//! The pinned gate measurement protocol.
2//!
3//! Every gate that consumes calibration numbers (P0.5 baselines, P1.4 legs b/c/e, P2.3)
4//! measures them this way. The constants are the contract; [`GateReport`] is the
5//! serializable artifact a gate run emits. Nothing here fits parameters — fitting is
6//! P2.2; this module defines *measurement* only.
7//!
8//! ## The protocol, pinned
9//!
10//! 1. **Binning**: [`GATE_BIN_COUNT`] equal-mass bins for ECE and reliability curves
11//!    ([`crate::bins::Binning::equal_mass`]). Equal-mass, not classic equal-width, so
12//!    bins stay populated under peaked confidence distributions.
13//! 2. **Uncertainty**: percentile bootstrap CIs at [`GATE_CI_LEVEL`] over
14//!    [`GATE_BOOTSTRAP_RESAMPLES`] resamples, seeded and recorded. Gate comparisons that
15//!    pin a bound (P1.4 leg b's "macro-ECE gap ≤ 2×") condition on the **upper CI
16//!    bound**, never the point estimate.
17//! 3. **Aggregation — per-field macro rule**: metrics are computed per question field
18//!    and averaged with equal field weight ([`macro_average`]). Rows are never pooled
19//!    across fields: a high-volume field must not dilute a miscalibrated rare one.
20//! 4. **Shift split** ([`ShiftSplit`]): the shift evaluation split is a held-out
21//!    workflow *family* — evaluation families disjoint from every family used for
22//!    fitting or calibration. Item-level disjointness is not the shift split.
23//! 5. **Per-primitive metrics** (P1.4 leg c): ECE for `noul`/`choice`; SCE/ACE + RPS for
24//!    `score` (top-label ECE is class-blind on ordinal tasks — S6b2).
25//! 6. **Joint coverage**: conformal coverage is marginal; cross-question joint coverage
26//!    is reported separately ([`crate::joint`]).
27
28use serde::{Deserialize, Serialize};
29
30use crate::bootstrap::{bootstrap_ci, ConfidenceInterval};
31use crate::error::MetricError;
32
33/// Pinned bin count for ECE and reliability curves (equal-mass).
34pub const GATE_BIN_COUNT: usize = 15;
35
36/// Pinned bootstrap resample count for gate confidence intervals.
37pub const GATE_BOOTSTRAP_RESAMPLES: usize = 1000;
38
39/// Pinned confidence level for gate intervals.
40pub const GATE_CI_LEVEL: f64 = 0.95;
41
42/// Per-field macro aggregation: unweighted mean of per-field metric values.
43/// Returns `None` for an empty field set (a report with no fields is malformed, not 0).
44#[must_use]
45pub fn macro_average(per_field: &[f64]) -> Option<f64> {
46    if per_field.is_empty() {
47        return None;
48    }
49    #[allow(clippy::cast_precision_loss)]
50    let n = per_field.len() as f64;
51    Some(per_field.iter().sum::<f64>() / n)
52}
53
54/// The shift-split definition: which workflow families may be used for
55/// fitting/calibration and which are held out for evaluation.
56///
57/// The **shift split** (P1.4 leg d, and the zero-shot leg e at family level) holds out
58/// whole workflow families: any family whose items were synthesized into training data,
59/// used for distillation targets, or used to fit calibration parameters is a *fit*
60/// family; the gate measures on the disjoint *evaluation* families. Construction enforces
61/// disjointness — a family on both sides is a protocol violation, not a warning.
62#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
63pub struct ShiftSplit {
64    /// Families used for fitting (training, distillation, calibration-parameter fitting).
65    pub fit_families: Vec<String>,
66    /// Held-out families reserved for shift/zero-shot evaluation.
67    pub eval_families: Vec<String>,
68}
69
70impl ShiftSplit {
71    /// Construct a shift split, rejecting any family present on both sides.
72    ///
73    /// # Errors
74    /// [`MetricError::ShiftSplitOverlap`] listing the overlapping families.
75    pub fn new(
76        fit_families: impl IntoIterator<Item = impl Into<String>>,
77        eval_families: impl IntoIterator<Item = impl Into<String>>,
78    ) -> Result<Self, MetricError> {
79        let fit: Vec<String> = fit_families.into_iter().map(Into::into).collect();
80        let eval: Vec<String> = eval_families.into_iter().map(Into::into).collect();
81        let mut overlap: Vec<String> = eval.iter().filter(|f| fit.contains(f)).cloned().collect();
82        overlap.sort();
83        overlap.dedup();
84        if !overlap.is_empty() {
85            return Err(MetricError::ShiftSplitOverlap { families: overlap });
86        }
87        Ok(Self {
88            fit_families: fit,
89            eval_families: eval,
90        })
91    }
92
93    /// Which side of the split a family belongs to.
94    #[must_use]
95    pub fn is_eval_family(&self, family: &str) -> bool {
96        self.eval_families.iter().any(|f| f == family)
97    }
98}
99
100/// One question field's measured metrics (one row of the macro rule).
101#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
102pub struct FieldMetrics {
103    /// Question field identifier (question id / field name in the question set).
104    pub field: String,
105    /// Workflow family this field's rows were drawn from.
106    pub family: String,
107    /// Number of observations measured.
108    pub n: usize,
109    /// Top-label ECE (equal-mass, [`GATE_BIN_COUNT`] bins). Primary for noul/choice.
110    pub ece: f64,
111    /// Multiclass Brier score.
112    pub brier: f64,
113    /// Class-wise static calibration error. Present for score fields (P1.4 leg c).
114    pub sce: Option<f64>,
115    /// Class-wise adaptive calibration error. Present for score fields.
116    pub ace: Option<f64>,
117    /// Ranked probability score. Present for score fields.
118    pub rps: Option<f64>,
119}
120
121/// Macro-averaged metric with its bootstrap CI, as reported by a gate.
122#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
123pub struct MacroMetrics {
124    /// Per-field macro mean (equal field weights).
125    pub mean: f64,
126    /// Bootstrap CI over the *field* values (fields are the resampling unit — resampling
127    /// rows would understate field-level variance).
128    pub ci: ConfidenceInterval,
129}
130
131impl MacroMetrics {
132    /// Macro-average a metric over fields, with the protocol bootstrap CI.
133    ///
134    /// # Errors
135    /// [`MetricError::EmptyInput`] if no field values are given; propagates bootstrap
136    /// errors.
137    pub fn from_fields(values: &[f64], seed: u64) -> Result<Self, MetricError> {
138        let (mean, ci) = bootstrap_ci(
139            values,
140            |v| {
141                #[allow(clippy::cast_precision_loss)]
142                let n = v.len() as f64;
143                v.iter().sum::<f64>() / n
144            },
145            GATE_BOOTSTRAP_RESAMPLES,
146            GATE_CI_LEVEL,
147            seed,
148        )?;
149        Ok(Self { mean, ci })
150    }
151
152    /// Upper CI bound — the value bound-style gate comparisons condition on.
153    #[must_use]
154    pub fn upper_bound(&self) -> f64 {
155        self.ci.high
156    }
157}
158
159/// A gate run's serializable measurement artifact.
160#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
161pub struct GateReport {
162    /// Name of the arm/subject measured (e.g. a model version, a baseline arm).
163    pub subject: String,
164    /// The split this report measured (e.g. `"deterministic"`, `"shift"`, `"zero-shot"`).
165    pub split: String,
166    /// The shift split in force for this report (fit/eval family designation).
167    pub shift_split: ShiftSplit,
168    /// Per-field measurements (the macro rule's inputs).
169    pub fields: Vec<FieldMetrics>,
170    /// Macro-averaged ECE with CI.
171    pub macro_ece: MacroMetrics,
172    /// Macro-averaged Brier with CI.
173    pub macro_brier: MacroMetrics,
174    /// Macro-averaged SCE over score fields with CI, if any score fields were measured.
175    pub macro_sce: Option<MacroMetrics>,
176    /// Macro-averaged RPS over score fields with CI, if any score fields were measured.
177    pub macro_rps: Option<MacroMetrics>,
178    /// Seed used for all bootstrap CIs in this report.
179    pub bootstrap_seed: u64,
180}
181
182impl GateReport {
183    /// Assemble a gate report from per-field measurements, applying the protocol's macro
184    /// rule and bootstrap CIs. SCE/ACE/RPS macro aggregates cover only the fields that
185    /// carry them (score fields), keeping the per-primitive split of P1.4 leg c.
186    ///
187    /// # Errors
188    /// [`MetricError::EmptyInput`] if `fields` is empty; propagates bootstrap errors.
189    pub fn assemble(
190        subject: impl Into<String>,
191        split: impl Into<String>,
192        shift_split: ShiftSplit,
193        fields: Vec<FieldMetrics>,
194        bootstrap_seed: u64,
195    ) -> Result<Self, MetricError> {
196        if fields.is_empty() {
197            return Err(MetricError::EmptyInput("gate report"));
198        }
199        let macro_ece = MacroMetrics::from_fields(
200            &fields.iter().map(|f| f.ece).collect::<Vec<_>>(),
201            bootstrap_seed,
202        )?;
203        let macro_brier = MacroMetrics::from_fields(
204            &fields.iter().map(|f| f.brier).collect::<Vec<_>>(),
205            bootstrap_seed ^ 0x5EED_0001,
206        )?;
207        let score_metric = |pick: fn(&FieldMetrics) -> Option<f64>, salt: u64| {
208            let values: Vec<f64> = fields.iter().filter_map(pick).collect();
209            if values.is_empty() {
210                return Ok(None);
211            }
212            MacroMetrics::from_fields(&values, bootstrap_seed ^ salt).map(Some)
213        };
214        let macro_sce = score_metric(|f| f.sce, 0x5EED_0002)?;
215        let macro_rps = score_metric(|f| f.rps, 0x5EED_0003)?;
216        Ok(Self {
217            subject: subject.into(),
218            split: split.into(),
219            shift_split,
220            fields,
221            macro_ece,
222            macro_brier,
223            macro_sce,
224            macro_rps,
225            bootstrap_seed,
226        })
227    }
228}