hyprstream_calibration/protocol.rs
1//! The pinned gate measurement protocol.
2//!
3//! Every gate that consumes calibration numbers (P0.5 baselines, P1.4 legs b/c/e, P2.3)
4//! measures them this way. The constants are the contract; [`GateReport`] is the
5//! serializable artifact a gate run emits. Nothing here fits parameters — fitting is
6//! P2.2; this module defines *measurement* only.
7//!
8//! ## The protocol, pinned
9//!
10//! 1. **Binning**: [`GATE_BIN_COUNT`] equal-mass bins for ECE and reliability curves
11//! ([`crate::bins::Binning::equal_mass`]). Equal-mass, not classic equal-width, so
12//! bins stay populated under peaked confidence distributions.
13//! 2. **Uncertainty**: percentile bootstrap CIs at [`GATE_CI_LEVEL`] over
14//! [`GATE_BOOTSTRAP_RESAMPLES`] resamples, seeded and recorded. Gate comparisons that
15//! pin a bound (P1.4 leg b's "macro-ECE gap ≤ 2×") condition on the **upper CI
16//! bound**, never the point estimate.
17//! 3. **Aggregation — per-field macro rule**: metrics are computed per question field
18//! and averaged with equal field weight ([`macro_average`]). Rows are never pooled
19//! across fields: a high-volume field must not dilute a miscalibrated rare one.
20//! 4. **Shift split** ([`ShiftSplit`]): the shift evaluation split is a held-out
21//! workflow *family* — evaluation families disjoint from every family used for
22//! fitting or calibration. Item-level disjointness is not the shift split.
23//! 5. **Per-primitive metrics** (P1.4 leg c): ECE for `noul`/`choice`; SCE/ACE + RPS for
24//! `score` (top-label ECE is class-blind on ordinal tasks — S6b2).
25//! 6. **Joint coverage**: conformal coverage is marginal; cross-question joint coverage
26//! is reported separately ([`crate::joint`]).
27
28use serde::{Deserialize, Serialize};
29
30use crate::bootstrap::{bootstrap_ci, ConfidenceInterval};
31use crate::error::MetricError;
32
33/// Pinned bin count for ECE and reliability curves (equal-mass).
34pub const GATE_BIN_COUNT: usize = 15;
35
36/// Pinned bootstrap resample count for gate confidence intervals.
37pub const GATE_BOOTSTRAP_RESAMPLES: usize = 1000;
38
39/// Pinned confidence level for gate intervals.
40pub const GATE_CI_LEVEL: f64 = 0.95;
41
42/// Per-field macro aggregation: unweighted mean of per-field metric values.
43/// Returns `None` for an empty field set (a report with no fields is malformed, not 0).
44#[must_use]
45pub fn macro_average(per_field: &[f64]) -> Option<f64> {
46 if per_field.is_empty() {
47 return None;
48 }
49 #[allow(clippy::cast_precision_loss)]
50 let n = per_field.len() as f64;
51 Some(per_field.iter().sum::<f64>() / n)
52}
53
54/// The shift-split definition: which workflow families may be used for
55/// fitting/calibration and which are held out for evaluation.
56///
57/// The **shift split** (P1.4 leg d, and the zero-shot leg e at family level) holds out
58/// whole workflow families: any family whose items were synthesized into training data,
59/// used for distillation targets, or used to fit calibration parameters is a *fit*
60/// family; the gate measures on the disjoint *evaluation* families. Construction enforces
61/// disjointness — a family on both sides is a protocol violation, not a warning.
62#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
63pub struct ShiftSplit {
64 /// Families used for fitting (training, distillation, calibration-parameter fitting).
65 pub fit_families: Vec<String>,
66 /// Held-out families reserved for shift/zero-shot evaluation.
67 pub eval_families: Vec<String>,
68}
69
70impl ShiftSplit {
71 /// Construct a shift split, rejecting any family present on both sides.
72 ///
73 /// # Errors
74 /// [`MetricError::ShiftSplitOverlap`] listing the overlapping families.
75 pub fn new(
76 fit_families: impl IntoIterator<Item = impl Into<String>>,
77 eval_families: impl IntoIterator<Item = impl Into<String>>,
78 ) -> Result<Self, MetricError> {
79 let fit: Vec<String> = fit_families.into_iter().map(Into::into).collect();
80 let eval: Vec<String> = eval_families.into_iter().map(Into::into).collect();
81 let mut overlap: Vec<String> = eval.iter().filter(|f| fit.contains(f)).cloned().collect();
82 overlap.sort();
83 overlap.dedup();
84 if !overlap.is_empty() {
85 return Err(MetricError::ShiftSplitOverlap { families: overlap });
86 }
87 Ok(Self {
88 fit_families: fit,
89 eval_families: eval,
90 })
91 }
92
93 /// Which side of the split a family belongs to.
94 #[must_use]
95 pub fn is_eval_family(&self, family: &str) -> bool {
96 self.eval_families.iter().any(|f| f == family)
97 }
98}
99
100/// One question field's measured metrics (one row of the macro rule).
101#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
102pub struct FieldMetrics {
103 /// Question field identifier (question id / field name in the question set).
104 pub field: String,
105 /// Workflow family this field's rows were drawn from.
106 pub family: String,
107 /// Number of observations measured.
108 pub n: usize,
109 /// Top-label ECE (equal-mass, [`GATE_BIN_COUNT`] bins). Primary for noul/choice.
110 pub ece: f64,
111 /// Multiclass Brier score.
112 pub brier: f64,
113 /// Class-wise static calibration error. Present for score fields (P1.4 leg c).
114 pub sce: Option<f64>,
115 /// Class-wise adaptive calibration error. Present for score fields.
116 pub ace: Option<f64>,
117 /// Ranked probability score. Present for score fields.
118 pub rps: Option<f64>,
119}
120
121/// Macro-averaged metric with its bootstrap CI, as reported by a gate.
122#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
123pub struct MacroMetrics {
124 /// Per-field macro mean (equal field weights).
125 pub mean: f64,
126 /// Bootstrap CI over the *field* values (fields are the resampling unit — resampling
127 /// rows would understate field-level variance).
128 pub ci: ConfidenceInterval,
129}
130
131impl MacroMetrics {
132 /// Macro-average a metric over fields, with the protocol bootstrap CI.
133 ///
134 /// # Errors
135 /// [`MetricError::EmptyInput`] if no field values are given; propagates bootstrap
136 /// errors.
137 pub fn from_fields(values: &[f64], seed: u64) -> Result<Self, MetricError> {
138 let (mean, ci) = bootstrap_ci(
139 values,
140 |v| {
141 #[allow(clippy::cast_precision_loss)]
142 let n = v.len() as f64;
143 v.iter().sum::<f64>() / n
144 },
145 GATE_BOOTSTRAP_RESAMPLES,
146 GATE_CI_LEVEL,
147 seed,
148 )?;
149 Ok(Self { mean, ci })
150 }
151
152 /// Upper CI bound — the value bound-style gate comparisons condition on.
153 #[must_use]
154 pub fn upper_bound(&self) -> f64 {
155 self.ci.high
156 }
157}
158
159/// A gate run's serializable measurement artifact.
160#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
161pub struct GateReport {
162 /// Name of the arm/subject measured (e.g. a model version, a baseline arm).
163 pub subject: String,
164 /// The split this report measured (e.g. `"deterministic"`, `"shift"`, `"zero-shot"`).
165 pub split: String,
166 /// The shift split in force for this report (fit/eval family designation).
167 pub shift_split: ShiftSplit,
168 /// Per-field measurements (the macro rule's inputs).
169 pub fields: Vec<FieldMetrics>,
170 /// Macro-averaged ECE with CI.
171 pub macro_ece: MacroMetrics,
172 /// Macro-averaged Brier with CI.
173 pub macro_brier: MacroMetrics,
174 /// Macro-averaged SCE over score fields with CI, if any score fields were measured.
175 pub macro_sce: Option<MacroMetrics>,
176 /// Macro-averaged RPS over score fields with CI, if any score fields were measured.
177 pub macro_rps: Option<MacroMetrics>,
178 /// Seed used for all bootstrap CIs in this report.
179 pub bootstrap_seed: u64,
180}
181
182impl GateReport {
183 /// Assemble a gate report from per-field measurements, applying the protocol's macro
184 /// rule and bootstrap CIs. SCE/ACE/RPS macro aggregates cover only the fields that
185 /// carry them (score fields), keeping the per-primitive split of P1.4 leg c.
186 ///
187 /// # Errors
188 /// [`MetricError::EmptyInput`] if `fields` is empty; propagates bootstrap errors.
189 pub fn assemble(
190 subject: impl Into<String>,
191 split: impl Into<String>,
192 shift_split: ShiftSplit,
193 fields: Vec<FieldMetrics>,
194 bootstrap_seed: u64,
195 ) -> Result<Self, MetricError> {
196 if fields.is_empty() {
197 return Err(MetricError::EmptyInput("gate report"));
198 }
199 let macro_ece = MacroMetrics::from_fields(
200 &fields.iter().map(|f| f.ece).collect::<Vec<_>>(),
201 bootstrap_seed,
202 )?;
203 let macro_brier = MacroMetrics::from_fields(
204 &fields.iter().map(|f| f.brier).collect::<Vec<_>>(),
205 bootstrap_seed ^ 0x5EED_0001,
206 )?;
207 let score_metric = |pick: fn(&FieldMetrics) -> Option<f64>, salt: u64| {
208 let values: Vec<f64> = fields.iter().filter_map(pick).collect();
209 if values.is_empty() {
210 return Ok(None);
211 }
212 MacroMetrics::from_fields(&values, bootstrap_seed ^ salt).map(Some)
213 };
214 let macro_sce = score_metric(|f| f.sce, 0x5EED_0002)?;
215 let macro_rps = score_metric(|f| f.rps, 0x5EED_0003)?;
216 Ok(Self {
217 subject: subject.into(),
218 split: split.into(),
219 shift_split,
220 fields,
221 macro_ece,
222 macro_brier,
223 macro_sce,
224 macro_rps,
225 bootstrap_seed,
226 })
227 }
228}