Skip to main content

hyprstream_calibration/
metrics.rs

1//! Calibration and proper-scoring metrics over probability vectors.
2//!
3//! All functions take parallel slices: one probability distribution per observation plus
4//! the realized label index. A `noul` answer is the 2-wide distribution
5//! `[P(false), P(true)]`; `choice` and `score` distributions are as wide as their
6//! option/level cardinality. Width may vary across observations (score levels are
7//! runtime-declared), except where a class-wise metric requires a shared class axis
8//! ([`sce`], [`ace`]).
9//!
10//! - **ECE** ([`ece`]) — top-label, over the pinned equal-mass protocol binning.
11//! - **Brier** ([`brier`]) — multiclass Brier score `Σₖ (pₖ − yₖ)²`.
12//! - **NLL** ([`nll`]) — negative log-likelihood of the realized label.
13//! - **RPS** ([`rps`]) — ranked probability score on the cumulative distribution,
14//!   normalized by `K − 1`; the proper ordinal metric for `score` (S6b2).
15//! - **SCE / ACE** ([`sce`], [`ace`]) — class-wise calibration error, static
16//!   (equal-width) vs adaptive (equal-mass) binning per Nixon et al. 2019; the metrics
17//!   that see ordinal tail miscalibration which top-label ECE misses (ORCU,
18//!   arXiv:2410.15658).
19//! - **Expected value** ([`expected_value`]) — the probability-weighted score readout
20//!   `Σₖ k·pₖ`, exposed so re-fits can be validated against weighted-value error
21//!   (S6b2 value guardrail: temperature moves the expectation without moving the mode).
22
23use crate::bins::{Bin, Binning};
24use crate::error::MetricError;
25
26/// Validate one observation: a non-empty normalized distribution and an in-range label.
27fn check_observation(probs: &[f64], label: usize, index: usize) -> Result<(), MetricError> {
28    if probs.is_empty() {
29        return Err(MetricError::InvalidDistribution {
30            index,
31            reason: "empty distribution".to_owned(),
32        });
33    }
34    let mut sum = 0.0;
35    for &p in probs {
36        if !p.is_finite() || p < 0.0 {
37            return Err(MetricError::InvalidDistribution {
38                index,
39                reason: format!("non-finite or negative probability {p}"),
40            });
41        }
42        sum += p;
43    }
44    // Consumer-side tolerance, matching the jev-1 profile decision D5.
45    if (sum - 1.0).abs() > 1e-2 {
46        return Err(MetricError::InvalidDistribution {
47            index,
48            reason: format!(
49                "probabilities sum to {sum}, outside the consumer tolerance |Σp − 1| ≤ 1e-2"
50            ),
51        });
52    }
53    if label >= probs.len() {
54        return Err(MetricError::InvalidLabel {
55            index,
56            label,
57            width: probs.len(),
58        });
59    }
60    Ok(())
61}
62
63/// Validate the parallel inputs and return the number of observations.
64fn check_inputs(probs: &[&[f64]], labels: &[usize]) -> Result<usize, MetricError> {
65    if probs.len() != labels.len() {
66        return Err(MetricError::LengthMismatch {
67            left_name: "probs",
68            left: probs.len(),
69            right_name: "labels",
70            right: labels.len(),
71        });
72    }
73    if probs.is_empty() {
74        return Err(MetricError::EmptyInput("metric inputs"));
75    }
76    for (i, (&p, &y)) in probs.iter().zip(labels).enumerate() {
77        check_observation(p, y, i)?;
78    }
79    Ok(probs.len())
80}
81
82/// Argmax index with earliest-index tie-breaking (jev-1 profile decision D6).
83#[must_use]
84pub fn argmax(probs: &[f64]) -> usize {
85    let mut best = 0usize;
86    for (i, &p) in probs.iter().enumerate().skip(1) {
87        if p > probs[best] {
88            best = i;
89        }
90    }
91    best
92}
93
94/// Top-label confidence and correctness per observation: `(max_k p_k, argmax == label)`.
95fn confidence_pairs(probs: &[&[f64]], labels: &[usize]) -> (Vec<f64>, Vec<bool>) {
96    probs
97        .iter()
98        .zip(labels)
99        .map(|(p, &y)| {
100            let top = argmax(p);
101            (p[top], top == y)
102        })
103        .unzip()
104}
105
106/// Top-label expected calibration error over `binning`.
107///
108/// `ECE = Σ_b (|B_b| / n) · |acc(B_b) − conf(B_b)|`.
109#[must_use]
110pub fn ece_from_bins(bins: &[Bin], n: usize) -> f64 {
111    if n == 0 {
112        return 0.0;
113    }
114    #[allow(clippy::cast_precision_loss)]
115    let n = n as f64;
116    bins.iter()
117        .map(|b| {
118            #[allow(clippy::cast_precision_loss)]
119            let weight = b.count as f64 / n;
120            weight * (b.mean_outcome - b.mean_confidence).abs()
121        })
122        .sum()
123}
124
125/// Top-label ECE under the pinned protocol binning (`n_bins` equal-mass bins).
126///
127/// # Errors
128/// Propagates validation and binning errors.
129pub fn ece(probs: &[&[f64]], labels: &[usize], n_bins: usize) -> Result<f64, MetricError> {
130    let n = check_inputs(probs, labels)?;
131    let (confs, correct) = confidence_pairs(probs, labels);
132    let binning = Binning::equal_mass(&confs, n_bins.min(n))?;
133    let bins = binning.summarize(&confs, &correct)?;
134    Ok(ece_from_bins(&bins, n))
135}
136
137/// Reliability curve over the top-label confidence axis.
138///
139/// # Errors
140/// Propagates validation and binning errors.
141pub fn reliability_curve(
142    probs: &[&[f64]],
143    labels: &[usize],
144    n_bins: usize,
145) -> Result<Vec<Bin>, MetricError> {
146    let n = check_inputs(probs, labels)?;
147    let (confs, correct) = confidence_pairs(probs, labels);
148    let binning = Binning::equal_mass(&confs, n_bins.min(n))?;
149    binning.summarize(&confs, &correct)
150}
151
152/// Mean multiclass Brier score `Σₖ (pₖ − yₖ)²` (lower is better; 0 is perfect).
153///
154/// # Errors
155/// Propagates validation errors.
156pub fn brier(probs: &[&[f64]], labels: &[usize]) -> Result<f64, MetricError> {
157    let n = check_inputs(probs, labels)?;
158    let total: f64 = probs
159        .iter()
160        .zip(labels)
161        .map(|(p, &y)| {
162            p.iter()
163                .enumerate()
164                .map(|(k, &pk)| {
165                    let target = f64::from(k == y);
166                    (pk - target).powi(2)
167                })
168                .sum::<f64>()
169        })
170        .sum();
171    #[allow(clippy::cast_precision_loss)]
172    Ok(total / n as f64)
173}
174
175/// Mean negative log-likelihood of the realized label.
176///
177/// # Errors
178/// Propagates validation errors. Returns [`MetricError::InvalidDistribution`] if a
179/// realized label was assigned exactly zero probability (infinite NLL).
180pub fn nll(probs: &[&[f64]], labels: &[usize]) -> Result<f64, MetricError> {
181    let n = check_inputs(probs, labels)?;
182    let mut total = 0.0;
183    for (i, (&p, &y)) in probs.iter().zip(labels).enumerate() {
184        let py = p[y];
185        if py <= 0.0 {
186            return Err(MetricError::InvalidDistribution {
187                index: i,
188                reason: "realized label has zero probability (infinite NLL)".to_owned(),
189            });
190        }
191        total -= py.ln();
192    }
193    #[allow(clippy::cast_precision_loss)]
194    Ok(total / n as f64)
195}
196
197/// Mean ranked probability score over the cumulative distribution, normalized by
198/// `K − 1` so scores are comparable across runtime-declared level cardinalities:
199/// `RPS = Σₖ₌₀..K₋₂ (Fₖ − 1{y ≤ k})² / (K − 1)` (lower is better; 0 is perfect).
200///
201/// This is the primary proper scoring rule for the ordinal `score` primitive (S6b2).
202/// For width-2 distributions (noul) it equals half the multiclass Brier score.
203///
204/// # Errors
205/// Propagates validation errors.
206pub fn rps(probs: &[&[f64]], labels: &[usize]) -> Result<f64, MetricError> {
207    let n = check_inputs(probs, labels)?;
208    let total: f64 = probs
209        .iter()
210        .zip(labels)
211        .map(|(p, &y)| rps_single(p, y))
212        .sum();
213    #[allow(clippy::cast_precision_loss)]
214    Ok(total / n as f64)
215}
216
217/// RPS of a single observation (normalized by `K − 1`; `K = 1` scores 0).
218#[must_use]
219pub fn rps_single(probs: &[f64], label: usize) -> f64 {
220    let k = probs.len();
221    if k < 2 {
222        return 0.0;
223    }
224    let mut cdf = 0.0;
225    let mut acc = 0.0;
226    for (i, &p) in probs.iter().enumerate().take(k - 1) {
227        cdf += p;
228        let target = f64::from(label <= i);
229        acc += (cdf - target).powi(2);
230    }
231    #[allow(clippy::cast_precision_loss)]
232    let denom = (k - 1) as f64;
233    acc / denom
234}
235
236/// Probability-weighted value of an ordinal distribution over level indices:
237/// `E = Σₖ k·pₖ` (level values default to their indices per the jev-1 profile).
238///
239/// A convenience readout, not a trust object — the served contract is the calibrated
240/// distribution plus a conformal interval (S6b2).
241#[must_use]
242pub fn expected_value(probs: &[f64]) -> f64 {
243    probs
244        .iter()
245        .enumerate()
246        .map(|(k, &p)| {
247            #[allow(clippy::cast_precision_loss)]
248            let v = k as f64;
249            v * p
250        })
251        .sum()
252}
253
254/// Class-wise calibration error with a shared binning over each class's confidence axis.
255///
256/// For every class `k`, treats `p_k` as the confidence and `1{label = k}` as the outcome,
257/// bins them, and accumulates the weighted per-bin gap. Returns the unweighted mean over
258/// classes. Requires a shared class axis (all distributions the same width).
259fn classwise_ce(
260    probs: &[&[f64]],
261    labels: &[usize],
262    n_bins: usize,
263    binning: fn(&[f64], usize) -> Result<Binning, MetricError>,
264) -> Result<f64, MetricError> {
265    let n = check_inputs(probs, labels)?;
266    let width = probs[0].len();
267    for (i, p) in probs.iter().enumerate() {
268        if p.len() != width {
269            return Err(MetricError::InvalidDistribution {
270                index: i,
271                reason: format!(
272                    "class-wise metrics require a shared class axis: width {} != {}",
273                    p.len(),
274                    width
275                ),
276            });
277        }
278    }
279    let mut total = 0.0;
280    for k in 0..width {
281        let confs: Vec<f64> = probs.iter().map(|p| p[k]).collect();
282        let outcomes: Vec<bool> = labels.iter().map(|&y| y == k).collect();
283        let b = binning(&confs, n_bins.min(n))?;
284        let bins = b.summarize(&confs, &outcomes)?;
285        total += ece_from_bins(&bins, n);
286    }
287    #[allow(clippy::cast_precision_loss)]
288    Ok(total / width as f64)
289}
290
291/// Static calibration error — class-wise, equal-width bins (Nixon et al. 2019).
292///
293/// # Errors
294/// Propagates validation and binning errors.
295pub fn sce(probs: &[&[f64]], labels: &[usize], n_bins: usize) -> Result<f64, MetricError> {
296    classwise_ce(probs, labels, n_bins, |confs, b| {
297        let _ = confs;
298        Ok(Binning::equal_width(b))
299    })
300}
301
302/// Adaptive calibration error — class-wise, equal-mass bins (Nixon et al. 2019).
303///
304/// # Errors
305/// Propagates validation and binning errors.
306pub fn ace(probs: &[&[f64]], labels: &[usize], n_bins: usize) -> Result<f64, MetricError> {
307    classwise_ce(probs, labels, n_bins, Binning::equal_mass)
308}