Skip to main content

hyprstream_calibration/
selective.rs

1//! Selective prediction: risk–coverage analysis over confidence-thresholded answers.
2//!
3//! Abstention is a first-class answer value in the jev-1 profile (`null` = abstained),
4//! and selective accuracy is the metric that prices it: accuracy among the answers a
5//! threshold policy would have *kept*, as a function of coverage. The curve is computed
6//! exactly (every distinct threshold), not sampled.
7
8use serde::{Deserialize, Serialize};
9
10use crate::error::MetricError;
11
12/// One point on the risk–coverage curve.
13#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
14pub struct RiskCoverage {
15    /// Fraction of observations kept (confidence ≥ threshold at this point).
16    pub coverage: f64,
17    /// Accuracy among the kept observations (1 − selective risk).
18    pub accuracy: f64,
19    /// The confidence threshold realizing this operating point.
20    pub threshold: f64,
21}
22
23/// Exact risk–coverage curve: sort by confidence descending; at each distinct threshold,
24/// coverage = kept fraction and accuracy = mean correctness among kept.
25///
26/// Points are returned from lowest coverage (most selective) to highest (keep all).
27///
28/// # Errors
29/// [`MetricError::LengthMismatch`] / [`MetricError::EmptyInput`] on malformed inputs.
30pub fn risk_coverage_curve(
31    confidences: &[f64],
32    correct: &[bool],
33) -> Result<Vec<RiskCoverage>, MetricError> {
34    if confidences.len() != correct.len() {
35        return Err(MetricError::LengthMismatch {
36            left_name: "confidences",
37            left: confidences.len(),
38            right_name: "correct",
39            right: correct.len(),
40        });
41    }
42    if confidences.is_empty() {
43        return Err(MetricError::EmptyInput("risk-coverage curve"));
44    }
45    let mut order: Vec<usize> = (0..confidences.len()).collect();
46    order.sort_by(|&a, &b| f64::total_cmp(&confidences[b], &confidences[a]));
47    #[allow(clippy::cast_precision_loss)]
48    let n = confidences.len() as f64;
49    let mut kept_correct = 0usize;
50    let mut curve = Vec::with_capacity(confidences.len());
51    for (rank, &i) in order.iter().enumerate() {
52        kept_correct += usize::from(correct[i]);
53        // Emit one point per distinct threshold value.
54        let is_last = rank + 1 == order.len();
55        if !is_last && confidences[order[rank + 1]] == confidences[i] {
56            continue;
57        }
58        #[allow(clippy::cast_precision_loss)]
59        let kept = (rank + 1) as f64;
60        curve.push(RiskCoverage {
61            coverage: kept / n,
62            #[allow(clippy::cast_precision_loss)]
63            accuracy: kept_correct as f64 / kept,
64            threshold: confidences[i],
65        });
66    }
67    Ok(curve)
68}
69
70/// Accuracy at the largest coverage `<= target_coverage` (step semantics: a threshold
71/// policy realizes only the curve's own points). Returns `None` if the target is below
72/// the smallest realized coverage.
73#[must_use]
74pub fn accuracy_at_coverage(curve: &[RiskCoverage], target_coverage: f64) -> Option<f64> {
75    curve
76        .iter()
77        .filter(|p| p.coverage <= target_coverage)
78        .max_by(|a, b| f64::total_cmp(&a.coverage, &b.coverage))
79        .map(|p| p.accuracy)
80}
81
82/// Area under the risk–coverage curve (coverage on x, accuracy on y), trapezoidal over
83/// the realized points with the left edge extended to coverage 0 at the first point's
84/// accuracy. Higher is better; 1.0 is an oracle ranker.
85#[must_use]
86pub fn selective_auc(curve: &[RiskCoverage]) -> f64 {
87    if curve.is_empty() {
88        return 0.0;
89    }
90    let mut area = curve[0].accuracy * curve[0].coverage;
91    for w in curve.windows(2) {
92        let (a, b) = (w[0], w[1]);
93        area += (b.coverage - a.coverage) * (a.accuracy + b.accuracy) / 2.0;
94    }
95    area
96}