1use crate::bins::{Bin, Binning};
24use crate::error::MetricError;
25
26fn check_observation(probs: &[f64], label: usize, index: usize) -> Result<(), MetricError> {
28 if probs.is_empty() {
29 return Err(MetricError::InvalidDistribution {
30 index,
31 reason: "empty distribution".to_owned(),
32 });
33 }
34 let mut sum = 0.0;
35 for &p in probs {
36 if !p.is_finite() || p < 0.0 {
37 return Err(MetricError::InvalidDistribution {
38 index,
39 reason: format!("non-finite or negative probability {p}"),
40 });
41 }
42 sum += p;
43 }
44 if (sum - 1.0).abs() > 1e-2 {
46 return Err(MetricError::InvalidDistribution {
47 index,
48 reason: format!(
49 "probabilities sum to {sum}, outside the consumer tolerance |Σp − 1| ≤ 1e-2"
50 ),
51 });
52 }
53 if label >= probs.len() {
54 return Err(MetricError::InvalidLabel {
55 index,
56 label,
57 width: probs.len(),
58 });
59 }
60 Ok(())
61}
62
63fn check_inputs(probs: &[&[f64]], labels: &[usize]) -> Result<usize, MetricError> {
65 if probs.len() != labels.len() {
66 return Err(MetricError::LengthMismatch {
67 left_name: "probs",
68 left: probs.len(),
69 right_name: "labels",
70 right: labels.len(),
71 });
72 }
73 if probs.is_empty() {
74 return Err(MetricError::EmptyInput("metric inputs"));
75 }
76 for (i, (&p, &y)) in probs.iter().zip(labels).enumerate() {
77 check_observation(p, y, i)?;
78 }
79 Ok(probs.len())
80}
81
82#[must_use]
84pub fn argmax(probs: &[f64]) -> usize {
85 let mut best = 0usize;
86 for (i, &p) in probs.iter().enumerate().skip(1) {
87 if p > probs[best] {
88 best = i;
89 }
90 }
91 best
92}
93
94fn confidence_pairs(probs: &[&[f64]], labels: &[usize]) -> (Vec<f64>, Vec<bool>) {
96 probs
97 .iter()
98 .zip(labels)
99 .map(|(p, &y)| {
100 let top = argmax(p);
101 (p[top], top == y)
102 })
103 .unzip()
104}
105
106#[must_use]
110pub fn ece_from_bins(bins: &[Bin], n: usize) -> f64 {
111 if n == 0 {
112 return 0.0;
113 }
114 #[allow(clippy::cast_precision_loss)]
115 let n = n as f64;
116 bins.iter()
117 .map(|b| {
118 #[allow(clippy::cast_precision_loss)]
119 let weight = b.count as f64 / n;
120 weight * (b.mean_outcome - b.mean_confidence).abs()
121 })
122 .sum()
123}
124
125pub fn ece(probs: &[&[f64]], labels: &[usize], n_bins: usize) -> Result<f64, MetricError> {
130 let n = check_inputs(probs, labels)?;
131 let (confs, correct) = confidence_pairs(probs, labels);
132 let binning = Binning::equal_mass(&confs, n_bins.min(n))?;
133 let bins = binning.summarize(&confs, &correct)?;
134 Ok(ece_from_bins(&bins, n))
135}
136
137pub fn reliability_curve(
142 probs: &[&[f64]],
143 labels: &[usize],
144 n_bins: usize,
145) -> Result<Vec<Bin>, MetricError> {
146 let n = check_inputs(probs, labels)?;
147 let (confs, correct) = confidence_pairs(probs, labels);
148 let binning = Binning::equal_mass(&confs, n_bins.min(n))?;
149 binning.summarize(&confs, &correct)
150}
151
152pub fn brier(probs: &[&[f64]], labels: &[usize]) -> Result<f64, MetricError> {
157 let n = check_inputs(probs, labels)?;
158 let total: f64 = probs
159 .iter()
160 .zip(labels)
161 .map(|(p, &y)| {
162 p.iter()
163 .enumerate()
164 .map(|(k, &pk)| {
165 let target = f64::from(k == y);
166 (pk - target).powi(2)
167 })
168 .sum::<f64>()
169 })
170 .sum();
171 #[allow(clippy::cast_precision_loss)]
172 Ok(total / n as f64)
173}
174
175pub fn nll(probs: &[&[f64]], labels: &[usize]) -> Result<f64, MetricError> {
181 let n = check_inputs(probs, labels)?;
182 let mut total = 0.0;
183 for (i, (&p, &y)) in probs.iter().zip(labels).enumerate() {
184 let py = p[y];
185 if py <= 0.0 {
186 return Err(MetricError::InvalidDistribution {
187 index: i,
188 reason: "realized label has zero probability (infinite NLL)".to_owned(),
189 });
190 }
191 total -= py.ln();
192 }
193 #[allow(clippy::cast_precision_loss)]
194 Ok(total / n as f64)
195}
196
197pub fn rps(probs: &[&[f64]], labels: &[usize]) -> Result<f64, MetricError> {
207 let n = check_inputs(probs, labels)?;
208 let total: f64 = probs
209 .iter()
210 .zip(labels)
211 .map(|(p, &y)| rps_single(p, y))
212 .sum();
213 #[allow(clippy::cast_precision_loss)]
214 Ok(total / n as f64)
215}
216
217#[must_use]
219pub fn rps_single(probs: &[f64], label: usize) -> f64 {
220 let k = probs.len();
221 if k < 2 {
222 return 0.0;
223 }
224 let mut cdf = 0.0;
225 let mut acc = 0.0;
226 for (i, &p) in probs.iter().enumerate().take(k - 1) {
227 cdf += p;
228 let target = f64::from(label <= i);
229 acc += (cdf - target).powi(2);
230 }
231 #[allow(clippy::cast_precision_loss)]
232 let denom = (k - 1) as f64;
233 acc / denom
234}
235
236#[must_use]
242pub fn expected_value(probs: &[f64]) -> f64 {
243 probs
244 .iter()
245 .enumerate()
246 .map(|(k, &p)| {
247 #[allow(clippy::cast_precision_loss)]
248 let v = k as f64;
249 v * p
250 })
251 .sum()
252}
253
254fn classwise_ce(
260 probs: &[&[f64]],
261 labels: &[usize],
262 n_bins: usize,
263 binning: fn(&[f64], usize) -> Result<Binning, MetricError>,
264) -> Result<f64, MetricError> {
265 let n = check_inputs(probs, labels)?;
266 let width = probs[0].len();
267 for (i, p) in probs.iter().enumerate() {
268 if p.len() != width {
269 return Err(MetricError::InvalidDistribution {
270 index: i,
271 reason: format!(
272 "class-wise metrics require a shared class axis: width {} != {}",
273 p.len(),
274 width
275 ),
276 });
277 }
278 }
279 let mut total = 0.0;
280 for k in 0..width {
281 let confs: Vec<f64> = probs.iter().map(|p| p[k]).collect();
282 let outcomes: Vec<bool> = labels.iter().map(|&y| y == k).collect();
283 let b = binning(&confs, n_bins.min(n))?;
284 let bins = b.summarize(&confs, &outcomes)?;
285 total += ece_from_bins(&bins, n);
286 }
287 #[allow(clippy::cast_precision_loss)]
288 Ok(total / width as f64)
289}
290
291pub fn sce(probs: &[&[f64]], labels: &[usize], n_bins: usize) -> Result<f64, MetricError> {
296 classwise_ce(probs, labels, n_bins, |confs, b| {
297 let _ = confs;
298 Ok(Binning::equal_width(b))
299 })
300}
301
302pub fn ace(probs: &[&[f64]], labels: &[usize], n_bins: usize) -> Result<f64, MetricError> {
307 classwise_ce(probs, labels, n_bins, Binning::equal_mass)
308}