Skip to main content

hyprstream_inference/runtime/
generation_metrics.rs

1//! Generation quality metrics for self-supervised training
2//!
3//! This module provides O(1) online metrics accumulation during text generation,
4//! capturing quality signals from logits that can be used for reinforcement learning.
5//!
6//! Key metrics:
7//! - Perplexity: Model confidence (lower = more confident)
8//! - Entropy: Decision uncertainty (lower = more certain)
9//! - Entropy variance: Consistency of confidence (lower = more consistent)
10//! - Repetition ratio: N-gram repetition detection (lower = less repetitive)
11
12use serde::{Deserialize, Serialize};
13use std::collections::VecDeque;
14
15/// Size of ring buffer for n-gram detection
16const NGRAM_BUFFER_SIZE: usize = 32;
17
18/// N-gram size for repetition detection (trigrams)
19const NGRAM_SIZE: usize = 3;
20
21/// Number of hash functions for bloom filter (k=3 gives ~2% FP rate at 50% fill)
22const BLOOM_K: usize = 3;
23
24/// Bloom filter size in bits (2048 bits = 256 bytes)
25const BLOOM_BITS: usize = 2048;
26
27/// O(1) online metrics accumulator using Welford's algorithm
28///
29/// This accumulator maintains running statistics without storing per-token data,
30/// using approximately 360 bytes of memory regardless of sequence length.
31///
32/// # Example
33/// ```ignore
34/// let mut acc = GenerationMetricsAccumulator::new();
35/// for token in generated_tokens {
36///     acc.add_token(log_prob, entropy, token_id);
37/// }
38/// let metrics = acc.finalize();
39/// println!("Perplexity: {}", metrics.perplexity);
40/// ```
41#[derive(Debug, Clone)]
42pub struct GenerationMetricsAccumulator {
43    // Welford's algorithm for log_prob statistics
44    log_prob_mean: f64,
45    log_prob_m2: f64,
46
47    // Welford's algorithm for entropy statistics
48    entropy_mean: f64,
49    entropy_m2: f64,
50
51    // Token count
52    count: u64,
53
54    // Ring buffer for n-gram detection
55    recent_tokens: VecDeque<u32>,
56
57    // N-gram repetition counter
58    ngram_repeats: u32,
59
60    // Bloom filter for n-gram tracking (256 bytes = 2048 bits)
61    // Uses k=3 hash functions for ~2% false positive rate at 50% fill
62    bloom_filter: [u64; 32],
63
64    // Count of unique n-grams inserted (approximate, counts insertions not unique)
65    ngram_insert_count: u32,
66}
67
68impl Default for GenerationMetricsAccumulator {
69    fn default() -> Self {
70        Self::new()
71    }
72}
73
74impl GenerationMetricsAccumulator {
75    /// Create a new metrics accumulator
76    pub fn new() -> Self {
77        Self {
78            log_prob_mean: 0.0,
79            log_prob_m2: 0.0,
80            entropy_mean: 0.0,
81            entropy_m2: 0.0,
82            count: 0,
83            recent_tokens: VecDeque::with_capacity(NGRAM_BUFFER_SIZE),
84            ngram_repeats: 0,
85            bloom_filter: [0u64; 32],
86            ngram_insert_count: 0,
87        }
88    }
89
90    /// Add a token's metrics to the accumulator
91    ///
92    /// # Arguments
93    /// * `log_prob` - Log probability of the sampled token (negative value)
94    /// * `entropy` - Entropy of the probability distribution (in nats)
95    /// * `token_id` - The sampled token ID (for n-gram tracking)
96    ///
97    /// # Complexity
98    /// O(1) time and space per token
99    pub fn add_token(&mut self, log_prob: f32, entropy: f32, token_id: u32) {
100        self.count += 1;
101        let n = self.count as f64;
102
103        // Welford's online algorithm for log_prob
104        let log_prob_f64 = log_prob as f64;
105        let delta_lp = log_prob_f64 - self.log_prob_mean;
106        self.log_prob_mean += delta_lp / n;
107        let delta2_lp = log_prob_f64 - self.log_prob_mean;
108        self.log_prob_m2 += delta_lp * delta2_lp;
109
110        // Welford's online algorithm for entropy
111        let entropy_f64 = entropy as f64;
112        let delta_e = entropy_f64 - self.entropy_mean;
113        self.entropy_mean += delta_e / n;
114        let delta2_e = entropy_f64 - self.entropy_mean;
115        self.entropy_m2 += delta_e * delta2_e;
116
117        // N-gram tracking
118        self.recent_tokens.push_back(token_id);
119        if self.recent_tokens.len() > NGRAM_BUFFER_SIZE {
120            self.recent_tokens.pop_front();
121        }
122
123        // Check for n-gram repetition when we have enough tokens
124        if self.recent_tokens.len() >= NGRAM_SIZE {
125            let ngram_hash = self.compute_current_ngram_hash();
126
127            // Check if this n-gram might already be in the bloom filter
128            if self.bloom_contains(ngram_hash) {
129                // Likely repeat (with ~2% false positive rate at 50% fill)
130                self.ngram_repeats += 1;
131            } else {
132                // New n-gram, add to bloom filter
133                self.bloom_insert(ngram_hash);
134                self.ngram_insert_count += 1;
135            }
136        }
137    }
138
139    /// Insert a hash into the bloom filter
140    ///
141    /// Uses k=3 hash functions derived from the base hash via mixing.
142    fn bloom_insert(&mut self, hash: u64) {
143        for i in 0..BLOOM_K {
144            let bit_index = self.bloom_hash(hash, i);
145            let word_index = bit_index / 64;
146            let bit_offset = bit_index % 64;
147            self.bloom_filter[word_index] |= 1 << bit_offset;
148        }
149    }
150
151    /// Check if a hash might be in the bloom filter
152    ///
153    /// Returns true if the element might be present (with some false positive rate),
154    /// false if the element is definitely not present.
155    fn bloom_contains(&self, hash: u64) -> bool {
156        for i in 0..BLOOM_K {
157            let bit_index = self.bloom_hash(hash, i);
158            let word_index = bit_index / 64;
159            let bit_offset = bit_index % 64;
160            if (self.bloom_filter[word_index] & (1 << bit_offset)) == 0 {
161                return false;
162            }
163        }
164        true
165    }
166
167    /// Compute the i-th hash function for the bloom filter
168    ///
169    /// Uses the double hashing technique: h_i(x) = h1(x) + i * h2(x)
170    /// where h1 and h2 are derived from the input hash.
171    fn bloom_hash(&self, hash: u64, i: usize) -> usize {
172        // Split hash into two 32-bit values for double hashing
173        let h1 = (hash & 0xFFFFFFFF) as usize;
174        let h2 = ((hash >> 32) | 1) as usize; // Ensure h2 is odd for better distribution
175
176        // Double hashing: h_i(x) = (h1 + i * h2) mod m
177        (h1.wrapping_add(i.wrapping_mul(h2))) % BLOOM_BITS
178    }
179
180    /// Compute hash of the current n-gram (last NGRAM_SIZE tokens)
181    fn compute_current_ngram_hash(&self) -> u64 {
182        let len = self.recent_tokens.len();
183        if len < NGRAM_SIZE {
184            return 0;
185        }
186
187        // Simple hash combining last NGRAM_SIZE tokens
188        let mut hash: u64 = 0;
189        for i in (len - NGRAM_SIZE)..len {
190            let token = self.recent_tokens[i] as u64;
191            hash = hash.wrapping_mul(31).wrapping_add(token);
192        }
193        // Ensure non-zero for bloom filter logic
194        if hash == 0 {
195            hash = 1;
196        }
197        hash
198    }
199
200    /// Finalize and return aggregated quality metrics
201    ///
202    /// # Returns
203    /// `GenerationQualityMetrics` containing perplexity, entropy stats, and repetition ratio
204    pub fn finalize(&self) -> GenerationQualityMetrics {
205        if self.count == 0 {
206            return GenerationQualityMetrics::default();
207        }
208
209        // Perplexity = exp(-avg_log_prob)
210        let perplexity = (-self.log_prob_mean).exp() as f32;
211
212        // Average entropy
213        let avg_entropy = self.entropy_mean as f32;
214
215        // Entropy variance (Welford's finalization)
216        let entropy_variance = if self.count > 1 {
217            (self.entropy_m2 / (self.count - 1) as f64) as f32
218        } else {
219            0.0
220        };
221
222        // Repetition ratio: ngram_repeats / total_possible_ngrams
223        let total_ngrams = if self.count >= NGRAM_SIZE as u64 {
224            self.count - NGRAM_SIZE as u64 + 1
225        } else {
226            0
227        };
228        let repetition_ratio = if total_ngrams > 0 {
229            self.ngram_repeats as f32 / total_ngrams as f32
230        } else {
231            0.0
232        };
233
234        GenerationQualityMetrics {
235            perplexity,
236            avg_entropy,
237            entropy_variance,
238            repetition_ratio,
239            token_count: self.count as u32,
240        }
241    }
242
243    /// Get current token count
244    pub fn token_count(&self) -> u64 {
245        self.count
246    }
247
248    /// Check if any tokens have been added
249    pub fn is_empty(&self) -> bool {
250        self.count == 0
251    }
252}
253
254/// Aggregated quality metrics from a generation (~24 bytes)
255///
256/// These metrics can be used for:
257/// - Quality scoring in self-supervised RL
258/// - Monitoring generation quality
259/// - Detecting degraded model performance
260#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
261pub struct GenerationQualityMetrics {
262    /// Perplexity: exp(-avg_log_prob)
263    /// Lower = model is more confident in its generations
264    /// Typical range: 1.0 (perfect) to 100+ (very uncertain)
265    pub perplexity: f32,
266
267    /// Average entropy of probability distributions (in nats)
268    /// Lower = more decisive token selection
269    /// Typical range: 0.0 (deterministic) to ~10.0 (very uncertain)
270    pub avg_entropy: f32,
271
272    /// Variance of entropy across tokens
273    /// Lower = more consistent confidence throughout generation
274    pub entropy_variance: f32,
275
276    /// Ratio of repeated n-grams to total n-grams
277    /// Lower = less repetitive output
278    /// Range: 0.0 (no repeats) to 1.0 (all repeats)
279    pub repetition_ratio: f32,
280
281    /// Number of tokens generated
282    pub token_count: u32,
283}
284
285impl GenerationQualityMetrics {
286    /// Check if metrics indicate potentially problematic generation
287    pub fn is_concerning(&self) -> bool {
288        self.perplexity > 50.0 || self.repetition_ratio > 0.3 || self.entropy_variance > 2.0
289    }
290}
291
292/// Per-session metrics storage for tracking quality across conversation turns
293#[derive(Debug, Clone, Default)]
294pub struct SessionMetrics {
295    /// Quality metrics for each turn in the session
296    turn_metrics: Vec<GenerationQualityMetrics>,
297
298    /// Running sum of perplexity for session average
299    session_perplexity_sum: f64,
300
301    /// Running sum of tokens for weighted averages
302    session_token_sum: u64,
303}
304
305impl SessionMetrics {
306    /// Create a new session metrics tracker
307    pub fn new() -> Self {
308        Self::default()
309    }
310
311    /// Add metrics from a completed generation turn
312    pub fn add_turn(&mut self, metrics: GenerationQualityMetrics) {
313        self.session_perplexity_sum += metrics.perplexity as f64 * metrics.token_count as f64;
314        self.session_token_sum += metrics.token_count as u64;
315        self.turn_metrics.push(metrics);
316    }
317
318    /// Get the number of turns in this session
319    pub fn turn_count(&self) -> usize {
320        self.turn_metrics.len()
321    }
322
323    /// Get metrics for a specific turn
324    pub fn get_turn(&self, index: usize) -> Option<&GenerationQualityMetrics> {
325        self.turn_metrics.get(index)
326    }
327
328    /// Get all turn metrics
329    pub fn turns(&self) -> &[GenerationQualityMetrics] {
330        &self.turn_metrics
331    }
332
333    /// Compute session-level average perplexity (token-weighted)
334    pub fn avg_perplexity(&self) -> f32 {
335        if self.session_token_sum == 0 {
336            return 0.0;
337        }
338        (self.session_perplexity_sum / self.session_token_sum as f64) as f32
339    }
340
341    /// Get total tokens generated in this session
342    pub fn total_tokens(&self) -> u64 {
343        self.session_token_sum
344    }
345}
346
347#[cfg(test)]
348mod tests {
349    use super::*;
350
351    #[test]
352    fn test_accumulator_empty() {
353        let acc = GenerationMetricsAccumulator::new();
354        assert!(acc.is_empty());
355        let metrics = acc.finalize();
356        assert_eq!(metrics.token_count, 0);
357    }
358
359    #[test]
360    fn test_accumulator_single_token() {
361        let mut acc = GenerationMetricsAccumulator::new();
362        acc.add_token(-2.0, 1.5, 100);
363
364        let metrics = acc.finalize();
365        assert_eq!(metrics.token_count, 1);
366        // perplexity = exp(-(-2.0)) = exp(2.0) ≈ 7.39
367        assert!((metrics.perplexity - 7.389).abs() < 0.01);
368        assert!((metrics.avg_entropy - 1.5).abs() < 0.01);
369        assert_eq!(metrics.entropy_variance, 0.0); // No variance with 1 sample
370    }
371
372    #[test]
373    fn test_accumulator_multiple_tokens() {
374        let mut acc = GenerationMetricsAccumulator::new();
375
376        // Add tokens with consistent log_prob and entropy
377        for i in 0..10 {
378            acc.add_token(-1.0, 2.0, i);
379        }
380
381        let metrics = acc.finalize();
382        assert_eq!(metrics.token_count, 10);
383        // perplexity = exp(-(-1.0)) = e ≈ 2.718
384        assert!((metrics.perplexity - std::f32::consts::E).abs() < 0.01);
385        assert!((metrics.avg_entropy - 2.0).abs() < 0.01);
386        // Variance should be near 0 with identical values
387        assert!(metrics.entropy_variance < 0.01);
388    }
389
390    #[test]
391    fn test_ngram_repetition() {
392        let mut acc = GenerationMetricsAccumulator::new();
393
394        // Generate pattern: 1,2,3,1,2,3,1,2,3 (repeated trigram)
395        for _ in 0..3 {
396            acc.add_token(-1.0, 1.0, 1);
397            acc.add_token(-1.0, 1.0, 2);
398            acc.add_token(-1.0, 1.0, 3);
399        }
400
401        let metrics = acc.finalize();
402        // Should detect repetition
403        assert!(metrics.repetition_ratio > 0.0);
404    }
405
406    #[test]
407    fn test_session_metrics() {
408        let mut session = SessionMetrics::new();
409
410        // Add some turns
411        session.add_turn(GenerationQualityMetrics {
412            perplexity: 5.0,
413            avg_entropy: 2.0,
414            entropy_variance: 0.5,
415            repetition_ratio: 0.1,
416            token_count: 50,
417        });
418
419        session.add_turn(GenerationQualityMetrics {
420            perplexity: 3.0,
421            avg_entropy: 1.5,
422            entropy_variance: 0.3,
423            repetition_ratio: 0.05,
424            token_count: 100,
425        });
426
427        assert_eq!(session.turn_count(), 2);
428        assert_eq!(session.total_tokens(), 150);
429
430        // Weighted average: (5*50 + 3*100) / 150 = 550/150 ≈ 3.67
431        assert!((session.avg_perplexity() - 3.67).abs() < 0.1);
432    }
433}