Skip to main content

hyprstream_inference/training/
quality_filter.rs

1//! Quality Filter Module
2//!
3//! Provides heuristic-based output filtering using generation metrics.
4//!
5//! # Important Distinction
6//!
7//! This module is for **filtering** outputs, NOT for training signals.
8//! The metrics (perplexity, entropy, repetition) measure model confidence,
9//! NOT correctness. They are useful for:
10//!
11//! - Filtering low-quality outputs before returning to users
12//! - Ranking multiple candidate outputs
13//! - Triggering regeneration for concerning outputs
14//!
15//! They should NOT be used as training rewards because:
16//! - Confidence != correctness (a model can be confident and wrong)
17//! - Training on confidence creates circular reinforcement
18//! - This leads to mode collapse, not improvement
19//!
20//! For research-valid training, see the `ttt` module (Test-Time Training).
21
22use crate::runtime::generation_metrics::GenerationQualityMetrics;
23use serde::{Deserialize, Serialize};
24
25/// Configuration for quality filtering
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct QualityFilterConfig {
28    /// Minimum confidence threshold (0.0 to 1.0)
29    /// Confidence = 1.0 / (1.0 + perplexity)
30    /// Higher = stricter filtering
31    #[serde(default = "default_min_confidence")]
32    pub min_confidence: f32,
33
34    /// Maximum repetition ratio (0.0 to 1.0)
35    /// Lower = stricter filtering for repetitive outputs
36    #[serde(default = "default_max_repetition")]
37    pub max_repetition: f32,
38
39    /// Maximum entropy variance
40    /// Lower = stricter filtering for inconsistent outputs
41    #[serde(default = "default_max_entropy_variance")]
42    pub max_entropy_variance: f32,
43
44    /// Maximum perplexity threshold
45    /// Lower = stricter filtering
46    #[serde(default = "default_max_perplexity")]
47    pub max_perplexity: f32,
48
49    /// Whether filtering is enabled
50    #[serde(default = "default_enabled")]
51    pub enabled: bool,
52}
53
54fn default_min_confidence() -> f32 {
55    0.1 // Very permissive default
56}
57fn default_max_repetition() -> f32 {
58    0.3 // Allow up to 30% repetition
59}
60fn default_max_entropy_variance() -> f32 {
61    2.0 // Matches is_concerning() threshold
62}
63fn default_max_perplexity() -> f32 {
64    50.0 // Matches is_concerning() threshold
65}
66fn default_enabled() -> bool {
67    false // Disabled by default - opt-in feature
68}
69
70impl Default for QualityFilterConfig {
71    fn default() -> Self {
72        Self {
73            min_confidence: default_min_confidence(),
74            max_repetition: default_max_repetition(),
75            max_entropy_variance: default_max_entropy_variance(),
76            max_perplexity: default_max_perplexity(),
77            enabled: default_enabled(),
78        }
79    }
80}
81
82/// Quality filter for generation outputs
83///
84/// Filters outputs based on heuristic metrics. This is NOT a training
85/// signal - it's a quality gate for user-facing outputs.
86///
87/// # Example
88///
89/// ```ignore
90/// let filter = QualityFilter::new(QualityFilterConfig::default());
91///
92/// // After generation
93/// let metrics = accumulator.finalize();
94///
95/// if filter.should_keep(&metrics) {
96///     // Return output to user
97/// } else {
98///     // Regenerate or warn user
99/// }
100/// ```
101#[derive(Debug, Clone)]
102pub struct QualityFilter {
103    config: QualityFilterConfig,
104}
105
106impl QualityFilter {
107    /// Create a new quality filter with the given configuration
108    pub fn new(config: QualityFilterConfig) -> Self {
109        Self { config }
110    }
111
112    /// Create a filter with default permissive settings
113    pub fn permissive() -> Self {
114        Self::new(QualityFilterConfig::default())
115    }
116
117    /// Create a filter with strict settings
118    pub fn strict() -> Self {
119        Self::new(QualityFilterConfig {
120            min_confidence: 0.3,
121            max_repetition: 0.15,
122            max_entropy_variance: 1.0,
123            max_perplexity: 20.0,
124            enabled: true,
125        })
126    }
127
128    /// Check if filtering is enabled
129    pub fn is_enabled(&self) -> bool {
130        self.config.enabled
131    }
132
133    /// Determine if an output should be kept based on quality metrics
134    ///
135    /// Returns true if the output passes all quality checks.
136    pub fn should_keep(&self, metrics: &GenerationQualityMetrics) -> bool {
137        if !self.config.enabled {
138            return true;
139        }
140
141        let confidence = self.compute_confidence(metrics.perplexity);
142
143        confidence >= self.config.min_confidence
144            && metrics.repetition_ratio <= self.config.max_repetition
145            && metrics.entropy_variance <= self.config.max_entropy_variance
146            && metrics.perplexity <= self.config.max_perplexity
147    }
148
149    /// Get detailed filter result with reasons
150    pub fn evaluate(&self, metrics: &GenerationQualityMetrics) -> FilterResult {
151        if !self.config.enabled {
152            return FilterResult {
153                passed: true,
154                confidence: self.compute_confidence(metrics.perplexity),
155                reasons: vec![],
156            };
157        }
158
159        let confidence = self.compute_confidence(metrics.perplexity);
160        let mut reasons = Vec::new();
161
162        if confidence < self.config.min_confidence {
163            reasons.push(FilterReason::LowConfidence {
164                actual: confidence,
165                threshold: self.config.min_confidence,
166            });
167        }
168
169        if metrics.repetition_ratio > self.config.max_repetition {
170            reasons.push(FilterReason::HighRepetition {
171                actual: metrics.repetition_ratio,
172                threshold: self.config.max_repetition,
173            });
174        }
175
176        if metrics.entropy_variance > self.config.max_entropy_variance {
177            reasons.push(FilterReason::HighEntropyVariance {
178                actual: metrics.entropy_variance,
179                threshold: self.config.max_entropy_variance,
180            });
181        }
182
183        if metrics.perplexity > self.config.max_perplexity {
184            reasons.push(FilterReason::HighPerplexity {
185                actual: metrics.perplexity,
186                threshold: self.config.max_perplexity,
187            });
188        }
189
190        FilterResult {
191            passed: reasons.is_empty(),
192            confidence,
193            reasons,
194        }
195    }
196
197    /// Compute confidence score from perplexity
198    ///
199    /// Formula: confidence = 1.0 / (1.0 + perplexity)
200    /// Range: (0.0, 1.0] where 1.0 is perfect confidence
201    fn compute_confidence(&self, perplexity: f32) -> f32 {
202        1.0 / (1.0 + perplexity)
203    }
204}
205
206/// Result of quality filter evaluation
207#[derive(Debug, Clone)]
208pub struct FilterResult {
209    /// Whether the output passed filtering
210    pub passed: bool,
211
212    /// Computed confidence score
213    pub confidence: f32,
214
215    /// Reasons for failing (empty if passed)
216    pub reasons: Vec<FilterReason>,
217}
218
219/// Reason for filter failure
220#[derive(Debug, Clone)]
221pub enum FilterReason {
222    LowConfidence { actual: f32, threshold: f32 },
223    HighRepetition { actual: f32, threshold: f32 },
224    HighEntropyVariance { actual: f32, threshold: f32 },
225    HighPerplexity { actual: f32, threshold: f32 },
226}
227
228impl std::fmt::Display for FilterReason {
229    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
230        match self {
231            Self::LowConfidence { actual, threshold } => {
232                write!(f, "Low confidence: {actual:.3} < {threshold:.3}")
233            }
234            Self::HighRepetition { actual, threshold } => {
235                write!(f, "High repetition: {actual:.3} > {threshold:.3}")
236            }
237            Self::HighEntropyVariance { actual, threshold } => {
238                write!(f, "High entropy variance: {actual:.3} > {threshold:.3}")
239            }
240            Self::HighPerplexity { actual, threshold } => {
241                write!(f, "High perplexity: {actual:.1} > {threshold:.1}")
242            }
243        }
244    }
245}
246
247#[cfg(test)]
248mod tests {
249    use super::*;
250
251    fn make_metrics(
252        perplexity: f32,
253        repetition: f32,
254        entropy_var: f32,
255    ) -> GenerationQualityMetrics {
256        GenerationQualityMetrics {
257            perplexity,
258            avg_entropy: 2.0,
259            entropy_variance: entropy_var,
260            repetition_ratio: repetition,
261            token_count: 100,
262        }
263    }
264
265    #[test]
266    fn test_filter_disabled_passes_all() {
267        let filter = QualityFilter::new(QualityFilterConfig {
268            enabled: false,
269            ..Default::default()
270        });
271
272        // Even terrible metrics should pass when disabled
273        let bad_metrics = make_metrics(1000.0, 0.9, 10.0);
274        assert!(filter.should_keep(&bad_metrics));
275    }
276
277    #[test]
278    fn test_filter_enabled_rejects_bad() {
279        let filter = QualityFilter::strict();
280
281        // High perplexity should fail
282        let high_ppl = make_metrics(100.0, 0.1, 0.5);
283        assert!(!filter.should_keep(&high_ppl));
284
285        // High repetition should fail
286        let high_rep = make_metrics(5.0, 0.5, 0.5);
287        assert!(!filter.should_keep(&high_rep));
288
289        // High entropy variance should fail
290        let high_var = make_metrics(5.0, 0.1, 5.0);
291        assert!(!filter.should_keep(&high_var));
292    }
293
294    #[test]
295    fn test_filter_accepts_good() {
296        let filter = QualityFilter::strict();
297
298        // Good metrics should pass
299        // Perplexity 2.0 -> confidence = 1/(1+2) = 0.33 > 0.3 (min_confidence)
300        let good = make_metrics(2.0, 0.1, 0.5);
301        assert!(filter.should_keep(&good));
302    }
303
304    #[test]
305    fn test_evaluate_gives_reasons() {
306        let filter = QualityFilter::strict();
307
308        let bad = make_metrics(100.0, 0.5, 5.0);
309        let result = filter.evaluate(&bad);
310
311        assert!(!result.passed);
312        assert!(result.reasons.len() >= 2); // Multiple failures
313    }
314
315    #[test]
316    fn test_confidence_calculation() {
317        let filter = QualityFilter::permissive();
318
319        // perplexity=0 -> confidence=1.0
320        let perfect = make_metrics(0.0, 0.0, 0.0);
321        let result = filter.evaluate(&perfect);
322        assert!((result.confidence - 1.0).abs() < 0.001);
323
324        // perplexity=9 -> confidence=0.1
325        let low = make_metrics(9.0, 0.0, 0.0);
326        let result = filter.evaluate(&low);
327        assert!((result.confidence - 0.1).abs() < 0.001);
328    }
329}