hyprstream_inference/training/
quality_filter.rs1use crate::runtime::generation_metrics::GenerationQualityMetrics;
23use serde::{Deserialize, Serialize};
24
25#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct QualityFilterConfig {
28 #[serde(default = "default_min_confidence")]
32 pub min_confidence: f32,
33
34 #[serde(default = "default_max_repetition")]
37 pub max_repetition: f32,
38
39 #[serde(default = "default_max_entropy_variance")]
42 pub max_entropy_variance: f32,
43
44 #[serde(default = "default_max_perplexity")]
47 pub max_perplexity: f32,
48
49 #[serde(default = "default_enabled")]
51 pub enabled: bool,
52}
53
54fn default_min_confidence() -> f32 {
55 0.1 }
57fn default_max_repetition() -> f32 {
58 0.3 }
60fn default_max_entropy_variance() -> f32 {
61 2.0 }
63fn default_max_perplexity() -> f32 {
64 50.0 }
66fn default_enabled() -> bool {
67 false }
69
70impl Default for QualityFilterConfig {
71 fn default() -> Self {
72 Self {
73 min_confidence: default_min_confidence(),
74 max_repetition: default_max_repetition(),
75 max_entropy_variance: default_max_entropy_variance(),
76 max_perplexity: default_max_perplexity(),
77 enabled: default_enabled(),
78 }
79 }
80}
81
82#[derive(Debug, Clone)]
102pub struct QualityFilter {
103 config: QualityFilterConfig,
104}
105
106impl QualityFilter {
107 pub fn new(config: QualityFilterConfig) -> Self {
109 Self { config }
110 }
111
112 pub fn permissive() -> Self {
114 Self::new(QualityFilterConfig::default())
115 }
116
117 pub fn strict() -> Self {
119 Self::new(QualityFilterConfig {
120 min_confidence: 0.3,
121 max_repetition: 0.15,
122 max_entropy_variance: 1.0,
123 max_perplexity: 20.0,
124 enabled: true,
125 })
126 }
127
128 pub fn is_enabled(&self) -> bool {
130 self.config.enabled
131 }
132
133 pub fn should_keep(&self, metrics: &GenerationQualityMetrics) -> bool {
137 if !self.config.enabled {
138 return true;
139 }
140
141 let confidence = self.compute_confidence(metrics.perplexity);
142
143 confidence >= self.config.min_confidence
144 && metrics.repetition_ratio <= self.config.max_repetition
145 && metrics.entropy_variance <= self.config.max_entropy_variance
146 && metrics.perplexity <= self.config.max_perplexity
147 }
148
149 pub fn evaluate(&self, metrics: &GenerationQualityMetrics) -> FilterResult {
151 if !self.config.enabled {
152 return FilterResult {
153 passed: true,
154 confidence: self.compute_confidence(metrics.perplexity),
155 reasons: vec![],
156 };
157 }
158
159 let confidence = self.compute_confidence(metrics.perplexity);
160 let mut reasons = Vec::new();
161
162 if confidence < self.config.min_confidence {
163 reasons.push(FilterReason::LowConfidence {
164 actual: confidence,
165 threshold: self.config.min_confidence,
166 });
167 }
168
169 if metrics.repetition_ratio > self.config.max_repetition {
170 reasons.push(FilterReason::HighRepetition {
171 actual: metrics.repetition_ratio,
172 threshold: self.config.max_repetition,
173 });
174 }
175
176 if metrics.entropy_variance > self.config.max_entropy_variance {
177 reasons.push(FilterReason::HighEntropyVariance {
178 actual: metrics.entropy_variance,
179 threshold: self.config.max_entropy_variance,
180 });
181 }
182
183 if metrics.perplexity > self.config.max_perplexity {
184 reasons.push(FilterReason::HighPerplexity {
185 actual: metrics.perplexity,
186 threshold: self.config.max_perplexity,
187 });
188 }
189
190 FilterResult {
191 passed: reasons.is_empty(),
192 confidence,
193 reasons,
194 }
195 }
196
197 fn compute_confidence(&self, perplexity: f32) -> f32 {
202 1.0 / (1.0 + perplexity)
203 }
204}
205
206#[derive(Debug, Clone)]
208pub struct FilterResult {
209 pub passed: bool,
211
212 pub confidence: f32,
214
215 pub reasons: Vec<FilterReason>,
217}
218
219#[derive(Debug, Clone)]
221pub enum FilterReason {
222 LowConfidence { actual: f32, threshold: f32 },
223 HighRepetition { actual: f32, threshold: f32 },
224 HighEntropyVariance { actual: f32, threshold: f32 },
225 HighPerplexity { actual: f32, threshold: f32 },
226}
227
228impl std::fmt::Display for FilterReason {
229 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
230 match self {
231 Self::LowConfidence { actual, threshold } => {
232 write!(f, "Low confidence: {actual:.3} < {threshold:.3}")
233 }
234 Self::HighRepetition { actual, threshold } => {
235 write!(f, "High repetition: {actual:.3} > {threshold:.3}")
236 }
237 Self::HighEntropyVariance { actual, threshold } => {
238 write!(f, "High entropy variance: {actual:.3} > {threshold:.3}")
239 }
240 Self::HighPerplexity { actual, threshold } => {
241 write!(f, "High perplexity: {actual:.1} > {threshold:.1}")
242 }
243 }
244 }
245}
246
247#[cfg(test)]
248mod tests {
249 use super::*;
250
251 fn make_metrics(
252 perplexity: f32,
253 repetition: f32,
254 entropy_var: f32,
255 ) -> GenerationQualityMetrics {
256 GenerationQualityMetrics {
257 perplexity,
258 avg_entropy: 2.0,
259 entropy_variance: entropy_var,
260 repetition_ratio: repetition,
261 token_count: 100,
262 }
263 }
264
265 #[test]
266 fn test_filter_disabled_passes_all() {
267 let filter = QualityFilter::new(QualityFilterConfig {
268 enabled: false,
269 ..Default::default()
270 });
271
272 let bad_metrics = make_metrics(1000.0, 0.9, 10.0);
274 assert!(filter.should_keep(&bad_metrics));
275 }
276
277 #[test]
278 fn test_filter_enabled_rejects_bad() {
279 let filter = QualityFilter::strict();
280
281 let high_ppl = make_metrics(100.0, 0.1, 0.5);
283 assert!(!filter.should_keep(&high_ppl));
284
285 let high_rep = make_metrics(5.0, 0.5, 0.5);
287 assert!(!filter.should_keep(&high_rep));
288
289 let high_var = make_metrics(5.0, 0.1, 5.0);
291 assert!(!filter.should_keep(&high_var));
292 }
293
294 #[test]
295 fn test_filter_accepts_good() {
296 let filter = QualityFilter::strict();
297
298 let good = make_metrics(2.0, 0.1, 0.5);
301 assert!(filter.should_keep(&good));
302 }
303
304 #[test]
305 fn test_evaluate_gives_reasons() {
306 let filter = QualityFilter::strict();
307
308 let bad = make_metrics(100.0, 0.5, 5.0);
309 let result = filter.evaluate(&bad);
310
311 assert!(!result.passed);
312 assert!(result.reasons.len() >= 2); }
314
315 #[test]
316 fn test_confidence_calculation() {
317 let filter = QualityFilter::permissive();
318
319 let perfect = make_metrics(0.0, 0.0, 0.0);
321 let result = filter.evaluate(&perfect);
322 assert!((result.confidence - 1.0).abs() < 0.001);
323
324 let low = make_metrics(9.0, 0.0, 0.0);
326 let result = filter.evaluate(&low);
327 assert!((result.confidence - 0.1).abs() < 0.001);
328 }
329}