hyprstream_inference/runtime/
tensor_sampling.rs1use anyhow::Result;
9use std::collections::HashSet;
10use tch::{Device, Tensor};
11
12#[derive(Clone)]
18pub struct TensorSampler {
19 device: Device, }
21
22impl TensorSampler {
23 pub fn new(device: Device) -> Self {
24 Self { device }
25 }
26
27 pub fn set_seed(seed: u64) {
30 tch::manual_seed(seed as i64);
32 tracing::info!("Set PyTorch random seed to {}", seed);
33 }
34
35 pub fn sample_token(
37 &self,
38 logits_tensor: &Tensor,
39 temperature: f32,
40 top_p: f32,
41 top_k: Option<usize>,
42 repeat_penalty: f32,
43 previous_tokens: &[i64],
44 ) -> Result<usize> {
45 let logits = self.squeeze_logits(logits_tensor);
46 let penalized_logits =
47 self.apply_repetition_penalty(logits, repeat_penalty, previous_tokens)?;
48 self.sample_from_logits(penalized_logits, temperature, top_p, top_k)
49 }
50
51 #[allow(clippy::too_many_arguments)]
57 pub fn sample_token_with_penalty_exemptions(
58 &self,
59 logits_tensor: &Tensor,
60 temperature: f32,
61 top_p: f32,
62 top_k: Option<usize>,
63 repeat_penalty: f32,
64 previous_tokens: &[i64],
65 exempt_tokens: &HashSet<i64>,
66 ) -> Result<usize> {
67 let logits = self.squeeze_logits(logits_tensor);
68 let penalized_logits = self.apply_repetition_penalty_tiered(
69 logits,
70 repeat_penalty,
71 previous_tokens,
72 exempt_tokens,
73 )?;
74 self.sample_from_logits(penalized_logits, temperature, top_p, top_k)
75 }
76
77 fn squeeze_logits(&self, logits_tensor: &Tensor) -> Tensor {
81 let logits = if logits_tensor.dim() > 1 {
82 let shape = logits_tensor.size();
83 if shape.len() == 3 {
84 logits_tensor.select(1, shape[1] - 1).squeeze_dim(0)
86 } else if shape.len() == 2 {
87 logits_tensor.squeeze_dim(0)
88 } else {
89 logits_tensor.shallow_clone()
90 }
91 } else {
92 logits_tensor.shallow_clone()
93 }
94 .to_device(self.device);
95
96 if logits.dim() != 1 {
97 tracing::warn!("Logits not 1D after squeeze: shape={:?}", logits.size());
98 }
99 logits
100 }
101
102 fn sample_from_logits(
105 &self,
106 penalized_logits: Tensor,
107 temperature: f32,
108 top_p: f32,
109 top_k: Option<usize>,
110 ) -> Result<usize> {
111 if temperature <= 0.01 || temperature.is_nan() || temperature.is_infinite() {
113 let vocab_size = penalized_logits.size()[0] as usize;
114 let (_, max_idx) = penalized_logits.max_dim(0, false);
115 let token_id = max_idx.int64_value(&[]) as usize;
116 if token_id >= vocab_size {
117 tracing::error!("Greedy: token_id {} >= vocab_size {}", token_id, vocab_size);
118 return Ok(0);
119 }
120 return Ok(token_id);
121 }
122
123 let filtered_logits = if let Some(k) = top_k {
125 self.apply_top_k_to_logits(&penalized_logits, k)?
126 } else {
127 penalized_logits
128 };
129
130 let scaled_logits = if (temperature - 1.0).abs() < 1e-6 {
132 filtered_logits
133 } else {
134 filtered_logits / (temperature as f64)
135 };
136
137 let probs = self.softmax_stable(&scaled_logits)?;
139
140 let final_probs = if top_p < 1.0 {
142 self.apply_top_p(&probs, top_p)?
143 } else {
144 probs
145 };
146
147 self.multinomial_sample(&final_probs)
148 }
149
150 fn apply_repetition_penalty(
157 &self,
158 logits: Tensor,
159 repeat_penalty: f32,
160 previous_tokens: &[i64],
161 ) -> Result<Tensor> {
162 if repeat_penalty <= 0.0
163 || (repeat_penalty - 1.0).abs() < 1e-6
164 || previous_tokens.is_empty()
165 {
166 return Ok(logits);
167 }
168
169 let vocab_size = logits.size()[0] as usize;
170 let logits_kind = logits.kind();
171
172 let mut seen = HashSet::new();
174 let unique_ids: Vec<usize> = previous_tokens
175 .iter()
176 .rev()
177 .filter_map(|&tid| {
178 let uid = tid as usize;
179 (tid >= 0 && uid < vocab_size && seen.insert(uid)).then_some(uid)
180 })
181 .collect();
182
183 if unique_ids.is_empty() {
184 return Ok(logits);
185 }
186
187 let mut result = logits.contiguous();
188 self.apply_penalty_to_indices(&mut result, &unique_ids, repeat_penalty, logits_kind)?;
189 Ok(result)
190 }
191
192 fn apply_repetition_penalty_tiered(
198 &self,
199 logits: Tensor,
200 repeat_penalty: f32,
201 previous_tokens: &[i64],
202 exempt_tokens: &HashSet<i64>,
203 ) -> Result<Tensor> {
204 if repeat_penalty <= 0.0
205 || (repeat_penalty - 1.0).abs() < 1e-6
206 || previous_tokens.is_empty()
207 {
208 return Ok(logits);
209 }
210
211 let vocab_size = logits.size()[0] as usize;
212 let logits_kind = logits.kind();
213
214 let mut full_penalty_ids = Vec::new();
216 let mut reduced_penalty_ids = Vec::new();
217 let mut seen = HashSet::new();
218
219 for &token_id in previous_tokens.iter().rev() {
220 let uid = token_id as usize;
221 if token_id >= 0 && uid < vocab_size && seen.insert(uid) {
222 if exempt_tokens.contains(&token_id) {
223 reduced_penalty_ids.push(uid);
224 } else {
225 full_penalty_ids.push(uid);
226 }
227 }
228 }
229
230 if full_penalty_ids.is_empty() && reduced_penalty_ids.is_empty() {
231 return Ok(logits);
232 }
233
234 let mut result = logits.contiguous();
235
236 if !full_penalty_ids.is_empty() {
237 self.apply_penalty_to_indices(
238 &mut result,
239 &full_penalty_ids,
240 repeat_penalty,
241 logits_kind,
242 )?;
243 }
244
245 if !reduced_penalty_ids.is_empty() {
246 let reduced = repeat_penalty.sqrt();
248 self.apply_penalty_to_indices(&mut result, &reduced_penalty_ids, reduced, logits_kind)?;
249 }
250
251 Ok(result)
252 }
253
254 fn apply_penalty_to_indices(
259 &self,
260 result: &mut Tensor,
261 token_ids: &[usize],
262 penalty: f32,
263 logits_kind: tch::Kind,
264 ) -> Result<()> {
265 let indices_vec: Vec<i64> = token_ids.iter().map(|&id| id as i64).collect();
266 let indices = Tensor::from_slice(&indices_vec).to_device(self.device);
267 let current_logits = result.index_select(0, &indices);
268
269 let penalty_tensor = Tensor::from_slice(&[penalty])
270 .to_device(self.device)
271 .to_kind(logits_kind);
272 let penalties =
273 Tensor::ones([indices_vec.len() as i64], (logits_kind, self.device)) * penalty_tensor;
274
275 let penalized_positive = ¤t_logits / &penalties;
276 let penalized_negative = ¤t_logits * &penalties;
277
278 let positive_mask = current_logits.gt(0.0).to_kind(logits_kind);
279 let negative_mask = Tensor::ones_like(&positive_mask) - &positive_mask;
280
281 let new_values =
282 &penalized_positive * &positive_mask + &penalized_negative * &negative_mask;
283
284 let _ = result.index_put_(&[Some(indices)], &new_values, false);
285 Ok(())
286 }
287
288 fn softmax_stable(&self, logits: &Tensor) -> Result<Tensor> {
296 let logits_fp32 = if logits.kind() != tch::Kind::Float {
297 logits.to_kind(tch::Kind::Float)
298 } else {
299 logits.shallow_clone()
300 };
301 Ok(logits_fp32.softmax(-1, tch::Kind::Float))
303 }
304
305 fn apply_top_k_to_logits(&self, logits: &Tensor, k: usize) -> Result<Tensor> {
308 let vocab_size = logits.size()[0] as usize;
309
310 if k >= vocab_size {
311 return Ok(logits.shallow_clone());
312 }
313
314 let (top_values, top_indices) = logits.topk(k as i64, -1, true, true);
316
317 let mut filtered_logits = Tensor::full(
319 [vocab_size as i64],
320 f64::NEG_INFINITY,
321 (logits.kind(), self.device),
322 );
323
324 let _ = filtered_logits.index_put_(&[Some(top_indices)], &top_values, false);
326
327 Ok(filtered_logits)
328 }
329
330 fn apply_top_p(&self, probs: &Tensor, top_p: f32) -> Result<Tensor> {
333 if top_p <= 0.0 || top_p >= 1.0 {
334 return Ok(probs.shallow_clone());
335 }
336
337 let (sorted_probs, sorted_indices) = probs.sort(-1, true);
339
340 let cumsum = sorted_probs.cumsum(-1, sorted_probs.kind());
342
343 let cumsum_size = cumsum.size()[0];
347
348 let cumsum_shifted = Tensor::cat(
349 &[
350 Tensor::zeros([1], (sorted_probs.kind(), self.device)),
351 cumsum.narrow(-1, 0, cumsum_size - 1),
352 ],
353 -1,
354 );
355
356 let mask = cumsum_shifted.lt(top_p as f64);
358
359 let filtered_sorted = &sorted_probs * &mask;
361
362 let mut filtered_probs = Tensor::zeros_like(probs);
364 let _ = filtered_probs.scatter_(-1, &sorted_indices, &filtered_sorted);
365
366 let sum = filtered_probs.sum(filtered_probs.kind());
369 Ok(&filtered_probs / &sum)
370 }
371
372 fn multinomial_sample(&self, probs: &Tensor) -> Result<usize> {
375 let vocab_size = probs.size()[0] as usize;
376
377 let sample = probs.multinomial(1, false); let token_id = sample.int64_value(&[0]) as usize; if token_id >= vocab_size {
384 tracing::error!(
386 "OUT OF BOUNDS! token_id={} >= vocab_size={}",
387 token_id,
388 vocab_size
389 );
390 let (_, max_idx) = probs.max_dim(0, false);
392 let safe_token_id = max_idx.int64_value(&[]) as usize;
393 if safe_token_id < vocab_size {
394 Ok(safe_token_id)
395 } else {
396 Ok(0) }
398 } else {
399 Ok(token_id)
400 }
401 }
402}
403
404#[cfg(test)]
405#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
406mod tests {
407 use super::*;
408 use tch::{Device, Kind, Tensor};
409
410 fn create_test_logits() -> Tensor {
411 Tensor::from_slice(&[1.0f32, 2.0, 3.0, 4.0, 5.0])
413 }
414
415 #[test]
416 fn test_temperature_scaling() -> Result<()> {
417 let sampler = TensorSampler::new(Device::Cpu);
418 let logits = create_test_logits();
419
420 let token = sampler.sample_token(
422 &logits,
423 0.01, 1.0,
425 None,
426 1.0,
427 &[],
428 )?;
429
430 assert_eq!(token, 4);
432 Ok(())
433 }
434
435 #[test]
436 fn test_top_k_masking() -> Result<()> {
437 let sampler = TensorSampler::new(Device::Cpu);
438 let logits = Tensor::from_slice(&[1.0f32, 2.0, 3.0, 2.5, 1.5]);
439
440 let filtered = sampler.apply_top_k_to_logits(&logits, 2)?;
441 let filtered_vec: Vec<f32> = Vec::try_from(filtered)?;
442
443 assert!(filtered_vec[2].is_finite() && filtered_vec[2] > 0.0); assert!(filtered_vec[3].is_finite() && filtered_vec[3] > 0.0); assert!(filtered_vec[0].is_infinite() && filtered_vec[0] < 0.0); assert!(filtered_vec[1].is_infinite() && filtered_vec[1] < 0.0); assert!(filtered_vec[4].is_infinite() && filtered_vec[4] < 0.0); Ok(())
451 }
452
453 #[test]
454 fn test_repetition_penalty() -> Result<()> {
455 let sampler = TensorSampler::new(Device::Cpu);
456 let logits = create_test_logits();
457
458 let token = sampler.sample_token(
460 &logits,
461 0.01, 1.0,
463 None,
464 10.0, &[4], )?;
467
468 assert_ne!(token, 4, "Repetition penalty not applied");
470 Ok(())
471 }
472
473 #[test]
474 fn test_repetition_penalty_values() -> Result<()> {
475 let sampler = TensorSampler::new(Device::Cpu);
477 let logits = Tensor::from_slice(&[1.0f32, 2.0, 3.0, 4.0, 5.0]);
478
479 let original_vec: Vec<f32> = Vec::try_from(&logits)?;
481
482 let penalized = sampler.apply_repetition_penalty(
484 logits, 2.0, &[4, 4, 4], )?;
488
489 let penalized_vec: Vec<f32> = Vec::try_from(penalized)?;
490
491 assert!(
495 penalized_vec[4] < original_vec[4],
496 "Token 4 logit should be reduced: {} vs {}",
497 penalized_vec[4],
498 original_vec[4]
499 );
500 assert!(
501 (penalized_vec[4] - 2.5).abs() < 0.01,
502 "Token 4 should be ~2.5 (uniform penalty), got {}",
503 penalized_vec[4]
504 );
505
506 for i in 0..4 {
508 assert_eq!(
509 penalized_vec[i], original_vec[i],
510 "Token {} should be unchanged",
511 i
512 );
513 }
514 Ok(())
515 }
516
517 #[test]
518 fn test_device_consistency() {
519 let cpu_sampler = TensorSampler::new(Device::Cpu);
521 assert_eq!(cpu_sampler.device, Device::Cpu);
522
523 let logits = Tensor::randn([100], (Kind::Float, Device::Cpu));
525 let _ = cpu_sampler.sample_token(&logits, 1.0, 1.0, None, 1.0, &[]);
526 }
528}