Skip to main content

hyprstream_inference/runtime/
tensor_sampling.rs

1//! Device-agnostic tensor sampling for token generation
2//!
3//! This module implements sampling algorithms directly on PyTorch tensors,
4//! working efficiently on both CPU and GPU devices. Despite the historical
5//! "GPU" naming, this sampler is fully device-agnostic and respects the
6//! device parameter provided during construction.
7
8use anyhow::Result;
9use std::collections::HashSet;
10use tch::{Device, Tensor};
11
12/// Device-agnostic token sampler operating on tensors
13///
14/// Performs all sampling operations (temperature scaling, top-k filtering,
15/// top-p nucleus sampling) directly on tensors, allowing the PyTorch backend
16/// to optimize for the target device (CPU, CUDA, ROCm, Metal, etc.).
17#[derive(Clone)]
18pub struct TensorSampler {
19    device: Device, // Not dead code - used throughout the implementation
20}
21
22impl TensorSampler {
23    pub fn new(device: Device) -> Self {
24        Self { device }
25    }
26
27    /// Set the random seed for deterministic sampling
28    /// This enables reproducible token generation for debugging
29    pub fn set_seed(seed: u64) {
30        // Use PyTorch's manual_seed via tch-rs
31        tch::manual_seed(seed as i64);
32        tracing::info!("Set PyTorch random seed to {}", seed);
33    }
34
35    /// Sample next token directly from logits tensor
36    pub fn sample_token(
37        &self,
38        logits_tensor: &Tensor,
39        temperature: f32,
40        top_p: f32,
41        top_k: Option<usize>,
42        repeat_penalty: f32,
43        previous_tokens: &[i64],
44    ) -> Result<usize> {
45        let logits = self.squeeze_logits(logits_tensor);
46        let penalized_logits =
47            self.apply_repetition_penalty(logits, repeat_penalty, previous_tokens)?;
48        self.sample_from_logits(penalized_logits, temperature, top_p, top_k)
49    }
50
51    /// Sample a token with reduced repeat penalty for single-character tokens.
52    ///
53    /// Exempt tokens (digits, punctuation) receive `sqrt(penalty)` instead of
54    /// the full penalty, preventing digit suppression in year generation while
55    /// still discouraging runaway single-character repetition.
56    #[allow(clippy::too_many_arguments)]
57    pub fn sample_token_with_penalty_exemptions(
58        &self,
59        logits_tensor: &Tensor,
60        temperature: f32,
61        top_p: f32,
62        top_k: Option<usize>,
63        repeat_penalty: f32,
64        previous_tokens: &[i64],
65        exempt_tokens: &HashSet<i64>,
66    ) -> Result<usize> {
67        let logits = self.squeeze_logits(logits_tensor);
68        let penalized_logits = self.apply_repetition_penalty_tiered(
69            logits,
70            repeat_penalty,
71            previous_tokens,
72            exempt_tokens,
73        )?;
74        self.sample_from_logits(penalized_logits, temperature, top_p, top_k)
75    }
76
77    // ── Shared pipeline helpers ──────────────────────────────────────────
78
79    /// Extract last-token logits from a potentially batched tensor, yielding a 1-D vector.
80    fn squeeze_logits(&self, logits_tensor: &Tensor) -> Tensor {
81        let logits = if logits_tensor.dim() > 1 {
82            let shape = logits_tensor.size();
83            if shape.len() == 3 {
84                // [batch, seq_len, vocab_size] → last position
85                logits_tensor.select(1, shape[1] - 1).squeeze_dim(0)
86            } else if shape.len() == 2 {
87                logits_tensor.squeeze_dim(0)
88            } else {
89                logits_tensor.shallow_clone()
90            }
91        } else {
92            logits_tensor.shallow_clone()
93        }
94        .to_device(self.device);
95
96        if logits.dim() != 1 {
97            tracing::warn!("Logits not 1D after squeeze: shape={:?}", logits.size());
98        }
99        logits
100    }
101
102    /// Run the sampling pipeline (greedy / top-k / temperature / softmax / top-p / multinomial)
103    /// on already-penalized logits.
104    fn sample_from_logits(
105        &self,
106        penalized_logits: Tensor,
107        temperature: f32,
108        top_p: f32,
109        top_k: Option<usize>,
110    ) -> Result<usize> {
111        // PERF: Greedy path - bypass softmax/multinomial when temperature is very low
112        if temperature <= 0.01 || temperature.is_nan() || temperature.is_infinite() {
113            let vocab_size = penalized_logits.size()[0] as usize;
114            let (_, max_idx) = penalized_logits.max_dim(0, false);
115            let token_id = max_idx.int64_value(&[]) as usize;
116            if token_id >= vocab_size {
117                tracing::error!("Greedy: token_id {} >= vocab_size {}", token_id, vocab_size);
118                return Ok(0);
119            }
120            return Ok(token_id);
121        }
122
123        // Top-k filtering (more efficient before softmax)
124        let filtered_logits = if let Some(k) = top_k {
125            self.apply_top_k_to_logits(&penalized_logits, k)?
126        } else {
127            penalized_logits
128        };
129
130        // Temperature scaling
131        let scaled_logits = if (temperature - 1.0).abs() < 1e-6 {
132            filtered_logits
133        } else {
134            filtered_logits / (temperature as f64)
135        };
136
137        // Softmax → probabilities (always FP32 for numerical stability)
138        let probs = self.softmax_stable(&scaled_logits)?;
139
140        // Top-p (nucleus) sampling
141        let final_probs = if top_p < 1.0 {
142            self.apply_top_p(&probs, top_p)?
143        } else {
144            probs
145        };
146
147        self.multinomial_sample(&final_probs)
148    }
149
150    // ── Repetition penalty ──────────────────────────────────────────────
151
152    /// Apply uniform repetition penalty to all tokens in the recent window.
153    ///
154    /// PERF: Takes ownership of logits to enable smart copy — only copies if
155    /// the tensor is a view.
156    fn apply_repetition_penalty(
157        &self,
158        logits: Tensor,
159        repeat_penalty: f32,
160        previous_tokens: &[i64],
161    ) -> Result<Tensor> {
162        if repeat_penalty <= 0.0
163            || (repeat_penalty - 1.0).abs() < 1e-6
164            || previous_tokens.is_empty()
165        {
166            return Ok(logits);
167        }
168
169        let vocab_size = logits.size()[0] as usize;
170        let logits_kind = logits.kind();
171
172        // Deduplicate tokens (penalty is applied once per token, not per occurrence)
173        let mut seen = HashSet::new();
174        let unique_ids: Vec<usize> = previous_tokens
175            .iter()
176            .rev()
177            .filter_map(|&tid| {
178                let uid = tid as usize;
179                (tid >= 0 && uid < vocab_size && seen.insert(uid)).then_some(uid)
180            })
181            .collect();
182
183        if unique_ids.is_empty() {
184            return Ok(logits);
185        }
186
187        let mut result = logits.contiguous();
188        self.apply_penalty_to_indices(&mut result, &unique_ids, repeat_penalty, logits_kind)?;
189        Ok(result)
190    }
191
192    /// Apply tiered repetition penalty: full penalty for normal tokens,
193    /// `sqrt(penalty)` for exempt tokens (single-character tokens like digits).
194    ///
195    /// This prevents digit suppression in year generation ("1917" → "209")
196    /// while still discouraging runaway digit repetition ("1822222…").
197    fn apply_repetition_penalty_tiered(
198        &self,
199        logits: Tensor,
200        repeat_penalty: f32,
201        previous_tokens: &[i64],
202        exempt_tokens: &HashSet<i64>,
203    ) -> Result<Tensor> {
204        if repeat_penalty <= 0.0
205            || (repeat_penalty - 1.0).abs() < 1e-6
206            || previous_tokens.is_empty()
207        {
208            return Ok(logits);
209        }
210
211        let vocab_size = logits.size()[0] as usize;
212        let logits_kind = logits.kind();
213
214        // Partition tokens into full-penalty and reduced-penalty groups
215        let mut full_penalty_ids = Vec::new();
216        let mut reduced_penalty_ids = Vec::new();
217        let mut seen = HashSet::new();
218
219        for &token_id in previous_tokens.iter().rev() {
220            let uid = token_id as usize;
221            if token_id >= 0 && uid < vocab_size && seen.insert(uid) {
222                if exempt_tokens.contains(&token_id) {
223                    reduced_penalty_ids.push(uid);
224                } else {
225                    full_penalty_ids.push(uid);
226                }
227            }
228        }
229
230        if full_penalty_ids.is_empty() && reduced_penalty_ids.is_empty() {
231            return Ok(logits);
232        }
233
234        let mut result = logits.contiguous();
235
236        if !full_penalty_ids.is_empty() {
237            self.apply_penalty_to_indices(
238                &mut result,
239                &full_penalty_ids,
240                repeat_penalty,
241                logits_kind,
242            )?;
243        }
244
245        if !reduced_penalty_ids.is_empty() {
246            // sqrt(penalty) gives a much gentler nudge: e.g. 1.1 → 1.049
247            let reduced = repeat_penalty.sqrt();
248            self.apply_penalty_to_indices(&mut result, &reduced_penalty_ids, reduced, logits_kind)?;
249        }
250
251        Ok(result)
252    }
253
254    /// Apply a scalar penalty to specific token indices in a logits tensor.
255    ///
256    /// For positive logits the value is divided by `penalty`;
257    /// for negative logits the value is multiplied by `penalty`.
258    fn apply_penalty_to_indices(
259        &self,
260        result: &mut Tensor,
261        token_ids: &[usize],
262        penalty: f32,
263        logits_kind: tch::Kind,
264    ) -> Result<()> {
265        let indices_vec: Vec<i64> = token_ids.iter().map(|&id| id as i64).collect();
266        let indices = Tensor::from_slice(&indices_vec).to_device(self.device);
267        let current_logits = result.index_select(0, &indices);
268
269        let penalty_tensor = Tensor::from_slice(&[penalty])
270            .to_device(self.device)
271            .to_kind(logits_kind);
272        let penalties =
273            Tensor::ones([indices_vec.len() as i64], (logits_kind, self.device)) * penalty_tensor;
274
275        let penalized_positive = &current_logits / &penalties;
276        let penalized_negative = &current_logits * &penalties;
277
278        let positive_mask = current_logits.gt(0.0).to_kind(logits_kind);
279        let negative_mask = Tensor::ones_like(&positive_mask) - &positive_mask;
280
281        let new_values =
282            &penalized_positive * &positive_mask + &penalized_negative * &negative_mask;
283
284        let _ = result.index_put_(&[Some(indices)], &new_values, false);
285        Ok(())
286    }
287
288    // ── Softmax / top-k / top-p / multinomial ───────────────────────────
289
290    /// Softmax in FP32 for numerical stability.
291    ///
292    /// BF16 softmax over ~150K vocabulary causes significant precision loss
293    /// (many small probabilities underflow to zero). This matches
294    /// HuggingFace/vLLM standard practice.
295    fn softmax_stable(&self, logits: &Tensor) -> Result<Tensor> {
296        let logits_fp32 = if logits.kind() != tch::Kind::Float {
297            logits.to_kind(tch::Kind::Float)
298        } else {
299            logits.shallow_clone()
300        };
301        // Keep FP32 for accurate multinomial sampling
302        Ok(logits_fp32.softmax(-1, tch::Kind::Float))
303    }
304
305    /// Apply top-k filtering to logits (before softmax)
306    /// More efficient than filtering probabilities after softmax
307    fn apply_top_k_to_logits(&self, logits: &Tensor, k: usize) -> Result<Tensor> {
308        let vocab_size = logits.size()[0] as usize;
309
310        if k >= vocab_size {
311            return Ok(logits.shallow_clone());
312        }
313
314        // Get top-k values and indices
315        let (top_values, top_indices) = logits.topk(k as i64, -1, true, true);
316
317        // Create filtered logits (all -inf except top-k)
318        let mut filtered_logits = Tensor::full(
319            [vocab_size as i64],
320            f64::NEG_INFINITY,
321            (logits.kind(), self.device),
322        );
323
324        // Set top-k logits to their original values
325        let _ = filtered_logits.index_put_(&[Some(top_indices)], &top_values, false);
326
327        Ok(filtered_logits)
328    }
329
330    /// Apply top-p (nucleus) sampling
331    /// Keeps tokens where cumulative probability BEFORE adding them is < top_p
332    fn apply_top_p(&self, probs: &Tensor, top_p: f32) -> Result<Tensor> {
333        if top_p <= 0.0 || top_p >= 1.0 {
334            return Ok(probs.shallow_clone());
335        }
336
337        // Sort probabilities in descending order
338        let (sorted_probs, sorted_indices) = probs.sort(-1, true);
339
340        // Compute cumulative sum
341        let cumsum = sorted_probs.cumsum(-1, sorted_probs.kind());
342
343        // Standard nucleus sampling: keep token i if cumsum[i-1] < top_p
344        // This means we include tokens BEFORE cumsum exceeds top_p
345        // Shift cumsum right by 1 position (first position gets 0)
346        let cumsum_size = cumsum.size()[0];
347
348        let cumsum_shifted = Tensor::cat(
349            &[
350                Tensor::zeros([1], (sorted_probs.kind(), self.device)),
351                cumsum.narrow(-1, 0, cumsum_size - 1),
352            ],
353            -1,
354        );
355
356        // Keep tokens where cumsum before adding them is < top_p
357        let mask = cumsum_shifted.lt(top_p as f64);
358
359        // Apply mask
360        let filtered_sorted = &sorted_probs * &mask;
361
362        // Scatter back to original order
363        let mut filtered_probs = Tensor::zeros_like(probs);
364        let _ = filtered_probs.scatter_(-1, &sorted_indices, &filtered_sorted);
365
366        // Renormalize - top-p filtering always keeps at least one token (highest prob)
367        // so sum > 0, avoiding the need for GPU-CPU sync to check
368        let sum = filtered_probs.sum(filtered_probs.kind());
369        Ok(&filtered_probs / &sum)
370    }
371
372    /// Sample from multinomial distribution on GPU
373    /// Optimized to minimize GPU-CPU synchronization
374    fn multinomial_sample(&self, probs: &Tensor) -> Result<usize> {
375        let vocab_size = probs.size()[0] as usize;
376
377        // Use PyTorch's multinomial sampling (GPU accelerated)
378        // This is the primary path - avoid validation overhead in hot path
379        let sample = probs.multinomial(1, false); // Sample 1 token
380        let token_id = sample.int64_value(&[0]) as usize; // Single GPU sync point
381
382        // Validate the sampled token is within vocabulary bounds
383        if token_id >= vocab_size {
384            // Rare error path - only log and do extra work here
385            tracing::error!(
386                "OUT OF BOUNDS! token_id={} >= vocab_size={}",
387                token_id,
388                vocab_size
389            );
390            // Fall back to argmax
391            let (_, max_idx) = probs.max_dim(0, false);
392            let safe_token_id = max_idx.int64_value(&[]) as usize;
393            if safe_token_id < vocab_size {
394                Ok(safe_token_id)
395            } else {
396                Ok(0) // Ultimate fallback to first token
397            }
398        } else {
399            Ok(token_id)
400        }
401    }
402}
403
404#[cfg(test)]
405#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
406mod tests {
407    use super::*;
408    use tch::{Device, Kind, Tensor};
409
410    fn create_test_logits() -> Tensor {
411        // Create deterministic test logits
412        Tensor::from_slice(&[1.0f32, 2.0, 3.0, 4.0, 5.0])
413    }
414
415    #[test]
416    fn test_temperature_scaling() -> Result<()> {
417        let sampler = TensorSampler::new(Device::Cpu);
418        let logits = create_test_logits();
419
420        // Very low temperature should approach greedy (highest logit wins)
421        let token = sampler.sample_token(
422            &logits,
423            0.01, // Very low temp
424            1.0,
425            None,
426            1.0,
427            &[],
428        )?;
429
430        // Should select token 4 (highest logit = 5.0)
431        assert_eq!(token, 4);
432        Ok(())
433    }
434
435    #[test]
436    fn test_top_k_masking() -> Result<()> {
437        let sampler = TensorSampler::new(Device::Cpu);
438        let logits = Tensor::from_slice(&[1.0f32, 2.0, 3.0, 2.5, 1.5]);
439
440        let filtered = sampler.apply_top_k_to_logits(&logits, 2)?;
441        let filtered_vec: Vec<f32> = Vec::try_from(filtered)?;
442
443        // Only top 2 logits should be kept (rest should be -inf)
444        // Index 2 (3.0) and 3 (2.5) are highest
445        assert!(filtered_vec[2].is_finite() && filtered_vec[2] > 0.0); // 3.0 (highest)
446        assert!(filtered_vec[3].is_finite() && filtered_vec[3] > 0.0); // 2.5 (second)
447        assert!(filtered_vec[0].is_infinite() && filtered_vec[0] < 0.0); // -inf
448        assert!(filtered_vec[1].is_infinite() && filtered_vec[1] < 0.0); // -inf
449        assert!(filtered_vec[4].is_infinite() && filtered_vec[4] < 0.0); // -inf
450        Ok(())
451    }
452
453    #[test]
454    fn test_repetition_penalty() -> Result<()> {
455        let sampler = TensorSampler::new(Device::Cpu);
456        let logits = create_test_logits();
457
458        // With high repetition penalty on token 4
459        let token = sampler.sample_token(
460            &logits,
461            0.01, // Very low temp for deterministic
462            1.0,
463            None,
464            10.0, // High penalty
465            &[4], // Previous token was 4
466        )?;
467
468        // Should NOT select token 4 despite it having highest logit
469        assert_ne!(token, 4, "Repetition penalty not applied");
470        Ok(())
471    }
472
473    #[test]
474    fn test_repetition_penalty_values() -> Result<()> {
475        // Verify that penalty actually modifies logit values
476        let sampler = TensorSampler::new(Device::Cpu);
477        let logits = Tensor::from_slice(&[1.0f32, 2.0, 3.0, 4.0, 5.0]);
478
479        // Save original values before penalty (shallow_clone shares storage, so save first)
480        let original_vec: Vec<f32> = Vec::try_from(&logits)?;
481
482        // Apply penalty to token 4 (highest logit = 5.0)
483        let penalized = sampler.apply_repetition_penalty(
484            logits,     // Pass ownership directly
485            2.0,        // 2x penalty
486            &[4, 4, 4], // Token 4 appeared 3 times (but penalty is uniform, not exponential)
487        )?;
488
489        let penalized_vec: Vec<f32> = Vec::try_from(penalized)?;
490
491        // Token 4 should be penalized with UNIFORM penalty (frequency ignored)
492        // Original: 5.0, Penalized: 5.0 / 2.0 = 2.5
493        // (NOT exponential: would be 5.0 / 2^3 = 0.625)
494        assert!(
495            penalized_vec[4] < original_vec[4],
496            "Token 4 logit should be reduced: {} vs {}",
497            penalized_vec[4],
498            original_vec[4]
499        );
500        assert!(
501            (penalized_vec[4] - 2.5).abs() < 0.01,
502            "Token 4 should be ~2.5 (uniform penalty), got {}",
503            penalized_vec[4]
504        );
505
506        // Other tokens should be unchanged
507        for i in 0..4 {
508            assert_eq!(
509                penalized_vec[i], original_vec[i],
510                "Token {} should be unchanged",
511                i
512            );
513        }
514        Ok(())
515    }
516
517    #[test]
518    fn test_device_consistency() {
519        // Test that sampler respects the device parameter
520        let cpu_sampler = TensorSampler::new(Device::Cpu);
521        assert_eq!(cpu_sampler.device, Device::Cpu);
522
523        // Operations should stay on CPU
524        let logits = Tensor::randn([100], (Kind::Float, Device::Cpu));
525        let _ = cpu_sampler.sample_token(&logits, 1.0, 1.0, None, 1.0, &[]);
526        // No panic = success (tensor device mismatch would panic)
527    }
528}