Skip to main content

hyprstream_inference/runtime/
rope.rs

1//! Rotary Position Embedding (RoPE) implementation for tch-rs
2//!
3//! Based on rotary-embedding-tchrs but updated for tch 0.20 compatibility.
4//! Original paper: https://arxiv.org/abs/2104.09864
5
6use anyhow::{anyhow, Result};
7use std::collections::HashMap;
8use tch::{Device, Kind, Tensor};
9
10/// RoPE (Rotary Position Embedding) implementation optimized for tch-rs
11pub struct RoPE {
12    /// Dimension of the embeddings (typically head_dim)
13    dim: i64,
14    /// Base frequency for the rotary embeddings (default 10000, but model-specific - read from config.json)
15    base: f64,
16    /// Maximum sequence length to precompute
17    #[allow(dead_code)]
18    max_seq_len: i64,
19    /// Device for computation
20    #[allow(dead_code)]
21    device: Device,
22    /// Data type for tensors (matches model dtype)
23    dtype: Kind,
24    /// Cached sin/cos values for efficiency
25    cache: Option<(Tensor, Tensor)>,
26    /// Cached sequence length to avoid recomputation
27    cached_seq_len: i64,
28}
29
30impl RoPE {
31    /// Create a new RoPE instance
32    pub fn new(dim: i64, base: f64, max_seq_len: i64, device: Device) -> Result<Self> {
33        if dim % 2 != 0 {
34            return Err(anyhow!("RoPE dimension must be even, got {}", dim));
35        }
36
37        Ok(Self {
38            dim,
39            base,
40            max_seq_len,
41            device,
42            dtype: Kind::BFloat16, // Default to BF16 to match model precision
43            cache: None,
44            cached_seq_len: 0,
45        })
46    }
47
48    /// Create a new RoPE instance with explicit dtype
49    pub fn new_with_dtype(
50        dim: i64,
51        base: f64,
52        max_seq_len: i64,
53        device: Device,
54        dtype: Kind,
55    ) -> Result<Self> {
56        if dim % 2 != 0 {
57            return Err(anyhow!("RoPE dimension must be even, got {}", dim));
58        }
59
60        Ok(Self {
61            dim,
62            base,
63            max_seq_len,
64            device,
65            dtype,
66            cache: None,
67            cached_seq_len: 0,
68        })
69    }
70
71    /// Create RoPE for standard models (base=10000)
72    ///
73    /// Note: For model-specific configurations, always read rope_theta from config.json
74    /// and use `new()` or `new_with_dtype()` with the actual value instead of hardcoding.
75    pub fn new_standard(dim: i64, max_seq_len: i64, device: Device) -> Result<Self> {
76        Self::new_with_dtype(dim, 10000.0, max_seq_len, device, Kind::BFloat16)
77    }
78
79    /// Create RoPE for standard models with dtype
80    ///
81    /// Note: For model-specific configurations, always read rope_theta from config.json
82    /// and use `new()` or `new_with_dtype()` with the actual value instead of hardcoding.
83    pub fn new_standard_with_dtype(
84        dim: i64,
85        max_seq_len: i64,
86        device: Device,
87        dtype: Kind,
88    ) -> Result<Self> {
89        Self::new_with_dtype(dim, 10000.0, max_seq_len, device, dtype)
90    }
91
92    /// Generate sin/cos embeddings for the given sequence length
93    ///
94    /// CRITICAL: Always computed in FP32 regardless of model dtype, then cast to target dtype.
95    /// BF16 computation causes catastrophic precision loss:
96    /// - Position indices > 256 get quantized (BF16 can't represent all integers)
97    /// - inv_freq computation with large base (e.g., 1000000) loses significant precision
98    /// - Compounding errors in angles → sin/cos cause progressive generation corruption
99    ///   This matches HuggingFace/vLLM/llama.cpp which all compute RoPE in FP32.
100    fn generate_embeddings(&self, seq_len: i64) -> Result<(Tensor, Tensor)> {
101        // Create frequency bands in FP32: 1 / (base ^ (2i / dim)) for i in [0, dim/2)
102        let half_dim = self.dim / 2;
103        let inv_freq = {
104            let positions = Tensor::arange(half_dim, (Kind::Float, self.device));
105            let exponents = &positions * 2.0 / (self.dim as f64);
106            let base_tensor = Tensor::full([half_dim], self.base, (Kind::Float, self.device));
107            base_tensor.pow(&(-exponents))
108        };
109
110        // Create position indices in FP32 [0, 1, 2, ..., seq_len-1]
111        let positions = Tensor::arange(seq_len, (Kind::Float, self.device));
112
113        // Compute angles in FP32: position[i] * inv_freq[j] for all i, j
114        // Result shape: [seq_len, dim/2]
115        let angles = positions.unsqueeze(-1).matmul(&inv_freq.unsqueeze(0));
116
117        // Compute sin and cos in FP32, then cast to target dtype
118        let sin_vals = angles.sin().to_kind(self.dtype);
119        let cos_vals = angles.cos().to_kind(self.dtype);
120
121        Ok((sin_vals, cos_vals))
122    }
123
124    /// Get cached or compute sin/cos embeddings for the sequence length
125    fn get_embeddings(&mut self, seq_len: i64) -> Result<(&Tensor, &Tensor)> {
126        // Check if we need to recompute
127        if self.cache.is_none() || seq_len > self.cached_seq_len {
128            let compute_len = seq_len.max(self.max_seq_len);
129            let (sin, cos) = self.generate_embeddings(compute_len)?;
130            self.cache = Some((sin, cos));
131            self.cached_seq_len = compute_len;
132        }
133
134        // Return reference to cached values, truncated to requested length
135        let (sin, cos) = self
136            .cache
137            .as_ref()
138            .ok_or_else(|| anyhow!("RoPE cache not initialized"))?;
139        Ok((sin, cos))
140    }
141
142    /// Apply rotary position embedding to a tensor
143    ///
144    /// # Arguments
145    /// * `x` - Input tensor with shape [..., seq_len, heads, head_dim]
146    /// * `position_ids` - Optional position indices, defaults to [0, 1, 2, ..., seq_len-1]
147    ///
148    /// # Returns
149    /// Tensor with same shape as input, with RoPE applied
150    pub fn forward(&mut self, x: &Tensor, position_ids: Option<&Tensor>) -> Result<Tensor> {
151        // Update dtype to match input tensor if needed
152        let x_dtype = x.kind();
153        if x_dtype != self.dtype {
154            self.dtype = x_dtype;
155            self.cache = None; // Invalidate cache since dtype changed
156            self.cached_seq_len = 0;
157        }
158
159        let x_shape = x.size();
160        let seq_len = x_shape[x_shape.len() - 3]; // [..., seq_len, heads, head_dim]
161        let head_dim = x_shape[x_shape.len() - 1];
162
163        if head_dim != self.dim {
164            return Err(anyhow!(
165                "Input head dimension {} doesn't match RoPE dimension {}",
166                head_dim,
167                self.dim
168            ));
169        }
170
171        // Get sin/cos embeddings
172        // Cache is pre-sized to max_seq_len during init, so we use that as the upper bound.
173        // This avoids GPU sync to extract max position from position_ids tensor.
174        // ASSUMPTION: position_ids never exceed max_seq_len (enforced by KV cache bounds).
175        // If this assumption is violated, index_select will fail at runtime.
176        let required_len = if position_ids.is_some() {
177            // Use pre-configured max_seq_len - positions are bounded by KV cache
178            self.max_seq_len.max(seq_len)
179        } else {
180            seq_len
181        };
182        let (sin_full, cos_full) = self.get_embeddings(required_len)?;
183
184        // Handle custom position_ids or use default sequence
185        let (sin, cos) = if let Some(pos_ids) = position_ids {
186            // Ensure position_ids are in the correct dtype for indexing
187            let pos_ids_int = if pos_ids.kind() != Kind::Int64 {
188                pos_ids.to_kind(Kind::Int64)
189            } else {
190                pos_ids.shallow_clone()
191            };
192            let sin_selected = sin_full.index_select(0, &pos_ids_int);
193            let cos_selected = cos_full.index_select(0, &pos_ids_int);
194            (sin_selected, cos_selected)
195        } else {
196            let sin_truncated = sin_full.narrow(0, 0, seq_len);
197            let cos_truncated = cos_full.narrow(0, 0, seq_len);
198            (sin_truncated, cos_truncated)
199        };
200
201        // Apply rotary embedding
202        self.apply_rotary_pos_emb(x, &sin, &cos)
203    }
204
205    /// Core rotary position embedding application
206    fn apply_rotary_pos_emb(&self, x: &Tensor, sin: &Tensor, cos: &Tensor) -> Result<Tensor> {
207        // Split x into two halves along the last dimension
208        let half_dim = self.dim / 2;
209        let x1 = x.narrow(-1, 0, half_dim); // First half
210        let x2 = x.narrow(-1, half_dim, half_dim); // Second half
211
212        // Expand sin/cos to match tensor dimensions
213        // Input sin/cos shape: [seq_len, dim/2]
214        // x1/x2 shape: [..., seq_len, heads, dim/2]
215        // Need to add dimensions and broadcast
216
217        // First, add dimensions to make sin/cos compatible with x1/x2
218        // sin/cos: [seq_len, dim/2] -> [seq_len, 1, dim/2]
219        let sin_reshaped = sin.unsqueeze(1); // Add heads dimension
220        let cos_reshaped = cos.unsqueeze(1); // Add heads dimension
221
222        // Now add batch dimensions if needed
223        let x_dims = x1.dim();
224        let mut sin_final = sin_reshaped.shallow_clone();
225        let mut cos_final = cos_reshaped.shallow_clone();
226
227        // If x has more than 3 dimensions, we need to add leading dimensions
228        if x_dims > 3 {
229            // Add leading dimensions (batch dimensions)
230            for _ in 0..(x_dims - 3) {
231                sin_final = sin_final.unsqueeze(0);
232                cos_final = cos_final.unsqueeze(0);
233            }
234        }
235
236        // Now expand to match x1/x2 shape
237        let sin_expanded = sin_final.expand_as(&x1);
238        let cos_expanded = cos_final.expand_as(&x1);
239
240        // Apply rotation:
241        // x1' = x1 * cos - x2 * sin
242        // x2' = x1 * sin + x2 * cos
243        let x1_rotated = &x1 * &cos_expanded - &x2 * &sin_expanded;
244        let x2_rotated = &x1 * &sin_expanded + &x2 * &cos_expanded;
245
246        // Concatenate the rotated halves back together
247        Ok(Tensor::cat(&[x1_rotated, x2_rotated], -1))
248    }
249
250    /// Update the base frequency (useful for different model variants)
251    pub fn set_base(&mut self, base: f64) {
252        if base != self.base {
253            self.base = base;
254            self.cache = None; // Invalidate cache
255            self.cached_seq_len = 0;
256        }
257    }
258
259    /// Get current dtype
260    pub fn dtype(&self) -> Kind {
261        self.dtype
262    }
263
264    /// Get current base frequency
265    pub fn base(&self) -> f64 {
266        self.base
267    }
268
269    /// Get embedding dimension
270    pub fn dim(&self) -> i64 {
271        self.dim
272    }
273}
274
275/// RoPE cache manager for multiple model layers
276pub struct RoPEManager {
277    ropes: HashMap<String, RoPE>,
278}
279
280impl Default for RoPEManager {
281    fn default() -> Self {
282        Self::new()
283    }
284}
285
286impl RoPEManager {
287    pub fn new() -> Self {
288        Self {
289            ropes: HashMap::new(),
290        }
291    }
292
293    /// Get or create a RoPE instance for a specific layer
294    pub fn get_rope(
295        &mut self,
296        layer_name: &str,
297        dim: i64,
298        base: f64,
299        max_seq_len: i64,
300        device: Device,
301    ) -> Result<&mut RoPE> {
302        if !self.ropes.contains_key(layer_name) {
303            let rope = RoPE::new(dim, base, max_seq_len, device)?;
304            self.ropes.insert(layer_name.to_owned(), rope);
305        }
306        self.ropes
307            .get_mut(layer_name)
308            .ok_or_else(|| anyhow!("RoPE not found for layer: {}", layer_name))
309    }
310
311    /// Get or create a RoPE instance with specific dtype
312    pub fn get_rope_with_dtype(
313        &mut self,
314        layer_name: &str,
315        dim: i64,
316        base: f64,
317        max_seq_len: i64,
318        device: Device,
319        dtype: Kind,
320    ) -> Result<&mut RoPE> {
321        if !self.ropes.contains_key(layer_name) {
322            let rope = RoPE::new_with_dtype(dim, base, max_seq_len, device, dtype)?;
323            self.ropes.insert(layer_name.to_owned(), rope);
324        }
325        self.ropes
326            .get_mut(layer_name)
327            .ok_or_else(|| anyhow!("RoPE not found for layer: {}", layer_name))
328    }
329
330    /// Apply RoPE to a tensor using the appropriate layer configuration
331    #[allow(clippy::too_many_arguments)]
332    pub fn apply_rope(
333        &mut self,
334        layer_name: &str,
335        x: &Tensor,
336        position_ids: Option<&Tensor>,
337        dim: i64,
338        base: f64,
339        max_seq_len: i64,
340        device: Device,
341    ) -> Result<Tensor> {
342        let rope = self.get_rope(layer_name, dim, base, max_seq_len, device)?;
343        rope.forward(x, position_ids)
344    }
345}
346
347#[cfg(test)]
348#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
349mod tests {
350    use super::*;
351
352    #[test]
353    fn test_rope_basic() -> Result<()> {
354        let device = Device::Cpu;
355        let mut rope = RoPE::new(64, 10000.0, 2048, device)?;
356
357        // Create test tensor: [batch=1, seq=8, heads=2, dim=64]
358        let x = Tensor::randn([1, 8, 2, 64], (Kind::Float, device));
359        let result = rope.forward(&x, None)?;
360
361        assert_eq!(result.size(), x.size());
362        Ok(())
363    }
364
365    #[test]
366    fn test_rope_with_large_base() -> Result<()> {
367        let device = Device::Cpu;
368        // Test with a large base value (e.g., for models with extended context)
369        let mut rope = RoPE::new(128, 1000000.0, 4096, device)?;
370
371        // Create test tensor: [batch=1, seq=16, heads=32, dim=128]
372        let x = Tensor::randn([1, 16, 32, 128], (Kind::Float, device));
373        let result = rope.forward(&x, None)?;
374
375        assert_eq!(result.size(), x.size());
376        assert_eq!(rope.base(), 1000000.0);
377        Ok(())
378    }
379
380    #[test]
381    fn test_rope_dimension_validation() {
382        let device = Device::Cpu;
383
384        // Should fail for odd dimensions
385        let result = RoPE::new(63, 10000.0, 1024, device);
386        assert!(result.is_err());
387
388        // Should succeed for even dimensions
389        let result = RoPE::new(64, 10000.0, 1024, device);
390        assert!(result.is_ok());
391    }
392
393    #[test]
394    fn test_rope_shape_broadcasting() -> Result<()> {
395        let device = Device::Cpu;
396        let mut rope = RoPE::new(64, 10000.0, 2048, device)?;
397
398        // Test the exact shape from the error: [1, 6, 32, 64]
399        let x = Tensor::randn([1, 6, 32, 64], (Kind::Float, device));
400        let result = rope.forward(&x, None)?;
401
402        assert_eq!(result.size(), vec![1, 6, 32, 64]);
403
404        // Test with different batch dimensions
405        let x2 = Tensor::randn([2, 8, 16, 64], (Kind::Float, device));
406        let result2 = rope.forward(&x2, None)?;
407        assert_eq!(result2.size(), vec![2, 8, 16, 64]);
408
409        // Test 3D tensor (no batch dimension)
410        let x3 = Tensor::randn([10, 4, 64], (Kind::Float, device));
411        let result3 = rope.forward(&x3, None)?;
412        assert_eq!(result3.size(), vec![10, 4, 64]);
413        Ok(())
414    }
415}