1use anyhow::{anyhow, Result};
7use std::collections::HashMap;
8use tch::{Device, Kind, Tensor};
9
10pub struct RoPE {
12 dim: i64,
14 base: f64,
16 #[allow(dead_code)]
18 max_seq_len: i64,
19 #[allow(dead_code)]
21 device: Device,
22 dtype: Kind,
24 cache: Option<(Tensor, Tensor)>,
26 cached_seq_len: i64,
28}
29
30impl RoPE {
31 pub fn new(dim: i64, base: f64, max_seq_len: i64, device: Device) -> Result<Self> {
33 if dim % 2 != 0 {
34 return Err(anyhow!("RoPE dimension must be even, got {}", dim));
35 }
36
37 Ok(Self {
38 dim,
39 base,
40 max_seq_len,
41 device,
42 dtype: Kind::BFloat16, cache: None,
44 cached_seq_len: 0,
45 })
46 }
47
48 pub fn new_with_dtype(
50 dim: i64,
51 base: f64,
52 max_seq_len: i64,
53 device: Device,
54 dtype: Kind,
55 ) -> Result<Self> {
56 if dim % 2 != 0 {
57 return Err(anyhow!("RoPE dimension must be even, got {}", dim));
58 }
59
60 Ok(Self {
61 dim,
62 base,
63 max_seq_len,
64 device,
65 dtype,
66 cache: None,
67 cached_seq_len: 0,
68 })
69 }
70
71 pub fn new_standard(dim: i64, max_seq_len: i64, device: Device) -> Result<Self> {
76 Self::new_with_dtype(dim, 10000.0, max_seq_len, device, Kind::BFloat16)
77 }
78
79 pub fn new_standard_with_dtype(
84 dim: i64,
85 max_seq_len: i64,
86 device: Device,
87 dtype: Kind,
88 ) -> Result<Self> {
89 Self::new_with_dtype(dim, 10000.0, max_seq_len, device, dtype)
90 }
91
92 fn generate_embeddings(&self, seq_len: i64) -> Result<(Tensor, Tensor)> {
101 let half_dim = self.dim / 2;
103 let inv_freq = {
104 let positions = Tensor::arange(half_dim, (Kind::Float, self.device));
105 let exponents = &positions * 2.0 / (self.dim as f64);
106 let base_tensor = Tensor::full([half_dim], self.base, (Kind::Float, self.device));
107 base_tensor.pow(&(-exponents))
108 };
109
110 let positions = Tensor::arange(seq_len, (Kind::Float, self.device));
112
113 let angles = positions.unsqueeze(-1).matmul(&inv_freq.unsqueeze(0));
116
117 let sin_vals = angles.sin().to_kind(self.dtype);
119 let cos_vals = angles.cos().to_kind(self.dtype);
120
121 Ok((sin_vals, cos_vals))
122 }
123
124 fn get_embeddings(&mut self, seq_len: i64) -> Result<(&Tensor, &Tensor)> {
126 if self.cache.is_none() || seq_len > self.cached_seq_len {
128 let compute_len = seq_len.max(self.max_seq_len);
129 let (sin, cos) = self.generate_embeddings(compute_len)?;
130 self.cache = Some((sin, cos));
131 self.cached_seq_len = compute_len;
132 }
133
134 let (sin, cos) = self
136 .cache
137 .as_ref()
138 .ok_or_else(|| anyhow!("RoPE cache not initialized"))?;
139 Ok((sin, cos))
140 }
141
142 pub fn forward(&mut self, x: &Tensor, position_ids: Option<&Tensor>) -> Result<Tensor> {
151 let x_dtype = x.kind();
153 if x_dtype != self.dtype {
154 self.dtype = x_dtype;
155 self.cache = None; self.cached_seq_len = 0;
157 }
158
159 let x_shape = x.size();
160 let seq_len = x_shape[x_shape.len() - 3]; let head_dim = x_shape[x_shape.len() - 1];
162
163 if head_dim != self.dim {
164 return Err(anyhow!(
165 "Input head dimension {} doesn't match RoPE dimension {}",
166 head_dim,
167 self.dim
168 ));
169 }
170
171 let required_len = if position_ids.is_some() {
177 self.max_seq_len.max(seq_len)
179 } else {
180 seq_len
181 };
182 let (sin_full, cos_full) = self.get_embeddings(required_len)?;
183
184 let (sin, cos) = if let Some(pos_ids) = position_ids {
186 let pos_ids_int = if pos_ids.kind() != Kind::Int64 {
188 pos_ids.to_kind(Kind::Int64)
189 } else {
190 pos_ids.shallow_clone()
191 };
192 let sin_selected = sin_full.index_select(0, &pos_ids_int);
193 let cos_selected = cos_full.index_select(0, &pos_ids_int);
194 (sin_selected, cos_selected)
195 } else {
196 let sin_truncated = sin_full.narrow(0, 0, seq_len);
197 let cos_truncated = cos_full.narrow(0, 0, seq_len);
198 (sin_truncated, cos_truncated)
199 };
200
201 self.apply_rotary_pos_emb(x, &sin, &cos)
203 }
204
205 fn apply_rotary_pos_emb(&self, x: &Tensor, sin: &Tensor, cos: &Tensor) -> Result<Tensor> {
207 let half_dim = self.dim / 2;
209 let x1 = x.narrow(-1, 0, half_dim); let x2 = x.narrow(-1, half_dim, half_dim); let sin_reshaped = sin.unsqueeze(1); let cos_reshaped = cos.unsqueeze(1); let x_dims = x1.dim();
224 let mut sin_final = sin_reshaped.shallow_clone();
225 let mut cos_final = cos_reshaped.shallow_clone();
226
227 if x_dims > 3 {
229 for _ in 0..(x_dims - 3) {
231 sin_final = sin_final.unsqueeze(0);
232 cos_final = cos_final.unsqueeze(0);
233 }
234 }
235
236 let sin_expanded = sin_final.expand_as(&x1);
238 let cos_expanded = cos_final.expand_as(&x1);
239
240 let x1_rotated = &x1 * &cos_expanded - &x2 * &sin_expanded;
244 let x2_rotated = &x1 * &sin_expanded + &x2 * &cos_expanded;
245
246 Ok(Tensor::cat(&[x1_rotated, x2_rotated], -1))
248 }
249
250 pub fn set_base(&mut self, base: f64) {
252 if base != self.base {
253 self.base = base;
254 self.cache = None; self.cached_seq_len = 0;
256 }
257 }
258
259 pub fn dtype(&self) -> Kind {
261 self.dtype
262 }
263
264 pub fn base(&self) -> f64 {
266 self.base
267 }
268
269 pub fn dim(&self) -> i64 {
271 self.dim
272 }
273}
274
275pub struct RoPEManager {
277 ropes: HashMap<String, RoPE>,
278}
279
280impl Default for RoPEManager {
281 fn default() -> Self {
282 Self::new()
283 }
284}
285
286impl RoPEManager {
287 pub fn new() -> Self {
288 Self {
289 ropes: HashMap::new(),
290 }
291 }
292
293 pub fn get_rope(
295 &mut self,
296 layer_name: &str,
297 dim: i64,
298 base: f64,
299 max_seq_len: i64,
300 device: Device,
301 ) -> Result<&mut RoPE> {
302 if !self.ropes.contains_key(layer_name) {
303 let rope = RoPE::new(dim, base, max_seq_len, device)?;
304 self.ropes.insert(layer_name.to_owned(), rope);
305 }
306 self.ropes
307 .get_mut(layer_name)
308 .ok_or_else(|| anyhow!("RoPE not found for layer: {}", layer_name))
309 }
310
311 pub fn get_rope_with_dtype(
313 &mut self,
314 layer_name: &str,
315 dim: i64,
316 base: f64,
317 max_seq_len: i64,
318 device: Device,
319 dtype: Kind,
320 ) -> Result<&mut RoPE> {
321 if !self.ropes.contains_key(layer_name) {
322 let rope = RoPE::new_with_dtype(dim, base, max_seq_len, device, dtype)?;
323 self.ropes.insert(layer_name.to_owned(), rope);
324 }
325 self.ropes
326 .get_mut(layer_name)
327 .ok_or_else(|| anyhow!("RoPE not found for layer: {}", layer_name))
328 }
329
330 #[allow(clippy::too_many_arguments)]
332 pub fn apply_rope(
333 &mut self,
334 layer_name: &str,
335 x: &Tensor,
336 position_ids: Option<&Tensor>,
337 dim: i64,
338 base: f64,
339 max_seq_len: i64,
340 device: Device,
341 ) -> Result<Tensor> {
342 let rope = self.get_rope(layer_name, dim, base, max_seq_len, device)?;
343 rope.forward(x, position_ids)
344 }
345}
346
347#[cfg(test)]
348#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
349mod tests {
350 use super::*;
351
352 #[test]
353 fn test_rope_basic() -> Result<()> {
354 let device = Device::Cpu;
355 let mut rope = RoPE::new(64, 10000.0, 2048, device)?;
356
357 let x = Tensor::randn([1, 8, 2, 64], (Kind::Float, device));
359 let result = rope.forward(&x, None)?;
360
361 assert_eq!(result.size(), x.size());
362 Ok(())
363 }
364
365 #[test]
366 fn test_rope_with_large_base() -> Result<()> {
367 let device = Device::Cpu;
368 let mut rope = RoPE::new(128, 1000000.0, 4096, device)?;
370
371 let x = Tensor::randn([1, 16, 32, 128], (Kind::Float, device));
373 let result = rope.forward(&x, None)?;
374
375 assert_eq!(result.size(), x.size());
376 assert_eq!(rope.base(), 1000000.0);
377 Ok(())
378 }
379
380 #[test]
381 fn test_rope_dimension_validation() {
382 let device = Device::Cpu;
383
384 let result = RoPE::new(63, 10000.0, 1024, device);
386 assert!(result.is_err());
387
388 let result = RoPE::new(64, 10000.0, 1024, device);
390 assert!(result.is_ok());
391 }
392
393 #[test]
394 fn test_rope_shape_broadcasting() -> Result<()> {
395 let device = Device::Cpu;
396 let mut rope = RoPE::new(64, 10000.0, 2048, device)?;
397
398 let x = Tensor::randn([1, 6, 32, 64], (Kind::Float, device));
400 let result = rope.forward(&x, None)?;
401
402 assert_eq!(result.size(), vec![1, 6, 32, 64]);
403
404 let x2 = Tensor::randn([2, 8, 16, 64], (Kind::Float, device));
406 let result2 = rope.forward(&x2, None)?;
407 assert_eq!(result2.size(), vec![2, 8, 16, 64]);
408
409 let x3 = Tensor::randn([10, 4, 64], (Kind::Float, device));
411 let result3 = rope.forward(&x3, None)?;
412 assert_eq!(result3.size(), vec![10, 4, 64]);
413 Ok(())
414 }
415}