Skip to main content

hyprstream_inference/runtime/architectures/
qwen3_5_vision.rs

1//! Qwen3.5 vision encoder
2#![allow(
3    dead_code,
4    unused_variables,
5    clippy::redundant_closure,
6    clippy::needless_borrows_for_generic_args,
7    clippy::redundant_closure_for_method_calls
8)]
9//!
10//! Architecture: Conv3d patch embedding → 27 transformer blocks (2D RoPE on Q/K) →
11//! VisionPatchMerger (2×2 spatial merge) → out_hidden_size projection.
12//!
13//! Output: [total_image_patches, out_hidden_size]
14
15use super::llama::{LinearProjection, LlamaMLP, RMSNorm};
16use anyhow::{anyhow, Result};
17use std::collections::HashMap;
18use tch::{Kind, Tensor};
19
20// ============================================================================
21// Vision config
22// ============================================================================
23
24pub struct Qwen3_5VisionConfig {
25    pub depth: usize,               // 27
26    pub hidden_size: usize,         // 1152
27    pub intermediate_size: usize,   // 4304
28    pub num_heads: usize,           // 16
29    pub head_dim: usize,            // hidden_size / num_heads = 72
30    pub patch_size: usize,          // 16
31    pub temporal_patch_size: usize, // 2 (image-only: use 1)
32    pub spatial_merge_size: usize,  // 2
33    pub out_hidden_size: usize,     // 3584 (9B) or 2048 (35B)
34    pub rms_norm_eps: f32,          // 1e-6
35}
36
37impl Qwen3_5VisionConfig {
38    pub fn from_json(json: &serde_json::Value) -> Self {
39        let hidden_size = json["hidden_size"].as_u64().unwrap_or(1152) as usize;
40        let num_heads = json["num_heads"].as_u64().unwrap_or(16) as usize;
41        Self {
42            depth: json["depth"].as_u64().unwrap_or(27) as usize,
43            hidden_size,
44            intermediate_size: json["intermediate_size"].as_u64().unwrap_or(4304) as usize,
45            num_heads,
46            head_dim: hidden_size / num_heads,
47            patch_size: json["patch_size"].as_u64().unwrap_or(16) as usize,
48            temporal_patch_size: json["temporal_patch_size"].as_u64().unwrap_or(2) as usize,
49            spatial_merge_size: json["spatial_merge_size"].as_u64().unwrap_or(2) as usize,
50            out_hidden_size: json["out_hidden_size"].as_u64().unwrap_or(3584) as usize,
51            rms_norm_eps: 1e-6,
52        }
53    }
54}
55
56// ============================================================================
57// Patch embedding (Conv3d)
58// ============================================================================
59
60struct VisionPatchEmbed {
61    proj_weight: Tensor, // [hidden, 3, temporal_patch_size, patch_size, patch_size]
62    proj_bias: Option<Tensor>,
63    patch_size: usize,
64    temporal_patch_size: usize,
65}
66
67impl VisionPatchEmbed {
68    fn load(
69        weights: &mut HashMap<String, Tensor>,
70        prefix: &str,
71        cfg: &Qwen3_5VisionConfig,
72    ) -> Result<Self> {
73        let pw = weights
74            .remove(&format!("{prefix}.patch_embed.proj.weight"))
75            .ok_or_else(|| anyhow!("Missing {prefix}.patch_embed.proj.weight"))?;
76        let pb = weights.remove(&format!("{prefix}.patch_embed.proj.bias"));
77        Ok(Self {
78            proj_weight: pw,
79            proj_bias: pb,
80            patch_size: cfg.patch_size,
81            temporal_patch_size: cfg.temporal_patch_size,
82        })
83    }
84
85    /// pixel_values: [B, C, H, W] for images
86    /// Returns [B, num_patches, hidden]
87    fn forward(&self, pixel_values: &Tensor) -> Result<Tensor> {
88        let (batch, _c, h, w) = {
89            let s = pixel_values.size();
90            (s[0], s[1], s[2], s[3])
91        };
92        let ps = self.patch_size as i64;
93        let tp = self.temporal_patch_size as i64;
94
95        // Reshape to 5D: [B, C, T=1, H, W] for Conv3d
96        let x5d = pixel_values.unsqueeze(2);
97
98        // Conv3d: kernel = [tp, ps, ps], stride = same
99        let out = x5d.conv3d(
100            &self.proj_weight,
101            self.proj_bias.as_ref(),
102            &[tp, ps, ps], // stride
103            &[0i64, 0, 0], // padding
104            &[1i64, 1, 1], // dilation
105            1,             // groups
106        );
107
108        // out: [B, hidden, T_out, H_out, W_out]
109        let out_shape = out.size();
110        let hidden = out_shape[1];
111        let total_patches = out_shape[2] * out_shape[3] * out_shape[4];
112
113        // Flatten to [B, num_patches, hidden]
114        let out = out
115            .permute([0, 2, 3, 4, 1])
116            .reshape([batch, total_patches, hidden]);
117        Ok(out)
118    }
119}
120
121unsafe impl Send for VisionPatchEmbed {}
122unsafe impl Sync for VisionPatchEmbed {}
123
124// ============================================================================
125// 2D Rotary Embeddings for vision
126// ============================================================================
127
128/// Compute 2D RoPE (cos, sin) for a patch grid
129/// Returns (cos, sin) each of shape [num_patches, head_dim]
130fn vision_rope_2d(
131    grid_h: i64,
132    grid_w: i64,
133    head_dim: i64,
134    device: tch::Device,
135) -> (Tensor, Tensor) {
136    // half of head_dim goes to rows, half to cols
137    let half = head_dim / 2;
138    let inv_freq = {
139        let pos = Tensor::arange(half / 2, (Kind::Float, device));
140        let exp = &pos * 2.0 / (half as f64);
141        Tensor::full([half / 2], 10000.0, (Kind::Float, device)).pow(&(-exp))
142    };
143
144    // Row positions: [grid_h, 1] × inv_freq → [grid_h, half/2]
145    let rows = Tensor::arange(grid_h, (Kind::Float, device)).unsqueeze(1);
146    let row_angles = rows.matmul(&inv_freq.unsqueeze(0)); // [grid_h, half/2]
147
148    // Col positions: [grid_w, 1] × inv_freq → [grid_w, half/2]
149    let cols = Tensor::arange(grid_w, (Kind::Float, device)).unsqueeze(1);
150    let col_angles = cols.matmul(&inv_freq.unsqueeze(0)); // [grid_w, half/2]
151
152    // For each patch (r, c), angle = cat(row_angle, col_angle) of total dim head_dim/2
153    // Expand: [grid_h, grid_w, half/2] for each
154    let row_exp = row_angles
155        .unsqueeze(1)
156        .expand([grid_h, grid_w, half / 2], false);
157    let col_exp = col_angles
158        .unsqueeze(0)
159        .expand([grid_h, grid_w, half / 2], false);
160
161    // Full angle for each patch: [grid_h*grid_w, half]
162    let angles = Tensor::cat(
163        &[
164            row_exp.reshape([grid_h * grid_w, half / 2]),
165            col_exp.reshape([grid_h * grid_w, half / 2]),
166        ],
167        1,
168    );
169
170    // Duplicate for cos/sin halves: [num_patches, head_dim]
171    let angles_full = Tensor::cat(&[&angles, &angles], 1);
172    let cos = angles_full.cos();
173    let sin = angles_full.sin();
174    (cos, sin)
175}
176
177/// Apply 2D RoPE to q or k: [seq, heads, head_dim]
178fn apply_rope_2d(x: &Tensor, cos: &Tensor, sin: &Tensor) -> Tensor {
179    // x: [seq, heads, head_dim]
180    // cos, sin: [seq, head_dim]
181    let cos = cos.unsqueeze(1).expand_as(x); // [seq, heads, head_dim]
182    let sin = sin.unsqueeze(1).expand_as(x);
183
184    let hd = x.size()[2];
185    let half = hd / 2;
186    let x1 = x.narrow(2, 0, half);
187    let x2 = x.narrow(2, half, half);
188    let x1_rot = &x1 * &cos.narrow(2, 0, half) - &x2 * &sin.narrow(2, 0, half);
189    let x2_rot = &x1 * &sin.narrow(2, half, half) + &x2 * &cos.narrow(2, half, half);
190    Tensor::cat(&[x1_rot, x2_rot], 2)
191}
192
193// ============================================================================
194// Vision attention block
195// ============================================================================
196
197struct VisionAttention {
198    qkv: LinearProjection,  // fused [3*hidden, hidden]
199    proj: LinearProjection, // [hidden, hidden]
200    num_heads: usize,
201    head_dim: usize,
202}
203
204impl VisionAttention {
205    fn forward(&self, x: &Tensor, cos: &Tensor, sin: &Tensor) -> Result<Tensor> {
206        let (seq, hidden) = {
207            let s = x.size();
208            (s[0], s[1])
209        };
210        let nh = self.num_heads as i64;
211        let hd = self.head_dim as i64;
212
213        let qkv = self.qkv.apply(x).reshape([seq, 3, nh, hd]);
214        let q = qkv.select(1, 0); // [seq, nh, hd]
215        let k = qkv.select(1, 1);
216        let v = qkv.select(1, 2);
217
218        // Apply 2D RoPE
219        let q = apply_rope_2d(&q, cos, sin).to_kind(Kind::Float);
220        let k = apply_rope_2d(&k, cos, sin).to_kind(Kind::Float);
221        let v = v.to_kind(Kind::Float);
222
223        // Attention [seq, nh, seq]
224        let scale = 1.0 / (hd as f64).sqrt();
225        let q_p = q.permute([1, 0, 2]); // [nh, seq, hd]
226        let k_p = k.permute([1, 2, 0]); // [nh, hd, seq]
227        let v_p = v.permute([1, 0, 2]); // [nh, seq, hd]
228
229        let scores = q_p.matmul(&k_p) * scale; // [nh, seq, seq]
230        let attn = scores.softmax(-1, Kind::Float);
231        let ctx = attn.matmul(&v_p); // [nh, seq, hd]
232        let ctx = ctx.permute([1, 0, 2]).contiguous().reshape([seq, hidden]);
233
234        let out = self.proj.apply(&ctx.to_kind(x.kind()));
235        Ok(out)
236    }
237}
238
239unsafe impl Send for VisionAttention {}
240unsafe impl Sync for VisionAttention {}
241
242// ============================================================================
243// Vision transformer block
244// ============================================================================
245
246struct VisionBlock {
247    norm1: RMSNorm,
248    attn: VisionAttention,
249    norm2: RMSNorm,
250    mlp: LlamaMLP,
251}
252
253impl VisionBlock {
254    fn load(
255        weights: &mut HashMap<String, Tensor>,
256        prefix: &str,
257        cfg: &Qwen3_5VisionConfig,
258        block_idx: usize,
259    ) -> Result<Self> {
260        let norm1_w = weights
261            .remove(&format!("{prefix}.norm1.weight"))
262            .ok_or_else(|| anyhow!("Missing {prefix}.norm1.weight"))?;
263        let norm2_w = weights
264            .remove(&format!("{prefix}.norm2.weight"))
265            .ok_or_else(|| anyhow!("Missing {prefix}.norm2.weight"))?;
266
267        Ok(Self {
268            norm1: RMSNorm {
269                weight: norm1_w,
270                eps: cfg.rms_norm_eps,
271            },
272            attn: VisionAttention {
273                qkv: LinearProjection::take(weights, &format!("{prefix}.attn.qkv.weight"))?,
274                proj: LinearProjection::take(weights, &format!("{prefix}.attn.proj.weight"))?,
275                num_heads: cfg.num_heads,
276                head_dim: cfg.head_dim,
277            },
278            norm2: RMSNorm {
279                weight: norm2_w,
280                eps: cfg.rms_norm_eps,
281            },
282            mlp: LlamaMLP {
283                gate_proj: LinearProjection::take(
284                    weights,
285                    &format!("{prefix}.mlp.gate_proj.weight"),
286                )?,
287                up_proj: LinearProjection::take(weights, &format!("{prefix}.mlp.up_proj.weight"))?,
288                down_proj: LinearProjection::take(
289                    weights,
290                    &format!("{prefix}.mlp.down_proj.weight"),
291                )?,
292                activation: "gelu_pytorch_tanh".to_owned(),
293                layer_idx: block_idx,
294            },
295        })
296    }
297
298    fn forward(&self, x: &Tensor, cos: &Tensor, sin: &Tensor) -> Result<Tensor> {
299        let attn_out = self.attn.forward(&self.norm1.forward(x)?, cos, sin)?;
300        let x = x + &attn_out;
301        let mlp_out = self.mlp.forward(&self.norm2.forward(&x)?, None)?;
302        Ok(&x + &mlp_out)
303    }
304}
305
306unsafe impl Send for VisionBlock {}
307unsafe impl Sync for VisionBlock {}
308
309// ============================================================================
310// Patch merger (spatial 2×2 → out_hidden_size)
311// ============================================================================
312
313struct VisionPatchMerger {
314    ln_q: RMSNorm,
315    mlp0: LinearProjection, // visual.merger.mlp.0  [(hidden*merge^2), out_hidden_size]
316    mlp2: LinearProjection, // visual.merger.mlp.2  [out_hidden_size, out_hidden_size]
317    spatial_merge_size: usize,
318    hidden_size: usize,
319    out_hidden_size: usize,
320}
321
322impl VisionPatchMerger {
323    fn load(
324        weights: &mut HashMap<String, Tensor>,
325        prefix: &str,
326        cfg: &Qwen3_5VisionConfig,
327    ) -> Result<Self> {
328        let ln_q_w = weights
329            .remove(&format!("{prefix}.ln_q.weight"))
330            .ok_or_else(|| anyhow!("Missing {prefix}.ln_q.weight"))?;
331
332        Ok(Self {
333            ln_q: RMSNorm {
334                weight: ln_q_w,
335                eps: cfg.rms_norm_eps,
336            },
337            mlp0: LinearProjection::take(weights, &format!("{prefix}.mlp.0.weight"))?,
338            mlp2: LinearProjection::take(weights, &format!("{prefix}.mlp.2.weight"))?,
339            spatial_merge_size: cfg.spatial_merge_size,
340            hidden_size: cfg.hidden_size,
341            out_hidden_size: cfg.out_hidden_size,
342        })
343    }
344
345    /// x: [num_patches, hidden_size], grid_hw = (H_patches, W_patches)
346    /// Returns [num_out_patches, out_hidden_size]
347    fn forward(&self, x: &Tensor, grid_h: i64, grid_w: i64) -> Result<Tensor> {
348        let x = self.ln_q.forward(x)?;
349        let s = self.spatial_merge_size as i64;
350        let h = self.hidden_size as i64;
351
352        // Reshape patches to [grid_h, grid_w, hidden], then merge s×s spatially
353        let x = x.reshape([grid_h, grid_w, h]);
354
355        // Only merge if grid dimensions are divisible by s
356        let out_h = grid_h / s;
357        let out_w = grid_w / s;
358
359        // Reshape: [out_h, s, out_w, s, h] → [out_h*out_w, s*s*h]
360        let merged = x
361            .reshape([out_h, s, out_w, s, h])
362            .permute([0, 2, 1, 3, 4])
363            .reshape([out_h * out_w, s * s * h]);
364
365        // Two-layer MLP with gelu activation
366        let out = self.mlp0.apply(&merged).gelu("none");
367        let out = self.mlp2.apply(&out);
368        Ok(out)
369    }
370}
371
372unsafe impl Send for VisionPatchMerger {}
373unsafe impl Sync for VisionPatchMerger {}
374
375// ============================================================================
376// Full vision encoder
377// ============================================================================
378
379pub struct Qwen3_5VisionEncoder {
380    config: Qwen3_5VisionConfig,
381    patch_embed: VisionPatchEmbed,
382    blocks: Vec<VisionBlock>,
383    merger: VisionPatchMerger,
384}
385
386impl Qwen3_5VisionEncoder {
387    /// Load encoder from weight map. All vision weights are under the "visual." prefix.
388    pub fn from_weights(
389        weights: &mut HashMap<String, Tensor>,
390        cfg: Qwen3_5VisionConfig,
391    ) -> Result<Self> {
392        let prefix = "visual";
393
394        let patch_embed = VisionPatchEmbed::load(weights, prefix, &cfg)?;
395
396        let mut blocks = Vec::with_capacity(cfg.depth);
397        for i in 0..cfg.depth {
398            let block_prefix = format!("{prefix}.blocks.{i}");
399            blocks.push(VisionBlock::load(weights, &block_prefix, &cfg, i)?);
400        }
401
402        let merger_prefix = format!("{prefix}.merger");
403        let merger = VisionPatchMerger::load(weights, &merger_prefix, &cfg)?;
404
405        Ok(Self {
406            config: cfg,
407            patch_embed,
408            blocks,
409            merger,
410        })
411    }
412
413    /// Encode pixel values to vision features.
414    ///
415    /// pixel_values: [B, C, H, W]
416    /// Returns: [B * num_out_patches, out_hidden_size]
417    pub fn forward(&self, pixel_values: &Tensor) -> Result<Tensor> {
418        let device = pixel_values.device();
419        let dtype = pixel_values.kind();
420
421        // Patch embedding: [B, num_patches, hidden]
422        let patches = self.patch_embed.forward(pixel_values)?;
423
424        let batch = patches.size()[0];
425        let num_patches = patches.size()[1];
426
427        // Grid dimensions (square images assumed)
428        let ps = self.config.patch_size;
429        let h_img = pixel_values.size()[2] as usize;
430        let w_img = pixel_values.size()[3] as usize;
431        let grid_h = (h_img / ps) as i64;
432        let grid_w = (w_img / ps) as i64;
433
434        // 2D RoPE for vision patches: [num_patches, head_dim]
435        let (cos, sin) = vision_rope_2d(grid_h, grid_w, self.config.head_dim as i64, device);
436        let cos = cos.to_kind(dtype);
437        let sin = sin.to_kind(dtype);
438
439        // Process each image separately through transformer blocks
440        let mut all_outputs = Vec::with_capacity(batch as usize);
441
442        for b in 0..batch {
443            let x = patches.select(0, b); // [num_patches, hidden]
444            let mut x = x;
445            for block in &self.blocks {
446                x = block.forward(&x, &cos, &sin)?;
447            }
448            // Merge spatially: [out_patches, out_hidden]
449            let merged = self.merger.forward(&x, grid_h, grid_w)?;
450            all_outputs.push(merged);
451        }
452
453        // Cat along batch: [B * out_patches, out_hidden]
454        Ok(Tensor::cat(&all_outputs, 0))
455    }
456
457    /// Returns the output hidden size (dimension of encoder output)
458    pub fn out_hidden_size(&self) -> usize {
459        self.config.out_hidden_size
460    }
461}
462
463unsafe impl Send for Qwen3_5VisionEncoder {}
464unsafe impl Sync for Qwen3_5VisionEncoder {}