1#![allow(
3 dead_code,
4 unused_variables,
5 clippy::redundant_closure,
6 clippy::needless_borrows_for_generic_args,
7 clippy::redundant_closure_for_method_calls
8)]
9use super::llama::{LinearProjection, LlamaMLP, RMSNorm};
16use anyhow::{anyhow, Result};
17use std::collections::HashMap;
18use tch::{Kind, Tensor};
19
20pub struct Qwen3_5VisionConfig {
25 pub depth: usize, pub hidden_size: usize, pub intermediate_size: usize, pub num_heads: usize, pub head_dim: usize, pub patch_size: usize, pub temporal_patch_size: usize, pub spatial_merge_size: usize, pub out_hidden_size: usize, pub rms_norm_eps: f32, }
36
37impl Qwen3_5VisionConfig {
38 pub fn from_json(json: &serde_json::Value) -> Self {
39 let hidden_size = json["hidden_size"].as_u64().unwrap_or(1152) as usize;
40 let num_heads = json["num_heads"].as_u64().unwrap_or(16) as usize;
41 Self {
42 depth: json["depth"].as_u64().unwrap_or(27) as usize,
43 hidden_size,
44 intermediate_size: json["intermediate_size"].as_u64().unwrap_or(4304) as usize,
45 num_heads,
46 head_dim: hidden_size / num_heads,
47 patch_size: json["patch_size"].as_u64().unwrap_or(16) as usize,
48 temporal_patch_size: json["temporal_patch_size"].as_u64().unwrap_or(2) as usize,
49 spatial_merge_size: json["spatial_merge_size"].as_u64().unwrap_or(2) as usize,
50 out_hidden_size: json["out_hidden_size"].as_u64().unwrap_or(3584) as usize,
51 rms_norm_eps: 1e-6,
52 }
53 }
54}
55
56struct VisionPatchEmbed {
61 proj_weight: Tensor, proj_bias: Option<Tensor>,
63 patch_size: usize,
64 temporal_patch_size: usize,
65}
66
67impl VisionPatchEmbed {
68 fn load(
69 weights: &mut HashMap<String, Tensor>,
70 prefix: &str,
71 cfg: &Qwen3_5VisionConfig,
72 ) -> Result<Self> {
73 let pw = weights
74 .remove(&format!("{prefix}.patch_embed.proj.weight"))
75 .ok_or_else(|| anyhow!("Missing {prefix}.patch_embed.proj.weight"))?;
76 let pb = weights.remove(&format!("{prefix}.patch_embed.proj.bias"));
77 Ok(Self {
78 proj_weight: pw,
79 proj_bias: pb,
80 patch_size: cfg.patch_size,
81 temporal_patch_size: cfg.temporal_patch_size,
82 })
83 }
84
85 fn forward(&self, pixel_values: &Tensor) -> Result<Tensor> {
88 let (batch, _c, h, w) = {
89 let s = pixel_values.size();
90 (s[0], s[1], s[2], s[3])
91 };
92 let ps = self.patch_size as i64;
93 let tp = self.temporal_patch_size as i64;
94
95 let x5d = pixel_values.unsqueeze(2);
97
98 let out = x5d.conv3d(
100 &self.proj_weight,
101 self.proj_bias.as_ref(),
102 &[tp, ps, ps], &[0i64, 0, 0], &[1i64, 1, 1], 1, );
107
108 let out_shape = out.size();
110 let hidden = out_shape[1];
111 let total_patches = out_shape[2] * out_shape[3] * out_shape[4];
112
113 let out = out
115 .permute([0, 2, 3, 4, 1])
116 .reshape([batch, total_patches, hidden]);
117 Ok(out)
118 }
119}
120
121unsafe impl Send for VisionPatchEmbed {}
122unsafe impl Sync for VisionPatchEmbed {}
123
124fn vision_rope_2d(
131 grid_h: i64,
132 grid_w: i64,
133 head_dim: i64,
134 device: tch::Device,
135) -> (Tensor, Tensor) {
136 let half = head_dim / 2;
138 let inv_freq = {
139 let pos = Tensor::arange(half / 2, (Kind::Float, device));
140 let exp = &pos * 2.0 / (half as f64);
141 Tensor::full([half / 2], 10000.0, (Kind::Float, device)).pow(&(-exp))
142 };
143
144 let rows = Tensor::arange(grid_h, (Kind::Float, device)).unsqueeze(1);
146 let row_angles = rows.matmul(&inv_freq.unsqueeze(0)); let cols = Tensor::arange(grid_w, (Kind::Float, device)).unsqueeze(1);
150 let col_angles = cols.matmul(&inv_freq.unsqueeze(0)); let row_exp = row_angles
155 .unsqueeze(1)
156 .expand([grid_h, grid_w, half / 2], false);
157 let col_exp = col_angles
158 .unsqueeze(0)
159 .expand([grid_h, grid_w, half / 2], false);
160
161 let angles = Tensor::cat(
163 &[
164 row_exp.reshape([grid_h * grid_w, half / 2]),
165 col_exp.reshape([grid_h * grid_w, half / 2]),
166 ],
167 1,
168 );
169
170 let angles_full = Tensor::cat(&[&angles, &angles], 1);
172 let cos = angles_full.cos();
173 let sin = angles_full.sin();
174 (cos, sin)
175}
176
177fn apply_rope_2d(x: &Tensor, cos: &Tensor, sin: &Tensor) -> Tensor {
179 let cos = cos.unsqueeze(1).expand_as(x); let sin = sin.unsqueeze(1).expand_as(x);
183
184 let hd = x.size()[2];
185 let half = hd / 2;
186 let x1 = x.narrow(2, 0, half);
187 let x2 = x.narrow(2, half, half);
188 let x1_rot = &x1 * &cos.narrow(2, 0, half) - &x2 * &sin.narrow(2, 0, half);
189 let x2_rot = &x1 * &sin.narrow(2, half, half) + &x2 * &cos.narrow(2, half, half);
190 Tensor::cat(&[x1_rot, x2_rot], 2)
191}
192
193struct VisionAttention {
198 qkv: LinearProjection, proj: LinearProjection, num_heads: usize,
201 head_dim: usize,
202}
203
204impl VisionAttention {
205 fn forward(&self, x: &Tensor, cos: &Tensor, sin: &Tensor) -> Result<Tensor> {
206 let (seq, hidden) = {
207 let s = x.size();
208 (s[0], s[1])
209 };
210 let nh = self.num_heads as i64;
211 let hd = self.head_dim as i64;
212
213 let qkv = self.qkv.apply(x).reshape([seq, 3, nh, hd]);
214 let q = qkv.select(1, 0); let k = qkv.select(1, 1);
216 let v = qkv.select(1, 2);
217
218 let q = apply_rope_2d(&q, cos, sin).to_kind(Kind::Float);
220 let k = apply_rope_2d(&k, cos, sin).to_kind(Kind::Float);
221 let v = v.to_kind(Kind::Float);
222
223 let scale = 1.0 / (hd as f64).sqrt();
225 let q_p = q.permute([1, 0, 2]); let k_p = k.permute([1, 2, 0]); let v_p = v.permute([1, 0, 2]); let scores = q_p.matmul(&k_p) * scale; let attn = scores.softmax(-1, Kind::Float);
231 let ctx = attn.matmul(&v_p); let ctx = ctx.permute([1, 0, 2]).contiguous().reshape([seq, hidden]);
233
234 let out = self.proj.apply(&ctx.to_kind(x.kind()));
235 Ok(out)
236 }
237}
238
239unsafe impl Send for VisionAttention {}
240unsafe impl Sync for VisionAttention {}
241
242struct VisionBlock {
247 norm1: RMSNorm,
248 attn: VisionAttention,
249 norm2: RMSNorm,
250 mlp: LlamaMLP,
251}
252
253impl VisionBlock {
254 fn load(
255 weights: &mut HashMap<String, Tensor>,
256 prefix: &str,
257 cfg: &Qwen3_5VisionConfig,
258 block_idx: usize,
259 ) -> Result<Self> {
260 let norm1_w = weights
261 .remove(&format!("{prefix}.norm1.weight"))
262 .ok_or_else(|| anyhow!("Missing {prefix}.norm1.weight"))?;
263 let norm2_w = weights
264 .remove(&format!("{prefix}.norm2.weight"))
265 .ok_or_else(|| anyhow!("Missing {prefix}.norm2.weight"))?;
266
267 Ok(Self {
268 norm1: RMSNorm {
269 weight: norm1_w,
270 eps: cfg.rms_norm_eps,
271 },
272 attn: VisionAttention {
273 qkv: LinearProjection::take(weights, &format!("{prefix}.attn.qkv.weight"))?,
274 proj: LinearProjection::take(weights, &format!("{prefix}.attn.proj.weight"))?,
275 num_heads: cfg.num_heads,
276 head_dim: cfg.head_dim,
277 },
278 norm2: RMSNorm {
279 weight: norm2_w,
280 eps: cfg.rms_norm_eps,
281 },
282 mlp: LlamaMLP {
283 gate_proj: LinearProjection::take(
284 weights,
285 &format!("{prefix}.mlp.gate_proj.weight"),
286 )?,
287 up_proj: LinearProjection::take(weights, &format!("{prefix}.mlp.up_proj.weight"))?,
288 down_proj: LinearProjection::take(
289 weights,
290 &format!("{prefix}.mlp.down_proj.weight"),
291 )?,
292 activation: "gelu_pytorch_tanh".to_owned(),
293 layer_idx: block_idx,
294 },
295 })
296 }
297
298 fn forward(&self, x: &Tensor, cos: &Tensor, sin: &Tensor) -> Result<Tensor> {
299 let attn_out = self.attn.forward(&self.norm1.forward(x)?, cos, sin)?;
300 let x = x + &attn_out;
301 let mlp_out = self.mlp.forward(&self.norm2.forward(&x)?, None)?;
302 Ok(&x + &mlp_out)
303 }
304}
305
306unsafe impl Send for VisionBlock {}
307unsafe impl Sync for VisionBlock {}
308
309struct VisionPatchMerger {
314 ln_q: RMSNorm,
315 mlp0: LinearProjection, mlp2: LinearProjection, spatial_merge_size: usize,
318 hidden_size: usize,
319 out_hidden_size: usize,
320}
321
322impl VisionPatchMerger {
323 fn load(
324 weights: &mut HashMap<String, Tensor>,
325 prefix: &str,
326 cfg: &Qwen3_5VisionConfig,
327 ) -> Result<Self> {
328 let ln_q_w = weights
329 .remove(&format!("{prefix}.ln_q.weight"))
330 .ok_or_else(|| anyhow!("Missing {prefix}.ln_q.weight"))?;
331
332 Ok(Self {
333 ln_q: RMSNorm {
334 weight: ln_q_w,
335 eps: cfg.rms_norm_eps,
336 },
337 mlp0: LinearProjection::take(weights, &format!("{prefix}.mlp.0.weight"))?,
338 mlp2: LinearProjection::take(weights, &format!("{prefix}.mlp.2.weight"))?,
339 spatial_merge_size: cfg.spatial_merge_size,
340 hidden_size: cfg.hidden_size,
341 out_hidden_size: cfg.out_hidden_size,
342 })
343 }
344
345 fn forward(&self, x: &Tensor, grid_h: i64, grid_w: i64) -> Result<Tensor> {
348 let x = self.ln_q.forward(x)?;
349 let s = self.spatial_merge_size as i64;
350 let h = self.hidden_size as i64;
351
352 let x = x.reshape([grid_h, grid_w, h]);
354
355 let out_h = grid_h / s;
357 let out_w = grid_w / s;
358
359 let merged = x
361 .reshape([out_h, s, out_w, s, h])
362 .permute([0, 2, 1, 3, 4])
363 .reshape([out_h * out_w, s * s * h]);
364
365 let out = self.mlp0.apply(&merged).gelu("none");
367 let out = self.mlp2.apply(&out);
368 Ok(out)
369 }
370}
371
372unsafe impl Send for VisionPatchMerger {}
373unsafe impl Sync for VisionPatchMerger {}
374
375pub struct Qwen3_5VisionEncoder {
380 config: Qwen3_5VisionConfig,
381 patch_embed: VisionPatchEmbed,
382 blocks: Vec<VisionBlock>,
383 merger: VisionPatchMerger,
384}
385
386impl Qwen3_5VisionEncoder {
387 pub fn from_weights(
389 weights: &mut HashMap<String, Tensor>,
390 cfg: Qwen3_5VisionConfig,
391 ) -> Result<Self> {
392 let prefix = "visual";
393
394 let patch_embed = VisionPatchEmbed::load(weights, prefix, &cfg)?;
395
396 let mut blocks = Vec::with_capacity(cfg.depth);
397 for i in 0..cfg.depth {
398 let block_prefix = format!("{prefix}.blocks.{i}");
399 blocks.push(VisionBlock::load(weights, &block_prefix, &cfg, i)?);
400 }
401
402 let merger_prefix = format!("{prefix}.merger");
403 let merger = VisionPatchMerger::load(weights, &merger_prefix, &cfg)?;
404
405 Ok(Self {
406 config: cfg,
407 patch_embed,
408 blocks,
409 merger,
410 })
411 }
412
413 pub fn forward(&self, pixel_values: &Tensor) -> Result<Tensor> {
418 let device = pixel_values.device();
419 let dtype = pixel_values.kind();
420
421 let patches = self.patch_embed.forward(pixel_values)?;
423
424 let batch = patches.size()[0];
425 let num_patches = patches.size()[1];
426
427 let ps = self.config.patch_size;
429 let h_img = pixel_values.size()[2] as usize;
430 let w_img = pixel_values.size()[3] as usize;
431 let grid_h = (h_img / ps) as i64;
432 let grid_w = (w_img / ps) as i64;
433
434 let (cos, sin) = vision_rope_2d(grid_h, grid_w, self.config.head_dim as i64, device);
436 let cos = cos.to_kind(dtype);
437 let sin = sin.to_kind(dtype);
438
439 let mut all_outputs = Vec::with_capacity(batch as usize);
441
442 for b in 0..batch {
443 let x = patches.select(0, b); let mut x = x;
445 for block in &self.blocks {
446 x = block.forward(&x, &cos, &sin)?;
447 }
448 let merged = self.merger.forward(&x, grid_h, grid_w)?;
450 all_outputs.push(merged);
451 }
452
453 Ok(Tensor::cat(&all_outputs, 0))
455 }
456
457 pub fn out_hidden_size(&self) -> usize {
459 self.config.out_hidden_size
460 }
461}
462
463unsafe impl Send for Qwen3_5VisionEncoder {}
464unsafe impl Sync for Qwen3_5VisionEncoder {}