1use anyhow::{anyhow, Result};
14use std::collections::HashMap;
15use tch::{Device, Kind as DType, Tensor};
16use tracing::debug;
17
18#[derive(Debug, Clone)]
20pub struct SigLIPConfig {
21 pub image_size: usize,
22 pub patch_size: usize,
23 pub hidden_size: usize,
24 pub num_hidden_layers: usize,
25 pub num_attention_heads: usize,
26 pub intermediate_size: usize,
27 pub use_attention_pool: bool,
28}
29
30impl Default for SigLIPConfig {
31 fn default() -> Self {
32 Self {
34 image_size: 384,
35 patch_size: 16,
36 hidden_size: 1024,
37 num_hidden_layers: 24,
38 num_attention_heads: 16,
39 intermediate_size: 4096,
40 use_attention_pool: true,
41 }
42 }
43}
44
45#[derive(Debug)]
47struct LayerNorm {
48 weight: Tensor,
49 bias: Tensor,
50 eps: f64,
51}
52
53impl LayerNorm {
54 fn from_weights(
55 weights: &HashMap<String, Tensor>,
56 prefix: &str,
57 device: Device,
58 dtype: DType,
59 ) -> Result<Self> {
60 let weight_key = format!("{prefix}.weight");
61 let bias_key = format!("{prefix}.bias");
62
63 let weight = weights
64 .get(&weight_key)
65 .ok_or_else(|| anyhow!("Missing LayerNorm weight: {}", weight_key))?
66 .to_device(device)
67 .to_kind(dtype);
68
69 let bias = weights
70 .get(&bias_key)
71 .ok_or_else(|| anyhow!("Missing LayerNorm bias: {}", bias_key))?
72 .to_device(device)
73 .to_kind(dtype);
74
75 Ok(Self {
76 weight,
77 bias,
78 eps: 1e-6,
79 })
80 }
81
82 fn forward(&self, x: &Tensor) -> Result<Tensor> {
83 Ok(x.layer_norm(
84 [self.weight.size()[0]],
85 Some(&self.weight),
86 Some(&self.bias),
87 self.eps,
88 true,
89 ))
90 }
91}
92
93#[derive(Debug)]
95struct SigLIPAttention {
96 qkv_weight: Tensor,
97 qkv_bias: Tensor,
98 proj_weight: Tensor,
99 proj_bias: Tensor,
100 num_heads: usize,
101 head_dim: usize,
102 scale: f64,
103}
104
105impl SigLIPAttention {
106 fn from_weights(
107 weights: &HashMap<String, Tensor>,
108 prefix: &str,
109 hidden_size: usize,
110 num_heads: usize,
111 device: Device,
112 dtype: DType,
113 ) -> Result<Self> {
114 let qkv_weight = weights
115 .get(&format!("{prefix}.qkv.weight"))
116 .ok_or_else(|| anyhow!("Missing QKV weight: {}.qkv.weight", prefix))?
117 .to_device(device)
118 .to_kind(dtype);
119
120 let qkv_bias = weights
121 .get(&format!("{prefix}.qkv.bias"))
122 .ok_or_else(|| anyhow!("Missing QKV bias: {}.qkv.bias", prefix))?
123 .to_device(device)
124 .to_kind(dtype);
125
126 let proj_weight = weights
127 .get(&format!("{prefix}.proj.weight"))
128 .ok_or_else(|| anyhow!("Missing proj weight: {}.proj.weight", prefix))?
129 .to_device(device)
130 .to_kind(dtype);
131
132 let proj_bias = weights
133 .get(&format!("{prefix}.proj.bias"))
134 .ok_or_else(|| anyhow!("Missing proj bias: {}.proj.bias", prefix))?
135 .to_device(device)
136 .to_kind(dtype);
137
138 let head_dim = hidden_size / num_heads;
139 let scale = 1.0 / (head_dim as f64).sqrt();
140
141 Ok(Self {
142 qkv_weight,
143 qkv_bias,
144 proj_weight,
145 proj_bias,
146 num_heads,
147 head_dim,
148 scale,
149 })
150 }
151
152 fn forward(&self, x: &Tensor) -> Result<Tensor> {
153 let (batch_size, seq_len, _hidden_size) = x.size3()?;
154
155 let qkv = x.matmul(&self.qkv_weight.tr()) + &self.qkv_bias;
157
158 let qkv = qkv.reshape([
160 batch_size,
161 seq_len,
162 3,
163 self.num_heads as i64,
164 self.head_dim as i64,
165 ]);
166
167 let qkv = qkv.permute([2, 0, 3, 1, 4]);
169
170 let q = qkv.get(0) * self.scale;
172 let k = qkv.get(1);
173 let v = qkv.get(2);
174
175 let attn = q.matmul(&k.transpose(-2, -1));
177 let attn = attn.softmax(-1, attn.kind());
178
179 let out = attn.matmul(&v);
181
182 let out = out.transpose(1, 2).contiguous().reshape([
184 batch_size,
185 seq_len,
186 (self.num_heads * self.head_dim) as i64,
187 ]);
188
189 let out = out.matmul(&self.proj_weight.tr()) + &self.proj_bias;
191
192 Ok(out)
193 }
194}
195
196#[derive(Debug)]
198struct SigLIPMlp {
199 fc1_weight: Tensor,
200 fc1_bias: Tensor,
201 fc2_weight: Tensor,
202 fc2_bias: Tensor,
203}
204
205impl SigLIPMlp {
206 fn from_weights(
207 weights: &HashMap<String, Tensor>,
208 prefix: &str,
209 device: Device,
210 dtype: DType,
211 ) -> Result<Self> {
212 let fc1_weight = weights
213 .get(&format!("{prefix}.fc1.weight"))
214 .ok_or_else(|| anyhow!("Missing MLP fc1 weight: {}.fc1.weight", prefix))?
215 .to_device(device)
216 .to_kind(dtype);
217
218 let fc1_bias = weights
219 .get(&format!("{prefix}.fc1.bias"))
220 .ok_or_else(|| anyhow!("Missing MLP fc1 bias: {}.fc1.bias", prefix))?
221 .to_device(device)
222 .to_kind(dtype);
223
224 let fc2_weight = weights
225 .get(&format!("{prefix}.fc2.weight"))
226 .ok_or_else(|| anyhow!("Missing MLP fc2 weight: {}.fc2.weight", prefix))?
227 .to_device(device)
228 .to_kind(dtype);
229
230 let fc2_bias = weights
231 .get(&format!("{prefix}.fc2.bias"))
232 .ok_or_else(|| anyhow!("Missing MLP fc2 bias: {}.fc2.bias", prefix))?
233 .to_device(device)
234 .to_kind(dtype);
235
236 Ok(Self {
237 fc1_weight,
238 fc1_bias,
239 fc2_weight,
240 fc2_bias,
241 })
242 }
243
244 fn forward(&self, x: &Tensor) -> Result<Tensor> {
245 let mut x = x.matmul(&self.fc1_weight.tr()) + &self.fc1_bias;
247 x = x.gelu("none");
248
249 x = x.matmul(&self.fc2_weight.tr()) + &self.fc2_bias;
251
252 Ok(x)
253 }
254}
255
256#[derive(Debug)]
258struct SigLIPBlock {
259 norm1: LayerNorm,
260 attn: SigLIPAttention,
261 norm2: LayerNorm,
262 mlp: SigLIPMlp,
263}
264
265impl SigLIPBlock {
266 fn from_weights(
267 weights: &HashMap<String, Tensor>,
268 prefix: &str,
269 hidden_size: usize,
270 num_heads: usize,
271 device: Device,
272 dtype: DType,
273 ) -> Result<Self> {
274 let norm1 = LayerNorm::from_weights(weights, &format!("{prefix}.norm1"), device, dtype)?;
275
276 let attn = SigLIPAttention::from_weights(
277 weights,
278 &format!("{prefix}.attn"),
279 hidden_size,
280 num_heads,
281 device,
282 dtype,
283 )?;
284
285 let norm2 = LayerNorm::from_weights(weights, &format!("{prefix}.norm2"), device, dtype)?;
286
287 let mlp = SigLIPMlp::from_weights(weights, &format!("{prefix}.mlp"), device, dtype)?;
288
289 Ok(Self {
290 norm1,
291 attn,
292 norm2,
293 mlp,
294 })
295 }
296
297 fn forward(&self, x: &Tensor) -> Result<Tensor> {
298 let x = x + self.attn.forward(&self.norm1.forward(x)?)?;
300
301 let x = &x + self.mlp.forward(&self.norm2.forward(&x)?)?;
303
304 Ok(x)
305 }
306}
307
308#[derive(Debug)]
310struct AttentionPool {
311 latent: Tensor,
312 q_weight: Tensor,
313 q_bias: Tensor,
314 kv_weight: Tensor,
315 kv_bias: Tensor,
316 proj_weight: Tensor,
317 proj_bias: Tensor,
318 norm: LayerNorm,
319 mlp: SigLIPMlp,
320 num_heads: usize,
321 head_dim: usize,
322}
323
324impl AttentionPool {
325 fn from_weights(
326 weights: &HashMap<String, Tensor>,
327 prefix: &str,
328 hidden_size: usize,
329 num_heads: usize,
330 device: Device,
331 dtype: DType,
332 ) -> Result<Self> {
333 let latent = weights
334 .get(&format!("{prefix}.latent"))
335 .ok_or_else(|| anyhow!("Missing attention pool latent: {}.latent", prefix))?
336 .to_device(device)
337 .to_kind(dtype);
338
339 let q_weight = weights
340 .get(&format!("{prefix}.q.weight"))
341 .ok_or_else(|| anyhow!("Missing attention pool Q weight: {}.q.weight", prefix))?
342 .to_device(device)
343 .to_kind(dtype);
344
345 let q_bias = weights
346 .get(&format!("{prefix}.q.bias"))
347 .ok_or_else(|| anyhow!("Missing attention pool Q bias: {}.q.bias", prefix))?
348 .to_device(device)
349 .to_kind(dtype);
350
351 let kv_weight = weights
352 .get(&format!("{prefix}.kv.weight"))
353 .ok_or_else(|| anyhow!("Missing attention pool KV weight: {}.kv.weight", prefix))?
354 .to_device(device)
355 .to_kind(dtype);
356
357 let kv_bias = weights
358 .get(&format!("{prefix}.kv.bias"))
359 .ok_or_else(|| anyhow!("Missing attention pool KV bias: {}.kv.bias", prefix))?
360 .to_device(device)
361 .to_kind(dtype);
362
363 let proj_weight = weights
364 .get(&format!("{prefix}.proj.weight"))
365 .ok_or_else(|| anyhow!("Missing attention pool proj weight: {}.proj.weight", prefix))?
366 .to_device(device)
367 .to_kind(dtype);
368
369 let proj_bias = weights
370 .get(&format!("{prefix}.proj.bias"))
371 .ok_or_else(|| anyhow!("Missing attention pool proj bias: {}.proj.bias", prefix))?
372 .to_device(device)
373 .to_kind(dtype);
374
375 let norm = LayerNorm::from_weights(weights, &format!("{prefix}.norm"), device, dtype)?;
376
377 let mlp = SigLIPMlp::from_weights(weights, &format!("{prefix}.mlp"), device, dtype)?;
378
379 let head_dim = hidden_size / num_heads;
380
381 Ok(Self {
382 latent,
383 q_weight,
384 q_bias,
385 kv_weight,
386 kv_bias,
387 proj_weight,
388 proj_bias,
389 norm,
390 mlp,
391 num_heads,
392 head_dim,
393 })
394 }
395
396 fn forward(&self, x: &Tensor) -> Result<Tensor> {
397 let (batch_size, seq_len, hidden_size) = x.size3()?;
398
399 let latent = self.latent.squeeze(); let latent = if latent.dim() == 1 {
403 latent.unsqueeze(0)
405 } else {
406 latent
408 };
409 let latent = latent.unsqueeze(0).expand([batch_size, -1, -1], false);
410
411 let q = latent.matmul(&self.q_weight.tr()) + &self.q_bias;
413 let num_queries = q.size()[1];
414
415 let kv = x.matmul(&self.kv_weight.tr()) + &self.kv_bias;
417 let kv = kv.reshape([
418 batch_size,
419 seq_len,
420 2,
421 self.num_heads as i64,
422 self.head_dim as i64,
423 ]);
424 let kv = kv.permute([2, 0, 3, 1, 4]);
425 let k = kv.get(0);
426 let v = kv.get(1);
427
428 let q = q.reshape([
430 batch_size,
431 num_queries,
432 self.num_heads as i64,
433 self.head_dim as i64,
434 ]);
435 let q = q.transpose(1, 2); let scale = 1.0 / (self.head_dim as f64).sqrt();
439 let attn = q.matmul(&k.transpose(-2, -1)) * scale;
440 let attn = attn.softmax(-1, attn.kind());
441
442 let out = attn.matmul(&v);
444
445 let out = out
447 .transpose(1, 2)
448 .contiguous()
449 .reshape([batch_size, num_queries, hidden_size]);
450
451 let out = out.matmul(&self.proj_weight.tr()) + &self.proj_bias;
453
454 let out = &out + self.mlp.forward(&self.norm.forward(&out)?)?;
456
457 Ok(out)
458 }
459}
460
461pub struct SigLIPVisionTransformer {
463 patch_embed_weight: Tensor,
464 patch_embed_bias: Option<Tensor>,
465 pos_embed: Tensor,
466 blocks: Vec<SigLIPBlock>,
467 norm: LayerNorm,
468 attn_pool: Option<AttentionPool>,
469 config: SigLIPConfig,
470}
471
472impl SigLIPVisionTransformer {
473 pub fn from_weights(
475 config: &SigLIPConfig,
476 weights: &HashMap<String, Tensor>,
477 prefix: &str,
478 device: Device,
479 dtype: DType,
480 ) -> Result<Self> {
481 debug!("Loading SigLIP vision transformer with prefix: {}", prefix);
482
483 let patch_embed_weight = weights
485 .get(&format!("{prefix}.patch_embed.proj.weight"))
486 .ok_or_else(|| {
487 anyhow!(
488 "Missing patch embed weight: {}.patch_embed.proj.weight",
489 prefix
490 )
491 })?
492 .to_device(device)
493 .to_kind(dtype);
494
495 let patch_embed_bias = weights
496 .get(&format!("{prefix}.patch_embed.proj.bias"))
497 .map(|t| t.to_device(device).to_kind(dtype));
498
499 let pos_embed = weights
501 .get(&format!("{prefix}.pos_embed"))
502 .ok_or_else(|| anyhow!("Missing position embeddings: {}.pos_embed", prefix))?
503 .to_device(device)
504 .to_kind(dtype);
505
506 debug!("Position embeddings shape: {:?}", pos_embed.size());
507
508 let mut blocks = Vec::new();
510 for i in 0..config.num_hidden_layers {
511 let block_prefix = format!("{prefix}.blocks.{i}");
512 let block = SigLIPBlock::from_weights(
513 weights,
514 &block_prefix,
515 config.hidden_size,
516 config.num_attention_heads,
517 device,
518 dtype,
519 )?;
520 blocks.push(block);
521 }
522 debug!("Loaded {} transformer blocks", blocks.len());
523
524 let norm = LayerNorm::from_weights(weights, &format!("{prefix}.norm"), device, dtype)?;
526
527 let attn_pool = if config.use_attention_pool {
529 Some(AttentionPool::from_weights(
530 weights,
531 &format!("{prefix}.attn_pool"),
532 config.hidden_size,
533 config.num_attention_heads,
534 device,
535 dtype,
536 )?)
537 } else {
538 None
539 };
540
541 Ok(Self {
542 patch_embed_weight,
543 patch_embed_bias,
544 pos_embed,
545 blocks,
546 norm,
547 attn_pool,
548 config: config.clone(),
549 })
550 }
551
552 pub fn forward(&self, images: &Tensor) -> Result<Tensor> {
554 let batch_size = images.size()[0];
555 debug!("SigLIP forward: input shape {:?}", images.size());
556
557 let images = images.to_kind(self.patch_embed_weight.kind());
559
560 let stride = self.config.patch_size as i64;
562 let mut x = images.conv2d(
563 &self.patch_embed_weight,
564 self.patch_embed_bias.as_ref(),
565 [stride, stride], [0, 0], [1, 1], 1, );
570
571 let (_, hidden_size, h, w) = x.size4()?;
573 let num_patches = h * w;
574 x = x
575 .permute([0, 2, 3, 1]) .reshape([batch_size, num_patches, hidden_size]);
577
578 debug!("After patch embed: {:?}", x.size());
579
580 x += &self.pos_embed;
583
584 for (i, block) in self.blocks.iter().enumerate() {
586 x = block.forward(&x)?;
587 if i == 0 || i == self.blocks.len() - 1 {
588 debug!("After block {}: {:?}", i, x.size());
589 }
590 }
591
592 x = self.norm.forward(&x)?;
594
595 if let Some(ref attn_pool) = self.attn_pool {
597 x = attn_pool.forward(&x)?;
598 debug!("After attention pooling: {:?}", x.size());
599 }
600
601 Ok(x)
602 }
603}
604
605#[cfg(test)]
606mod tests {
607 use super::*;
608
609 #[test]
610 fn test_siglip_config_default() {
611 let config = SigLIPConfig::default();
612 assert_eq!(config.image_size, 384);
613 assert_eq!(config.patch_size, 16);
614 assert_eq!(config.hidden_size, 1024);
615 assert_eq!(config.num_hidden_layers, 24);
616 }
617}