Skip to main content

hyprstream_inference/runtime/architectures/
siglip.rs

1//! SigLIP (Sigmoid Loss for Language-Image Pre-training) Vision Transformer
2//!
3//! Implementation based on the timm (PyTorch Image Models) SigLIP architecture.
4//! This is a standalone implementation that can be used by multimodal models like Janus.
5//!
6//! Architecture:
7//! - Patch embedding (Conv2d)
8//! - Position embeddings (learned)
9//! - Transformer blocks (self-attention + MLP with pre-norm)
10//! - Final layer norm
11//! - Optional attention pooling head
12
13use anyhow::{anyhow, Result};
14use std::collections::HashMap;
15use tch::{Device, Kind as DType, Tensor};
16use tracing::debug;
17
18/// Configuration for SigLIP vision encoder
19#[derive(Debug, Clone)]
20pub struct SigLIPConfig {
21    pub image_size: usize,
22    pub patch_size: usize,
23    pub hidden_size: usize,
24    pub num_hidden_layers: usize,
25    pub num_attention_heads: usize,
26    pub intermediate_size: usize,
27    pub use_attention_pool: bool,
28}
29
30impl Default for SigLIPConfig {
31    fn default() -> Self {
32        // SigLIP Large/16 @ 384px defaults
33        Self {
34            image_size: 384,
35            patch_size: 16,
36            hidden_size: 1024,
37            num_hidden_layers: 24,
38            num_attention_heads: 16,
39            intermediate_size: 4096,
40            use_attention_pool: true,
41        }
42    }
43}
44
45/// Layer normalization
46#[derive(Debug)]
47struct LayerNorm {
48    weight: Tensor,
49    bias: Tensor,
50    eps: f64,
51}
52
53impl LayerNorm {
54    fn from_weights(
55        weights: &HashMap<String, Tensor>,
56        prefix: &str,
57        device: Device,
58        dtype: DType,
59    ) -> Result<Self> {
60        let weight_key = format!("{prefix}.weight");
61        let bias_key = format!("{prefix}.bias");
62
63        let weight = weights
64            .get(&weight_key)
65            .ok_or_else(|| anyhow!("Missing LayerNorm weight: {}", weight_key))?
66            .to_device(device)
67            .to_kind(dtype);
68
69        let bias = weights
70            .get(&bias_key)
71            .ok_or_else(|| anyhow!("Missing LayerNorm bias: {}", bias_key))?
72            .to_device(device)
73            .to_kind(dtype);
74
75        Ok(Self {
76            weight,
77            bias,
78            eps: 1e-6,
79        })
80    }
81
82    fn forward(&self, x: &Tensor) -> Result<Tensor> {
83        Ok(x.layer_norm(
84            [self.weight.size()[0]],
85            Some(&self.weight),
86            Some(&self.bias),
87            self.eps,
88            true,
89        ))
90    }
91}
92
93/// Self-attention layer with fused QKV projection
94#[derive(Debug)]
95struct SigLIPAttention {
96    qkv_weight: Tensor,
97    qkv_bias: Tensor,
98    proj_weight: Tensor,
99    proj_bias: Tensor,
100    num_heads: usize,
101    head_dim: usize,
102    scale: f64,
103}
104
105impl SigLIPAttention {
106    fn from_weights(
107        weights: &HashMap<String, Tensor>,
108        prefix: &str,
109        hidden_size: usize,
110        num_heads: usize,
111        device: Device,
112        dtype: DType,
113    ) -> Result<Self> {
114        let qkv_weight = weights
115            .get(&format!("{prefix}.qkv.weight"))
116            .ok_or_else(|| anyhow!("Missing QKV weight: {}.qkv.weight", prefix))?
117            .to_device(device)
118            .to_kind(dtype);
119
120        let qkv_bias = weights
121            .get(&format!("{prefix}.qkv.bias"))
122            .ok_or_else(|| anyhow!("Missing QKV bias: {}.qkv.bias", prefix))?
123            .to_device(device)
124            .to_kind(dtype);
125
126        let proj_weight = weights
127            .get(&format!("{prefix}.proj.weight"))
128            .ok_or_else(|| anyhow!("Missing proj weight: {}.proj.weight", prefix))?
129            .to_device(device)
130            .to_kind(dtype);
131
132        let proj_bias = weights
133            .get(&format!("{prefix}.proj.bias"))
134            .ok_or_else(|| anyhow!("Missing proj bias: {}.proj.bias", prefix))?
135            .to_device(device)
136            .to_kind(dtype);
137
138        let head_dim = hidden_size / num_heads;
139        let scale = 1.0 / (head_dim as f64).sqrt();
140
141        Ok(Self {
142            qkv_weight,
143            qkv_bias,
144            proj_weight,
145            proj_bias,
146            num_heads,
147            head_dim,
148            scale,
149        })
150    }
151
152    fn forward(&self, x: &Tensor) -> Result<Tensor> {
153        let (batch_size, seq_len, _hidden_size) = x.size3()?;
154
155        // QKV projection: [B, N, hidden_size] -> [B, N, 3 * hidden_size]
156        let qkv = x.matmul(&self.qkv_weight.tr()) + &self.qkv_bias;
157
158        // Reshape to [B, N, 3, num_heads, head_dim]
159        let qkv = qkv.reshape([
160            batch_size,
161            seq_len,
162            3,
163            self.num_heads as i64,
164            self.head_dim as i64,
165        ]);
166
167        // Permute to [3, B, num_heads, N, head_dim]
168        let qkv = qkv.permute([2, 0, 3, 1, 4]);
169
170        // Split into Q, K, V
171        let q = qkv.get(0) * self.scale;
172        let k = qkv.get(1);
173        let v = qkv.get(2);
174
175        // Attention scores: [B, num_heads, N, N]
176        let attn = q.matmul(&k.transpose(-2, -1));
177        let attn = attn.softmax(-1, attn.kind());
178
179        // Apply attention to values: [B, num_heads, N, head_dim]
180        let out = attn.matmul(&v);
181
182        // Reshape back: [B, N, hidden_size]
183        let out = out.transpose(1, 2).contiguous().reshape([
184            batch_size,
185            seq_len,
186            (self.num_heads * self.head_dim) as i64,
187        ]);
188
189        // Output projection
190        let out = out.matmul(&self.proj_weight.tr()) + &self.proj_bias;
191
192        Ok(out)
193    }
194}
195
196/// MLP block with GELU activation
197#[derive(Debug)]
198struct SigLIPMlp {
199    fc1_weight: Tensor,
200    fc1_bias: Tensor,
201    fc2_weight: Tensor,
202    fc2_bias: Tensor,
203}
204
205impl SigLIPMlp {
206    fn from_weights(
207        weights: &HashMap<String, Tensor>,
208        prefix: &str,
209        device: Device,
210        dtype: DType,
211    ) -> Result<Self> {
212        let fc1_weight = weights
213            .get(&format!("{prefix}.fc1.weight"))
214            .ok_or_else(|| anyhow!("Missing MLP fc1 weight: {}.fc1.weight", prefix))?
215            .to_device(device)
216            .to_kind(dtype);
217
218        let fc1_bias = weights
219            .get(&format!("{prefix}.fc1.bias"))
220            .ok_or_else(|| anyhow!("Missing MLP fc1 bias: {}.fc1.bias", prefix))?
221            .to_device(device)
222            .to_kind(dtype);
223
224        let fc2_weight = weights
225            .get(&format!("{prefix}.fc2.weight"))
226            .ok_or_else(|| anyhow!("Missing MLP fc2 weight: {}.fc2.weight", prefix))?
227            .to_device(device)
228            .to_kind(dtype);
229
230        let fc2_bias = weights
231            .get(&format!("{prefix}.fc2.bias"))
232            .ok_or_else(|| anyhow!("Missing MLP fc2 bias: {}.fc2.bias", prefix))?
233            .to_device(device)
234            .to_kind(dtype);
235
236        Ok(Self {
237            fc1_weight,
238            fc1_bias,
239            fc2_weight,
240            fc2_bias,
241        })
242    }
243
244    fn forward(&self, x: &Tensor) -> Result<Tensor> {
245        // fc1 + GELU
246        let mut x = x.matmul(&self.fc1_weight.tr()) + &self.fc1_bias;
247        x = x.gelu("none");
248
249        // fc2
250        x = x.matmul(&self.fc2_weight.tr()) + &self.fc2_bias;
251
252        Ok(x)
253    }
254}
255
256/// Transformer block (attention + MLP with pre-norm and residual)
257#[derive(Debug)]
258struct SigLIPBlock {
259    norm1: LayerNorm,
260    attn: SigLIPAttention,
261    norm2: LayerNorm,
262    mlp: SigLIPMlp,
263}
264
265impl SigLIPBlock {
266    fn from_weights(
267        weights: &HashMap<String, Tensor>,
268        prefix: &str,
269        hidden_size: usize,
270        num_heads: usize,
271        device: Device,
272        dtype: DType,
273    ) -> Result<Self> {
274        let norm1 = LayerNorm::from_weights(weights, &format!("{prefix}.norm1"), device, dtype)?;
275
276        let attn = SigLIPAttention::from_weights(
277            weights,
278            &format!("{prefix}.attn"),
279            hidden_size,
280            num_heads,
281            device,
282            dtype,
283        )?;
284
285        let norm2 = LayerNorm::from_weights(weights, &format!("{prefix}.norm2"), device, dtype)?;
286
287        let mlp = SigLIPMlp::from_weights(weights, &format!("{prefix}.mlp"), device, dtype)?;
288
289        Ok(Self {
290            norm1,
291            attn,
292            norm2,
293            mlp,
294        })
295    }
296
297    fn forward(&self, x: &Tensor) -> Result<Tensor> {
298        // Pre-norm + attention + residual
299        let x = x + self.attn.forward(&self.norm1.forward(x)?)?;
300
301        // Pre-norm + MLP + residual
302        let x = &x + self.mlp.forward(&self.norm2.forward(&x)?)?;
303
304        Ok(x)
305    }
306}
307
308/// Attention pooling head (MAPool - Multihead Attention Pooling)
309#[derive(Debug)]
310struct AttentionPool {
311    latent: Tensor,
312    q_weight: Tensor,
313    q_bias: Tensor,
314    kv_weight: Tensor,
315    kv_bias: Tensor,
316    proj_weight: Tensor,
317    proj_bias: Tensor,
318    norm: LayerNorm,
319    mlp: SigLIPMlp,
320    num_heads: usize,
321    head_dim: usize,
322}
323
324impl AttentionPool {
325    fn from_weights(
326        weights: &HashMap<String, Tensor>,
327        prefix: &str,
328        hidden_size: usize,
329        num_heads: usize,
330        device: Device,
331        dtype: DType,
332    ) -> Result<Self> {
333        let latent = weights
334            .get(&format!("{prefix}.latent"))
335            .ok_or_else(|| anyhow!("Missing attention pool latent: {}.latent", prefix))?
336            .to_device(device)
337            .to_kind(dtype);
338
339        let q_weight = weights
340            .get(&format!("{prefix}.q.weight"))
341            .ok_or_else(|| anyhow!("Missing attention pool Q weight: {}.q.weight", prefix))?
342            .to_device(device)
343            .to_kind(dtype);
344
345        let q_bias = weights
346            .get(&format!("{prefix}.q.bias"))
347            .ok_or_else(|| anyhow!("Missing attention pool Q bias: {}.q.bias", prefix))?
348            .to_device(device)
349            .to_kind(dtype);
350
351        let kv_weight = weights
352            .get(&format!("{prefix}.kv.weight"))
353            .ok_or_else(|| anyhow!("Missing attention pool KV weight: {}.kv.weight", prefix))?
354            .to_device(device)
355            .to_kind(dtype);
356
357        let kv_bias = weights
358            .get(&format!("{prefix}.kv.bias"))
359            .ok_or_else(|| anyhow!("Missing attention pool KV bias: {}.kv.bias", prefix))?
360            .to_device(device)
361            .to_kind(dtype);
362
363        let proj_weight = weights
364            .get(&format!("{prefix}.proj.weight"))
365            .ok_or_else(|| anyhow!("Missing attention pool proj weight: {}.proj.weight", prefix))?
366            .to_device(device)
367            .to_kind(dtype);
368
369        let proj_bias = weights
370            .get(&format!("{prefix}.proj.bias"))
371            .ok_or_else(|| anyhow!("Missing attention pool proj bias: {}.proj.bias", prefix))?
372            .to_device(device)
373            .to_kind(dtype);
374
375        let norm = LayerNorm::from_weights(weights, &format!("{prefix}.norm"), device, dtype)?;
376
377        let mlp = SigLIPMlp::from_weights(weights, &format!("{prefix}.mlp"), device, dtype)?;
378
379        let head_dim = hidden_size / num_heads;
380
381        Ok(Self {
382            latent,
383            q_weight,
384            q_bias,
385            kv_weight,
386            kv_bias,
387            proj_weight,
388            proj_bias,
389            norm,
390            mlp,
391            num_heads,
392            head_dim,
393        })
394    }
395
396    fn forward(&self, x: &Tensor) -> Result<Tensor> {
397        let (batch_size, seq_len, hidden_size) = x.size3()?;
398
399        // Expand latent queries: [num_queries, hidden_size] -> [B, num_queries, hidden_size]
400        // The latent tensor may have extra dimensions, so squeeze first
401        let latent = self.latent.squeeze(); // Remove any singleton dimensions
402        let latent = if latent.dim() == 1 {
403            // If it's 1D [hidden_size], reshape to [1, hidden_size]
404            latent.unsqueeze(0)
405        } else {
406            // Should be 2D [num_queries, hidden_size]
407            latent
408        };
409        let latent = latent.unsqueeze(0).expand([batch_size, -1, -1], false);
410
411        // Q from latent
412        let q = latent.matmul(&self.q_weight.tr()) + &self.q_bias;
413        let num_queries = q.size()[1];
414
415        // K, V from input
416        let kv = x.matmul(&self.kv_weight.tr()) + &self.kv_bias;
417        let kv = kv.reshape([
418            batch_size,
419            seq_len,
420            2,
421            self.num_heads as i64,
422            self.head_dim as i64,
423        ]);
424        let kv = kv.permute([2, 0, 3, 1, 4]);
425        let k = kv.get(0);
426        let v = kv.get(1);
427
428        // Reshape Q: [B, num_queries, num_heads, head_dim]
429        let q = q.reshape([
430            batch_size,
431            num_queries,
432            self.num_heads as i64,
433            self.head_dim as i64,
434        ]);
435        let q = q.transpose(1, 2); // [B, num_heads, num_queries, head_dim]
436
437        // Attention
438        let scale = 1.0 / (self.head_dim as f64).sqrt();
439        let attn = q.matmul(&k.transpose(-2, -1)) * scale;
440        let attn = attn.softmax(-1, attn.kind());
441
442        // Apply attention: [B, num_heads, num_queries, head_dim]
443        let out = attn.matmul(&v);
444
445        // Reshape: [B, num_queries, hidden_size]
446        let out = out
447            .transpose(1, 2)
448            .contiguous()
449            .reshape([batch_size, num_queries, hidden_size]);
450
451        // Output projection
452        let out = out.matmul(&self.proj_weight.tr()) + &self.proj_bias;
453
454        // MLP (with residual and norm)
455        let out = &out + self.mlp.forward(&self.norm.forward(&out)?)?;
456
457        Ok(out)
458    }
459}
460
461/// SigLIP Vision Transformer
462pub struct SigLIPVisionTransformer {
463    patch_embed_weight: Tensor,
464    patch_embed_bias: Option<Tensor>,
465    pos_embed: Tensor,
466    blocks: Vec<SigLIPBlock>,
467    norm: LayerNorm,
468    attn_pool: Option<AttentionPool>,
469    config: SigLIPConfig,
470}
471
472impl SigLIPVisionTransformer {
473    /// Load SigLIP model from weights
474    pub fn from_weights(
475        config: &SigLIPConfig,
476        weights: &HashMap<String, Tensor>,
477        prefix: &str,
478        device: Device,
479        dtype: DType,
480    ) -> Result<Self> {
481        debug!("Loading SigLIP vision transformer with prefix: {}", prefix);
482
483        // Patch embedding (Conv2d projection)
484        let patch_embed_weight = weights
485            .get(&format!("{prefix}.patch_embed.proj.weight"))
486            .ok_or_else(|| {
487                anyhow!(
488                    "Missing patch embed weight: {}.patch_embed.proj.weight",
489                    prefix
490                )
491            })?
492            .to_device(device)
493            .to_kind(dtype);
494
495        let patch_embed_bias = weights
496            .get(&format!("{prefix}.patch_embed.proj.bias"))
497            .map(|t| t.to_device(device).to_kind(dtype));
498
499        // Position embeddings
500        let pos_embed = weights
501            .get(&format!("{prefix}.pos_embed"))
502            .ok_or_else(|| anyhow!("Missing position embeddings: {}.pos_embed", prefix))?
503            .to_device(device)
504            .to_kind(dtype);
505
506        debug!("Position embeddings shape: {:?}", pos_embed.size());
507
508        // Transformer blocks
509        let mut blocks = Vec::new();
510        for i in 0..config.num_hidden_layers {
511            let block_prefix = format!("{prefix}.blocks.{i}");
512            let block = SigLIPBlock::from_weights(
513                weights,
514                &block_prefix,
515                config.hidden_size,
516                config.num_attention_heads,
517                device,
518                dtype,
519            )?;
520            blocks.push(block);
521        }
522        debug!("Loaded {} transformer blocks", blocks.len());
523
524        // Final layer norm
525        let norm = LayerNorm::from_weights(weights, &format!("{prefix}.norm"), device, dtype)?;
526
527        // Optional attention pooling
528        let attn_pool = if config.use_attention_pool {
529            Some(AttentionPool::from_weights(
530                weights,
531                &format!("{prefix}.attn_pool"),
532                config.hidden_size,
533                config.num_attention_heads,
534                device,
535                dtype,
536            )?)
537        } else {
538            None
539        };
540
541        Ok(Self {
542            patch_embed_weight,
543            patch_embed_bias,
544            pos_embed,
545            blocks,
546            norm,
547            attn_pool,
548            config: config.clone(),
549        })
550    }
551
552    /// Forward pass
553    pub fn forward(&self, images: &Tensor) -> Result<Tensor> {
554        let batch_size = images.size()[0];
555        debug!("SigLIP forward: input shape {:?}", images.size());
556
557        // Convert input to match model dtype (weights are typically F16/BF16)
558        let images = images.to_kind(self.patch_embed_weight.kind());
559
560        // Patch embedding: [B, 3, H, W] -> [B, hidden_size, num_patches_h, num_patches_w]
561        let stride = self.config.patch_size as i64;
562        let mut x = images.conv2d(
563            &self.patch_embed_weight,
564            self.patch_embed_bias.as_ref(),
565            [stride, stride], // stride
566            [0, 0],           // padding
567            [1, 1],           // dilation
568            1,                // groups
569        );
570
571        // Reshape: [B, hidden_size, H', W'] -> [B, num_patches, hidden_size]
572        let (_, hidden_size, h, w) = x.size4()?;
573        let num_patches = h * w;
574        x = x
575            .permute([0, 2, 3, 1]) // [B, H', W', hidden_size]
576            .reshape([batch_size, num_patches, hidden_size]);
577
578        debug!("After patch embed: {:?}", x.size());
579
580        // Add position embeddings
581        // pos_embed is typically [1, num_patches, hidden_size]
582        x += &self.pos_embed;
583
584        // Transformer blocks
585        for (i, block) in self.blocks.iter().enumerate() {
586            x = block.forward(&x)?;
587            if i == 0 || i == self.blocks.len() - 1 {
588                debug!("After block {}: {:?}", i, x.size());
589            }
590        }
591
592        // Final layer norm
593        x = self.norm.forward(&x)?;
594
595        // Optional attention pooling
596        if let Some(ref attn_pool) = self.attn_pool {
597            x = attn_pool.forward(&x)?;
598            debug!("After attention pooling: {:?}", x.size());
599        }
600
601        Ok(x)
602    }
603}
604
605#[cfg(test)]
606mod tests {
607    use super::*;
608
609    #[test]
610    fn test_siglip_config_default() {
611        let config = SigLIPConfig::default();
612        assert_eq!(config.image_size, 384);
613        assert_eq!(config.patch_size, 16);
614        assert_eq!(config.hidden_size, 1024);
615        assert_eq!(config.num_hidden_layers, 24);
616    }
617}