Skip to main content

hyprstream_inference/runtime/architectures/
janus.rs

1//! Janus multimodal vision-language model implementation
2//!
3//! Janus combines vision and language understanding with optional image generation.
4//! It uses a composition pattern to reuse existing language models (typically Llama)
5//! while adding vision encoders and cross-modal alignment.
6//!
7//! NOTE: This module contains vision transformer components that are planned for
8//! future multimodal support. They are currently unused but retained for the
9//! upcoming Janus VLM integration.
10
11#![allow(dead_code, clippy::type_complexity)]
12
13use super::config::AttentionType;
14use super::{ArchitectureConfig, ModelArchitecture, ModelOperations, VisionEncoderType};
15use crate::runtime::kv_cache::LayerKVCache;
16use anyhow::{anyhow, Result};
17use std::collections::HashMap;
18use tch::{Device, Kind as DType, Tensor};
19
20/// Janus model configuration
21pub struct JanusConfig {
22    /// Configuration for the base language model
23    pub language_config: Box<dyn ArchitectureConfig>,
24    /// Vision encoder configuration
25    pub vision_config: VisionEncoderConfig,
26    /// Aligner configuration (vision to language)
27    pub aligner_config: ProjectorConfig,
28    /// Generation configuration (optional)
29    pub generation_config: Option<GenerationConfig>,
30    /// Device for computation
31    pub device: Device,
32    /// Data type for computation
33    pub dtype: DType,
34}
35
36impl ArchitectureConfig for JanusConfig {
37    fn hidden_size(&self) -> usize {
38        self.language_config.hidden_size()
39    }
40
41    fn intermediate_size(&self) -> usize {
42        self.language_config.intermediate_size()
43    }
44
45    fn num_attention_heads(&self) -> usize {
46        self.language_config.num_attention_heads()
47    }
48
49    fn num_key_value_heads(&self) -> usize {
50        self.language_config.num_key_value_heads()
51    }
52
53    fn head_dim(&self) -> usize {
54        self.language_config.head_dim()
55    }
56
57    fn vocab_size(&self) -> usize {
58        self.language_config.vocab_size()
59    }
60
61    fn max_position_embeddings(&self) -> usize {
62        self.language_config.max_position_embeddings()
63    }
64
65    fn rope_theta(&self) -> Option<f32> {
66        self.language_config.rope_theta()
67    }
68
69    fn rope_dim(&self) -> Option<usize> {
70        self.language_config.rope_dim()
71    }
72
73    fn layer_norm_eps(&self) -> f32 {
74        self.language_config.layer_norm_eps()
75    }
76
77    fn use_rms_norm(&self) -> bool {
78        self.language_config.use_rms_norm()
79    }
80
81    fn attention_type(&self) -> AttentionType {
82        self.language_config.attention_type()
83    }
84}
85
86/// Vision encoder configuration
87#[derive(Debug, Clone)]
88pub struct VisionEncoderConfig {
89    pub encoder_type: VisionEncoderType,
90    pub hidden_size: usize,
91    pub image_size: usize,
92    pub patch_size: usize,
93    pub num_layers: usize,
94    pub num_patches: usize, // (image_size / patch_size)^2
95    pub num_attention_heads: Option<usize>,
96    pub intermediate_size: Option<usize>,
97}
98
99/// MLP projector configuration (for aligners)
100#[derive(Debug, Clone)]
101pub struct ProjectorConfig {
102    pub input_dim: usize,
103    pub output_dim: usize,
104    pub hidden_dim: Option<usize>, // For 2-layer MLP
105}
106
107/// Generation pipeline configuration
108#[derive(Debug, Clone)]
109pub struct GenerationConfig {
110    pub vq_config: VQModelConfig,
111    pub gen_aligner_config: ProjectorConfig,
112    pub gen_head_config: VisionHeadConfig,
113    pub image_token_size: usize, // Vocabulary size for image tokens
114}
115
116/// VQ-VAE model configuration
117#[derive(Debug, Clone)]
118pub struct VQModelConfig {
119    pub codebook_size: usize,
120    pub latent_dim: usize,
121}
122
123/// Vision head configuration for image generation
124#[derive(Debug, Clone)]
125pub struct VisionHeadConfig {
126    pub input_dim: usize,
127    pub hidden_dim: usize,
128    pub output_dim: usize, // image_token_size
129}
130
131/// Main Janus multimodal model
132pub struct JanusModel {
133    /// Core components (required)
134    language_model: Box<dyn ModelOperations>, // Reuse existing models!
135    vision_encoder: VisionEncoder,
136    vision_aligner: MlpProjector,
137
138    /// Generation components (optional)
139    generation: Option<GenerationPipeline>,
140
141    /// Model metadata
142    device: Device,
143    dtype: DType,
144    architecture: ModelArchitecture,
145    config: JanusConfig,
146
147    /// Optimization: Cache last vision encoding
148    /// Uses Mutex instead of RefCell for thread safety (Send requirement)
149    vision_cache: parking_lot::Mutex<Option<(Vec<u8>, Tensor)>>, // (input_hash, features)
150}
151
152/// Vision encoder implementations
153#[allow(clippy::large_enum_variant)] // Created once, used frequently - boxing adds unwanted indirection
154pub enum VisionEncoder {
155    SigLIP(SigLIPVisionTower),
156    CLIP(CLIPVisionTower),
157}
158
159impl VisionEncoder {
160    fn forward(&self, images: &Tensor) -> Result<Tensor> {
161        match self {
162            Self::SigLIP(model) => model.forward(images),
163            Self::CLIP(model) => model.forward(images),
164        }
165    }
166
167    fn from_weights(
168        config: &VisionEncoderConfig,
169        weights: &HashMap<String, Tensor>,
170        prefix: &str,
171        device: Device,
172        dtype: DType,
173    ) -> Result<Self> {
174        match &config.encoder_type {
175            VisionEncoderType::SigLIP { .. } => Ok(Self::SigLIP(SigLIPVisionTower::from_weights(
176                config, weights, prefix, device, dtype,
177            )?)),
178            VisionEncoderType::CLIP { .. } => Ok(Self::CLIP(CLIPVisionTower::from_weights(
179                config, weights, prefix, device, dtype,
180            )?)),
181            _ => Err(anyhow!("Unsupported vision encoder type")),
182        }
183    }
184}
185
186/// SigLIP vision encoder (wrapper around siglip module)
187pub struct SigLIPVisionTower {
188    model: super::siglip::SigLIPVisionTransformer,
189    config: VisionEncoderConfig,
190}
191
192impl SigLIPVisionTower {
193    fn from_weights(
194        config: &VisionEncoderConfig,
195        weights: &HashMap<String, Tensor>,
196        prefix: &str,
197        device: Device,
198        dtype: DType,
199    ) -> Result<Self> {
200        // Convert VisionEncoderConfig to SigLIPConfig
201        let siglip_config = super::siglip::SigLIPConfig {
202            image_size: config.image_size,
203            patch_size: config.patch_size,
204            hidden_size: config.hidden_size,
205            num_hidden_layers: config.num_layers,
206            num_attention_heads: config.num_attention_heads.unwrap_or(16),
207            intermediate_size: config.intermediate_size.unwrap_or(config.hidden_size * 4),
208            use_attention_pool: false, // Janus does NOT use attention pooling - needs all 576 patches
209        };
210
211        let model = super::siglip::SigLIPVisionTransformer::from_weights(
212            &siglip_config,
213            weights,
214            prefix,
215            device,
216            dtype,
217        )?;
218
219        Ok(Self {
220            model,
221            config: config.clone(),
222        })
223    }
224
225    fn forward(&self, images: &Tensor) -> Result<Tensor> {
226        self.model.forward(images)
227    }
228}
229
230/// CLIP vision encoder (placeholder - not yet implemented with new architecture)
231pub struct CLIPVisionTower {
232    config: VisionEncoderConfig,
233}
234
235impl CLIPVisionTower {
236    fn from_weights(
237        _config: &VisionEncoderConfig,
238        _weights: &HashMap<String, Tensor>,
239        _prefix: &str,
240        _device: Device,
241        _dtype: DType,
242    ) -> Result<Self> {
243        // TODO: Implement CLIP vision encoder similar to SigLIP
244        // CLIP and SigLIP have very similar architectures
245        Err(anyhow!(
246            "CLIP vision encoder not yet implemented - use SigLIP instead"
247        ))
248    }
249
250    fn forward(&self, _images: &Tensor) -> Result<Tensor> {
251        Err(anyhow!(
252            "CLIP vision encoder not yet implemented - use SigLIP instead"
253        ))
254    }
255}
256
257/// Vision transformer layer
258struct VisionTransformerLayer {
259    self_attn: MultiHeadAttention,
260    mlp: Mlp,
261    ln_1: LayerNorm,
262    ln_2: LayerNorm,
263}
264
265impl VisionTransformerLayer {
266    fn from_weights(
267        weights: &HashMap<String, Tensor>,
268        prefix: &str,
269        hidden_size: usize,
270        device: Device,
271        dtype: DType,
272    ) -> Result<Self> {
273        let self_attn = MultiHeadAttention::from_weights(
274            weights,
275            &format!("{prefix}.self_attn"),
276            hidden_size,
277            device,
278            dtype,
279        )?;
280
281        let mlp = Mlp::from_weights(
282            weights,
283            &format!("{prefix}.mlp"),
284            hidden_size,
285            hidden_size * 4, // Standard 4x expansion
286            device,
287            dtype,
288        )?;
289
290        let ln_1 = LayerNorm::from_weights(
291            weights,
292            &format!("{prefix}.ln_1"),
293            hidden_size,
294            device,
295            dtype,
296        )?;
297
298        let ln_2 = LayerNorm::from_weights(
299            weights,
300            &format!("{prefix}.ln_2"),
301            hidden_size,
302            device,
303            dtype,
304        )?;
305
306        Ok(Self {
307            self_attn,
308            mlp,
309            ln_1,
310            ln_2,
311        })
312    }
313
314    fn forward(&self, x: &Tensor) -> Result<Tensor> {
315        // Pre-norm architecture
316        let mut output = x.shallow_clone();
317
318        // Self-attention with residual
319        let attn_out = self.self_attn.forward(&self.ln_1.forward(&output)?)?;
320        output = &output + &attn_out;
321
322        // MLP with residual
323        let mlp_out = self.mlp.forward(&self.ln_2.forward(&output)?)?;
324        output = &output + &mlp_out;
325
326        Ok(output)
327    }
328}
329
330/// Multi-head attention for vision transformer
331struct MultiHeadAttention {
332    q_proj: Tensor,
333    k_proj: Tensor,
334    v_proj: Tensor,
335    out_proj: Tensor,
336    num_heads: usize,
337    head_dim: usize,
338}
339
340impl MultiHeadAttention {
341    fn from_weights(
342        weights: &HashMap<String, Tensor>,
343        prefix: &str,
344        hidden_size: usize,
345        device: Device,
346        dtype: DType,
347    ) -> Result<Self> {
348        let mut q_proj = weights
349            .get(&format!("{prefix}.q_proj.weight"))
350            .ok_or_else(|| anyhow!("Missing q_proj weight"))?
351            .shallow_clone();
352        q_proj = q_proj.to_device(device);
353        q_proj = q_proj.to_kind(dtype);
354
355        let mut k_proj = weights
356            .get(&format!("{prefix}.k_proj.weight"))
357            .ok_or_else(|| anyhow!("Missing k_proj weight"))?
358            .shallow_clone();
359        k_proj = k_proj.to_device(device);
360        k_proj = k_proj.to_kind(dtype);
361
362        let mut v_proj = weights
363            .get(&format!("{prefix}.v_proj.weight"))
364            .ok_or_else(|| anyhow!("Missing v_proj weight"))?
365            .shallow_clone();
366        v_proj = v_proj.to_device(device);
367        v_proj = v_proj.to_kind(dtype);
368
369        let mut out_proj = weights
370            .get(&format!("{prefix}.out_proj.weight"))
371            .ok_or_else(|| anyhow!("Missing out_proj weight"))?
372            .shallow_clone();
373        out_proj = out_proj.to_device(device);
374        out_proj = out_proj.to_kind(dtype);
375
376        // Infer num_heads from weight shapes
377        let num_heads = 16; // Standard for many vision models, could be made configurable
378        let head_dim = hidden_size / num_heads;
379
380        Ok(Self {
381            q_proj,
382            k_proj,
383            v_proj,
384            out_proj,
385            num_heads,
386            head_dim,
387        })
388    }
389
390    fn forward(&self, x: &Tensor) -> Result<Tensor> {
391        let (batch_size, seq_len, _) = x.size3()?;
392
393        // Project to Q, K, V
394        let q = x.matmul(&self.q_proj.transpose(0, 1));
395        let k = x.matmul(&self.k_proj.transpose(0, 1));
396        let v = x.matmul(&self.v_proj.transpose(0, 1));
397
398        // Reshape for multi-head attention
399        let q = q
400            .reshape([
401                batch_size,
402                seq_len,
403                self.num_heads as i64,
404                self.head_dim as i64,
405            ])
406            .transpose(1, 2); // [B, H, N, D]
407        let k = k
408            .reshape([
409                batch_size,
410                seq_len,
411                self.num_heads as i64,
412                self.head_dim as i64,
413            ])
414            .transpose(1, 2);
415        let v = v
416            .reshape([
417                batch_size,
418                seq_len,
419                self.num_heads as i64,
420                self.head_dim as i64,
421            ])
422            .transpose(1, 2);
423
424        // Scaled dot-product attention
425        let scale = (self.head_dim as f64).sqrt();
426        let scores = q.matmul(&k.transpose(-2, -1)).divide_scalar(scale);
427        let attn_weights = scores.softmax(-1, scores.kind());
428        let attn_output = attn_weights.matmul(&v);
429
430        // Reshape back
431        let attn_output = attn_output
432            .transpose(1, 2)
433            .reshape([batch_size, seq_len, -1]);
434
435        // Output projection
436        let output = attn_output.matmul(&self.out_proj.transpose(0, 1));
437
438        Ok(output)
439    }
440}
441
442/// MLP for vision transformer
443struct Mlp {
444    fc1: Tensor,
445    fc2: Tensor,
446}
447
448impl Mlp {
449    fn from_weights(
450        weights: &HashMap<String, Tensor>,
451        prefix: &str,
452        _hidden_size: usize,
453        _intermediate_size: usize,
454        device: Device,
455        dtype: DType,
456    ) -> Result<Self> {
457        let mut fc1 = weights
458            .get(&format!("{prefix}.fc1.weight"))
459            .ok_or_else(|| anyhow!("Missing fc1 weight"))?
460            .shallow_clone();
461        fc1 = fc1.to_device(device);
462        fc1 = fc1.to_kind(dtype);
463
464        let mut fc2 = weights
465            .get(&format!("{prefix}.fc2.weight"))
466            .ok_or_else(|| anyhow!("Missing fc2 weight"))?
467            .shallow_clone();
468        fc2 = fc2.to_device(device);
469        fc2 = fc2.to_kind(dtype);
470
471        Ok(Self { fc1, fc2 })
472    }
473
474    fn forward(&self, x: &Tensor) -> Result<Tensor> {
475        let mut output = x.matmul(&self.fc1.transpose(0, 1));
476        output = output.gelu("none"); // GELU activation
477        output = output.matmul(&self.fc2.transpose(0, 1));
478        Ok(output)
479    }
480}
481
482/// Layer normalization
483struct LayerNorm {
484    weight: Tensor,
485    bias: Option<Tensor>,
486    eps: f64,
487}
488
489impl LayerNorm {
490    fn from_weights(
491        weights: &HashMap<String, Tensor>,
492        prefix: &str,
493        _hidden_size: usize,
494        device: Device,
495        dtype: DType,
496    ) -> Result<Self> {
497        let mut weight = weights
498            .get(&format!("{prefix}.weight"))
499            .ok_or_else(|| anyhow!("Missing layer norm weight"))?
500            .shallow_clone();
501        weight = weight.to_device(device);
502        weight = weight.to_kind(dtype);
503
504        let bias = weights.get(&format!("{prefix}.bias")).map(|b| {
505            let mut b = b.shallow_clone();
506            b = b.to_device(device);
507            b.to_kind(dtype)
508        });
509
510        Ok(Self {
511            weight,
512            bias,
513            eps: 1e-5, // Standard epsilon
514        })
515    }
516
517    fn forward(&self, x: &Tensor) -> Result<Tensor> {
518        // Simplified layer norm implementation
519        // TODO: Use proper layer_norm once we figure out the signature
520        let mean = x.mean_dim(&[-1i64][..], true, x.kind());
521        let var = x.var_dim(&[-1i64][..], false, true);
522        let normalized = (x - &mean) / (&var + self.eps).sqrt();
523
524        let output = &normalized * &self.weight;
525        let output = if let Some(ref bias) = self.bias {
526            output + bias
527        } else {
528            output
529        };
530        Ok(output)
531    }
532}
533
534/// MLP projector for vision-language alignment
535pub struct MlpProjector {
536    layers: Vec<Tensor>,
537    config: ProjectorConfig,
538}
539
540impl MlpProjector {
541    pub fn from_weights(
542        weights: &HashMap<String, Tensor>,
543        prefix: &str,
544        config: &ProjectorConfig,
545        device: Device,
546        dtype: DType,
547    ) -> Result<Self> {
548        use tracing::debug;
549
550        // Debug: log available keys
551        debug!("Loading MLP projector with prefix: {}", prefix);
552        let aligner_keys: Vec<_> = weights.keys().filter(|k| k.starts_with(prefix)).collect();
553        debug!("Available aligner keys: {:?}", aligner_keys);
554
555        let mut layers = Vec::new();
556
557        if let Some(_hidden_dim) = config.hidden_dim {
558            // 2-layer MLP with GELU
559            // Structure: Linear(0) -> GELU(1) -> Linear(2)
560            // Only Linear layers have weights
561
562            // First linear layer
563            let mut layer0_weight = weights
564                .get(&format!("{prefix}.layers.0.weight"))
565                .ok_or_else(|| anyhow!("Missing projector layer 0 weight"))?
566                .shallow_clone();
567            layer0_weight = layer0_weight.to_device(device);
568            layer0_weight = layer0_weight.to_kind(dtype);
569
570            let mut layer0_bias = weights
571                .get(&format!("{prefix}.layers.0.bias"))
572                .ok_or_else(|| anyhow!("Missing projector layer 0 bias"))?
573                .shallow_clone();
574            layer0_bias = layer0_bias.to_device(device);
575            layer0_bias = layer0_bias.to_kind(dtype);
576
577            // Second linear layer (index 2 in Sequential, after GELU at index 1)
578            let mut layer2_weight = weights
579                .get(&format!("{prefix}.layers.2.weight"))
580                .ok_or_else(|| anyhow!("Missing projector layer 2 weight"))?
581                .shallow_clone();
582            layer2_weight = layer2_weight.to_device(device);
583            layer2_weight = layer2_weight.to_kind(dtype);
584
585            let mut layer2_bias = weights
586                .get(&format!("{prefix}.layers.2.bias"))
587                .ok_or_else(|| anyhow!("Missing projector layer 2 bias"))?
588                .shallow_clone();
589            layer2_bias = layer2_bias.to_device(device);
590            layer2_bias = layer2_bias.to_kind(dtype);
591
592            // Store weights and biases as tuples
593            layers.push(layer0_weight);
594            layers.push(layer0_bias);
595            layers.push(layer2_weight);
596            layers.push(layer2_bias);
597        } else {
598            // Single linear layer
599            let mut weight = weights
600                .get(&format!("{prefix}.layers.0.weight"))
601                .ok_or_else(|| anyhow!("Missing projector weight"))?
602                .shallow_clone();
603            weight = weight.to_device(device);
604            weight = weight.to_kind(dtype);
605
606            let mut bias = weights
607                .get(&format!("{prefix}.layers.0.bias"))
608                .ok_or_else(|| anyhow!("Missing projector bias"))?
609                .shallow_clone();
610            bias = bias.to_device(device);
611            bias = bias.to_kind(dtype);
612
613            layers.push(weight);
614            layers.push(bias);
615        }
616
617        Ok(Self {
618            layers,
619            config: config.clone(),
620        })
621    }
622
623    pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
624        if self.config.hidden_dim.is_some() {
625            // 2-layer MLP: Linear -> GELU -> Linear
626            // layers = [weight0, bias0, weight2, bias2]
627
628            // First linear layer
629            let mut output = x.matmul(&self.layers[0].transpose(0, 1)) + &self.layers[1];
630
631            // GELU activation
632            output = output.gelu("none");
633
634            // Second linear layer
635            output = output.matmul(&self.layers[2].transpose(0, 1)) + &self.layers[3];
636
637            Ok(output)
638        } else {
639            // Single linear layer
640            // layers = [weight, bias]
641            let output = x.matmul(&self.layers[0].transpose(0, 1)) + &self.layers[1];
642            Ok(output)
643        }
644    }
645}
646
647/// Generation pipeline for image generation
648pub struct GenerationPipeline {
649    vq_model: VQModel,
650    gen_aligner: MlpProjector,
651    gen_head: VisionHead,
652    gen_embed: Tensor, // [image_token_size, hidden_size]
653}
654
655/// VQ-VAE model for image generation
656pub struct VQModel {
657    // Simplified VQ model - would need full encoder/decoder in practice
658    codebook: Tensor,
659    config: VQModelConfig,
660}
661
662/// Vision head for generating image tokens
663pub struct VisionHead {
664    output_mlp_projector: Tensor,
665    vision_head: Tensor,
666}
667
668impl JanusModel {
669    /// Create Janus model from weights
670    pub fn from_weights(
671        weights: HashMap<String, Tensor>,
672        config: JanusConfig,
673        device: Device,
674        dtype: DType,
675    ) -> Result<Self> {
676        // Extract weights by prefix
677        let (language_weights, vision_weights, aligner_weights, _gen_weights) =
678            Self::extract_weights_by_prefix(&weights)?;
679
680        // Load language model
681        let language_model = Self::load_language_model(language_weights, &config, device, dtype)?;
682
683        // Load vision encoder
684        // Janus stores vision weights under vision_model.vision_tower prefix
685        let vision_encoder = VisionEncoder::from_weights(
686            &config.vision_config,
687            &vision_weights,
688            "vision_model.vision_tower",
689            device,
690            dtype,
691        )?;
692
693        // Load vision aligner
694        let vision_aligner = MlpProjector::from_weights(
695            &aligner_weights,
696            "aligner",
697            &config.aligner_config,
698            device,
699            dtype,
700        )?;
701
702        // Load generation components if configured
703        let generation = if let Some(_gen_config) = &config.generation_config {
704            // TODO: Implement generation pipeline loading
705            None
706        } else {
707            None
708        };
709
710        // Create architecture enum
711        let architecture = ModelArchitecture::Janus {
712            base_architecture: Box::new(ModelArchitecture::Llama { version: 3 }), // TODO: Detect from config
713            vision_encoder: config.vision_config.encoder_type.clone(),
714            has_generation: generation.is_some(),
715        };
716
717        Ok(Self {
718            language_model,
719            vision_encoder,
720            vision_aligner,
721            generation,
722            device,
723            dtype,
724            architecture,
725            config,
726            vision_cache: parking_lot::Mutex::new(None),
727        })
728    }
729
730    /// Extract weights by component prefix
731    fn extract_weights_by_prefix(
732        weights: &HashMap<String, Tensor>,
733    ) -> Result<(
734        HashMap<String, Tensor>, // language_model.*
735        HashMap<String, Tensor>, // vision_model.*
736        HashMap<String, Tensor>, // aligner.*
737        HashMap<String, Tensor>, // gen_* components - currently unused
738    )> {
739        let mut language_weights = HashMap::new();
740        let mut vision_weights = HashMap::new();
741        let mut aligner_weights = HashMap::new();
742        let mut _gen_weights = HashMap::new();
743
744        for (key, tensor) in weights {
745            if let Some(suffix) = key.strip_prefix("language_model.") {
746                // Remove prefix for language model
747                language_weights.insert(suffix.to_owned(), tensor.shallow_clone());
748            } else if key.starts_with("vision_model.") {
749                vision_weights.insert(key.clone(), tensor.shallow_clone());
750            } else if key.starts_with("aligner.") {
751                aligner_weights.insert(key.clone(), tensor.shallow_clone());
752            } else if key.starts_with("gen_") {
753                _gen_weights.insert(key.clone(), tensor.shallow_clone());
754            }
755        }
756
757        Ok((
758            language_weights,
759            vision_weights,
760            aligner_weights,
761            _gen_weights,
762        ))
763    }
764
765    /// Load the language model component
766    fn load_language_model(
767        mut weights: HashMap<String, Tensor>,
768        _config: &JanusConfig,
769        device: Device,
770        dtype: DType,
771    ) -> Result<Box<dyn ModelOperations>> {
772        // For now, assume Llama - could be made more flexible
773        use super::llama::LlamaModel;
774
775        // Create Llama config from Janus language config
776        // This would need proper conversion logic
777        let llama_model = LlamaModel::from_weights(&mut weights, &device, dtype)?;
778
779        Ok(Box::new(llama_model))
780    }
781
782    /// Prepare inputs by combining vision and text embeddings
783    /// Prepare inputs by combining vision and text embeddings
784    pub fn prepare_inputs_embeds(
785        &self,
786        input_ids: &Tensor,
787        pixel_values: Option<&Tensor>,
788        images_seq_mask: Option<&Tensor>,
789        images_emb_mask: Option<&Tensor>,
790    ) -> Result<Tensor> {
791        use tracing::debug;
792
793        // Get text embeddings from language model
794        let mut inputs_embeds = self.language_model.embed_tokens(input_ids)?;
795
796        // If we have images, process them and insert into sequence
797        if let (Some(images), Some(seq_mask), Some(emb_mask)) =
798            (pixel_values, images_seq_mask, images_emb_mask)
799        {
800            let images_shape = images.size();
801
802            // Handle both [batch, 3, h, w] and [batch, n_images, 3, h, w] formats
803            let (batch_size, n_images, channels, height, width) = if images_shape.len() == 4 {
804                // Single image: [batch, 3, h, w] → treat as [batch, 1, 3, h, w]
805                (
806                    images_shape[0],
807                    1,
808                    images_shape[1],
809                    images_shape[2],
810                    images_shape[3],
811                )
812            } else if images_shape.len() == 5 {
813                // Multiple images: [batch, n_images, 3, h, w]
814                (
815                    images_shape[0],
816                    images_shape[1],
817                    images_shape[2],
818                    images_shape[3],
819                    images_shape[4],
820                )
821            } else {
822                return Err(anyhow!(
823                    "Unexpected pixel_values shape: {:?}. Expected [batch, 3, h, w] or [batch, n_images, 3, h, w]",
824                    images_shape
825                ));
826            };
827
828            debug!(
829                "Processing {} images: batch={}, n_images={}, shape=[{}, {}, {}]",
830                n_images * batch_size,
831                batch_size,
832                n_images,
833                height,
834                width,
835                channels
836            );
837
838            // Reshape from [batch, n_images, 3, h, w] → [batch*n_images, 3, h, w]
839            // For 4D input, this is just a simple reshape
840            let images_flat = if images_shape.len() == 4 {
841                // [batch, 3, h, w] → [batch, 3, h, w] (no change needed, batch=batch*1)
842                images.shallow_clone()
843            } else {
844                // [batch, n_images, 3, h, w] → [batch*n_images, 3, h, w]
845                images.reshape([batch_size * n_images, channels, height, width])
846            };
847
848            // Encode images through vision encoder
849            let image_features = self.vision_encoder.forward(&images_flat)?;
850
851            // image_features shape: [batch*n_images, num_patches, encoder_hidden_size]
852            let feat_shape = image_features.size();
853            let num_patches = feat_shape[1];
854            let encoder_hidden = feat_shape[2];
855
856            debug!(
857                "Vision encoder output: num_patches={}, hidden_size={}",
858                num_patches, encoder_hidden
859            );
860
861            // Project to language space
862            let image_embeds = self.vision_aligner.forward(&image_features)?;
863
864            // image_embeds shape: [batch*n_images, num_patches, language_hidden_size]
865            let lang_shape = image_embeds.size();
866            let language_hidden = lang_shape[2];
867
868            // Reshape from [batch*n_images, num_patches, hidden]
869            // to [batch, n_images*num_patches, hidden]
870            let image_embeds_reshaped =
871                image_embeds.reshape([batch_size, n_images * num_patches, language_hidden]);
872
873            debug!(
874                "Image embeddings reshaped: {:?}",
875                image_embeds_reshaped.size()
876            );
877
878            // Verify dimension compatibility
879            let input_shape = inputs_embeds.size();
880            let input_hidden = input_shape[2];
881
882            if language_hidden != input_hidden {
883                return Err(anyhow!(
884                    "Language model hidden size {} != vision aligner output {}",
885                    input_hidden,
886                    language_hidden
887                ));
888            }
889
890            // Reshape emb_mask to [batch, n_images*num_patches]
891            // Input can be either [batch, num_patches] or [batch, n_images, num_patches]
892            // IMPORTANT: We use emb_mask's dimensions, not vision encoder output,
893            // because emb_mask contains the ground truth number of image tokens (576)
894            let emb_mask_shape = emb_mask.size();
895            let emb_mask_patches = if emb_mask_shape.len() == 3 {
896                emb_mask_shape[2] // [batch, n_images, num_patches] → extract num_patches
897            } else {
898                emb_mask_shape[1] // [batch, num_patches] → extract num_patches
899            };
900
901            let emb_mask_flat = if emb_mask_shape.len() == 2 {
902                // Already [batch, num_patches] for single image
903                emb_mask.shallow_clone()
904            } else {
905                // [batch, n_images, num_patches] → [batch, n_images*num_patches]
906                emb_mask.reshape([batch_size, n_images * emb_mask_patches])
907            };
908
909            // Replace image placeholders with actual image embeddings
910            inputs_embeds = self.merge_embeddings(
911                &inputs_embeds,
912                &image_embeds_reshaped,
913                seq_mask,
914                &emb_mask_flat,
915            )?;
916
917            debug!("Merged embeddings: {:?}", inputs_embeds.size());
918        }
919
920        Ok(inputs_embeds)
921    }
922
923    /// Merge text and image embeddings based on masks
924    ///
925    /// This function takes text embeddings and vision embeddings and combines them
926    /// by replacing positions marked in seq_mask with corresponding vision embeddings.
927    ///
928    /// # Arguments
929    /// * `text_embeds` - Text token embeddings [batch, text_seq_len, hidden_dim]
930    /// * `image_embeds` - Vision embeddings [batch, num_images * num_patches, hidden_dim]
931    /// * `seq_mask` - Boolean mask [batch, text_seq_len] indicating positions to replace
932    /// * `emb_mask` - Boolean mask [batch, num_images * num_patches] indicating which vision embeddings to use
933    ///
934    /// # Returns
935    /// Combined embedding sequence where image positions are replaced with vision embeddings
936    /// Merge text and image embeddings using in-place updates
937    ///
938    /// # Implementation Note
939    /// This function uses shallow_clone() which creates a view sharing the same underlying data.
940    /// Mutations through copy_() will modify this view, which is the intended behavior.
941    /// The caller should ensure text_embeds is not aliased elsewhere if this is undesired.
942    ///
943    /// This approach avoids tch-rs index operation issues by processing batch-by-batch
944    /// and using copy_ operations carefully.
945    fn merge_embeddings(
946        &self,
947        text_embeds: &Tensor,
948        image_embeds: &Tensor,
949        seq_mask: &Tensor,
950        emb_mask: &Tensor,
951    ) -> Result<Tensor> {
952        use tracing::debug;
953
954        let text_shape = text_embeds.size();
955        let image_shape = image_embeds.size();
956
957        let batch_size = text_shape[0];
958        let _seq_len = text_shape[1];
959        let hidden_dim = text_shape[2];
960
961        debug!(
962            "merge_embeddings: text={:?}, image={:?}",
963            text_shape, image_shape
964        );
965
966        // Verify dimensions match
967        if image_shape[2] != hidden_dim {
968            return Err(anyhow!(
969                "Hidden dimension mismatch: text={}, image={}",
970                hidden_dim,
971                image_shape[2]
972            ));
973        }
974
975        // Create a view we can modify in-place
976        // NOTE: shallow_clone() shares underlying data, mutations will affect the view
977        let result = text_embeds.shallow_clone();
978
979        // Process batch by batch for safety
980        for b in 0..batch_size {
981            let batch_result = result.get(b);
982            let batch_seq_mask = seq_mask.get(b);
983            let batch_image_embeds = image_embeds.get(b);
984            let batch_emb_mask = emb_mask.get(b);
985
986            // Find all positions where seq_mask is True
987            let seq_positions = batch_seq_mask.nonzero(); // [N, 1]
988            let num_positions = seq_positions.size()[0];
989
990            if num_positions == 0 {
991                continue;
992            }
993
994            // Find all valid image embeddings
995            let emb_mask_flat = batch_emb_mask.reshape([-1]);
996            let image_flat = batch_image_embeds.reshape([-1, hidden_dim]);
997            let valid_image_embeds = image_flat
998                .masked_select(&emb_mask_flat.unsqueeze(-1))
999                .reshape([num_positions, hidden_dim]);
1000
1001            debug!(
1002                "Batch {}: Merging {} image embeddings at {} positions",
1003                b,
1004                valid_image_embeds.size()[0],
1005                num_positions
1006            );
1007
1008            // Do the replacement: for each True position, use corresponding image embedding
1009            for i in 0..num_positions {
1010                let pos = seq_positions.int64_value(&[i, 0]);
1011                let mut row = batch_result.get(pos);
1012                let img_row = valid_image_embeds.get(i);
1013
1014                // Update in-place
1015                row.copy_(&img_row);
1016            }
1017        }
1018
1019        Ok(result)
1020    }
1021}
1022
1023impl ModelOperations for JanusModel {
1024    fn architecture(&self) -> ModelArchitecture {
1025        self.architecture.clone()
1026    }
1027
1028    fn config(&self) -> &dyn ArchitectureConfig {
1029        &self.config
1030    }
1031
1032    fn forward(&self, input: &Tensor, past_kv: Option<&Tensor>) -> Result<Tensor> {
1033        // For text-only forward, delegate to language model
1034        self.language_model.forward(input, past_kv)
1035    }
1036
1037    fn embed_tokens(&self, input_ids: &Tensor) -> Result<Tensor> {
1038        self.language_model.embed_tokens(input_ids)
1039    }
1040
1041    fn forward_from_embeddings(&self, embeddings: &Tensor, start_pos: usize) -> Result<Tensor> {
1042        // Delegate to language model's forward_from_embeddings
1043        self.language_model
1044            .forward_from_embeddings(embeddings, start_pos)
1045    }
1046
1047    fn forward_with_cache(&self, input: &Tensor, start_pos: usize) -> Result<Tensor> {
1048        // CRITICAL: Delegate to language model's forward_with_cache
1049        // The default trait implementation calls forward(input, None) which loses start_pos!
1050        self.language_model.forward_with_cache(input, start_pos)
1051    }
1052
1053    fn prepare_multimodal_inputs(
1054        &self,
1055        input_ids: &Tensor,
1056        pixel_values: Option<&Tensor>,
1057        images_seq_mask: Option<&Tensor>,
1058        images_emb_mask: Option<&Tensor>,
1059    ) -> Result<Tensor> {
1060        // Call the existing prepare_inputs_embeds method
1061        self.prepare_inputs_embeds(input_ids, pixel_values, images_seq_mask, images_emb_mask)
1062    }
1063
1064    fn decode_layer(
1065        &self,
1066        layer_idx: usize,
1067        hidden_states: &Tensor,
1068        attention_mask: Option<&Tensor>,
1069        position_ids: Option<&Tensor>,
1070        past_kv: Option<&LayerKVCache>,
1071    ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
1072        self.language_model.decode_layer(
1073            layer_idx,
1074            hidden_states,
1075            attention_mask,
1076            position_ids,
1077            past_kv,
1078        )
1079    }
1080
1081    fn apply_final_norm(&self, hidden_states: &Tensor) -> Result<Tensor> {
1082        self.language_model.apply_final_norm(hidden_states)
1083    }
1084
1085    fn lm_head(&self, hidden_states: &Tensor) -> Result<Tensor> {
1086        self.language_model.lm_head(hidden_states)
1087    }
1088
1089    fn num_layers(&self) -> usize {
1090        self.language_model.num_layers()
1091    }
1092
1093    fn reshape_for_attention(&self, tensor: &Tensor, is_key_value: bool) -> Result<Tensor> {
1094        self.language_model
1095            .reshape_for_attention(tensor, is_key_value)
1096    }
1097
1098    fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor> {
1099        self.language_model.apply_rope(tensor, position_ids)
1100    }
1101
1102    fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
1103        self.language_model.normalize(tensor)
1104    }
1105
1106    fn get_attention_mask(&self, seq_len: usize, past_kv_len: usize) -> Result<Tensor> {
1107        self.language_model.get_attention_mask(seq_len, past_kv_len)
1108    }
1109
1110    fn encode_vision(&self, pixel_values: &Tensor) -> Result<Tensor> {
1111        let features = self.vision_encoder.forward(pixel_values)?;
1112        // features shape depends on model: [B, num_patches, hidden_size]
1113        // Mean pool over patches to get [B, hidden_size]
1114        let pooled = features.mean_dim(1, false, tch::Kind::Float);
1115        Ok(pooled)
1116    }
1117
1118    fn as_any(&self) -> &dyn std::any::Any {
1119        self
1120    }
1121}
1122
1123// SAFETY: JanusModel is Send because:
1124// - All Tensor fields are Send (guaranteed by tch-rs)
1125// - language_model (Box<dyn ModelOperations>) is Send (required by trait bound)
1126// - vision_cache uses Mutex (which is Send) instead of RefCell
1127// - No thread-local state or non-Send fields
1128unsafe impl Send for JanusModel {}
1129
1130// SAFETY: VisionEncoder is Send because both enum variants are Send
1131unsafe impl Send for VisionEncoder {}
1132
1133// SAFETY: SigLIPVisionTower is Send because all fields are Send:
1134// - Tensors are Send (tch-rs guarantee)
1135// - SigLIPConfig contains only primitives
1136unsafe impl Send for SigLIPVisionTower {}
1137
1138// SAFETY: CLIPVisionTower is Send because VisionEncoderConfig is Send
1139unsafe impl Send for CLIPVisionTower {}
1140
1141// SAFETY: Vision transformer components are Send because all Tensor fields are Send
1142unsafe impl Send for VisionTransformerLayer {}
1143unsafe impl Send for MultiHeadAttention {}
1144unsafe impl Send for Mlp {}
1145unsafe impl Send for LayerNorm {}
1146unsafe impl Send for MlpProjector {}
1147unsafe impl Send for GenerationPipeline {}
1148unsafe impl Send for VQModel {}
1149
1150// ============================================================================
1151// Janus-specific placeholder token utilities
1152// ============================================================================
1153
1154/// Configuration for Janus image placeholder replacement
1155///
1156/// Janus uses special tokens to mark where images should be injected:
1157/// - `<image_placeholder>` - User writes this in prompts
1158/// - `<begin_of_image>` - Marks start of image token sequence
1159/// - `<end_of_image>` - Marks end of image token sequence
1160/// - Each placeholder is replaced with: <boi> + 576 image tokens + <eoi>
1161#[derive(Debug, Clone)]
1162pub struct JanusPlaceholderConfig {
1163    /// Token ID for `<image_placeholder>`
1164    pub image_placeholder_id: u32,
1165    /// Token ID for `<begin_of_image>`
1166    pub image_start_id: u32,
1167    /// Token ID for `<end_of_image>`
1168    pub image_end_id: u32,
1169    /// Token ID used for actual image embeddings (typically same as placeholder)
1170    pub image_token_id: u32,
1171    /// Number of image tokens per image (typically 576 for 24x24 patches)
1172    pub num_image_tokens: usize,
1173}
1174
1175impl JanusPlaceholderConfig {
1176    /// Create configuration from a tokenizer
1177    pub fn from_tokenizer(tokenizer: &tokenizers::Tokenizer) -> Result<Self> {
1178        let vocab = tokenizer.get_vocab(true);
1179
1180        let image_placeholder_id = vocab
1181            .get("<image_placeholder>")
1182            .copied()
1183            .ok_or_else(|| anyhow!("Tokenizer missing <image_placeholder> token"))?;
1184
1185        let image_start_id = vocab
1186            .get("<begin_of_image>")
1187            .copied()
1188            .ok_or_else(|| anyhow!("Tokenizer missing <begin_of_image> token"))?;
1189
1190        let image_end_id = vocab
1191            .get("<end_of_image>")
1192            .copied()
1193            .ok_or_else(|| anyhow!("Tokenizer missing <end_of_image> token"))?;
1194
1195        Ok(Self {
1196            image_placeholder_id,
1197            image_start_id,
1198            image_end_id,
1199            image_token_id: image_placeholder_id, // Use same ID for actual image tokens
1200            num_image_tokens: 576, // Standard for 384x384 images with 16x16 patches = 24x24 = 576
1201        })
1202    }
1203}
1204
1205/// Result of Janus placeholder token replacement
1206#[derive(Debug)]
1207pub struct JanusPlaceholderReplacement {
1208    /// Modified input IDs with placeholders replaced
1209    /// Shape: [batch, expanded_seq_len]
1210    pub input_ids: Tensor,
1211
1212    /// Mask indicating where image tokens are in the sequence
1213    /// Shape: [batch, expanded_seq_len]
1214    pub images_seq_mask: Tensor,
1215
1216    /// Mask indicating which vision embeddings to use
1217    /// Shape: [batch, n_images, num_image_tokens]
1218    pub images_emb_mask: Tensor,
1219
1220    /// Number of images detected
1221    pub num_images: usize,
1222}
1223
1224/// Replace Janus image placeholder tokens with proper token sequence
1225///
1226/// This follows the official Janus design:
1227/// 1. Find all `<image_placeholder>` tokens in input_ids
1228/// 2. Replace each with: <begin_of_image> + 576 x <image_token> + <end_of_image>
1229/// 3. Create masks for embedding replacement
1230///
1231/// # Arguments
1232/// * `input_ids` - Original token IDs [seq_len]
1233/// * `config` - Placeholder configuration with token IDs
1234/// * `device` - Target device for tensors
1235///
1236/// # Returns
1237/// JanusPlaceholderReplacement with expanded tokens and masks
1238pub fn replace_janus_placeholders(
1239    input_ids: &[i64],
1240    config: &JanusPlaceholderConfig,
1241    device: Device,
1242) -> Result<JanusPlaceholderReplacement> {
1243    use tracing::{debug, warn};
1244
1245    // Find indices of all placeholder tokens
1246    let placeholder_indices: Vec<usize> = input_ids
1247        .iter()
1248        .enumerate()
1249        .filter(|(_, &id)| id == config.image_placeholder_id as i64)
1250        .map(|(idx, _)| idx)
1251        .collect();
1252
1253    let num_images = placeholder_indices.len();
1254
1255    if num_images == 0 {
1256        warn!("No image placeholder tokens found in input");
1257        // Return original input with empty masks
1258        let input_ids_tensor = Tensor::from_slice(input_ids)
1259            .to_kind(DType::Int64)
1260            .to_device(device)
1261            .unsqueeze(0); // [1, seq_len]
1262
1263        let seq_len = input_ids.len();
1264        let images_seq_mask = Tensor::zeros([1, seq_len as i64], (DType::Bool, device));
1265        let images_emb_mask = Tensor::zeros(
1266            [1, 1, config.num_image_tokens as i64],
1267            (DType::Bool, device),
1268        );
1269
1270        return Ok(JanusPlaceholderReplacement {
1271            input_ids: input_ids_tensor,
1272            images_seq_mask,
1273            images_emb_mask,
1274            num_images: 0,
1275        });
1276    }
1277
1278    debug!(
1279        "Found {} image placeholder(s) at positions: {:?}",
1280        num_images, placeholder_indices
1281    );
1282
1283    // Build new token sequence with replacements
1284    let mut new_tokens = Vec::new();
1285    let mut last_pos = 0;
1286
1287    for &placeholder_idx in &placeholder_indices {
1288        // Add tokens before placeholder
1289        new_tokens.extend_from_slice(&input_ids[last_pos..placeholder_idx]);
1290
1291        // Replace placeholder with: <boi> + 576 x <image> + <eoi>
1292        new_tokens.push(config.image_start_id as i64);
1293        new_tokens.extend(std::iter::repeat_n(
1294            config.image_token_id as i64,
1295            config.num_image_tokens,
1296        ));
1297        new_tokens.push(config.image_end_id as i64);
1298
1299        last_pos = placeholder_idx + 1;
1300    }
1301
1302    // Add remaining tokens after last placeholder
1303    new_tokens.extend_from_slice(&input_ids[last_pos..]);
1304
1305    let expanded_seq_len = new_tokens.len();
1306    debug!(
1307        "Expanded sequence length: {} -> {} (added {} tokens for {} images)",
1308        input_ids.len(),
1309        expanded_seq_len,
1310        expanded_seq_len - input_ids.len(),
1311        num_images
1312    );
1313
1314    // Create input_ids tensor [1, expanded_seq_len]
1315    let input_ids_tensor = Tensor::from_slice(&new_tokens)
1316        .to_kind(DType::Int64)
1317        .to_device(device)
1318        .unsqueeze(0);
1319
1320    // Create images_seq_mask: mark positions where image tokens are
1321    // This includes the 576 image tokens, NOT the <boi> and <eoi> tokens
1322    let mut seq_mask = vec![false; expanded_seq_len];
1323    let mut current_pos = 0;
1324
1325    for (img_idx, &placeholder_idx) in placeholder_indices.iter().enumerate() {
1326        // Calculate how many tokens were added before this placeholder
1327        let tokens_before = if img_idx == 0 {
1328            placeholder_idx
1329        } else {
1330            placeholder_idx - placeholder_indices[img_idx - 1] - 1
1331        };
1332
1333        current_pos += tokens_before;
1334
1335        // Skip <boi> token
1336        current_pos += 1;
1337
1338        // Mark the 576 image tokens with bounds checking
1339        for _ in 0..config.num_image_tokens {
1340            if current_pos >= seq_mask.len() {
1341                return Err(anyhow!(
1342                    "Placeholder replacement exceeded sequence length: pos={}, len={}, image={}/{}",
1343                    current_pos,
1344                    seq_mask.len(),
1345                    img_idx + 1,
1346                    num_images
1347                ));
1348            }
1349            seq_mask[current_pos] = true;
1350            current_pos += 1;
1351        }
1352
1353        // Skip <eoi> token
1354        current_pos += 1;
1355    }
1356
1357    let images_seq_mask = Tensor::from_slice(
1358        &seq_mask
1359            .iter()
1360            .map(|&b| if b { 1u8 } else { 0u8 })
1361            .collect::<Vec<_>>(),
1362    )
1363    .to_kind(DType::Bool)
1364    .to_device(device)
1365    .unsqueeze(0); // [1, expanded_seq_len]
1366
1367    // Create images_emb_mask: all vision embeddings are valid
1368    // Shape: [1, num_images, num_image_tokens] - all True
1369    let images_emb_mask = Tensor::ones(
1370        [1, num_images as i64, config.num_image_tokens as i64],
1371        (DType::Bool, device),
1372    );
1373
1374    Ok(JanusPlaceholderReplacement {
1375        input_ids: input_ids_tensor,
1376        images_seq_mask,
1377        images_emb_mask,
1378        num_images,
1379    })
1380}
1381
1382// SAFETY: VisionHead is Send because all Tensor fields are Send
1383unsafe impl Send for VisionHead {}