1#![allow(dead_code, clippy::type_complexity)]
12
13use super::config::AttentionType;
14use super::{ArchitectureConfig, ModelArchitecture, ModelOperations, VisionEncoderType};
15use crate::runtime::kv_cache::LayerKVCache;
16use anyhow::{anyhow, Result};
17use std::collections::HashMap;
18use tch::{Device, Kind as DType, Tensor};
19
20pub struct JanusConfig {
22 pub language_config: Box<dyn ArchitectureConfig>,
24 pub vision_config: VisionEncoderConfig,
26 pub aligner_config: ProjectorConfig,
28 pub generation_config: Option<GenerationConfig>,
30 pub device: Device,
32 pub dtype: DType,
34}
35
36impl ArchitectureConfig for JanusConfig {
37 fn hidden_size(&self) -> usize {
38 self.language_config.hidden_size()
39 }
40
41 fn intermediate_size(&self) -> usize {
42 self.language_config.intermediate_size()
43 }
44
45 fn num_attention_heads(&self) -> usize {
46 self.language_config.num_attention_heads()
47 }
48
49 fn num_key_value_heads(&self) -> usize {
50 self.language_config.num_key_value_heads()
51 }
52
53 fn head_dim(&self) -> usize {
54 self.language_config.head_dim()
55 }
56
57 fn vocab_size(&self) -> usize {
58 self.language_config.vocab_size()
59 }
60
61 fn max_position_embeddings(&self) -> usize {
62 self.language_config.max_position_embeddings()
63 }
64
65 fn rope_theta(&self) -> Option<f32> {
66 self.language_config.rope_theta()
67 }
68
69 fn rope_dim(&self) -> Option<usize> {
70 self.language_config.rope_dim()
71 }
72
73 fn layer_norm_eps(&self) -> f32 {
74 self.language_config.layer_norm_eps()
75 }
76
77 fn use_rms_norm(&self) -> bool {
78 self.language_config.use_rms_norm()
79 }
80
81 fn attention_type(&self) -> AttentionType {
82 self.language_config.attention_type()
83 }
84}
85
86#[derive(Debug, Clone)]
88pub struct VisionEncoderConfig {
89 pub encoder_type: VisionEncoderType,
90 pub hidden_size: usize,
91 pub image_size: usize,
92 pub patch_size: usize,
93 pub num_layers: usize,
94 pub num_patches: usize, pub num_attention_heads: Option<usize>,
96 pub intermediate_size: Option<usize>,
97}
98
99#[derive(Debug, Clone)]
101pub struct ProjectorConfig {
102 pub input_dim: usize,
103 pub output_dim: usize,
104 pub hidden_dim: Option<usize>, }
106
107#[derive(Debug, Clone)]
109pub struct GenerationConfig {
110 pub vq_config: VQModelConfig,
111 pub gen_aligner_config: ProjectorConfig,
112 pub gen_head_config: VisionHeadConfig,
113 pub image_token_size: usize, }
115
116#[derive(Debug, Clone)]
118pub struct VQModelConfig {
119 pub codebook_size: usize,
120 pub latent_dim: usize,
121}
122
123#[derive(Debug, Clone)]
125pub struct VisionHeadConfig {
126 pub input_dim: usize,
127 pub hidden_dim: usize,
128 pub output_dim: usize, }
130
131pub struct JanusModel {
133 language_model: Box<dyn ModelOperations>, vision_encoder: VisionEncoder,
136 vision_aligner: MlpProjector,
137
138 generation: Option<GenerationPipeline>,
140
141 device: Device,
143 dtype: DType,
144 architecture: ModelArchitecture,
145 config: JanusConfig,
146
147 vision_cache: parking_lot::Mutex<Option<(Vec<u8>, Tensor)>>, }
151
152#[allow(clippy::large_enum_variant)] pub enum VisionEncoder {
155 SigLIP(SigLIPVisionTower),
156 CLIP(CLIPVisionTower),
157}
158
159impl VisionEncoder {
160 fn forward(&self, images: &Tensor) -> Result<Tensor> {
161 match self {
162 Self::SigLIP(model) => model.forward(images),
163 Self::CLIP(model) => model.forward(images),
164 }
165 }
166
167 fn from_weights(
168 config: &VisionEncoderConfig,
169 weights: &HashMap<String, Tensor>,
170 prefix: &str,
171 device: Device,
172 dtype: DType,
173 ) -> Result<Self> {
174 match &config.encoder_type {
175 VisionEncoderType::SigLIP { .. } => Ok(Self::SigLIP(SigLIPVisionTower::from_weights(
176 config, weights, prefix, device, dtype,
177 )?)),
178 VisionEncoderType::CLIP { .. } => Ok(Self::CLIP(CLIPVisionTower::from_weights(
179 config, weights, prefix, device, dtype,
180 )?)),
181 _ => Err(anyhow!("Unsupported vision encoder type")),
182 }
183 }
184}
185
186pub struct SigLIPVisionTower {
188 model: super::siglip::SigLIPVisionTransformer,
189 config: VisionEncoderConfig,
190}
191
192impl SigLIPVisionTower {
193 fn from_weights(
194 config: &VisionEncoderConfig,
195 weights: &HashMap<String, Tensor>,
196 prefix: &str,
197 device: Device,
198 dtype: DType,
199 ) -> Result<Self> {
200 let siglip_config = super::siglip::SigLIPConfig {
202 image_size: config.image_size,
203 patch_size: config.patch_size,
204 hidden_size: config.hidden_size,
205 num_hidden_layers: config.num_layers,
206 num_attention_heads: config.num_attention_heads.unwrap_or(16),
207 intermediate_size: config.intermediate_size.unwrap_or(config.hidden_size * 4),
208 use_attention_pool: false, };
210
211 let model = super::siglip::SigLIPVisionTransformer::from_weights(
212 &siglip_config,
213 weights,
214 prefix,
215 device,
216 dtype,
217 )?;
218
219 Ok(Self {
220 model,
221 config: config.clone(),
222 })
223 }
224
225 fn forward(&self, images: &Tensor) -> Result<Tensor> {
226 self.model.forward(images)
227 }
228}
229
230pub struct CLIPVisionTower {
232 config: VisionEncoderConfig,
233}
234
235impl CLIPVisionTower {
236 fn from_weights(
237 _config: &VisionEncoderConfig,
238 _weights: &HashMap<String, Tensor>,
239 _prefix: &str,
240 _device: Device,
241 _dtype: DType,
242 ) -> Result<Self> {
243 Err(anyhow!(
246 "CLIP vision encoder not yet implemented - use SigLIP instead"
247 ))
248 }
249
250 fn forward(&self, _images: &Tensor) -> Result<Tensor> {
251 Err(anyhow!(
252 "CLIP vision encoder not yet implemented - use SigLIP instead"
253 ))
254 }
255}
256
257struct VisionTransformerLayer {
259 self_attn: MultiHeadAttention,
260 mlp: Mlp,
261 ln_1: LayerNorm,
262 ln_2: LayerNorm,
263}
264
265impl VisionTransformerLayer {
266 fn from_weights(
267 weights: &HashMap<String, Tensor>,
268 prefix: &str,
269 hidden_size: usize,
270 device: Device,
271 dtype: DType,
272 ) -> Result<Self> {
273 let self_attn = MultiHeadAttention::from_weights(
274 weights,
275 &format!("{prefix}.self_attn"),
276 hidden_size,
277 device,
278 dtype,
279 )?;
280
281 let mlp = Mlp::from_weights(
282 weights,
283 &format!("{prefix}.mlp"),
284 hidden_size,
285 hidden_size * 4, device,
287 dtype,
288 )?;
289
290 let ln_1 = LayerNorm::from_weights(
291 weights,
292 &format!("{prefix}.ln_1"),
293 hidden_size,
294 device,
295 dtype,
296 )?;
297
298 let ln_2 = LayerNorm::from_weights(
299 weights,
300 &format!("{prefix}.ln_2"),
301 hidden_size,
302 device,
303 dtype,
304 )?;
305
306 Ok(Self {
307 self_attn,
308 mlp,
309 ln_1,
310 ln_2,
311 })
312 }
313
314 fn forward(&self, x: &Tensor) -> Result<Tensor> {
315 let mut output = x.shallow_clone();
317
318 let attn_out = self.self_attn.forward(&self.ln_1.forward(&output)?)?;
320 output = &output + &attn_out;
321
322 let mlp_out = self.mlp.forward(&self.ln_2.forward(&output)?)?;
324 output = &output + &mlp_out;
325
326 Ok(output)
327 }
328}
329
330struct MultiHeadAttention {
332 q_proj: Tensor,
333 k_proj: Tensor,
334 v_proj: Tensor,
335 out_proj: Tensor,
336 num_heads: usize,
337 head_dim: usize,
338}
339
340impl MultiHeadAttention {
341 fn from_weights(
342 weights: &HashMap<String, Tensor>,
343 prefix: &str,
344 hidden_size: usize,
345 device: Device,
346 dtype: DType,
347 ) -> Result<Self> {
348 let mut q_proj = weights
349 .get(&format!("{prefix}.q_proj.weight"))
350 .ok_or_else(|| anyhow!("Missing q_proj weight"))?
351 .shallow_clone();
352 q_proj = q_proj.to_device(device);
353 q_proj = q_proj.to_kind(dtype);
354
355 let mut k_proj = weights
356 .get(&format!("{prefix}.k_proj.weight"))
357 .ok_or_else(|| anyhow!("Missing k_proj weight"))?
358 .shallow_clone();
359 k_proj = k_proj.to_device(device);
360 k_proj = k_proj.to_kind(dtype);
361
362 let mut v_proj = weights
363 .get(&format!("{prefix}.v_proj.weight"))
364 .ok_or_else(|| anyhow!("Missing v_proj weight"))?
365 .shallow_clone();
366 v_proj = v_proj.to_device(device);
367 v_proj = v_proj.to_kind(dtype);
368
369 let mut out_proj = weights
370 .get(&format!("{prefix}.out_proj.weight"))
371 .ok_or_else(|| anyhow!("Missing out_proj weight"))?
372 .shallow_clone();
373 out_proj = out_proj.to_device(device);
374 out_proj = out_proj.to_kind(dtype);
375
376 let num_heads = 16; let head_dim = hidden_size / num_heads;
379
380 Ok(Self {
381 q_proj,
382 k_proj,
383 v_proj,
384 out_proj,
385 num_heads,
386 head_dim,
387 })
388 }
389
390 fn forward(&self, x: &Tensor) -> Result<Tensor> {
391 let (batch_size, seq_len, _) = x.size3()?;
392
393 let q = x.matmul(&self.q_proj.transpose(0, 1));
395 let k = x.matmul(&self.k_proj.transpose(0, 1));
396 let v = x.matmul(&self.v_proj.transpose(0, 1));
397
398 let q = q
400 .reshape([
401 batch_size,
402 seq_len,
403 self.num_heads as i64,
404 self.head_dim as i64,
405 ])
406 .transpose(1, 2); let k = k
408 .reshape([
409 batch_size,
410 seq_len,
411 self.num_heads as i64,
412 self.head_dim as i64,
413 ])
414 .transpose(1, 2);
415 let v = v
416 .reshape([
417 batch_size,
418 seq_len,
419 self.num_heads as i64,
420 self.head_dim as i64,
421 ])
422 .transpose(1, 2);
423
424 let scale = (self.head_dim as f64).sqrt();
426 let scores = q.matmul(&k.transpose(-2, -1)).divide_scalar(scale);
427 let attn_weights = scores.softmax(-1, scores.kind());
428 let attn_output = attn_weights.matmul(&v);
429
430 let attn_output = attn_output
432 .transpose(1, 2)
433 .reshape([batch_size, seq_len, -1]);
434
435 let output = attn_output.matmul(&self.out_proj.transpose(0, 1));
437
438 Ok(output)
439 }
440}
441
442struct Mlp {
444 fc1: Tensor,
445 fc2: Tensor,
446}
447
448impl Mlp {
449 fn from_weights(
450 weights: &HashMap<String, Tensor>,
451 prefix: &str,
452 _hidden_size: usize,
453 _intermediate_size: usize,
454 device: Device,
455 dtype: DType,
456 ) -> Result<Self> {
457 let mut fc1 = weights
458 .get(&format!("{prefix}.fc1.weight"))
459 .ok_or_else(|| anyhow!("Missing fc1 weight"))?
460 .shallow_clone();
461 fc1 = fc1.to_device(device);
462 fc1 = fc1.to_kind(dtype);
463
464 let mut fc2 = weights
465 .get(&format!("{prefix}.fc2.weight"))
466 .ok_or_else(|| anyhow!("Missing fc2 weight"))?
467 .shallow_clone();
468 fc2 = fc2.to_device(device);
469 fc2 = fc2.to_kind(dtype);
470
471 Ok(Self { fc1, fc2 })
472 }
473
474 fn forward(&self, x: &Tensor) -> Result<Tensor> {
475 let mut output = x.matmul(&self.fc1.transpose(0, 1));
476 output = output.gelu("none"); output = output.matmul(&self.fc2.transpose(0, 1));
478 Ok(output)
479 }
480}
481
482struct LayerNorm {
484 weight: Tensor,
485 bias: Option<Tensor>,
486 eps: f64,
487}
488
489impl LayerNorm {
490 fn from_weights(
491 weights: &HashMap<String, Tensor>,
492 prefix: &str,
493 _hidden_size: usize,
494 device: Device,
495 dtype: DType,
496 ) -> Result<Self> {
497 let mut weight = weights
498 .get(&format!("{prefix}.weight"))
499 .ok_or_else(|| anyhow!("Missing layer norm weight"))?
500 .shallow_clone();
501 weight = weight.to_device(device);
502 weight = weight.to_kind(dtype);
503
504 let bias = weights.get(&format!("{prefix}.bias")).map(|b| {
505 let mut b = b.shallow_clone();
506 b = b.to_device(device);
507 b.to_kind(dtype)
508 });
509
510 Ok(Self {
511 weight,
512 bias,
513 eps: 1e-5, })
515 }
516
517 fn forward(&self, x: &Tensor) -> Result<Tensor> {
518 let mean = x.mean_dim(&[-1i64][..], true, x.kind());
521 let var = x.var_dim(&[-1i64][..], false, true);
522 let normalized = (x - &mean) / (&var + self.eps).sqrt();
523
524 let output = &normalized * &self.weight;
525 let output = if let Some(ref bias) = self.bias {
526 output + bias
527 } else {
528 output
529 };
530 Ok(output)
531 }
532}
533
534pub struct MlpProjector {
536 layers: Vec<Tensor>,
537 config: ProjectorConfig,
538}
539
540impl MlpProjector {
541 pub fn from_weights(
542 weights: &HashMap<String, Tensor>,
543 prefix: &str,
544 config: &ProjectorConfig,
545 device: Device,
546 dtype: DType,
547 ) -> Result<Self> {
548 use tracing::debug;
549
550 debug!("Loading MLP projector with prefix: {}", prefix);
552 let aligner_keys: Vec<_> = weights.keys().filter(|k| k.starts_with(prefix)).collect();
553 debug!("Available aligner keys: {:?}", aligner_keys);
554
555 let mut layers = Vec::new();
556
557 if let Some(_hidden_dim) = config.hidden_dim {
558 let mut layer0_weight = weights
564 .get(&format!("{prefix}.layers.0.weight"))
565 .ok_or_else(|| anyhow!("Missing projector layer 0 weight"))?
566 .shallow_clone();
567 layer0_weight = layer0_weight.to_device(device);
568 layer0_weight = layer0_weight.to_kind(dtype);
569
570 let mut layer0_bias = weights
571 .get(&format!("{prefix}.layers.0.bias"))
572 .ok_or_else(|| anyhow!("Missing projector layer 0 bias"))?
573 .shallow_clone();
574 layer0_bias = layer0_bias.to_device(device);
575 layer0_bias = layer0_bias.to_kind(dtype);
576
577 let mut layer2_weight = weights
579 .get(&format!("{prefix}.layers.2.weight"))
580 .ok_or_else(|| anyhow!("Missing projector layer 2 weight"))?
581 .shallow_clone();
582 layer2_weight = layer2_weight.to_device(device);
583 layer2_weight = layer2_weight.to_kind(dtype);
584
585 let mut layer2_bias = weights
586 .get(&format!("{prefix}.layers.2.bias"))
587 .ok_or_else(|| anyhow!("Missing projector layer 2 bias"))?
588 .shallow_clone();
589 layer2_bias = layer2_bias.to_device(device);
590 layer2_bias = layer2_bias.to_kind(dtype);
591
592 layers.push(layer0_weight);
594 layers.push(layer0_bias);
595 layers.push(layer2_weight);
596 layers.push(layer2_bias);
597 } else {
598 let mut weight = weights
600 .get(&format!("{prefix}.layers.0.weight"))
601 .ok_or_else(|| anyhow!("Missing projector weight"))?
602 .shallow_clone();
603 weight = weight.to_device(device);
604 weight = weight.to_kind(dtype);
605
606 let mut bias = weights
607 .get(&format!("{prefix}.layers.0.bias"))
608 .ok_or_else(|| anyhow!("Missing projector bias"))?
609 .shallow_clone();
610 bias = bias.to_device(device);
611 bias = bias.to_kind(dtype);
612
613 layers.push(weight);
614 layers.push(bias);
615 }
616
617 Ok(Self {
618 layers,
619 config: config.clone(),
620 })
621 }
622
623 pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
624 if self.config.hidden_dim.is_some() {
625 let mut output = x.matmul(&self.layers[0].transpose(0, 1)) + &self.layers[1];
630
631 output = output.gelu("none");
633
634 output = output.matmul(&self.layers[2].transpose(0, 1)) + &self.layers[3];
636
637 Ok(output)
638 } else {
639 let output = x.matmul(&self.layers[0].transpose(0, 1)) + &self.layers[1];
642 Ok(output)
643 }
644 }
645}
646
647pub struct GenerationPipeline {
649 vq_model: VQModel,
650 gen_aligner: MlpProjector,
651 gen_head: VisionHead,
652 gen_embed: Tensor, }
654
655pub struct VQModel {
657 codebook: Tensor,
659 config: VQModelConfig,
660}
661
662pub struct VisionHead {
664 output_mlp_projector: Tensor,
665 vision_head: Tensor,
666}
667
668impl JanusModel {
669 pub fn from_weights(
671 weights: HashMap<String, Tensor>,
672 config: JanusConfig,
673 device: Device,
674 dtype: DType,
675 ) -> Result<Self> {
676 let (language_weights, vision_weights, aligner_weights, _gen_weights) =
678 Self::extract_weights_by_prefix(&weights)?;
679
680 let language_model = Self::load_language_model(language_weights, &config, device, dtype)?;
682
683 let vision_encoder = VisionEncoder::from_weights(
686 &config.vision_config,
687 &vision_weights,
688 "vision_model.vision_tower",
689 device,
690 dtype,
691 )?;
692
693 let vision_aligner = MlpProjector::from_weights(
695 &aligner_weights,
696 "aligner",
697 &config.aligner_config,
698 device,
699 dtype,
700 )?;
701
702 let generation = if let Some(_gen_config) = &config.generation_config {
704 None
706 } else {
707 None
708 };
709
710 let architecture = ModelArchitecture::Janus {
712 base_architecture: Box::new(ModelArchitecture::Llama { version: 3 }), vision_encoder: config.vision_config.encoder_type.clone(),
714 has_generation: generation.is_some(),
715 };
716
717 Ok(Self {
718 language_model,
719 vision_encoder,
720 vision_aligner,
721 generation,
722 device,
723 dtype,
724 architecture,
725 config,
726 vision_cache: parking_lot::Mutex::new(None),
727 })
728 }
729
730 fn extract_weights_by_prefix(
732 weights: &HashMap<String, Tensor>,
733 ) -> Result<(
734 HashMap<String, Tensor>, HashMap<String, Tensor>, HashMap<String, Tensor>, HashMap<String, Tensor>, )> {
739 let mut language_weights = HashMap::new();
740 let mut vision_weights = HashMap::new();
741 let mut aligner_weights = HashMap::new();
742 let mut _gen_weights = HashMap::new();
743
744 for (key, tensor) in weights {
745 if let Some(suffix) = key.strip_prefix("language_model.") {
746 language_weights.insert(suffix.to_owned(), tensor.shallow_clone());
748 } else if key.starts_with("vision_model.") {
749 vision_weights.insert(key.clone(), tensor.shallow_clone());
750 } else if key.starts_with("aligner.") {
751 aligner_weights.insert(key.clone(), tensor.shallow_clone());
752 } else if key.starts_with("gen_") {
753 _gen_weights.insert(key.clone(), tensor.shallow_clone());
754 }
755 }
756
757 Ok((
758 language_weights,
759 vision_weights,
760 aligner_weights,
761 _gen_weights,
762 ))
763 }
764
765 fn load_language_model(
767 mut weights: HashMap<String, Tensor>,
768 _config: &JanusConfig,
769 device: Device,
770 dtype: DType,
771 ) -> Result<Box<dyn ModelOperations>> {
772 use super::llama::LlamaModel;
774
775 let llama_model = LlamaModel::from_weights(&mut weights, &device, dtype)?;
778
779 Ok(Box::new(llama_model))
780 }
781
782 pub fn prepare_inputs_embeds(
785 &self,
786 input_ids: &Tensor,
787 pixel_values: Option<&Tensor>,
788 images_seq_mask: Option<&Tensor>,
789 images_emb_mask: Option<&Tensor>,
790 ) -> Result<Tensor> {
791 use tracing::debug;
792
793 let mut inputs_embeds = self.language_model.embed_tokens(input_ids)?;
795
796 if let (Some(images), Some(seq_mask), Some(emb_mask)) =
798 (pixel_values, images_seq_mask, images_emb_mask)
799 {
800 let images_shape = images.size();
801
802 let (batch_size, n_images, channels, height, width) = if images_shape.len() == 4 {
804 (
806 images_shape[0],
807 1,
808 images_shape[1],
809 images_shape[2],
810 images_shape[3],
811 )
812 } else if images_shape.len() == 5 {
813 (
815 images_shape[0],
816 images_shape[1],
817 images_shape[2],
818 images_shape[3],
819 images_shape[4],
820 )
821 } else {
822 return Err(anyhow!(
823 "Unexpected pixel_values shape: {:?}. Expected [batch, 3, h, w] or [batch, n_images, 3, h, w]",
824 images_shape
825 ));
826 };
827
828 debug!(
829 "Processing {} images: batch={}, n_images={}, shape=[{}, {}, {}]",
830 n_images * batch_size,
831 batch_size,
832 n_images,
833 height,
834 width,
835 channels
836 );
837
838 let images_flat = if images_shape.len() == 4 {
841 images.shallow_clone()
843 } else {
844 images.reshape([batch_size * n_images, channels, height, width])
846 };
847
848 let image_features = self.vision_encoder.forward(&images_flat)?;
850
851 let feat_shape = image_features.size();
853 let num_patches = feat_shape[1];
854 let encoder_hidden = feat_shape[2];
855
856 debug!(
857 "Vision encoder output: num_patches={}, hidden_size={}",
858 num_patches, encoder_hidden
859 );
860
861 let image_embeds = self.vision_aligner.forward(&image_features)?;
863
864 let lang_shape = image_embeds.size();
866 let language_hidden = lang_shape[2];
867
868 let image_embeds_reshaped =
871 image_embeds.reshape([batch_size, n_images * num_patches, language_hidden]);
872
873 debug!(
874 "Image embeddings reshaped: {:?}",
875 image_embeds_reshaped.size()
876 );
877
878 let input_shape = inputs_embeds.size();
880 let input_hidden = input_shape[2];
881
882 if language_hidden != input_hidden {
883 return Err(anyhow!(
884 "Language model hidden size {} != vision aligner output {}",
885 input_hidden,
886 language_hidden
887 ));
888 }
889
890 let emb_mask_shape = emb_mask.size();
895 let emb_mask_patches = if emb_mask_shape.len() == 3 {
896 emb_mask_shape[2] } else {
898 emb_mask_shape[1] };
900
901 let emb_mask_flat = if emb_mask_shape.len() == 2 {
902 emb_mask.shallow_clone()
904 } else {
905 emb_mask.reshape([batch_size, n_images * emb_mask_patches])
907 };
908
909 inputs_embeds = self.merge_embeddings(
911 &inputs_embeds,
912 &image_embeds_reshaped,
913 seq_mask,
914 &emb_mask_flat,
915 )?;
916
917 debug!("Merged embeddings: {:?}", inputs_embeds.size());
918 }
919
920 Ok(inputs_embeds)
921 }
922
923 fn merge_embeddings(
946 &self,
947 text_embeds: &Tensor,
948 image_embeds: &Tensor,
949 seq_mask: &Tensor,
950 emb_mask: &Tensor,
951 ) -> Result<Tensor> {
952 use tracing::debug;
953
954 let text_shape = text_embeds.size();
955 let image_shape = image_embeds.size();
956
957 let batch_size = text_shape[0];
958 let _seq_len = text_shape[1];
959 let hidden_dim = text_shape[2];
960
961 debug!(
962 "merge_embeddings: text={:?}, image={:?}",
963 text_shape, image_shape
964 );
965
966 if image_shape[2] != hidden_dim {
968 return Err(anyhow!(
969 "Hidden dimension mismatch: text={}, image={}",
970 hidden_dim,
971 image_shape[2]
972 ));
973 }
974
975 let result = text_embeds.shallow_clone();
978
979 for b in 0..batch_size {
981 let batch_result = result.get(b);
982 let batch_seq_mask = seq_mask.get(b);
983 let batch_image_embeds = image_embeds.get(b);
984 let batch_emb_mask = emb_mask.get(b);
985
986 let seq_positions = batch_seq_mask.nonzero(); let num_positions = seq_positions.size()[0];
989
990 if num_positions == 0 {
991 continue;
992 }
993
994 let emb_mask_flat = batch_emb_mask.reshape([-1]);
996 let image_flat = batch_image_embeds.reshape([-1, hidden_dim]);
997 let valid_image_embeds = image_flat
998 .masked_select(&emb_mask_flat.unsqueeze(-1))
999 .reshape([num_positions, hidden_dim]);
1000
1001 debug!(
1002 "Batch {}: Merging {} image embeddings at {} positions",
1003 b,
1004 valid_image_embeds.size()[0],
1005 num_positions
1006 );
1007
1008 for i in 0..num_positions {
1010 let pos = seq_positions.int64_value(&[i, 0]);
1011 let mut row = batch_result.get(pos);
1012 let img_row = valid_image_embeds.get(i);
1013
1014 row.copy_(&img_row);
1016 }
1017 }
1018
1019 Ok(result)
1020 }
1021}
1022
1023impl ModelOperations for JanusModel {
1024 fn architecture(&self) -> ModelArchitecture {
1025 self.architecture.clone()
1026 }
1027
1028 fn config(&self) -> &dyn ArchitectureConfig {
1029 &self.config
1030 }
1031
1032 fn forward(&self, input: &Tensor, past_kv: Option<&Tensor>) -> Result<Tensor> {
1033 self.language_model.forward(input, past_kv)
1035 }
1036
1037 fn embed_tokens(&self, input_ids: &Tensor) -> Result<Tensor> {
1038 self.language_model.embed_tokens(input_ids)
1039 }
1040
1041 fn forward_from_embeddings(&self, embeddings: &Tensor, start_pos: usize) -> Result<Tensor> {
1042 self.language_model
1044 .forward_from_embeddings(embeddings, start_pos)
1045 }
1046
1047 fn forward_with_cache(&self, input: &Tensor, start_pos: usize) -> Result<Tensor> {
1048 self.language_model.forward_with_cache(input, start_pos)
1051 }
1052
1053 fn prepare_multimodal_inputs(
1054 &self,
1055 input_ids: &Tensor,
1056 pixel_values: Option<&Tensor>,
1057 images_seq_mask: Option<&Tensor>,
1058 images_emb_mask: Option<&Tensor>,
1059 ) -> Result<Tensor> {
1060 self.prepare_inputs_embeds(input_ids, pixel_values, images_seq_mask, images_emb_mask)
1062 }
1063
1064 fn decode_layer(
1065 &self,
1066 layer_idx: usize,
1067 hidden_states: &Tensor,
1068 attention_mask: Option<&Tensor>,
1069 position_ids: Option<&Tensor>,
1070 past_kv: Option<&LayerKVCache>,
1071 ) -> Result<(Tensor, Option<(Tensor, Tensor)>)> {
1072 self.language_model.decode_layer(
1073 layer_idx,
1074 hidden_states,
1075 attention_mask,
1076 position_ids,
1077 past_kv,
1078 )
1079 }
1080
1081 fn apply_final_norm(&self, hidden_states: &Tensor) -> Result<Tensor> {
1082 self.language_model.apply_final_norm(hidden_states)
1083 }
1084
1085 fn lm_head(&self, hidden_states: &Tensor) -> Result<Tensor> {
1086 self.language_model.lm_head(hidden_states)
1087 }
1088
1089 fn num_layers(&self) -> usize {
1090 self.language_model.num_layers()
1091 }
1092
1093 fn reshape_for_attention(&self, tensor: &Tensor, is_key_value: bool) -> Result<Tensor> {
1094 self.language_model
1095 .reshape_for_attention(tensor, is_key_value)
1096 }
1097
1098 fn apply_rope(&self, tensor: &Tensor, position_ids: &Tensor) -> Result<Tensor> {
1099 self.language_model.apply_rope(tensor, position_ids)
1100 }
1101
1102 fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
1103 self.language_model.normalize(tensor)
1104 }
1105
1106 fn get_attention_mask(&self, seq_len: usize, past_kv_len: usize) -> Result<Tensor> {
1107 self.language_model.get_attention_mask(seq_len, past_kv_len)
1108 }
1109
1110 fn encode_vision(&self, pixel_values: &Tensor) -> Result<Tensor> {
1111 let features = self.vision_encoder.forward(pixel_values)?;
1112 let pooled = features.mean_dim(1, false, tch::Kind::Float);
1115 Ok(pooled)
1116 }
1117
1118 fn as_any(&self) -> &dyn std::any::Any {
1119 self
1120 }
1121}
1122
1123unsafe impl Send for JanusModel {}
1129
1130unsafe impl Send for VisionEncoder {}
1132
1133unsafe impl Send for SigLIPVisionTower {}
1137
1138unsafe impl Send for CLIPVisionTower {}
1140
1141unsafe impl Send for VisionTransformerLayer {}
1143unsafe impl Send for MultiHeadAttention {}
1144unsafe impl Send for Mlp {}
1145unsafe impl Send for LayerNorm {}
1146unsafe impl Send for MlpProjector {}
1147unsafe impl Send for GenerationPipeline {}
1148unsafe impl Send for VQModel {}
1149
1150#[derive(Debug, Clone)]
1162pub struct JanusPlaceholderConfig {
1163 pub image_placeholder_id: u32,
1165 pub image_start_id: u32,
1167 pub image_end_id: u32,
1169 pub image_token_id: u32,
1171 pub num_image_tokens: usize,
1173}
1174
1175impl JanusPlaceholderConfig {
1176 pub fn from_tokenizer(tokenizer: &tokenizers::Tokenizer) -> Result<Self> {
1178 let vocab = tokenizer.get_vocab(true);
1179
1180 let image_placeholder_id = vocab
1181 .get("<image_placeholder>")
1182 .copied()
1183 .ok_or_else(|| anyhow!("Tokenizer missing <image_placeholder> token"))?;
1184
1185 let image_start_id = vocab
1186 .get("<begin_of_image>")
1187 .copied()
1188 .ok_or_else(|| anyhow!("Tokenizer missing <begin_of_image> token"))?;
1189
1190 let image_end_id = vocab
1191 .get("<end_of_image>")
1192 .copied()
1193 .ok_or_else(|| anyhow!("Tokenizer missing <end_of_image> token"))?;
1194
1195 Ok(Self {
1196 image_placeholder_id,
1197 image_start_id,
1198 image_end_id,
1199 image_token_id: image_placeholder_id, num_image_tokens: 576, })
1202 }
1203}
1204
1205#[derive(Debug)]
1207pub struct JanusPlaceholderReplacement {
1208 pub input_ids: Tensor,
1211
1212 pub images_seq_mask: Tensor,
1215
1216 pub images_emb_mask: Tensor,
1219
1220 pub num_images: usize,
1222}
1223
1224pub fn replace_janus_placeholders(
1239 input_ids: &[i64],
1240 config: &JanusPlaceholderConfig,
1241 device: Device,
1242) -> Result<JanusPlaceholderReplacement> {
1243 use tracing::{debug, warn};
1244
1245 let placeholder_indices: Vec<usize> = input_ids
1247 .iter()
1248 .enumerate()
1249 .filter(|(_, &id)| id == config.image_placeholder_id as i64)
1250 .map(|(idx, _)| idx)
1251 .collect();
1252
1253 let num_images = placeholder_indices.len();
1254
1255 if num_images == 0 {
1256 warn!("No image placeholder tokens found in input");
1257 let input_ids_tensor = Tensor::from_slice(input_ids)
1259 .to_kind(DType::Int64)
1260 .to_device(device)
1261 .unsqueeze(0); let seq_len = input_ids.len();
1264 let images_seq_mask = Tensor::zeros([1, seq_len as i64], (DType::Bool, device));
1265 let images_emb_mask = Tensor::zeros(
1266 [1, 1, config.num_image_tokens as i64],
1267 (DType::Bool, device),
1268 );
1269
1270 return Ok(JanusPlaceholderReplacement {
1271 input_ids: input_ids_tensor,
1272 images_seq_mask,
1273 images_emb_mask,
1274 num_images: 0,
1275 });
1276 }
1277
1278 debug!(
1279 "Found {} image placeholder(s) at positions: {:?}",
1280 num_images, placeholder_indices
1281 );
1282
1283 let mut new_tokens = Vec::new();
1285 let mut last_pos = 0;
1286
1287 for &placeholder_idx in &placeholder_indices {
1288 new_tokens.extend_from_slice(&input_ids[last_pos..placeholder_idx]);
1290
1291 new_tokens.push(config.image_start_id as i64);
1293 new_tokens.extend(std::iter::repeat_n(
1294 config.image_token_id as i64,
1295 config.num_image_tokens,
1296 ));
1297 new_tokens.push(config.image_end_id as i64);
1298
1299 last_pos = placeholder_idx + 1;
1300 }
1301
1302 new_tokens.extend_from_slice(&input_ids[last_pos..]);
1304
1305 let expanded_seq_len = new_tokens.len();
1306 debug!(
1307 "Expanded sequence length: {} -> {} (added {} tokens for {} images)",
1308 input_ids.len(),
1309 expanded_seq_len,
1310 expanded_seq_len - input_ids.len(),
1311 num_images
1312 );
1313
1314 let input_ids_tensor = Tensor::from_slice(&new_tokens)
1316 .to_kind(DType::Int64)
1317 .to_device(device)
1318 .unsqueeze(0);
1319
1320 let mut seq_mask = vec![false; expanded_seq_len];
1323 let mut current_pos = 0;
1324
1325 for (img_idx, &placeholder_idx) in placeholder_indices.iter().enumerate() {
1326 let tokens_before = if img_idx == 0 {
1328 placeholder_idx
1329 } else {
1330 placeholder_idx - placeholder_indices[img_idx - 1] - 1
1331 };
1332
1333 current_pos += tokens_before;
1334
1335 current_pos += 1;
1337
1338 for _ in 0..config.num_image_tokens {
1340 if current_pos >= seq_mask.len() {
1341 return Err(anyhow!(
1342 "Placeholder replacement exceeded sequence length: pos={}, len={}, image={}/{}",
1343 current_pos,
1344 seq_mask.len(),
1345 img_idx + 1,
1346 num_images
1347 ));
1348 }
1349 seq_mask[current_pos] = true;
1350 current_pos += 1;
1351 }
1352
1353 current_pos += 1;
1355 }
1356
1357 let images_seq_mask = Tensor::from_slice(
1358 &seq_mask
1359 .iter()
1360 .map(|&b| if b { 1u8 } else { 0u8 })
1361 .collect::<Vec<_>>(),
1362 )
1363 .to_kind(DType::Bool)
1364 .to_device(device)
1365 .unsqueeze(0); let images_emb_mask = Tensor::ones(
1370 [1, num_images as i64, config.num_image_tokens as i64],
1371 (DType::Bool, device),
1372 );
1373
1374 Ok(JanusPlaceholderReplacement {
1375 input_ids: input_ids_tensor,
1376 images_seq_mask,
1377 images_emb_mask,
1378 num_images,
1379 })
1380}
1381
1382unsafe impl Send for VisionHead {}