hyprstream_inference/runtime/architectures/qwen.rs
1//! Qwen model adapter that reuses Llama implementation with Qwen-specific configurations
2//!
3//! Qwen models are architecturally similar to Llama but with key differences:
4//! - RoPE base (rope_theta) varies by model and version - always read from config.json
5//! - Qwen2.5 models typically use 1,000,000
6//! - Qwen3 models typically use 10,000 (standard)
7//! - Context window varies by model (read from max_position_embeddings in config.json)
8//! - Qwen3-4B: 40,960 tokens
9//! - Can extend via YARN scaling for longer context in some variants
10//! - Vocabulary size of 151,936 tokens (Qwen2/Qwen3)
11//! - SiLU activation function (same as Llama)
12//! - GQA (Grouped-Query Attention) used in Qwen3 models (num_key_value_heads < num_attention_heads)
13
14use super::llama::LlamaModel;
15use super::ModelOperations;
16use anyhow::Result;
17use std::collections::HashMap;
18use tch::{Device, Kind as DType, Tensor};
19
20/// Create a Qwen model by adapting Llama implementation
21pub struct QwenAdapter;
22
23impl QwenAdapter {
24 /// Create a Qwen model from weights and config.json, using Llama as the base implementation
25 /// Takes mutable reference to weights to enable incremental tensor freeing during construction
26 #[allow(clippy::too_many_arguments)]
27 pub fn from_weights(
28 weights: &mut HashMap<String, Tensor>,
29 config_json: &str,
30 version: u8,
31 is_moe: bool,
32 _context_length: usize,
33 device: &Device,
34 dtype: DType,
35 kv_quant_type: hyprstream_rpc_std::model_client::KVQuantType,
36 ) -> Result<Box<dyn ModelOperations>> {
37 // Parse the config.json to get proper rope_theta and other settings
38 let config = LlamaModel::parse_config(config_json)?;
39
40 tracing::info!(
41 "Parsed config from config.json: rope_theta={}, vocab_size={}, hidden_size={}",
42 config.rope_theta,
43 config.vocab_size,
44 config.hidden_size
45 );
46
47 // Version-specific notes (all configurations loaded from config.json):
48 // - Qwen3: rope_theta, max_position_embeddings (e.g., 40,960 for Qwen3-4B), GQA settings
49 // - Qwen2/Qwen1: All values from config.json
50 // No runtime overrides needed - config.json values are authoritative
51
52 // Handle MoE configurations - currently not implemented
53 if is_moe {
54 return Err(anyhow::anyhow!(
55 "Qwen MoE models (e.g., Qwen3-30B-A3B, Qwen3-235B-A22B) are not yet supported. \
56 MoE (Mixture of Experts) implementation is deferred for future development. \
57 Please use a dense Qwen model instead."
58 ));
59 }
60
61 // Log the configuration being used
62 tracing::info!(
63 "[from_weights] Creating Qwen{} model with config: hidden_size={}, num_heads={}, num_kv_heads={}, context_length={}, rope_theta={}, kv_quant={:?}",
64 version,
65 config.hidden_size,
66 config.num_attention_heads,
67 config.num_key_value_heads,
68 config.max_position_embeddings,
69 config.rope_theta,
70 kv_quant_type
71 );
72
73 // Create the model using Llama implementation with Qwen config
74 let model =
75 LlamaModel::from_weights_with_config(weights, config, device, dtype, kv_quant_type)?;
76 Ok(Box::new(model))
77 }
78
79 /// Check if weights contain Qwen-specific patterns
80 pub fn is_qwen_model(weights: &HashMap<String, Tensor>) -> bool {
81 // Check for Qwen-specific weight patterns
82 weights.keys().any(|k| {
83 k.contains("c_attn") || // Combined attention weights
84 k.contains("qwen") || // Explicit Qwen naming
85 k.contains("c_proj") // Combined projection
86 })
87 }
88}
89
90#[cfg(test)]
91#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
92mod tests {
93 use crate::runtime::architectures::llama::LlamaConfig;
94
95 #[test]
96 fn test_qwen_config_adaptation() {
97 // Test that Qwen3 gets correct RoPE configuration
98 let config = LlamaConfig {
99 hidden_size: 2560,
100 num_attention_heads: 32,
101 num_key_value_heads: 32,
102 rope_theta: 10_000.0, // Default Llama value
103 max_position_embeddings: 4096,
104 ..Default::default()
105 };
106
107 // After Qwen3 adaptation, rope_theta should be 1M
108 assert_eq!(config.hidden_size, 2560);
109 // In actual implementation, the adapter would modify these values
110 }
111}