Skip to main content

hyprstream_inference/runtime/architectures/
qwen.rs

1//! Qwen model adapter that reuses Llama implementation with Qwen-specific configurations
2//!
3//! Qwen models are architecturally similar to Llama but with key differences:
4//! - RoPE base (rope_theta) varies by model and version - always read from config.json
5//!   - Qwen2.5 models typically use 1,000,000
6//!   - Qwen3 models typically use 10,000 (standard)
7//! - Context window varies by model (read from max_position_embeddings in config.json)
8//!   - Qwen3-4B: 40,960 tokens
9//!   - Can extend via YARN scaling for longer context in some variants
10//! - Vocabulary size of 151,936 tokens (Qwen2/Qwen3)
11//! - SiLU activation function (same as Llama)
12//! - GQA (Grouped-Query Attention) used in Qwen3 models (num_key_value_heads < num_attention_heads)
13
14use super::llama::LlamaModel;
15use super::ModelOperations;
16use anyhow::Result;
17use std::collections::HashMap;
18use tch::{Device, Kind as DType, Tensor};
19
20/// Create a Qwen model by adapting Llama implementation
21pub struct QwenAdapter;
22
23impl QwenAdapter {
24    /// Create a Qwen model from weights and config.json, using Llama as the base implementation
25    /// Takes mutable reference to weights to enable incremental tensor freeing during construction
26    #[allow(clippy::too_many_arguments)]
27    pub fn from_weights(
28        weights: &mut HashMap<String, Tensor>,
29        config_json: &str,
30        version: u8,
31        is_moe: bool,
32        _context_length: usize,
33        device: &Device,
34        dtype: DType,
35        kv_quant_type: hyprstream_rpc_std::model_client::KVQuantType,
36    ) -> Result<Box<dyn ModelOperations>> {
37        // Parse the config.json to get proper rope_theta and other settings
38        let config = LlamaModel::parse_config(config_json)?;
39
40        tracing::info!(
41            "Parsed config from config.json: rope_theta={}, vocab_size={}, hidden_size={}",
42            config.rope_theta,
43            config.vocab_size,
44            config.hidden_size
45        );
46
47        // Version-specific notes (all configurations loaded from config.json):
48        // - Qwen3: rope_theta, max_position_embeddings (e.g., 40,960 for Qwen3-4B), GQA settings
49        // - Qwen2/Qwen1: All values from config.json
50        // No runtime overrides needed - config.json values are authoritative
51
52        // Handle MoE configurations - currently not implemented
53        if is_moe {
54            return Err(anyhow::anyhow!(
55                "Qwen MoE models (e.g., Qwen3-30B-A3B, Qwen3-235B-A22B) are not yet supported. \
56                 MoE (Mixture of Experts) implementation is deferred for future development. \
57                 Please use a dense Qwen model instead."
58            ));
59        }
60
61        // Log the configuration being used
62        tracing::info!(
63            "[from_weights] Creating Qwen{} model with config: hidden_size={}, num_heads={}, num_kv_heads={}, context_length={}, rope_theta={}, kv_quant={:?}",
64            version,
65            config.hidden_size,
66            config.num_attention_heads,
67            config.num_key_value_heads,
68            config.max_position_embeddings,
69            config.rope_theta,
70            kv_quant_type
71        );
72
73        // Create the model using Llama implementation with Qwen config
74        let model =
75            LlamaModel::from_weights_with_config(weights, config, device, dtype, kv_quant_type)?;
76        Ok(Box::new(model))
77    }
78
79    /// Check if weights contain Qwen-specific patterns
80    pub fn is_qwen_model(weights: &HashMap<String, Tensor>) -> bool {
81        // Check for Qwen-specific weight patterns
82        weights.keys().any(|k| {
83            k.contains("c_attn") ||  // Combined attention weights
84            k.contains("qwen") ||     // Explicit Qwen naming
85            k.contains("c_proj") // Combined projection
86        })
87    }
88}
89
90#[cfg(test)]
91#[allow(clippy::expect_used, clippy::unwrap_used, clippy::print_stdout)]
92mod tests {
93    use crate::runtime::architectures::llama::LlamaConfig;
94
95    #[test]
96    fn test_qwen_config_adaptation() {
97        // Test that Qwen3 gets correct RoPE configuration
98        let config = LlamaConfig {
99            hidden_size: 2560,
100            num_attention_heads: 32,
101            num_key_value_heads: 32,
102            rope_theta: 10_000.0, // Default Llama value
103            max_position_embeddings: 4096,
104            ..Default::default()
105        };
106
107        // After Qwen3 adaptation, rope_theta should be 1M
108        assert_eq!(config.hidden_size, 2560);
109        // In actual implementation, the adapter would modify these values
110    }
111}