Skip to main content

hyprstream_inference/runtime/architectures/
config.rs

1//! Architecture configuration traits and implementations
2
3use tch::Kind as DType;
4
5/// Core configuration for model architectures
6pub trait ArchitectureConfig: Send + Sync {
7    /// Number of attention heads for queries
8    fn num_attention_heads(&self) -> usize;
9
10    /// Number of key-value heads (for MQA/GQA)
11    fn num_key_value_heads(&self) -> usize;
12
13    /// Hidden dimension size
14    fn hidden_size(&self) -> usize;
15
16    /// Intermediate/FFN dimension size
17    fn intermediate_size(&self) -> usize;
18
19    /// Head dimension (hidden_size / num_attention_heads)
20    fn head_dim(&self) -> usize {
21        self.hidden_size() / self.num_attention_heads()
22    }
23
24    /// Vocabulary size
25    fn vocab_size(&self) -> usize;
26
27    /// Maximum position embeddings
28    fn max_position_embeddings(&self) -> usize;
29
30    /// RoPE base frequency (if applicable)
31    fn rope_theta(&self) -> Option<f32>;
32
33    /// RoPE dimension count (if different from head_dim)
34    fn rope_dim(&self) -> Option<usize>;
35
36    /// Layer normalization epsilon
37    fn layer_norm_eps(&self) -> f32;
38
39    /// Whether to use RMSNorm instead of LayerNorm
40    fn use_rms_norm(&self) -> bool;
41
42    /// Whether this architecture uses parallel attention and FFN
43    fn use_parallel_residual(&self) -> bool {
44        false
45    }
46
47    /// Get attention type (MHA, MQA, GQA)
48    fn attention_type(&self) -> AttentionType {
49        let num_heads = self.num_attention_heads();
50        let num_kv_heads = self.num_key_value_heads();
51
52        if num_kv_heads == 1 {
53            AttentionType::MQA
54        } else if num_kv_heads < num_heads {
55            AttentionType::GQA {
56                num_groups: num_heads / num_kv_heads,
57            }
58        } else {
59            AttentionType::MHA
60        }
61    }
62
63    /// Default dtype for this architecture  
64    fn default_dtype(&self) -> DType {
65        DType::Half // F16 equivalent in tch
66    }
67}
68
69/// Attention configuration
70#[derive(Debug, Clone)]
71pub struct AttentionConfig {
72    pub num_heads: usize,
73    pub num_kv_heads: usize,
74    pub head_dim: usize,
75    pub rotary_dim: Option<usize>,
76    pub max_position_embeddings: usize,
77    pub rope_theta: f32,
78    pub use_flash_attention: bool,
79    pub use_sliding_window: bool,
80    pub sliding_window_size: Option<usize>,
81}
82
83impl Default for AttentionConfig {
84    fn default() -> Self {
85        Self {
86            num_heads: 32,
87            num_kv_heads: 32,
88            head_dim: 128,
89            rotary_dim: None,
90            max_position_embeddings: 4096,
91            rope_theta: 10000.0,
92            use_flash_attention: false,
93            use_sliding_window: false,
94            sliding_window_size: None,
95        }
96    }
97}
98
99/// Attention mechanism types
100#[derive(Debug, Clone, PartialEq)]
101pub enum AttentionType {
102    /// Multi-Head Attention (all heads have separate K,V)
103    MHA,
104    /// Multi-Query Attention (single K,V shared across all Q heads)
105    MQA,
106    /// Grouped-Query Attention (groups of Q heads share K,V)
107    GQA { num_groups: usize },
108}
109
110impl AttentionType {
111    /// Get the expansion factor for key-value heads
112    pub fn kv_expansion_factor(&self, num_heads: usize) -> usize {
113        match self {
114            AttentionType::MHA => 1,
115            AttentionType::MQA => num_heads,
116            AttentionType::GQA { num_groups } => num_heads / num_groups,
117        }
118    }
119
120    /// Check if KV cache can be shared
121    pub fn supports_kv_cache_sharing(&self) -> bool {
122        !matches!(self, AttentionType::MHA)
123    }
124}
125
126/// Normalization types
127#[derive(Debug, Clone, PartialEq)]
128pub enum NormalizationType {
129    LayerNorm,
130    RMSNorm,
131    GroupNorm { num_groups: usize },
132}
133
134/// Feed-forward network configuration
135#[derive(Debug, Clone)]
136pub struct FFNConfig {
137    pub intermediate_size: usize,
138    pub hidden_size: usize,
139    pub activation: ActivationType,
140    pub use_gate: bool,
141    pub dropout: f32,
142}
143
144/// Activation function types
145#[derive(Debug, Clone, PartialEq)]
146pub enum ActivationType {
147    GELU,
148    SiLU, // Swish
149    ReLU,
150    GEGLU,
151    Swish,
152}
153
154#[cfg(test)]
155mod tests {
156    use super::*;
157
158    #[test]
159    fn test_attention_type_detection() {
160        struct TestConfig {
161            heads: usize,
162            kv_heads: usize,
163        }
164
165        impl ArchitectureConfig for TestConfig {
166            fn num_attention_heads(&self) -> usize {
167                self.heads
168            }
169            fn num_key_value_heads(&self) -> usize {
170                self.kv_heads
171            }
172            fn hidden_size(&self) -> usize {
173                4096
174            }
175            fn intermediate_size(&self) -> usize {
176                11008
177            }
178            fn vocab_size(&self) -> usize {
179                32000
180            }
181            fn max_position_embeddings(&self) -> usize {
182                4096
183            }
184            fn rope_theta(&self) -> Option<f32> {
185                Some(10000.0)
186            }
187            fn rope_dim(&self) -> Option<usize> {
188                None
189            }
190            fn layer_norm_eps(&self) -> f32 {
191                1e-6
192            }
193            fn use_rms_norm(&self) -> bool {
194                true
195            }
196        }
197
198        // MHA: equal heads
199        let config = TestConfig {
200            heads: 32,
201            kv_heads: 32,
202        };
203        assert_eq!(config.attention_type(), AttentionType::MHA);
204
205        // MQA: single KV head
206        let config = TestConfig {
207            heads: 32,
208            kv_heads: 1,
209        };
210        assert_eq!(config.attention_type(), AttentionType::MQA);
211
212        // GQA: grouped heads
213        let config = TestConfig {
214            heads: 32,
215            kv_heads: 8,
216        };
217        assert_eq!(
218            config.attention_type(),
219            AttentionType::GQA { num_groups: 4 }
220        );
221    }
222
223    #[test]
224    fn test_kv_expansion_factor() {
225        assert_eq!(AttentionType::MHA.kv_expansion_factor(32), 1);
226        assert_eq!(AttentionType::MQA.kv_expansion_factor(32), 32);
227        assert_eq!(
228            AttentionType::GQA { num_groups: 4 }.kv_expansion_factor(32),
229            8
230        );
231    }
232}