hyprstream_inference/runtime/architectures/
config.rs1use tch::Kind as DType;
4
5pub trait ArchitectureConfig: Send + Sync {
7 fn num_attention_heads(&self) -> usize;
9
10 fn num_key_value_heads(&self) -> usize;
12
13 fn hidden_size(&self) -> usize;
15
16 fn intermediate_size(&self) -> usize;
18
19 fn head_dim(&self) -> usize {
21 self.hidden_size() / self.num_attention_heads()
22 }
23
24 fn vocab_size(&self) -> usize;
26
27 fn max_position_embeddings(&self) -> usize;
29
30 fn rope_theta(&self) -> Option<f32>;
32
33 fn rope_dim(&self) -> Option<usize>;
35
36 fn layer_norm_eps(&self) -> f32;
38
39 fn use_rms_norm(&self) -> bool;
41
42 fn use_parallel_residual(&self) -> bool {
44 false
45 }
46
47 fn attention_type(&self) -> AttentionType {
49 let num_heads = self.num_attention_heads();
50 let num_kv_heads = self.num_key_value_heads();
51
52 if num_kv_heads == 1 {
53 AttentionType::MQA
54 } else if num_kv_heads < num_heads {
55 AttentionType::GQA {
56 num_groups: num_heads / num_kv_heads,
57 }
58 } else {
59 AttentionType::MHA
60 }
61 }
62
63 fn default_dtype(&self) -> DType {
65 DType::Half }
67}
68
69#[derive(Debug, Clone)]
71pub struct AttentionConfig {
72 pub num_heads: usize,
73 pub num_kv_heads: usize,
74 pub head_dim: usize,
75 pub rotary_dim: Option<usize>,
76 pub max_position_embeddings: usize,
77 pub rope_theta: f32,
78 pub use_flash_attention: bool,
79 pub use_sliding_window: bool,
80 pub sliding_window_size: Option<usize>,
81}
82
83impl Default for AttentionConfig {
84 fn default() -> Self {
85 Self {
86 num_heads: 32,
87 num_kv_heads: 32,
88 head_dim: 128,
89 rotary_dim: None,
90 max_position_embeddings: 4096,
91 rope_theta: 10000.0,
92 use_flash_attention: false,
93 use_sliding_window: false,
94 sliding_window_size: None,
95 }
96 }
97}
98
99#[derive(Debug, Clone, PartialEq)]
101pub enum AttentionType {
102 MHA,
104 MQA,
106 GQA { num_groups: usize },
108}
109
110impl AttentionType {
111 pub fn kv_expansion_factor(&self, num_heads: usize) -> usize {
113 match self {
114 AttentionType::MHA => 1,
115 AttentionType::MQA => num_heads,
116 AttentionType::GQA { num_groups } => num_heads / num_groups,
117 }
118 }
119
120 pub fn supports_kv_cache_sharing(&self) -> bool {
122 !matches!(self, AttentionType::MHA)
123 }
124}
125
126#[derive(Debug, Clone, PartialEq)]
128pub enum NormalizationType {
129 LayerNorm,
130 RMSNorm,
131 GroupNorm { num_groups: usize },
132}
133
134#[derive(Debug, Clone)]
136pub struct FFNConfig {
137 pub intermediate_size: usize,
138 pub hidden_size: usize,
139 pub activation: ActivationType,
140 pub use_gate: bool,
141 pub dropout: f32,
142}
143
144#[derive(Debug, Clone, PartialEq)]
146pub enum ActivationType {
147 GELU,
148 SiLU, ReLU,
150 GEGLU,
151 Swish,
152}
153
154#[cfg(test)]
155mod tests {
156 use super::*;
157
158 #[test]
159 fn test_attention_type_detection() {
160 struct TestConfig {
161 heads: usize,
162 kv_heads: usize,
163 }
164
165 impl ArchitectureConfig for TestConfig {
166 fn num_attention_heads(&self) -> usize {
167 self.heads
168 }
169 fn num_key_value_heads(&self) -> usize {
170 self.kv_heads
171 }
172 fn hidden_size(&self) -> usize {
173 4096
174 }
175 fn intermediate_size(&self) -> usize {
176 11008
177 }
178 fn vocab_size(&self) -> usize {
179 32000
180 }
181 fn max_position_embeddings(&self) -> usize {
182 4096
183 }
184 fn rope_theta(&self) -> Option<f32> {
185 Some(10000.0)
186 }
187 fn rope_dim(&self) -> Option<usize> {
188 None
189 }
190 fn layer_norm_eps(&self) -> f32 {
191 1e-6
192 }
193 fn use_rms_norm(&self) -> bool {
194 true
195 }
196 }
197
198 let config = TestConfig {
200 heads: 32,
201 kv_heads: 32,
202 };
203 assert_eq!(config.attention_type(), AttentionType::MHA);
204
205 let config = TestConfig {
207 heads: 32,
208 kv_heads: 1,
209 };
210 assert_eq!(config.attention_type(), AttentionType::MQA);
211
212 let config = TestConfig {
214 heads: 32,
215 kv_heads: 8,
216 };
217 assert_eq!(
218 config.attention_type(),
219 AttentionType::GQA { num_groups: 4 }
220 );
221 }
222
223 #[test]
224 fn test_kv_expansion_factor() {
225 assert_eq!(AttentionType::MHA.kv_expansion_factor(32), 1);
226 assert_eq!(AttentionType::MQA.kv_expansion_factor(32), 32);
227 assert_eq!(
228 AttentionType::GQA { num_groups: 4 }.kv_expansion_factor(32),
229 8
230 );
231 }
232}