hyprstream_inference/runtime/tokenizer_config.rs
1//! Trait-based tokenizer configuration for different model architectures
2//!
3//! This module provides a clean abstraction for model-specific tokenizer configuration,
4//! allowing each model architecture to customize tokenizer behavior without polluting
5//! the global runtime code.
6
7use anyhow::Result;
8use tokenizers::{AddedToken, Tokenizer};
9
10/// Trait for model-specific tokenizer configuration
11///
12/// Each model architecture can implement this trait to provide custom
13/// tokenizer configuration, including adding special tokens, configuring
14/// post-processors, and handling vocabulary mismatches.
15pub trait TokenizerConfig: Send + Sync {
16 /// Configure the tokenizer for this specific model architecture
17 ///
18 /// This method is called after loading the base tokenizer from file,
19 /// allowing the model to add special tokens, configure post-processing,
20 /// or make other model-specific adjustments.
21 ///
22 /// # Arguments
23 /// * `tokenizer` - The tokenizer to configure (mutable reference)
24 /// * `model_vocab_size` - The vocabulary size from the model's config
25 ///
26 /// # Returns
27 /// * `Result<()>` - Success or error during configuration
28 fn configure_tokenizer(&self, tokenizer: &mut Tokenizer, model_vocab_size: usize)
29 -> Result<()>;
30
31 /// Get special tokens that should be added for this model
32 ///
33 /// This is called by `configure_tokenizer` but can also be used
34 /// independently if needed.
35 ///
36 /// # Arguments
37 /// * `model_vocab_size` - Total vocabulary size expected by the model
38 /// * `tokenizer_vocab_size` - Current size of the tokenizer vocabulary
39 ///
40 /// # Returns
41 /// * Vector of `AddedToken` objects to add to the tokenizer
42 fn get_special_tokens(
43 &self,
44 _model_vocab_size: usize,
45 _tokenizer_vocab_size: usize,
46 ) -> Vec<AddedToken> {
47 Vec::new() // Default: no special tokens
48 }
49
50 /// Check if a token ID represents a special token for this model
51 ///
52 /// # Arguments
53 /// * `token_id` - The token ID to check
54 /// * `vocab_size` - The base vocabulary size (without special tokens)
55 ///
56 /// # Returns
57 /// * `true` if this is a special token, `false` otherwise
58 fn is_special_token(&self, _token_id: usize, _vocab_size: usize) -> bool {
59 false // Default: no special tokens
60 }
61
62 /// Get display representation for out-of-vocabulary or special tokens
63 ///
64 /// This is useful for debugging or when special tokens need custom display.
65 ///
66 /// # Arguments
67 /// * `token_id` - The token ID to format
68 /// * `vocab_size` - The base vocabulary size
69 ///
70 /// # Returns
71 /// * `Some(String)` with the formatted token, or `None` if no special formatting
72 fn format_special_token(&self, _token_id: usize, _vocab_size: usize) -> Option<String> {
73 None // Default: no special formatting
74 }
75
76 /// Decode a token that might be out of vocabulary
77 ///
78 /// This method is called when a token needs to be decoded. It can handle
79 /// tokens that are beyond the tokenizer's vocabulary by returning a special
80 /// representation.
81 ///
82 /// # Arguments
83 /// * `tokenizer` - The tokenizer to use for decoding
84 /// * `token_id` - The token ID to decode
85 ///
86 /// # Returns
87 /// * `Ok(Some(String))` - Successfully decoded token or special representation
88 /// * `Ok(None)` - Token should be skipped
89 /// * `Err` - Decoding error
90 fn decode_token(&self, tokenizer: &Tokenizer, token_id: u32) -> Result<Option<String>> {
91 // Default: try normal decoding
92 match tokenizer.decode(&[token_id], false) {
93 Ok(text) => Ok(Some(text)),
94 Err(_) => {
95 // If decoding fails, check if it's a special token
96 let vocab_size = tokenizer.get_vocab_size(true);
97 if let Some(special) = self.format_special_token(token_id as usize, vocab_size) {
98 Ok(Some(special))
99 } else {
100 Ok(None) // Skip unknown tokens
101 }
102 }
103 }
104 }
105}
106
107// ============================================================================
108// Model-specific implementations
109// ============================================================================
110
111/// Qwen model tokenizer configuration
112///
113/// Qwen models have a vocabulary mismatch between the model (151,936 tokens)
114/// and the distributed tokenizer (151,643 tokens). The missing 293 tokens are
115/// special tokens following the <|extra_N|> convention.
116pub struct QwenTokenizerConfig;
117
118impl TokenizerConfig for QwenTokenizerConfig {
119 fn configure_tokenizer(
120 &self,
121 tokenizer: &mut Tokenizer,
122 model_vocab_size: usize,
123 ) -> Result<()> {
124 let current_vocab_size = tokenizer.get_vocab_size(true);
125
126 // Qwen models have special tokens beyond the base vocabulary
127 if model_vocab_size > current_vocab_size {
128 tracing::info!(
129 "Qwen model vocabulary mismatch detected: model has {} tokens, tokenizer has {} tokens",
130 model_vocab_size, current_vocab_size
131 );
132
133 let missing_count = model_vocab_size - current_vocab_size;
134
135 // Create special tokens for the missing vocabulary entries
136 // Qwen uses <|extra_N|> convention for additional tokens
137 let mut tokens_to_add = Vec::with_capacity(missing_count);
138 for i in 0..missing_count {
139 // Create as special tokens (special=true)
140 // They ARE special tokens in the model's vocabulary
141 let token = AddedToken::from(format!("<|extra_{i}|>"), true)
142 .single_word(true) // Treat as single word
143 .lstrip(false)
144 .rstrip(false)
145 .normalized(false);
146 tokens_to_add.push(token);
147 }
148
149 // Add as special tokens to the tokenizer
150 // They will be properly tracked as special tokens with correct IDs
151 let added = tokenizer.add_special_tokens(&tokens_to_add);
152
153 tracing::info!(
154 "Added {} special tokens (<|extra_0|> through <|extra_{}|>) to Qwen tokenizer",
155 added,
156 missing_count - 1
157 );
158
159 // Verify they were actually added. A tokenizer/embedding vocab
160 // mismatch is the documented cause of CUDA index-out-of-bounds on
161 // the embedding lookup (#143): the model can sample or be fed a
162 // token id in [tokenizer_vocab, embedding_vocab), which the
163 // tokenizer cannot decode and which — if the embedding matrix was
164 // allocated to the smaller tokenizer size — faults the GPU. So a
165 // short-fall here is a hard error, not a warning.
166 let new_size = tokenizer.get_vocab_size(true);
167 if new_size == model_vocab_size {
168 tracing::info!(
169 "✅ Vocabulary successfully expanded from {} to {}",
170 current_vocab_size,
171 new_size
172 );
173 } else if new_size > current_vocab_size {
174 tracing::info!(
175 "✅ Vocabulary expanded from {} to {} (added {} tokens)",
176 current_vocab_size,
177 new_size,
178 new_size - current_vocab_size
179 );
180 }
181 // new_size < current_vocab_size only happens if add_special_tokens
182 // no-ops entirely (e.g. all candidates already present) — still a
183 // mismatch when model_vocab_size > new_size.
184 if new_size != model_vocab_size {
185 anyhow::bail!(
186 "Qwen tokenizer vocab ({}) could not be expanded to the model's \
187 embedding size ({}): add_special_tokens reported {}, final size {}. \
188 A tokenizer/embedding vocab mismatch causes a CUDA \
189 index-out-of-bounds on the embedding lookup; refusing to load.",
190 current_vocab_size,
191 model_vocab_size,
192 added,
193 new_size
194 );
195 }
196 }
197
198 Ok(())
199 }
200
201 fn get_special_tokens(
202 &self,
203 model_vocab_size: usize,
204 tokenizer_vocab_size: usize,
205 ) -> Vec<AddedToken> {
206 if model_vocab_size <= tokenizer_vocab_size {
207 return Vec::new();
208 }
209
210 let missing_count = model_vocab_size - tokenizer_vocab_size;
211 let mut tokens = Vec::with_capacity(missing_count);
212
213 // Qwen uses <|extra_N|> convention for additional tokens
214 // These include both reserved tokens and control tokens
215 for i in 0..missing_count {
216 let token = AddedToken::from(format!("<|extra_{i}|>"), true)
217 .single_word(false)
218 .lstrip(false)
219 .rstrip(false)
220 .normalized(false);
221 tokens.push(token);
222 }
223
224 tokens
225 }
226
227 fn is_special_token(&self, token_id: usize, vocab_size: usize) -> bool {
228 // Tokens beyond the base vocabulary are special tokens
229 token_id >= vocab_size
230 }
231
232 fn format_special_token(&self, token_id: usize, vocab_size: usize) -> Option<String> {
233 if token_id >= vocab_size {
234 // Format as <|extra_N|> for display/debugging
235 Some(format!("<|extra_{}|>", token_id - vocab_size))
236 } else {
237 None
238 }
239 }
240
241 fn decode_token(&self, tokenizer: &Tokenizer, token_id: u32) -> Result<Option<String>> {
242 // First try normal decoding (including special tokens)
243 // Since we added <|extra_N|> as special tokens, they should decode properly
244 match tokenizer.decode(&[token_id], false) {
245 Ok(text) => Ok(Some(text)),
246 Err(_) => {
247 // If decoding fails, check if it's an out-of-bounds token
248 let vocab_size = tokenizer.get_vocab_size(true);
249 if (token_id as usize) >= vocab_size {
250 // Token is beyond even the expanded vocabulary
251 // This shouldn't happen if we configured the tokenizer correctly
252 tracing::warn!(
253 "Token {} is beyond expanded vocabulary size {}, formatting as <|extra_N|>",
254 token_id,
255 vocab_size
256 );
257 let extra_idx = (token_id as usize) - vocab_size;
258 Ok(Some(format!("<|extra_{extra_idx}|>")))
259 } else {
260 tracing::warn!("Failed to decode token {} within vocab", token_id);
261 Ok(None) // Skip tokens that can't be decoded
262 }
263 }
264 }
265 }
266}
267
268/// Llama model tokenizer configuration
269///
270/// Llama models typically have matching vocabulary sizes between
271/// the model and tokenizer, requiring no special configuration.
272pub struct LlamaTokenizerConfig;
273
274impl TokenizerConfig for LlamaTokenizerConfig {
275 fn configure_tokenizer(
276 &self,
277 tokenizer: &mut Tokenizer,
278 model_vocab_size: usize,
279 ) -> Result<()> {
280 let current_vocab_size = tokenizer.get_vocab_size(true);
281
282 if model_vocab_size != current_vocab_size {
283 tracing::debug!(
284 "Llama model vocab size ({}) differs from tokenizer vocab size ({}), but this is typically OK",
285 model_vocab_size, current_vocab_size
286 );
287 }
288
289 // Llama tokenizers typically don't need special configuration
290 // The tokenizer.json should already be complete
291 Ok(())
292 }
293}
294
295/// Gemma model tokenizer configuration
296pub struct GemmaTokenizerConfig;
297
298impl TokenizerConfig for GemmaTokenizerConfig {
299 fn configure_tokenizer(
300 &self,
301 tokenizer: &mut Tokenizer,
302 model_vocab_size: usize,
303 ) -> Result<()> {
304 let current_vocab_size = tokenizer.get_vocab_size(true);
305
306 if model_vocab_size != current_vocab_size {
307 tracing::debug!(
308 "Gemma model vocab size ({}) differs from tokenizer vocab size ({})",
309 model_vocab_size,
310 current_vocab_size
311 );
312 }
313
314 // Gemma models may have their own special tokens or configuration
315 // Add Gemma-specific setup here if needed
316 Ok(())
317 }
318}
319
320/// Default tokenizer configuration for unknown model types
321///
322/// This provides a fallback that doesn't modify the tokenizer,
323/// suitable for models that don't require special handling.
324pub struct DefaultTokenizerConfig;
325
326impl TokenizerConfig for DefaultTokenizerConfig {
327 fn configure_tokenizer(
328 &self,
329 _tokenizer: &mut Tokenizer,
330 _model_vocab_size: usize,
331 ) -> Result<()> {
332 // No special configuration for unknown models
333 tracing::debug!("Using default tokenizer configuration (no modifications)");
334 Ok(())
335 }
336}