Skip to main content

hyprstream_inference/runtime/
tokenizer_config.rs

1//! Trait-based tokenizer configuration for different model architectures
2//!
3//! This module provides a clean abstraction for model-specific tokenizer configuration,
4//! allowing each model architecture to customize tokenizer behavior without polluting
5//! the global runtime code.
6
7use anyhow::Result;
8use tokenizers::{AddedToken, Tokenizer};
9
10/// Trait for model-specific tokenizer configuration
11///
12/// Each model architecture can implement this trait to provide custom
13/// tokenizer configuration, including adding special tokens, configuring
14/// post-processors, and handling vocabulary mismatches.
15pub trait TokenizerConfig: Send + Sync {
16    /// Configure the tokenizer for this specific model architecture
17    ///
18    /// This method is called after loading the base tokenizer from file,
19    /// allowing the model to add special tokens, configure post-processing,
20    /// or make other model-specific adjustments.
21    ///
22    /// # Arguments
23    /// * `tokenizer` - The tokenizer to configure (mutable reference)
24    /// * `model_vocab_size` - The vocabulary size from the model's config
25    ///
26    /// # Returns
27    /// * `Result<()>` - Success or error during configuration
28    fn configure_tokenizer(&self, tokenizer: &mut Tokenizer, model_vocab_size: usize)
29        -> Result<()>;
30
31    /// Get special tokens that should be added for this model
32    ///
33    /// This is called by `configure_tokenizer` but can also be used
34    /// independently if needed.
35    ///
36    /// # Arguments
37    /// * `model_vocab_size` - Total vocabulary size expected by the model
38    /// * `tokenizer_vocab_size` - Current size of the tokenizer vocabulary
39    ///
40    /// # Returns
41    /// * Vector of `AddedToken` objects to add to the tokenizer
42    fn get_special_tokens(
43        &self,
44        _model_vocab_size: usize,
45        _tokenizer_vocab_size: usize,
46    ) -> Vec<AddedToken> {
47        Vec::new() // Default: no special tokens
48    }
49
50    /// Check if a token ID represents a special token for this model
51    ///
52    /// # Arguments
53    /// * `token_id` - The token ID to check
54    /// * `vocab_size` - The base vocabulary size (without special tokens)
55    ///
56    /// # Returns
57    /// * `true` if this is a special token, `false` otherwise
58    fn is_special_token(&self, _token_id: usize, _vocab_size: usize) -> bool {
59        false // Default: no special tokens
60    }
61
62    /// Get display representation for out-of-vocabulary or special tokens
63    ///
64    /// This is useful for debugging or when special tokens need custom display.
65    ///
66    /// # Arguments
67    /// * `token_id` - The token ID to format
68    /// * `vocab_size` - The base vocabulary size
69    ///
70    /// # Returns
71    /// * `Some(String)` with the formatted token, or `None` if no special formatting
72    fn format_special_token(&self, _token_id: usize, _vocab_size: usize) -> Option<String> {
73        None // Default: no special formatting
74    }
75
76    /// Decode a token that might be out of vocabulary
77    ///
78    /// This method is called when a token needs to be decoded. It can handle
79    /// tokens that are beyond the tokenizer's vocabulary by returning a special
80    /// representation.
81    ///
82    /// # Arguments
83    /// * `tokenizer` - The tokenizer to use for decoding
84    /// * `token_id` - The token ID to decode
85    ///
86    /// # Returns
87    /// * `Ok(Some(String))` - Successfully decoded token or special representation
88    /// * `Ok(None)` - Token should be skipped
89    /// * `Err` - Decoding error
90    fn decode_token(&self, tokenizer: &Tokenizer, token_id: u32) -> Result<Option<String>> {
91        // Default: try normal decoding
92        match tokenizer.decode(&[token_id], false) {
93            Ok(text) => Ok(Some(text)),
94            Err(_) => {
95                // If decoding fails, check if it's a special token
96                let vocab_size = tokenizer.get_vocab_size(true);
97                if let Some(special) = self.format_special_token(token_id as usize, vocab_size) {
98                    Ok(Some(special))
99                } else {
100                    Ok(None) // Skip unknown tokens
101                }
102            }
103        }
104    }
105}
106
107// ============================================================================
108// Model-specific implementations
109// ============================================================================
110
111/// Qwen model tokenizer configuration
112///
113/// Qwen models have a vocabulary mismatch between the model (151,936 tokens)
114/// and the distributed tokenizer (151,643 tokens). The missing 293 tokens are
115/// special tokens following the <|extra_N|> convention.
116pub struct QwenTokenizerConfig;
117
118impl TokenizerConfig for QwenTokenizerConfig {
119    fn configure_tokenizer(
120        &self,
121        tokenizer: &mut Tokenizer,
122        model_vocab_size: usize,
123    ) -> Result<()> {
124        let current_vocab_size = tokenizer.get_vocab_size(true);
125
126        // Qwen models have special tokens beyond the base vocabulary
127        if model_vocab_size > current_vocab_size {
128            tracing::info!(
129                "Qwen model vocabulary mismatch detected: model has {} tokens, tokenizer has {} tokens",
130                model_vocab_size, current_vocab_size
131            );
132
133            let missing_count = model_vocab_size - current_vocab_size;
134
135            // Create special tokens for the missing vocabulary entries
136            // Qwen uses <|extra_N|> convention for additional tokens
137            let mut tokens_to_add = Vec::with_capacity(missing_count);
138            for i in 0..missing_count {
139                // Create as special tokens (special=true)
140                // They ARE special tokens in the model's vocabulary
141                let token = AddedToken::from(format!("<|extra_{i}|>"), true)
142                    .single_word(true) // Treat as single word
143                    .lstrip(false)
144                    .rstrip(false)
145                    .normalized(false);
146                tokens_to_add.push(token);
147            }
148
149            // Add as special tokens to the tokenizer
150            // They will be properly tracked as special tokens with correct IDs
151            let added = tokenizer.add_special_tokens(&tokens_to_add);
152
153            tracing::info!(
154                "Added {} special tokens (<|extra_0|> through <|extra_{}|>) to Qwen tokenizer",
155                added,
156                missing_count - 1
157            );
158
159            // Verify they were actually added. A tokenizer/embedding vocab
160            // mismatch is the documented cause of CUDA index-out-of-bounds on
161            // the embedding lookup (#143): the model can sample or be fed a
162            // token id in [tokenizer_vocab, embedding_vocab), which the
163            // tokenizer cannot decode and which — if the embedding matrix was
164            // allocated to the smaller tokenizer size — faults the GPU. So a
165            // short-fall here is a hard error, not a warning.
166            let new_size = tokenizer.get_vocab_size(true);
167            if new_size == model_vocab_size {
168                tracing::info!(
169                    "✅ Vocabulary successfully expanded from {} to {}",
170                    current_vocab_size,
171                    new_size
172                );
173            } else if new_size > current_vocab_size {
174                tracing::info!(
175                    "✅ Vocabulary expanded from {} to {} (added {} tokens)",
176                    current_vocab_size,
177                    new_size,
178                    new_size - current_vocab_size
179                );
180            }
181            // new_size < current_vocab_size only happens if add_special_tokens
182            // no-ops entirely (e.g. all candidates already present) — still a
183            // mismatch when model_vocab_size > new_size.
184            if new_size != model_vocab_size {
185                anyhow::bail!(
186                    "Qwen tokenizer vocab ({}) could not be expanded to the model's \
187                     embedding size ({}): add_special_tokens reported {}, final size {}. \
188                     A tokenizer/embedding vocab mismatch causes a CUDA \
189                     index-out-of-bounds on the embedding lookup; refusing to load.",
190                    current_vocab_size,
191                    model_vocab_size,
192                    added,
193                    new_size
194                );
195            }
196        }
197
198        Ok(())
199    }
200
201    fn get_special_tokens(
202        &self,
203        model_vocab_size: usize,
204        tokenizer_vocab_size: usize,
205    ) -> Vec<AddedToken> {
206        if model_vocab_size <= tokenizer_vocab_size {
207            return Vec::new();
208        }
209
210        let missing_count = model_vocab_size - tokenizer_vocab_size;
211        let mut tokens = Vec::with_capacity(missing_count);
212
213        // Qwen uses <|extra_N|> convention for additional tokens
214        // These include both reserved tokens and control tokens
215        for i in 0..missing_count {
216            let token = AddedToken::from(format!("<|extra_{i}|>"), true)
217                .single_word(false)
218                .lstrip(false)
219                .rstrip(false)
220                .normalized(false);
221            tokens.push(token);
222        }
223
224        tokens
225    }
226
227    fn is_special_token(&self, token_id: usize, vocab_size: usize) -> bool {
228        // Tokens beyond the base vocabulary are special tokens
229        token_id >= vocab_size
230    }
231
232    fn format_special_token(&self, token_id: usize, vocab_size: usize) -> Option<String> {
233        if token_id >= vocab_size {
234            // Format as <|extra_N|> for display/debugging
235            Some(format!("<|extra_{}|>", token_id - vocab_size))
236        } else {
237            None
238        }
239    }
240
241    fn decode_token(&self, tokenizer: &Tokenizer, token_id: u32) -> Result<Option<String>> {
242        // First try normal decoding (including special tokens)
243        // Since we added <|extra_N|> as special tokens, they should decode properly
244        match tokenizer.decode(&[token_id], false) {
245            Ok(text) => Ok(Some(text)),
246            Err(_) => {
247                // If decoding fails, check if it's an out-of-bounds token
248                let vocab_size = tokenizer.get_vocab_size(true);
249                if (token_id as usize) >= vocab_size {
250                    // Token is beyond even the expanded vocabulary
251                    // This shouldn't happen if we configured the tokenizer correctly
252                    tracing::warn!(
253                        "Token {} is beyond expanded vocabulary size {}, formatting as <|extra_N|>",
254                        token_id,
255                        vocab_size
256                    );
257                    let extra_idx = (token_id as usize) - vocab_size;
258                    Ok(Some(format!("<|extra_{extra_idx}|>")))
259                } else {
260                    tracing::warn!("Failed to decode token {} within vocab", token_id);
261                    Ok(None) // Skip tokens that can't be decoded
262                }
263            }
264        }
265    }
266}
267
268/// Llama model tokenizer configuration
269///
270/// Llama models typically have matching vocabulary sizes between
271/// the model and tokenizer, requiring no special configuration.
272pub struct LlamaTokenizerConfig;
273
274impl TokenizerConfig for LlamaTokenizerConfig {
275    fn configure_tokenizer(
276        &self,
277        tokenizer: &mut Tokenizer,
278        model_vocab_size: usize,
279    ) -> Result<()> {
280        let current_vocab_size = tokenizer.get_vocab_size(true);
281
282        if model_vocab_size != current_vocab_size {
283            tracing::debug!(
284                "Llama model vocab size ({}) differs from tokenizer vocab size ({}), but this is typically OK",
285                model_vocab_size, current_vocab_size
286            );
287        }
288
289        // Llama tokenizers typically don't need special configuration
290        // The tokenizer.json should already be complete
291        Ok(())
292    }
293}
294
295/// Gemma model tokenizer configuration
296pub struct GemmaTokenizerConfig;
297
298impl TokenizerConfig for GemmaTokenizerConfig {
299    fn configure_tokenizer(
300        &self,
301        tokenizer: &mut Tokenizer,
302        model_vocab_size: usize,
303    ) -> Result<()> {
304        let current_vocab_size = tokenizer.get_vocab_size(true);
305
306        if model_vocab_size != current_vocab_size {
307            tracing::debug!(
308                "Gemma model vocab size ({}) differs from tokenizer vocab size ({})",
309                model_vocab_size,
310                current_vocab_size
311            );
312        }
313
314        // Gemma models may have their own special tokens or configuration
315        // Add Gemma-specific setup here if needed
316        Ok(())
317    }
318}
319
320/// Default tokenizer configuration for unknown model types
321///
322/// This provides a fallback that doesn't modify the tokenizer,
323/// suitable for models that don't require special handling.
324pub struct DefaultTokenizerConfig;
325
326impl TokenizerConfig for DefaultTokenizerConfig {
327    fn configure_tokenizer(
328        &self,
329        _tokenizer: &mut Tokenizer,
330        _model_vocab_size: usize,
331    ) -> Result<()> {
332        // No special configuration for unknown models
333        tracing::debug!("Using default tokenizer configuration (no modifications)");
334        Ok(())
335    }
336}