hyprstream_inference/runtime/torch_engine.rs
1//! PyTorch-based inference engine using tch-rs
2
3use crate::config::{FinishReason, GenerationConfig, GenerationResult, RuntimeConfig};
4use crate::runtime::architectures::ModelOperations;
5use crate::runtime::template_engine::{ChatMessage, TemplateEngine};
6use crate::runtime::tensor_sampling::TensorSampler;
7use crate::runtime::RuntimeEngine;
8use anyhow::{anyhow, Result};
9use async_trait::async_trait;
10use hyprstream_rpc_std::inference_client::GenerationRequest;
11use hyprstream_rpc_std::inference_client::ModelInfo;
12use json_threat_protection as jtp;
13use parking_lot::Mutex;
14use serde_json;
15use std::collections::{HashMap, HashSet};
16use std::path::Path;
17use std::sync::{
18 atomic::{AtomicU32, AtomicUsize, Ordering},
19 Arc,
20};
21use tch::{nn::VarStore, Device, Tensor};
22use tokenizers::Tokenizer;
23use tracing::{info, instrument, warn};
24
25/// Resolve the compute dtype independently from checkpoint weight storage.
26/// This pure policy accepts the documented case-insensitive precision aliases;
27/// it performs no device allocation. `fp8` retains legacy BF16 compute except
28/// for qualified channel-FP8 CPU checkpoints, whose activation QDQ/matmuls run
29/// in Float32. It never requests hardware Float8 activations on CPU.
30/// Serving adapters must still enforce their own qualified-device/dtype gate.
31pub fn resolve_inference_compute_kind(
32 precision_mode: Option<&str>,
33 device: Device,
34 channel_fp8: bool,
35) -> Result<tch::Kind> {
36 match precision_mode
37 .unwrap_or("auto")
38 .to_ascii_lowercase()
39 .as_str()
40 {
41 "fp32" | "float32" => Ok(tch::Kind::Float),
42 "fp16" | "float16" => Ok(tch::Kind::Half),
43 "auto" | "fp8" if device == Device::Cpu && channel_fp8 => Ok(tch::Kind::Float),
44 "bf16" | "bfloat16" | "auto" | "fp8" => Ok(tch::Kind::BFloat16),
45 mode => anyhow::bail!("unsupported inference precision mode: {mode}"),
46 }
47}
48
49/// Build a self-explaining diagnostic for why CUDA/ROCm is unavailable.
50///
51/// `tch::Cuda::is_available()` returns a bare `false` with no cause, which made
52/// the GPU→CPU fallback nearly impossible to diagnose in the field (e.g. an RTX
53/// 5090 node with a CUDA libtorch bundled that still logged a one-line "not
54/// available"). This collects the underlying signals so the operator can tell
55/// *why*:
56///
57/// - `has_cuda` / `has_cudart`: was this libtorch even compiled with CUDA? If
58/// `false`, an install/active-backend mismatch bundled a CPU build.
59/// - `device_count`: 0 with a CUDA build almost always means the driver lib
60/// (`libcuda.so.1`, shipped by the host NVIDIA driver — *never* by the
61/// AppImage) could not be loaded, i.e. `LD_LIBRARY_PATH` does not reach the
62/// host driver path. A negative count is a CUDA-internal error.
63/// - The current `LD_LIBRARY_PATH` is echoed so the operator can immediately
64/// see whether the host driver path is missing.
65fn cuda_unavailable_diagnostics() -> String {
66 let has_cuda = tch::utils::has_cuda();
67 let has_cudart = tch::utils::has_cudart();
68 let has_hip = tch::utils::has_hip();
69 let device_count = tch::Cuda::device_count();
70 let ld_path = std::env::var("LD_LIBRARY_PATH").unwrap_or_else(|_| "<unset>".to_owned());
71
72 // Best-effort, actionable interpretation of the signals above.
73 let hint = if !has_cuda && !has_hip {
74 "this libtorch was NOT built with CUDA/ROCm support (CPU-only build). \
75 Check the installed/active backend — an install likely selected a 'cpu' \
76 variant (verify ~/.local/share/hyprstream/active-backend / active-version)."
77 } else if device_count <= 0 {
78 "libtorch has CUDA/ROCm support but reports 0 devices. This is almost \
79 always the host driver library (libcuda.so.1) failing to load: it ships \
80 with the NVIDIA driver and is never bundled in the AppImage. Ensure the \
81 host driver lib dir is on LD_LIBRARY_PATH (e.g. /usr/lib64, \
82 /usr/lib/x86_64-linux-gnu, or the libcuda.so.1 location from \
83 `ldconfig -p | grep libcuda`), or that ldconfig has it cached."
84 } else {
85 "CUDA/ROCm reports devices but is_available() is false — possible \
86 driver/runtime version mismatch or a CUDA-internal init error."
87 };
88
89 format!(
90 "is_available()=false; has_cuda={has_cuda}, has_cudart={has_cudart}, \
91 has_hip={has_hip}, device_count={device_count}, \
92 LD_LIBRARY_PATH={ld_path}. Likely cause: {hint}"
93 )
94}
95
96/// Basic context state for tracking generation state
97#[derive(Debug, Clone)]
98pub struct ContextState {
99 /// Current sequence length
100 pub sequence_length: usize,
101 /// Context window size
102 pub context_window: usize,
103 /// Whether model is initialized
104 pub initialized: bool,
105}
106
107/// PyTorch inference engine using tch-rs
108///
109/// Thread-safe implementation using proper synchronization primitives.
110/// All mutable state is protected by mutexes with poisoning recovery.
111#[derive(Clone)]
112pub struct TorchEngine {
113 /// VarStore for native PyTorch weight management - not thread safe, requires external sync
114 var_store: Arc<Mutex<Option<VarStore>>>,
115 /// SafeTensors raw data for on-demand tensor creation - thread safe after initialization
116 /// Detected model architecture - thread safe after initialization
117 model_architecture: Arc<Mutex<Option<String>>>,
118 /// Persistent model instance to avoid recreation on every forward pass
119 /// Using Arc<Mutex<>> for interior mutability since ModelOperations has mutable methods
120 persistent_model: Option<Arc<Mutex<Box<dyn ModelOperations>>>>,
121 /// Basic KV cache storage for context tracking - thread safe with mutex
122 context_state: Arc<Mutex<Option<ContextState>>>,
123 /// Tokenizer for text processing - thread safe after initialization
124 tokenizer: Arc<Mutex<Option<Tokenizer>>>,
125 /// Template engine for chat formatting
126 template_engine: Arc<Mutex<Option<TemplateEngine>>>,
127 /// Device for computation (CPU/CUDA/ROCm) - immutable after construction.
128 ///
129 /// For a multi-device engine this is the pool's **primary** (first) device —
130 /// the home of non-layer weights (embeddings, final norm, lm_head) and the
131 /// device inputs are placed on before the first decoder layer. Per-layer
132 /// placement is driven by the model's own `LayerDeviceMap` (`forward_layers`).
133 device: Device,
134 /// Resolved multi-device pool, when the engine was built from >1 device
135 /// (#314 wiring). `None` for single-device engines (the common case). When
136 /// `Some` and non-single, the loaded model is built as a single pipeline
137 /// stage spanning `[0..num_hidden_layers)` with a layer→device map derived
138 /// from this pool. Held as `Option<Arc<...>>` because `TorchEngine` is
139 /// `Clone` and the pool is `Send + Sync` (it holds only `Device` values).
140 device_pool: Option<Arc<crate::runtime::DevicePool>>,
141 /// Runtime configuration - immutable after construction
142 config: RuntimeConfig,
143 /// Generation configuration with defaults
144 generation_config: GenerationConfig,
145 /// Loaded model information - protected by mutex
146 model_info: Arc<Mutex<ModelInfo>>,
147 /// Active LoRA adapter name - thread safe with mutex
148 pub active_lora: Arc<Mutex<Option<String>>>,
149 /// GPU sampler for efficient token sampling - thread safe after initialization
150 sampler: TensorSampler, // Renamed from gpu_sampler - works on both CPU and GPU
151 /// Cached tokenizer vocabulary size for lock-free access
152 /// 0 means not yet initialized
153 tokenizer_vocab_size: Arc<AtomicUsize>,
154 /// Cached EOS token ID for lock-free access
155 /// 0 means not yet initialized (actual EOS tokens are typically > 0)
156 eos_token_id: Arc<AtomicU32>,
157 /// KV cache registry for session-based cache isolation
158 /// Enables concurrent inference across multiple sessions
159 kv_cache_registry: Option<Arc<crate::runtime::kv_cache::KVCacheRegistry>>,
160 /// Current active session owner for KV cache selection
161 /// If set, generation uses the corresponding cache from the registry
162 active_cache_owner: Arc<Mutex<Option<crate::runtime::kv_cache::CacheOwner>>>,
163 /// KV-cache compatibility descriptor for the loaded model (#1277). The
164 /// authoritative identity every reusable/durable KV lookup is checked
165 /// against; `None` until a model is loaded. Built from the model config +
166 /// runtime config at load time, finalized with the tokenizer identity.
167 kv_compat: Arc<Mutex<Option<crate::runtime::kv_compat::KvCompatDescriptor>>>,
168 // Note: XET/LFS handled by git-xet-filter + ModelFactory::load_file_with_pointer_detection()
169 // Note: Pre-training not supported (persistent_model doesn't expose VarStore), LoRA only
170}
171
172/// Tokenize `text` with `tokenizer`, writing no caller content to the process
173/// log.
174///
175/// Token-burn counts (prompt token count) are now emitted as OTel metrics at the
176/// call site (`TextStream::new_with_delta`), not as log lines (#1261) — the log
177/// is no longer the metering source. This helper logs only on the two
178/// content-free diagnostic outcomes: an empty token sequence (failure) and a
179/// tokenize/decode roundtrip mismatch (a real tokenizer-bug signal), in both
180/// cases carrying only bounded lengths, never the text.
181///
182/// #1253: the prompt and its decoded token text are tenant-private and must
183/// never reach process-wide logs — a shared inference process exposes them
184/// outside the caller's request boundary. This helper is split out from
185/// [`TorchEngine::tokenize`] so its no-leak behavior is unit-testable without
186/// a full engine/model.
187fn tokenize_redacted(tokenizer: &Tokenizer, text: &str) -> Result<Vec<i64>> {
188 let bytes = text.len();
189 let chars = text.chars().count();
190
191 let encoding = tokenizer
192 .encode(text, false)
193 .map_err(|e| anyhow!("Tokenization failed: {}", e))?;
194 let token_ids: Vec<i64> = encoding.get_ids().iter().map(|&id| id as i64).collect();
195
196 if token_ids.is_empty() {
197 tracing::warn!(bytes, chars, "Tokenization produced empty token sequence");
198 return Err(anyhow!(
199 "Tokenization produced empty token sequence ({} bytes, {} chars)",
200 bytes,
201 chars
202 ));
203 }
204
205 // Surface roundtrip mismatches for diagnosis, but never the original or
206 // decoded text — only that they diverged and the bounded lengths.
207 if let Ok(decoded) = tokenizer.decode(
208 &token_ids.iter().map(|&id| id as u32).collect::<Vec<_>>(),
209 false,
210 ) {
211 if decoded != text {
212 tracing::warn!(
213 bytes,
214 chars,
215 tokens = token_ids.len(),
216 decoded_bytes = decoded.len(),
217 "Tokenization roundtrip mismatch (content redacted)"
218 );
219 }
220 }
221
222 Ok(token_ids)
223}
224
225/// Helper functions for tensor operations
226impl TorchEngine {
227 /// Set the random seed for deterministic generation
228 /// Useful for debugging - enables reproducible token sequences
229 pub fn set_seed(&self, seed: u64) {
230 TensorSampler::set_seed(seed);
231 }
232
233 /// Get cached vocabulary size without locking
234 pub fn get_vocab_size(&self) -> usize {
235 let size = self.tokenizer_vocab_size.load(Ordering::Relaxed);
236 if size > 0 {
237 size
238 } else {
239 // Fallback: try to get from tokenizer (shouldn't happen after load_tokenizer)
240 self.tokenizer
241 .lock()
242 .as_ref()
243 .map(|t| t.get_vocab_size(true))
244 .unwrap_or(0) // Return 0 if tokenizer not loaded (caller should error)
245 }
246 }
247
248 /// Get the device this engine is using for computation
249 pub fn device(&self) -> Device {
250 self.device
251 }
252
253 /// The resolved multi-device pool, if this engine was built with >1 device.
254 ///
255 /// Returns `None` for single-device engines (the common case, including the
256 /// legacy single-`gpu_device_id` path and CPU). When `Some` and non-single,
257 /// the loaded model is built as a single pipeline stage with a layer→device
258 /// map derived from this pool (#314 wiring). The pool's primary equals
259 /// [`Self::device`].
260 #[must_use]
261 pub fn device_pool(&self) -> Option<&crate::runtime::DevicePool> {
262 self.device_pool.as_deref()
263 }
264
265 /// Whether this engine is configured to build a multi-device pipeline.
266 ///
267 /// True exactly when [`Self::device_pool`] is `Some` and holds >1 device.
268 /// The model is constructed with `LayerDeviceMap::even_split` across the
269 /// pool; single-device engines return `false` and build the whole model on
270 /// [`Self::device`].
271 #[must_use]
272 pub fn is_multi_device(&self) -> bool {
273 self.device_pool.as_deref().is_some_and(|p| !p.is_single())
274 }
275
276 // ============================================================================
277 // Session-Based KV Cache Management
278 // ============================================================================
279
280 /// Initialize the KV cache registry for session-based cache isolation.
281 ///
282 /// This should be called after loading a model when the configuration is known.
283 /// The registry will manage separate KV caches for different sessions/requests.
284 pub fn initialize_kv_registry(
285 &mut self,
286 num_layers: usize,
287 max_seq_len: usize,
288 quant_type: hyprstream_rpc_std::model_client::KVQuantType,
289 memory_budget: Option<usize>,
290 ) {
291 let config = crate::runtime::kv_cache::CacheConfig::new(num_layers, max_seq_len)
292 .with_quant_type(quant_type);
293
294 let registry = crate::runtime::kv_cache::KVCacheRegistry::new(config, memory_budget);
295 self.kv_cache_registry = Some(Arc::new(registry));
296
297 tracing::info!(
298 "[TorchEngine] Initialized KV cache registry: {} layers, max_seq_len={}, budget={:?}",
299 num_layers,
300 max_seq_len,
301 memory_budget
302 );
303 }
304
305 /// Expected KV-cache compatibility fingerprint for the loaded model under the
306 /// given adapter/delta generation, or `None` when no authoritative identity is
307 /// available.
308 ///
309 /// The effective identity folds in the base-weight content digest, the full
310 /// effective architecture/runtime config, and the live `adapter_generation`
311 /// actually applied to the request. Returns `None` when no descriptor was
312 /// built (the base-weight digest could not be computed) or when the descriptor
313 /// is not yet authoritatively complete (tokenizer identity not yet folded in,
314 /// or tokenizer hashing failed). The session-reuse path treats `None` as
315 /// **fail-closed**: it discards a cache rather than reusing it under a
316 /// missing or uncertain identity — it never treats absence as "no gating".
317 pub fn kv_compat_fingerprint_for(
318 &self,
319 adapter_generation: u64,
320 ) -> Option<crate::runtime::kv_compat::KvCompatFingerprint> {
321 let guard = self.kv_compat.lock();
322 let desc = guard.as_ref()?;
323 if !desc.is_authoritatively_complete() {
324 return None;
325 }
326 let mut effective = desc.clone();
327 effective.weights.adapter_generation = adapter_generation;
328 Some(crate::runtime::kv_compat::KvCompatFingerprint::of(
329 &effective,
330 ))
331 }
332
333 /// Compute a KV cache memory budget based on model size and available GPU memory.
334 ///
335 /// Uses `HYPRSTREAM_KV_BUDGET_MB` env var if set, otherwise estimates:
336 /// GPU total (from common card sizes) minus model weights, times 0.7.
337 /// Returns None on CPU (no budget needed).
338 pub fn compute_kv_budget(&self) -> Option<usize> {
339 if self.device == Device::Cpu {
340 return None;
341 }
342
343 // Check for explicit override
344 if let Ok(mb_str) = std::env::var("HYPRSTREAM_KV_BUDGET_MB") {
345 if let Ok(mb) = mb_str.parse::<usize>() {
346 tracing::info!("KV cache budget from HYPRSTREAM_KV_BUDGET_MB: {} MB", mb);
347 return Some(mb * 1024 * 1024);
348 }
349 }
350
351 let model_bytes = self.model_memory_usage();
352
353 // Estimate total GPU memory from common card sizes.
354 // Without cudaMemGetInfo FFI, we use a conservative heuristic:
355 // assume the GPU has enough memory to hold the model + some headroom.
356 // We estimate total VRAM as model_bytes * 1.75 (model is typically ~55-60% of VRAM).
357 // This is conservative — on larger GPUs it underestimates, leaving more headroom.
358 let estimated_total = model_bytes * 7 / 4; // 1.75x model size
359 let remaining = estimated_total.saturating_sub(model_bytes);
360 let budget = remaining * 7 / 10; // 70% of remaining
361
362 tracing::info!(
363 "KV cache budget: {} MB (estimated from {} MB model weights, ~{} MB total VRAM)",
364 budget / (1024 * 1024),
365 model_bytes / (1024 * 1024),
366 estimated_total / (1024 * 1024),
367 );
368
369 Some(budget)
370 }
371
372 /// Get the KV cache registry (for external access)
373 pub fn kv_registry(&self) -> Option<Arc<crate::runtime::kv_cache::KVCacheRegistry>> {
374 self.kv_cache_registry.clone()
375 }
376
377 /// Set the active session for generation.
378 ///
379 /// This sets which KV cache will be used for the next generation call.
380 /// Use `CacheOwner::Session(session_id)` for conversational sessions that
381 /// should preserve context across multiple requests.
382 ///
383 /// # Example
384 /// ```ignore
385 /// // For a conversational session (preserves context)
386 /// engine.set_session(CacheOwner::Session("conversation-123".into()));
387 ///
388 /// // For a stateless request (cache discarded after)
389 /// engine.set_session(CacheOwner::new_stateless());
390 /// ```
391 pub fn set_session(&self, owner: crate::runtime::kv_cache::CacheOwner) -> Result<()> {
392 if self.kv_cache_registry.is_none() {
393 return Err(anyhow!(
394 "KV cache registry not initialized. Call initialize_kv_registry first."
395 ));
396 }
397
398 let mut active_owner = self.active_cache_owner.lock();
399
400 tracing::debug!("Setting active session to: {:?}", owner);
401 *active_owner = Some(owner);
402
403 Ok(())
404 }
405
406 /// Get the current active session owner
407 pub fn get_session(&self) -> Option<crate::runtime::kv_cache::CacheOwner> {
408 self.active_cache_owner.lock().clone()
409 }
410
411 /// Clear the active session (subsequent generations will use the model's internal cache)
412 pub fn clear_session(&self) -> Result<()> {
413 let mut active_owner = self.active_cache_owner.lock();
414 *active_owner = None;
415 Ok(())
416 }
417
418 /// Release a session's KV cache (for cleanup after conversation ends)
419 pub fn release_session(&self, owner: &crate::runtime::kv_cache::CacheOwner) -> Result<()> {
420 if let Some(registry) = &self.kv_cache_registry {
421 registry.release(owner);
422 tracing::debug!("Released session cache: {:?}", owner);
423 }
424 Ok(())
425 }
426
427 /// Get total memory usage across all session caches
428 pub fn session_cache_memory_usage(&self) -> usize {
429 self.kv_cache_registry
430 .as_ref()
431 .map(|r| r.total_memory_usage())
432 .unwrap_or(0)
433 }
434
435 /// Get number of active session caches
436 pub fn active_session_count(&self) -> usize {
437 self.kv_cache_registry
438 .as_ref()
439 .map(|r| r.cache_count())
440 .unwrap_or(0)
441 }
442
443 /// Evict LRU session caches to stay within memory budget
444 pub fn evict_session_caches(&self) {
445 if let Some(registry) = &self.kv_cache_registry {
446 registry.evict_to_budget();
447 }
448 }
449
450 /// Swap the active session's KV cache into the model.
451 ///
452 /// If an active session owner is set and the registry is initialized,
453 /// retrieves (or creates) the session's cache from the registry and
454 /// installs it in the model via `set_kv_cache()`. Returns true if
455 /// a session cache was swapped in.
456 ///
457 /// `tenant_id` is the effective tenant whose delta contributed to this
458 /// request's weights (`None` when no tenant delta contributes), and
459 /// `weight_epoch` is that delta's current weight epoch (ignored when
460 /// `tenant_id` is `None`). The pair is registered against the owner cache
461 /// *before* the cache is installed in the model, so the cache is tracked
462 /// for invalidation (and cleared if its previously-computed KV is stale
463 /// w.r.t. a different tenant or an older weight epoch) before generation
464 /// can read a single token from it.
465 ///
466 /// **Atomicity invariant (F2):** the register→install sequence below is
467 /// *not* guarded by a lock spanning both operations. It is race-free today
468 /// because `InferenceService` runs on a single-threaded `current_thread`
469 /// Tokio runtime inside a `LocalSet`, and this function is await-free — no
470 /// other request task can interleave between `register_delta_dependency`
471 /// and `set_kv_cache`, so the pair executes atomically with respect to
472 /// other requests. If `InferenceService` ever moves to a multi-threaded
473 /// runtime (or an `.await` is introduced here), this invariant breaks and a
474 /// spanning lock or an epoch handshake would be required.
475 pub fn swap_session_cache(&self, tenant_id: Option<String>, weight_epoch: u64) -> bool {
476 let owner = match self.active_cache_owner.lock().clone() {
477 Some(o) => o,
478 None => return false,
479 };
480 let registry = match &self.kv_cache_registry {
481 Some(r) => r,
482 None => return false,
483 };
484 let model_arc = match &self.persistent_model {
485 Some(m) => m,
486 None => return false,
487 };
488
489 let session_cache = registry.get_or_create(owner.clone());
490 // Register the delta dependency BEFORE installing the cache. This both
491 // records the owner for tenant-delta invalidation and, if the effective
492 // tenant changed since this cache was last computed, clears the stale KV
493 // in place. Ordering matters: registration/staleness-clear precede
494 // set_kv_cache, so the model can never publish output from an untracked
495 // or weight-stale cache.
496 let cleared_stale = registry.register_delta_dependency(&owner, tenant_id, weight_epoch);
497 let mut model = model_arc.lock();
498 model.set_kv_cache(session_cache);
499 tracing::debug!(
500 "Swapped session KV cache into model (stale_cleared={})",
501 cleared_stale
502 );
503 true
504 }
505
506 /// Estimate model memory usage in bytes.
507 ///
508 /// Sums the byte sizes of all parameters in the VarStore.
509 pub fn model_memory_usage(&self) -> usize {
510 let vs_guard = self.var_store.lock();
511 {
512 if let Some(vs) = vs_guard.as_ref() {
513 return vs
514 .variables()
515 .values()
516 .map(|t| {
517 let numel = t.numel();
518 let elem_size = match t.kind() {
519 tch::Kind::Half | tch::Kind::BFloat16 => 2,
520 tch::Kind::Double => 8,
521 tch::Kind::Int8 | tch::Kind::Uint8 => 1,
522 _ => 4, // Float and others
523 };
524 numel * elem_size
525 })
526 .sum();
527 }
528 }
529 0
530 }
531
532 /// Create new PyTorch engine
533 pub fn new(config: RuntimeConfig) -> Result<Self> {
534 Self::new_sync(config)
535 }
536
537 /// Create new PyTorch engine (async version)
538 pub async fn new_async(config: RuntimeConfig) -> Result<Self> {
539 Self::new_sync(config)
540 }
541
542 /// Internal sync constructor
543 fn new_sync(config: RuntimeConfig) -> Result<Self> {
544 // Multi-GPU foundation (#313/#314, epic #310): when devices are
545 // *explicitly* requested via `RuntimeConfig.devices` /
546 // `HYPRSTREAM_GPU_DEVICES`, resolve them through `DevicePool`, which
547 // validates them fail-fast (no silent CPU downgrade). When the pool
548 // holds >1 device the engine keeps the whole pool and threads it into
549 // model construction so the model is built as a single multi-device
550 // pipeline stage (layers spread via `LayerDeviceMap::even_split`,
551 // `forward_layers` performs the stage-boundary copies). Single-pool and
552 // legacy single-`gpu_device_id` paths are unchanged.
553 if config.use_gpu {
554 if let Some(indices) = config.resolve_explicit_multi_device_indices()? {
555 let pool = crate::runtime::DevicePool::from_cuda_indices(&indices)?;
556 let device = pool.primary();
557 if pool.is_single() {
558 info!(
559 "🚀 DevicePool resolved a single device {:?}; building single-device engine",
560 device
561 );
562 return Self::with_device(config, device, None);
563 }
564 info!(
565 "🚀 Multi-GPU DevicePool resolved {} device(s) {:?}; building a multi-device \
566 pipeline (primary {:?} hosts embeddings/lm_head)",
567 pool.len(),
568 pool.devices(),
569 device
570 );
571 return Self::with_device(config, device, Some(Arc::new(pool)));
572 }
573 }
574
575 // Determine device based on configuration
576 let device = if config.use_gpu {
577 // Use specified GPU device ID, or auto-detect
578 let gpu_device = if let Some(device_id) = config.gpu_device_id {
579 // A GPU was *explicitly* requested. Check if CUDA/ROCm is available.
580 if Device::cuda_if_available() != Device::Cpu {
581 Device::Cuda(device_id)
582 } else if config.strict_device {
583 // No-fragile-fallbacks (#315): a process told to use GPU N that
584 // silently lands on CPU tanks the pipeline. Fail fast instead,
585 // and surface the *actual* reason CUDA was unavailable so the
586 // operator can fix it (driver lib / wrong backend / mismatch).
587 let diag = cuda_unavailable_diagnostics();
588 return Err(anyhow!(
589 "GPU {device_id} explicitly requested but CUDA/ROCm is not available; \
590 refusing to silently fall back to CPU (strict_device). \
591 Set HYPRSTREAM_STRICT_DEVICE=0 to allow CPU fallback. \
592 Diagnostics: {diag}"
593 ));
594 } else {
595 warn!(
596 "⚠️ GPU {} requested but CUDA/ROCm not available, falling back to CPU. {}",
597 device_id,
598 cuda_unavailable_diagnostics()
599 );
600 Device::Cpu
601 }
602 } else {
603 // Pure auto-detect ("use a GPU if there is one"): CPU fallback is
604 // the intended behavior here even under strict_device, since no
605 // specific device was requested.
606 Device::cuda_if_available()
607 };
608
609 if gpu_device != Device::Cpu {
610 // Check if this is actually ROCm/HIP
611 let device_id = match gpu_device {
612 Device::Cuda(id) => id,
613 _ => 0,
614 };
615 if tch::utils::has_hip() {
616 info!("🚀 Using ROCm/HIP GPU {} acceleration", device_id);
617 } else {
618 info!("🚀 Using CUDA GPU {} acceleration", device_id);
619 }
620 gpu_device
621 } else {
622 // The live RTX 5090 / cuda130 node hit exactly this branch: a
623 // bare "not available" with no cause. Always surface *why* now
624 // (most commonly libcuda.so.1 missing from LD_LIBRARY_PATH).
625 warn!(
626 "⚠️ GPU requested but not available, falling back to CPU. {}",
627 cuda_unavailable_diagnostics()
628 );
629 Device::Cpu
630 }
631 } else {
632 info!("💻 Using CPU inference");
633 Device::Cpu
634 };
635
636 Self::with_device(config, device, None)
637 }
638
639 /// Construct an engine pinned to an already-resolved primary [`Device`].
640 ///
641 /// Pass `device_pool = Some(pool)` with a multi-device pool to build the
642 /// model as a single pipeline stage spanning the whole decoder stack with a
643 /// layer→device map (the #314 multi-device path). The engine's `device`
644 /// remains the pool's primary (first) device — the home of embeddings, final
645 /// norm, and lm_head, and where inputs land before the first layer.
646 /// `device_pool = None` (or a single-device pool) preserves the original
647 /// whole-model-on-one-device behavior.
648 fn with_device(
649 config: RuntimeConfig,
650 device: Device,
651 device_pool: Option<Arc<crate::runtime::DevicePool>>,
652 ) -> Result<Self> {
653 Ok(Self {
654 var_store: Arc::new(Mutex::new(None)),
655 model_architecture: Arc::new(Mutex::new(None)),
656 persistent_model: None,
657 context_state: Arc::new(Mutex::new(None)),
658 tokenizer: Arc::new(Mutex::new(None)),
659 template_engine: Arc::new(Mutex::new(None)),
660 device,
661 device_pool,
662 config: config.clone(),
663 generation_config: GenerationConfig {
664 max_tokens: 2048,
665 temperature: 0.7,
666 top_p: 0.9,
667 top_k: Some(40),
668 repeat_penalty: 1.1,
669 stop_tokens: vec!["</s>".to_owned()],
670 seed: None,
671 stream: false,
672 },
673 model_info: Arc::new(Mutex::new(ModelInfo {
674 name: "unloaded".to_owned(),
675 architecture: "unknown".to_owned(),
676 parameters: Some(0),
677 context_length: 2048,
678 vocab_size: 32000,
679 hidden_size: 768,
680 ..Default::default()
681 })),
682 active_lora: Arc::new(Mutex::new(None)),
683 sampler: TensorSampler::new(device),
684 tokenizer_vocab_size: Arc::new(AtomicUsize::new(0)),
685 eos_token_id: Arc::new(AtomicU32::new(0)),
686 kv_cache_registry: None, // Initialized after model load when config is known
687 active_cache_owner: Arc::new(Mutex::new(None)),
688 kv_compat: Arc::new(Mutex::new(None)), // Built from model config at load time (#1277)
689 })
690 }
691
692 /// Load model from safetensors or torchscript
693 async fn load_model_file(&mut self, path: &Path) -> Result<()> {
694 let ext = path
695 .extension()
696 .and_then(|e| e.to_str())
697 .ok_or_else(|| anyhow!("Invalid file extension"))?;
698
699 match ext {
700 "pt" | "pth" => {
701 return Err(anyhow!("TorchScript models (.pt/.pth) are no longer supported. Please use SafeTensors format (.safetensors) instead."));
702 }
703 "safetensors" => {
704 // Load SafeTensors model directly
705 info!("Loading weights: {}", path.display());
706 self.load_safetensors(path).await?;
707 }
708 _ => {
709 return Err(anyhow!("Unsupported model format: {}", ext));
710 }
711 }
712
713 Ok(())
714 }
715
716 /// Load SafeTensors model using ModelFactory for unified weight loading
717 async fn load_safetensors(&mut self, path: &Path) -> Result<()> {
718 info!("Loading weights: {}", path.display());
719
720 // Get the model directory (parent of the safetensors file)
721 let model_dir = path.parent().unwrap_or(path);
722
723 // Use ModelFactory to load the model (handles all weight loading internally)
724 self.initialize_persistent_model(model_dir).await?;
725
726 // Extract model info from the loaded model
727 if let Some(model) = &self.persistent_model {
728 let _model_guard = model.lock();
729
730 // Get architecture info from the model
731 // For now, use a default since we don't have a method to query this
732 let architecture = "auto".to_owned();
733
734 // Update model info
735 {
736 let mut model_info_guard = self.model_info.lock();
737 model_info_guard.architecture = architecture.clone();
738 }
739
740 // Set architecture
741 {
742 let mut arch_guard = self.model_architecture.lock();
743 *arch_guard = Some(architecture.clone());
744 }
745 }
746
747 // Get context window from model info that was populated from config
748 let context_window = self.effective_context_limit()?;
749
750 // Initialize context state
751 {
752 let mut context_guard = self.context_state.lock();
753 *context_guard = Some(ContextState {
754 sequence_length: 0,
755 context_window,
756 initialized: true,
757 });
758 }
759
760 // Create dummy VarStore for backward compatibility
761 {
762 let vs = VarStore::new(self.device);
763 let mut var_store_guard = self.var_store.lock();
764 *var_store_guard = Some(vs);
765 }
766
767 info!("✅ SafeTensors model loaded via ModelFactory");
768 info!("🚀 Model initialized and ready for inference");
769 Ok(())
770 }
771
772 /// Get tensor from VarStore by name (for inference) - thread safe
773 pub fn get_tensor(&self, name: &str) -> Option<Tensor> {
774 let var_store_guard = self.var_store.lock();
775 let vs = var_store_guard.as_ref()?;
776 vs.variables().get(name).map(tch::Tensor::shallow_clone)
777 }
778
779 /// List all available tensor names in VarStore - thread safe
780 pub fn list_tensor_names(&self) -> Vec<String> {
781 let var_store_guard = self.var_store.lock();
782 if let Some(vs) = var_store_guard.as_ref() {
783 vs.variables().keys().cloned().collect()
784 } else {
785 Vec::new()
786 }
787 }
788
789 /// Check if model is loaded via VarStore - thread safe
790 pub fn has_varstore(&self) -> bool {
791 self.var_store.lock().is_some()
792 }
793
794 /// Authoritative content digest of the base weights actually loaded from
795 /// `model_path` — a canonical SHA-256 over the **loader-selected**
796 /// `.safetensors` shards' **resolved** bytes (shard count + length-prefixed
797 /// name + per-shard content digest), in sorted name order. This is the
798 /// base-weight identity for the KV-compat fingerprint (#1277): two snapshots
799 /// sharing a basename but differing in content diverge here, unlike a
800 /// path/label.
801 ///
802 /// The shard set comes from the **same selector the loader uses**
803 /// (`ModelFactory::find_shard_files_for_digest`, the digest-authoritative
804 /// variant of the loader's selector: `model.safetensors.index.json` →
805 /// `model.safetensors` → shard-name glob), so the digest covers EXACTLY the
806 /// shards the loader reads — an extra/unreferenced `.safetensors` in the
807 /// directory (stale shard, alt-precision copy) cannot shift the fingerprint
808 /// while the loaded weights are identical. Unlike the loader, the digest
809 /// REJECTS an ambiguous manifest-less multi-shard glob set regardless of
810 /// `HYPRSTREAM_STRICT_LOADER` — fingerprint authority is stricter than
811 /// loader permissiveness.
812 ///
813 /// The digest hashes the **resolved** bytes — the identical resolution path
814 /// the loader walks (`ModelFactory::resolve_weight_for_digest` transparently
815 /// resolves an un-smudged LFS/XET pointer to its object content), so it never
816 /// hashes a pointer stub in place of the real tensors. It is **fail-closed**:
817 /// any selection/read/resolve error, an empty selection, an
818 /// index-referenced shard that is missing, or an ambiguous manifest-less
819 /// multi-shard set propagates, so the caller declines to mint a descriptor
820 /// and KV reuse fails closed rather than hashing a partial shard set.
821 async fn weights_content_digest(model_path: &Path) -> Result<String> {
822 use crate::runtime::kv_compat;
823 use crate::runtime::model_factory::{ModelFactory, ResolvedWeight};
824
825 // Resolve the loader-selected shard set (the digest-authoritative
826 // variant of the loader's own selector).
827 let shards = ModelFactory::find_shard_files_for_digest(model_path)?;
828
829 // Fail closed on an empty or inconsistent selection: the loader would
830 // have nothing authoritative to load (or the index disagrees with the
831 // directory), so reuse must be declined rather than minted over a
832 // partial/ambiguous set.
833 if shards.is_empty() {
834 anyhow::bail!(
835 "no loader-selected .safetensors weight shards in {}",
836 model_path.display()
837 );
838 }
839 for shard in &shards {
840 if !shard.is_file() {
841 anyhow::bail!(
842 "loader-selected shard {} is missing or not a regular file",
843 shard.display()
844 );
845 }
846 }
847
848 let mut digested: Vec<(String, [u8; 32])> = Vec::with_capacity(shards.len());
849 for shard in &shards {
850 let name = shard
851 .file_name()
852 .and_then(|s| s.to_str())
853 .unwrap_or("")
854 .to_owned();
855 // Resolve transparently (LFS/XET pointer → object content), then
856 // hash the resolved bytes. Large ordinary shards stream; resolved
857 // pointers and small files hash their owned bytes.
858 let shard_digest = match ModelFactory::resolve_weight_for_digest(shard).await? {
859 ResolvedWeight::Owned(bytes) => kv_compat::shard_content_digest(&bytes),
860 ResolvedWeight::File(file) => Self::stream_file_digest(file)?,
861 };
862 digested.push((name, shard_digest));
863 }
864 Ok(kv_compat::base_weight_digest_from_shards(&digested))
865 }
866
867 /// SHA-256 of a file's bytes, streamed in 64 KiB chunks so a multi-GB shard
868 /// is never fully buffered into memory just to hash it. Returns the resolved
869 /// read error (fail-closed) rather than a partial digest.
870 fn stream_file_digest(file: std::fs::File) -> std::result::Result<[u8; 32], std::io::Error> {
871 use sha2::Digest;
872 use std::io::Read;
873 let mut hasher = sha2::Sha256::new();
874 let mut f = file;
875 let mut buf = [0u8; 65536];
876 loop {
877 let n = f.read(&mut buf)?;
878 if n == 0 {
879 break;
880 }
881 hasher.update(&buf[..n]);
882 }
883 let mut out = [0u8; 32];
884 out.copy_from_slice(hasher.finalize().as_slice());
885 Ok(out)
886 }
887
888 /// Initialize XET storage with default configuration
889 /// Initialize the persistent model instance using ModelFactory
890 async fn initialize_persistent_model(&mut self, model_path: &Path) -> Result<()> {
891 use crate::runtime::model_config::ModelConfig;
892 use crate::runtime::model_factory::ModelFactory;
893 use crate::runtime::torch_utils::preflight_gpu_check;
894
895 info!("Initializing model");
896
897 // XET/LFS handled automatically by git-xet-filter + ModelFactory fallback
898 // Load model config first to get model parameters
899 let empty_weights = HashMap::new();
900 let config = ModelConfig::load(model_path, &empty_weights)?;
901
902 // Effective context length (CLI override or model default)
903 let effective_max_context = self
904 .config
905 .max_context
906 .map(|v| v as usize)
907 .unwrap_or(config.max_position_embeddings);
908 if self.config.max_context.is_some() {
909 info!(
910 "Using max_context override: {} tokens (model default: {})",
911 effective_max_context, config.max_position_embeddings
912 );
913 }
914
915 // Estimate model memory requirements
916 let estimated_weights_mb = {
917 // Rough estimate: vocab_size * hidden_size (embeddings)
918 // + num_layers * hidden_size * intermediate_size * 3 (MLP)
919 // + num_layers * hidden_size * hidden_size * 4 (attention)
920 let embedding_params = config.vocab_size * config.hidden_size;
921 let mlp_params_per_layer = config.hidden_size * config.intermediate_size * 3;
922 let attn_params_per_layer = config.hidden_size * config.hidden_size * 4;
923 let params_per_layer = mlp_params_per_layer + attn_params_per_layer;
924 let total_params = embedding_params + (config.num_hidden_layers * params_per_layer);
925
926 // BF16 = 2 bytes per parameter
927 (total_params * 2) as f64 / (1024.0 * 1024.0)
928 };
929
930 let kv_cache_mb = {
931 // KV cache: 2 (keys+values) * num_layers * batch_size * max_seq_len * num_heads * head_dim * 2 (BF16)
932 let batch_size = 1;
933 let kv_per_layer = 2
934 * batch_size
935 * effective_max_context
936 * config.num_attention_heads
937 * config.head_dim
938 * 2;
939 let total_kv = config.num_hidden_layers * kv_per_layer;
940 total_kv as f64 / (1024.0 * 1024.0)
941 };
942
943 let total_estimated_mb = estimated_weights_mb + kv_cache_mb;
944
945 info!(
946 "Model memory estimate:\n\
947 - Weights: {:.2} MB\n\
948 - KV cache: {:.2} MB (max_seq_len={})\n\
949 - Total: {:.2} MB",
950 estimated_weights_mb, kv_cache_mb, effective_max_context, total_estimated_mb
951 );
952
953 // Pre-flight GPU memory check (best-effort)
954 if let Err(e) = preflight_gpu_check(self.device, total_estimated_mb) {
955 warn!("GPU memory pre-flight check failed: {}", e);
956 // Continue anyway - the check might not be accurate
957 }
958
959 // Update ModelInfo with actual values from config
960 {
961 let mut model_info = self.model_info.lock();
962 model_info.hidden_size = config.hidden_size as u32;
963 model_info.intermediate_size = Some(config.intermediate_size as u32);
964 model_info.num_attention_heads = Some(config.num_attention_heads as u32);
965 model_info.num_key_value_heads = Some(config.num_key_value_heads as u32);
966 model_info.head_dim = Some(config.head_dim as u32);
967 model_info.num_hidden_layers = Some(config.num_hidden_layers as u32);
968 model_info.vocab_size = config.vocab_size as u32;
969 model_info.context_length = effective_max_context as u32;
970 model_info.architecture = config.model_type.clone();
971 }
972
973 // Build the KV-cache compatibility descriptor (#1277): the authoritative
974 // identity of the model + runtime config that produces this engine's KV.
975 // Every reusable/durable KV lookup is checked against its fingerprint.
976 // Built here (where the full `ModelConfig` is in scope) from authoritative
977 // inputs only — never a human label. The base-weight identity is a content
978 // digest of the resolved shards; the tokenizer identity is folded in later
979 // by `load_tokenizer`, before any generation reads the fingerprint. If no
980 // authoritative base digest can be computed, the descriptor is left unset
981 // and the cache boundary fails closed (KV recomputed, not reused).
982 match Self::weights_content_digest(model_path).await {
983 Ok(base_revision) => {
984 use crate::runtime::kv_compat::{
985 KvCompatDescriptor, KvDtype, KvQuantMode, MoeRouting, RopeScalingFp,
986 WeightIdentity, KV_COMPAT_FORMAT_VERSION,
987 };
988 let kv_quant = match self.config.kv_quant_type {
989 hyprstream_rpc_std::model_client::KVQuantType::None => KvQuantMode::None,
990 hyprstream_rpc_std::model_client::KVQuantType::Int8 => KvQuantMode::Int8,
991 hyprstream_rpc_std::model_client::KVQuantType::Nf4 => KvQuantMode::Nf4,
992 hyprstream_rpc_std::model_client::KVQuantType::Fp4 => KvQuantMode::Fp4,
993 };
994 // `model_name` is a display label for mismatch messages; the
995 // authoritative weight identity is the content digest above.
996 let model_name = model_path
997 .file_stem()
998 .and_then(|s| s.to_str())
999 .unwrap_or("")
1000 .to_owned();
1001 let descriptor = KvCompatDescriptor {
1002 format_version: KV_COMPAT_FORMAT_VERSION,
1003 weights: WeightIdentity::base_only(base_revision),
1004 model_name,
1005 architecture: config.model_type.clone(),
1006 model_version: config.version,
1007 num_hidden_layers: config.num_hidden_layers,
1008 num_attention_heads: config.num_attention_heads,
1009 num_key_value_heads: config.num_key_value_heads,
1010 head_dim: config.head_dim,
1011 hidden_size: config.hidden_size,
1012 // Effective attention behavior — every field below can change
1013 // KV *values* without changing coarse geometry (#1277 review).
1014 use_qk_norm: config.use_qk_norm,
1015 partial_rotary_factor_bits: config.partial_rotary_factor.map(f32::to_bits),
1016 layer_types: config.layer_types.clone(),
1017 linear_key_head_dim: config.linear_key_head_dim,
1018 linear_value_head_dim: config.linear_value_head_dim,
1019 linear_num_key_heads: config.linear_num_key_heads,
1020 linear_num_value_heads: config.linear_num_value_heads,
1021 linear_conv_kernel_dim: config.linear_conv_kernel_dim,
1022 // Effective model config — each demonstrably changes hidden
1023 // states / downstream K/V without changing coarse geometry
1024 // (#1277 review finding F3): normalization epsilon, activation,
1025 // embedding/attention scaling, MoE routing, and local-RoPE /
1026 // sliding-window. Taken verbatim from the resolved ModelConfig
1027 // that constructs the model.
1028 rms_norm_eps_bits: config.rms_norm_eps.to_bits(),
1029 layer_norm_eps_bits: config.layer_norm_eps.map(f32::to_bits),
1030 hidden_activation: config.hidden_activation.clone(),
1031 scale_embeddings: config.scale_embeddings,
1032 query_pre_attn_scalar_bits: config.query_pre_attn_scalar.map(f32::to_bits),
1033 moe: MoeRouting {
1034 is_moe: config.is_moe,
1035 num_experts: config.num_experts,
1036 num_experts_per_tok: config.num_experts_per_tok,
1037 moe_intermediate_size: config.moe_intermediate_size,
1038 shared_expert_intermediate_size: config.shared_expert_intermediate_size,
1039 },
1040 sliding_window: config.sliding_window,
1041 rope_local_base_freq_bits: config.rope_local_base_freq.map(f32::to_bits),
1042 rope_theta_bits: config.rope_theta.to_bits(),
1043 rope_scaling: config
1044 .rope_scaling
1045 .as_ref()
1046 .map(|rs| RopeScalingFp::new(&rs.rope_type, rs.factor)),
1047 max_position_embeddings: config.max_position_embeddings,
1048 dtype: KvDtype::from_name(&config.dtype),
1049 kv_quant,
1050 block_size: crate::runtime::kv_cache::BLOCK_SIZE,
1051 max_context: effective_max_context,
1052 vocab_size: config.vocab_size,
1053 // Folded in by `load_tokenizer`; descriptor is non-authoritative
1054 // (fingerprint = None) until then, so reuse fails closed.
1055 tokenizer_hash: None,
1056 };
1057 *self.kv_compat.lock() = Some(descriptor);
1058 }
1059 Err(e) => {
1060 warn!(
1061 "Cannot compute authoritative base-weight digest for {}: {} \
1062 — KV-cache reuse disabled (fail-closed)",
1063 model_path.display(),
1064 e
1065 );
1066 // Leave kv_compat unset (None): the reuse boundary will discard
1067 // rather than adopt any cache, since there is no authoritative
1068 // identity to compare against.
1069 *self.kv_compat.lock() = None;
1070 }
1071 }
1072
1073 // Use the factory to create the model. When `device_pool` is set and
1074 // multi-device, the factory builds the model as a single pipeline stage
1075 // spanning all decoder layers with a layer→device map (#314 wiring).
1076 let factory_start = std::time::Instant::now();
1077 let model = ModelFactory::create(
1078 model_path,
1079 &self.device,
1080 resolve_inference_compute_kind(
1081 self.config.precision_mode.as_deref(),
1082 self.device,
1083 super::channel_fp8::ChannelFp8Profile::read(model_path)?.is_some(),
1084 )?,
1085 self.config.max_context.map(|v| v as usize),
1086 self.config.kv_quant_type,
1087 self.device_pool.as_deref(),
1088 self.config.fp8_dequant_load,
1089 self.config.speculative_decoding,
1090 )
1091 .await?;
1092 let factory_time = factory_start.elapsed();
1093 info!(
1094 "✅ Model weights loaded in {:.2}s",
1095 factory_time.as_secs_f64()
1096 );
1097
1098 self.persistent_model = Some(Arc::new(Mutex::new(model)));
1099
1100 // Load generation_config.json if present (model-specific sampling defaults)
1101 let gen_config_path = model_path.join("generation_config.json");
1102 if gen_config_path.exists() {
1103 match std::fs::read_to_string(&gen_config_path) {
1104 Ok(contents) => {
1105 if let Ok(json) = serde_json::from_str::<serde_json::Value>(&contents) {
1106 if let Some(t) = json["temperature"].as_f64() {
1107 self.generation_config.temperature = t as f32;
1108 }
1109 if let Some(p) = json["top_p"].as_f64() {
1110 self.generation_config.top_p = p as f32;
1111 }
1112 if let Some(k) = json["top_k"].as_u64() {
1113 self.generation_config.top_k = Some(k as usize);
1114 }
1115 if let Some(rp) = json["repetition_penalty"].as_f64() {
1116 self.generation_config.repeat_penalty = rp as f32;
1117 }
1118 if let Some(mt) = json["max_new_tokens"].as_u64() {
1119 self.generation_config.max_tokens = mt as usize;
1120 } else if let Some(ml) = json["max_length"].as_u64() {
1121 self.generation_config.max_tokens = ml as usize;
1122 }
1123 }
1124 }
1125 Err(e) => warn!("Failed to read generation_config.json: {}", e),
1126 }
1127 }
1128
1129 if !gen_config_path.exists() && config.model_type.contains("qwen3_5") {
1130 // Qwen3.5 recommended defaults (non-thinking mode) when no
1131 // generation_config.json is present. Qwen3 models ship their own
1132 // generation_config.json so this only applies to Qwen3.5.
1133 self.generation_config.temperature = 0.7;
1134 self.generation_config.top_p = 0.8;
1135 self.generation_config.top_k = Some(20);
1136 self.generation_config.repeat_penalty = 1.5;
1137 }
1138
1139 info!("Generation config: temperature={}, top_p={}, top_k={:?}, repeat_penalty={} (model_type={})",
1140 self.generation_config.temperature, self.generation_config.top_p,
1141 self.generation_config.top_k, self.generation_config.repeat_penalty,
1142 config.model_type);
1143
1144 Ok(())
1145 }
1146
1147 /// Load tokenizer and template configuration - thread safe
1148 #[instrument(name = "torch_engine.load_tokenizer", skip(self), fields(model_path = %model_path.display()))]
1149 async fn load_tokenizer(&mut self, model_path: &Path) -> Result<()> {
1150 // Try to find tokenizer.json - if model_path is a directory, look inside it
1151 // If it's a file, look in the parent directory
1152 let search_dir = if model_path.is_dir() {
1153 model_path
1154 } else {
1155 model_path.parent().unwrap_or(model_path)
1156 };
1157
1158 let tokenizer_path = search_dir.join("tokenizer.json");
1159 let tokenizer_config_path = search_dir.join("tokenizer_config.json");
1160
1161 if tokenizer_path.exists() {
1162 info!("Loading tokenizer: {}", tokenizer_path.display());
1163 let mut tokenizer = Tokenizer::from_file(&tokenizer_path)
1164 .map_err(|e| anyhow!("Failed to load tokenizer: {}", e))?;
1165
1166 // Log initial vocabulary size
1167 let initial_vocab_size = tokenizer.get_vocab_size(true);
1168 info!("Initial tokenizer vocabulary size: {}", initial_vocab_size);
1169
1170 // Get model's configured vocab size
1171 let model_vocab_size = {
1172 let model_info = self.model_info.lock();
1173 model_info.vocab_size as usize
1174 };
1175
1176 // Apply model-specific tokenizer configuration if model is loaded
1177 if let Some(model) = &self.persistent_model {
1178 let model_guard = model.lock();
1179 let tokenizer_config = model_guard.get_tokenizer_config();
1180
1181 // Configure the tokenizer based on the model architecture
1182 tokenizer_config.configure_tokenizer(&mut tokenizer, model_vocab_size)?;
1183
1184 tracing::debug!("Applied model-specific tokenizer configuration");
1185 } else {
1186 // Model not loaded yet - log vocab mismatch if any
1187 if model_vocab_size != initial_vocab_size {
1188 tracing::debug!(
1189 "Vocabulary size mismatch detected (model: {}, tokenizer: {}), but model not loaded yet to apply configuration",
1190 model_vocab_size, initial_vocab_size
1191 );
1192 }
1193 }
1194
1195 // Cache the final vocabulary size for lock-free access
1196 let final_vocab_size = tokenizer.get_vocab_size(true);
1197 self.tokenizer_vocab_size
1198 .store(final_vocab_size, Ordering::Relaxed);
1199
1200 info!("Final tokenizer vocabulary size: {}", final_vocab_size);
1201
1202 // Fold the tokenizer identity into the KV-compat descriptor (#1277).
1203 // Cached token ids are only meaningful relative to one tokenizer, so
1204 // a different tokenizer (even with the same vocab size) must reject.
1205 // The on-disk `tokenizer.json` bytes are the authoritative identity;
1206 // deterministic model-driven mutation (e.g. Qwen vocab padding) is
1207 // already captured by `vocab_size`. FAIL-CLOSED: if the bytes cannot
1208 // be hashed, the tokenizer identity is left unset, the descriptor is
1209 // non-authoritative (fingerprint = None), and the cache boundary
1210 // discards rather than adopts any cache — we never silently degrade
1211 // to a weaker vocab-only identity.
1212 {
1213 let mut guard = self.kv_compat.lock();
1214 let tokenizer_hash = match std::fs::read(&tokenizer_path) {
1215 Ok(bytes) => {
1216 use sha2::{Digest, Sha256};
1217 let mut hasher = Sha256::new();
1218 hasher.update(&bytes);
1219 Some(hex::encode(hasher.finalize()))
1220 }
1221 Err(e) => {
1222 tracing::error!(
1223 "Could not read {} for KV-compat tokenizer identity ({}): \
1224 KV-cache reuse disabled (fail-closed)",
1225 tokenizer_path.display(),
1226 e
1227 );
1228 None
1229 }
1230 };
1231 if let Some(desc) = guard.as_mut() {
1232 desc.set_tokenizer(final_vocab_size, tokenizer_hash);
1233 }
1234 }
1235
1236 // Thread safe assignment
1237 let mut tokenizer_guard = self.tokenizer.lock();
1238 *tokenizer_guard = Some(tokenizer);
1239 } else {
1240 return Err(anyhow!(
1241 "Tokenizer not found at {}. A proper tokenizer.json file is required for inference.",
1242 tokenizer_path.display()
1243 ));
1244 }
1245
1246 // Load template configuration if available
1247 if tokenizer_config_path.exists() {
1248 info!(
1249 "Loading tokenizer config: {}",
1250 tokenizer_config_path.display()
1251 );
1252 let config_content = tokio::fs::read_to_string(&tokenizer_config_path).await?;
1253
1254 // Validate before parsing
1255 jtp::from_str(&config_content)
1256 .with_max_depth(10)
1257 .with_max_string_length(50000)
1258 .validate()
1259 .map_err(|e| anyhow!("Invalid tokenizer config: {:?}", e))?;
1260
1261 let config_json: serde_json::Value = serde_json::from_str(&config_content)?;
1262
1263 // Parse template configuration
1264 let template_config = TemplateEngine::from_tokenizer_config(&config_json)?;
1265
1266 // Cache EOS token ID for lock-free access during generation
1267 if let Some(ref eos_str) = template_config.eos_token {
1268 let tokenizer_guard = self.tokenizer.lock();
1269 if let Some(ref tokenizer) = *tokenizer_guard {
1270 if let Some(eos_id) = tokenizer.token_to_id(eos_str) {
1271 self.eos_token_id.store(eos_id, Ordering::Relaxed);
1272 info!("Cached EOS token: '{}' -> ID {}", eos_str, eos_id);
1273 } else {
1274 warn!("EOS token '{}' not found in tokenizer vocabulary", eos_str);
1275 }
1276 }
1277 }
1278
1279 // Create template engine
1280 let template_engine = TemplateEngine::new(template_config)?;
1281
1282 // Store template engine
1283 let mut template_guard = self.template_engine.lock();
1284 *template_guard = Some(template_engine);
1285
1286 info!("✅ Template engine initialized");
1287 } else {
1288 info!("⚠️ No tokenizer_config.json found, using fallback templates");
1289 }
1290
1291 Ok(())
1292 }
1293
1294 /// Tokenize text to input IDs - thread safe
1295 ///
1296 /// Only bounded, content-redacted metadata (byte/char length, token count)
1297 /// is written to the process log. The prompt itself and its token text are
1298 /// tenant-private and never logged (#1253).
1299 fn tokenize(&self, text: &str) -> Result<Vec<i64>> {
1300 let tokenizer_guard = self.tokenizer.lock();
1301 let tokenizer = tokenizer_guard
1302 .as_ref()
1303 .ok_or_else(|| anyhow!("Tokenizer not loaded. Call load_tokenizer() first."))?;
1304
1305 tokenize_redacted(tokenizer, text)
1306 }
1307
1308 /// Format text with dynamic chat template
1309 fn format_chat_message(&self, system: Option<&str>, user: &str) -> String {
1310 // Try to use template engine if available
1311 {
1312 let template_guard = self.template_engine.lock();
1313 if let Some(ref engine) = *template_guard {
1314 let mut messages = Vec::new();
1315
1316 // Add system message if provided
1317 if let Some(system_msg) = system {
1318 messages.push(ChatMessage {
1319 role: "system".into(),
1320 content: Some(system_msg.into()),
1321 ..Default::default()
1322 });
1323 }
1324
1325 // Add user message
1326 messages.push(ChatMessage {
1327 role: "user".into(),
1328 content: Some(user.into()),
1329 ..Default::default()
1330 });
1331
1332 // Apply template (no tools in this code path)
1333 if let Ok(formatted) = engine.apply_chat_template(&messages, Some(true), None) {
1334 return formatted;
1335 }
1336 }
1337 }
1338
1339 // Generic fallback template (model-agnostic)
1340 let mut formatted = String::new();
1341
1342 // Add system message if provided
1343 if let Some(system_msg) = system {
1344 formatted.push_str("System: ");
1345 formatted.push_str(system_msg);
1346 formatted.push_str("\n\n");
1347 }
1348
1349 // Add user message
1350 formatted.push_str("User: ");
1351 formatted.push_str(user);
1352 formatted.push_str("\n\nAssistant: ");
1353
1354 formatted
1355 }
1356
1357 /// Check if a token ID is the EOS token (lock-free using cached ID)
1358 pub fn is_eos_token(&self, token_id: usize) -> bool {
1359 let cached_eos = self.eos_token_id.load(Ordering::Relaxed);
1360 // cached_eos == 0 means not initialized; actual EOS tokens are > 0
1361 cached_eos > 0 && token_id as u32 == cached_eos
1362 }
1363
1364 /// Get the tokenizer for streaming decoding - CoW makes this cheap!
1365 ///
1366 /// Returns a cloned tokenizer (cheap due to copy-on-write)
1367 pub fn get_tokenizer(&self) -> Result<Tokenizer> {
1368 let tokenizer_guard = self.tokenizer.lock();
1369 tokenizer_guard
1370 .as_ref()
1371 .cloned() // Cheap clone due to CoW
1372 .ok_or_else(|| anyhow!("Tokenizer not loaded. Call load_tokenizer() first."))
1373 }
1374
1375 /// Run inference on the model (supports both TorchScript and VarStore models) - thread safe
1376 pub fn forward(&self, input_ids: &[i64]) -> Result<Tensor> {
1377 // Try VarStore-based inference first (preferred for SafeTensors models)
1378 if self.has_varstore() {
1379 return self.forward_varstore(input_ids);
1380 }
1381
1382 Err(anyhow!("No model loaded - call load_model() first"))
1383 }
1384
1385 /// Run inference using VarStore (SafeTensors models) with persistent model - thread safe
1386 fn forward_varstore(&self, input_ids: &[i64]) -> Result<Tensor> {
1387 // Use the persistent model instance - NO recreation!
1388 let model_arc = self
1389 .persistent_model
1390 .as_ref()
1391 .ok_or_else(|| anyhow!("Persistent model not initialized - call load_model() first"))?;
1392
1393 // Verify context state with thread safety
1394 {
1395 let context_guard = self.context_state.lock();
1396 let _context_state = context_guard
1397 .as_ref()
1398 .ok_or_else(|| anyhow!("Context state not initialized"))?;
1399 }
1400
1401 if !self.is_persistent_model_ready() {
1402 return Err(anyhow!("Model not properly initialized"));
1403 }
1404
1405 // Convert input IDs to tensor (keep as int64 for embeddings)
1406 let input_tensor = Tensor::from_slice(input_ids)
1407 .to_kind(tch::Kind::Int64) // Ensure int64 for embedding lookup
1408 .to_device(self.device)
1409 .unsqueeze(0); // Add batch dimension: [1, seq_len]
1410
1411 // Lock the model and run forward pass (efficient!) with poison recovery
1412 let model = model_arc.lock();
1413 // RAII guard: panic-safe — Drop restores GradMode during unwinding
1414 let logits = {
1415 let _no_grad = tch::no_grad_guard();
1416 model.forward(&input_tensor, None)?
1417 };
1418
1419 // Extract logits for the last token
1420 let logits_shape = logits.size();
1421 let seq_len = logits_shape[1];
1422 let _vocab_size = logits_shape[2] as usize;
1423
1424 // Get logits for last token: [batch=1, last_seq_pos, vocab_size]
1425 let last_token_logits = logits.narrow(1, seq_len - 1, 1).squeeze_dim(1); // [1, vocab_size]
1426
1427 Ok(last_token_logits)
1428 }
1429
1430 /// Run optimized inference with KV caching - only process new tokens
1431 pub fn forward_cached(&self, input_ids: &[i64], start_pos: usize) -> Result<Tensor> {
1432 // Use the persistent model instance
1433 let model_arc = self
1434 .persistent_model
1435 .as_ref()
1436 .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1437
1438 if !self.is_persistent_model_ready() {
1439 return Err(anyhow!("Model not properly initialized"));
1440 }
1441
1442 // Process all provided tokens. The caller is responsible for passing
1443 // only the tokens that need processing:
1444 // - Decode mode: caller passes &[last_token] (single token)
1445 // - Partial prefill: caller passes &prompt_tokens[prefix_len..] (new suffix)
1446 // - Full prefill: caller passes &prompt_tokens (all tokens)
1447 let tokens_to_process = input_ids;
1448
1449 // Convert to tensor
1450 let input_tensor = Tensor::from_slice(tokens_to_process)
1451 .to_kind(tch::Kind::Int64)
1452 .to_device(self.device)
1453 .unsqueeze(0); // [1, seq_len]
1454
1455 // Run forward pass with position info for proper KV cache usage
1456 let model = model_arc.lock();
1457
1458 // RAII guard: panic-safe — Drop restores GradMode during unwinding
1459 let logits = {
1460 let _no_grad = tch::no_grad_guard();
1461 model.forward_with_cache(&input_tensor, start_pos)?
1462 };
1463
1464 // Extract logits for the last token
1465 let logits_shape = logits.size();
1466 let seq_len = logits_shape[1];
1467 let _vocab_size = logits_shape[2] as usize;
1468
1469 // Get logits for last token
1470 let last_token_logits = logits.narrow(1, seq_len - 1, 1).squeeze_dim(1);
1471
1472 Ok(last_token_logits)
1473 }
1474
1475 /// Run inference with KV caching and optional delta injection
1476 ///
1477 /// Same as `forward_cached()` but calls `forward_with_cache_and_delta()` on the model,
1478 /// injecting per-tenant LoRA corrections at each attention layer.
1479 pub fn forward_with_delta_cached(
1480 &self,
1481 input_ids: &[i64],
1482 start_pos: usize,
1483 delta: Option<&crate::training::TenantDelta>,
1484 ) -> Result<Tensor> {
1485 let model_arc = self
1486 .persistent_model
1487 .as_ref()
1488 .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1489
1490 if !self.is_persistent_model_ready() {
1491 return Err(anyhow!("Model not properly initialized"));
1492 }
1493
1494 // Process all provided tokens — caller passes the appropriate slice
1495 let input_tensor = Tensor::from_slice(input_ids)
1496 .to_kind(tch::Kind::Int64)
1497 .to_device(self.device)
1498 .unsqueeze(0);
1499
1500 let model = model_arc.lock();
1501
1502 let logits = {
1503 let _no_grad = tch::no_grad_guard();
1504 model.forward_with_cache_and_delta(&input_tensor, start_pos, delta)?
1505 };
1506
1507 let logits_shape = logits.size();
1508 let seq_len = logits_shape[1];
1509
1510 let last_token_logits = logits.narrow(1, seq_len - 1, 1).squeeze_dim(1);
1511
1512 Ok(last_token_logits)
1513 }
1514
1515 /// Batched ragged decode step over several same-delta sequences (#329).
1516 ///
1517 /// Each entry is `(new_token_ids, start_pos, per-sequence KVCacheManager)`.
1518 /// Returns last-token logits per row `[B, vocab]`. Delegates to the model's
1519 /// `forward_batched` (Llama-only; other architectures return an error so the
1520 /// caller falls back to per-stream `forward_with_delta_cached`). Tensors stay
1521 /// on this (bridge) thread; only the Send token/logits metadata crosses.
1522 pub fn forward_batched_step(
1523 &self,
1524 sequences: &mut [(
1525 Vec<i64>,
1526 usize,
1527 std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
1528 )],
1529 delta: Option<&crate::training::TenantDelta>,
1530 ) -> Result<Tensor> {
1531 let model_arc = self
1532 .persistent_model
1533 .as_ref()
1534 .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1535 if !self.is_persistent_model_ready() {
1536 return Err(anyhow!("Model not properly initialized"));
1537 }
1538
1539 let model = model_arc.lock();
1540 let logits = {
1541 let _no_grad = tch::no_grad_guard();
1542 model.forward_batched(sequences, delta)?
1543 };
1544
1545 // logits: [B, q, vocab] -> last-token logits [B, vocab].
1546 let seq_len = logits.size()[1];
1547 Ok(logits.narrow(1, seq_len - 1, 1).squeeze_dim(1))
1548 }
1549
1550 /// Run full forward pass with optional delta injection (no KV cache)
1551 pub fn forward_with_delta_full(
1552 &self,
1553 input_ids: &[i64],
1554 delta: Option<&crate::training::TenantDelta>,
1555 ) -> Result<Tensor> {
1556 let model_arc = self
1557 .persistent_model
1558 .as_ref()
1559 .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1560
1561 if !self.is_persistent_model_ready() {
1562 return Err(anyhow!("Model not properly initialized"));
1563 }
1564
1565 let input_tensor = Tensor::from_slice(input_ids)
1566 .to_kind(tch::Kind::Int64)
1567 .to_device(self.device)
1568 .unsqueeze(0);
1569
1570 let model = model_arc.lock();
1571
1572 let logits = {
1573 let _no_grad = tch::no_grad_guard();
1574 model.forward_with_cache_and_delta(&input_tensor, 0, delta)?
1575 };
1576
1577 let logits_shape = logits.size();
1578 let seq_len = logits_shape[1];
1579
1580 let last_token_logits = logits.narrow(1, seq_len - 1, 1).squeeze_dim(1);
1581
1582 Ok(last_token_logits)
1583 }
1584
1585 /// Clear KV cache before new generation to prevent context pollution
1586 pub fn clear_kv_cache(&self) {
1587 if let Some(model_arc) = &self.persistent_model {
1588 let model = model_arc.lock();
1589 let model_any = model.as_any();
1590 if let Some(llama_model) =
1591 model_any.downcast_ref::<crate::runtime::architectures::llama::LlamaModel>()
1592 {
1593 llama_model.clear_kv_cache();
1594 tracing::debug!("Cleared KV cache before generation");
1595 } else if let Some(q35_model) =
1596 model_any.downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
1597 {
1598 q35_model.clear_kv_cache();
1599 tracing::debug!("Cleared Qwen3.5 KV cache + SSM states before generation");
1600 }
1601 }
1602 }
1603
1604 /// Snapshot Qwen3.5 SSM (conv/rec) states for TTT adaptation.
1605 ///
1606 /// Returns `Some((conv, rec))` if the loaded model is Qwen3.5; `None` otherwise.
1607 /// Each `Tensor` is deep-copied via `.copy()` (C4 fix) so the snapshot is
1608 /// independent of subsequent forward-pass mutations during the TTT loop.
1609 pub fn snapshot_ssm_states(&self) -> Option<(Vec<Option<Tensor>>, Vec<Option<Tensor>>)> {
1610 let model_arc = self.persistent_model.as_ref()?;
1611 let model = model_arc.lock();
1612 let q35 = model
1613 .as_any()
1614 .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()?;
1615 Some(q35.snapshot_ssm_states())
1616 }
1617
1618 /// Restore Qwen3.5 SSM states from a snapshot produced by `snapshot_ssm_states`.
1619 ///
1620 /// No-op if the loaded model is not Qwen3.5 or snapshot is `None`.
1621 pub fn restore_ssm_states(&self, snapshot: Option<(Vec<Option<Tensor>>, Vec<Option<Tensor>>)>) {
1622 let Some((conv_snap, rec_snap)) = snapshot else {
1623 return;
1624 };
1625 let Some(model_arc) = &self.persistent_model else {
1626 return;
1627 };
1628 let model = model_arc.lock();
1629 if let Some(q35) = model
1630 .as_any()
1631 .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
1632 {
1633 q35.restore_ssm_states(conv_snap, rec_snap);
1634 tracing::debug!("Restored Qwen3.5 SSM states after TTT adaptation");
1635 }
1636 }
1637
1638 // ========================================================================
1639 // MTP self-speculative decoding (Qwen3.5, v1: dense + greedy + batch=1)
1640 // ========================================================================
1641
1642 /// Whether the loaded model can self-speculate: a Qwen3.5 with a loaded
1643 /// (dense) MTP draft head. Pipeline/multi-device stage loads never own the
1644 /// MTP head, so this is false there too.
1645 pub fn speculative_supported(&self) -> bool {
1646 let Some(model_arc) = &self.persistent_model else {
1647 return false;
1648 };
1649 let model = model_arc.lock();
1650 model
1651 .as_any()
1652 .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
1653 .is_some_and(crate::runtime::architectures::qwen3_5::Qwen3_5Model::has_mtp)
1654 }
1655
1656 /// Speculative prefill/verify forward: returns per-position logits plus the
1657 /// pre-final-norm hidden states the MTP head consumes.
1658 ///
1659 /// Never delta-adapted: speculation is disabled for tenant-delta streams at
1660 /// stream construction, so this always runs the base model.
1661 ///
1662 /// Returns `(logits [1, seq|1, vocab], hidden [1, seq, hidden_size])`.
1663 pub fn forward_speculative(
1664 &self,
1665 input_ids: &[i64],
1666 start_pos: usize,
1667 last_logits_only: bool,
1668 ) -> Result<(Tensor, Tensor)> {
1669 let model_arc = self
1670 .persistent_model
1671 .as_ref()
1672 .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1673 if !self.is_persistent_model_ready() {
1674 return Err(anyhow!("Model not properly initialized"));
1675 }
1676
1677 let input_tensor = Tensor::from_slice(input_ids)
1678 .to_kind(tch::Kind::Int64)
1679 .to_device(self.device)
1680 .unsqueeze(0);
1681
1682 let model = model_arc.lock();
1683 let q35 = model
1684 .as_any()
1685 .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
1686 .ok_or_else(|| anyhow!("Speculative decoding requires a Qwen3.5 model"))?;
1687 let _no_grad = tch::no_grad_guard();
1688 q35.forward_with_cache_hidden(&input_tensor, start_pos, last_logits_only)
1689 }
1690
1691 /// MTP draft forward over `token_ids.len()` consecutive slots. Slot `i`
1692 /// pairs `hiddens[.., i, ..]` with `embed(token_ids[i])` at absolute
1693 /// position `base + i`. Returns the last slot's logits `[1, 1, vocab]`.
1694 pub fn forward_mtp_draft(
1695 &self,
1696 token_ids: &[i64],
1697 hiddens: &Tensor,
1698 base: usize,
1699 ) -> Result<Tensor> {
1700 let model_arc = self
1701 .persistent_model
1702 .as_ref()
1703 .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1704 if !self.is_persistent_model_ready() {
1705 return Err(anyhow!("Model not properly initialized"));
1706 }
1707
1708 let ids_tensor = Tensor::from_slice(token_ids)
1709 .to_kind(tch::Kind::Int64)
1710 .to_device(self.device)
1711 .unsqueeze(0);
1712
1713 let model = model_arc.lock();
1714 let q35 = model
1715 .as_any()
1716 .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
1717 .ok_or_else(|| anyhow!("Speculative decoding requires a Qwen3.5 model"))?;
1718 let _no_grad = tch::no_grad_guard();
1719 q35.mtp_forward(&ids_tensor, hiddens, base)
1720 }
1721
1722 /// Sample next token using bundled parameters with tiered repeat penalty.
1723 fn sample_token_with_params(
1724 &self,
1725 logits_tensor: &Tensor,
1726 params: &SamplingParams,
1727 previous_tokens: &[i64],
1728 penalty_exempt_tokens: &HashSet<i64>,
1729 ) -> Result<usize> {
1730 self.sampler.sample_token_with_penalty_exemptions(
1731 logits_tensor,
1732 params.temperature,
1733 params.top_p,
1734 params.top_k,
1735 params.repeat_penalty,
1736 previous_tokens,
1737 penalty_exempt_tokens,
1738 )
1739 }
1740}
1741
1742/// Rewind the model's installed session cache to a matched prefix.
1743///
1744/// Pure-attention models: truncating the KV cache to `prefix_len` is exact, so
1745/// any prefix length is reusable (unchanged behavior).
1746///
1747/// Hybrid recurrent models (Qwen3.5): the GDN conv/rec state is Markovian and
1748/// cannot be truncated to an arbitrary position — it can only be restored to
1749/// the end-of-prefill snapshot stored with the cached tokens (see
1750/// [`KVCacheManager::set_cached_tokens_with_ssm`]). A prefix hit is therefore
1751/// reusable only when the FULL cached sequence matched
1752/// (`prefix_len == cached_token_count()`) and a snapshot is present; anything
1753/// else discards KV + SSM state and forces a full recompute, because decoding
1754/// from a truncated KV with stale recurrent state silently produces tokens
1755/// conditioned on the wrong context.
1756///
1757/// Returns the number of reusable prefix tokens (0 = cache cleared, full
1758/// prefill required). The caller must guarantee a strict partial hit
1759/// (`prefix_len < prompt_len`); the exact-prompt-repeat decision lives in
1760/// [`resolve_session_prefill_start`].
1761pub(crate) fn rewind_session_state(model: &dyn ModelOperations, prefix_len: usize) -> usize {
1762 let Some(cache) = model.get_kv_cache() else {
1763 return 0;
1764 };
1765 let q35 = model
1766 .as_any()
1767 .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>();
1768 let Some(q35) = q35 else {
1769 cache.lock().truncate_to(prefix_len);
1770 return prefix_len;
1771 };
1772
1773 let snapshot = {
1774 let cache_guard = cache.lock();
1775 if prefix_len == cache_guard.cached_token_count() {
1776 cache_guard.ssm_snapshot()
1777 } else {
1778 None
1779 }
1780 };
1781 match snapshot {
1782 Some((conv_snap, rec_snap)) => {
1783 cache.lock().truncate_to(prefix_len);
1784 q35.restore_ssm_states(conv_snap, rec_snap);
1785 prefix_len
1786 }
1787 None => {
1788 // Partial prefix match (or no snapshot): recurrent state cannot be
1789 // rewound to an intermediate position — recompute from scratch.
1790 model.clear_kv_cache();
1791 0
1792 }
1793 }
1794}
1795
1796/// Decide the prefill start position for a session-cache prefix match.
1797///
1798/// A strict partial hit (`0 < prefix_len < prompt_len`) is reusable: there is
1799/// a nonempty suffix left to prefill, and [`rewind_session_state`] puts the
1800/// cache and model into the exact end-of-prefix state.
1801///
1802/// An exact-prompt repeat (`prefix_len == prompt_len` — the session resending
1803/// its previous prompt verbatim, `prefix_len == cached_token_count()`) is NOT
1804/// reusable even though the full cached sequence matched: no tokens remain to
1805/// prefill, and sampling the first new token needs logits for the last prompt
1806/// position, which the cache does not retain. The engine must re-run the whole
1807/// prompt, and that re-run must start from CLEARED state — prefilling onto the
1808/// restored KV plus end-of-prefill GDN snapshot would advance the recurrent
1809/// state over the prompt a second time, producing wrong logits (and a wrong
1810/// end-of-prefill snapshot for the next turn). A miss (`prefix_len == 0`)
1811/// clears and starts fresh as before.
1812///
1813/// Returns the prefill start position (0 = cache cleared, full prefill).
1814pub(crate) fn resolve_session_prefill_start(
1815 model: &dyn ModelOperations,
1816 prefix_len: usize,
1817 prompt_len: usize,
1818) -> usize {
1819 if prefix_len > 0 && prefix_len < prompt_len {
1820 rewind_session_state(model, prefix_len)
1821 } else {
1822 model.clear_kv_cache();
1823 0
1824 }
1825}
1826
1827impl TorchEngine {
1828 /// Stable, content-free label identifying the loaded model for OTel metric
1829 /// attributes (#1261). Prefers the model name (set on load); falls back to
1830 /// architecture, then "unknown". Never the prompt or generated text.
1831 fn model_label(&self) -> String {
1832 let info = self.model_info.lock().clone();
1833 if !info.name.is_empty() && info.name != "unloaded" {
1834 info.name
1835 } else if !info.architecture.is_empty() && info.architecture != "unknown" {
1836 info.architecture
1837 } else {
1838 "unknown".to_owned()
1839 }
1840 }
1841}
1842
1843#[async_trait]
1844impl RuntimeEngine for TorchEngine {
1845 #[instrument(name = "torch_engine.load_model", skip(self), fields(path = %path.display()))]
1846 async fn load_model(&mut self, path: &Path) -> Result<()> {
1847 // Store the original path for model naming
1848 let original_path = path.to_path_buf();
1849
1850 // If path is a directory, find the model file inside it
1851 let model_file_path = if path.is_dir() {
1852 // First check for single file patterns
1853 let single_files = [
1854 "model.safetensors",
1855 "pytorch_model.bin",
1856 "model.bin",
1857 "model.pt",
1858 "model.pth",
1859 ];
1860
1861 let mut found_file = None;
1862 for filename in &single_files {
1863 let candidate = path.join(filename);
1864 if candidate.exists() {
1865 found_file = Some(candidate);
1866 break;
1867 }
1868 }
1869
1870 // If no single file found, check for sharded SafeTensors
1871 if found_file.is_none() {
1872 if let Ok(entries) = std::fs::read_dir(path) {
1873 for entry in entries.flatten() {
1874 let filename = entry.file_name();
1875 if let Some(name) = filename.to_str() {
1876 // Match standard shard pattern: model-00001-of-NNNNN.safetensors
1877 // Also match Qwen3.5 pattern: model.safetensors-00001-of-NNNNN.safetensors
1878 let is_first_shard = (name.starts_with("model-00001-of-")
1879 || name.starts_with("model.safetensors-00001-of-"))
1880 && name.ends_with(".safetensors");
1881 if is_first_shard {
1882 info!(
1883 "🔍 Detected sharded SafeTensors model starting with: {}",
1884 name
1885 );
1886 found_file = Some(entry.path());
1887 break;
1888 }
1889 }
1890 }
1891 }
1892 }
1893
1894 found_file.ok_or_else(|| {
1895 anyhow!(
1896 "No supported model file found in directory: {}",
1897 path.display()
1898 )
1899 })?
1900 } else {
1901 path.to_path_buf()
1902 };
1903
1904 info!("Loading model: {}", model_file_path.display());
1905 self.load_model_file(&model_file_path).await?;
1906
1907 // Set the model name based on the canonical path
1908 // Use directory name if loading from directory, otherwise use filename
1909 let model_name = if original_path.is_dir() {
1910 // Get the last component of the directory path as the model name
1911 original_path
1912 .file_name()
1913 .and_then(|s| s.to_str())
1914 .unwrap_or("unknown")
1915 .to_owned()
1916 } else {
1917 // Use the file stem for single files
1918 original_path
1919 .file_stem()
1920 .and_then(|s| s.to_str())
1921 .unwrap_or("unknown")
1922 .to_owned()
1923 };
1924
1925 // Update model info with the correct name
1926 {
1927 let mut model_info_guard = self.model_info.lock();
1928 model_info_guard.name = model_name;
1929 }
1930
1931 self.load_tokenizer(path).await?;
1932 Ok(())
1933 }
1934
1935 async fn generate(&self, prompt: &str, max_tokens: usize) -> Result<String> {
1936 // Format prompt with model-agnostic chat template
1937 let formatted_prompt =
1938 self.format_chat_message(Some("You are a helpful assistant."), prompt);
1939
1940 let request = GenerationRequest {
1941 prompt: formatted_prompt,
1942 max_tokens: Some(max_tokens as u32),
1943 temperature: Some(self.generation_config.temperature),
1944 top_p: Some(self.generation_config.top_p),
1945 top_k: self.generation_config.top_k.map(|v| v as u32),
1946 repeat_penalty: Some(self.generation_config.repeat_penalty),
1947 stop_tokens: Some(self.generation_config.stop_tokens.clone()),
1948 ..Default::default()
1949 };
1950
1951 let result = self.generate_with_params(request).await?;
1952 Ok(result.text)
1953 }
1954
1955 async fn generate_with_params(&self, request: GenerationRequest) -> Result<GenerationResult> {
1956 use futures::StreamExt;
1957
1958 if !self.is_persistent_model_ready() {
1959 return Err(anyhow!(
1960 "Model not properly initialized - persistent model not ready"
1961 ));
1962 }
1963
1964 let mut stream = self.generate(request)?;
1965 let mut accumulated_text = String::new();
1966
1967 while let Some(text_chunk) = stream.next().await {
1968 accumulated_text.push_str(&text_chunk?);
1969 }
1970
1971 let stats = stream.stats();
1972 Ok(GenerationResult {
1973 text: accumulated_text,
1974 tokens_generated: stats.tokens_generated,
1975 finish_reason: stats.finish_reason.unwrap_or(FinishReason::Stop),
1976 generation_time_ms: stats.generation_time_ms,
1977 tokens_per_second: stats.tokens_per_second,
1978 quality_metrics: stats.quality_metrics,
1979 prefill_tokens: stats.prefill_tokens,
1980 prefill_time_ms: stats.prefill_time_ms,
1981 prefill_tokens_per_sec: stats.prefill_tokens_per_sec,
1982 inference_tokens: stats.inference_tokens,
1983 inference_time_ms: stats.inference_time_ms,
1984 inference_tokens_per_sec: stats.inference_tokens_per_sec,
1985 ttt_metrics: None, // TTT metrics attached by InferenceService
1986 })
1987 }
1988
1989 fn model_info(&self) -> ModelInfo {
1990 self.model_info.lock().clone()
1991 }
1992
1993 fn is_loaded(&self) -> bool {
1994 let varstore_loaded = self.has_varstore();
1995 let persistent_loaded = self.persistent_model.is_some();
1996
1997 varstore_loaded || persistent_loaded
1998 }
1999
2000 fn apply_chat_template(
2001 &self,
2002 messages: &[ChatMessage],
2003 add_generation_prompt: bool,
2004 tools: Option<&serde_json::Value>,
2005 ) -> Result<String> {
2006 self.apply_chat_template_with_vars(messages, add_generation_prompt, tools, None, None)
2007 }
2008
2009 fn apply_chat_template_with_vars(
2010 &self,
2011 messages: &[ChatMessage],
2012 add_generation_prompt: bool,
2013 tools: Option<&serde_json::Value>,
2014 enable_thinking: Option<bool>,
2015 template_vars_json: Option<&str>,
2016 ) -> Result<String> {
2017 // Use our template engine if available
2018 let template_guard = self.template_engine.lock();
2019
2020 if let Some(ref engine) = *template_guard {
2021 engine.apply_chat_template_with_vars(
2022 messages,
2023 Some(add_generation_prompt),
2024 tools,
2025 enable_thinking,
2026 template_vars_json,
2027 )
2028 } else {
2029 // Fallback to simple formatting
2030 let mut formatted = String::new();
2031 for msg in messages {
2032 formatted.push_str(&format!(
2033 "{}: {}\n",
2034 msg.role,
2035 msg.content.as_deref().unwrap_or("")
2036 ));
2037 }
2038 if add_generation_prompt {
2039 formatted.push_str("assistant: ");
2040 }
2041 Ok(formatted)
2042 }
2043 }
2044}
2045
2046// Training-specific methods
2047impl TorchEngine {
2048 /// Enable training mode for pre-training
2049 pub fn enable_training(&mut self, _learning_rate: f64) -> Result<()> {
2050 // Ensure model is loaded
2051 if !self.is_persistent_model_ready() {
2052 return Err(anyhow!(
2053 "Model not loaded - persistent model not initialized"
2054 ));
2055 }
2056
2057 // For pre-training without VarStore:
2058 // 1. Set requires_grad on model tensors
2059 // 2. Use manual SGD or implement custom optimizer
2060 //
2061 // The challenge: tch-rs optimizers require VarStore
2062 // Solutions:
2063 // - Create a temporary VarStore and register model weights
2064 // - Implement manual gradient descent
2065 // - Use LoRA for all training (recommended)
2066
2067 tracing::warn!("Pre-training without VarStore requires manual optimizer implementation");
2068 tracing::info!("For now, use LoRA training which has proper VarStore support");
2069
2070 Ok(())
2071 }
2072
2073 /// Perform a manual SGD step without optimizer
2074 pub fn manual_sgd_step(&mut self, _learning_rate: f32) -> Result<()> {
2075 // This would manually update weights:
2076 // weight = weight - learning_rate * weight.grad()
2077 // But we don't have direct access to the model's tensors
2078
2079 tracing::warn!("Manual SGD not implemented - model tensors not accessible");
2080 Ok(())
2081 }
2082
2083 /// Forward pass for training (with gradient tracking)
2084 pub fn forward_training(&self, input_ids: &Tensor, track_gradients: bool) -> Result<Tensor> {
2085 if !self.is_persistent_model_ready() {
2086 return Err(anyhow!("Model not initialized"));
2087 }
2088
2089 let model = self
2090 .persistent_model
2091 .as_ref()
2092 .ok_or_else(|| anyhow!("Persistent model not available"))?;
2093 let model_guard = model.lock();
2094
2095 // Forward pass - gradients tracked if tensors have requires_grad
2096 if track_gradients {
2097 // Gradients are tracked by default when tensors have requires_grad
2098 model_guard.forward(input_ids, None)
2099 } else {
2100 {
2101 let _no_grad = tch::no_grad_guard();
2102 model_guard.forward(input_ids, None)
2103 }
2104 }
2105 }
2106
2107 /// Compute loss and backward (without optimizer step)
2108 pub fn compute_loss_and_backward(&self, input_ids: &Tensor, labels: &Tensor) -> Result<f64> {
2109 // Forward pass with gradients
2110 let logits = self.forward_training(input_ids, true)?;
2111
2112 // Compute cross-entropy loss
2113 let batch_size = logits.size()[0];
2114 let seq_len = logits.size()[1];
2115 let vocab_size = logits.size()[2];
2116
2117 let logits_flat = logits.view([batch_size * seq_len, vocab_size]);
2118 let labels_flat = labels.view([batch_size * seq_len]);
2119
2120 let loss = logits_flat.cross_entropy_loss::<Tensor>(
2121 &labels_flat,
2122 None,
2123 tch::Reduction::Mean,
2124 -100, // ignore_index for padding
2125 0.0, // label_smoothing
2126 );
2127
2128 let loss_value = loss.double_value(&[]);
2129
2130 // Backward pass computes gradients
2131 loss.backward();
2132
2133 Ok(loss_value)
2134 }
2135
2136 /// Disable training mode
2137 pub fn disable_training(&mut self) -> Result<()> {
2138 // Gradient tracking is controlled per tensor, not globally
2139 tracing::info!("Training mode disabled");
2140 Ok(())
2141 }
2142
2143 /// Check if training is enabled
2144 pub fn is_training_enabled(&self) -> bool {
2145 false // Pre-training not fully supported without VarStore
2146 }
2147}
2148
2149// Additional methods needed by inference layer
2150impl TorchEngine {
2151 // Old callback-based streaming APIs removed - use generate() for all streaming use cases
2152
2153 /// Check if persistent model is initialized - thread safe
2154 pub fn is_persistent_model_ready(&self) -> bool {
2155 let persistent_ready = self.persistent_model.is_some();
2156 let context_ready = self
2157 .context_state
2158 .lock()
2159 .as_ref()
2160 .is_some_and(|c| c.initialized);
2161
2162 persistent_ready && context_ready
2163 }
2164
2165 /// Generate text as a stream of decoded UTF-8 text chunks
2166 ///
2167 /// Returns a Stream that yields `Result<String>` with properly decoded text.
2168 /// The stream automatically handles:
2169 /// - Multi-byte UTF-8 sequences (emojis, CJK characters, etc.)
2170 /// - EOS and stop token detection
2171 /// - Max tokens limit
2172 /// - Special token filtering
2173 ///
2174 /// Dropping the stream stops generation automatically (no manual cancellation needed).
2175 ///
2176 /// # Example
2177 /// ```no_run
2178 /// use futures::StreamExt;
2179 /// use hyprstream_rpc_std::inference_client::GenerationRequest;
2180 ///
2181 /// # async fn example(engine: &hyprstream_inference::runtime::torch_engine::TorchEngine) -> anyhow::Result<()> {
2182 /// let request = GenerationRequest::default();
2183 /// let mut stream = engine.generate(request)?;
2184 ///
2185 /// while let Some(text_chunk) = stream.next().await {
2186 /// print!("{}", text_chunk?);
2187 /// }
2188 ///
2189 /// let stats = stream.stats();
2190 /// println!("Generated {} tokens", stats.tokens_generated);
2191 /// # Ok(())
2192 /// # }
2193 /// ```
2194 pub fn generate(&self, mut request: GenerationRequest) -> Result<TextStream<'_>> {
2195 // Set random seed if provided for deterministic generation
2196 if let Some(seed) = request.seed {
2197 self.set_seed(seed as u64);
2198 }
2199
2200 // Apply server defaults if not specified in request
2201 if request.timeout_ms.is_none() {
2202 request.timeout_ms = Some(self.config.default_generation_timeout_ms);
2203 }
2204
2205 TextStream::new(self, request)
2206 }
2207
2208 /// Generate with optional per-tenant delta for delta-aware inference
2209 ///
2210 /// When `delta` is Some, LoRA corrections are injected at each attention layer
2211 /// during generation. The delta is locked per-token (not held across the entire
2212 /// generation) to allow concurrent training updates.
2213 ///
2214 /// `adapter_generation` is the live LoRA/delta generation actually applied to
2215 /// this request; it is folded into the KV-compat expected identity so a
2216 /// re-adapted model is detected as incompatible and its stale KV rejected.
2217 ///
2218 /// `tenant_id` is the effective tenant whose delta contributed to the weights
2219 /// (`Some` only when a per-tenant delta is in play, `None` otherwise), and
2220 /// `weight_epoch` is that delta's current weight epoch. The pair is threaded
2221 /// to the session-cache swap so the resulting KV is registered for
2222 /// tenant-delta invalidation and cleared if the weights changed under a
2223 /// different tenant or an older weight epoch (same-tenant in-place
2224 /// re-adaptation). The caller must derive the tenant id from the verified
2225 /// request subject, not invent a new identity source.
2226 pub fn generate_with_delta(
2227 &self,
2228 mut request: GenerationRequest,
2229 delta: Option<std::sync::Arc<parking_lot::Mutex<crate::training::TenantDelta>>>,
2230 tenant_id: Option<String>,
2231 weight_epoch: u64,
2232 adapter_generation: u64,
2233 ) -> Result<TextStream<'_>> {
2234 if let Some(seed) = request.seed {
2235 self.set_seed(seed as u64);
2236 }
2237
2238 if request.timeout_ms.is_none() {
2239 request.timeout_ms = Some(self.config.default_generation_timeout_ms);
2240 }
2241
2242 TextStream::new_with_delta(
2243 self,
2244 request,
2245 delta,
2246 tenant_id,
2247 weight_epoch,
2248 adapter_generation,
2249 )
2250 }
2251
2252 /// Non-streaming generation with optional delta (convenience wrapper)
2253 pub async fn generate_with_delta_params(
2254 &self,
2255 request: GenerationRequest,
2256 delta: Option<std::sync::Arc<parking_lot::Mutex<crate::training::TenantDelta>>>,
2257 tenant_id: Option<String>,
2258 weight_epoch: u64,
2259 adapter_generation: u64,
2260 ) -> Result<crate::config::GenerationResult> {
2261 use futures::StreamExt;
2262
2263 if !self.is_persistent_model_ready() {
2264 return Err(anyhow!(
2265 "Model not properly initialized - persistent model not ready"
2266 ));
2267 }
2268
2269 let mut stream =
2270 self.generate_with_delta(request, delta, tenant_id, weight_epoch, adapter_generation)?;
2271 let mut accumulated_text = String::new();
2272
2273 while let Some(text_chunk) = stream.next().await {
2274 accumulated_text.push_str(&text_chunk?);
2275 }
2276
2277 let stats = stream.stats();
2278 Ok(crate::config::GenerationResult {
2279 text: accumulated_text,
2280 tokens_generated: stats.tokens_generated,
2281 finish_reason: stats
2282 .finish_reason
2283 .unwrap_or(crate::config::FinishReason::Stop),
2284 generation_time_ms: stats.generation_time_ms,
2285 tokens_per_second: stats.tokens_per_second,
2286 quality_metrics: stats.quality_metrics,
2287 prefill_tokens: stats.prefill_tokens,
2288 prefill_time_ms: stats.prefill_time_ms,
2289 prefill_tokens_per_sec: stats.prefill_tokens_per_sec,
2290 inference_tokens: stats.inference_tokens,
2291 inference_time_ms: stats.inference_time_ms,
2292 inference_tokens_per_sec: stats.inference_tokens_per_sec,
2293 ttt_metrics: None,
2294 })
2295 }
2296
2297 /// Forward pass with per-layer delta injection for TTT training.
2298 ///
2299 /// Uses `decode_layer_with_delta()` to inject the tenant delta's A/B matrices
2300 /// after q_proj/v_proj projections inside each attention layer. This creates a
2301 /// differentiable path from the loss back to the delta parameters, enabling
2302 /// gradient-based training via `loss.backward()`.
2303 ///
2304 /// # Arguments
2305 /// * `input_ids` - Token IDs [1, seq_len]
2306 /// * `delta` - Tenant's LoRA delta with trainable A/B matrices
2307 ///
2308 /// # Returns
2309 /// Logits tensor [1, seq_len, vocab_size] with gradient path to delta parameters
2310 pub fn forward_with_delta(
2311 &self,
2312 input_ids: &Tensor,
2313 delta: &crate::training::TenantDelta,
2314 ) -> Result<Tensor> {
2315 let model = self
2316 .persistent_model
2317 .as_ref()
2318 .ok_or_else(|| anyhow!("Model not loaded"))?;
2319 let model_guard = model.lock();
2320
2321 // Get embeddings from the base model
2322 let input = input_ids.to(self.device);
2323 let mut hidden_states = model_guard.embed_tokens(&input)?;
2324
2325 // NOTE: f32 upcast is NOT done here because model weights are bf16 and
2326 // ROCm/HIP matmul requires matching dtypes. Gradient precision is handled
2327 // by the delta injection path (LoRA A/B matrices) operating in the model's
2328 // native dtype with the with_grad() wrapper ensuring autograd is active.
2329
2330 // Generate position IDs
2331 let seq_len = hidden_states.size()[1];
2332 let position_ids = Tensor::arange(seq_len, (tch::Kind::Int64, hidden_states.device()));
2333
2334 let num_layers = model_guard.num_layers();
2335
2336 // Process each layer with delta injection
2337 for layer_idx in 0..num_layers {
2338 let (new_hidden, _kv) = model_guard.decode_layer_with_delta(
2339 layer_idx,
2340 &hidden_states,
2341 None, // attention_mask
2342 Some(&position_ids), // position_ids
2343 None, // past_kv (no cache during training)
2344 delta,
2345 )?;
2346 hidden_states = new_hidden;
2347 }
2348
2349 // Apply final norm + LM head
2350 hidden_states = model_guard.apply_final_norm(&hidden_states)?;
2351 let logits = model_guard.lm_head(&hidden_states)?;
2352
2353 Ok(logits)
2354 }
2355
2356 /// Authoritative context capacity of the loaded model. ModelInfo is populated
2357 /// from the checkpoint configuration and optional max_context override during
2358 /// loading; RuntimeConfig.context_length is only a constructor default.
2359 /// Serving encoders and native decision admission must use this same limit.
2360 pub fn effective_context_limit(&self) -> Result<usize> {
2361 anyhow::ensure!(self.persistent_model.is_some(), "Model not loaded");
2362 let limit = self.model_info.lock().context_length as usize;
2363 anyhow::ensure!(limit > 0, "loaded model has an invalid context limit");
2364 Ok(limit)
2365 }
2366
2367 /// Execute a decision head against fresh full-sequence hidden states while
2368 /// holding the backbone lock. The supplied gather reads original LM-head
2369 /// rows without allocating token-by-vocabulary logits. No cache is reused.
2370 pub fn with_decision_backbone<T>(
2371 &self,
2372 token_ids: &[i64],
2373 head: impl FnOnce(&Tensor, &dyn Fn(&[i64]) -> Result<Tensor>) -> Result<T>,
2374 ) -> Result<T> {
2375 self.with_decision_backbone_checked(token_ids, &|| Ok(()), head)
2376 }
2377
2378 /// Cooperative cancellation/deadline variant. The checker runs before
2379 /// admission, between decoder layers, and around head evaluation. An active
2380 /// libtorch kernel completes before the next check; dropping a future alone
2381 /// is not a compute-cancellation mechanism.
2382 pub fn with_decision_backbone_checked<T>(
2383 &self,
2384 token_ids: &[i64],
2385 check: &dyn Fn() -> Result<()>,
2386 head: impl FnOnce(&Tensor, &dyn Fn(&[i64]) -> Result<Tensor>) -> Result<T>,
2387 ) -> Result<T> {
2388 check()?;
2389 anyhow::ensure!(!token_ids.is_empty(), "decision token sequence is empty");
2390 let limit = self.effective_context_limit()?;
2391 anyhow::ensure!(
2392 token_ids.len() <= limit,
2393 "decision input exceeds context limit"
2394 );
2395 anyhow::ensure!(
2396 self.device == tch::Device::Cpu,
2397 "decision backbone is CPU-only pending CPU acceptance"
2398 );
2399 let model = self
2400 .persistent_model
2401 .as_ref()
2402 .ok_or_else(|| anyhow!("Model not loaded"))?;
2403 let model = model.lock();
2404 let _guard = tch::no_grad_guard();
2405 let input = Tensor::from_slice(token_ids)
2406 .reshape([1, token_ids.len() as i64])
2407 .to_device(self.device);
2408 let hidden = model.decision_hidden_states_checked(&input, check)?;
2409 check()?;
2410 let gather = |ids: &[i64]| model.decision_output_embedding_rows(ids);
2411 let output = head(&hidden, &gather)?;
2412 check()?;
2413 Ok(output)
2414 }
2415
2416 /// Get the number of layers in the loaded model.
2417 pub fn get_num_layers(&self) -> Result<usize> {
2418 let model = self
2419 .persistent_model
2420 .as_ref()
2421 .ok_or_else(|| anyhow!("Model not loaded"))?;
2422 Ok(model.lock().num_layers())
2423 }
2424
2425 /// Get module dimensions for LoRA target modules from the loaded model.
2426 ///
2427 /// Returns a map of module_name -> (in_features, out_features) for all
2428 /// supported LoRA target modules. Used by DeltaPool to initialize
2429 /// per-tenant LoRA deltas with correct dimensions.
2430 ///
2431 /// Supported modules: q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj
2432 pub fn get_lora_module_dims(
2433 &self,
2434 ) -> Result<std::collections::HashMap<String, (usize, usize)>> {
2435 let model_info = self.model_info.lock();
2436 let hidden_size = model_info.hidden_size as usize;
2437
2438 let num_heads = model_info
2439 .num_attention_heads
2440 .ok_or_else(|| anyhow!("num_attention_heads not set in ModelInfo"))?
2441 as usize;
2442 let num_kv_heads = model_info.num_key_value_heads.unwrap_or(num_heads as u32) as usize;
2443 let head_dim = model_info
2444 .head_dim
2445 .map(|d| d as usize)
2446 .unwrap_or(hidden_size / num_heads);
2447
2448 let q_out = num_heads * head_dim;
2449 let kv_out = num_kv_heads * head_dim;
2450
2451 let intermediate_size = if let Some(intermediate) = model_info.intermediate_size {
2452 intermediate as usize
2453 } else {
2454 match model_info.architecture.as_str() {
2455 "Qwen2ForCausalLM" => (hidden_size as f32 * 2.6667) as usize,
2456 "LlamaForCausalLM" => (hidden_size as f32 * 2.75) as usize,
2457 _ => hidden_size * 4,
2458 }
2459 };
2460
2461 let mut dims = std::collections::HashMap::new();
2462 // Self-attention projections with correct dimensions
2463 // q_proj: in=hidden_size, out=num_heads * head_dim
2464 dims.insert("q_proj".to_owned(), (hidden_size, q_out));
2465 // k_proj/v_proj: in=hidden_size, out=num_kv_heads * head_dim (GQA support)
2466 dims.insert("k_proj".to_owned(), (hidden_size, kv_out));
2467 dims.insert("v_proj".to_owned(), (hidden_size, kv_out));
2468 // o_proj: in=num_heads * head_dim, out=hidden_size
2469 dims.insert("o_proj".to_owned(), (q_out, hidden_size));
2470 // MLP projections (expand)
2471 for name in &["gate_proj", "up_proj"] {
2472 dims.insert((*name).to_owned(), (hidden_size, intermediate_size));
2473 }
2474 // MLP projection (contract)
2475 dims.insert("down_proj".to_owned(), (intermediate_size, hidden_size));
2476
2477 Ok(dims)
2478 }
2479
2480 /// Get per-layer LoRA module dimensions for hybrid architectures (e.g., Qwen3.5).
2481 ///
2482 /// Returns `None` for uniform architectures (Llama, Gemma, etc.) where all layers
2483 /// have identical module dims. Returns per-layer overrides for Qwen3.5 where GDN
2484 /// layers have different `o_proj` input dims than full-attention layers.
2485 pub fn get_per_layer_lora_dims(
2486 &self,
2487 ) -> Option<std::collections::HashMap<usize, std::collections::HashMap<String, (usize, usize)>>>
2488 {
2489 let model_arc = self.persistent_model.as_ref()?;
2490 let model = model_arc.lock();
2491 let model_any = model.as_any();
2492
2493 if let Some(q35) =
2494 model_any.downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
2495 {
2496 let qcfg = q35.text_config();
2497 let hidden = qcfg.hidden_size as usize;
2498 let nh = qcfg.num_attention_heads as usize;
2499 let hd = qcfg.head_dim as usize;
2500 // q_proj outputs num_heads * head_dim * 2 (first half = Q, second half = gate).
2501 // The LoRA delta targets only the Q half, so lora_b out_features = nh*hd (the half-dim).
2502 // o_proj in_features is also nh*hd (attention output after gate application).
2503 let q_half_dim = nh * hd;
2504
2505 let gdn_out_proj_in = qcfg.linear_num_value_heads * qcfg.linear_value_head_dim;
2506 let num_layers = qcfg.num_hidden_layers as usize;
2507
2508 let mut per_layer: std::collections::HashMap<
2509 usize,
2510 std::collections::HashMap<String, (usize, usize)>,
2511 > = std::collections::HashMap::new();
2512
2513 let nkv_out = (qcfg.num_key_value_heads as usize) * hd;
2514 for (layer_idx, lt) in qcfg.layer_types.iter().enumerate() {
2515 let mut layer_dims: std::collections::HashMap<String, (usize, usize)> =
2516 std::collections::HashMap::new();
2517 if lt == "full_attention" {
2518 // Full-attention: q correction targets the Q half (not the doubled projection)
2519 layer_dims.insert("q_proj".to_owned(), (hidden, q_half_dim));
2520 layer_dims.insert("v_proj".to_owned(), (hidden, nkv_out));
2521 layer_dims.insert("o_proj".to_owned(), (q_half_dim, hidden));
2522 } else {
2523 // GDN (linear_attention): only o_proj; maps to struct field out_proj.
2524 // in_features = val_dim (input to out_proj), out_features = hidden_size.
2525 layer_dims.insert("o_proj".to_owned(), (gdn_out_proj_in, hidden));
2526 }
2527 per_layer.insert(layer_idx, layer_dims);
2528 }
2529
2530 // Fill any layers not covered by layer_types (every 4th is full_attn)
2531 for layer_idx in 0..num_layers {
2532 per_layer.entry(layer_idx).or_insert_with(|| {
2533 let is_full_attn = (layer_idx + 1) % 4 == 0;
2534 let mut layer_dims: std::collections::HashMap<String, (usize, usize)> =
2535 std::collections::HashMap::new();
2536 if is_full_attn {
2537 layer_dims.insert("q_proj".to_owned(), (hidden, q_half_dim));
2538 layer_dims.insert("v_proj".to_owned(), (hidden, nkv_out));
2539 layer_dims.insert("o_proj".to_owned(), (q_half_dim, hidden));
2540 } else {
2541 layer_dims.insert("o_proj".to_owned(), (gdn_out_proj_in, hidden));
2542 }
2543 layer_dims
2544 });
2545 }
2546
2547 Some(per_layer)
2548 } else {
2549 None
2550 }
2551 }
2552
2553 /// Validate LoRA configuration against model dimensions.
2554 ///
2555 /// Checks that the requested target modules are valid for this model.
2556 /// The actual LoRA delta creation is handled by DeltaPool/TenantDelta.
2557 pub fn create_lora(&mut self, config: crate::training::TenantDeltaConfig) -> Result<()> {
2558 let all_dims = self.get_lora_module_dims()?;
2559
2560 for module_name in &config.target_modules {
2561 if !all_dims.contains_key(module_name.as_str()) {
2562 return Err(anyhow!(
2563 "Unknown module '{}'. Supported modules: q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj",
2564 module_name
2565 ));
2566 }
2567 }
2568
2569 tracing::info!(
2570 rank = config.rank,
2571 alpha = config.alpha,
2572 target_modules = ?config.target_modules,
2573 "LoRA configuration validated - delta creation handled by DeltaPool"
2574 );
2575 Ok(())
2576 }
2577
2578 // ============================================================================
2579 // Embedding Extraction for RAG/CAG
2580 // ============================================================================
2581
2582 /// Extract embedding from text using the model's hidden states.
2583 ///
2584 /// This method tokenizes the input text, runs a forward pass through the model,
2585 /// and extracts the final hidden states. The embedding is computed by mean-pooling
2586 /// the hidden states over the sequence dimension.
2587 ///
2588 /// # Arguments
2589 /// - `text` - Input text to embed
2590 ///
2591 /// # Returns
2592 /// - Vec<f32> - The embedding vector (dimension = model's hidden_size)
2593 ///
2594 /// # Example
2595 /// ```ignore
2596 /// let embedding = engine.extract_embedding("What is machine learning?")?;
2597 /// println!("Embedding dimension: {}", embedding.len());
2598 /// ```
2599 pub fn extract_embedding(&self, text: &str) -> Result<Vec<f32>> {
2600 // Tokenize input
2601 let tokenizer_guard = self.tokenizer.lock();
2602 let tokenizer = tokenizer_guard
2603 .as_ref()
2604 .ok_or_else(|| anyhow!("No tokenizer loaded"))?;
2605
2606 let encoding = tokenizer
2607 .encode(text, false)
2608 .map_err(|e| anyhow!("Tokenization failed: {}", e))?;
2609
2610 let token_ids: Vec<i64> = encoding.get_ids().iter().map(|&id| id as i64).collect();
2611 drop(tokenizer_guard);
2612
2613 self.extract_embedding_from_tokens(&token_ids)
2614 }
2615
2616 /// Extract embedding from pre-tokenized input.
2617 ///
2618 /// This is a lower-level method for when tokens are already available
2619 /// (e.g., from a previous generation). Uses mean pooling over the hidden states.
2620 ///
2621 /// # Arguments
2622 /// - `token_ids` - Pre-tokenized input IDs
2623 ///
2624 /// # Returns
2625 /// - Vec<f32> - The embedding vector (dimension = model's hidden_size)
2626 pub fn extract_embedding_from_tokens(&self, token_ids: &[i64]) -> Result<Vec<f32>> {
2627 if token_ids.is_empty() {
2628 return Err(anyhow!("Empty token list"));
2629 }
2630
2631 let input_tensor = Tensor::from_slice(token_ids)
2632 .to_device(self.device)
2633 .unsqueeze(0); // [1, seq_len]
2634
2635 let model_guard = self
2636 .persistent_model
2637 .as_ref()
2638 .ok_or_else(|| anyhow!("No model loaded"))?
2639 .lock();
2640
2641 {
2642 let _no_grad = tch::no_grad_guard();
2643
2644 // Get token embeddings
2645 let embeddings = model_guard.embed_tokens(&input_tensor)?;
2646
2647 // Process through layers to get hidden states
2648 let hidden_states = self
2649 .process_through_layers_for_embedding(&**model_guard, &embeddings)
2650 .unwrap_or_else(|_| embeddings.shallow_clone());
2651
2652 // Apply final normalization
2653 let normalized = model_guard
2654 .apply_final_norm(&hidden_states)
2655 .unwrap_or(hidden_states);
2656
2657 // Mean pool over sequence dimension: [1, seq_len, hidden_size] -> [1, hidden_size]
2658 let pooled = normalized.mean_dim(1, false, tch::Kind::Float);
2659
2660 // Squeeze batch dimension
2661 let pooled_1d = pooled.squeeze_dim(0); // [hidden_size]
2662
2663 // L2 normalize for cosine similarity
2664 let norm = pooled_1d.norm();
2665 let normalized_embedding = if norm.double_value(&[]) > 1e-12 {
2666 &pooled_1d / norm
2667 } else {
2668 pooled_1d
2669 };
2670
2671 // Convert to CPU and extract values
2672 let cpu_tensor = normalized_embedding.to_device(tch::Device::Cpu);
2673 let numel = cpu_tensor.numel();
2674 let mut embedding_vec = vec![0.0f32; numel];
2675 cpu_tensor.copy_data(&mut embedding_vec, numel);
2676
2677 Ok(embedding_vec)
2678 }
2679 }
2680
2681 /// Compute vision embeddings for one or more images.
2682 ///
2683 /// Preprocesses raw image bytes and runs them through the model's vision
2684 /// encoder (e.g. SigLIP). Returns one embedding vector per image.
2685 ///
2686 /// # Arguments
2687 /// * `images` - Raw image bytes (PNG/JPEG/RGB) per image
2688 ///
2689 /// # Returns
2690 /// * Vec<Vec<f32>> - Embedding vectors, one per input image
2691 pub fn embed_images(&self, images: &[Vec<u8>]) -> Result<Vec<Vec<f32>>> {
2692 use crate::runtime::image_utils::{load_image_from_bytes, ImagePreprocessConfig};
2693
2694 if images.is_empty() {
2695 return Ok(Vec::new());
2696 }
2697
2698 let model_guard = self
2699 .persistent_model
2700 .as_ref()
2701 .ok_or_else(|| anyhow!("No model loaded"))?
2702 .lock();
2703
2704 // Preprocess images to tensors
2705 let config = ImagePreprocessConfig::siglip();
2706 let mut image_tensors = Vec::with_capacity(images.len());
2707 for img_bytes in images {
2708 let tensor = load_image_from_bytes(img_bytes, &config, self.device)?;
2709 image_tensors.push(tensor);
2710 }
2711 let batch = Tensor::cat(&image_tensors, 0); // [B, 3, H, W]
2712
2713 let _no_grad = tch::no_grad_guard();
2714
2715 // Run through vision encoder
2716 let embeddings_tensor = model_guard.encode_vision(&batch)?;
2717 // embeddings_tensor shape: [B, hidden_size]
2718
2719 let batch_size = embeddings_tensor.size()[0] as usize;
2720 let dim = embeddings_tensor.size()[1] as usize;
2721 let cpu_tensor = embeddings_tensor
2722 .to_device(tch::Device::Cpu)
2723 .to_kind(tch::Kind::Float);
2724
2725 let mut results = Vec::with_capacity(batch_size);
2726 for i in 0..batch_size {
2727 let mut vec = vec![0.0f32; dim];
2728 let row = cpu_tensor.get(i as i64);
2729 row.copy_data(&mut vec, dim);
2730 results.push(vec);
2731 }
2732
2733 Ok(results)
2734 }
2735
2736 /// Process embeddings through all model layers for embedding extraction
2737 fn process_through_layers_for_embedding(
2738 &self,
2739 model: &dyn crate::runtime::architectures::ModelOperations,
2740 embeddings: &Tensor,
2741 ) -> Result<Tensor> {
2742 let num_layers = model.num_layers();
2743 let mut hidden_states = embeddings.shallow_clone();
2744
2745 for layer_idx in 0..num_layers {
2746 let (new_hidden, _kv) = model.decode_layer(
2747 layer_idx,
2748 &hidden_states,
2749 None, // attention_mask
2750 None, // position_ids
2751 None, // past_kv
2752 )?;
2753 hidden_states = new_hidden;
2754 }
2755
2756 Ok(hidden_states)
2757 }
2758
2759 /// Explicitly drop every CUDA/device tensor the engine owns or has cached
2760 /// on this thread, **now** — while the CUDA context is still alive.
2761 ///
2762 /// # Why this exists (#429)
2763 ///
2764 /// `tch`/libtorch defers CUDA context teardown to an `atexit` handler that
2765 /// runs during *process* exit. But two classes of device tensor outlive the
2766 /// engine struct and only drop later:
2767 ///
2768 /// 1. **Thread-local RoPE caches** (`llama::ROPE_CACHE`, `qwen3_5::ROPE_CACHE`)
2769 /// — sin/cos tables cached on the compute device. They are TLS, so they
2770 /// are not touched by `Drop` and survive until the owning thread exits.
2771 /// 2. **The inference service thread itself** — it is spawned detached in
2772 /// `LocalInferenceService::start`, so its TLS destructors (and thus the
2773 /// RoPE cache drop) can fire during process-exit cleanup, *after* the
2774 /// CUDA context is gone → `c10::Error: invalid device pointer` (SIGABRT).
2775 ///
2776 /// The engine-only GPU smoke (`e6_gpu_engine_is_single_device_on_one_gpu_host`)
2777 /// never loads weights nor runs a forward pass, so it populates no RoPE
2778 /// cache and exits cleanly; the streaming test does, which is why only it
2779 /// aborts.
2780 ///
2781 /// Calling this on the service thread at `Shutdown` receipt — *before*
2782 /// replying to the client — drops the RoPE tables and the model weights
2783 /// synchronously while the caller is still blocked on the reply (process
2784 /// indubitably alive, context valid). After it returns only empty shells
2785 /// remain for any late TLS/static destruction, so there is nothing left to
2786 /// free against a dead context.
2787 ///
2788 /// # Contract
2789 ///
2790 /// This is a **terminal** teardown: the engine is unusable afterward
2791 /// (`model_info` reports unloaded, generation will error). It must only be
2792 /// called once, at shutdown. Unlike [`Drop`], it deliberately *does* clear
2793 /// shared `Arc<Mutex<Option<T>>>` contents — that is safe precisely because
2794 /// the engine is being retired, not because the contents are unused. (The
2795 /// production inference path holds a single engine instance per service; a
2796 /// cached clone would see `None` and fail closed with "No model loaded",
2797 /// which is the correct behavior for a retired engine.)
2798 pub fn release_device_resources(&mut self) {
2799 // 1. Clear the model's own KV cache while the model still exists (the
2800 // cache tensors live inside the architecture model's KVCacheManager).
2801 self.clear_kv_cache();
2802
2803 // 2. Drop the architecture model (all decoder-layer weights, embeddings,
2804 // lm_head) — the bulk of the device memory.
2805 self.persistent_model = None;
2806
2807 // 3. Drop the (dummy) VarStore and clear the session KV registry so any
2808 // device tensors held there release before context teardown too.
2809 {
2810 let mut vs_guard = self.var_store.lock();
2811 *vs_guard = None;
2812 }
2813 if let Some(registry) = self.kv_cache_registry.clone() {
2814 registry.clear_all();
2815 }
2816
2817 // 4. Drop the thread-local RoPE sin/cos tables on *this* thread. This is
2818 // the load-bearing step for #429: without it these CUDA tensors would
2819 // only free in the detached service thread's TLS destructor.
2820 crate::runtime::architectures::llama::clear_rope_cache();
2821 crate::runtime::architectures::qwen3_5::clear_rope_cache();
2822
2823 // 5. Drain any in-flight CUDA kernels (frees completed, makes the above
2824 // drops effective on the device) before we hand control back. CPU is
2825 // a no-op. Best-effort: a failure to synchronize is logged, not fatal
2826 // — the tensors are already dropped on the Rust side.
2827 if let Device::Cuda(idx) = self.device {
2828 // atc_synchronize panics on internal error; never propagate.
2829 let res = std::panic::catch_unwind(|| tch::Cuda::synchronize(idx as i64));
2830 if res.is_err() {
2831 warn!("CUDA synchronize during teardown returned an error (ignored)");
2832 }
2833 }
2834 }
2835}
2836
2837impl Drop for TorchEngine {
2838 fn drop(&mut self) {
2839 // Drop our Arc references to shared resources.
2840 // The actual cleanup happens when the LAST Arc reference drops.
2841 //
2842 // IMPORTANT: Do NOT clear the contents of Arc<Mutex<Option<T>>> fields!
2843 // All TorchEngine clones share the same Arc instances. Clearing the contents
2844 // would break other clones (especially the cached engine in model_cache).
2845 //
2846 // Previous implementation cleared shared Arc contents, causing:
2847 // - First request: clone for spawn_blocking drops, clears var_store
2848 // - Second request: cached engine has var_store = None, fails with "No model loaded"
2849
2850 // Drop our Arc references (NOT the contents)
2851 self.persistent_model = None; // Drops our Option<Arc<...>> reference
2852
2853 // Arc<Mutex<...>> fields are automatically cleaned up via Arc reference counting
2854 // when the last TorchEngine clone drops. No manual intervention needed.
2855 }
2856}
2857
2858#[cfg(test)]
2859#[path = "torch_engine_mtp_session_tests.rs"]
2860mod mtp_session_tests;
2861
2862#[cfg(test)]
2863#[allow(clippy::unwrap_used, clippy::expect_used)]
2864mod tests {
2865 use super::*;
2866 use parking_lot::Mutex;
2867 use std::io::Write;
2868 use std::sync::Arc;
2869
2870 #[test]
2871 fn precision_policy_preserves_aliases_and_non_cpu_modes_without_gpu_execution() {
2872 for device in [Device::Cpu, Device::Cuda(0), Device::Mps] {
2873 for channel in [false, true] {
2874 for (aliases, expected) in [
2875 (
2876 &["fp32", "float32", "FP32", "Float32"][..],
2877 tch::Kind::Float,
2878 ),
2879 (&["fp16", "float16", "FP16", "Float16"][..], tch::Kind::Half),
2880 (
2881 &["bf16", "bfloat16", "BF16", "BFloat16"][..],
2882 tch::Kind::BFloat16,
2883 ),
2884 ] {
2885 for alias in aliases {
2886 assert_eq!(
2887 resolve_inference_compute_kind(Some(alias), device, channel).unwrap(),
2888 expected
2889 );
2890 }
2891 }
2892 let automatic = if device == Device::Cpu && channel {
2893 tch::Kind::Float
2894 } else {
2895 tch::Kind::BFloat16
2896 };
2897 for alias in [None, Some("auto"), Some("AUTO"), Some("fp8"), Some("FP8")] {
2898 assert_eq!(
2899 resolve_inference_compute_kind(alias, device, channel).unwrap(),
2900 automatic
2901 );
2902 }
2903 for invalid in ["", "fp64", "float8", "surprise"] {
2904 assert!(
2905 resolve_inference_compute_kind(Some(invalid), device, channel).is_err()
2906 );
2907 }
2908 }
2909 }
2910 }
2911
2912 #[tokio::test]
2913 async fn decision_context_uses_actual_loaded_config_and_override() {
2914 let directory = tempfile::tempdir().unwrap();
2915 let path = directory.path().join("model.safetensors");
2916 let shapes = [
2917 ("model.embed_tokens.weight", vec![8, 4]),
2918 ("model.norm.weight", vec![4]),
2919 ("model.layers.0.input_layernorm.weight", vec![4]),
2920 ("model.layers.0.post_attention_layernorm.weight", vec![4]),
2921 ("model.layers.0.self_attn.q_proj.weight", vec![4, 4]),
2922 ("model.layers.0.self_attn.k_proj.weight", vec![4, 4]),
2923 ("model.layers.0.self_attn.v_proj.weight", vec![4, 4]),
2924 ("model.layers.0.self_attn.o_proj.weight", vec![4, 4]),
2925 ("model.layers.0.mlp.gate_proj.weight", vec![8, 4]),
2926 ("model.layers.0.mlp.up_proj.weight", vec![8, 4]),
2927 ("model.layers.0.mlp.down_proj.weight", vec![4, 8]),
2928 ];
2929 let weights: Vec<_> = shapes
2930 .into_iter()
2931 .map(|(key, shape)| {
2932 (
2933 key.to_owned(),
2934 Tensor::ones(shape.as_slice(), (tch::Kind::Float, Device::Cpu)),
2935 )
2936 })
2937 .collect();
2938 Tensor::write_safetensors(&weights, &path).unwrap();
2939 for (model_limit, override_limit, expected_limit, admitted_tokens) in [
2940 (64usize, None, 64usize, 64usize),
2941 (8192, None, 8192, 4097),
2942 (8192, Some(32), 32, 32),
2943 ] {
2944 let config = serde_json::json!({
2945 "model_type":"llama","architectures":["LlamaForCausalLM"],
2946 "hidden_size":4,"intermediate_size":8,"num_hidden_layers":1,
2947 "num_attention_heads":2,"num_key_value_heads":2,"head_dim":2,
2948 "vocab_size":8,"max_position_embeddings":model_limit,
2949 "rms_norm_eps":1e-5,"rope_theta":10000.0,"tie_word_embeddings":true
2950 });
2951 std::fs::write(
2952 directory.path().join("config.json"),
2953 serde_json::to_vec(&config).unwrap(),
2954 )
2955 .unwrap();
2956 let mut engine = TorchEngine::new(RuntimeConfig {
2957 use_gpu: false,
2958 context_length: 4096,
2959 max_context: override_limit,
2960 precision_mode: Some("fp32".to_owned()),
2961 speculative_decoding: false,
2962 ..Default::default()
2963 })
2964 .unwrap();
2965 assert!(engine.effective_context_limit().is_err());
2966 engine.load_model_file(&path).await.unwrap();
2967 assert_eq!(engine.effective_context_limit().unwrap(), expected_limit);
2968 assert_eq!(
2969 engine.context_state.lock().as_ref().unwrap().context_window,
2970 expected_limit
2971 );
2972 let overflow = vec![0i64; expected_limit + 1];
2973 let rejected = engine
2974 .with_decision_backbone_checked(&overflow, &|| Ok(()), |_, _| Ok(()))
2975 .unwrap_err();
2976 assert!(rejected.to_string().contains("exceeds context limit"));
2977 // Llama deliberately lacks decision hidden states. Reaching that
2978 // downstream error proves valid context was admitted without doing
2979 // a long forward. The8192case causally admits4097tokens, which the
2980 // previous RuntimeConfig default4096 incorrectly rejected.
2981 let admitted = vec![0i64; admitted_tokens];
2982 let downstream = engine
2983 .with_decision_backbone_checked(&admitted, &|| Ok(()), |_, _| Ok(()))
2984 .unwrap_err();
2985 assert!(downstream
2986 .to_string()
2987 .contains("decision hidden states unsupported"));
2988 }
2989 }
2990
2991 #[test]
2992 fn mtp_bonus_sampling_matches_serial_repetition_window() {
2993 let engine = TorchEngine::new(RuntimeConfig {
2994 use_gpu: false,
2995 speculative_decoding: false,
2996 ..Default::default()
2997 })
2998 .unwrap();
2999 const TOKENIZER: &str = r#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"oldest":0,"recent":1,"accepted":2,"[UNK]":3},"unk_token":"[UNK]"}}"#;
3000 *engine.tokenizer.lock() = Some(Tokenizer::from_bytes(TOKENIZER.as_bytes()).unwrap());
3001 let request = GenerationRequest {
3002 prompt: "recent".into(),
3003 temperature: Some(0.0),
3004 repeat_penalty: Some(2.0),
3005 repeat_last_n: Some(2),
3006 ..Default::default()
3007 };
3008 let mut stream = TextStream::new(&engine, request).unwrap();
3009 stream.recent_tokens = VecDeque::from([0, 1]);
3010 // The accepted first token makes serial evict token 0. Penalizing that
3011 // stale token changes the bonus argmax from 0 to 3 in this fixture.
3012 let logits = Tensor::from_slice(&[10.0_f32, 0.0, 0.0, 7.0]);
3013 let serial = engine
3014 .sample_token_with_params(
3015 &logits.copy(),
3016 &stream.sampling_params,
3017 &[1, 2],
3018 &stream.penalty_exempt_tokens,
3019 )
3020 .unwrap();
3021 let stale = engine
3022 .sample_token_with_params(
3023 &logits.copy(),
3024 &stream.sampling_params,
3025 &[0, 1, 2],
3026 &stream.penalty_exempt_tokens,
3027 )
3028 .unwrap();
3029 assert_ne!(stale, serial, "fixture must detect the pre-fix history");
3030 let bonus = stream.sample_spec_position(&logits.copy(), &[2]).unwrap();
3031 assert_eq!(bonus as usize, serial);
3032 assert_eq!(stream.recent_tokens_buffer, vec![1, 2]);
3033 assert_eq!(stream.recent_tokens, VecDeque::from([0, 1]));
3034 // More round-local tokens than the configured window still stay bounded.
3035 stream
3036 .sample_spec_position(&logits.copy(), &[2, 3, 1])
3037 .unwrap();
3038 assert_eq!(stream.recent_tokens_buffer, vec![3, 1]);
3039 }
3040
3041 // ===== #1253: prompt/token text must never reach process logs =====
3042
3043 /// Distinctive canary planted in prompt inputs; the assertions below prove
3044 /// it never appears in captured process logs.
3045 const LEAK_CANARY: &str = "PROMPTLEAKCANARY7q3z";
3046
3047 /// Minimal in-memory WordLevel tokenizer (no model files needed). The canary
3048 /// is a known vocab entry so it tokenizes cleanly on the success path; any
3049 /// other word maps to `[UNK]`, exercising the roundtrip-mismatch path.
3050 fn redact_test_tokenizer() -> Tokenizer {
3051 const TOK_JSON: &str = r#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"hello":0,"world":1,"[UNK]":3,"PROMPTLEAKCANARY7q3z":7},"unk_token":"[UNK]"}}"#;
3052 Tokenizer::from_bytes(TOK_JSON.as_bytes()).expect("valid in-memory tokenizer")
3053 }
3054
3055 /// `tracing_subscriber` writer that buffers all output into a shared Vec.
3056 struct CapturingWriter(Arc<Mutex<Vec<u8>>>);
3057 impl Write for CapturingWriter {
3058 fn write(&mut self, b: &[u8]) -> std::io::Result<usize> {
3059 self.0.lock().extend_from_slice(b);
3060 Ok(b.len())
3061 }
3062 fn flush(&mut self) -> std::io::Result<()> {
3063 Ok(())
3064 }
3065 }
3066
3067 /// Newtype wrapper so we can implement the foreign `MakeWriter` trait for a
3068 /// local type (orphan rule).
3069 #[derive(Clone)]
3070 struct CaptureSink(Arc<Mutex<Vec<u8>>>);
3071
3072 impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CaptureSink {
3073 type Writer = CapturingWriter;
3074 fn make_writer(&'a self) -> Self::Writer {
3075 CapturingWriter(self.0.clone())
3076 }
3077 }
3078
3079 // ===== Session-cache recording vs cancellation-before-first-poll =====
3080
3081 /// Minimal `ArchitectureConfig` for the recording stub below: no model
3082 /// math is exercised, only the configuration surface the trait requires.
3083 struct StubArchConfig;
3084 impl crate::runtime::architectures::ArchitectureConfig for StubArchConfig {
3085 fn num_attention_heads(&self) -> usize {
3086 4
3087 }
3088 fn num_key_value_heads(&self) -> usize {
3089 4
3090 }
3091 fn hidden_size(&self) -> usize {
3092 16
3093 }
3094 fn intermediate_size(&self) -> usize {
3095 32
3096 }
3097 fn vocab_size(&self) -> usize {
3098 16
3099 }
3100 fn max_position_embeddings(&self) -> usize {
3101 64
3102 }
3103 fn rope_theta(&self) -> Option<f32> {
3104 None
3105 }
3106 fn rope_dim(&self) -> Option<usize> {
3107 None
3108 }
3109 fn layer_norm_eps(&self) -> f32 {
3110 1e-5
3111 }
3112 fn use_rms_norm(&self) -> bool {
3113 true
3114 }
3115 }
3116
3117 /// Model stub for cache-recording tests: holds a real (empty)
3118 /// `KVCacheManager` and serves it to the engine's save path via
3119 /// `get_kv_cache`. No forward is ever called — these tests exercise stream
3120 /// construction, drop, and cache recording only, so every compute method
3121 /// stays on the trait's `Err`/no-op defaults.
3122 struct RecordingStubModel {
3123 cache: Option<Arc<Mutex<crate::runtime::KVCacheManager>>>,
3124 }
3125 impl ModelOperations for RecordingStubModel {
3126 fn architecture(&self) -> crate::runtime::architectures::ModelArchitecture {
3127 crate::runtime::architectures::ModelArchitecture::Llama { version: 3 }
3128 }
3129 fn config(&self) -> &dyn crate::runtime::architectures::ArchitectureConfig {
3130 &StubArchConfig
3131 }
3132 fn forward(&self, _input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor> {
3133 Err(anyhow!("recording tests never poll the stream"))
3134 }
3135 fn reshape_for_attention(&self, tensor: &Tensor, _is_key_value: bool) -> Result<Tensor> {
3136 Ok(tensor.shallow_clone())
3137 }
3138 fn apply_rope(&self, tensor: &Tensor, _position_ids: &Tensor) -> Result<Tensor> {
3139 Ok(tensor.shallow_clone())
3140 }
3141 fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
3142 Ok(tensor.shallow_clone())
3143 }
3144 fn get_attention_mask(&self, _seq_len: usize, _past_kv_len: usize) -> Result<Tensor> {
3145 Err(anyhow!("recording tests never poll the stream"))
3146 }
3147 fn as_any(&self) -> &dyn std::any::Any {
3148 self
3149 }
3150 fn get_kv_cache(&self) -> Option<Arc<Mutex<crate::runtime::KVCacheManager>>> {
3151 self.cache.clone()
3152 }
3153 fn set_kv_cache(&mut self, cache: Arc<Mutex<crate::runtime::KVCacheManager>>) {
3154 self.cache = Some(cache);
3155 }
3156 fn clear_kv_cache(&self) {
3157 // Same shape as the real architectures: route the clear through
3158 // the installed KVCacheManager so the metadata contract under
3159 // test is the production `clear_all` path.
3160 if let Some(cache) = &self.cache {
3161 let mut guard = cache.lock();
3162 guard.clear_all();
3163 }
3164 }
3165 }
3166
3167 /// Regression (cancellation before first poll): dropping a stream that was
3168 /// never polled must NOT record its prompt IDs into the session cache.
3169 /// Exact-hit/miss resolution in `TextStream::new` clears the cache at
3170 /// generation setup, before any forward — recording IDs over that empty
3171 /// state would let a later extending prompt claim a prefix_len against KV
3172 /// that does not exist (the pure-attention hit path would partial-prefill
3173 /// from mid-prompt with no history behind it). `prefill_time_ms` is Some
3174 /// exactly when the first prefill forward completed, so it gates recording.
3175 #[test]
3176 fn drop_before_first_poll_does_not_record_session_cache() {
3177 let mut engine = TorchEngine::new(RuntimeConfig {
3178 use_gpu: false,
3179 ..RuntimeConfig::default()
3180 })
3181 .expect("CPU engine should construct without a model");
3182
3183 // In-memory tokenizer (same pattern as the log-redaction tests);
3184 // "hello world" → [0, 1].
3185 *engine.tokenizer.lock() = Some(redact_test_tokenizer());
3186
3187 // A real empty KV cache served by the stub model — the recording target.
3188 let cache: Arc<Mutex<crate::runtime::KVCacheManager>> = Arc::new(Mutex::new(
3189 crate::runtime::KVCacheManager::new(2, 64, crate::runtime::KVQuantType::None),
3190 ));
3191 engine.persistent_model = Some(Arc::new(Mutex::new(Box::new(RecordingStubModel {
3192 cache: Some(cache.clone()),
3193 }))));
3194
3195 let request = GenerationRequest {
3196 prompt: "hello world".to_owned(),
3197 ..Default::default()
3198 };
3199 let stream = TextStream::new(&engine, request).expect("stream should construct");
3200 // Cancellation: dropped before the first poll — no prefill ever ran.
3201 drop(stream);
3202
3203 assert_eq!(
3204 cache.lock().cached_token_count(),
3205 0,
3206 "a drop before the first poll must not record prompt IDs over the cleared cache"
3207 );
3208
3209 // The downstream hazard: a later extending prompt must not claim a
3210 // reusable prefix over state that was never prefilled.
3211 let extended = vec![0i64, 1, 5, 9];
3212 assert_eq!(
3213 cache.lock().prefix_match_len(&extended),
3214 0,
3215 "no prefix may be claimed over cache state that was never computed"
3216 );
3217 }
3218
3219 /// Regression (prior-session metadata vs cancellation): a cache populated
3220 /// by an earlier turn (prompt IDs + end-of-prefill SSM snapshot) must not
3221 /// retain that metadata after the set-up clear that a later turn's exact
3222 /// hit or miss performs at generation setup. That clear runs before any
3223 /// forward; if the turn is then cancelled before its first poll, nothing
3224 /// new is recorded — so a subsequent extending prompt would otherwise
3225 /// claim a prefix (and, for hybrid models, restore a stale snapshot)
3226 /// against KV that no longer exists.
3227 #[test]
3228 fn setup_clear_drops_prior_session_metadata_on_cancelled_turn() {
3229 let mut engine = TorchEngine::new(RuntimeConfig {
3230 use_gpu: false,
3231 ..RuntimeConfig::default()
3232 })
3233 .expect("CPU engine should construct without a model");
3234 *engine.tokenizer.lock() = Some(redact_test_tokenizer());
3235
3236 // Registry + session owner so the swap path engages: the set-up clear
3237 // then runs through resolve_session_prefill_start exactly as the
3238 // production prefix-detection branch drives it.
3239 let registry = Arc::new(crate::runtime::kv_cache::KVCacheRegistry::new(
3240 crate::runtime::kv_cache::CacheConfig {
3241 num_layers: 2,
3242 max_seq_len: 64,
3243 quant_type: crate::runtime::KVQuantType::None,
3244 paged: false,
3245 },
3246 None,
3247 ));
3248 let owner = crate::runtime::kv_cache::CacheOwner::Session("s1".to_owned());
3249 let prior = registry.get_or_create(owner.clone());
3250 // Stamp the cache compatibly so the #1277 reuse policy keeps it and
3251 // prefix detection actually sees the prior metadata.
3252 let mut desc = crate::runtime::kv_compat::KvCompatDescriptor::default();
3253 desc.weights.base_revision = "test-base".to_owned();
3254 desc.set_tokenizer(16, Some("test-tok-hash".to_owned()));
3255 *engine.kv_compat.lock() = Some(desc);
3256 {
3257 let mut prior_guard = prior.lock();
3258 prior_guard.set_compat_fingerprint(engine.kv_compat_fingerprint_for(0).unwrap());
3259 let opt = (tch::Kind::Float, Device::Cpu);
3260 let conv = vec![Some(Tensor::zeros([2, 3], opt)), None];
3261 let rec = vec![Some(Tensor::zeros([4], opt)), None];
3262 // Prior turn's prompt [0, 1, 7] strictly extends this turn's
3263 // ("hello world" → [0, 1]): the request is an exact hit, whose
3264 // set-up clear routes through resolve_session_prefill_start →
3265 // model.clear_kv_cache() — the production boundary under test.
3266 prior_guard.set_cached_tokens_with_ssm(vec![0, 1, 7], Some((conv, rec)));
3267 }
3268 engine.kv_cache_registry = Some(registry);
3269 *engine.active_cache_owner.lock() = Some(owner);
3270 engine.persistent_model = Some(Arc::new(Mutex::new(Box::new(RecordingStubModel {
3271 cache: None,
3272 }))));
3273
3274 let request = GenerationRequest {
3275 prompt: "hello world".to_owned(),
3276 ..Default::default()
3277 };
3278 let stream = TextStream::new(&engine, request).expect("stream should construct");
3279 // Cancelled before the first poll: the set-up clear has run, and no
3280 // new recording can happen.
3281 drop(stream);
3282
3283 // `prior` is the same Arc the swap installed into the model.
3284 let guard = prior.lock();
3285 assert_eq!(
3286 guard.cached_token_count(),
3287 0,
3288 "prior prompt IDs must not survive the set-up clear"
3289 );
3290 assert!(
3291 guard.ssm_snapshot().is_none(),
3292 "prior end-of-prefill SSM snapshot must not survive the set-up clear"
3293 );
3294 assert_eq!(
3295 guard.prefix_match_len(&[0, 1, 7, 4]),
3296 0,
3297 "an extending prompt must not claim a prefix over cleared KV"
3298 );
3299 }
3300
3301 /// Run `f` under a thread-local TRACE subscriber whose output is captured.
3302 /// TRACE is deliberate: #1253 forbids relocating prompt text to DEBUG, so
3303 /// the canary must be absent even at the lowest log level.
3304 fn capture_logs<F: FnOnce()>(f: F) -> String {
3305 let sink = CaptureSink(Arc::new(Mutex::new(Vec::new())));
3306 let subscriber = tracing_subscriber::fmt()
3307 .with_max_level(tracing::Level::TRACE)
3308 .with_ansi(false)
3309 .with_writer(sink.clone())
3310 .finish();
3311 let _guard = tracing::subscriber::set_default(subscriber);
3312 f();
3313 // Drain the captured bytes into a local so the MutexGuard drops before
3314 // `sink` itself (block-end drop order).
3315 let cells = std::mem::take(&mut *sink.0.lock());
3316 String::from_utf8(cells).unwrap()
3317 }
3318
3319 /// #1253: a distinctive prompt secret must never appear in process logs,
3320 /// regardless of tokenization outcome — success, empty-token failure, or
3321 /// roundtrip mismatch.
3322 ///
3323 /// (Generation, stop, cancellation, and stream-error paths were audited:
3324 /// after the #1261 redaction they log only positions/counts and error
3325 /// objects — never prompt text, generated text, or token IDs, which are
3326 /// reversible to text via the loaded tokenizer. Those paths require a full
3327 /// model to drive, so the content-bearing tokenization outcomes — which are
3328 /// exactly where the prompt is handled — are covered hermetically here.)
3329 #[test]
3330 fn tokenize_redacted_never_logs_prompt_content() {
3331 let tokenizer = redact_test_tokenizer();
3332
3333 // --- success: every word is a known token ---
3334 let logs = capture_logs(|| {
3335 let ids = tokenize_redacted(&tokenizer, &format!("hello {LEAK_CANARY} world"));
3336 assert_eq!(ids.unwrap().len(), 3);
3337 });
3338 assert!(
3339 !logs.contains("Tokenized prompt"),
3340 "success path must no longer emit a metering log line (token counts are OTel metrics now, #1261): {logs}"
3341 );
3342 assert!(
3343 !logs.contains(LEAK_CANARY),
3344 "prompt secret leaked into logs on success: {logs}"
3345 );
3346
3347 // --- roundtrip mismatch: unknown word decodes to [UNK] ---
3348 let logs = capture_logs(|| {
3349 let ids = tokenize_redacted(&tokenizer, &format!("{LEAK_CANARY} zzz_unknown"));
3350 // non-empty -> Ok, but the decode diverges -> mismatch warning
3351 assert_eq!(ids.unwrap().len(), 2);
3352 });
3353 assert!(
3354 logs.contains("roundtrip mismatch"),
3355 "mismatch path must still emit bounded metadata: {logs}"
3356 );
3357 assert!(
3358 !logs.contains(LEAK_CANARY),
3359 "prompt secret leaked into logs on mismatch: {logs}"
3360 );
3361 assert!(
3362 !logs.contains("zzz_unknown"),
3363 "decoded token text leaked into logs on mismatch: {logs}"
3364 );
3365
3366 // --- empty-token failure: 0 tokens (pure whitespace input) ---
3367 // An input that carries the canary always yields >=1 token with this
3368 // tokenizer, so the failure path is exercised with whitespace; we still
3369 // assert it logs only bounded metadata ("empty token sequence") and no
3370 // prompt content.
3371 let logs = capture_logs(|| {
3372 let res = tokenize_redacted(&tokenizer, " ");
3373 assert!(res.is_err(), "whitespace-only input must fail");
3374 });
3375 assert!(
3376 logs.contains("empty token sequence"),
3377 "failure path must still emit bounded metadata: {logs}"
3378 );
3379 }
3380
3381 // ---- base-weight content digest (#1277, finding F1) ----
3382 //
3383 // The pure framing/digest helpers live in `kv_compat`; these cover the engine
3384 // seam: loader-selected shard-set resolution (extra/unreferenced `.safetensors`
3385 // files are never hashed), fail-closed selection, resolved-byte hashing
3386 // (incl. LFS/XET pointer handling), determinism, and parity with the pure
3387 // helpers.
3388
3389 use crate::runtime::kv_compat;
3390
3391 /// Write `bytes` to `dir/shard_name`.
3392 fn write_shard(dir: &std::path::Path, shard_name: &str, bytes: &[u8]) -> std::path::PathBuf {
3393 use std::io::Write;
3394 let p = dir.join(shard_name);
3395 let mut f = std::fs::File::create(&p).unwrap();
3396 f.write_all(bytes).unwrap();
3397 p
3398 }
3399
3400 /// Write a minimal `model.safetensors.index.json` whose `weight_map`
3401 /// references exactly `shard_names`.
3402 fn write_index(dir: &std::path::Path, shard_names: &[&str]) {
3403 let weight_map: serde_json::Map<String, serde_json::Value> = shard_names
3404 .iter()
3405 .enumerate()
3406 .map(|(i, name)| {
3407 (
3408 format!("tensor.{i}"),
3409 serde_json::Value::String((*name).to_owned()),
3410 )
3411 })
3412 .collect();
3413 let index = serde_json::json!({ "weight_map": weight_map });
3414 std::fs::write(
3415 dir.join("model.safetensors.index.json"),
3416 serde_json::to_string_pretty(&index).unwrap(),
3417 )
3418 .unwrap();
3419 }
3420
3421 #[tokio::test]
3422 async fn weights_content_digest_is_deterministic_and_matches_pure() {
3423 let dir = tempfile::tempdir().unwrap();
3424 write_shard(
3425 dir.path(),
3426 "model-00001-of-00002.safetensors",
3427 b"AAAA-content",
3428 );
3429 write_shard(
3430 dir.path(),
3431 "model-00002-of-00002.safetensors",
3432 b"BBBB-content",
3433 );
3434 write_index(
3435 dir.path(),
3436 &[
3437 "model-00001-of-00002.safetensors",
3438 "model-00002-of-00002.safetensors",
3439 ],
3440 );
3441 let a = TorchEngine::weights_content_digest(dir.path())
3442 .await
3443 .unwrap();
3444 let b = TorchEngine::weights_content_digest(dir.path())
3445 .await
3446 .unwrap();
3447 assert_eq!(a, b, "digest must be deterministic across runs");
3448 // Parity with the pure framing helper (engine resolves+hashes, pure fn frames).
3449 let expected = kv_compat::base_weight_digest_from_shards(&[
3450 (
3451 "model-00001-of-00002.safetensors".into(),
3452 kv_compat::shard_content_digest(b"AAAA-content"),
3453 ),
3454 (
3455 "model-00002-of-00002.safetensors".into(),
3456 kv_compat::shard_content_digest(b"BBBB-content"),
3457 ),
3458 ]);
3459 assert_eq!(
3460 a, expected,
3461 "engine digest must equal the pure framing over resolved bytes"
3462 );
3463 }
3464
3465 #[tokio::test]
3466 async fn weight_digest_reads_sealed_tree_after_checkout_mutation() {
3467 let dir = tempfile::tempdir().unwrap();
3468 let repo = git2db::Repository::init(dir.path()).unwrap();
3469 repo.config().unwrap().set_str("user.name", "test").unwrap();
3470 repo.config()
3471 .unwrap()
3472 .set_str("user.email", "test@example.invalid")
3473 .unwrap();
3474 write_shard(dir.path(), "model.safetensors", b"reviewed weights");
3475 let mut index = repo.index().unwrap();
3476 index
3477 .add_path(std::path::Path::new("model.safetensors"))
3478 .unwrap();
3479 index.write().unwrap();
3480 let tree = repo.find_tree(index.write_tree().unwrap()).unwrap();
3481 let sig = repo.signature().unwrap();
3482 let commit = repo
3483 .commit(Some("HEAD"), &sig, &sig, "reviewed", &tree, &[])
3484 .unwrap();
3485 drop(tree);
3486 drop(repo);
3487
3488 let artifact = git2db::pinned_tree::PinnedTree::acquire(dir.path(), commit)
3489 .await
3490 .unwrap()
3491 .into_sealed_projection()
3492 .unwrap();
3493 let baseline = TorchEngine::weights_content_digest(artifact.root())
3494 .await
3495 .unwrap();
3496 std::fs::write(dir.path().join("model.safetensors"), b"replacement weights").unwrap();
3497 let after = TorchEngine::weights_content_digest(artifact.root())
3498 .await
3499 .unwrap();
3500 assert_eq!(baseline, after);
3501 assert_ne!(
3502 baseline,
3503 TorchEngine::weights_content_digest(dir.path())
3504 .await
3505 .unwrap()
3506 );
3507 }
3508
3509 #[tokio::test]
3510 async fn weights_content_digest_same_basename_different_bytes_diverge() {
3511 let d1 = tempfile::tempdir().unwrap();
3512 let d2 = tempfile::tempdir().unwrap();
3513 write_shard(d1.path(), "model-00001-of-00002.safetensors", b"content-A");
3514 write_shard(d1.path(), "model-00002-of-00002.safetensors", b"content-B");
3515 write_index(
3516 d1.path(),
3517 &[
3518 "model-00001-of-00002.safetensors",
3519 "model-00002-of-00002.safetensors",
3520 ],
3521 );
3522 write_shard(d2.path(), "model-00001-of-00002.safetensors", b"content-A!");
3523 write_shard(d2.path(), "model-00002-of-00002.safetensors", b"content-B");
3524 write_index(
3525 d2.path(),
3526 &[
3527 "model-00001-of-00002.safetensors",
3528 "model-00002-of-00002.safetensors",
3529 ],
3530 );
3531 let a = TorchEngine::weights_content_digest(d1.path())
3532 .await
3533 .unwrap();
3534 let b = TorchEngine::weights_content_digest(d2.path())
3535 .await
3536 .unwrap();
3537 assert_ne!(a, b, "same basenames, different content must diverge");
3538 }
3539
3540 /// Two shard sets equal as a (name → content) map but enumerated in
3541 /// different creation order must produce the same digest (sorted determinism).
3542 #[tokio::test]
3543 async fn weights_content_digest_is_order_independent() {
3544 let d1 = tempfile::tempdir().unwrap();
3545 let d2 = tempfile::tempdir().unwrap();
3546 write_shard(d1.path(), "model-00001-of-00002.safetensors", b"X");
3547 write_shard(d1.path(), "model-00002-of-00002.safetensors", b"Y");
3548 write_index(
3549 d1.path(),
3550 &[
3551 "model-00001-of-00002.safetensors",
3552 "model-00002-of-00002.safetensors",
3553 ],
3554 );
3555 // Reverse creation order in d2.
3556 write_shard(d2.path(), "model-00002-of-00002.safetensors", b"Y");
3557 write_shard(d2.path(), "model-00001-of-00002.safetensors", b"X");
3558 write_index(
3559 d2.path(),
3560 &[
3561 "model-00001-of-00002.safetensors",
3562 "model-00002-of-00002.safetensors",
3563 ],
3564 );
3565 let a = TorchEngine::weights_content_digest(d1.path())
3566 .await
3567 .unwrap();
3568 let b = TorchEngine::weights_content_digest(d2.path())
3569 .await
3570 .unwrap();
3571 assert_eq!(a, b, "digest must be independent of enumeration order");
3572 }
3573
3574 #[tokio::test]
3575 async fn weights_content_digest_empty_dir_fails_closed() {
3576 let dir = tempfile::tempdir().unwrap();
3577 assert!(
3578 TorchEngine::weights_content_digest(dir.path())
3579 .await
3580 .is_err(),
3581 "an empty model dir must fail closed, not mint a partial digest"
3582 );
3583 }
3584
3585 #[tokio::test]
3586 async fn weights_content_digest_no_safetensors_fails_closed() {
3587 let dir = tempfile::tempdir().unwrap();
3588 write_shard(dir.path(), "config.json", b"{}");
3589 write_shard(dir.path(), "tokenizer.json", b"{}");
3590 assert!(
3591 TorchEngine::weights_content_digest(dir.path())
3592 .await
3593 .is_err(),
3594 "a dir with no .safetensors shards must fail closed"
3595 );
3596 }
3597
3598 /// A small (<1 KiB) ordinary shard that is NOT an LFS pointer must be hashed
3599 /// as content (not skipped, not treated as a pointer).
3600 #[tokio::test]
3601 async fn weights_content_digest_small_plain_shard_hashed_as_content() {
3602 let dir = tempfile::tempdir().unwrap();
3603 let bytes = b"tiny-but-real-weights";
3604 write_shard(dir.path(), "model.safetensors", bytes);
3605 let digest = TorchEngine::weights_content_digest(dir.path())
3606 .await
3607 .unwrap();
3608 let expected = kv_compat::base_weight_digest_from_shards(&[(
3609 "model.safetensors".into(),
3610 kv_compat::shard_content_digest(bytes),
3611 )]);
3612 assert_eq!(
3613 digest, expected,
3614 "a small non-pointer shard must hash its content bytes"
3615 );
3616 }
3617
3618 /// An un-smudged LFS/XET pointer shard must fail closed (never be hashed as a
3619 /// pointer stub). With no resolvable object store in the test environment,
3620 /// resolution errors out → the digest declines.
3621 #[tokio::test]
3622 async fn weights_content_digest_pointer_fails_closed() {
3623 let dir = tempfile::tempdir().unwrap();
3624 // A canonical LFS pointer header (<1 KiB). It is not resolvable here.
3625 let pointer = b"version https://git-lfs.github.com/spec/v1\noid sha256:0000000000000000000000000000000000000000000000000000000000000000\nsize 1234\n";
3626 write_shard(dir.path(), "model.safetensors", pointer);
3627 assert!(
3628 TorchEngine::weights_content_digest(dir.path())
3629 .await
3630 .is_err(),
3631 "an un-smudged LFS pointer must fail closed, not be hashed as a stub"
3632 );
3633 }
3634
3635 /// An unreadable shard must fail closed (read error propagates), not produce
3636 /// a digest over the readable remainder. Skipped when running as root (root
3637 /// bypasses the 0o000 permission).
3638 #[cfg(unix)]
3639 #[tokio::test]
3640 async fn weights_content_digest_unreadable_shard_fails_closed() {
3641 // Root bypasses the 0o000 permission, so the read would succeed and the
3642 // fail-closed assertion could not hold — skip silently under root.
3643 if unsafe { libc::geteuid() } == 0 {
3644 return;
3645 }
3646 let dir = tempfile::tempdir().unwrap();
3647 let p = write_shard(dir.path(), "model.safetensors", b"real-content");
3648 use std::os::unix::fs::PermissionsExt;
3649 std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o000)).unwrap();
3650 let res = TorchEngine::weights_content_digest(dir.path()).await;
3651 // Restore so tempdir cleanup can remove it.
3652 let _ = std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o644));
3653 assert!(
3654 res.is_err(),
3655 "an unreadable shard must fail closed, not hash a partial set"
3656 );
3657 }
3658
3659 /// An extra/unreferenced `.safetensors` in the directory must NOT shift the
3660 /// digest: only the index-referenced (loader-selected) shards are hashed.
3661 #[tokio::test]
3662 async fn weights_content_digest_ignores_unreferenced_extra_shard() {
3663 let dir = tempfile::tempdir().unwrap();
3664 write_shard(
3665 dir.path(),
3666 "model-00001-of-00002.safetensors",
3667 b"AAAA-content",
3668 );
3669 write_shard(
3670 dir.path(),
3671 "model-00002-of-00002.safetensors",
3672 b"BBBB-content",
3673 );
3674 write_index(
3675 dir.path(),
3676 &[
3677 "model-00001-of-00002.safetensors",
3678 "model-00002-of-00002.safetensors",
3679 ],
3680 );
3681 let selected_only = TorchEngine::weights_content_digest(dir.path())
3682 .await
3683 .unwrap();
3684
3685 // A stale shard with a shard-like name the loader never references.
3686 write_shard(
3687 dir.path(),
3688 "model-00003-of-00002.safetensors",
3689 b"STALE-alt-precision",
3690 );
3691 let with_extra = TorchEngine::weights_content_digest(dir.path())
3692 .await
3693 .unwrap();
3694
3695 assert_eq!(
3696 selected_only, with_extra,
3697 "an unreferenced extra shard must be invariant to the digest"
3698 );
3699 let expected = kv_compat::base_weight_digest_from_shards(&[
3700 (
3701 "model-00001-of-00002.safetensors".into(),
3702 kv_compat::shard_content_digest(b"AAAA-content"),
3703 ),
3704 (
3705 "model-00002-of-00002.safetensors".into(),
3706 kv_compat::shard_content_digest(b"BBBB-content"),
3707 ),
3708 ]);
3709 assert_eq!(
3710 with_extra, expected,
3711 "digest must cover exactly the loader-selected set, no more"
3712 );
3713 }
3714
3715 /// With a single `model.safetensors`, an extra shard-like file is not part
3716 /// of the loader's selection and must not shift the digest.
3717 #[tokio::test]
3718 async fn weights_content_digest_single_file_ignores_extra_shard() {
3719 let dir = tempfile::tempdir().unwrap();
3720 write_shard(dir.path(), "model.safetensors", b"single-file-weights");
3721 let baseline = TorchEngine::weights_content_digest(dir.path())
3722 .await
3723 .unwrap();
3724 let expected = kv_compat::base_weight_digest_from_shards(&[(
3725 "model.safetensors".into(),
3726 kv_compat::shard_content_digest(b"single-file-weights"),
3727 )]);
3728 assert_eq!(baseline, expected);
3729
3730 // Even a name matching the shard glob is not selected when
3731 // `model.safetensors` is the loader's pick.
3732 write_shard(
3733 dir.path(),
3734 "model-00001-of-00001.safetensors",
3735 b"UNREFERENCED",
3736 );
3737 let with_extra = TorchEngine::weights_content_digest(dir.path())
3738 .await
3739 .unwrap();
3740 assert_eq!(
3741 baseline, with_extra,
3742 "an extra shard next to model.safetensors must be invariant"
3743 );
3744 }
3745
3746 /// Changing a SELECTED shard changes the digest; adding, removing, or
3747 /// mutating an UNREFERENCED shard does not.
3748 #[tokio::test]
3749 async fn weights_content_digest_selected_mutations_diverge_unreferenced_do_not() {
3750 let dir = tempfile::tempdir().unwrap();
3751 write_shard(
3752 dir.path(),
3753 "model-00001-of-00002.safetensors",
3754 b"AAAA-content",
3755 );
3756 write_shard(
3757 dir.path(),
3758 "model-00002-of-00002.safetensors",
3759 b"BBBB-content",
3760 );
3761 write_index(
3762 dir.path(),
3763 &[
3764 "model-00001-of-00002.safetensors",
3765 "model-00002-of-00002.safetensors",
3766 ],
3767 );
3768 let baseline = TorchEngine::weights_content_digest(dir.path())
3769 .await
3770 .unwrap();
3771
3772 // Add an unreferenced shard.
3773 let extra = write_shard(dir.path(), "stale-backup.safetensors", b"stale");
3774 assert_eq!(
3775 TorchEngine::weights_content_digest(dir.path())
3776 .await
3777 .unwrap(),
3778 baseline,
3779 "adding an unreferenced shard must not change the digest"
3780 );
3781 // Mutate the unreferenced shard.
3782 write_shard(dir.path(), "stale-backup.safetensors", b"stale-CHANGED");
3783 assert_eq!(
3784 TorchEngine::weights_content_digest(dir.path())
3785 .await
3786 .unwrap(),
3787 baseline,
3788 "mutating an unreferenced shard must not change the digest"
3789 );
3790 // Remove the unreferenced shard.
3791 std::fs::remove_file(&extra).unwrap();
3792 assert_eq!(
3793 TorchEngine::weights_content_digest(dir.path())
3794 .await
3795 .unwrap(),
3796 baseline,
3797 "removing an unreferenced shard must not change the digest"
3798 );
3799 // Change a SELECTED shard.
3800 write_shard(
3801 dir.path(),
3802 "model-00002-of-00002.safetensors",
3803 b"BBBB-CHANGED",
3804 );
3805 assert_ne!(
3806 TorchEngine::weights_content_digest(dir.path())
3807 .await
3808 .unwrap(),
3809 baseline,
3810 "changing a selected shard must change the digest"
3811 );
3812 }
3813
3814 /// A directory whose only `.safetensors` files match no loader selection
3815 /// pattern has ZERO selected shards and must fail closed (reuse declined).
3816 #[tokio::test]
3817 async fn weights_content_digest_zero_selected_shards_fails_closed() {
3818 let dir = tempfile::tempdir().unwrap();
3819 write_shard(dir.path(), "backup.safetensors", b"not-a-selected-shard");
3820 write_shard(dir.path(), "optimizer.safetensors", b"also-not-selected");
3821 assert!(
3822 TorchEngine::weights_content_digest(dir.path())
3823 .await
3824 .is_err(),
3825 "a dir with no loader-selected shards must fail closed"
3826 );
3827 }
3828
3829 /// An index referencing a shard that does not exist on disk is an
3830 /// inconsistent selection and must fail closed (reuse declined).
3831 #[tokio::test]
3832 async fn weights_content_digest_missing_index_shard_fails_closed() {
3833 let dir = tempfile::tempdir().unwrap();
3834 write_shard(
3835 dir.path(),
3836 "model-00001-of-00002.safetensors",
3837 b"AAAA-content",
3838 );
3839 write_index(
3840 dir.path(),
3841 &[
3842 "model-00001-of-00002.safetensors",
3843 "model-00002-of-00002.safetensors", // referenced but absent
3844 ],
3845 );
3846 assert!(
3847 TorchEngine::weights_content_digest(dir.path())
3848 .await
3849 .is_err(),
3850 "an index referencing a missing shard must fail closed"
3851 );
3852 }
3853
3854 /// A manifest-less MULTI-shard glob selection is ambiguous (the glob can
3855 /// silently drop/duplicate shards), so the digest must fail closed —
3856 /// decline reuse — even though the LOADER only warns in its permissive
3857 /// default mode. Fingerprint authority is stricter than loader
3858 /// permissiveness (#1277 revise-4): this holds regardless of
3859 /// `HYPRSTREAM_STRICT_LOADER` (the test env does not set it, exercising
3860 /// the permissive loader path).
3861 #[tokio::test]
3862 async fn weights_content_digest_manifestless_multi_shard_fails_closed() {
3863 let dir = tempfile::tempdir().unwrap();
3864 write_shard(
3865 dir.path(),
3866 "model-00001-of-00002.safetensors",
3867 b"AAAA-content",
3868 );
3869 write_shard(
3870 dir.path(),
3871 "model-00002-of-00002.safetensors",
3872 b"BBBB-content",
3873 );
3874 // NO model.safetensors.index.json — the loader warns and continues;
3875 // the digest must refuse.
3876 assert!(
3877 TorchEngine::weights_content_digest(dir.path())
3878 .await
3879 .is_err(),
3880 "an ambiguous manifest-less multi-shard glob set must fail closed"
3881 );
3882 }
3883
3884 /// A SINGLE globbed shard (no index, no model.safetensors) is unambiguous
3885 /// and must still digest — only multi-shard glob sets are ambiguous.
3886 #[tokio::test]
3887 async fn weights_content_digest_single_globbed_shard_passes() {
3888 let dir = tempfile::tempdir().unwrap();
3889 write_shard(
3890 dir.path(),
3891 "model-00001-of-00001.safetensors",
3892 b"single-shard-content",
3893 );
3894 let digest = TorchEngine::weights_content_digest(dir.path())
3895 .await
3896 .unwrap();
3897 let expected = kv_compat::base_weight_digest_from_shards(&[(
3898 "model-00001-of-00001.safetensors".into(),
3899 kv_compat::shard_content_digest(b"single-shard-content"),
3900 )]);
3901 assert_eq!(
3902 digest, expected,
3903 "a single globbed shard is unambiguous and must digest"
3904 );
3905 }
3906
3907 #[test]
3908 fn test_generation_config_in_engine_has_correct_max_tokens() {
3909 // Create a minimal runtime config
3910 let _runtime_config = RuntimeConfig::default();
3911
3912 // Create engine (this will fail without LIBTORCH but we can test the constructor logic)
3913 // We're just verifying the default generation_config
3914 let gen_config = GenerationConfig {
3915 max_tokens: 2048,
3916 temperature: 0.7,
3917 top_p: 0.9,
3918 top_k: Some(40),
3919 repeat_penalty: 1.1,
3920 stop_tokens: vec!["</s>".to_owned()],
3921 seed: None,
3922 stream: false,
3923 };
3924
3925 // Verify the defaults we set
3926 assert_eq!(
3927 gen_config.max_tokens, 2048,
3928 "TorchEngine should initialize with max_tokens=2048"
3929 );
3930 }
3931
3932 /// `with_device(.., None)` — the single-device path used by every legacy
3933 /// caller — must produce an engine that reports single-device and exposes
3934 /// no pool. This is the regression guard that the #314 wiring does not
3935 /// perturb single-GPU/CPU construction.
3936 #[test]
3937 fn single_device_engine_is_not_multi_device() {
3938 let engine =
3939 super::TorchEngine::with_device(RuntimeConfig::default(), Device::Cpu, None).unwrap();
3940 assert!(!engine.is_multi_device());
3941 assert!(engine.device_pool().is_none());
3942 assert_eq!(engine.device(), Device::Cpu);
3943 }
3944
3945 /// #429: `release_device_resources` is the terminal teardown that drops
3946 /// CUDA tensors (weights, KV cache, thread-local RoPE tables) on the
3947 /// service thread at shutdown. It must:
3948 /// - be safe on an engine that never loaded a model (no-op for the
3949 /// weight/KV paths, but must still clear RoPE TLS),
3950 /// - be idempotent (double-call must not panic / double-free),
3951 /// - leave the engine visibly unloaded afterward.
3952 ///
3953 /// This runs on CPU (no CUDA context), so it can't reproduce the SIGABRT
3954 /// itself; it pins the *ordering contract* the GPU fix relies on. The
3955 /// thread-local RoPE caches are populated on whichever thread runs the
3956 /// forward pass — this test clears the test thread's own TLS, which is
3957 /// exactly what the production Shutdown handler does on the service thread.
3958 #[test]
3959 fn release_device_resources_is_safe_and_idempotent() {
3960 let mut engine =
3961 super::TorchEngine::with_device(RuntimeConfig::default(), Device::Cpu, None).unwrap();
3962
3963 // Never loaded a model — these start empty.
3964 assert!(!engine.has_varstore());
3965
3966 // First call: no model weights, no KV registry, no RoPE entries on this
3967 // thread — must still succeed and clear the (empty) RoPE TLS.
3968 engine.release_device_resources();
3969 assert!(
3970 !engine.has_varstore(),
3971 "an unloaded engine stays unloaded after teardown"
3972 );
3973
3974 // Second call: idempotent — must not panic or double-free.
3975 engine.release_device_resources();
3976 engine.release_device_resources();
3977
3978 // Dropping the engine afterward must also be clean (no double-drop of
3979 // the already-None persistent_model / var_store).
3980 drop(engine);
3981 }
3982
3983 /// #429: a loaded-model teardown must actually drop the VarStore contents.
3984 /// We can't load a real model without weights/libtorch-on-this-host, but we
3985 /// can set the dummy VarStore (the same step `load_model_file` performs) and
3986 /// confirm `release_device_resources` clears it — the contract the GPU path
3987 /// relies on to free weight tensors before CUDA context teardown.
3988 #[test]
3989 fn release_device_resources_clears_loaded_varstore() {
3990 let mut engine =
3991 super::TorchEngine::with_device(RuntimeConfig::default(), Device::Cpu, None).unwrap();
3992 // Mimic the load path: install a dummy VarStore on the engine's device.
3993 {
3994 let vs = VarStore::new(Device::Cpu);
3995 let mut guard = engine.var_store.lock();
3996 *guard = Some(vs);
3997 }
3998 assert!(engine.has_varstore());
3999
4000 engine.release_device_resources();
4001
4002 assert!(
4003 !engine.has_varstore(),
4004 "release_device_resources must drop the VarStore"
4005 );
4006 }
4007
4008 /// `with_device(.., Some(multi_pool))` must mark the engine multi-device and
4009 /// expose the pool with the right primary. We build the pool from explicit
4010 /// `Device::Cuda` values via `DevicePool::from_devices` (which does not probe
4011 /// CUDA), so this runs on CPU-only CI — we never construct a model here, we
4012 /// only exercise the engine-struct wiring (#314 selection logic).
4013 #[test]
4014 fn multi_device_pool_marks_engine_multi_device() {
4015 use crate::runtime::device_pool::DevicePool;
4016 let pool = DevicePool::from_devices(vec![Device::Cuda(0), Device::Cuda(1)]).unwrap();
4017 assert_eq!(pool.len(), 2);
4018 let primary = pool.primary();
4019 let engine = super::TorchEngine::with_device(
4020 RuntimeConfig::default(),
4021 primary,
4022 Some(std::sync::Arc::new(pool.clone())),
4023 )
4024 .unwrap();
4025 assert!(engine.is_multi_device());
4026 let exposed = engine
4027 .device_pool()
4028 .expect("multi-device engine must expose its pool");
4029 assert_eq!(exposed.devices(), pool.devices());
4030 assert_eq!(exposed.primary(), primary);
4031 assert_eq!(engine.device(), primary);
4032 }
4033
4034 /// The map the factory would build for a 2-device pool must be a contiguous
4035 /// prefix-heavy split with exactly one stage boundary — the invariant the
4036 /// `forward_layers` path relies on. This pins the planner the wiring hands
4037 /// to model construction (#314 → #310 epic).
4038 #[test]
4039 fn engine_multi_device_map_has_one_boundary_per_stage_gap() {
4040 use crate::runtime::device_pool::{DevicePool, LayerDeviceMap};
4041 let pool = DevicePool::from_devices(vec![Device::Cuda(0), Device::Cuda(1)]).unwrap();
4042 // 7 layers over 2 devices → [0,0,0,0, 1,1,1] (prefix gets the extra).
4043 let map = LayerDeviceMap::even_split(&pool, 7).unwrap();
4044 assert_eq!(map.len(), 7);
4045 let mut boundaries = 0;
4046 let mut prev = map.device_for(0);
4047 for g in 1..map.len() {
4048 if map.is_boundary(prev, g) {
4049 boundaries += 1;
4050 }
4051 prev = map.device_for(g);
4052 }
4053 assert_eq!(
4054 boundaries, 1,
4055 "2-device even_split has exactly one cross-device copy per forward"
4056 );
4057 assert!(!map.is_single_device());
4058 }
4059}
4060
4061/// Internal sampling parameters passed to TensorSampler
4062#[derive(Debug, Clone, Copy)]
4063struct SamplingParams {
4064 temperature: f32,
4065 top_p: f32,
4066 top_k: Option<usize>,
4067 repeat_penalty: f32,
4068}
4069
4070use futures::Stream;
4071use std::collections::VecDeque;
4072use std::pin::Pin;
4073use std::task::{Context as TaskContext, Poll};
4074
4075/// Statistics about text generation with separate prefill/inference metrics
4076#[derive(Debug, Clone)]
4077pub struct GenerationStats {
4078 // Overall metrics
4079 pub tokens_generated: usize,
4080 pub generation_time_ms: u64,
4081 pub tokens_per_second: f32,
4082 pub finish_reason: Option<FinishReason>,
4083 /// Quality metrics captured during generation (for self-supervised training)
4084 pub quality_metrics: Option<crate::runtime::generation_metrics::GenerationQualityMetrics>,
4085
4086 // Prefill metrics (processing the prompt)
4087 pub prefill_tokens: usize,
4088 pub prefill_time_ms: u64,
4089 pub prefill_tokens_per_sec: f32,
4090
4091 // Inference metrics (generating new tokens, excluding prefill)
4092 pub inference_tokens: usize,
4093 pub inference_time_ms: u64,
4094 /// Cumulative average: tokens / time (accurate for final reporting)
4095 pub inference_tokens_per_sec: f32,
4096 /// Exponential moving average (responsive for real-time adaptive batching)
4097 pub inference_tokens_per_sec_ema: f32,
4098
4099 // MTP self-speculative decoding outcomes (0 when speculation is off)
4100 /// Draft tokens accepted by greedy exact-match verify this request
4101 pub speculative_accepted: u64,
4102 /// Draft tokens rejected (and rewound) this request
4103 pub speculative_rejected: u64,
4104}
4105
4106/// Stream that yields decoded UTF-8 text chunks during generation.
4107///
4108/// Automatically handles UTF-8 buffering, stop tokens, EOS detection,
4109/// timeout handling, cancellation, and all generation complexities.
4110/// Just iterate and get text!
4111pub struct TextStream<'a> {
4112 engine: &'a TorchEngine,
4113
4114 /// Optional per-tenant delta for delta-aware inference.
4115 /// When present, LoRA corrections are injected at each attention layer.
4116 delta: Option<std::sync::Arc<parking_lot::Mutex<crate::training::TenantDelta>>>,
4117
4118 prompt_tokens: Vec<i64>,
4119 last_generated: Option<i64>,
4120
4121 recent_tokens: VecDeque<i64>,
4122 repeat_last_n: usize,
4123
4124 // PERF: Individual sampling params removed - now stored in sampling_params field below
4125 // This avoids struct recreation per token while keeping backward compatibility
4126 max_tokens: usize,
4127 stop_token_ids: Vec<u32>,
4128
4129 // Store tokenizer as Arc for safe sharing across streams
4130 // IMPORTANT: This field is required for the unsafe transmute in TextStream::new()
4131 // DO NOT REMOVE - it ensures the tokenizer lives long enough for the decode_stream lifetime 'a
4132 #[allow(dead_code)]
4133 tokenizer: Arc<Tokenizer>,
4134 decode_stream: tokenizers::tokenizer::DecodeStream<
4135 'a,
4136 tokenizers::models::ModelWrapper,
4137 tokenizers::normalizers::NormalizerWrapper,
4138 tokenizers::pre_tokenizers::PreTokenizerWrapper,
4139 tokenizers::processors::PostProcessorWrapper,
4140 tokenizers::decoders::DecoderWrapper,
4141 >,
4142
4143 prompt_len: usize,
4144 /// Position from which prefill should start (0 = full prefill, >0 = partial via prefix cache hit)
4145 prefill_start_pos: usize,
4146 /// SSM (conv/rec) state snapshot captured at end-of-prefill (hybrid
4147 /// recurrent models only; `None` for pure-attention models). Saved into
4148 /// the session cache by `save_cached_tokens` so a later prefix hit can
4149 /// rewind the recurrent state alongside the KV truncation
4150 /// (`rewind_session_state`).
4151 prefill_ssm_snapshot: Option<(Vec<Option<Tensor>>, Vec<Option<Tensor>>)>,
4152 /// KV cache position tracking for this stream
4153 /// Each stream has exclusive access via &mut self, so no atomic needed
4154 kv_cache_position: usize,
4155 /// Total tokens generated (including buffered UTF-8) - for statistics only
4156 tokens_generated: usize,
4157 start_time: std::time::Instant,
4158 finished: bool,
4159 finish_reason: Option<FinishReason>,
4160
4161 // Timeout handling
4162 timeout_ms: Option<u64>,
4163
4164 /// Whether to collect per-token quality metrics (expensive - ~10x overhead)
4165 collect_metrics: bool,
4166
4167 /// Metrics accumulator for self-supervised training quality signals
4168 metrics_accumulator: crate::runtime::generation_metrics::GenerationMetricsAccumulator,
4169
4170 // PERF: Cached values to avoid per-token recomputation
4171 /// Pre-created sampling params - avoids struct creation per token
4172 sampling_params: SamplingParams,
4173 /// Cached tokenizer vocab size - avoids lock acquisition per token
4174 vocab_size: usize,
4175 /// Cached model vocab size (from logits shape) - set after first forward
4176 model_vocab_size: usize,
4177 /// Reusable buffer for recent_tokens when VecDeque wraps around
4178 recent_tokens_buffer: Vec<i64>,
4179 /// Token IDs exempt from repeat penalty (single-character tokens like digits 0-9,
4180 /// punctuation, etc.) These tokens appear frequently in number generation and
4181 /// should not be penalized just for reoccurring.
4182 penalty_exempt_tokens: HashSet<i64>,
4183
4184 // Timing for prefill/inference separation
4185 /// Time spent on prefill (processing prompt), set after first forward pass
4186 prefill_time_ms: Option<u64>,
4187 /// Timestamp when first token was sampled (after prefill completes)
4188 first_token_time: Option<std::time::Instant>,
4189
4190 // EMA rate tracking for adaptive batching
4191 /// Timestamp when the last token was generated (for inter-token timing)
4192 last_token_time: Option<std::time::Instant>,
4193 /// Exponential moving average of tokens per second
4194 ema_tokens_per_sec: f32,
4195
4196 /// Model label for OTel token-burn metric attributes (#1261).
4197 model_label: String,
4198 /// Opaque (hashed) tenant id for token-burn metric attributes — never the
4199 /// raw request subject (#1253). `None` when no tenant is in scope.
4200 tenant_hash: Option<u64>,
4201
4202 // MTP self-speculative decoding state (v1: greedy, batch=1, dense Qwen3.5).
4203 /// Whether this stream decodes via MTP draft + verify. Decided once at
4204 /// construction from the engine config, model capability, and request
4205 /// sampling params (see `new_with_delta`).
4206 speculative: bool,
4207 /// Draft token proposed by the MTP head for the position after
4208 /// `last_generated`; always `Some` between speculative decode rounds.
4209 spec_pending_draft: Option<u32>,
4210 /// Accepted tokens from the current speculative round not yet consumed by
4211 /// `poll_next` (1 on reject, up to 2 on accept).
4212 spec_out_queue: VecDeque<u32>,
4213 /// Draft outcome counters for this request (OTel `record_speculative` at
4214 /// drop + `GenerationStats`). Plain fields, never gated on collect_metrics.
4215 spec_accepted: u64,
4216 spec_rejected: u64,
4217}
4218
4219impl<'a> TextStream<'a> {
4220 fn new(engine: &'a TorchEngine, request: GenerationRequest) -> Result<Self> {
4221 // No delta path: no tenant dependency and adapter generation 0 (base
4222 // weights only).
4223 Self::new_with_delta(engine, request, None, None, 0, 0)
4224 }
4225
4226 fn new_with_delta(
4227 engine: &'a TorchEngine,
4228 request: GenerationRequest,
4229 delta: Option<std::sync::Arc<parking_lot::Mutex<crate::training::TenantDelta>>>,
4230 tenant_id: Option<String>,
4231 weight_epoch: u64,
4232 adapter_generation: u64,
4233 ) -> Result<Self> {
4234 let prompt_tokens = engine.tokenize(&request.prompt)?;
4235 let prompt_len = prompt_tokens.len();
4236
4237 // Token-burn metering (#1261): emit the prompt-token count at the
4238 // tokenization boundary as an OTel metric. The raw tenant subject is never
4239 // a metric attribute — it is reduced to an opaque hash (#1253); the
4240 // authoritative spend ledger is #1264.
4241 let tenant_hash = crate::runtime::token_metrics::opaque_tenant_id(&tenant_id);
4242 let model_label = engine.model_label();
4243 crate::runtime::token_metrics::global_meter().record_prompt(
4244 &model_label,
4245 tenant_hash,
4246 prompt_len as u64,
4247 );
4248
4249 let tokenizer = engine.get_tokenizer()?;
4250 let stop_token_ids: Vec<u32> = request.stop_tokens.as_deref().unwrap_or(&[])
4251 .iter()
4252 .filter_map(|stop_str| {
4253 let encoding = tokenizer.encode(stop_str.as_str(), false).ok()?;
4254 let ids = encoding.get_ids();
4255 if ids.len() == 1 {
4256 Some(ids[0])
4257 } else {
4258 // The stop sequence is caller-supplied content and is never
4259 // logged (#1253); only its byte length and resulting token
4260 // count are retained. The tenant is referenced only by its
4261 // opaque hash — the raw subject is dropped from logs.
4262 tracing::warn!(
4263 stop_bytes = stop_str.len(),
4264 encoded_tokens = ids.len(),
4265 tenant_hash = ?tenant_hash,
4266 "Stop token encodes to multiple tokens, skipping (only single-token stops supported; content redacted)"
4267 );
4268 None
4269 }
4270 })
4271 .collect();
4272
4273 // Evict LRU session caches if memory budget is exceeded.
4274 // This must happen before allocating/swapping in a new cache.
4275 engine.evict_session_caches();
4276
4277 // Session cache management with prefix detection:
4278 // If an active session exists, swap its KV cache into the model and check
4279 // if the new prompt shares a prefix with the cached tokens. This avoids
4280 // re-computing attention for unchanged conversation history.
4281 let prefill_start_pos = if engine.swap_session_cache(tenant_id, weight_epoch) {
4282 // Session cache swapped in — check for prefix match
4283 let prefix_len = if let Some(model_arc) = &engine.persistent_model {
4284 let model = model_arc.lock();
4285 if let Some(cache) = model.get_kv_cache() {
4286 let mut cache_guard = cache.lock();
4287
4288 // KV-cache compatibility guard (#1277): KV produced under an
4289 // incompatible (or unproven) model/runtime config is never
4290 // reused. The policy is **fail-closed** via
4291 // `decide_cache_reuse`: a stamp mismatch, an unstamped but
4292 // *populated* cache, or the absence of an authoritative
4293 // expected identity all discard the KV and recompute. Only a
4294 // genuinely fresh (empty) cache under an authoritative
4295 // identity is stamped for reuse. This is the pull-based
4296 // complement to the registry's push-based delta invalidation.
4297 let expected = engine.kv_compat_fingerprint_for(adapter_generation);
4298 cache_guard.apply_reuse_policy(expected);
4299
4300 let matched = cache_guard.prefix_match_len(&prompt_tokens);
4301 if matched > 0 {
4302 tracing::info!(
4303 "Prefix cache hit: {} of {} tokens reusable (skipping {:.0}% of prefill)",
4304 matched, prompt_len,
4305 (matched as f64 / prompt_len as f64) * 100.0
4306 );
4307 }
4308 matched
4309 } else {
4310 0
4311 }
4312 } else {
4313 0
4314 };
4315
4316 if prefix_len > 0 && prefix_len <= prompt_len {
4317 // Resolve the hit: a strict partial prefix rewinds cached
4318 // state (KV truncation; hybrid recurrent models additionally
4319 // restore the end-of-prefill GDN conv/rec snapshot). An
4320 // exact-prompt repeat (`prefix_len == prompt_len`) must
4321 // recompute from cleared state instead — there is no suffix
4322 // left to prefill, and the fresh forward would double-advance
4323 // restored recurrent state. Both decisions live in
4324 // `resolve_session_prefill_start`.
4325 if let Some(model_arc) = &engine.persistent_model {
4326 let model = model_arc.lock();
4327 resolve_session_prefill_start(model.as_ref(), prefix_len, prompt_len)
4328 } else {
4329 0
4330 }
4331 } else {
4332 // No match — clear and start fresh
4333 engine.clear_kv_cache();
4334 0
4335 }
4336 } else {
4337 engine.clear_kv_cache();
4338 0
4339 };
4340
4341 let repeat_last_n = request
4342 .repeat_last_n
4343 .map(|v| v as usize)
4344 .filter(|&v| v > 0)
4345 .unwrap_or(64);
4346
4347 // Use Arc for safe tokenizer sharing
4348 // This Arc is REQUIRED by the TextStream struct - see the comment on the tokenizer field
4349 let tokenizer_arc = Arc::new(tokenizer);
4350
4351 // Create DecodeStream with proper lifetime management
4352 // The Arc<Tokenizer> is stored in the TextStream, ensuring the tokenizer lives as long as 'a
4353 // DEPENDENCY: The unsafe transmute below depends on tokenizer_arc being stored in the struct
4354 let decode_stream = {
4355 // We need to extend the tokenizer lifetime to match 'a
4356 // Since we have Arc<Tokenizer> stored in the struct, we can safely create a reference
4357 let tokenizer_ref = unsafe {
4358 // SAFETY: This is safe because:
4359 // 1. The tokenizer_arc is stored in the TextStream struct, ensuring it lives as long as 'a
4360 // 2. We're not moving or deallocating the tokenizer while the stream exists
4361 // 3. Arc guarantees thread-safe reference counting and proper lifetime management
4362 std::mem::transmute::<&Tokenizer, &'a Tokenizer>(tokenizer_arc.as_ref())
4363 };
4364
4365 // Use skip_special_tokens=false because <|extra_N|> tokens are special tokens
4366 // that should appear in output (they represent actual model vocabulary)
4367 tokenizer_ref.decode_stream(false) // skip_special_tokens=false
4368 };
4369
4370 // Build set of token IDs exempt from repeat penalty.
4371 // Single-character tokens (digits, punctuation, single letters) appear frequently
4372 // in number generation. Penalizing them causes digit suppression: "1917" → "209" → "41".
4373 // We exempt any token whose decoded form is a single character.
4374 let penalty_exempt_tokens = {
4375 let vocab = tokenizer_arc.get_vocab(true);
4376 let mut exempt = HashSet::new();
4377 for (token_str, &id) in &vocab {
4378 // Check if this token decodes to a single character
4379 // Token strings in the vocab may have special prefixes (like Ġ for space+char)
4380 // We want to exempt raw single characters: digits, punctuation, single letters
4381 let clean = token_str.trim();
4382 if clean.chars().count() == 1 {
4383 exempt.insert(id as i64);
4384 }
4385 }
4386 tracing::debug!(
4387 "Built penalty-exempt token set: {} single-char tokens",
4388 exempt.len()
4389 );
4390 exempt
4391 };
4392
4393 // PERF: Pre-create sampling params to avoid struct allocation per token.
4394 // Fall back to model's generation_config defaults (from generation_config.json
4395 // or engine defaults) when the request doesn't specify a value.
4396 let gc = &engine.generation_config;
4397 let sampling_params = SamplingParams {
4398 temperature: request.temperature.unwrap_or(gc.temperature),
4399 top_p: request.top_p.unwrap_or(gc.top_p),
4400 top_k: request.top_k.map(|v| v as usize).or(gc.top_k),
4401 repeat_penalty: request.repeat_penalty.unwrap_or(gc.repeat_penalty),
4402 };
4403
4404 // PERF: Cache vocab_size to avoid lock acquisition per token
4405 let vocab_size = engine.get_vocab_size();
4406
4407 // MTP self-speculative decoding (v1): opt-in via config, and only when
4408 // every correctness prerequisite holds. Reasons are logged once here
4409 // (counts/config only — never content, #1253).
4410 let speculative = if !engine.config.speculative_decoding {
4411 false
4412 } else if delta.is_some() {
4413 // The MTP head is never delta-adapted, so acceptance would degrade
4414 // under a tenant delta. Correctness would be preserved (verify runs
4415 // the delta'd model), but v1 keeps the two features disjoint.
4416 info!("Speculative decoding disabled: tenant LoRA delta active (MTP head is not delta-adapted)");
4417 false
4418 } else if engine.config.kv_quant_type != crate::runtime::KVQuantType::None {
4419 // Quantized LayerKVCache::truncate_to falls back to a full clear(),
4420 // which would silently break the reject rewind.
4421 info!("Speculative decoding disabled: quantized KV cache (reject rewind needs O(1) truncate)");
4422 false
4423 } else if sampling_params.temperature > 0.01 {
4424 // Greedy-only v1: exact-match verify is only exact under argmax.
4425 info!("Speculative decoding disabled: non-greedy sampling (v1 is greedy-only)");
4426 false
4427 } else if !engine.speculative_supported() {
4428 info!("Speculative decoding disabled: model has no dense MTP draft head");
4429 false
4430 } else {
4431 info!("Speculative decoding enabled (MTP self-draft, k=1, greedy)");
4432 true
4433 };
4434
4435 Ok(Self {
4436 engine,
4437 delta,
4438 prompt_tokens,
4439 last_generated: None,
4440 recent_tokens: VecDeque::with_capacity(repeat_last_n),
4441 repeat_last_n,
4442 max_tokens: request.max_tokens.unwrap_or(2048) as usize,
4443 stop_token_ids,
4444 tokenizer: tokenizer_arc,
4445 decode_stream,
4446 prompt_len,
4447 prefill_start_pos,
4448 prefill_ssm_snapshot: None, // Captured after successful serial or speculative prefill
4449 // KV cache starts with prompt already in it after first forward
4450 kv_cache_position: prompt_len,
4451 tokens_generated: 0,
4452 start_time: std::time::Instant::now(),
4453 finished: false,
4454 finish_reason: None,
4455 timeout_ms: request.timeout_ms,
4456 collect_metrics: false, // Server-side concern, not per-request
4457 metrics_accumulator:
4458 crate::runtime::generation_metrics::GenerationMetricsAccumulator::new(),
4459 // PERF: Cached values initialized here, avoid per-token recomputation
4460 sampling_params,
4461 vocab_size,
4462 model_vocab_size: 0, // Set after first forward pass from logits shape
4463 recent_tokens_buffer: Vec::with_capacity(repeat_last_n),
4464 penalty_exempt_tokens,
4465 prefill_time_ms: None,
4466 first_token_time: None,
4467 last_token_time: None,
4468 ema_tokens_per_sec: 0.0,
4469 model_label,
4470 tenant_hash,
4471 speculative,
4472 spec_pending_draft: None,
4473 spec_out_queue: VecDeque::new(),
4474 spec_accepted: 0,
4475 spec_rejected: 0,
4476 })
4477 }
4478
4479 /// Get generation statistics (call after stream exhausted)
4480 pub fn stats(&self) -> GenerationStats {
4481 let total_time = self.start_time.elapsed();
4482
4483 // Prefill metrics
4484 let prefill_time_ms = self.prefill_time_ms.unwrap_or(0);
4485 let prefill_tokens = self.prompt_len;
4486 let prefill_tokens_per_sec = if prefill_time_ms > 0 {
4487 (prefill_tokens as f32 * 1000.0) / prefill_time_ms as f32
4488 } else {
4489 0.0
4490 };
4491
4492 // Inference metrics (time since first token was sampled)
4493 let inference_time_ms = self
4494 .first_token_time
4495 .map(|t| t.elapsed().as_millis() as u64)
4496 .unwrap_or(0);
4497 let inference_tokens = self.tokens_generated;
4498
4499 // Cumulative average for final reporting
4500 let inference_tokens_per_sec = if inference_time_ms > 0 {
4501 (inference_tokens as f32 * 1000.0) / inference_time_ms as f32
4502 } else {
4503 0.0
4504 };
4505
4506 // EMA for real-time adaptive batching (falls back to cumulative if not yet initialized)
4507 let inference_tokens_per_sec_ema = if self.ema_tokens_per_sec > 0.0 {
4508 self.ema_tokens_per_sec
4509 } else {
4510 inference_tokens_per_sec
4511 };
4512
4513 // Finalize quality metrics from accumulator
4514 let quality_metrics = if !self.metrics_accumulator.is_empty() {
4515 Some(self.metrics_accumulator.finalize())
4516 } else {
4517 None
4518 };
4519
4520 GenerationStats {
4521 tokens_generated: self.tokens_generated,
4522 generation_time_ms: total_time.as_millis() as u64,
4523 tokens_per_second: if total_time.as_secs_f32() > 0.0 {
4524 self.tokens_generated as f32 / total_time.as_secs_f32()
4525 } else {
4526 0.0
4527 },
4528 finish_reason: self.finish_reason.clone(),
4529 quality_metrics,
4530
4531 // Separate prefill/inference metrics
4532 prefill_tokens,
4533 prefill_time_ms,
4534 prefill_tokens_per_sec,
4535 inference_tokens,
4536 inference_time_ms,
4537 inference_tokens_per_sec,
4538 inference_tokens_per_sec_ema,
4539
4540 speculative_accepted: self.spec_accepted,
4541 speculative_rejected: self.spec_rejected,
4542 }
4543 }
4544
4545 /// Update EMA rate after generating a token.
4546 ///
4547 /// Uses exponential moving average with alpha=0.3 to smooth rate measurements.
4548 /// This handles initial acceleration gracefully without needing warmup hacks.
4549 /// Save the current token sequence (prompt + generated) to the session KV cache.
4550 ///
4551 /// Called when generation finishes so the next turn can detect prefix overlap.
4552 /// For hybrid recurrent models (Qwen3.5) the end-of-prefill SSM snapshot
4553 /// captured by either prefill path is stored alongside the tokens, making
4554 /// the next turn's prefix hit rewindable (`rewind_session_state`).
4555 fn save_cached_tokens(&mut self) {
4556 // Session-cache recording describes the model's live KV/SSM state,
4557 // which exists only once this stream's first prefill completed. A drop
4558 // before the first poll (cancellation, client disconnect) reaches this
4559 // Drop with the cache already cleared — exact-hit/miss resolution runs
4560 // at generation setup, before any forward — and a failed first prefill
4561 // can leave partial forward state matching no token count. Recording
4562 // prompt IDs in either case lets a later extending prompt claim a
4563 // prefix_len over missing or wrong state (a pure-attention partial
4564 // prefill would then resume with no history behind it). `prefill_time_ms`
4565 // is set exactly when the first prefill forward returned successfully
4566 // on either decoding path, so it is the fail-safe recording gate.
4567 if self.prefill_time_ms.is_none() {
4568 return;
4569 }
4570 if let Some(model_arc) = &self.engine.persistent_model {
4571 let model = model_arc.lock();
4572 if let Some(cache) = model.get_kv_cache() {
4573 let mut cache_guard = cache.lock();
4574 // Save the prompt token IDs for prefix matching on the next turn.
4575 // We only save the prompt (not generated tokens) because the next turn's
4576 // prompt will include the assistant's response via the chat template —
4577 // so the entire current prompt becomes a prefix of the next turn's prompt.
4578 cache_guard.set_cached_tokens_with_ssm(
4579 self.prompt_tokens.clone(),
4580 self.prefill_ssm_snapshot.take(),
4581 );
4582 tracing::debug!(
4583 "Saved {} cached tokens for prefix matching on next turn",
4584 cache_guard.cached_token_count()
4585 );
4586 }
4587 }
4588 }
4589
4590 fn update_ema_rate(&mut self) {
4591 let now = std::time::Instant::now();
4592
4593 if let Some(last_time) = self.last_token_time {
4594 let elapsed_secs = last_time.elapsed().as_secs_f32();
4595 if elapsed_secs > 0.0 {
4596 let instantaneous_rate = 1.0 / elapsed_secs;
4597
4598 // EMA: alpha=0.3 gives good responsiveness while smoothing noise
4599 const ALPHA: f32 = 0.3;
4600 if self.ema_tokens_per_sec > 0.0 {
4601 self.ema_tokens_per_sec =
4602 ALPHA * instantaneous_rate + (1.0 - ALPHA) * self.ema_tokens_per_sec;
4603 } else {
4604 // First measurement - use instantaneous rate
4605 self.ema_tokens_per_sec = instantaneous_rate;
4606 }
4607 }
4608 }
4609
4610 self.last_token_time = Some(now);
4611 }
4612
4613 fn sample_next_token(&mut self) -> Result<u32> {
4614 // Determine KV position for this forward pass
4615 let current_kv_pos = if self.tokens_generated == 0 {
4616 if self.prefill_start_pos > 0 {
4617 self.prefill_start_pos // Resume from prefix cache hit
4618 } else {
4619 0 // Full prefill from start
4620 }
4621 } else {
4622 self.kv_cache_position
4623 };
4624
4625 // Lock delta once for this token's forward pass (if present)
4626 let delta_guard = self.delta.as_ref().map(|d| d.lock());
4627
4628 let logits = if self.tokens_generated == 0 {
4629 // PREFILL: Process prompt (full or partial based on prefix cache hit)
4630 let prefill_start = std::time::Instant::now();
4631
4632 let result = if self.prefill_start_pos > 0
4633 && self.prefill_start_pos < self.prompt_tokens.len()
4634 {
4635 // PARTIAL PREFILL: prefix is cached, only forward new tokens
4636 let new_tokens = &self.prompt_tokens[self.prefill_start_pos..];
4637 tracing::info!(
4638 "Partial prefill: processing {} new tokens (skipped {} cached)",
4639 new_tokens.len(),
4640 self.prefill_start_pos
4641 );
4642 if delta_guard.is_some() {
4643 let delta_ref = delta_guard.as_deref();
4644 self.engine.forward_with_delta_cached(
4645 new_tokens,
4646 self.prefill_start_pos,
4647 delta_ref,
4648 )?
4649 } else {
4650 self.engine
4651 .forward_cached(new_tokens, self.prefill_start_pos)?
4652 }
4653 } else {
4654 // FULL PREFILL: no prefix cache hit
4655 if delta_guard.is_some() {
4656 let delta_ref = delta_guard.as_deref();
4657 self.engine
4658 .forward_with_delta_full(&self.prompt_tokens, delta_ref)?
4659 } else {
4660 self.engine.forward(&self.prompt_tokens)?
4661 }
4662 };
4663 let prefill_elapsed = prefill_start.elapsed();
4664
4665 // Hybrid recurrent models (Qwen3.5): capture the SSM conv/rec state
4666 // as of end-of-prefill, before decode advances it. `save_cached_tokens`
4667 // stores it with the prompt tokens so a later session prefix hit can
4668 // rewind the recurrent state (KV truncation alone cannot). This is a
4669 // cheap downcast returning `None` for pure-attention models.
4670 self.prefill_ssm_snapshot = self.engine.snapshot_ssm_states();
4671
4672 // Store prefill timing
4673 self.prefill_time_ms = Some(prefill_elapsed.as_millis() as u64);
4674 self.first_token_time = Some(std::time::Instant::now());
4675
4676 let prefill_token_count = if self.prefill_start_pos > 0 {
4677 self.prompt_tokens.len() - self.prefill_start_pos
4678 } else {
4679 self.prompt_tokens.len()
4680 };
4681 tracing::info!(
4682 "PREFILL: {} tokens in {:?} ({:.2} tok/sec){}{}",
4683 prefill_token_count,
4684 prefill_elapsed,
4685 if prefill_elapsed.as_secs_f32() > 0.0 {
4686 prefill_token_count as f32 / prefill_elapsed.as_secs_f32()
4687 } else {
4688 0.0
4689 },
4690 if self.prefill_start_pos > 0 {
4691 format!(" [prefix cached: {} tokens]", self.prefill_start_pos)
4692 } else {
4693 String::new()
4694 },
4695 if delta_guard.is_some() {
4696 " [delta-aware]"
4697 } else {
4698 ""
4699 }
4700 );
4701
4702 result
4703 } else {
4704 let last_token = self.last_generated.ok_or_else(|| {
4705 anyhow::anyhow!(
4706 "Internal error: last_generated not set after {} tokens",
4707 self.tokens_generated
4708 )
4709 })?;
4710
4711 if self.tokens_generated.is_multiple_of(50) {
4712 // Counts/positions only — token IDs are reversible to text via the
4713 // loaded tokenizer and must never hit process logs (#1253/#1261).
4714 tracing::debug!(
4715 "🔵 KV cache position: {}, tokens_generated: {}",
4716 self.kv_cache_position,
4717 self.tokens_generated
4718 );
4719 }
4720
4721 // Use kv_cache_position directly - this is where the next token will be written
4722 if delta_guard.is_some() {
4723 let delta_ref = delta_guard.as_deref();
4724 self.engine
4725 .forward_with_delta_cached(&[last_token], current_kv_pos, delta_ref)?
4726 } else {
4727 self.engine.forward_cached(&[last_token], current_kv_pos)?
4728 }
4729 };
4730
4731 // Release delta lock before sampling (no longer needed)
4732 drop(delta_guard);
4733
4734 // NOTE: Logits truncation has been DISABLED (Nov 6, 2025)
4735 //
4736 // Previous code tried to truncate logits from model vocab (151936) to tokenizer vocab (151669).
4737 // This was causing year tokens to be excluded from sampling, leading to "1 7 7 6" instead of "1776".
4738 //
4739 // vLLM with the same model does NOT have this issue, suggesting the truncation was too aggressive.
4740 //
4741 // Solution: Use model vocab directly. The sampler will only pick valid tokens anyway.
4742 // The tokenizer's get_vocab_size() should match model vocab if properly configured.
4743
4744 // PERF: Use cached vocab_size - avoids lock acquisition per token
4745 let vocab_size = self.vocab_size;
4746
4747 // PERF: Cache model_vocab_size on first call (from logits shape)
4748 let model_vocab_size = if self.model_vocab_size == 0 {
4749 let logits_shape = logits.size();
4750 let size = logits_shape[logits_shape.len() - 1] as usize;
4751 self.model_vocab_size = size;
4752 size
4753 } else {
4754 self.model_vocab_size
4755 };
4756
4757 if vocab_size == 0 {
4758 // Tokenizer not loaded - this should never happen during generation
4759 return Err(anyhow::anyhow!(
4760 "Cannot sample tokens: tokenizer vocabulary size is 0 (tokenizer not loaded)"
4761 ));
4762 }
4763
4764 // Log mismatch for debugging but don't truncate
4765 if model_vocab_size != vocab_size {
4766 tracing::debug!(
4767 "Vocab mismatch: model_vocab_size={}, tokenizer_vocab_size={}. Using model vocab directly.",
4768 model_vocab_size, vocab_size
4769 );
4770 }
4771
4772 // UTF-8 reranking DISABLED - DecodeStream handles UTF-8 correctly
4773 // The reranker was causing number corruption by manipulating logits
4774 // for non-UTF-8 related tokens (like digits)
4775 //
4776 // YEAR CORRUPTION BUG FIX (Nov 6, 2025):
4777 // The real issue is vocab mismatch: model (151936) vs tokenizer (151669).
4778 // Years like "1776" are split into individual digits "1 7 7 6" because:
4779 // 1. Year tokens may be in the truncated range (>151669) if model's vocab
4780 // extends beyond tokenizer's
4781 // 2. OR the model's logits heavily favor digit tokens over year tokens
4782 // 3. This is NOT a UTF-8 issue - it's a vocab/tokenization issue
4783 //
4784 // Current approach: Keep reranker disabled (it corrupted numbers), but
4785 // we need to investigate why year tokens are disfavored in the logits.
4786 // The truncation to vocab_size is necessary to prevent out-of-bounds,
4787 // but may be filtering out valid year token IDs.
4788
4789 // PERF: Use pre-created sampling params - avoids struct allocation per token
4790 let params = &self.sampling_params;
4791
4792 // PERF: Optimize VecDeque slice handling
4793 // Pass penalty_exempt_tokens to sampler for reduced penalty on single-char tokens
4794 let (slice1, slice2) = self.recent_tokens.as_slices();
4795 let recent_tokens_slice = if slice2.is_empty() {
4796 slice1
4797 } else {
4798 self.recent_tokens_buffer.clear();
4799 self.recent_tokens_buffer
4800 .extend(self.recent_tokens.iter().copied());
4801 &self.recent_tokens_buffer
4802 };
4803 let next_token = self.engine.sample_token_with_params(
4804 &logits,
4805 params,
4806 recent_tokens_slice,
4807 &self.penalty_exempt_tokens,
4808 )?;
4809
4810 // Validate sampled token is within model vocabulary
4811 if model_vocab_size > 0 && next_token >= model_vocab_size {
4812 return Err(anyhow::anyhow!(
4813 "Generated out-of-bounds token {}: exceeds model vocab size {}",
4814 next_token,
4815 model_vocab_size
4816 ));
4817 }
4818
4819 // Warn if token is beyond tokenizer vocabulary (but allow it)
4820 if next_token >= vocab_size {
4821 tracing::warn!(
4822 "⚠️ Sampled token {} is beyond tokenizer vocab ({}) but within model vocab ({}). This may indicate a vocab mismatch.",
4823 next_token, vocab_size, model_vocab_size
4824 );
4825 }
4826
4827 // Capture metrics for self-supervised training (EXPENSIVE - only if explicitly enabled)
4828 // Computes softmax + entropy over full vocabulary (~150K) with GPU->CPU syncs
4829 // This adds ~10x overhead per token, so only enable for training
4830 if self.collect_metrics {
4831 // Get last token's logits (already squeezed to 1D in sampler, but handle 2D case)
4832 let last_logits = if logits.dim() > 1 {
4833 let shape = logits.size();
4834 if shape.len() == 3 {
4835 // [batch, seq_len, vocab] -> take [-1, -1, :] -> [vocab]
4836 logits.select(1, shape[1] - 1).squeeze_dim(0)
4837 } else if shape.len() == 2 {
4838 // [seq_len, vocab] or [batch, vocab] -> squeeze
4839 logits.squeeze_dim(0)
4840 } else {
4841 logits.shallow_clone()
4842 }
4843 } else {
4844 logits.shallow_clone()
4845 };
4846
4847 // Compute stable softmax: exp(x - max(x)) / sum(exp(x - max(x)))
4848 let max_logit = last_logits.max();
4849 let shifted = &last_logits - &max_logit;
4850 let exp_logits = shifted.exp();
4851 let sum_exp = exp_logits.sum(tch::Kind::Float);
4852 let probs = &exp_logits / &sum_exp;
4853
4854 // Compute log_prob of sampled token (in nats)
4855 // log_prob = log(probs[next_token])
4856 let log_prob = probs.select(0, next_token as i64).log().double_value(&[]) as f32;
4857
4858 // Compute entropy: -sum(p * log(p)) where p > 0
4859 // Use a small epsilon to avoid log(0)
4860 let eps = 1e-10f64;
4861 let safe_probs = probs.clamp(eps, 1.0 - eps);
4862 let log_probs = safe_probs.log();
4863 let entropy = (-(&probs * &log_probs))
4864 .sum(tch::Kind::Float)
4865 .double_value(&[]) as f32;
4866
4867 // Add to accumulator (O(1) per token)
4868 self.metrics_accumulator
4869 .add_token(log_prob, entropy, next_token as u32);
4870 }
4871
4872 Ok(next_token as u32)
4873 }
4874
4875 /// Sample one token for the speculative path with the standard tiered
4876 /// repeat penalty. `extra_window` holds tokens accepted earlier in the
4877 /// current round but not yet pushed to `recent_tokens` by `poll_next`, so
4878 /// the penalty window matches what serial decoding would see at this
4879 /// position. Mirrors the sampling section of `sample_next_token`.
4880 fn sample_spec_position(&mut self, logits: &Tensor, extra_window: &[i64]) -> Result<u32> {
4881 let vocab_size = self.vocab_size;
4882
4883 // PERF: Cache model_vocab_size on first call (from logits shape)
4884 let model_vocab_size = if self.model_vocab_size == 0 {
4885 let logits_shape = logits.size();
4886 let size = logits_shape[logits_shape.len() - 1] as usize;
4887 self.model_vocab_size = size;
4888 size
4889 } else {
4890 self.model_vocab_size
4891 };
4892
4893 if vocab_size == 0 {
4894 return Err(anyhow::anyhow!(
4895 "Cannot sample tokens: tokenizer vocabulary size is 0 (tokenizer not loaded)"
4896 ));
4897 }
4898
4899 let params = self.sampling_params;
4900 // Reuse the PERF buffer: recent_tokens (penalty window) + round-local extras.
4901 self.recent_tokens_buffer.clear();
4902 self.recent_tokens_buffer
4903 .extend(self.recent_tokens.iter().copied());
4904 self.recent_tokens_buffer
4905 .extend(extra_window.iter().copied());
4906 let excess = self
4907 .recent_tokens_buffer
4908 .len()
4909 .saturating_sub(self.repeat_last_n);
4910 self.recent_tokens_buffer.drain(..excess);
4911 let next_token = self.engine.sample_token_with_params(
4912 logits,
4913 ¶ms,
4914 &self.recent_tokens_buffer,
4915 &self.penalty_exempt_tokens,
4916 )?;
4917
4918 if model_vocab_size > 0 && next_token >= model_vocab_size {
4919 return Err(anyhow::anyhow!(
4920 "Generated out-of-bounds token {}: exceeds model vocab size {}",
4921 next_token,
4922 model_vocab_size
4923 ));
4924 }
4925 if next_token >= vocab_size {
4926 tracing::warn!(
4927 "⚠️ Sampled token {} is beyond tokenizer vocab ({}) but within model vocab ({}). This may indicate a vocab mismatch.",
4928 next_token, vocab_size, model_vocab_size
4929 );
4930 }
4931 Ok(next_token as u32)
4932 }
4933
4934 /// Greedy argmax of draft logits `[1, 1, vocab]` (no repeat penalty — the
4935 /// draft only proposes; the penalty-applied verify decides acceptance).
4936 fn spec_argmax_draft(draft_logits: &Tensor) -> u32 {
4937 draft_logits
4938 .reshape([-1i64])
4939 .argmax(-1, false)
4940 .int64_value(&[]) as u32
4941 }
4942
4943 /// One speculative decode step (MTP self-draft, k=1, greedy exact-match
4944 /// verify). Returns the 1–2 tokens accepted this round, in order.
4945 ///
4946 /// Round structure (decode phase; `last_generated` = x_pos is emitted but
4947 /// not yet forwarded, `kv_cache_position` = pos, and `spec_pending_draft`
4948 /// holds the MTP head's candidate for position pos+1):
4949 ///
4950 /// 1. Snapshot the GDN conv/rec state (before the verify forward mutates it).
4951 /// 2. Verify: one main forward over `[x_pos, draft]` at `pos`. Logits at
4952 /// position 0 check the draft; logits at position 1 yield the bonus
4953 /// token when the draft matches.
4954 /// 3. Accept (draft == verifier's token): emit both tokens; the verify
4955 /// forward already consumed exactly the true sequence, so KV and SSM
4956 /// state are consistent. Draft the next round from the two new slots.
4957 /// Reject: emit only the verifier's correction token, restore the SSM
4958 /// snapshot, truncate KV to `pos + 1` (dropping the rejected draft's
4959 /// KV), then re-forward `x_pos` alone — the 2-token verify advanced the
4960 /// GDN state through the rejected draft as well, so the restore alone
4961 /// would leave SSM one accepted token behind the kept KV. Re-draft from
4962 /// the correction token's slot.
4963 ///
4964 /// Invariants: `last_generated`/`tokens_generated`/`recent_tokens`/
4965 /// `decode_stream` advance only per accepted token (the rejected draft
4966 /// never enters the penalty window or decode stream);
4967 /// `kv_cache_position` advances by exactly the accepted count.
4968 fn sample_speculative_tokens(&mut self) -> Result<Vec<u32>> {
4969 if self.tokens_generated == 0 {
4970 // ---- PREFILL (full, or partial on a session prefix-cache hit) ----
4971 let prefill_start = std::time::Instant::now();
4972 let prompt_len = self.prompt_tokens.len();
4973 let partial = self.prefill_start_pos > 0 && self.prefill_start_pos < prompt_len;
4974 // Owned copy: the forward + sampling below need `&mut self`.
4975 let (slice, start_pos) = if partial {
4976 tracing::info!(
4977 "Partial prefill: processing {} new tokens (skipped {} cached)",
4978 prompt_len - self.prefill_start_pos,
4979 self.prefill_start_pos
4980 );
4981 (
4982 self.prompt_tokens[self.prefill_start_pos..].to_vec(),
4983 self.prefill_start_pos,
4984 )
4985 } else {
4986 (self.prompt_tokens.clone(), 0)
4987 };
4988
4989 let (logits, hidden) = self.engine.forward_speculative(&slice, start_pos, true)?;
4990
4991 // Retain the prompt's recurrent state before verification advances
4992 // it beyond the emitted-token queue. Session Drop persists this
4993 // deep snapshot alongside the prompt IDs, just like serial prefill.
4994 self.prefill_ssm_snapshot = self.engine.snapshot_ssm_states();
4995
4996 let prefill_elapsed = prefill_start.elapsed();
4997 self.prefill_time_ms = Some(prefill_elapsed.as_millis() as u64);
4998 self.first_token_time = Some(std::time::Instant::now());
4999 tracing::info!(
5000 "PREFILL: {} tokens in {:?} ({:.2} tok/sec){} [speculative]",
5001 slice.len(),
5002 prefill_elapsed,
5003 if prefill_elapsed.as_secs_f32() > 0.0 {
5004 slice.len() as f32 / prefill_elapsed.as_secs_f32()
5005 } else {
5006 0.0
5007 },
5008 if partial {
5009 format!(" [prefix cached: {} tokens]", self.prefill_start_pos)
5010 } else {
5011 String::new()
5012 },
5013 );
5014
5015 let next = self.sample_spec_position(&logits, &[])?;
5016
5017 // MTP prompt prefill + first draft in one call: slot t pairs
5018 // (hidden after x_t, embed(x_{t+1})); the final slot pairs the
5019 // last hidden with the just-sampled token, yielding the draft for
5020 // the following position. On a prefix-cache hit the pre-hit slots
5021 // keep the swapped-in cache's contents (stale or zero entries only
5022 // degrade acceptance, never correctness — the verify is decisive).
5023 let mut mtp_tokens: Vec<i64> = slice[1..].to_vec();
5024 mtp_tokens.push(next as i64);
5025 let draft_logits = self
5026 .engine
5027 .forward_mtp_draft(&mtp_tokens, &hidden, start_pos)?;
5028 self.spec_pending_draft = Some(Self::spec_argmax_draft(&draft_logits));
5029
5030 Ok(vec![next])
5031 } else {
5032 // ---- VERIFY ROUND ----
5033 let pos = self.kv_cache_position;
5034
5035 // Context boundary: the 2-token verify forward writes KV at `pos`
5036 // and `pos + 1`, and LayerKVCache::update hard-errors past
5037 // max_seq_len. When fewer than two slots remain, the serial path
5038 // can still forward `last` in the final slot — fall back to a
5039 // single-token step for THIS round (not a permanent latch; the
5040 // next round re-checks and ends exactly as the serial path would).
5041 if let Some(cap) = self.spec_kv_capacity() {
5042 if pos + 2 > cap {
5043 let token = self.sample_next_token()?;
5044 // poll_next's per-token increment is skipped on the
5045 // speculative path, so advance here like a serial step.
5046 self.kv_cache_position += 1;
5047 return Ok(vec![token]);
5048 }
5049 }
5050
5051 let last = self.last_generated.ok_or_else(|| {
5052 anyhow::anyhow!(
5053 "Internal error: last_generated not set after {} tokens",
5054 self.tokens_generated
5055 )
5056 })?;
5057 let draft = self.spec_pending_draft.take().ok_or_else(|| {
5058 anyhow::anyhow!("Internal error: speculative round without a pending draft")
5059 })?;
5060
5061 let snapshot = self.engine.snapshot_ssm_states();
5062
5063 let (logits, hidden) =
5064 self.engine
5065 .forward_speculative(&[last, draft as i64], pos, false)?;
5066 let v1 = self.sample_spec_position(&logits.select(1, 0), &[])?;
5067
5068 if v1 == draft {
5069 // ACCEPT: the draft was the verifier's own greedy choice, so the
5070 // verify forward consumed exactly the true sequence (KV + SSM
5071 // consistent). Emit the verified draft plus the bonus token.
5072 let v2 = self.sample_spec_position(&logits.select(1, 1), &[v1 as i64])?;
5073 let next_draft_logits =
5074 self.engine
5075 .forward_mtp_draft(&[v1 as i64, v2 as i64], &hidden, pos)?;
5076 self.spec_pending_draft = Some(Self::spec_argmax_draft(&next_draft_logits));
5077 self.kv_cache_position += 2;
5078 self.spec_accepted += 1;
5079 Ok(vec![v1, v2])
5080 } else {
5081 // REJECT: emit only the verifier's correction token and rewind.
5082 self.engine.restore_ssm_states(snapshot);
5083 // Drop the rejected draft's KV; the +1 keeps the KV the verify
5084 // forward wrote for `last`. The MTP slot lags one position
5085 // behind the main cache (it never consumed the rejected draft),
5086 // so this manager-wide truncate is a no-op for it.
5087 if let Some(model_arc) = &self.engine.persistent_model {
5088 let model = model_arc.lock();
5089 if let Some(cache) = model.get_kv_cache() {
5090 cache.lock().truncate_to(pos + 1);
5091 }
5092 }
5093 // Re-forward `last` alone to re-sync the GDN conv/rec state
5094 // with the kept KV (see fn docs); rewrites identical KV at pos.
5095 let (_logits, hidden2) = self.engine.forward_speculative(&[last], pos, true)?;
5096 let next_draft_logits =
5097 self.engine.forward_mtp_draft(&[v1 as i64], &hidden2, pos)?;
5098 self.spec_pending_draft = Some(Self::spec_argmax_draft(&next_draft_logits));
5099 self.kv_cache_position += 1;
5100 self.spec_rejected += 1;
5101 Ok(vec![v1])
5102 }
5103 }
5104 }
5105
5106 /// Next token from the speculative path: drains the current round's
5107 /// accepted-token queue, running a new speculative round when empty.
5108 /// `kv_cache_position` is advanced inside `sample_speculative_tokens`
5109 /// (per accepted token), so unlike the serial path no increment happens here.
5110 fn next_speculative_token(&mut self) -> Result<u32> {
5111 if self.spec_out_queue.is_empty() {
5112 let tokens = self.sample_speculative_tokens()?;
5113 self.spec_out_queue.extend(tokens);
5114 }
5115 self.spec_out_queue
5116 .pop_front()
5117 .ok_or_else(|| anyhow::anyhow!("Speculative sampler produced no tokens"))
5118 }
5119
5120 /// KV capacity (max sequence length) of the model's cache manager — the
5121 /// hard bound `LayerKVCache::update` enforces. `None` when no model/cache
5122 /// is installed (caller then skips the boundary check).
5123 fn spec_kv_capacity(&self) -> Option<usize> {
5124 let model_arc = self.engine.persistent_model.as_ref()?;
5125 let model = model_arc.lock();
5126 let cache = model.get_kv_cache()?;
5127 let cap = cache.lock().max_seq_len();
5128 Some(cap)
5129 }
5130}
5131
5132// SAFETY: TextStream can be Send because:
5133// - tokenizer pointer points to heap data that won't move
5134// - decode_stream is tied to the tokenizer lifetime
5135// - Tokenizer itself is Send
5136unsafe impl<'a> Send for TextStream<'a> {}
5137
5138// No custom Drop needed - Arc handles automatic cleanup
5139
5140impl<'a> Stream for TextStream<'a> {
5141 type Item = Result<String>;
5142
5143 fn poll_next(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {
5144 loop {
5145 if self.finished {
5146 return Poll::Ready(None);
5147 }
5148
5149 // Check timeout
5150 if let Some(timeout_ms) = self.timeout_ms {
5151 let elapsed = self.start_time.elapsed();
5152 if elapsed.as_millis() >= timeout_ms as u128 {
5153 tracing::debug!("Generation timed out after {}ms", timeout_ms);
5154 self.finished = true;
5155 self.finish_reason = Some(FinishReason::Error(format!(
5156 "Generation timed out after {timeout_ms}ms"
5157 )));
5158 return Poll::Ready(Some(Err(anyhow::anyhow!(
5159 "Generation timed out after {}ms",
5160 timeout_ms
5161 ))));
5162 }
5163 }
5164
5165 // Check max tokens
5166 if self.max_tokens > 0 && self.tokens_generated >= self.max_tokens {
5167 tracing::debug!("Reached max tokens: {}", self.max_tokens);
5168 self.finished = true;
5169 self.finish_reason = Some(FinishReason::MaxTokens);
5170 return Poll::Ready(None);
5171 }
5172
5173 // Sample next token
5174 // The `!self.collect_metrics` guard keeps metrics-collecting
5175 // (training-quality) requests on the serial path: the speculative
5176 // sampler does not replicate the per-token logprob/entropy
5177 // accumulator block from `sample_next_token`.
5178 let next_token = if self.speculative && !self.collect_metrics {
5179 match self.next_speculative_token() {
5180 Ok(token) => token,
5181 Err(e) => {
5182 self.finished = true;
5183 self.finish_reason = Some(FinishReason::Error(e.to_string()));
5184 return Poll::Ready(Some(Err(e)));
5185 }
5186 }
5187 } else {
5188 match self.sample_next_token() {
5189 Ok(token) => {
5190 // FIX: Increment KV cache position after successful token sampling
5191 // This ensures KV cache stays synchronized with generation state
5192 if self.tokens_generated > 0 {
5193 // Don't increment on initial prompt
5194 self.kv_cache_position += 1;
5195 }
5196 token
5197 }
5198 Err(e) => {
5199 self.finished = true;
5200 self.finish_reason = Some(FinishReason::Error(e.to_string()));
5201 return Poll::Ready(Some(Err(e)));
5202 }
5203 }
5204 };
5205
5206 // Check EOS
5207 if self.engine.is_eos_token(next_token as usize) {
5208 tracing::debug!(
5209 "EOS token detected at generated position {}",
5210 self.tokens_generated
5211 );
5212 self.finished = true;
5213 self.finish_reason = Some(FinishReason::EndOfSequence);
5214 return Poll::Ready(None);
5215 }
5216
5217 // Check stop tokens
5218 if self.stop_token_ids.contains(&next_token) {
5219 // Never log the matched stop-token ID: it identifies a
5220 // caller-supplied stop string (#1253/#1261). Position only.
5221 tracing::debug!(
5222 "Stop token matched at generated position {}",
5223 self.tokens_generated
5224 );
5225 self.finished = true;
5226 self.finish_reason = Some(FinishReason::StopToken(format!("{next_token}")));
5227 return Poll::Ready(None);
5228 }
5229
5230 // KV cache position is already tracked correctly in sample_next_token
5231 // No need to increment here
5232
5233 // Process decode_stream FIRST, then update state
5234 // This prevents state corruption if decode_stream fails
5235 match self.decode_stream.step(next_token) {
5236 Ok(Some(text)) => {
5237 // DecodeStream succeeded - now update state
5238 let token_i64 = next_token as i64;
5239 self.last_generated = Some(token_i64);
5240 self.tokens_generated += 1;
5241 self.update_ema_rate();
5242
5243 self.recent_tokens.push_back(token_i64);
5244 if self.recent_tokens.len() > self.repeat_last_n {
5245 self.recent_tokens.pop_front();
5246 }
5247
5248 // DecodeStream returned text - emit it. The chunk text is
5249 // generated content and is never logged (#1253); generated-
5250 // token counts are now emitted as OTel metrics at completion
5251 // (#1261), so the per-chunk count log is dropped.
5252 return Poll::Ready(Some(Ok(text)));
5253 }
5254 Ok(None) => {
5255 // DecodeStream is buffering incomplete UTF-8 - update state and continue
5256 let token_i64 = next_token as i64;
5257 self.last_generated = Some(token_i64);
5258 self.tokens_generated += 1;
5259 self.update_ema_rate();
5260
5261 self.recent_tokens.push_back(token_i64);
5262 if self.recent_tokens.len() > self.repeat_last_n {
5263 self.recent_tokens.pop_front();
5264 }
5265
5266 // Simple trace without expensive tokenizer lock or decode.
5267 // Counts/positions only — token IDs are reversible to text
5268 // and must never hit process logs (#1253/#1261).
5269 tracing::trace!(
5270 "Token at generated position {} buffering (incomplete UTF-8)",
5271 self.tokens_generated
5272 );
5273 continue;
5274 }
5275 Err(e) => {
5276 // CRITICAL: Do NOT update state on decode_stream error
5277 // This prevents state corruption and keeps generation consistent
5278 tracing::error!(
5279 "DecodeStream error at generated position {}: {}",
5280 self.tokens_generated,
5281 e
5282 );
5283 self.finished = true;
5284 self.finish_reason = Some(FinishReason::Error(e.to_string()));
5285 return Poll::Ready(Some(Err(anyhow::anyhow!("Decode error: {}", e))));
5286 }
5287 }
5288 }
5289 }
5290}
5291
5292impl<'a> Drop for TextStream<'a> {
5293 fn drop(&mut self) {
5294 // Save token IDs to the session cache for prefix matching on the next turn.
5295 // This runs on all exit paths (EOS, stop token, max tokens, timeout, error).
5296 self.save_cached_tokens();
5297
5298 // Token-burn metering (#1261): emit the generated-token count and the
5299 // per-request total at completion. Drop runs on every exit path, so this
5300 // covers normal EOS, stop-token, max-tokens, timeout, and error finishes.
5301 // Only integer counts and the opaque tenant hash are metric attributes (#1253).
5302 let generated = self.tokens_generated as u64;
5303 let meter = crate::runtime::token_metrics::global_meter();
5304 meter.record_generated(&self.model_label, self.tenant_hash, generated);
5305 meter.record_request_total(
5306 &self.model_label,
5307 self.tenant_hash,
5308 self.prompt_len as u64 + generated,
5309 );
5310 // MTP self-speculative draft outcomes (integer counts only, #1253).
5311 meter.record_speculative(
5312 &self.model_label,
5313 self.tenant_hash,
5314 self.spec_accepted,
5315 self.spec_rejected,
5316 );
5317 }
5318}