Skip to main content

hyprstream_inference/runtime/
torch_engine.rs

1//! PyTorch-based inference engine using tch-rs
2
3use crate::config::{FinishReason, GenerationConfig, GenerationResult, RuntimeConfig};
4use crate::runtime::architectures::ModelOperations;
5use crate::runtime::template_engine::{ChatMessage, TemplateEngine};
6use crate::runtime::tensor_sampling::TensorSampler;
7use crate::runtime::RuntimeEngine;
8use anyhow::{anyhow, Result};
9use async_trait::async_trait;
10use hyprstream_rpc_std::inference_client::GenerationRequest;
11use hyprstream_rpc_std::inference_client::ModelInfo;
12use json_threat_protection as jtp;
13use parking_lot::Mutex;
14use serde_json;
15use std::collections::{HashMap, HashSet};
16use std::path::Path;
17use std::sync::{
18    atomic::{AtomicU32, AtomicUsize, Ordering},
19    Arc,
20};
21use tch::{nn::VarStore, Device, Tensor};
22use tokenizers::Tokenizer;
23use tracing::{info, instrument, warn};
24
25/// Resolve the compute dtype independently from checkpoint weight storage.
26/// This pure policy accepts the documented case-insensitive precision aliases;
27/// it performs no device allocation. `fp8` retains legacy BF16 compute except
28/// for qualified channel-FP8 CPU checkpoints, whose activation QDQ/matmuls run
29/// in Float32. It never requests hardware Float8 activations on CPU.
30/// Serving adapters must still enforce their own qualified-device/dtype gate.
31pub fn resolve_inference_compute_kind(
32    precision_mode: Option<&str>,
33    device: Device,
34    channel_fp8: bool,
35) -> Result<tch::Kind> {
36    match precision_mode
37        .unwrap_or("auto")
38        .to_ascii_lowercase()
39        .as_str()
40    {
41        "fp32" | "float32" => Ok(tch::Kind::Float),
42        "fp16" | "float16" => Ok(tch::Kind::Half),
43        "auto" | "fp8" if device == Device::Cpu && channel_fp8 => Ok(tch::Kind::Float),
44        "bf16" | "bfloat16" | "auto" | "fp8" => Ok(tch::Kind::BFloat16),
45        mode => anyhow::bail!("unsupported inference precision mode: {mode}"),
46    }
47}
48
49/// Build a self-explaining diagnostic for why CUDA/ROCm is unavailable.
50///
51/// `tch::Cuda::is_available()` returns a bare `false` with no cause, which made
52/// the GPU→CPU fallback nearly impossible to diagnose in the field (e.g. an RTX
53/// 5090 node with a CUDA libtorch bundled that still logged a one-line "not
54/// available"). This collects the underlying signals so the operator can tell
55/// *why*:
56///
57/// - `has_cuda` / `has_cudart`: was this libtorch even compiled with CUDA? If
58///   `false`, an install/active-backend mismatch bundled a CPU build.
59/// - `device_count`: 0 with a CUDA build almost always means the driver lib
60///   (`libcuda.so.1`, shipped by the host NVIDIA driver — *never* by the
61///   AppImage) could not be loaded, i.e. `LD_LIBRARY_PATH` does not reach the
62///   host driver path. A negative count is a CUDA-internal error.
63/// - The current `LD_LIBRARY_PATH` is echoed so the operator can immediately
64///   see whether the host driver path is missing.
65fn cuda_unavailable_diagnostics() -> String {
66    let has_cuda = tch::utils::has_cuda();
67    let has_cudart = tch::utils::has_cudart();
68    let has_hip = tch::utils::has_hip();
69    let device_count = tch::Cuda::device_count();
70    let ld_path = std::env::var("LD_LIBRARY_PATH").unwrap_or_else(|_| "<unset>".to_owned());
71
72    // Best-effort, actionable interpretation of the signals above.
73    let hint = if !has_cuda && !has_hip {
74        "this libtorch was NOT built with CUDA/ROCm support (CPU-only build). \
75         Check the installed/active backend — an install likely selected a 'cpu' \
76         variant (verify ~/.local/share/hyprstream/active-backend / active-version)."
77    } else if device_count <= 0 {
78        "libtorch has CUDA/ROCm support but reports 0 devices. This is almost \
79         always the host driver library (libcuda.so.1) failing to load: it ships \
80         with the NVIDIA driver and is never bundled in the AppImage. Ensure the \
81         host driver lib dir is on LD_LIBRARY_PATH (e.g. /usr/lib64, \
82         /usr/lib/x86_64-linux-gnu, or the libcuda.so.1 location from \
83         `ldconfig -p | grep libcuda`), or that ldconfig has it cached."
84    } else {
85        "CUDA/ROCm reports devices but is_available() is false — possible \
86         driver/runtime version mismatch or a CUDA-internal init error."
87    };
88
89    format!(
90        "is_available()=false; has_cuda={has_cuda}, has_cudart={has_cudart}, \
91         has_hip={has_hip}, device_count={device_count}, \
92         LD_LIBRARY_PATH={ld_path}. Likely cause: {hint}"
93    )
94}
95
96/// Basic context state for tracking generation state
97#[derive(Debug, Clone)]
98pub struct ContextState {
99    /// Current sequence length
100    pub sequence_length: usize,
101    /// Context window size
102    pub context_window: usize,
103    /// Whether model is initialized
104    pub initialized: bool,
105}
106
107/// PyTorch inference engine using tch-rs
108///
109/// Thread-safe implementation using proper synchronization primitives.
110/// All mutable state is protected by mutexes with poisoning recovery.
111#[derive(Clone)]
112pub struct TorchEngine {
113    /// VarStore for native PyTorch weight management - not thread safe, requires external sync
114    var_store: Arc<Mutex<Option<VarStore>>>,
115    /// SafeTensors raw data for on-demand tensor creation - thread safe after initialization
116    /// Detected model architecture - thread safe after initialization
117    model_architecture: Arc<Mutex<Option<String>>>,
118    /// Persistent model instance to avoid recreation on every forward pass
119    /// Using Arc<Mutex<>> for interior mutability since ModelOperations has mutable methods
120    persistent_model: Option<Arc<Mutex<Box<dyn ModelOperations>>>>,
121    /// Basic KV cache storage for context tracking - thread safe with mutex
122    context_state: Arc<Mutex<Option<ContextState>>>,
123    /// Tokenizer for text processing - thread safe after initialization
124    tokenizer: Arc<Mutex<Option<Tokenizer>>>,
125    /// Template engine for chat formatting
126    template_engine: Arc<Mutex<Option<TemplateEngine>>>,
127    /// Device for computation (CPU/CUDA/ROCm) - immutable after construction.
128    ///
129    /// For a multi-device engine this is the pool's **primary** (first) device —
130    /// the home of non-layer weights (embeddings, final norm, lm_head) and the
131    /// device inputs are placed on before the first decoder layer. Per-layer
132    /// placement is driven by the model's own `LayerDeviceMap` (`forward_layers`).
133    device: Device,
134    /// Resolved multi-device pool, when the engine was built from >1 device
135    /// (#314 wiring). `None` for single-device engines (the common case). When
136    /// `Some` and non-single, the loaded model is built as a single pipeline
137    /// stage spanning `[0..num_hidden_layers)` with a layer→device map derived
138    /// from this pool. Held as `Option<Arc<...>>` because `TorchEngine` is
139    /// `Clone` and the pool is `Send + Sync` (it holds only `Device` values).
140    device_pool: Option<Arc<crate::runtime::DevicePool>>,
141    /// Runtime configuration - immutable after construction
142    config: RuntimeConfig,
143    /// Generation configuration with defaults
144    generation_config: GenerationConfig,
145    /// Loaded model information - protected by mutex
146    model_info: Arc<Mutex<ModelInfo>>,
147    /// Active LoRA adapter name - thread safe with mutex
148    pub active_lora: Arc<Mutex<Option<String>>>,
149    /// GPU sampler for efficient token sampling - thread safe after initialization
150    sampler: TensorSampler, // Renamed from gpu_sampler - works on both CPU and GPU
151    /// Cached tokenizer vocabulary size for lock-free access
152    /// 0 means not yet initialized
153    tokenizer_vocab_size: Arc<AtomicUsize>,
154    /// Cached EOS token ID for lock-free access
155    /// 0 means not yet initialized (actual EOS tokens are typically > 0)
156    eos_token_id: Arc<AtomicU32>,
157    /// KV cache registry for session-based cache isolation
158    /// Enables concurrent inference across multiple sessions
159    kv_cache_registry: Option<Arc<crate::runtime::kv_cache::KVCacheRegistry>>,
160    /// Current active session owner for KV cache selection
161    /// If set, generation uses the corresponding cache from the registry
162    active_cache_owner: Arc<Mutex<Option<crate::runtime::kv_cache::CacheOwner>>>,
163    /// KV-cache compatibility descriptor for the loaded model (#1277). The
164    /// authoritative identity every reusable/durable KV lookup is checked
165    /// against; `None` until a model is loaded. Built from the model config +
166    /// runtime config at load time, finalized with the tokenizer identity.
167    kv_compat: Arc<Mutex<Option<crate::runtime::kv_compat::KvCompatDescriptor>>>,
168    // Note: XET/LFS handled by git-xet-filter + ModelFactory::load_file_with_pointer_detection()
169    // Note: Pre-training not supported (persistent_model doesn't expose VarStore), LoRA only
170}
171
172/// Tokenize `text` with `tokenizer`, writing no caller content to the process
173/// log.
174///
175/// Token-burn counts (prompt token count) are now emitted as OTel metrics at the
176/// call site (`TextStream::new_with_delta`), not as log lines (#1261) — the log
177/// is no longer the metering source. This helper logs only on the two
178/// content-free diagnostic outcomes: an empty token sequence (failure) and a
179/// tokenize/decode roundtrip mismatch (a real tokenizer-bug signal), in both
180/// cases carrying only bounded lengths, never the text.
181///
182/// #1253: the prompt and its decoded token text are tenant-private and must
183/// never reach process-wide logs — a shared inference process exposes them
184/// outside the caller's request boundary. This helper is split out from
185/// [`TorchEngine::tokenize`] so its no-leak behavior is unit-testable without
186/// a full engine/model.
187fn tokenize_redacted(tokenizer: &Tokenizer, text: &str) -> Result<Vec<i64>> {
188    let bytes = text.len();
189    let chars = text.chars().count();
190
191    let encoding = tokenizer
192        .encode(text, false)
193        .map_err(|e| anyhow!("Tokenization failed: {}", e))?;
194    let token_ids: Vec<i64> = encoding.get_ids().iter().map(|&id| id as i64).collect();
195
196    if token_ids.is_empty() {
197        tracing::warn!(bytes, chars, "Tokenization produced empty token sequence");
198        return Err(anyhow!(
199            "Tokenization produced empty token sequence ({} bytes, {} chars)",
200            bytes,
201            chars
202        ));
203    }
204
205    // Surface roundtrip mismatches for diagnosis, but never the original or
206    // decoded text — only that they diverged and the bounded lengths.
207    if let Ok(decoded) = tokenizer.decode(
208        &token_ids.iter().map(|&id| id as u32).collect::<Vec<_>>(),
209        false,
210    ) {
211        if decoded != text {
212            tracing::warn!(
213                bytes,
214                chars,
215                tokens = token_ids.len(),
216                decoded_bytes = decoded.len(),
217                "Tokenization roundtrip mismatch (content redacted)"
218            );
219        }
220    }
221
222    Ok(token_ids)
223}
224
225/// Helper functions for tensor operations
226impl TorchEngine {
227    /// Set the random seed for deterministic generation
228    /// Useful for debugging - enables reproducible token sequences
229    pub fn set_seed(&self, seed: u64) {
230        TensorSampler::set_seed(seed);
231    }
232
233    /// Get cached vocabulary size without locking
234    pub fn get_vocab_size(&self) -> usize {
235        let size = self.tokenizer_vocab_size.load(Ordering::Relaxed);
236        if size > 0 {
237            size
238        } else {
239            // Fallback: try to get from tokenizer (shouldn't happen after load_tokenizer)
240            self.tokenizer
241                .lock()
242                .as_ref()
243                .map(|t| t.get_vocab_size(true))
244                .unwrap_or(0) // Return 0 if tokenizer not loaded (caller should error)
245        }
246    }
247
248    /// Get the device this engine is using for computation
249    pub fn device(&self) -> Device {
250        self.device
251    }
252
253    /// The resolved multi-device pool, if this engine was built with >1 device.
254    ///
255    /// Returns `None` for single-device engines (the common case, including the
256    /// legacy single-`gpu_device_id` path and CPU). When `Some` and non-single,
257    /// the loaded model is built as a single pipeline stage with a layer→device
258    /// map derived from this pool (#314 wiring). The pool's primary equals
259    /// [`Self::device`].
260    #[must_use]
261    pub fn device_pool(&self) -> Option<&crate::runtime::DevicePool> {
262        self.device_pool.as_deref()
263    }
264
265    /// Whether this engine is configured to build a multi-device pipeline.
266    ///
267    /// True exactly when [`Self::device_pool`] is `Some` and holds >1 device.
268    /// The model is constructed with `LayerDeviceMap::even_split` across the
269    /// pool; single-device engines return `false` and build the whole model on
270    /// [`Self::device`].
271    #[must_use]
272    pub fn is_multi_device(&self) -> bool {
273        self.device_pool.as_deref().is_some_and(|p| !p.is_single())
274    }
275
276    // ============================================================================
277    // Session-Based KV Cache Management
278    // ============================================================================
279
280    /// Initialize the KV cache registry for session-based cache isolation.
281    ///
282    /// This should be called after loading a model when the configuration is known.
283    /// The registry will manage separate KV caches for different sessions/requests.
284    pub fn initialize_kv_registry(
285        &mut self,
286        num_layers: usize,
287        max_seq_len: usize,
288        quant_type: hyprstream_rpc_std::model_client::KVQuantType,
289        memory_budget: Option<usize>,
290    ) {
291        let config = crate::runtime::kv_cache::CacheConfig::new(num_layers, max_seq_len)
292            .with_quant_type(quant_type);
293
294        let registry = crate::runtime::kv_cache::KVCacheRegistry::new(config, memory_budget);
295        self.kv_cache_registry = Some(Arc::new(registry));
296
297        tracing::info!(
298            "[TorchEngine] Initialized KV cache registry: {} layers, max_seq_len={}, budget={:?}",
299            num_layers,
300            max_seq_len,
301            memory_budget
302        );
303    }
304
305    /// Expected KV-cache compatibility fingerprint for the loaded model under the
306    /// given adapter/delta generation, or `None` when no authoritative identity is
307    /// available.
308    ///
309    /// The effective identity folds in the base-weight content digest, the full
310    /// effective architecture/runtime config, and the live `adapter_generation`
311    /// actually applied to the request. Returns `None` when no descriptor was
312    /// built (the base-weight digest could not be computed) or when the descriptor
313    /// is not yet authoritatively complete (tokenizer identity not yet folded in,
314    /// or tokenizer hashing failed). The session-reuse path treats `None` as
315    /// **fail-closed**: it discards a cache rather than reusing it under a
316    /// missing or uncertain identity — it never treats absence as "no gating".
317    pub fn kv_compat_fingerprint_for(
318        &self,
319        adapter_generation: u64,
320    ) -> Option<crate::runtime::kv_compat::KvCompatFingerprint> {
321        let guard = self.kv_compat.lock();
322        let desc = guard.as_ref()?;
323        if !desc.is_authoritatively_complete() {
324            return None;
325        }
326        let mut effective = desc.clone();
327        effective.weights.adapter_generation = adapter_generation;
328        Some(crate::runtime::kv_compat::KvCompatFingerprint::of(
329            &effective,
330        ))
331    }
332
333    /// Compute a KV cache memory budget based on model size and available GPU memory.
334    ///
335    /// Uses `HYPRSTREAM_KV_BUDGET_MB` env var if set, otherwise estimates:
336    /// GPU total (from common card sizes) minus model weights, times 0.7.
337    /// Returns None on CPU (no budget needed).
338    pub fn compute_kv_budget(&self) -> Option<usize> {
339        if self.device == Device::Cpu {
340            return None;
341        }
342
343        // Check for explicit override
344        if let Ok(mb_str) = std::env::var("HYPRSTREAM_KV_BUDGET_MB") {
345            if let Ok(mb) = mb_str.parse::<usize>() {
346                tracing::info!("KV cache budget from HYPRSTREAM_KV_BUDGET_MB: {} MB", mb);
347                return Some(mb * 1024 * 1024);
348            }
349        }
350
351        let model_bytes = self.model_memory_usage();
352
353        // Estimate total GPU memory from common card sizes.
354        // Without cudaMemGetInfo FFI, we use a conservative heuristic:
355        // assume the GPU has enough memory to hold the model + some headroom.
356        // We estimate total VRAM as model_bytes * 1.75 (model is typically ~55-60% of VRAM).
357        // This is conservative — on larger GPUs it underestimates, leaving more headroom.
358        let estimated_total = model_bytes * 7 / 4; // 1.75x model size
359        let remaining = estimated_total.saturating_sub(model_bytes);
360        let budget = remaining * 7 / 10; // 70% of remaining
361
362        tracing::info!(
363            "KV cache budget: {} MB (estimated from {} MB model weights, ~{} MB total VRAM)",
364            budget / (1024 * 1024),
365            model_bytes / (1024 * 1024),
366            estimated_total / (1024 * 1024),
367        );
368
369        Some(budget)
370    }
371
372    /// Get the KV cache registry (for external access)
373    pub fn kv_registry(&self) -> Option<Arc<crate::runtime::kv_cache::KVCacheRegistry>> {
374        self.kv_cache_registry.clone()
375    }
376
377    /// Set the active session for generation.
378    ///
379    /// This sets which KV cache will be used for the next generation call.
380    /// Use `CacheOwner::Session(session_id)` for conversational sessions that
381    /// should preserve context across multiple requests.
382    ///
383    /// # Example
384    /// ```ignore
385    /// // For a conversational session (preserves context)
386    /// engine.set_session(CacheOwner::Session("conversation-123".into()));
387    ///
388    /// // For a stateless request (cache discarded after)
389    /// engine.set_session(CacheOwner::new_stateless());
390    /// ```
391    pub fn set_session(&self, owner: crate::runtime::kv_cache::CacheOwner) -> Result<()> {
392        if self.kv_cache_registry.is_none() {
393            return Err(anyhow!(
394                "KV cache registry not initialized. Call initialize_kv_registry first."
395            ));
396        }
397
398        let mut active_owner = self.active_cache_owner.lock();
399
400        tracing::debug!("Setting active session to: {:?}", owner);
401        *active_owner = Some(owner);
402
403        Ok(())
404    }
405
406    /// Get the current active session owner
407    pub fn get_session(&self) -> Option<crate::runtime::kv_cache::CacheOwner> {
408        self.active_cache_owner.lock().clone()
409    }
410
411    /// Clear the active session (subsequent generations will use the model's internal cache)
412    pub fn clear_session(&self) -> Result<()> {
413        let mut active_owner = self.active_cache_owner.lock();
414        *active_owner = None;
415        Ok(())
416    }
417
418    /// Release a session's KV cache (for cleanup after conversation ends)
419    pub fn release_session(&self, owner: &crate::runtime::kv_cache::CacheOwner) -> Result<()> {
420        if let Some(registry) = &self.kv_cache_registry {
421            registry.release(owner);
422            tracing::debug!("Released session cache: {:?}", owner);
423        }
424        Ok(())
425    }
426
427    /// Get total memory usage across all session caches
428    pub fn session_cache_memory_usage(&self) -> usize {
429        self.kv_cache_registry
430            .as_ref()
431            .map(|r| r.total_memory_usage())
432            .unwrap_or(0)
433    }
434
435    /// Get number of active session caches
436    pub fn active_session_count(&self) -> usize {
437        self.kv_cache_registry
438            .as_ref()
439            .map(|r| r.cache_count())
440            .unwrap_or(0)
441    }
442
443    /// Evict LRU session caches to stay within memory budget
444    pub fn evict_session_caches(&self) {
445        if let Some(registry) = &self.kv_cache_registry {
446            registry.evict_to_budget();
447        }
448    }
449
450    /// Swap the active session's KV cache into the model.
451    ///
452    /// If an active session owner is set and the registry is initialized,
453    /// retrieves (or creates) the session's cache from the registry and
454    /// installs it in the model via `set_kv_cache()`. Returns true if
455    /// a session cache was swapped in.
456    ///
457    /// `tenant_id` is the effective tenant whose delta contributed to this
458    /// request's weights (`None` when no tenant delta contributes), and
459    /// `weight_epoch` is that delta's current weight epoch (ignored when
460    /// `tenant_id` is `None`). The pair is registered against the owner cache
461    /// *before* the cache is installed in the model, so the cache is tracked
462    /// for invalidation (and cleared if its previously-computed KV is stale
463    /// w.r.t. a different tenant or an older weight epoch) before generation
464    /// can read a single token from it.
465    ///
466    /// **Atomicity invariant (F2):** the register→install sequence below is
467    /// *not* guarded by a lock spanning both operations. It is race-free today
468    /// because `InferenceService` runs on a single-threaded `current_thread`
469    /// Tokio runtime inside a `LocalSet`, and this function is await-free — no
470    /// other request task can interleave between `register_delta_dependency`
471    /// and `set_kv_cache`, so the pair executes atomically with respect to
472    /// other requests. If `InferenceService` ever moves to a multi-threaded
473    /// runtime (or an `.await` is introduced here), this invariant breaks and a
474    /// spanning lock or an epoch handshake would be required.
475    pub fn swap_session_cache(&self, tenant_id: Option<String>, weight_epoch: u64) -> bool {
476        let owner = match self.active_cache_owner.lock().clone() {
477            Some(o) => o,
478            None => return false,
479        };
480        let registry = match &self.kv_cache_registry {
481            Some(r) => r,
482            None => return false,
483        };
484        let model_arc = match &self.persistent_model {
485            Some(m) => m,
486            None => return false,
487        };
488
489        let session_cache = registry.get_or_create(owner.clone());
490        // Register the delta dependency BEFORE installing the cache. This both
491        // records the owner for tenant-delta invalidation and, if the effective
492        // tenant changed since this cache was last computed, clears the stale KV
493        // in place. Ordering matters: registration/staleness-clear precede
494        // set_kv_cache, so the model can never publish output from an untracked
495        // or weight-stale cache.
496        let cleared_stale = registry.register_delta_dependency(&owner, tenant_id, weight_epoch);
497        let mut model = model_arc.lock();
498        model.set_kv_cache(session_cache);
499        tracing::debug!(
500            "Swapped session KV cache into model (stale_cleared={})",
501            cleared_stale
502        );
503        true
504    }
505
506    /// Estimate model memory usage in bytes.
507    ///
508    /// Sums the byte sizes of all parameters in the VarStore.
509    pub fn model_memory_usage(&self) -> usize {
510        let vs_guard = self.var_store.lock();
511        {
512            if let Some(vs) = vs_guard.as_ref() {
513                return vs
514                    .variables()
515                    .values()
516                    .map(|t| {
517                        let numel = t.numel();
518                        let elem_size = match t.kind() {
519                            tch::Kind::Half | tch::Kind::BFloat16 => 2,
520                            tch::Kind::Double => 8,
521                            tch::Kind::Int8 | tch::Kind::Uint8 => 1,
522                            _ => 4, // Float and others
523                        };
524                        numel * elem_size
525                    })
526                    .sum();
527            }
528        }
529        0
530    }
531
532    /// Create new PyTorch engine
533    pub fn new(config: RuntimeConfig) -> Result<Self> {
534        Self::new_sync(config)
535    }
536
537    /// Create new PyTorch engine (async version)
538    pub async fn new_async(config: RuntimeConfig) -> Result<Self> {
539        Self::new_sync(config)
540    }
541
542    /// Internal sync constructor
543    fn new_sync(config: RuntimeConfig) -> Result<Self> {
544        // Multi-GPU foundation (#313/#314, epic #310): when devices are
545        // *explicitly* requested via `RuntimeConfig.devices` /
546        // `HYPRSTREAM_GPU_DEVICES`, resolve them through `DevicePool`, which
547        // validates them fail-fast (no silent CPU downgrade). When the pool
548        // holds >1 device the engine keeps the whole pool and threads it into
549        // model construction so the model is built as a single multi-device
550        // pipeline stage (layers spread via `LayerDeviceMap::even_split`,
551        // `forward_layers` performs the stage-boundary copies). Single-pool and
552        // legacy single-`gpu_device_id` paths are unchanged.
553        if config.use_gpu {
554            if let Some(indices) = config.resolve_explicit_multi_device_indices()? {
555                let pool = crate::runtime::DevicePool::from_cuda_indices(&indices)?;
556                let device = pool.primary();
557                if pool.is_single() {
558                    info!(
559                        "🚀 DevicePool resolved a single device {:?}; building single-device engine",
560                        device
561                    );
562                    return Self::with_device(config, device, None);
563                }
564                info!(
565                    "🚀 Multi-GPU DevicePool resolved {} device(s) {:?}; building a multi-device \
566                     pipeline (primary {:?} hosts embeddings/lm_head)",
567                    pool.len(),
568                    pool.devices(),
569                    device
570                );
571                return Self::with_device(config, device, Some(Arc::new(pool)));
572            }
573        }
574
575        // Determine device based on configuration
576        let device = if config.use_gpu {
577            // Use specified GPU device ID, or auto-detect
578            let gpu_device = if let Some(device_id) = config.gpu_device_id {
579                // A GPU was *explicitly* requested. Check if CUDA/ROCm is available.
580                if Device::cuda_if_available() != Device::Cpu {
581                    Device::Cuda(device_id)
582                } else if config.strict_device {
583                    // No-fragile-fallbacks (#315): a process told to use GPU N that
584                    // silently lands on CPU tanks the pipeline. Fail fast instead,
585                    // and surface the *actual* reason CUDA was unavailable so the
586                    // operator can fix it (driver lib / wrong backend / mismatch).
587                    let diag = cuda_unavailable_diagnostics();
588                    return Err(anyhow!(
589                        "GPU {device_id} explicitly requested but CUDA/ROCm is not available; \
590                         refusing to silently fall back to CPU (strict_device). \
591                         Set HYPRSTREAM_STRICT_DEVICE=0 to allow CPU fallback. \
592                         Diagnostics: {diag}"
593                    ));
594                } else {
595                    warn!(
596                        "⚠️  GPU {} requested but CUDA/ROCm not available, falling back to CPU. {}",
597                        device_id,
598                        cuda_unavailable_diagnostics()
599                    );
600                    Device::Cpu
601                }
602            } else {
603                // Pure auto-detect ("use a GPU if there is one"): CPU fallback is
604                // the intended behavior here even under strict_device, since no
605                // specific device was requested.
606                Device::cuda_if_available()
607            };
608
609            if gpu_device != Device::Cpu {
610                // Check if this is actually ROCm/HIP
611                let device_id = match gpu_device {
612                    Device::Cuda(id) => id,
613                    _ => 0,
614                };
615                if tch::utils::has_hip() {
616                    info!("🚀 Using ROCm/HIP GPU {} acceleration", device_id);
617                } else {
618                    info!("🚀 Using CUDA GPU {} acceleration", device_id);
619                }
620                gpu_device
621            } else {
622                // The live RTX 5090 / cuda130 node hit exactly this branch: a
623                // bare "not available" with no cause. Always surface *why* now
624                // (most commonly libcuda.so.1 missing from LD_LIBRARY_PATH).
625                warn!(
626                    "⚠️  GPU requested but not available, falling back to CPU. {}",
627                    cuda_unavailable_diagnostics()
628                );
629                Device::Cpu
630            }
631        } else {
632            info!("💻 Using CPU inference");
633            Device::Cpu
634        };
635
636        Self::with_device(config, device, None)
637    }
638
639    /// Construct an engine pinned to an already-resolved primary [`Device`].
640    ///
641    /// Pass `device_pool = Some(pool)` with a multi-device pool to build the
642    /// model as a single pipeline stage spanning the whole decoder stack with a
643    /// layer→device map (the #314 multi-device path). The engine's `device`
644    /// remains the pool's primary (first) device — the home of embeddings, final
645    /// norm, and lm_head, and where inputs land before the first layer.
646    /// `device_pool = None` (or a single-device pool) preserves the original
647    /// whole-model-on-one-device behavior.
648    fn with_device(
649        config: RuntimeConfig,
650        device: Device,
651        device_pool: Option<Arc<crate::runtime::DevicePool>>,
652    ) -> Result<Self> {
653        Ok(Self {
654            var_store: Arc::new(Mutex::new(None)),
655            model_architecture: Arc::new(Mutex::new(None)),
656            persistent_model: None,
657            context_state: Arc::new(Mutex::new(None)),
658            tokenizer: Arc::new(Mutex::new(None)),
659            template_engine: Arc::new(Mutex::new(None)),
660            device,
661            device_pool,
662            config: config.clone(),
663            generation_config: GenerationConfig {
664                max_tokens: 2048,
665                temperature: 0.7,
666                top_p: 0.9,
667                top_k: Some(40),
668                repeat_penalty: 1.1,
669                stop_tokens: vec!["</s>".to_owned()],
670                seed: None,
671                stream: false,
672            },
673            model_info: Arc::new(Mutex::new(ModelInfo {
674                name: "unloaded".to_owned(),
675                architecture: "unknown".to_owned(),
676                parameters: Some(0),
677                context_length: 2048,
678                vocab_size: 32000,
679                hidden_size: 768,
680                ..Default::default()
681            })),
682            active_lora: Arc::new(Mutex::new(None)),
683            sampler: TensorSampler::new(device),
684            tokenizer_vocab_size: Arc::new(AtomicUsize::new(0)),
685            eos_token_id: Arc::new(AtomicU32::new(0)),
686            kv_cache_registry: None, // Initialized after model load when config is known
687            active_cache_owner: Arc::new(Mutex::new(None)),
688            kv_compat: Arc::new(Mutex::new(None)), // Built from model config at load time (#1277)
689        })
690    }
691
692    /// Load model from safetensors or torchscript
693    async fn load_model_file(&mut self, path: &Path) -> Result<()> {
694        let ext = path
695            .extension()
696            .and_then(|e| e.to_str())
697            .ok_or_else(|| anyhow!("Invalid file extension"))?;
698
699        match ext {
700            "pt" | "pth" => {
701                return Err(anyhow!("TorchScript models (.pt/.pth) are no longer supported. Please use SafeTensors format (.safetensors) instead."));
702            }
703            "safetensors" => {
704                // Load SafeTensors model directly
705                info!("Loading weights: {}", path.display());
706                self.load_safetensors(path).await?;
707            }
708            _ => {
709                return Err(anyhow!("Unsupported model format: {}", ext));
710            }
711        }
712
713        Ok(())
714    }
715
716    /// Load SafeTensors model using ModelFactory for unified weight loading
717    async fn load_safetensors(&mut self, path: &Path) -> Result<()> {
718        info!("Loading weights: {}", path.display());
719
720        // Get the model directory (parent of the safetensors file)
721        let model_dir = path.parent().unwrap_or(path);
722
723        // Use ModelFactory to load the model (handles all weight loading internally)
724        self.initialize_persistent_model(model_dir).await?;
725
726        // Extract model info from the loaded model
727        if let Some(model) = &self.persistent_model {
728            let _model_guard = model.lock();
729
730            // Get architecture info from the model
731            // For now, use a default since we don't have a method to query this
732            let architecture = "auto".to_owned();
733
734            // Update model info
735            {
736                let mut model_info_guard = self.model_info.lock();
737                model_info_guard.architecture = architecture.clone();
738            }
739
740            // Set architecture
741            {
742                let mut arch_guard = self.model_architecture.lock();
743                *arch_guard = Some(architecture.clone());
744            }
745        }
746
747        // Get context window from model info that was populated from config
748        let context_window = self.effective_context_limit()?;
749
750        // Initialize context state
751        {
752            let mut context_guard = self.context_state.lock();
753            *context_guard = Some(ContextState {
754                sequence_length: 0,
755                context_window,
756                initialized: true,
757            });
758        }
759
760        // Create dummy VarStore for backward compatibility
761        {
762            let vs = VarStore::new(self.device);
763            let mut var_store_guard = self.var_store.lock();
764            *var_store_guard = Some(vs);
765        }
766
767        info!("✅ SafeTensors model loaded via ModelFactory");
768        info!("🚀 Model initialized and ready for inference");
769        Ok(())
770    }
771
772    /// Get tensor from VarStore by name (for inference) - thread safe
773    pub fn get_tensor(&self, name: &str) -> Option<Tensor> {
774        let var_store_guard = self.var_store.lock();
775        let vs = var_store_guard.as_ref()?;
776        vs.variables().get(name).map(tch::Tensor::shallow_clone)
777    }
778
779    /// List all available tensor names in VarStore - thread safe
780    pub fn list_tensor_names(&self) -> Vec<String> {
781        let var_store_guard = self.var_store.lock();
782        if let Some(vs) = var_store_guard.as_ref() {
783            vs.variables().keys().cloned().collect()
784        } else {
785            Vec::new()
786        }
787    }
788
789    /// Check if model is loaded via VarStore - thread safe
790    pub fn has_varstore(&self) -> bool {
791        self.var_store.lock().is_some()
792    }
793
794    /// Authoritative content digest of the base weights actually loaded from
795    /// `model_path` — a canonical SHA-256 over the **loader-selected**
796    /// `.safetensors` shards' **resolved** bytes (shard count + length-prefixed
797    /// name + per-shard content digest), in sorted name order. This is the
798    /// base-weight identity for the KV-compat fingerprint (#1277): two snapshots
799    /// sharing a basename but differing in content diverge here, unlike a
800    /// path/label.
801    ///
802    /// The shard set comes from the **same selector the loader uses**
803    /// (`ModelFactory::find_shard_files_for_digest`, the digest-authoritative
804    /// variant of the loader's selector: `model.safetensors.index.json` →
805    /// `model.safetensors` → shard-name glob), so the digest covers EXACTLY the
806    /// shards the loader reads — an extra/unreferenced `.safetensors` in the
807    /// directory (stale shard, alt-precision copy) cannot shift the fingerprint
808    /// while the loaded weights are identical. Unlike the loader, the digest
809    /// REJECTS an ambiguous manifest-less multi-shard glob set regardless of
810    /// `HYPRSTREAM_STRICT_LOADER` — fingerprint authority is stricter than
811    /// loader permissiveness.
812    ///
813    /// The digest hashes the **resolved** bytes — the identical resolution path
814    /// the loader walks (`ModelFactory::resolve_weight_for_digest` transparently
815    /// resolves an un-smudged LFS/XET pointer to its object content), so it never
816    /// hashes a pointer stub in place of the real tensors. It is **fail-closed**:
817    /// any selection/read/resolve error, an empty selection, an
818    /// index-referenced shard that is missing, or an ambiguous manifest-less
819    /// multi-shard set propagates, so the caller declines to mint a descriptor
820    /// and KV reuse fails closed rather than hashing a partial shard set.
821    async fn weights_content_digest(model_path: &Path) -> Result<String> {
822        use crate::runtime::kv_compat;
823        use crate::runtime::model_factory::{ModelFactory, ResolvedWeight};
824
825        // Resolve the loader-selected shard set (the digest-authoritative
826        // variant of the loader's own selector).
827        let shards = ModelFactory::find_shard_files_for_digest(model_path)?;
828
829        // Fail closed on an empty or inconsistent selection: the loader would
830        // have nothing authoritative to load (or the index disagrees with the
831        // directory), so reuse must be declined rather than minted over a
832        // partial/ambiguous set.
833        if shards.is_empty() {
834            anyhow::bail!(
835                "no loader-selected .safetensors weight shards in {}",
836                model_path.display()
837            );
838        }
839        for shard in &shards {
840            if !shard.is_file() {
841                anyhow::bail!(
842                    "loader-selected shard {} is missing or not a regular file",
843                    shard.display()
844                );
845            }
846        }
847
848        let mut digested: Vec<(String, [u8; 32])> = Vec::with_capacity(shards.len());
849        for shard in &shards {
850            let name = shard
851                .file_name()
852                .and_then(|s| s.to_str())
853                .unwrap_or("")
854                .to_owned();
855            // Resolve transparently (LFS/XET pointer → object content), then
856            // hash the resolved bytes. Large ordinary shards stream; resolved
857            // pointers and small files hash their owned bytes.
858            let shard_digest = match ModelFactory::resolve_weight_for_digest(shard).await? {
859                ResolvedWeight::Owned(bytes) => kv_compat::shard_content_digest(&bytes),
860                ResolvedWeight::File(file) => Self::stream_file_digest(file)?,
861            };
862            digested.push((name, shard_digest));
863        }
864        Ok(kv_compat::base_weight_digest_from_shards(&digested))
865    }
866
867    /// SHA-256 of a file's bytes, streamed in 64 KiB chunks so a multi-GB shard
868    /// is never fully buffered into memory just to hash it. Returns the resolved
869    /// read error (fail-closed) rather than a partial digest.
870    fn stream_file_digest(file: std::fs::File) -> std::result::Result<[u8; 32], std::io::Error> {
871        use sha2::Digest;
872        use std::io::Read;
873        let mut hasher = sha2::Sha256::new();
874        let mut f = file;
875        let mut buf = [0u8; 65536];
876        loop {
877            let n = f.read(&mut buf)?;
878            if n == 0 {
879                break;
880            }
881            hasher.update(&buf[..n]);
882        }
883        let mut out = [0u8; 32];
884        out.copy_from_slice(hasher.finalize().as_slice());
885        Ok(out)
886    }
887
888    /// Initialize XET storage with default configuration
889    /// Initialize the persistent model instance using ModelFactory
890    async fn initialize_persistent_model(&mut self, model_path: &Path) -> Result<()> {
891        use crate::runtime::model_config::ModelConfig;
892        use crate::runtime::model_factory::ModelFactory;
893        use crate::runtime::torch_utils::preflight_gpu_check;
894
895        info!("Initializing model");
896
897        // XET/LFS handled automatically by git-xet-filter + ModelFactory fallback
898        // Load model config first to get model parameters
899        let empty_weights = HashMap::new();
900        let config = ModelConfig::load(model_path, &empty_weights)?;
901
902        // Effective context length (CLI override or model default)
903        let effective_max_context = self
904            .config
905            .max_context
906            .map(|v| v as usize)
907            .unwrap_or(config.max_position_embeddings);
908        if self.config.max_context.is_some() {
909            info!(
910                "Using max_context override: {} tokens (model default: {})",
911                effective_max_context, config.max_position_embeddings
912            );
913        }
914
915        // Estimate model memory requirements
916        let estimated_weights_mb = {
917            // Rough estimate: vocab_size * hidden_size (embeddings)
918            //                 + num_layers * hidden_size * intermediate_size * 3 (MLP)
919            //                 + num_layers * hidden_size * hidden_size * 4 (attention)
920            let embedding_params = config.vocab_size * config.hidden_size;
921            let mlp_params_per_layer = config.hidden_size * config.intermediate_size * 3;
922            let attn_params_per_layer = config.hidden_size * config.hidden_size * 4;
923            let params_per_layer = mlp_params_per_layer + attn_params_per_layer;
924            let total_params = embedding_params + (config.num_hidden_layers * params_per_layer);
925
926            // BF16 = 2 bytes per parameter
927            (total_params * 2) as f64 / (1024.0 * 1024.0)
928        };
929
930        let kv_cache_mb = {
931            // KV cache: 2 (keys+values) * num_layers * batch_size * max_seq_len * num_heads * head_dim * 2 (BF16)
932            let batch_size = 1;
933            let kv_per_layer = 2
934                * batch_size
935                * effective_max_context
936                * config.num_attention_heads
937                * config.head_dim
938                * 2;
939            let total_kv = config.num_hidden_layers * kv_per_layer;
940            total_kv as f64 / (1024.0 * 1024.0)
941        };
942
943        let total_estimated_mb = estimated_weights_mb + kv_cache_mb;
944
945        info!(
946            "Model memory estimate:\n\
947             - Weights: {:.2} MB\n\
948             - KV cache: {:.2} MB (max_seq_len={})\n\
949             - Total: {:.2} MB",
950            estimated_weights_mb, kv_cache_mb, effective_max_context, total_estimated_mb
951        );
952
953        // Pre-flight GPU memory check (best-effort)
954        if let Err(e) = preflight_gpu_check(self.device, total_estimated_mb) {
955            warn!("GPU memory pre-flight check failed: {}", e);
956            // Continue anyway - the check might not be accurate
957        }
958
959        // Update ModelInfo with actual values from config
960        {
961            let mut model_info = self.model_info.lock();
962            model_info.hidden_size = config.hidden_size as u32;
963            model_info.intermediate_size = Some(config.intermediate_size as u32);
964            model_info.num_attention_heads = Some(config.num_attention_heads as u32);
965            model_info.num_key_value_heads = Some(config.num_key_value_heads as u32);
966            model_info.head_dim = Some(config.head_dim as u32);
967            model_info.num_hidden_layers = Some(config.num_hidden_layers as u32);
968            model_info.vocab_size = config.vocab_size as u32;
969            model_info.context_length = effective_max_context as u32;
970            model_info.architecture = config.model_type.clone();
971        }
972
973        // Build the KV-cache compatibility descriptor (#1277): the authoritative
974        // identity of the model + runtime config that produces this engine's KV.
975        // Every reusable/durable KV lookup is checked against its fingerprint.
976        // Built here (where the full `ModelConfig` is in scope) from authoritative
977        // inputs only — never a human label. The base-weight identity is a content
978        // digest of the resolved shards; the tokenizer identity is folded in later
979        // by `load_tokenizer`, before any generation reads the fingerprint. If no
980        // authoritative base digest can be computed, the descriptor is left unset
981        // and the cache boundary fails closed (KV recomputed, not reused).
982        match Self::weights_content_digest(model_path).await {
983            Ok(base_revision) => {
984                use crate::runtime::kv_compat::{
985                    KvCompatDescriptor, KvDtype, KvQuantMode, MoeRouting, RopeScalingFp,
986                    WeightIdentity, KV_COMPAT_FORMAT_VERSION,
987                };
988                let kv_quant = match self.config.kv_quant_type {
989                    hyprstream_rpc_std::model_client::KVQuantType::None => KvQuantMode::None,
990                    hyprstream_rpc_std::model_client::KVQuantType::Int8 => KvQuantMode::Int8,
991                    hyprstream_rpc_std::model_client::KVQuantType::Nf4 => KvQuantMode::Nf4,
992                    hyprstream_rpc_std::model_client::KVQuantType::Fp4 => KvQuantMode::Fp4,
993                };
994                // `model_name` is a display label for mismatch messages; the
995                // authoritative weight identity is the content digest above.
996                let model_name = model_path
997                    .file_stem()
998                    .and_then(|s| s.to_str())
999                    .unwrap_or("")
1000                    .to_owned();
1001                let descriptor = KvCompatDescriptor {
1002                    format_version: KV_COMPAT_FORMAT_VERSION,
1003                    weights: WeightIdentity::base_only(base_revision),
1004                    model_name,
1005                    architecture: config.model_type.clone(),
1006                    model_version: config.version,
1007                    num_hidden_layers: config.num_hidden_layers,
1008                    num_attention_heads: config.num_attention_heads,
1009                    num_key_value_heads: config.num_key_value_heads,
1010                    head_dim: config.head_dim,
1011                    hidden_size: config.hidden_size,
1012                    // Effective attention behavior — every field below can change
1013                    // KV *values* without changing coarse geometry (#1277 review).
1014                    use_qk_norm: config.use_qk_norm,
1015                    partial_rotary_factor_bits: config.partial_rotary_factor.map(f32::to_bits),
1016                    layer_types: config.layer_types.clone(),
1017                    linear_key_head_dim: config.linear_key_head_dim,
1018                    linear_value_head_dim: config.linear_value_head_dim,
1019                    linear_num_key_heads: config.linear_num_key_heads,
1020                    linear_num_value_heads: config.linear_num_value_heads,
1021                    linear_conv_kernel_dim: config.linear_conv_kernel_dim,
1022                    // Effective model config — each demonstrably changes hidden
1023                    // states / downstream K/V without changing coarse geometry
1024                    // (#1277 review finding F3): normalization epsilon, activation,
1025                    // embedding/attention scaling, MoE routing, and local-RoPE /
1026                    // sliding-window. Taken verbatim from the resolved ModelConfig
1027                    // that constructs the model.
1028                    rms_norm_eps_bits: config.rms_norm_eps.to_bits(),
1029                    layer_norm_eps_bits: config.layer_norm_eps.map(f32::to_bits),
1030                    hidden_activation: config.hidden_activation.clone(),
1031                    scale_embeddings: config.scale_embeddings,
1032                    query_pre_attn_scalar_bits: config.query_pre_attn_scalar.map(f32::to_bits),
1033                    moe: MoeRouting {
1034                        is_moe: config.is_moe,
1035                        num_experts: config.num_experts,
1036                        num_experts_per_tok: config.num_experts_per_tok,
1037                        moe_intermediate_size: config.moe_intermediate_size,
1038                        shared_expert_intermediate_size: config.shared_expert_intermediate_size,
1039                    },
1040                    sliding_window: config.sliding_window,
1041                    rope_local_base_freq_bits: config.rope_local_base_freq.map(f32::to_bits),
1042                    rope_theta_bits: config.rope_theta.to_bits(),
1043                    rope_scaling: config
1044                        .rope_scaling
1045                        .as_ref()
1046                        .map(|rs| RopeScalingFp::new(&rs.rope_type, rs.factor)),
1047                    max_position_embeddings: config.max_position_embeddings,
1048                    dtype: KvDtype::from_name(&config.dtype),
1049                    kv_quant,
1050                    block_size: crate::runtime::kv_cache::BLOCK_SIZE,
1051                    max_context: effective_max_context,
1052                    vocab_size: config.vocab_size,
1053                    // Folded in by `load_tokenizer`; descriptor is non-authoritative
1054                    // (fingerprint = None) until then, so reuse fails closed.
1055                    tokenizer_hash: None,
1056                };
1057                *self.kv_compat.lock() = Some(descriptor);
1058            }
1059            Err(e) => {
1060                warn!(
1061                    "Cannot compute authoritative base-weight digest for {}: {} \
1062                     — KV-cache reuse disabled (fail-closed)",
1063                    model_path.display(),
1064                    e
1065                );
1066                // Leave kv_compat unset (None): the reuse boundary will discard
1067                // rather than adopt any cache, since there is no authoritative
1068                // identity to compare against.
1069                *self.kv_compat.lock() = None;
1070            }
1071        }
1072
1073        // Use the factory to create the model. When `device_pool` is set and
1074        // multi-device, the factory builds the model as a single pipeline stage
1075        // spanning all decoder layers with a layer→device map (#314 wiring).
1076        let factory_start = std::time::Instant::now();
1077        let model = ModelFactory::create(
1078            model_path,
1079            &self.device,
1080            resolve_inference_compute_kind(
1081                self.config.precision_mode.as_deref(),
1082                self.device,
1083                super::channel_fp8::ChannelFp8Profile::read(model_path)?.is_some(),
1084            )?,
1085            self.config.max_context.map(|v| v as usize),
1086            self.config.kv_quant_type,
1087            self.device_pool.as_deref(),
1088            self.config.fp8_dequant_load,
1089            self.config.speculative_decoding,
1090        )
1091        .await?;
1092        let factory_time = factory_start.elapsed();
1093        info!(
1094            "✅ Model weights loaded in {:.2}s",
1095            factory_time.as_secs_f64()
1096        );
1097
1098        self.persistent_model = Some(Arc::new(Mutex::new(model)));
1099
1100        // Load generation_config.json if present (model-specific sampling defaults)
1101        let gen_config_path = model_path.join("generation_config.json");
1102        if gen_config_path.exists() {
1103            match std::fs::read_to_string(&gen_config_path) {
1104                Ok(contents) => {
1105                    if let Ok(json) = serde_json::from_str::<serde_json::Value>(&contents) {
1106                        if let Some(t) = json["temperature"].as_f64() {
1107                            self.generation_config.temperature = t as f32;
1108                        }
1109                        if let Some(p) = json["top_p"].as_f64() {
1110                            self.generation_config.top_p = p as f32;
1111                        }
1112                        if let Some(k) = json["top_k"].as_u64() {
1113                            self.generation_config.top_k = Some(k as usize);
1114                        }
1115                        if let Some(rp) = json["repetition_penalty"].as_f64() {
1116                            self.generation_config.repeat_penalty = rp as f32;
1117                        }
1118                        if let Some(mt) = json["max_new_tokens"].as_u64() {
1119                            self.generation_config.max_tokens = mt as usize;
1120                        } else if let Some(ml) = json["max_length"].as_u64() {
1121                            self.generation_config.max_tokens = ml as usize;
1122                        }
1123                    }
1124                }
1125                Err(e) => warn!("Failed to read generation_config.json: {}", e),
1126            }
1127        }
1128
1129        if !gen_config_path.exists() && config.model_type.contains("qwen3_5") {
1130            // Qwen3.5 recommended defaults (non-thinking mode) when no
1131            // generation_config.json is present. Qwen3 models ship their own
1132            // generation_config.json so this only applies to Qwen3.5.
1133            self.generation_config.temperature = 0.7;
1134            self.generation_config.top_p = 0.8;
1135            self.generation_config.top_k = Some(20);
1136            self.generation_config.repeat_penalty = 1.5;
1137        }
1138
1139        info!("Generation config: temperature={}, top_p={}, top_k={:?}, repeat_penalty={} (model_type={})",
1140            self.generation_config.temperature, self.generation_config.top_p,
1141            self.generation_config.top_k, self.generation_config.repeat_penalty,
1142            config.model_type);
1143
1144        Ok(())
1145    }
1146
1147    /// Load tokenizer and template configuration - thread safe
1148    #[instrument(name = "torch_engine.load_tokenizer", skip(self), fields(model_path = %model_path.display()))]
1149    async fn load_tokenizer(&mut self, model_path: &Path) -> Result<()> {
1150        // Try to find tokenizer.json - if model_path is a directory, look inside it
1151        // If it's a file, look in the parent directory
1152        let search_dir = if model_path.is_dir() {
1153            model_path
1154        } else {
1155            model_path.parent().unwrap_or(model_path)
1156        };
1157
1158        let tokenizer_path = search_dir.join("tokenizer.json");
1159        let tokenizer_config_path = search_dir.join("tokenizer_config.json");
1160
1161        if tokenizer_path.exists() {
1162            info!("Loading tokenizer: {}", tokenizer_path.display());
1163            let mut tokenizer = Tokenizer::from_file(&tokenizer_path)
1164                .map_err(|e| anyhow!("Failed to load tokenizer: {}", e))?;
1165
1166            // Log initial vocabulary size
1167            let initial_vocab_size = tokenizer.get_vocab_size(true);
1168            info!("Initial tokenizer vocabulary size: {}", initial_vocab_size);
1169
1170            // Get model's configured vocab size
1171            let model_vocab_size = {
1172                let model_info = self.model_info.lock();
1173                model_info.vocab_size as usize
1174            };
1175
1176            // Apply model-specific tokenizer configuration if model is loaded
1177            if let Some(model) = &self.persistent_model {
1178                let model_guard = model.lock();
1179                let tokenizer_config = model_guard.get_tokenizer_config();
1180
1181                // Configure the tokenizer based on the model architecture
1182                tokenizer_config.configure_tokenizer(&mut tokenizer, model_vocab_size)?;
1183
1184                tracing::debug!("Applied model-specific tokenizer configuration");
1185            } else {
1186                // Model not loaded yet - log vocab mismatch if any
1187                if model_vocab_size != initial_vocab_size {
1188                    tracing::debug!(
1189                        "Vocabulary size mismatch detected (model: {}, tokenizer: {}), but model not loaded yet to apply configuration",
1190                        model_vocab_size, initial_vocab_size
1191                    );
1192                }
1193            }
1194
1195            // Cache the final vocabulary size for lock-free access
1196            let final_vocab_size = tokenizer.get_vocab_size(true);
1197            self.tokenizer_vocab_size
1198                .store(final_vocab_size, Ordering::Relaxed);
1199
1200            info!("Final tokenizer vocabulary size: {}", final_vocab_size);
1201
1202            // Fold the tokenizer identity into the KV-compat descriptor (#1277).
1203            // Cached token ids are only meaningful relative to one tokenizer, so
1204            // a different tokenizer (even with the same vocab size) must reject.
1205            // The on-disk `tokenizer.json` bytes are the authoritative identity;
1206            // deterministic model-driven mutation (e.g. Qwen vocab padding) is
1207            // already captured by `vocab_size`. FAIL-CLOSED: if the bytes cannot
1208            // be hashed, the tokenizer identity is left unset, the descriptor is
1209            // non-authoritative (fingerprint = None), and the cache boundary
1210            // discards rather than adopts any cache — we never silently degrade
1211            // to a weaker vocab-only identity.
1212            {
1213                let mut guard = self.kv_compat.lock();
1214                let tokenizer_hash = match std::fs::read(&tokenizer_path) {
1215                    Ok(bytes) => {
1216                        use sha2::{Digest, Sha256};
1217                        let mut hasher = Sha256::new();
1218                        hasher.update(&bytes);
1219                        Some(hex::encode(hasher.finalize()))
1220                    }
1221                    Err(e) => {
1222                        tracing::error!(
1223                            "Could not read {} for KV-compat tokenizer identity ({}): \
1224                             KV-cache reuse disabled (fail-closed)",
1225                            tokenizer_path.display(),
1226                            e
1227                        );
1228                        None
1229                    }
1230                };
1231                if let Some(desc) = guard.as_mut() {
1232                    desc.set_tokenizer(final_vocab_size, tokenizer_hash);
1233                }
1234            }
1235
1236            // Thread safe assignment
1237            let mut tokenizer_guard = self.tokenizer.lock();
1238            *tokenizer_guard = Some(tokenizer);
1239        } else {
1240            return Err(anyhow!(
1241                "Tokenizer not found at {}. A proper tokenizer.json file is required for inference.",
1242                tokenizer_path.display()
1243            ));
1244        }
1245
1246        // Load template configuration if available
1247        if tokenizer_config_path.exists() {
1248            info!(
1249                "Loading tokenizer config: {}",
1250                tokenizer_config_path.display()
1251            );
1252            let config_content = tokio::fs::read_to_string(&tokenizer_config_path).await?;
1253
1254            // Validate before parsing
1255            jtp::from_str(&config_content)
1256                .with_max_depth(10)
1257                .with_max_string_length(50000)
1258                .validate()
1259                .map_err(|e| anyhow!("Invalid tokenizer config: {:?}", e))?;
1260
1261            let config_json: serde_json::Value = serde_json::from_str(&config_content)?;
1262
1263            // Parse template configuration
1264            let template_config = TemplateEngine::from_tokenizer_config(&config_json)?;
1265
1266            // Cache EOS token ID for lock-free access during generation
1267            if let Some(ref eos_str) = template_config.eos_token {
1268                let tokenizer_guard = self.tokenizer.lock();
1269                if let Some(ref tokenizer) = *tokenizer_guard {
1270                    if let Some(eos_id) = tokenizer.token_to_id(eos_str) {
1271                        self.eos_token_id.store(eos_id, Ordering::Relaxed);
1272                        info!("Cached EOS token: '{}' -> ID {}", eos_str, eos_id);
1273                    } else {
1274                        warn!("EOS token '{}' not found in tokenizer vocabulary", eos_str);
1275                    }
1276                }
1277            }
1278
1279            // Create template engine
1280            let template_engine = TemplateEngine::new(template_config)?;
1281
1282            // Store template engine
1283            let mut template_guard = self.template_engine.lock();
1284            *template_guard = Some(template_engine);
1285
1286            info!("✅ Template engine initialized");
1287        } else {
1288            info!("⚠️ No tokenizer_config.json found, using fallback templates");
1289        }
1290
1291        Ok(())
1292    }
1293
1294    /// Tokenize text to input IDs - thread safe
1295    ///
1296    /// Only bounded, content-redacted metadata (byte/char length, token count)
1297    /// is written to the process log. The prompt itself and its token text are
1298    /// tenant-private and never logged (#1253).
1299    fn tokenize(&self, text: &str) -> Result<Vec<i64>> {
1300        let tokenizer_guard = self.tokenizer.lock();
1301        let tokenizer = tokenizer_guard
1302            .as_ref()
1303            .ok_or_else(|| anyhow!("Tokenizer not loaded. Call load_tokenizer() first."))?;
1304
1305        tokenize_redacted(tokenizer, text)
1306    }
1307
1308    /// Format text with dynamic chat template
1309    fn format_chat_message(&self, system: Option<&str>, user: &str) -> String {
1310        // Try to use template engine if available
1311        {
1312            let template_guard = self.template_engine.lock();
1313            if let Some(ref engine) = *template_guard {
1314                let mut messages = Vec::new();
1315
1316                // Add system message if provided
1317                if let Some(system_msg) = system {
1318                    messages.push(ChatMessage {
1319                        role: "system".into(),
1320                        content: Some(system_msg.into()),
1321                        ..Default::default()
1322                    });
1323                }
1324
1325                // Add user message
1326                messages.push(ChatMessage {
1327                    role: "user".into(),
1328                    content: Some(user.into()),
1329                    ..Default::default()
1330                });
1331
1332                // Apply template (no tools in this code path)
1333                if let Ok(formatted) = engine.apply_chat_template(&messages, Some(true), None) {
1334                    return formatted;
1335                }
1336            }
1337        }
1338
1339        // Generic fallback template (model-agnostic)
1340        let mut formatted = String::new();
1341
1342        // Add system message if provided
1343        if let Some(system_msg) = system {
1344            formatted.push_str("System: ");
1345            formatted.push_str(system_msg);
1346            formatted.push_str("\n\n");
1347        }
1348
1349        // Add user message
1350        formatted.push_str("User: ");
1351        formatted.push_str(user);
1352        formatted.push_str("\n\nAssistant: ");
1353
1354        formatted
1355    }
1356
1357    /// Check if a token ID is the EOS token (lock-free using cached ID)
1358    pub fn is_eos_token(&self, token_id: usize) -> bool {
1359        let cached_eos = self.eos_token_id.load(Ordering::Relaxed);
1360        // cached_eos == 0 means not initialized; actual EOS tokens are > 0
1361        cached_eos > 0 && token_id as u32 == cached_eos
1362    }
1363
1364    /// Get the tokenizer for streaming decoding - CoW makes this cheap!
1365    ///
1366    /// Returns a cloned tokenizer (cheap due to copy-on-write)
1367    pub fn get_tokenizer(&self) -> Result<Tokenizer> {
1368        let tokenizer_guard = self.tokenizer.lock();
1369        tokenizer_guard
1370            .as_ref()
1371            .cloned() // Cheap clone due to CoW
1372            .ok_or_else(|| anyhow!("Tokenizer not loaded. Call load_tokenizer() first."))
1373    }
1374
1375    /// Run inference on the model (supports both TorchScript and VarStore models) - thread safe
1376    pub fn forward(&self, input_ids: &[i64]) -> Result<Tensor> {
1377        // Try VarStore-based inference first (preferred for SafeTensors models)
1378        if self.has_varstore() {
1379            return self.forward_varstore(input_ids);
1380        }
1381
1382        Err(anyhow!("No model loaded - call load_model() first"))
1383    }
1384
1385    /// Run inference using VarStore (SafeTensors models) with persistent model - thread safe
1386    fn forward_varstore(&self, input_ids: &[i64]) -> Result<Tensor> {
1387        // Use the persistent model instance - NO recreation!
1388        let model_arc = self
1389            .persistent_model
1390            .as_ref()
1391            .ok_or_else(|| anyhow!("Persistent model not initialized - call load_model() first"))?;
1392
1393        // Verify context state with thread safety
1394        {
1395            let context_guard = self.context_state.lock();
1396            let _context_state = context_guard
1397                .as_ref()
1398                .ok_or_else(|| anyhow!("Context state not initialized"))?;
1399        }
1400
1401        if !self.is_persistent_model_ready() {
1402            return Err(anyhow!("Model not properly initialized"));
1403        }
1404
1405        // Convert input IDs to tensor (keep as int64 for embeddings)
1406        let input_tensor = Tensor::from_slice(input_ids)
1407            .to_kind(tch::Kind::Int64) // Ensure int64 for embedding lookup
1408            .to_device(self.device)
1409            .unsqueeze(0); // Add batch dimension: [1, seq_len]
1410
1411        // Lock the model and run forward pass (efficient!) with poison recovery
1412        let model = model_arc.lock();
1413        // RAII guard: panic-safe — Drop restores GradMode during unwinding
1414        let logits = {
1415            let _no_grad = tch::no_grad_guard();
1416            model.forward(&input_tensor, None)?
1417        };
1418
1419        // Extract logits for the last token
1420        let logits_shape = logits.size();
1421        let seq_len = logits_shape[1];
1422        let _vocab_size = logits_shape[2] as usize;
1423
1424        // Get logits for last token: [batch=1, last_seq_pos, vocab_size]
1425        let last_token_logits = logits.narrow(1, seq_len - 1, 1).squeeze_dim(1); // [1, vocab_size]
1426
1427        Ok(last_token_logits)
1428    }
1429
1430    /// Run optimized inference with KV caching - only process new tokens
1431    pub fn forward_cached(&self, input_ids: &[i64], start_pos: usize) -> Result<Tensor> {
1432        // Use the persistent model instance
1433        let model_arc = self
1434            .persistent_model
1435            .as_ref()
1436            .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1437
1438        if !self.is_persistent_model_ready() {
1439            return Err(anyhow!("Model not properly initialized"));
1440        }
1441
1442        // Process all provided tokens. The caller is responsible for passing
1443        // only the tokens that need processing:
1444        // - Decode mode: caller passes &[last_token] (single token)
1445        // - Partial prefill: caller passes &prompt_tokens[prefix_len..] (new suffix)
1446        // - Full prefill: caller passes &prompt_tokens (all tokens)
1447        let tokens_to_process = input_ids;
1448
1449        // Convert to tensor
1450        let input_tensor = Tensor::from_slice(tokens_to_process)
1451            .to_kind(tch::Kind::Int64)
1452            .to_device(self.device)
1453            .unsqueeze(0); // [1, seq_len]
1454
1455        // Run forward pass with position info for proper KV cache usage
1456        let model = model_arc.lock();
1457
1458        // RAII guard: panic-safe — Drop restores GradMode during unwinding
1459        let logits = {
1460            let _no_grad = tch::no_grad_guard();
1461            model.forward_with_cache(&input_tensor, start_pos)?
1462        };
1463
1464        // Extract logits for the last token
1465        let logits_shape = logits.size();
1466        let seq_len = logits_shape[1];
1467        let _vocab_size = logits_shape[2] as usize;
1468
1469        // Get logits for last token
1470        let last_token_logits = logits.narrow(1, seq_len - 1, 1).squeeze_dim(1);
1471
1472        Ok(last_token_logits)
1473    }
1474
1475    /// Run inference with KV caching and optional delta injection
1476    ///
1477    /// Same as `forward_cached()` but calls `forward_with_cache_and_delta()` on the model,
1478    /// injecting per-tenant LoRA corrections at each attention layer.
1479    pub fn forward_with_delta_cached(
1480        &self,
1481        input_ids: &[i64],
1482        start_pos: usize,
1483        delta: Option<&crate::training::TenantDelta>,
1484    ) -> Result<Tensor> {
1485        let model_arc = self
1486            .persistent_model
1487            .as_ref()
1488            .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1489
1490        if !self.is_persistent_model_ready() {
1491            return Err(anyhow!("Model not properly initialized"));
1492        }
1493
1494        // Process all provided tokens — caller passes the appropriate slice
1495        let input_tensor = Tensor::from_slice(input_ids)
1496            .to_kind(tch::Kind::Int64)
1497            .to_device(self.device)
1498            .unsqueeze(0);
1499
1500        let model = model_arc.lock();
1501
1502        let logits = {
1503            let _no_grad = tch::no_grad_guard();
1504            model.forward_with_cache_and_delta(&input_tensor, start_pos, delta)?
1505        };
1506
1507        let logits_shape = logits.size();
1508        let seq_len = logits_shape[1];
1509
1510        let last_token_logits = logits.narrow(1, seq_len - 1, 1).squeeze_dim(1);
1511
1512        Ok(last_token_logits)
1513    }
1514
1515    /// Batched ragged decode step over several same-delta sequences (#329).
1516    ///
1517    /// Each entry is `(new_token_ids, start_pos, per-sequence KVCacheManager)`.
1518    /// Returns last-token logits per row `[B, vocab]`. Delegates to the model's
1519    /// `forward_batched` (Llama-only; other architectures return an error so the
1520    /// caller falls back to per-stream `forward_with_delta_cached`). Tensors stay
1521    /// on this (bridge) thread; only the Send token/logits metadata crosses.
1522    pub fn forward_batched_step(
1523        &self,
1524        sequences: &mut [(
1525            Vec<i64>,
1526            usize,
1527            std::sync::Arc<parking_lot::Mutex<crate::runtime::kv_cache::KVCacheManager>>,
1528        )],
1529        delta: Option<&crate::training::TenantDelta>,
1530    ) -> Result<Tensor> {
1531        let model_arc = self
1532            .persistent_model
1533            .as_ref()
1534            .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1535        if !self.is_persistent_model_ready() {
1536            return Err(anyhow!("Model not properly initialized"));
1537        }
1538
1539        let model = model_arc.lock();
1540        let logits = {
1541            let _no_grad = tch::no_grad_guard();
1542            model.forward_batched(sequences, delta)?
1543        };
1544
1545        // logits: [B, q, vocab] -> last-token logits [B, vocab].
1546        let seq_len = logits.size()[1];
1547        Ok(logits.narrow(1, seq_len - 1, 1).squeeze_dim(1))
1548    }
1549
1550    /// Run full forward pass with optional delta injection (no KV cache)
1551    pub fn forward_with_delta_full(
1552        &self,
1553        input_ids: &[i64],
1554        delta: Option<&crate::training::TenantDelta>,
1555    ) -> Result<Tensor> {
1556        let model_arc = self
1557            .persistent_model
1558            .as_ref()
1559            .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1560
1561        if !self.is_persistent_model_ready() {
1562            return Err(anyhow!("Model not properly initialized"));
1563        }
1564
1565        let input_tensor = Tensor::from_slice(input_ids)
1566            .to_kind(tch::Kind::Int64)
1567            .to_device(self.device)
1568            .unsqueeze(0);
1569
1570        let model = model_arc.lock();
1571
1572        let logits = {
1573            let _no_grad = tch::no_grad_guard();
1574            model.forward_with_cache_and_delta(&input_tensor, 0, delta)?
1575        };
1576
1577        let logits_shape = logits.size();
1578        let seq_len = logits_shape[1];
1579
1580        let last_token_logits = logits.narrow(1, seq_len - 1, 1).squeeze_dim(1);
1581
1582        Ok(last_token_logits)
1583    }
1584
1585    /// Clear KV cache before new generation to prevent context pollution
1586    pub fn clear_kv_cache(&self) {
1587        if let Some(model_arc) = &self.persistent_model {
1588            let model = model_arc.lock();
1589            let model_any = model.as_any();
1590            if let Some(llama_model) =
1591                model_any.downcast_ref::<crate::runtime::architectures::llama::LlamaModel>()
1592            {
1593                llama_model.clear_kv_cache();
1594                tracing::debug!("Cleared KV cache before generation");
1595            } else if let Some(q35_model) =
1596                model_any.downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
1597            {
1598                q35_model.clear_kv_cache();
1599                tracing::debug!("Cleared Qwen3.5 KV cache + SSM states before generation");
1600            }
1601        }
1602    }
1603
1604    /// Snapshot Qwen3.5 SSM (conv/rec) states for TTT adaptation.
1605    ///
1606    /// Returns `Some((conv, rec))` if the loaded model is Qwen3.5; `None` otherwise.
1607    /// Each `Tensor` is deep-copied via `.copy()` (C4 fix) so the snapshot is
1608    /// independent of subsequent forward-pass mutations during the TTT loop.
1609    pub fn snapshot_ssm_states(&self) -> Option<(Vec<Option<Tensor>>, Vec<Option<Tensor>>)> {
1610        let model_arc = self.persistent_model.as_ref()?;
1611        let model = model_arc.lock();
1612        let q35 = model
1613            .as_any()
1614            .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()?;
1615        Some(q35.snapshot_ssm_states())
1616    }
1617
1618    /// Restore Qwen3.5 SSM states from a snapshot produced by `snapshot_ssm_states`.
1619    ///
1620    /// No-op if the loaded model is not Qwen3.5 or snapshot is `None`.
1621    pub fn restore_ssm_states(&self, snapshot: Option<(Vec<Option<Tensor>>, Vec<Option<Tensor>>)>) {
1622        let Some((conv_snap, rec_snap)) = snapshot else {
1623            return;
1624        };
1625        let Some(model_arc) = &self.persistent_model else {
1626            return;
1627        };
1628        let model = model_arc.lock();
1629        if let Some(q35) = model
1630            .as_any()
1631            .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
1632        {
1633            q35.restore_ssm_states(conv_snap, rec_snap);
1634            tracing::debug!("Restored Qwen3.5 SSM states after TTT adaptation");
1635        }
1636    }
1637
1638    // ========================================================================
1639    // MTP self-speculative decoding (Qwen3.5, v1: dense + greedy + batch=1)
1640    // ========================================================================
1641
1642    /// Whether the loaded model can self-speculate: a Qwen3.5 with a loaded
1643    /// (dense) MTP draft head. Pipeline/multi-device stage loads never own the
1644    /// MTP head, so this is false there too.
1645    pub fn speculative_supported(&self) -> bool {
1646        let Some(model_arc) = &self.persistent_model else {
1647            return false;
1648        };
1649        let model = model_arc.lock();
1650        model
1651            .as_any()
1652            .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
1653            .is_some_and(crate::runtime::architectures::qwen3_5::Qwen3_5Model::has_mtp)
1654    }
1655
1656    /// Speculative prefill/verify forward: returns per-position logits plus the
1657    /// pre-final-norm hidden states the MTP head consumes.
1658    ///
1659    /// Never delta-adapted: speculation is disabled for tenant-delta streams at
1660    /// stream construction, so this always runs the base model.
1661    ///
1662    /// Returns `(logits [1, seq|1, vocab], hidden [1, seq, hidden_size])`.
1663    pub fn forward_speculative(
1664        &self,
1665        input_ids: &[i64],
1666        start_pos: usize,
1667        last_logits_only: bool,
1668    ) -> Result<(Tensor, Tensor)> {
1669        let model_arc = self
1670            .persistent_model
1671            .as_ref()
1672            .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1673        if !self.is_persistent_model_ready() {
1674            return Err(anyhow!("Model not properly initialized"));
1675        }
1676
1677        let input_tensor = Tensor::from_slice(input_ids)
1678            .to_kind(tch::Kind::Int64)
1679            .to_device(self.device)
1680            .unsqueeze(0);
1681
1682        let model = model_arc.lock();
1683        let q35 = model
1684            .as_any()
1685            .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
1686            .ok_or_else(|| anyhow!("Speculative decoding requires a Qwen3.5 model"))?;
1687        let _no_grad = tch::no_grad_guard();
1688        q35.forward_with_cache_hidden(&input_tensor, start_pos, last_logits_only)
1689    }
1690
1691    /// MTP draft forward over `token_ids.len()` consecutive slots. Slot `i`
1692    /// pairs `hiddens[.., i, ..]` with `embed(token_ids[i])` at absolute
1693    /// position `base + i`. Returns the last slot's logits `[1, 1, vocab]`.
1694    pub fn forward_mtp_draft(
1695        &self,
1696        token_ids: &[i64],
1697        hiddens: &Tensor,
1698        base: usize,
1699    ) -> Result<Tensor> {
1700        let model_arc = self
1701            .persistent_model
1702            .as_ref()
1703            .ok_or_else(|| anyhow!("Persistent model not initialized"))?;
1704        if !self.is_persistent_model_ready() {
1705            return Err(anyhow!("Model not properly initialized"));
1706        }
1707
1708        let ids_tensor = Tensor::from_slice(token_ids)
1709            .to_kind(tch::Kind::Int64)
1710            .to_device(self.device)
1711            .unsqueeze(0);
1712
1713        let model = model_arc.lock();
1714        let q35 = model
1715            .as_any()
1716            .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
1717            .ok_or_else(|| anyhow!("Speculative decoding requires a Qwen3.5 model"))?;
1718        let _no_grad = tch::no_grad_guard();
1719        q35.mtp_forward(&ids_tensor, hiddens, base)
1720    }
1721
1722    /// Sample next token using bundled parameters with tiered repeat penalty.
1723    fn sample_token_with_params(
1724        &self,
1725        logits_tensor: &Tensor,
1726        params: &SamplingParams,
1727        previous_tokens: &[i64],
1728        penalty_exempt_tokens: &HashSet<i64>,
1729    ) -> Result<usize> {
1730        self.sampler.sample_token_with_penalty_exemptions(
1731            logits_tensor,
1732            params.temperature,
1733            params.top_p,
1734            params.top_k,
1735            params.repeat_penalty,
1736            previous_tokens,
1737            penalty_exempt_tokens,
1738        )
1739    }
1740}
1741
1742/// Rewind the model's installed session cache to a matched prefix.
1743///
1744/// Pure-attention models: truncating the KV cache to `prefix_len` is exact, so
1745/// any prefix length is reusable (unchanged behavior).
1746///
1747/// Hybrid recurrent models (Qwen3.5): the GDN conv/rec state is Markovian and
1748/// cannot be truncated to an arbitrary position — it can only be restored to
1749/// the end-of-prefill snapshot stored with the cached tokens (see
1750/// [`KVCacheManager::set_cached_tokens_with_ssm`]). A prefix hit is therefore
1751/// reusable only when the FULL cached sequence matched
1752/// (`prefix_len == cached_token_count()`) and a snapshot is present; anything
1753/// else discards KV + SSM state and forces a full recompute, because decoding
1754/// from a truncated KV with stale recurrent state silently produces tokens
1755/// conditioned on the wrong context.
1756///
1757/// Returns the number of reusable prefix tokens (0 = cache cleared, full
1758/// prefill required). The caller must guarantee a strict partial hit
1759/// (`prefix_len < prompt_len`); the exact-prompt-repeat decision lives in
1760/// [`resolve_session_prefill_start`].
1761pub(crate) fn rewind_session_state(model: &dyn ModelOperations, prefix_len: usize) -> usize {
1762    let Some(cache) = model.get_kv_cache() else {
1763        return 0;
1764    };
1765    let q35 = model
1766        .as_any()
1767        .downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>();
1768    let Some(q35) = q35 else {
1769        cache.lock().truncate_to(prefix_len);
1770        return prefix_len;
1771    };
1772
1773    let snapshot = {
1774        let cache_guard = cache.lock();
1775        if prefix_len == cache_guard.cached_token_count() {
1776            cache_guard.ssm_snapshot()
1777        } else {
1778            None
1779        }
1780    };
1781    match snapshot {
1782        Some((conv_snap, rec_snap)) => {
1783            cache.lock().truncate_to(prefix_len);
1784            q35.restore_ssm_states(conv_snap, rec_snap);
1785            prefix_len
1786        }
1787        None => {
1788            // Partial prefix match (or no snapshot): recurrent state cannot be
1789            // rewound to an intermediate position — recompute from scratch.
1790            model.clear_kv_cache();
1791            0
1792        }
1793    }
1794}
1795
1796/// Decide the prefill start position for a session-cache prefix match.
1797///
1798/// A strict partial hit (`0 < prefix_len < prompt_len`) is reusable: there is
1799/// a nonempty suffix left to prefill, and [`rewind_session_state`] puts the
1800/// cache and model into the exact end-of-prefix state.
1801///
1802/// An exact-prompt repeat (`prefix_len == prompt_len` — the session resending
1803/// its previous prompt verbatim, `prefix_len == cached_token_count()`) is NOT
1804/// reusable even though the full cached sequence matched: no tokens remain to
1805/// prefill, and sampling the first new token needs logits for the last prompt
1806/// position, which the cache does not retain. The engine must re-run the whole
1807/// prompt, and that re-run must start from CLEARED state — prefilling onto the
1808/// restored KV plus end-of-prefill GDN snapshot would advance the recurrent
1809/// state over the prompt a second time, producing wrong logits (and a wrong
1810/// end-of-prefill snapshot for the next turn). A miss (`prefix_len == 0`)
1811/// clears and starts fresh as before.
1812///
1813/// Returns the prefill start position (0 = cache cleared, full prefill).
1814pub(crate) fn resolve_session_prefill_start(
1815    model: &dyn ModelOperations,
1816    prefix_len: usize,
1817    prompt_len: usize,
1818) -> usize {
1819    if prefix_len > 0 && prefix_len < prompt_len {
1820        rewind_session_state(model, prefix_len)
1821    } else {
1822        model.clear_kv_cache();
1823        0
1824    }
1825}
1826
1827impl TorchEngine {
1828    /// Stable, content-free label identifying the loaded model for OTel metric
1829    /// attributes (#1261). Prefers the model name (set on load); falls back to
1830    /// architecture, then "unknown". Never the prompt or generated text.
1831    fn model_label(&self) -> String {
1832        let info = self.model_info.lock().clone();
1833        if !info.name.is_empty() && info.name != "unloaded" {
1834            info.name
1835        } else if !info.architecture.is_empty() && info.architecture != "unknown" {
1836            info.architecture
1837        } else {
1838            "unknown".to_owned()
1839        }
1840    }
1841}
1842
1843#[async_trait]
1844impl RuntimeEngine for TorchEngine {
1845    #[instrument(name = "torch_engine.load_model", skip(self), fields(path = %path.display()))]
1846    async fn load_model(&mut self, path: &Path) -> Result<()> {
1847        // Store the original path for model naming
1848        let original_path = path.to_path_buf();
1849
1850        // If path is a directory, find the model file inside it
1851        let model_file_path = if path.is_dir() {
1852            // First check for single file patterns
1853            let single_files = [
1854                "model.safetensors",
1855                "pytorch_model.bin",
1856                "model.bin",
1857                "model.pt",
1858                "model.pth",
1859            ];
1860
1861            let mut found_file = None;
1862            for filename in &single_files {
1863                let candidate = path.join(filename);
1864                if candidate.exists() {
1865                    found_file = Some(candidate);
1866                    break;
1867                }
1868            }
1869
1870            // If no single file found, check for sharded SafeTensors
1871            if found_file.is_none() {
1872                if let Ok(entries) = std::fs::read_dir(path) {
1873                    for entry in entries.flatten() {
1874                        let filename = entry.file_name();
1875                        if let Some(name) = filename.to_str() {
1876                            // Match standard shard pattern: model-00001-of-NNNNN.safetensors
1877                            // Also match Qwen3.5 pattern: model.safetensors-00001-of-NNNNN.safetensors
1878                            let is_first_shard = (name.starts_with("model-00001-of-")
1879                                || name.starts_with("model.safetensors-00001-of-"))
1880                                && name.ends_with(".safetensors");
1881                            if is_first_shard {
1882                                info!(
1883                                    "🔍 Detected sharded SafeTensors model starting with: {}",
1884                                    name
1885                                );
1886                                found_file = Some(entry.path());
1887                                break;
1888                            }
1889                        }
1890                    }
1891                }
1892            }
1893
1894            found_file.ok_or_else(|| {
1895                anyhow!(
1896                    "No supported model file found in directory: {}",
1897                    path.display()
1898                )
1899            })?
1900        } else {
1901            path.to_path_buf()
1902        };
1903
1904        info!("Loading model: {}", model_file_path.display());
1905        self.load_model_file(&model_file_path).await?;
1906
1907        // Set the model name based on the canonical path
1908        // Use directory name if loading from directory, otherwise use filename
1909        let model_name = if original_path.is_dir() {
1910            // Get the last component of the directory path as the model name
1911            original_path
1912                .file_name()
1913                .and_then(|s| s.to_str())
1914                .unwrap_or("unknown")
1915                .to_owned()
1916        } else {
1917            // Use the file stem for single files
1918            original_path
1919                .file_stem()
1920                .and_then(|s| s.to_str())
1921                .unwrap_or("unknown")
1922                .to_owned()
1923        };
1924
1925        // Update model info with the correct name
1926        {
1927            let mut model_info_guard = self.model_info.lock();
1928            model_info_guard.name = model_name;
1929        }
1930
1931        self.load_tokenizer(path).await?;
1932        Ok(())
1933    }
1934
1935    async fn generate(&self, prompt: &str, max_tokens: usize) -> Result<String> {
1936        // Format prompt with model-agnostic chat template
1937        let formatted_prompt =
1938            self.format_chat_message(Some("You are a helpful assistant."), prompt);
1939
1940        let request = GenerationRequest {
1941            prompt: formatted_prompt,
1942            max_tokens: Some(max_tokens as u32),
1943            temperature: Some(self.generation_config.temperature),
1944            top_p: Some(self.generation_config.top_p),
1945            top_k: self.generation_config.top_k.map(|v| v as u32),
1946            repeat_penalty: Some(self.generation_config.repeat_penalty),
1947            stop_tokens: Some(self.generation_config.stop_tokens.clone()),
1948            ..Default::default()
1949        };
1950
1951        let result = self.generate_with_params(request).await?;
1952        Ok(result.text)
1953    }
1954
1955    async fn generate_with_params(&self, request: GenerationRequest) -> Result<GenerationResult> {
1956        use futures::StreamExt;
1957
1958        if !self.is_persistent_model_ready() {
1959            return Err(anyhow!(
1960                "Model not properly initialized - persistent model not ready"
1961            ));
1962        }
1963
1964        let mut stream = self.generate(request)?;
1965        let mut accumulated_text = String::new();
1966
1967        while let Some(text_chunk) = stream.next().await {
1968            accumulated_text.push_str(&text_chunk?);
1969        }
1970
1971        let stats = stream.stats();
1972        Ok(GenerationResult {
1973            text: accumulated_text,
1974            tokens_generated: stats.tokens_generated,
1975            finish_reason: stats.finish_reason.unwrap_or(FinishReason::Stop),
1976            generation_time_ms: stats.generation_time_ms,
1977            tokens_per_second: stats.tokens_per_second,
1978            quality_metrics: stats.quality_metrics,
1979            prefill_tokens: stats.prefill_tokens,
1980            prefill_time_ms: stats.prefill_time_ms,
1981            prefill_tokens_per_sec: stats.prefill_tokens_per_sec,
1982            inference_tokens: stats.inference_tokens,
1983            inference_time_ms: stats.inference_time_ms,
1984            inference_tokens_per_sec: stats.inference_tokens_per_sec,
1985            ttt_metrics: None, // TTT metrics attached by InferenceService
1986        })
1987    }
1988
1989    fn model_info(&self) -> ModelInfo {
1990        self.model_info.lock().clone()
1991    }
1992
1993    fn is_loaded(&self) -> bool {
1994        let varstore_loaded = self.has_varstore();
1995        let persistent_loaded = self.persistent_model.is_some();
1996
1997        varstore_loaded || persistent_loaded
1998    }
1999
2000    fn apply_chat_template(
2001        &self,
2002        messages: &[ChatMessage],
2003        add_generation_prompt: bool,
2004        tools: Option<&serde_json::Value>,
2005    ) -> Result<String> {
2006        self.apply_chat_template_with_vars(messages, add_generation_prompt, tools, None, None)
2007    }
2008
2009    fn apply_chat_template_with_vars(
2010        &self,
2011        messages: &[ChatMessage],
2012        add_generation_prompt: bool,
2013        tools: Option<&serde_json::Value>,
2014        enable_thinking: Option<bool>,
2015        template_vars_json: Option<&str>,
2016    ) -> Result<String> {
2017        // Use our template engine if available
2018        let template_guard = self.template_engine.lock();
2019
2020        if let Some(ref engine) = *template_guard {
2021            engine.apply_chat_template_with_vars(
2022                messages,
2023                Some(add_generation_prompt),
2024                tools,
2025                enable_thinking,
2026                template_vars_json,
2027            )
2028        } else {
2029            // Fallback to simple formatting
2030            let mut formatted = String::new();
2031            for msg in messages {
2032                formatted.push_str(&format!(
2033                    "{}: {}\n",
2034                    msg.role,
2035                    msg.content.as_deref().unwrap_or("")
2036                ));
2037            }
2038            if add_generation_prompt {
2039                formatted.push_str("assistant: ");
2040            }
2041            Ok(formatted)
2042        }
2043    }
2044}
2045
2046// Training-specific methods
2047impl TorchEngine {
2048    /// Enable training mode for pre-training
2049    pub fn enable_training(&mut self, _learning_rate: f64) -> Result<()> {
2050        // Ensure model is loaded
2051        if !self.is_persistent_model_ready() {
2052            return Err(anyhow!(
2053                "Model not loaded - persistent model not initialized"
2054            ));
2055        }
2056
2057        // For pre-training without VarStore:
2058        // 1. Set requires_grad on model tensors
2059        // 2. Use manual SGD or implement custom optimizer
2060        //
2061        // The challenge: tch-rs optimizers require VarStore
2062        // Solutions:
2063        // - Create a temporary VarStore and register model weights
2064        // - Implement manual gradient descent
2065        // - Use LoRA for all training (recommended)
2066
2067        tracing::warn!("Pre-training without VarStore requires manual optimizer implementation");
2068        tracing::info!("For now, use LoRA training which has proper VarStore support");
2069
2070        Ok(())
2071    }
2072
2073    /// Perform a manual SGD step without optimizer
2074    pub fn manual_sgd_step(&mut self, _learning_rate: f32) -> Result<()> {
2075        // This would manually update weights:
2076        // weight = weight - learning_rate * weight.grad()
2077        // But we don't have direct access to the model's tensors
2078
2079        tracing::warn!("Manual SGD not implemented - model tensors not accessible");
2080        Ok(())
2081    }
2082
2083    /// Forward pass for training (with gradient tracking)
2084    pub fn forward_training(&self, input_ids: &Tensor, track_gradients: bool) -> Result<Tensor> {
2085        if !self.is_persistent_model_ready() {
2086            return Err(anyhow!("Model not initialized"));
2087        }
2088
2089        let model = self
2090            .persistent_model
2091            .as_ref()
2092            .ok_or_else(|| anyhow!("Persistent model not available"))?;
2093        let model_guard = model.lock();
2094
2095        // Forward pass - gradients tracked if tensors have requires_grad
2096        if track_gradients {
2097            // Gradients are tracked by default when tensors have requires_grad
2098            model_guard.forward(input_ids, None)
2099        } else {
2100            {
2101                let _no_grad = tch::no_grad_guard();
2102                model_guard.forward(input_ids, None)
2103            }
2104        }
2105    }
2106
2107    /// Compute loss and backward (without optimizer step)
2108    pub fn compute_loss_and_backward(&self, input_ids: &Tensor, labels: &Tensor) -> Result<f64> {
2109        // Forward pass with gradients
2110        let logits = self.forward_training(input_ids, true)?;
2111
2112        // Compute cross-entropy loss
2113        let batch_size = logits.size()[0];
2114        let seq_len = logits.size()[1];
2115        let vocab_size = logits.size()[2];
2116
2117        let logits_flat = logits.view([batch_size * seq_len, vocab_size]);
2118        let labels_flat = labels.view([batch_size * seq_len]);
2119
2120        let loss = logits_flat.cross_entropy_loss::<Tensor>(
2121            &labels_flat,
2122            None,
2123            tch::Reduction::Mean,
2124            -100, // ignore_index for padding
2125            0.0,  // label_smoothing
2126        );
2127
2128        let loss_value = loss.double_value(&[]);
2129
2130        // Backward pass computes gradients
2131        loss.backward();
2132
2133        Ok(loss_value)
2134    }
2135
2136    /// Disable training mode
2137    pub fn disable_training(&mut self) -> Result<()> {
2138        // Gradient tracking is controlled per tensor, not globally
2139        tracing::info!("Training mode disabled");
2140        Ok(())
2141    }
2142
2143    /// Check if training is enabled
2144    pub fn is_training_enabled(&self) -> bool {
2145        false // Pre-training not fully supported without VarStore
2146    }
2147}
2148
2149// Additional methods needed by inference layer
2150impl TorchEngine {
2151    // Old callback-based streaming APIs removed - use generate() for all streaming use cases
2152
2153    /// Check if persistent model is initialized - thread safe
2154    pub fn is_persistent_model_ready(&self) -> bool {
2155        let persistent_ready = self.persistent_model.is_some();
2156        let context_ready = self
2157            .context_state
2158            .lock()
2159            .as_ref()
2160            .is_some_and(|c| c.initialized);
2161
2162        persistent_ready && context_ready
2163    }
2164
2165    /// Generate text as a stream of decoded UTF-8 text chunks
2166    ///
2167    /// Returns a Stream that yields `Result<String>` with properly decoded text.
2168    /// The stream automatically handles:
2169    /// - Multi-byte UTF-8 sequences (emojis, CJK characters, etc.)
2170    /// - EOS and stop token detection
2171    /// - Max tokens limit
2172    /// - Special token filtering
2173    ///
2174    /// Dropping the stream stops generation automatically (no manual cancellation needed).
2175    ///
2176    /// # Example
2177    /// ```no_run
2178    /// use futures::StreamExt;
2179    /// use hyprstream_rpc_std::inference_client::GenerationRequest;
2180    ///
2181    /// # async fn example(engine: &hyprstream_inference::runtime::torch_engine::TorchEngine) -> anyhow::Result<()> {
2182    /// let request = GenerationRequest::default();
2183    /// let mut stream = engine.generate(request)?;
2184    ///
2185    /// while let Some(text_chunk) = stream.next().await {
2186    ///     print!("{}", text_chunk?);
2187    /// }
2188    ///
2189    /// let stats = stream.stats();
2190    /// println!("Generated {} tokens", stats.tokens_generated);
2191    /// # Ok(())
2192    /// # }
2193    /// ```
2194    pub fn generate(&self, mut request: GenerationRequest) -> Result<TextStream<'_>> {
2195        // Set random seed if provided for deterministic generation
2196        if let Some(seed) = request.seed {
2197            self.set_seed(seed as u64);
2198        }
2199
2200        // Apply server defaults if not specified in request
2201        if request.timeout_ms.is_none() {
2202            request.timeout_ms = Some(self.config.default_generation_timeout_ms);
2203        }
2204
2205        TextStream::new(self, request)
2206    }
2207
2208    /// Generate with optional per-tenant delta for delta-aware inference
2209    ///
2210    /// When `delta` is Some, LoRA corrections are injected at each attention layer
2211    /// during generation. The delta is locked per-token (not held across the entire
2212    /// generation) to allow concurrent training updates.
2213    ///
2214    /// `adapter_generation` is the live LoRA/delta generation actually applied to
2215    /// this request; it is folded into the KV-compat expected identity so a
2216    /// re-adapted model is detected as incompatible and its stale KV rejected.
2217    ///
2218    /// `tenant_id` is the effective tenant whose delta contributed to the weights
2219    /// (`Some` only when a per-tenant delta is in play, `None` otherwise), and
2220    /// `weight_epoch` is that delta's current weight epoch. The pair is threaded
2221    /// to the session-cache swap so the resulting KV is registered for
2222    /// tenant-delta invalidation and cleared if the weights changed under a
2223    /// different tenant or an older weight epoch (same-tenant in-place
2224    /// re-adaptation). The caller must derive the tenant id from the verified
2225    /// request subject, not invent a new identity source.
2226    pub fn generate_with_delta(
2227        &self,
2228        mut request: GenerationRequest,
2229        delta: Option<std::sync::Arc<parking_lot::Mutex<crate::training::TenantDelta>>>,
2230        tenant_id: Option<String>,
2231        weight_epoch: u64,
2232        adapter_generation: u64,
2233    ) -> Result<TextStream<'_>> {
2234        if let Some(seed) = request.seed {
2235            self.set_seed(seed as u64);
2236        }
2237
2238        if request.timeout_ms.is_none() {
2239            request.timeout_ms = Some(self.config.default_generation_timeout_ms);
2240        }
2241
2242        TextStream::new_with_delta(
2243            self,
2244            request,
2245            delta,
2246            tenant_id,
2247            weight_epoch,
2248            adapter_generation,
2249        )
2250    }
2251
2252    /// Non-streaming generation with optional delta (convenience wrapper)
2253    pub async fn generate_with_delta_params(
2254        &self,
2255        request: GenerationRequest,
2256        delta: Option<std::sync::Arc<parking_lot::Mutex<crate::training::TenantDelta>>>,
2257        tenant_id: Option<String>,
2258        weight_epoch: u64,
2259        adapter_generation: u64,
2260    ) -> Result<crate::config::GenerationResult> {
2261        use futures::StreamExt;
2262
2263        if !self.is_persistent_model_ready() {
2264            return Err(anyhow!(
2265                "Model not properly initialized - persistent model not ready"
2266            ));
2267        }
2268
2269        let mut stream =
2270            self.generate_with_delta(request, delta, tenant_id, weight_epoch, adapter_generation)?;
2271        let mut accumulated_text = String::new();
2272
2273        while let Some(text_chunk) = stream.next().await {
2274            accumulated_text.push_str(&text_chunk?);
2275        }
2276
2277        let stats = stream.stats();
2278        Ok(crate::config::GenerationResult {
2279            text: accumulated_text,
2280            tokens_generated: stats.tokens_generated,
2281            finish_reason: stats
2282                .finish_reason
2283                .unwrap_or(crate::config::FinishReason::Stop),
2284            generation_time_ms: stats.generation_time_ms,
2285            tokens_per_second: stats.tokens_per_second,
2286            quality_metrics: stats.quality_metrics,
2287            prefill_tokens: stats.prefill_tokens,
2288            prefill_time_ms: stats.prefill_time_ms,
2289            prefill_tokens_per_sec: stats.prefill_tokens_per_sec,
2290            inference_tokens: stats.inference_tokens,
2291            inference_time_ms: stats.inference_time_ms,
2292            inference_tokens_per_sec: stats.inference_tokens_per_sec,
2293            ttt_metrics: None,
2294        })
2295    }
2296
2297    /// Forward pass with per-layer delta injection for TTT training.
2298    ///
2299    /// Uses `decode_layer_with_delta()` to inject the tenant delta's A/B matrices
2300    /// after q_proj/v_proj projections inside each attention layer. This creates a
2301    /// differentiable path from the loss back to the delta parameters, enabling
2302    /// gradient-based training via `loss.backward()`.
2303    ///
2304    /// # Arguments
2305    /// * `input_ids` - Token IDs [1, seq_len]
2306    /// * `delta` - Tenant's LoRA delta with trainable A/B matrices
2307    ///
2308    /// # Returns
2309    /// Logits tensor [1, seq_len, vocab_size] with gradient path to delta parameters
2310    pub fn forward_with_delta(
2311        &self,
2312        input_ids: &Tensor,
2313        delta: &crate::training::TenantDelta,
2314    ) -> Result<Tensor> {
2315        let model = self
2316            .persistent_model
2317            .as_ref()
2318            .ok_or_else(|| anyhow!("Model not loaded"))?;
2319        let model_guard = model.lock();
2320
2321        // Get embeddings from the base model
2322        let input = input_ids.to(self.device);
2323        let mut hidden_states = model_guard.embed_tokens(&input)?;
2324
2325        // NOTE: f32 upcast is NOT done here because model weights are bf16 and
2326        // ROCm/HIP matmul requires matching dtypes. Gradient precision is handled
2327        // by the delta injection path (LoRA A/B matrices) operating in the model's
2328        // native dtype with the with_grad() wrapper ensuring autograd is active.
2329
2330        // Generate position IDs
2331        let seq_len = hidden_states.size()[1];
2332        let position_ids = Tensor::arange(seq_len, (tch::Kind::Int64, hidden_states.device()));
2333
2334        let num_layers = model_guard.num_layers();
2335
2336        // Process each layer with delta injection
2337        for layer_idx in 0..num_layers {
2338            let (new_hidden, _kv) = model_guard.decode_layer_with_delta(
2339                layer_idx,
2340                &hidden_states,
2341                None,                // attention_mask
2342                Some(&position_ids), // position_ids
2343                None,                // past_kv (no cache during training)
2344                delta,
2345            )?;
2346            hidden_states = new_hidden;
2347        }
2348
2349        // Apply final norm + LM head
2350        hidden_states = model_guard.apply_final_norm(&hidden_states)?;
2351        let logits = model_guard.lm_head(&hidden_states)?;
2352
2353        Ok(logits)
2354    }
2355
2356    /// Authoritative context capacity of the loaded model. ModelInfo is populated
2357    /// from the checkpoint configuration and optional max_context override during
2358    /// loading; RuntimeConfig.context_length is only a constructor default.
2359    /// Serving encoders and native decision admission must use this same limit.
2360    pub fn effective_context_limit(&self) -> Result<usize> {
2361        anyhow::ensure!(self.persistent_model.is_some(), "Model not loaded");
2362        let limit = self.model_info.lock().context_length as usize;
2363        anyhow::ensure!(limit > 0, "loaded model has an invalid context limit");
2364        Ok(limit)
2365    }
2366
2367    /// Execute a decision head against fresh full-sequence hidden states while
2368    /// holding the backbone lock. The supplied gather reads original LM-head
2369    /// rows without allocating token-by-vocabulary logits. No cache is reused.
2370    pub fn with_decision_backbone<T>(
2371        &self,
2372        token_ids: &[i64],
2373        head: impl FnOnce(&Tensor, &dyn Fn(&[i64]) -> Result<Tensor>) -> Result<T>,
2374    ) -> Result<T> {
2375        self.with_decision_backbone_checked(token_ids, &|| Ok(()), head)
2376    }
2377
2378    /// Cooperative cancellation/deadline variant. The checker runs before
2379    /// admission, between decoder layers, and around head evaluation. An active
2380    /// libtorch kernel completes before the next check; dropping a future alone
2381    /// is not a compute-cancellation mechanism.
2382    pub fn with_decision_backbone_checked<T>(
2383        &self,
2384        token_ids: &[i64],
2385        check: &dyn Fn() -> Result<()>,
2386        head: impl FnOnce(&Tensor, &dyn Fn(&[i64]) -> Result<Tensor>) -> Result<T>,
2387    ) -> Result<T> {
2388        check()?;
2389        anyhow::ensure!(!token_ids.is_empty(), "decision token sequence is empty");
2390        let limit = self.effective_context_limit()?;
2391        anyhow::ensure!(
2392            token_ids.len() <= limit,
2393            "decision input exceeds context limit"
2394        );
2395        anyhow::ensure!(
2396            self.device == tch::Device::Cpu,
2397            "decision backbone is CPU-only pending CPU acceptance"
2398        );
2399        let model = self
2400            .persistent_model
2401            .as_ref()
2402            .ok_or_else(|| anyhow!("Model not loaded"))?;
2403        let model = model.lock();
2404        let _guard = tch::no_grad_guard();
2405        let input = Tensor::from_slice(token_ids)
2406            .reshape([1, token_ids.len() as i64])
2407            .to_device(self.device);
2408        let hidden = model.decision_hidden_states_checked(&input, check)?;
2409        check()?;
2410        let gather = |ids: &[i64]| model.decision_output_embedding_rows(ids);
2411        let output = head(&hidden, &gather)?;
2412        check()?;
2413        Ok(output)
2414    }
2415
2416    /// Get the number of layers in the loaded model.
2417    pub fn get_num_layers(&self) -> Result<usize> {
2418        let model = self
2419            .persistent_model
2420            .as_ref()
2421            .ok_or_else(|| anyhow!("Model not loaded"))?;
2422        Ok(model.lock().num_layers())
2423    }
2424
2425    /// Get module dimensions for LoRA target modules from the loaded model.
2426    ///
2427    /// Returns a map of module_name -> (in_features, out_features) for all
2428    /// supported LoRA target modules. Used by DeltaPool to initialize
2429    /// per-tenant LoRA deltas with correct dimensions.
2430    ///
2431    /// Supported modules: q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj
2432    pub fn get_lora_module_dims(
2433        &self,
2434    ) -> Result<std::collections::HashMap<String, (usize, usize)>> {
2435        let model_info = self.model_info.lock();
2436        let hidden_size = model_info.hidden_size as usize;
2437
2438        let num_heads = model_info
2439            .num_attention_heads
2440            .ok_or_else(|| anyhow!("num_attention_heads not set in ModelInfo"))?
2441            as usize;
2442        let num_kv_heads = model_info.num_key_value_heads.unwrap_or(num_heads as u32) as usize;
2443        let head_dim = model_info
2444            .head_dim
2445            .map(|d| d as usize)
2446            .unwrap_or(hidden_size / num_heads);
2447
2448        let q_out = num_heads * head_dim;
2449        let kv_out = num_kv_heads * head_dim;
2450
2451        let intermediate_size = if let Some(intermediate) = model_info.intermediate_size {
2452            intermediate as usize
2453        } else {
2454            match model_info.architecture.as_str() {
2455                "Qwen2ForCausalLM" => (hidden_size as f32 * 2.6667) as usize,
2456                "LlamaForCausalLM" => (hidden_size as f32 * 2.75) as usize,
2457                _ => hidden_size * 4,
2458            }
2459        };
2460
2461        let mut dims = std::collections::HashMap::new();
2462        // Self-attention projections with correct dimensions
2463        // q_proj: in=hidden_size, out=num_heads * head_dim
2464        dims.insert("q_proj".to_owned(), (hidden_size, q_out));
2465        // k_proj/v_proj: in=hidden_size, out=num_kv_heads * head_dim (GQA support)
2466        dims.insert("k_proj".to_owned(), (hidden_size, kv_out));
2467        dims.insert("v_proj".to_owned(), (hidden_size, kv_out));
2468        // o_proj: in=num_heads * head_dim, out=hidden_size
2469        dims.insert("o_proj".to_owned(), (q_out, hidden_size));
2470        // MLP projections (expand)
2471        for name in &["gate_proj", "up_proj"] {
2472            dims.insert((*name).to_owned(), (hidden_size, intermediate_size));
2473        }
2474        // MLP projection (contract)
2475        dims.insert("down_proj".to_owned(), (intermediate_size, hidden_size));
2476
2477        Ok(dims)
2478    }
2479
2480    /// Get per-layer LoRA module dimensions for hybrid architectures (e.g., Qwen3.5).
2481    ///
2482    /// Returns `None` for uniform architectures (Llama, Gemma, etc.) where all layers
2483    /// have identical module dims. Returns per-layer overrides for Qwen3.5 where GDN
2484    /// layers have different `o_proj` input dims than full-attention layers.
2485    pub fn get_per_layer_lora_dims(
2486        &self,
2487    ) -> Option<std::collections::HashMap<usize, std::collections::HashMap<String, (usize, usize)>>>
2488    {
2489        let model_arc = self.persistent_model.as_ref()?;
2490        let model = model_arc.lock();
2491        let model_any = model.as_any();
2492
2493        if let Some(q35) =
2494            model_any.downcast_ref::<crate::runtime::architectures::qwen3_5::Qwen3_5Model>()
2495        {
2496            let qcfg = q35.text_config();
2497            let hidden = qcfg.hidden_size as usize;
2498            let nh = qcfg.num_attention_heads as usize;
2499            let hd = qcfg.head_dim as usize;
2500            // q_proj outputs num_heads * head_dim * 2 (first half = Q, second half = gate).
2501            // The LoRA delta targets only the Q half, so lora_b out_features = nh*hd (the half-dim).
2502            // o_proj in_features is also nh*hd (attention output after gate application).
2503            let q_half_dim = nh * hd;
2504
2505            let gdn_out_proj_in = qcfg.linear_num_value_heads * qcfg.linear_value_head_dim;
2506            let num_layers = qcfg.num_hidden_layers as usize;
2507
2508            let mut per_layer: std::collections::HashMap<
2509                usize,
2510                std::collections::HashMap<String, (usize, usize)>,
2511            > = std::collections::HashMap::new();
2512
2513            let nkv_out = (qcfg.num_key_value_heads as usize) * hd;
2514            for (layer_idx, lt) in qcfg.layer_types.iter().enumerate() {
2515                let mut layer_dims: std::collections::HashMap<String, (usize, usize)> =
2516                    std::collections::HashMap::new();
2517                if lt == "full_attention" {
2518                    // Full-attention: q correction targets the Q half (not the doubled projection)
2519                    layer_dims.insert("q_proj".to_owned(), (hidden, q_half_dim));
2520                    layer_dims.insert("v_proj".to_owned(), (hidden, nkv_out));
2521                    layer_dims.insert("o_proj".to_owned(), (q_half_dim, hidden));
2522                } else {
2523                    // GDN (linear_attention): only o_proj; maps to struct field out_proj.
2524                    // in_features = val_dim (input to out_proj), out_features = hidden_size.
2525                    layer_dims.insert("o_proj".to_owned(), (gdn_out_proj_in, hidden));
2526                }
2527                per_layer.insert(layer_idx, layer_dims);
2528            }
2529
2530            // Fill any layers not covered by layer_types (every 4th is full_attn)
2531            for layer_idx in 0..num_layers {
2532                per_layer.entry(layer_idx).or_insert_with(|| {
2533                    let is_full_attn = (layer_idx + 1) % 4 == 0;
2534                    let mut layer_dims: std::collections::HashMap<String, (usize, usize)> =
2535                        std::collections::HashMap::new();
2536                    if is_full_attn {
2537                        layer_dims.insert("q_proj".to_owned(), (hidden, q_half_dim));
2538                        layer_dims.insert("v_proj".to_owned(), (hidden, nkv_out));
2539                        layer_dims.insert("o_proj".to_owned(), (q_half_dim, hidden));
2540                    } else {
2541                        layer_dims.insert("o_proj".to_owned(), (gdn_out_proj_in, hidden));
2542                    }
2543                    layer_dims
2544                });
2545            }
2546
2547            Some(per_layer)
2548        } else {
2549            None
2550        }
2551    }
2552
2553    /// Validate LoRA configuration against model dimensions.
2554    ///
2555    /// Checks that the requested target modules are valid for this model.
2556    /// The actual LoRA delta creation is handled by DeltaPool/TenantDelta.
2557    pub fn create_lora(&mut self, config: crate::training::TenantDeltaConfig) -> Result<()> {
2558        let all_dims = self.get_lora_module_dims()?;
2559
2560        for module_name in &config.target_modules {
2561            if !all_dims.contains_key(module_name.as_str()) {
2562                return Err(anyhow!(
2563                    "Unknown module '{}'. Supported modules: q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj",
2564                    module_name
2565                ));
2566            }
2567        }
2568
2569        tracing::info!(
2570            rank = config.rank,
2571            alpha = config.alpha,
2572            target_modules = ?config.target_modules,
2573            "LoRA configuration validated - delta creation handled by DeltaPool"
2574        );
2575        Ok(())
2576    }
2577
2578    // ============================================================================
2579    // Embedding Extraction for RAG/CAG
2580    // ============================================================================
2581
2582    /// Extract embedding from text using the model's hidden states.
2583    ///
2584    /// This method tokenizes the input text, runs a forward pass through the model,
2585    /// and extracts the final hidden states. The embedding is computed by mean-pooling
2586    /// the hidden states over the sequence dimension.
2587    ///
2588    /// # Arguments
2589    /// - `text` - Input text to embed
2590    ///
2591    /// # Returns
2592    /// - Vec<f32> - The embedding vector (dimension = model's hidden_size)
2593    ///
2594    /// # Example
2595    /// ```ignore
2596    /// let embedding = engine.extract_embedding("What is machine learning?")?;
2597    /// println!("Embedding dimension: {}", embedding.len());
2598    /// ```
2599    pub fn extract_embedding(&self, text: &str) -> Result<Vec<f32>> {
2600        // Tokenize input
2601        let tokenizer_guard = self.tokenizer.lock();
2602        let tokenizer = tokenizer_guard
2603            .as_ref()
2604            .ok_or_else(|| anyhow!("No tokenizer loaded"))?;
2605
2606        let encoding = tokenizer
2607            .encode(text, false)
2608            .map_err(|e| anyhow!("Tokenization failed: {}", e))?;
2609
2610        let token_ids: Vec<i64> = encoding.get_ids().iter().map(|&id| id as i64).collect();
2611        drop(tokenizer_guard);
2612
2613        self.extract_embedding_from_tokens(&token_ids)
2614    }
2615
2616    /// Extract embedding from pre-tokenized input.
2617    ///
2618    /// This is a lower-level method for when tokens are already available
2619    /// (e.g., from a previous generation). Uses mean pooling over the hidden states.
2620    ///
2621    /// # Arguments
2622    /// - `token_ids` - Pre-tokenized input IDs
2623    ///
2624    /// # Returns
2625    /// - Vec<f32> - The embedding vector (dimension = model's hidden_size)
2626    pub fn extract_embedding_from_tokens(&self, token_ids: &[i64]) -> Result<Vec<f32>> {
2627        if token_ids.is_empty() {
2628            return Err(anyhow!("Empty token list"));
2629        }
2630
2631        let input_tensor = Tensor::from_slice(token_ids)
2632            .to_device(self.device)
2633            .unsqueeze(0); // [1, seq_len]
2634
2635        let model_guard = self
2636            .persistent_model
2637            .as_ref()
2638            .ok_or_else(|| anyhow!("No model loaded"))?
2639            .lock();
2640
2641        {
2642            let _no_grad = tch::no_grad_guard();
2643
2644            // Get token embeddings
2645            let embeddings = model_guard.embed_tokens(&input_tensor)?;
2646
2647            // Process through layers to get hidden states
2648            let hidden_states = self
2649                .process_through_layers_for_embedding(&**model_guard, &embeddings)
2650                .unwrap_or_else(|_| embeddings.shallow_clone());
2651
2652            // Apply final normalization
2653            let normalized = model_guard
2654                .apply_final_norm(&hidden_states)
2655                .unwrap_or(hidden_states);
2656
2657            // Mean pool over sequence dimension: [1, seq_len, hidden_size] -> [1, hidden_size]
2658            let pooled = normalized.mean_dim(1, false, tch::Kind::Float);
2659
2660            // Squeeze batch dimension
2661            let pooled_1d = pooled.squeeze_dim(0); // [hidden_size]
2662
2663            // L2 normalize for cosine similarity
2664            let norm = pooled_1d.norm();
2665            let normalized_embedding = if norm.double_value(&[]) > 1e-12 {
2666                &pooled_1d / norm
2667            } else {
2668                pooled_1d
2669            };
2670
2671            // Convert to CPU and extract values
2672            let cpu_tensor = normalized_embedding.to_device(tch::Device::Cpu);
2673            let numel = cpu_tensor.numel();
2674            let mut embedding_vec = vec![0.0f32; numel];
2675            cpu_tensor.copy_data(&mut embedding_vec, numel);
2676
2677            Ok(embedding_vec)
2678        }
2679    }
2680
2681    /// Compute vision embeddings for one or more images.
2682    ///
2683    /// Preprocesses raw image bytes and runs them through the model's vision
2684    /// encoder (e.g. SigLIP). Returns one embedding vector per image.
2685    ///
2686    /// # Arguments
2687    /// * `images` - Raw image bytes (PNG/JPEG/RGB) per image
2688    ///
2689    /// # Returns
2690    /// * Vec<Vec<f32>> - Embedding vectors, one per input image
2691    pub fn embed_images(&self, images: &[Vec<u8>]) -> Result<Vec<Vec<f32>>> {
2692        use crate::runtime::image_utils::{load_image_from_bytes, ImagePreprocessConfig};
2693
2694        if images.is_empty() {
2695            return Ok(Vec::new());
2696        }
2697
2698        let model_guard = self
2699            .persistent_model
2700            .as_ref()
2701            .ok_or_else(|| anyhow!("No model loaded"))?
2702            .lock();
2703
2704        // Preprocess images to tensors
2705        let config = ImagePreprocessConfig::siglip();
2706        let mut image_tensors = Vec::with_capacity(images.len());
2707        for img_bytes in images {
2708            let tensor = load_image_from_bytes(img_bytes, &config, self.device)?;
2709            image_tensors.push(tensor);
2710        }
2711        let batch = Tensor::cat(&image_tensors, 0); // [B, 3, H, W]
2712
2713        let _no_grad = tch::no_grad_guard();
2714
2715        // Run through vision encoder
2716        let embeddings_tensor = model_guard.encode_vision(&batch)?;
2717        // embeddings_tensor shape: [B, hidden_size]
2718
2719        let batch_size = embeddings_tensor.size()[0] as usize;
2720        let dim = embeddings_tensor.size()[1] as usize;
2721        let cpu_tensor = embeddings_tensor
2722            .to_device(tch::Device::Cpu)
2723            .to_kind(tch::Kind::Float);
2724
2725        let mut results = Vec::with_capacity(batch_size);
2726        for i in 0..batch_size {
2727            let mut vec = vec![0.0f32; dim];
2728            let row = cpu_tensor.get(i as i64);
2729            row.copy_data(&mut vec, dim);
2730            results.push(vec);
2731        }
2732
2733        Ok(results)
2734    }
2735
2736    /// Process embeddings through all model layers for embedding extraction
2737    fn process_through_layers_for_embedding(
2738        &self,
2739        model: &dyn crate::runtime::architectures::ModelOperations,
2740        embeddings: &Tensor,
2741    ) -> Result<Tensor> {
2742        let num_layers = model.num_layers();
2743        let mut hidden_states = embeddings.shallow_clone();
2744
2745        for layer_idx in 0..num_layers {
2746            let (new_hidden, _kv) = model.decode_layer(
2747                layer_idx,
2748                &hidden_states,
2749                None, // attention_mask
2750                None, // position_ids
2751                None, // past_kv
2752            )?;
2753            hidden_states = new_hidden;
2754        }
2755
2756        Ok(hidden_states)
2757    }
2758
2759    /// Explicitly drop every CUDA/device tensor the engine owns or has cached
2760    /// on this thread, **now** — while the CUDA context is still alive.
2761    ///
2762    /// # Why this exists (#429)
2763    ///
2764    /// `tch`/libtorch defers CUDA context teardown to an `atexit` handler that
2765    /// runs during *process* exit. But two classes of device tensor outlive the
2766    /// engine struct and only drop later:
2767    ///
2768    /// 1. **Thread-local RoPE caches** (`llama::ROPE_CACHE`, `qwen3_5::ROPE_CACHE`)
2769    ///    — sin/cos tables cached on the compute device. They are TLS, so they
2770    ///    are not touched by `Drop` and survive until the owning thread exits.
2771    /// 2. **The inference service thread itself** — it is spawned detached in
2772    ///    `LocalInferenceService::start`, so its TLS destructors (and thus the
2773    ///    RoPE cache drop) can fire during process-exit cleanup, *after* the
2774    ///    CUDA context is gone → `c10::Error: invalid device pointer` (SIGABRT).
2775    ///
2776    /// The engine-only GPU smoke (`e6_gpu_engine_is_single_device_on_one_gpu_host`)
2777    /// never loads weights nor runs a forward pass, so it populates no RoPE
2778    /// cache and exits cleanly; the streaming test does, which is why only it
2779    /// aborts.
2780    ///
2781    /// Calling this on the service thread at `Shutdown` receipt — *before*
2782    /// replying to the client — drops the RoPE tables and the model weights
2783    /// synchronously while the caller is still blocked on the reply (process
2784    /// indubitably alive, context valid). After it returns only empty shells
2785    /// remain for any late TLS/static destruction, so there is nothing left to
2786    /// free against a dead context.
2787    ///
2788    /// # Contract
2789    ///
2790    /// This is a **terminal** teardown: the engine is unusable afterward
2791    /// (`model_info` reports unloaded, generation will error). It must only be
2792    /// called once, at shutdown. Unlike [`Drop`], it deliberately *does* clear
2793    /// shared `Arc<Mutex<Option<T>>>` contents — that is safe precisely because
2794    /// the engine is being retired, not because the contents are unused. (The
2795    /// production inference path holds a single engine instance per service; a
2796    /// cached clone would see `None` and fail closed with "No model loaded",
2797    /// which is the correct behavior for a retired engine.)
2798    pub fn release_device_resources(&mut self) {
2799        // 1. Clear the model's own KV cache while the model still exists (the
2800        //    cache tensors live inside the architecture model's KVCacheManager).
2801        self.clear_kv_cache();
2802
2803        // 2. Drop the architecture model (all decoder-layer weights, embeddings,
2804        //    lm_head) — the bulk of the device memory.
2805        self.persistent_model = None;
2806
2807        // 3. Drop the (dummy) VarStore and clear the session KV registry so any
2808        //    device tensors held there release before context teardown too.
2809        {
2810            let mut vs_guard = self.var_store.lock();
2811            *vs_guard = None;
2812        }
2813        if let Some(registry) = self.kv_cache_registry.clone() {
2814            registry.clear_all();
2815        }
2816
2817        // 4. Drop the thread-local RoPE sin/cos tables on *this* thread. This is
2818        //    the load-bearing step for #429: without it these CUDA tensors would
2819        //    only free in the detached service thread's TLS destructor.
2820        crate::runtime::architectures::llama::clear_rope_cache();
2821        crate::runtime::architectures::qwen3_5::clear_rope_cache();
2822
2823        // 5. Drain any in-flight CUDA kernels (frees completed, makes the above
2824        //    drops effective on the device) before we hand control back. CPU is
2825        //    a no-op. Best-effort: a failure to synchronize is logged, not fatal
2826        //    — the tensors are already dropped on the Rust side.
2827        if let Device::Cuda(idx) = self.device {
2828            // atc_synchronize panics on internal error; never propagate.
2829            let res = std::panic::catch_unwind(|| tch::Cuda::synchronize(idx as i64));
2830            if res.is_err() {
2831                warn!("CUDA synchronize during teardown returned an error (ignored)");
2832            }
2833        }
2834    }
2835}
2836
2837impl Drop for TorchEngine {
2838    fn drop(&mut self) {
2839        // Drop our Arc references to shared resources.
2840        // The actual cleanup happens when the LAST Arc reference drops.
2841        //
2842        // IMPORTANT: Do NOT clear the contents of Arc<Mutex<Option<T>>> fields!
2843        // All TorchEngine clones share the same Arc instances. Clearing the contents
2844        // would break other clones (especially the cached engine in model_cache).
2845        //
2846        // Previous implementation cleared shared Arc contents, causing:
2847        // - First request: clone for spawn_blocking drops, clears var_store
2848        // - Second request: cached engine has var_store = None, fails with "No model loaded"
2849
2850        // Drop our Arc references (NOT the contents)
2851        self.persistent_model = None; // Drops our Option<Arc<...>> reference
2852
2853        // Arc<Mutex<...>> fields are automatically cleaned up via Arc reference counting
2854        // when the last TorchEngine clone drops. No manual intervention needed.
2855    }
2856}
2857
2858#[cfg(test)]
2859#[path = "torch_engine_mtp_session_tests.rs"]
2860mod mtp_session_tests;
2861
2862#[cfg(test)]
2863#[allow(clippy::unwrap_used, clippy::expect_used)]
2864mod tests {
2865    use super::*;
2866    use parking_lot::Mutex;
2867    use std::io::Write;
2868    use std::sync::Arc;
2869
2870    #[test]
2871    fn precision_policy_preserves_aliases_and_non_cpu_modes_without_gpu_execution() {
2872        for device in [Device::Cpu, Device::Cuda(0), Device::Mps] {
2873            for channel in [false, true] {
2874                for (aliases, expected) in [
2875                    (
2876                        &["fp32", "float32", "FP32", "Float32"][..],
2877                        tch::Kind::Float,
2878                    ),
2879                    (&["fp16", "float16", "FP16", "Float16"][..], tch::Kind::Half),
2880                    (
2881                        &["bf16", "bfloat16", "BF16", "BFloat16"][..],
2882                        tch::Kind::BFloat16,
2883                    ),
2884                ] {
2885                    for alias in aliases {
2886                        assert_eq!(
2887                            resolve_inference_compute_kind(Some(alias), device, channel).unwrap(),
2888                            expected
2889                        );
2890                    }
2891                }
2892                let automatic = if device == Device::Cpu && channel {
2893                    tch::Kind::Float
2894                } else {
2895                    tch::Kind::BFloat16
2896                };
2897                for alias in [None, Some("auto"), Some("AUTO"), Some("fp8"), Some("FP8")] {
2898                    assert_eq!(
2899                        resolve_inference_compute_kind(alias, device, channel).unwrap(),
2900                        automatic
2901                    );
2902                }
2903                for invalid in ["", "fp64", "float8", "surprise"] {
2904                    assert!(
2905                        resolve_inference_compute_kind(Some(invalid), device, channel).is_err()
2906                    );
2907                }
2908            }
2909        }
2910    }
2911
2912    #[tokio::test]
2913    async fn decision_context_uses_actual_loaded_config_and_override() {
2914        let directory = tempfile::tempdir().unwrap();
2915        let path = directory.path().join("model.safetensors");
2916        let shapes = [
2917            ("model.embed_tokens.weight", vec![8, 4]),
2918            ("model.norm.weight", vec![4]),
2919            ("model.layers.0.input_layernorm.weight", vec![4]),
2920            ("model.layers.0.post_attention_layernorm.weight", vec![4]),
2921            ("model.layers.0.self_attn.q_proj.weight", vec![4, 4]),
2922            ("model.layers.0.self_attn.k_proj.weight", vec![4, 4]),
2923            ("model.layers.0.self_attn.v_proj.weight", vec![4, 4]),
2924            ("model.layers.0.self_attn.o_proj.weight", vec![4, 4]),
2925            ("model.layers.0.mlp.gate_proj.weight", vec![8, 4]),
2926            ("model.layers.0.mlp.up_proj.weight", vec![8, 4]),
2927            ("model.layers.0.mlp.down_proj.weight", vec![4, 8]),
2928        ];
2929        let weights: Vec<_> = shapes
2930            .into_iter()
2931            .map(|(key, shape)| {
2932                (
2933                    key.to_owned(),
2934                    Tensor::ones(shape.as_slice(), (tch::Kind::Float, Device::Cpu)),
2935                )
2936            })
2937            .collect();
2938        Tensor::write_safetensors(&weights, &path).unwrap();
2939        for (model_limit, override_limit, expected_limit, admitted_tokens) in [
2940            (64usize, None, 64usize, 64usize),
2941            (8192, None, 8192, 4097),
2942            (8192, Some(32), 32, 32),
2943        ] {
2944            let config = serde_json::json!({
2945                "model_type":"llama","architectures":["LlamaForCausalLM"],
2946                "hidden_size":4,"intermediate_size":8,"num_hidden_layers":1,
2947                "num_attention_heads":2,"num_key_value_heads":2,"head_dim":2,
2948                "vocab_size":8,"max_position_embeddings":model_limit,
2949                "rms_norm_eps":1e-5,"rope_theta":10000.0,"tie_word_embeddings":true
2950            });
2951            std::fs::write(
2952                directory.path().join("config.json"),
2953                serde_json::to_vec(&config).unwrap(),
2954            )
2955            .unwrap();
2956            let mut engine = TorchEngine::new(RuntimeConfig {
2957                use_gpu: false,
2958                context_length: 4096,
2959                max_context: override_limit,
2960                precision_mode: Some("fp32".to_owned()),
2961                speculative_decoding: false,
2962                ..Default::default()
2963            })
2964            .unwrap();
2965            assert!(engine.effective_context_limit().is_err());
2966            engine.load_model_file(&path).await.unwrap();
2967            assert_eq!(engine.effective_context_limit().unwrap(), expected_limit);
2968            assert_eq!(
2969                engine.context_state.lock().as_ref().unwrap().context_window,
2970                expected_limit
2971            );
2972            let overflow = vec![0i64; expected_limit + 1];
2973            let rejected = engine
2974                .with_decision_backbone_checked(&overflow, &|| Ok(()), |_, _| Ok(()))
2975                .unwrap_err();
2976            assert!(rejected.to_string().contains("exceeds context limit"));
2977            // Llama deliberately lacks decision hidden states. Reaching that
2978            // downstream error proves valid context was admitted without doing
2979            // a long forward. The8192case causally admits4097tokens, which the
2980            // previous RuntimeConfig default4096 incorrectly rejected.
2981            let admitted = vec![0i64; admitted_tokens];
2982            let downstream = engine
2983                .with_decision_backbone_checked(&admitted, &|| Ok(()), |_, _| Ok(()))
2984                .unwrap_err();
2985            assert!(downstream
2986                .to_string()
2987                .contains("decision hidden states unsupported"));
2988        }
2989    }
2990
2991    #[test]
2992    fn mtp_bonus_sampling_matches_serial_repetition_window() {
2993        let engine = TorchEngine::new(RuntimeConfig {
2994            use_gpu: false,
2995            speculative_decoding: false,
2996            ..Default::default()
2997        })
2998        .unwrap();
2999        const TOKENIZER: &str = r#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"oldest":0,"recent":1,"accepted":2,"[UNK]":3},"unk_token":"[UNK]"}}"#;
3000        *engine.tokenizer.lock() = Some(Tokenizer::from_bytes(TOKENIZER.as_bytes()).unwrap());
3001        let request = GenerationRequest {
3002            prompt: "recent".into(),
3003            temperature: Some(0.0),
3004            repeat_penalty: Some(2.0),
3005            repeat_last_n: Some(2),
3006            ..Default::default()
3007        };
3008        let mut stream = TextStream::new(&engine, request).unwrap();
3009        stream.recent_tokens = VecDeque::from([0, 1]);
3010        // The accepted first token makes serial evict token 0. Penalizing that
3011        // stale token changes the bonus argmax from 0 to 3 in this fixture.
3012        let logits = Tensor::from_slice(&[10.0_f32, 0.0, 0.0, 7.0]);
3013        let serial = engine
3014            .sample_token_with_params(
3015                &logits.copy(),
3016                &stream.sampling_params,
3017                &[1, 2],
3018                &stream.penalty_exempt_tokens,
3019            )
3020            .unwrap();
3021        let stale = engine
3022            .sample_token_with_params(
3023                &logits.copy(),
3024                &stream.sampling_params,
3025                &[0, 1, 2],
3026                &stream.penalty_exempt_tokens,
3027            )
3028            .unwrap();
3029        assert_ne!(stale, serial, "fixture must detect the pre-fix history");
3030        let bonus = stream.sample_spec_position(&logits.copy(), &[2]).unwrap();
3031        assert_eq!(bonus as usize, serial);
3032        assert_eq!(stream.recent_tokens_buffer, vec![1, 2]);
3033        assert_eq!(stream.recent_tokens, VecDeque::from([0, 1]));
3034        // More round-local tokens than the configured window still stay bounded.
3035        stream
3036            .sample_spec_position(&logits.copy(), &[2, 3, 1])
3037            .unwrap();
3038        assert_eq!(stream.recent_tokens_buffer, vec![3, 1]);
3039    }
3040
3041    // ===== #1253: prompt/token text must never reach process logs =====
3042
3043    /// Distinctive canary planted in prompt inputs; the assertions below prove
3044    /// it never appears in captured process logs.
3045    const LEAK_CANARY: &str = "PROMPTLEAKCANARY7q3z";
3046
3047    /// Minimal in-memory WordLevel tokenizer (no model files needed). The canary
3048    /// is a known vocab entry so it tokenizes cleanly on the success path; any
3049    /// other word maps to `[UNK]`, exercising the roundtrip-mismatch path.
3050    fn redact_test_tokenizer() -> Tokenizer {
3051        const TOK_JSON: &str = r#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"hello":0,"world":1,"[UNK]":3,"PROMPTLEAKCANARY7q3z":7},"unk_token":"[UNK]"}}"#;
3052        Tokenizer::from_bytes(TOK_JSON.as_bytes()).expect("valid in-memory tokenizer")
3053    }
3054
3055    /// `tracing_subscriber` writer that buffers all output into a shared Vec.
3056    struct CapturingWriter(Arc<Mutex<Vec<u8>>>);
3057    impl Write for CapturingWriter {
3058        fn write(&mut self, b: &[u8]) -> std::io::Result<usize> {
3059            self.0.lock().extend_from_slice(b);
3060            Ok(b.len())
3061        }
3062        fn flush(&mut self) -> std::io::Result<()> {
3063            Ok(())
3064        }
3065    }
3066
3067    /// Newtype wrapper so we can implement the foreign `MakeWriter` trait for a
3068    /// local type (orphan rule).
3069    #[derive(Clone)]
3070    struct CaptureSink(Arc<Mutex<Vec<u8>>>);
3071
3072    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CaptureSink {
3073        type Writer = CapturingWriter;
3074        fn make_writer(&'a self) -> Self::Writer {
3075            CapturingWriter(self.0.clone())
3076        }
3077    }
3078
3079    // ===== Session-cache recording vs cancellation-before-first-poll =====
3080
3081    /// Minimal `ArchitectureConfig` for the recording stub below: no model
3082    /// math is exercised, only the configuration surface the trait requires.
3083    struct StubArchConfig;
3084    impl crate::runtime::architectures::ArchitectureConfig for StubArchConfig {
3085        fn num_attention_heads(&self) -> usize {
3086            4
3087        }
3088        fn num_key_value_heads(&self) -> usize {
3089            4
3090        }
3091        fn hidden_size(&self) -> usize {
3092            16
3093        }
3094        fn intermediate_size(&self) -> usize {
3095            32
3096        }
3097        fn vocab_size(&self) -> usize {
3098            16
3099        }
3100        fn max_position_embeddings(&self) -> usize {
3101            64
3102        }
3103        fn rope_theta(&self) -> Option<f32> {
3104            None
3105        }
3106        fn rope_dim(&self) -> Option<usize> {
3107            None
3108        }
3109        fn layer_norm_eps(&self) -> f32 {
3110            1e-5
3111        }
3112        fn use_rms_norm(&self) -> bool {
3113            true
3114        }
3115    }
3116
3117    /// Model stub for cache-recording tests: holds a real (empty)
3118    /// `KVCacheManager` and serves it to the engine's save path via
3119    /// `get_kv_cache`. No forward is ever called — these tests exercise stream
3120    /// construction, drop, and cache recording only, so every compute method
3121    /// stays on the trait's `Err`/no-op defaults.
3122    struct RecordingStubModel {
3123        cache: Option<Arc<Mutex<crate::runtime::KVCacheManager>>>,
3124    }
3125    impl ModelOperations for RecordingStubModel {
3126        fn architecture(&self) -> crate::runtime::architectures::ModelArchitecture {
3127            crate::runtime::architectures::ModelArchitecture::Llama { version: 3 }
3128        }
3129        fn config(&self) -> &dyn crate::runtime::architectures::ArchitectureConfig {
3130            &StubArchConfig
3131        }
3132        fn forward(&self, _input: &Tensor, _past_kv: Option<&Tensor>) -> Result<Tensor> {
3133            Err(anyhow!("recording tests never poll the stream"))
3134        }
3135        fn reshape_for_attention(&self, tensor: &Tensor, _is_key_value: bool) -> Result<Tensor> {
3136            Ok(tensor.shallow_clone())
3137        }
3138        fn apply_rope(&self, tensor: &Tensor, _position_ids: &Tensor) -> Result<Tensor> {
3139            Ok(tensor.shallow_clone())
3140        }
3141        fn normalize(&self, tensor: &Tensor) -> Result<Tensor> {
3142            Ok(tensor.shallow_clone())
3143        }
3144        fn get_attention_mask(&self, _seq_len: usize, _past_kv_len: usize) -> Result<Tensor> {
3145            Err(anyhow!("recording tests never poll the stream"))
3146        }
3147        fn as_any(&self) -> &dyn std::any::Any {
3148            self
3149        }
3150        fn get_kv_cache(&self) -> Option<Arc<Mutex<crate::runtime::KVCacheManager>>> {
3151            self.cache.clone()
3152        }
3153        fn set_kv_cache(&mut self, cache: Arc<Mutex<crate::runtime::KVCacheManager>>) {
3154            self.cache = Some(cache);
3155        }
3156        fn clear_kv_cache(&self) {
3157            // Same shape as the real architectures: route the clear through
3158            // the installed KVCacheManager so the metadata contract under
3159            // test is the production `clear_all` path.
3160            if let Some(cache) = &self.cache {
3161                let mut guard = cache.lock();
3162                guard.clear_all();
3163            }
3164        }
3165    }
3166
3167    /// Regression (cancellation before first poll): dropping a stream that was
3168    /// never polled must NOT record its prompt IDs into the session cache.
3169    /// Exact-hit/miss resolution in `TextStream::new` clears the cache at
3170    /// generation setup, before any forward — recording IDs over that empty
3171    /// state would let a later extending prompt claim a prefix_len against KV
3172    /// that does not exist (the pure-attention hit path would partial-prefill
3173    /// from mid-prompt with no history behind it). `prefill_time_ms` is Some
3174    /// exactly when the first prefill forward completed, so it gates recording.
3175    #[test]
3176    fn drop_before_first_poll_does_not_record_session_cache() {
3177        let mut engine = TorchEngine::new(RuntimeConfig {
3178            use_gpu: false,
3179            ..RuntimeConfig::default()
3180        })
3181        .expect("CPU engine should construct without a model");
3182
3183        // In-memory tokenizer (same pattern as the log-redaction tests);
3184        // "hello world" → [0, 1].
3185        *engine.tokenizer.lock() = Some(redact_test_tokenizer());
3186
3187        // A real empty KV cache served by the stub model — the recording target.
3188        let cache: Arc<Mutex<crate::runtime::KVCacheManager>> = Arc::new(Mutex::new(
3189            crate::runtime::KVCacheManager::new(2, 64, crate::runtime::KVQuantType::None),
3190        ));
3191        engine.persistent_model = Some(Arc::new(Mutex::new(Box::new(RecordingStubModel {
3192            cache: Some(cache.clone()),
3193        }))));
3194
3195        let request = GenerationRequest {
3196            prompt: "hello world".to_owned(),
3197            ..Default::default()
3198        };
3199        let stream = TextStream::new(&engine, request).expect("stream should construct");
3200        // Cancellation: dropped before the first poll — no prefill ever ran.
3201        drop(stream);
3202
3203        assert_eq!(
3204            cache.lock().cached_token_count(),
3205            0,
3206            "a drop before the first poll must not record prompt IDs over the cleared cache"
3207        );
3208
3209        // The downstream hazard: a later extending prompt must not claim a
3210        // reusable prefix over state that was never prefilled.
3211        let extended = vec![0i64, 1, 5, 9];
3212        assert_eq!(
3213            cache.lock().prefix_match_len(&extended),
3214            0,
3215            "no prefix may be claimed over cache state that was never computed"
3216        );
3217    }
3218
3219    /// Regression (prior-session metadata vs cancellation): a cache populated
3220    /// by an earlier turn (prompt IDs + end-of-prefill SSM snapshot) must not
3221    /// retain that metadata after the set-up clear that a later turn's exact
3222    /// hit or miss performs at generation setup. That clear runs before any
3223    /// forward; if the turn is then cancelled before its first poll, nothing
3224    /// new is recorded — so a subsequent extending prompt would otherwise
3225    /// claim a prefix (and, for hybrid models, restore a stale snapshot)
3226    /// against KV that no longer exists.
3227    #[test]
3228    fn setup_clear_drops_prior_session_metadata_on_cancelled_turn() {
3229        let mut engine = TorchEngine::new(RuntimeConfig {
3230            use_gpu: false,
3231            ..RuntimeConfig::default()
3232        })
3233        .expect("CPU engine should construct without a model");
3234        *engine.tokenizer.lock() = Some(redact_test_tokenizer());
3235
3236        // Registry + session owner so the swap path engages: the set-up clear
3237        // then runs through resolve_session_prefill_start exactly as the
3238        // production prefix-detection branch drives it.
3239        let registry = Arc::new(crate::runtime::kv_cache::KVCacheRegistry::new(
3240            crate::runtime::kv_cache::CacheConfig {
3241                num_layers: 2,
3242                max_seq_len: 64,
3243                quant_type: crate::runtime::KVQuantType::None,
3244                paged: false,
3245            },
3246            None,
3247        ));
3248        let owner = crate::runtime::kv_cache::CacheOwner::Session("s1".to_owned());
3249        let prior = registry.get_or_create(owner.clone());
3250        // Stamp the cache compatibly so the #1277 reuse policy keeps it and
3251        // prefix detection actually sees the prior metadata.
3252        let mut desc = crate::runtime::kv_compat::KvCompatDescriptor::default();
3253        desc.weights.base_revision = "test-base".to_owned();
3254        desc.set_tokenizer(16, Some("test-tok-hash".to_owned()));
3255        *engine.kv_compat.lock() = Some(desc);
3256        {
3257            let mut prior_guard = prior.lock();
3258            prior_guard.set_compat_fingerprint(engine.kv_compat_fingerprint_for(0).unwrap());
3259            let opt = (tch::Kind::Float, Device::Cpu);
3260            let conv = vec![Some(Tensor::zeros([2, 3], opt)), None];
3261            let rec = vec![Some(Tensor::zeros([4], opt)), None];
3262            // Prior turn's prompt [0, 1, 7] strictly extends this turn's
3263            // ("hello world" → [0, 1]): the request is an exact hit, whose
3264            // set-up clear routes through resolve_session_prefill_start →
3265            // model.clear_kv_cache() — the production boundary under test.
3266            prior_guard.set_cached_tokens_with_ssm(vec![0, 1, 7], Some((conv, rec)));
3267        }
3268        engine.kv_cache_registry = Some(registry);
3269        *engine.active_cache_owner.lock() = Some(owner);
3270        engine.persistent_model = Some(Arc::new(Mutex::new(Box::new(RecordingStubModel {
3271            cache: None,
3272        }))));
3273
3274        let request = GenerationRequest {
3275            prompt: "hello world".to_owned(),
3276            ..Default::default()
3277        };
3278        let stream = TextStream::new(&engine, request).expect("stream should construct");
3279        // Cancelled before the first poll: the set-up clear has run, and no
3280        // new recording can happen.
3281        drop(stream);
3282
3283        // `prior` is the same Arc the swap installed into the model.
3284        let guard = prior.lock();
3285        assert_eq!(
3286            guard.cached_token_count(),
3287            0,
3288            "prior prompt IDs must not survive the set-up clear"
3289        );
3290        assert!(
3291            guard.ssm_snapshot().is_none(),
3292            "prior end-of-prefill SSM snapshot must not survive the set-up clear"
3293        );
3294        assert_eq!(
3295            guard.prefix_match_len(&[0, 1, 7, 4]),
3296            0,
3297            "an extending prompt must not claim a prefix over cleared KV"
3298        );
3299    }
3300
3301    /// Run `f` under a thread-local TRACE subscriber whose output is captured.
3302    /// TRACE is deliberate: #1253 forbids relocating prompt text to DEBUG, so
3303    /// the canary must be absent even at the lowest log level.
3304    fn capture_logs<F: FnOnce()>(f: F) -> String {
3305        let sink = CaptureSink(Arc::new(Mutex::new(Vec::new())));
3306        let subscriber = tracing_subscriber::fmt()
3307            .with_max_level(tracing::Level::TRACE)
3308            .with_ansi(false)
3309            .with_writer(sink.clone())
3310            .finish();
3311        let _guard = tracing::subscriber::set_default(subscriber);
3312        f();
3313        // Drain the captured bytes into a local so the MutexGuard drops before
3314        // `sink` itself (block-end drop order).
3315        let cells = std::mem::take(&mut *sink.0.lock());
3316        String::from_utf8(cells).unwrap()
3317    }
3318
3319    /// #1253: a distinctive prompt secret must never appear in process logs,
3320    /// regardless of tokenization outcome — success, empty-token failure, or
3321    /// roundtrip mismatch.
3322    ///
3323    /// (Generation, stop, cancellation, and stream-error paths were audited:
3324    /// after the #1261 redaction they log only positions/counts and error
3325    /// objects — never prompt text, generated text, or token IDs, which are
3326    /// reversible to text via the loaded tokenizer. Those paths require a full
3327    /// model to drive, so the content-bearing tokenization outcomes — which are
3328    /// exactly where the prompt is handled — are covered hermetically here.)
3329    #[test]
3330    fn tokenize_redacted_never_logs_prompt_content() {
3331        let tokenizer = redact_test_tokenizer();
3332
3333        // --- success: every word is a known token ---
3334        let logs = capture_logs(|| {
3335            let ids = tokenize_redacted(&tokenizer, &format!("hello {LEAK_CANARY} world"));
3336            assert_eq!(ids.unwrap().len(), 3);
3337        });
3338        assert!(
3339            !logs.contains("Tokenized prompt"),
3340            "success path must no longer emit a metering log line (token counts are OTel metrics now, #1261): {logs}"
3341        );
3342        assert!(
3343            !logs.contains(LEAK_CANARY),
3344            "prompt secret leaked into logs on success: {logs}"
3345        );
3346
3347        // --- roundtrip mismatch: unknown word decodes to [UNK] ---
3348        let logs = capture_logs(|| {
3349            let ids = tokenize_redacted(&tokenizer, &format!("{LEAK_CANARY} zzz_unknown"));
3350            // non-empty -> Ok, but the decode diverges -> mismatch warning
3351            assert_eq!(ids.unwrap().len(), 2);
3352        });
3353        assert!(
3354            logs.contains("roundtrip mismatch"),
3355            "mismatch path must still emit bounded metadata: {logs}"
3356        );
3357        assert!(
3358            !logs.contains(LEAK_CANARY),
3359            "prompt secret leaked into logs on mismatch: {logs}"
3360        );
3361        assert!(
3362            !logs.contains("zzz_unknown"),
3363            "decoded token text leaked into logs on mismatch: {logs}"
3364        );
3365
3366        // --- empty-token failure: 0 tokens (pure whitespace input) ---
3367        // An input that carries the canary always yields >=1 token with this
3368        // tokenizer, so the failure path is exercised with whitespace; we still
3369        // assert it logs only bounded metadata ("empty token sequence") and no
3370        // prompt content.
3371        let logs = capture_logs(|| {
3372            let res = tokenize_redacted(&tokenizer, "   ");
3373            assert!(res.is_err(), "whitespace-only input must fail");
3374        });
3375        assert!(
3376            logs.contains("empty token sequence"),
3377            "failure path must still emit bounded metadata: {logs}"
3378        );
3379    }
3380
3381    // ---- base-weight content digest (#1277, finding F1) ----
3382    //
3383    // The pure framing/digest helpers live in `kv_compat`; these cover the engine
3384    // seam: loader-selected shard-set resolution (extra/unreferenced `.safetensors`
3385    // files are never hashed), fail-closed selection, resolved-byte hashing
3386    // (incl. LFS/XET pointer handling), determinism, and parity with the pure
3387    // helpers.
3388
3389    use crate::runtime::kv_compat;
3390
3391    /// Write `bytes` to `dir/shard_name`.
3392    fn write_shard(dir: &std::path::Path, shard_name: &str, bytes: &[u8]) -> std::path::PathBuf {
3393        use std::io::Write;
3394        let p = dir.join(shard_name);
3395        let mut f = std::fs::File::create(&p).unwrap();
3396        f.write_all(bytes).unwrap();
3397        p
3398    }
3399
3400    /// Write a minimal `model.safetensors.index.json` whose `weight_map`
3401    /// references exactly `shard_names`.
3402    fn write_index(dir: &std::path::Path, shard_names: &[&str]) {
3403        let weight_map: serde_json::Map<String, serde_json::Value> = shard_names
3404            .iter()
3405            .enumerate()
3406            .map(|(i, name)| {
3407                (
3408                    format!("tensor.{i}"),
3409                    serde_json::Value::String((*name).to_owned()),
3410                )
3411            })
3412            .collect();
3413        let index = serde_json::json!({ "weight_map": weight_map });
3414        std::fs::write(
3415            dir.join("model.safetensors.index.json"),
3416            serde_json::to_string_pretty(&index).unwrap(),
3417        )
3418        .unwrap();
3419    }
3420
3421    #[tokio::test]
3422    async fn weights_content_digest_is_deterministic_and_matches_pure() {
3423        let dir = tempfile::tempdir().unwrap();
3424        write_shard(
3425            dir.path(),
3426            "model-00001-of-00002.safetensors",
3427            b"AAAA-content",
3428        );
3429        write_shard(
3430            dir.path(),
3431            "model-00002-of-00002.safetensors",
3432            b"BBBB-content",
3433        );
3434        write_index(
3435            dir.path(),
3436            &[
3437                "model-00001-of-00002.safetensors",
3438                "model-00002-of-00002.safetensors",
3439            ],
3440        );
3441        let a = TorchEngine::weights_content_digest(dir.path())
3442            .await
3443            .unwrap();
3444        let b = TorchEngine::weights_content_digest(dir.path())
3445            .await
3446            .unwrap();
3447        assert_eq!(a, b, "digest must be deterministic across runs");
3448        // Parity with the pure framing helper (engine resolves+hashes, pure fn frames).
3449        let expected = kv_compat::base_weight_digest_from_shards(&[
3450            (
3451                "model-00001-of-00002.safetensors".into(),
3452                kv_compat::shard_content_digest(b"AAAA-content"),
3453            ),
3454            (
3455                "model-00002-of-00002.safetensors".into(),
3456                kv_compat::shard_content_digest(b"BBBB-content"),
3457            ),
3458        ]);
3459        assert_eq!(
3460            a, expected,
3461            "engine digest must equal the pure framing over resolved bytes"
3462        );
3463    }
3464
3465    #[tokio::test]
3466    async fn weight_digest_reads_sealed_tree_after_checkout_mutation() {
3467        let dir = tempfile::tempdir().unwrap();
3468        let repo = git2db::Repository::init(dir.path()).unwrap();
3469        repo.config().unwrap().set_str("user.name", "test").unwrap();
3470        repo.config()
3471            .unwrap()
3472            .set_str("user.email", "test@example.invalid")
3473            .unwrap();
3474        write_shard(dir.path(), "model.safetensors", b"reviewed weights");
3475        let mut index = repo.index().unwrap();
3476        index
3477            .add_path(std::path::Path::new("model.safetensors"))
3478            .unwrap();
3479        index.write().unwrap();
3480        let tree = repo.find_tree(index.write_tree().unwrap()).unwrap();
3481        let sig = repo.signature().unwrap();
3482        let commit = repo
3483            .commit(Some("HEAD"), &sig, &sig, "reviewed", &tree, &[])
3484            .unwrap();
3485        drop(tree);
3486        drop(repo);
3487
3488        let artifact = git2db::pinned_tree::PinnedTree::acquire(dir.path(), commit)
3489            .await
3490            .unwrap()
3491            .into_sealed_projection()
3492            .unwrap();
3493        let baseline = TorchEngine::weights_content_digest(artifact.root())
3494            .await
3495            .unwrap();
3496        std::fs::write(dir.path().join("model.safetensors"), b"replacement weights").unwrap();
3497        let after = TorchEngine::weights_content_digest(artifact.root())
3498            .await
3499            .unwrap();
3500        assert_eq!(baseline, after);
3501        assert_ne!(
3502            baseline,
3503            TorchEngine::weights_content_digest(dir.path())
3504                .await
3505                .unwrap()
3506        );
3507    }
3508
3509    #[tokio::test]
3510    async fn weights_content_digest_same_basename_different_bytes_diverge() {
3511        let d1 = tempfile::tempdir().unwrap();
3512        let d2 = tempfile::tempdir().unwrap();
3513        write_shard(d1.path(), "model-00001-of-00002.safetensors", b"content-A");
3514        write_shard(d1.path(), "model-00002-of-00002.safetensors", b"content-B");
3515        write_index(
3516            d1.path(),
3517            &[
3518                "model-00001-of-00002.safetensors",
3519                "model-00002-of-00002.safetensors",
3520            ],
3521        );
3522        write_shard(d2.path(), "model-00001-of-00002.safetensors", b"content-A!");
3523        write_shard(d2.path(), "model-00002-of-00002.safetensors", b"content-B");
3524        write_index(
3525            d2.path(),
3526            &[
3527                "model-00001-of-00002.safetensors",
3528                "model-00002-of-00002.safetensors",
3529            ],
3530        );
3531        let a = TorchEngine::weights_content_digest(d1.path())
3532            .await
3533            .unwrap();
3534        let b = TorchEngine::weights_content_digest(d2.path())
3535            .await
3536            .unwrap();
3537        assert_ne!(a, b, "same basenames, different content must diverge");
3538    }
3539
3540    /// Two shard sets equal as a (name → content) map but enumerated in
3541    /// different creation order must produce the same digest (sorted determinism).
3542    #[tokio::test]
3543    async fn weights_content_digest_is_order_independent() {
3544        let d1 = tempfile::tempdir().unwrap();
3545        let d2 = tempfile::tempdir().unwrap();
3546        write_shard(d1.path(), "model-00001-of-00002.safetensors", b"X");
3547        write_shard(d1.path(), "model-00002-of-00002.safetensors", b"Y");
3548        write_index(
3549            d1.path(),
3550            &[
3551                "model-00001-of-00002.safetensors",
3552                "model-00002-of-00002.safetensors",
3553            ],
3554        );
3555        // Reverse creation order in d2.
3556        write_shard(d2.path(), "model-00002-of-00002.safetensors", b"Y");
3557        write_shard(d2.path(), "model-00001-of-00002.safetensors", b"X");
3558        write_index(
3559            d2.path(),
3560            &[
3561                "model-00001-of-00002.safetensors",
3562                "model-00002-of-00002.safetensors",
3563            ],
3564        );
3565        let a = TorchEngine::weights_content_digest(d1.path())
3566            .await
3567            .unwrap();
3568        let b = TorchEngine::weights_content_digest(d2.path())
3569            .await
3570            .unwrap();
3571        assert_eq!(a, b, "digest must be independent of enumeration order");
3572    }
3573
3574    #[tokio::test]
3575    async fn weights_content_digest_empty_dir_fails_closed() {
3576        let dir = tempfile::tempdir().unwrap();
3577        assert!(
3578            TorchEngine::weights_content_digest(dir.path())
3579                .await
3580                .is_err(),
3581            "an empty model dir must fail closed, not mint a partial digest"
3582        );
3583    }
3584
3585    #[tokio::test]
3586    async fn weights_content_digest_no_safetensors_fails_closed() {
3587        let dir = tempfile::tempdir().unwrap();
3588        write_shard(dir.path(), "config.json", b"{}");
3589        write_shard(dir.path(), "tokenizer.json", b"{}");
3590        assert!(
3591            TorchEngine::weights_content_digest(dir.path())
3592                .await
3593                .is_err(),
3594            "a dir with no .safetensors shards must fail closed"
3595        );
3596    }
3597
3598    /// A small (<1 KiB) ordinary shard that is NOT an LFS pointer must be hashed
3599    /// as content (not skipped, not treated as a pointer).
3600    #[tokio::test]
3601    async fn weights_content_digest_small_plain_shard_hashed_as_content() {
3602        let dir = tempfile::tempdir().unwrap();
3603        let bytes = b"tiny-but-real-weights";
3604        write_shard(dir.path(), "model.safetensors", bytes);
3605        let digest = TorchEngine::weights_content_digest(dir.path())
3606            .await
3607            .unwrap();
3608        let expected = kv_compat::base_weight_digest_from_shards(&[(
3609            "model.safetensors".into(),
3610            kv_compat::shard_content_digest(bytes),
3611        )]);
3612        assert_eq!(
3613            digest, expected,
3614            "a small non-pointer shard must hash its content bytes"
3615        );
3616    }
3617
3618    /// An un-smudged LFS/XET pointer shard must fail closed (never be hashed as a
3619    /// pointer stub). With no resolvable object store in the test environment,
3620    /// resolution errors out → the digest declines.
3621    #[tokio::test]
3622    async fn weights_content_digest_pointer_fails_closed() {
3623        let dir = tempfile::tempdir().unwrap();
3624        // A canonical LFS pointer header (<1 KiB). It is not resolvable here.
3625        let pointer = b"version https://git-lfs.github.com/spec/v1\noid sha256:0000000000000000000000000000000000000000000000000000000000000000\nsize 1234\n";
3626        write_shard(dir.path(), "model.safetensors", pointer);
3627        assert!(
3628            TorchEngine::weights_content_digest(dir.path())
3629                .await
3630                .is_err(),
3631            "an un-smudged LFS pointer must fail closed, not be hashed as a stub"
3632        );
3633    }
3634
3635    /// An unreadable shard must fail closed (read error propagates), not produce
3636    /// a digest over the readable remainder. Skipped when running as root (root
3637    /// bypasses the 0o000 permission).
3638    #[cfg(unix)]
3639    #[tokio::test]
3640    async fn weights_content_digest_unreadable_shard_fails_closed() {
3641        // Root bypasses the 0o000 permission, so the read would succeed and the
3642        // fail-closed assertion could not hold — skip silently under root.
3643        if unsafe { libc::geteuid() } == 0 {
3644            return;
3645        }
3646        let dir = tempfile::tempdir().unwrap();
3647        let p = write_shard(dir.path(), "model.safetensors", b"real-content");
3648        use std::os::unix::fs::PermissionsExt;
3649        std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o000)).unwrap();
3650        let res = TorchEngine::weights_content_digest(dir.path()).await;
3651        // Restore so tempdir cleanup can remove it.
3652        let _ = std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o644));
3653        assert!(
3654            res.is_err(),
3655            "an unreadable shard must fail closed, not hash a partial set"
3656        );
3657    }
3658
3659    /// An extra/unreferenced `.safetensors` in the directory must NOT shift the
3660    /// digest: only the index-referenced (loader-selected) shards are hashed.
3661    #[tokio::test]
3662    async fn weights_content_digest_ignores_unreferenced_extra_shard() {
3663        let dir = tempfile::tempdir().unwrap();
3664        write_shard(
3665            dir.path(),
3666            "model-00001-of-00002.safetensors",
3667            b"AAAA-content",
3668        );
3669        write_shard(
3670            dir.path(),
3671            "model-00002-of-00002.safetensors",
3672            b"BBBB-content",
3673        );
3674        write_index(
3675            dir.path(),
3676            &[
3677                "model-00001-of-00002.safetensors",
3678                "model-00002-of-00002.safetensors",
3679            ],
3680        );
3681        let selected_only = TorchEngine::weights_content_digest(dir.path())
3682            .await
3683            .unwrap();
3684
3685        // A stale shard with a shard-like name the loader never references.
3686        write_shard(
3687            dir.path(),
3688            "model-00003-of-00002.safetensors",
3689            b"STALE-alt-precision",
3690        );
3691        let with_extra = TorchEngine::weights_content_digest(dir.path())
3692            .await
3693            .unwrap();
3694
3695        assert_eq!(
3696            selected_only, with_extra,
3697            "an unreferenced extra shard must be invariant to the digest"
3698        );
3699        let expected = kv_compat::base_weight_digest_from_shards(&[
3700            (
3701                "model-00001-of-00002.safetensors".into(),
3702                kv_compat::shard_content_digest(b"AAAA-content"),
3703            ),
3704            (
3705                "model-00002-of-00002.safetensors".into(),
3706                kv_compat::shard_content_digest(b"BBBB-content"),
3707            ),
3708        ]);
3709        assert_eq!(
3710            with_extra, expected,
3711            "digest must cover exactly the loader-selected set, no more"
3712        );
3713    }
3714
3715    /// With a single `model.safetensors`, an extra shard-like file is not part
3716    /// of the loader's selection and must not shift the digest.
3717    #[tokio::test]
3718    async fn weights_content_digest_single_file_ignores_extra_shard() {
3719        let dir = tempfile::tempdir().unwrap();
3720        write_shard(dir.path(), "model.safetensors", b"single-file-weights");
3721        let baseline = TorchEngine::weights_content_digest(dir.path())
3722            .await
3723            .unwrap();
3724        let expected = kv_compat::base_weight_digest_from_shards(&[(
3725            "model.safetensors".into(),
3726            kv_compat::shard_content_digest(b"single-file-weights"),
3727        )]);
3728        assert_eq!(baseline, expected);
3729
3730        // Even a name matching the shard glob is not selected when
3731        // `model.safetensors` is the loader's pick.
3732        write_shard(
3733            dir.path(),
3734            "model-00001-of-00001.safetensors",
3735            b"UNREFERENCED",
3736        );
3737        let with_extra = TorchEngine::weights_content_digest(dir.path())
3738            .await
3739            .unwrap();
3740        assert_eq!(
3741            baseline, with_extra,
3742            "an extra shard next to model.safetensors must be invariant"
3743        );
3744    }
3745
3746    /// Changing a SELECTED shard changes the digest; adding, removing, or
3747    /// mutating an UNREFERENCED shard does not.
3748    #[tokio::test]
3749    async fn weights_content_digest_selected_mutations_diverge_unreferenced_do_not() {
3750        let dir = tempfile::tempdir().unwrap();
3751        write_shard(
3752            dir.path(),
3753            "model-00001-of-00002.safetensors",
3754            b"AAAA-content",
3755        );
3756        write_shard(
3757            dir.path(),
3758            "model-00002-of-00002.safetensors",
3759            b"BBBB-content",
3760        );
3761        write_index(
3762            dir.path(),
3763            &[
3764                "model-00001-of-00002.safetensors",
3765                "model-00002-of-00002.safetensors",
3766            ],
3767        );
3768        let baseline = TorchEngine::weights_content_digest(dir.path())
3769            .await
3770            .unwrap();
3771
3772        // Add an unreferenced shard.
3773        let extra = write_shard(dir.path(), "stale-backup.safetensors", b"stale");
3774        assert_eq!(
3775            TorchEngine::weights_content_digest(dir.path())
3776                .await
3777                .unwrap(),
3778            baseline,
3779            "adding an unreferenced shard must not change the digest"
3780        );
3781        // Mutate the unreferenced shard.
3782        write_shard(dir.path(), "stale-backup.safetensors", b"stale-CHANGED");
3783        assert_eq!(
3784            TorchEngine::weights_content_digest(dir.path())
3785                .await
3786                .unwrap(),
3787            baseline,
3788            "mutating an unreferenced shard must not change the digest"
3789        );
3790        // Remove the unreferenced shard.
3791        std::fs::remove_file(&extra).unwrap();
3792        assert_eq!(
3793            TorchEngine::weights_content_digest(dir.path())
3794                .await
3795                .unwrap(),
3796            baseline,
3797            "removing an unreferenced shard must not change the digest"
3798        );
3799        // Change a SELECTED shard.
3800        write_shard(
3801            dir.path(),
3802            "model-00002-of-00002.safetensors",
3803            b"BBBB-CHANGED",
3804        );
3805        assert_ne!(
3806            TorchEngine::weights_content_digest(dir.path())
3807                .await
3808                .unwrap(),
3809            baseline,
3810            "changing a selected shard must change the digest"
3811        );
3812    }
3813
3814    /// A directory whose only `.safetensors` files match no loader selection
3815    /// pattern has ZERO selected shards and must fail closed (reuse declined).
3816    #[tokio::test]
3817    async fn weights_content_digest_zero_selected_shards_fails_closed() {
3818        let dir = tempfile::tempdir().unwrap();
3819        write_shard(dir.path(), "backup.safetensors", b"not-a-selected-shard");
3820        write_shard(dir.path(), "optimizer.safetensors", b"also-not-selected");
3821        assert!(
3822            TorchEngine::weights_content_digest(dir.path())
3823                .await
3824                .is_err(),
3825            "a dir with no loader-selected shards must fail closed"
3826        );
3827    }
3828
3829    /// An index referencing a shard that does not exist on disk is an
3830    /// inconsistent selection and must fail closed (reuse declined).
3831    #[tokio::test]
3832    async fn weights_content_digest_missing_index_shard_fails_closed() {
3833        let dir = tempfile::tempdir().unwrap();
3834        write_shard(
3835            dir.path(),
3836            "model-00001-of-00002.safetensors",
3837            b"AAAA-content",
3838        );
3839        write_index(
3840            dir.path(),
3841            &[
3842                "model-00001-of-00002.safetensors",
3843                "model-00002-of-00002.safetensors", // referenced but absent
3844            ],
3845        );
3846        assert!(
3847            TorchEngine::weights_content_digest(dir.path())
3848                .await
3849                .is_err(),
3850            "an index referencing a missing shard must fail closed"
3851        );
3852    }
3853
3854    /// A manifest-less MULTI-shard glob selection is ambiguous (the glob can
3855    /// silently drop/duplicate shards), so the digest must fail closed —
3856    /// decline reuse — even though the LOADER only warns in its permissive
3857    /// default mode. Fingerprint authority is stricter than loader
3858    /// permissiveness (#1277 revise-4): this holds regardless of
3859    /// `HYPRSTREAM_STRICT_LOADER` (the test env does not set it, exercising
3860    /// the permissive loader path).
3861    #[tokio::test]
3862    async fn weights_content_digest_manifestless_multi_shard_fails_closed() {
3863        let dir = tempfile::tempdir().unwrap();
3864        write_shard(
3865            dir.path(),
3866            "model-00001-of-00002.safetensors",
3867            b"AAAA-content",
3868        );
3869        write_shard(
3870            dir.path(),
3871            "model-00002-of-00002.safetensors",
3872            b"BBBB-content",
3873        );
3874        // NO model.safetensors.index.json — the loader warns and continues;
3875        // the digest must refuse.
3876        assert!(
3877            TorchEngine::weights_content_digest(dir.path())
3878                .await
3879                .is_err(),
3880            "an ambiguous manifest-less multi-shard glob set must fail closed"
3881        );
3882    }
3883
3884    /// A SINGLE globbed shard (no index, no model.safetensors) is unambiguous
3885    /// and must still digest — only multi-shard glob sets are ambiguous.
3886    #[tokio::test]
3887    async fn weights_content_digest_single_globbed_shard_passes() {
3888        let dir = tempfile::tempdir().unwrap();
3889        write_shard(
3890            dir.path(),
3891            "model-00001-of-00001.safetensors",
3892            b"single-shard-content",
3893        );
3894        let digest = TorchEngine::weights_content_digest(dir.path())
3895            .await
3896            .unwrap();
3897        let expected = kv_compat::base_weight_digest_from_shards(&[(
3898            "model-00001-of-00001.safetensors".into(),
3899            kv_compat::shard_content_digest(b"single-shard-content"),
3900        )]);
3901        assert_eq!(
3902            digest, expected,
3903            "a single globbed shard is unambiguous and must digest"
3904        );
3905    }
3906
3907    #[test]
3908    fn test_generation_config_in_engine_has_correct_max_tokens() {
3909        // Create a minimal runtime config
3910        let _runtime_config = RuntimeConfig::default();
3911
3912        // Create engine (this will fail without LIBTORCH but we can test the constructor logic)
3913        // We're just verifying the default generation_config
3914        let gen_config = GenerationConfig {
3915            max_tokens: 2048,
3916            temperature: 0.7,
3917            top_p: 0.9,
3918            top_k: Some(40),
3919            repeat_penalty: 1.1,
3920            stop_tokens: vec!["</s>".to_owned()],
3921            seed: None,
3922            stream: false,
3923        };
3924
3925        // Verify the defaults we set
3926        assert_eq!(
3927            gen_config.max_tokens, 2048,
3928            "TorchEngine should initialize with max_tokens=2048"
3929        );
3930    }
3931
3932    /// `with_device(.., None)` — the single-device path used by every legacy
3933    /// caller — must produce an engine that reports single-device and exposes
3934    /// no pool. This is the regression guard that the #314 wiring does not
3935    /// perturb single-GPU/CPU construction.
3936    #[test]
3937    fn single_device_engine_is_not_multi_device() {
3938        let engine =
3939            super::TorchEngine::with_device(RuntimeConfig::default(), Device::Cpu, None).unwrap();
3940        assert!(!engine.is_multi_device());
3941        assert!(engine.device_pool().is_none());
3942        assert_eq!(engine.device(), Device::Cpu);
3943    }
3944
3945    /// #429: `release_device_resources` is the terminal teardown that drops
3946    /// CUDA tensors (weights, KV cache, thread-local RoPE tables) on the
3947    /// service thread at shutdown. It must:
3948    /// - be safe on an engine that never loaded a model (no-op for the
3949    ///   weight/KV paths, but must still clear RoPE TLS),
3950    /// - be idempotent (double-call must not panic / double-free),
3951    /// - leave the engine visibly unloaded afterward.
3952    ///
3953    /// This runs on CPU (no CUDA context), so it can't reproduce the SIGABRT
3954    /// itself; it pins the *ordering contract* the GPU fix relies on. The
3955    /// thread-local RoPE caches are populated on whichever thread runs the
3956    /// forward pass — this test clears the test thread's own TLS, which is
3957    /// exactly what the production Shutdown handler does on the service thread.
3958    #[test]
3959    fn release_device_resources_is_safe_and_idempotent() {
3960        let mut engine =
3961            super::TorchEngine::with_device(RuntimeConfig::default(), Device::Cpu, None).unwrap();
3962
3963        // Never loaded a model — these start empty.
3964        assert!(!engine.has_varstore());
3965
3966        // First call: no model weights, no KV registry, no RoPE entries on this
3967        // thread — must still succeed and clear the (empty) RoPE TLS.
3968        engine.release_device_resources();
3969        assert!(
3970            !engine.has_varstore(),
3971            "an unloaded engine stays unloaded after teardown"
3972        );
3973
3974        // Second call: idempotent — must not panic or double-free.
3975        engine.release_device_resources();
3976        engine.release_device_resources();
3977
3978        // Dropping the engine afterward must also be clean (no double-drop of
3979        // the already-None persistent_model / var_store).
3980        drop(engine);
3981    }
3982
3983    /// #429: a loaded-model teardown must actually drop the VarStore contents.
3984    /// We can't load a real model without weights/libtorch-on-this-host, but we
3985    /// can set the dummy VarStore (the same step `load_model_file` performs) and
3986    /// confirm `release_device_resources` clears it — the contract the GPU path
3987    /// relies on to free weight tensors before CUDA context teardown.
3988    #[test]
3989    fn release_device_resources_clears_loaded_varstore() {
3990        let mut engine =
3991            super::TorchEngine::with_device(RuntimeConfig::default(), Device::Cpu, None).unwrap();
3992        // Mimic the load path: install a dummy VarStore on the engine's device.
3993        {
3994            let vs = VarStore::new(Device::Cpu);
3995            let mut guard = engine.var_store.lock();
3996            *guard = Some(vs);
3997        }
3998        assert!(engine.has_varstore());
3999
4000        engine.release_device_resources();
4001
4002        assert!(
4003            !engine.has_varstore(),
4004            "release_device_resources must drop the VarStore"
4005        );
4006    }
4007
4008    /// `with_device(.., Some(multi_pool))` must mark the engine multi-device and
4009    /// expose the pool with the right primary. We build the pool from explicit
4010    /// `Device::Cuda` values via `DevicePool::from_devices` (which does not probe
4011    /// CUDA), so this runs on CPU-only CI — we never construct a model here, we
4012    /// only exercise the engine-struct wiring (#314 selection logic).
4013    #[test]
4014    fn multi_device_pool_marks_engine_multi_device() {
4015        use crate::runtime::device_pool::DevicePool;
4016        let pool = DevicePool::from_devices(vec![Device::Cuda(0), Device::Cuda(1)]).unwrap();
4017        assert_eq!(pool.len(), 2);
4018        let primary = pool.primary();
4019        let engine = super::TorchEngine::with_device(
4020            RuntimeConfig::default(),
4021            primary,
4022            Some(std::sync::Arc::new(pool.clone())),
4023        )
4024        .unwrap();
4025        assert!(engine.is_multi_device());
4026        let exposed = engine
4027            .device_pool()
4028            .expect("multi-device engine must expose its pool");
4029        assert_eq!(exposed.devices(), pool.devices());
4030        assert_eq!(exposed.primary(), primary);
4031        assert_eq!(engine.device(), primary);
4032    }
4033
4034    /// The map the factory would build for a 2-device pool must be a contiguous
4035    /// prefix-heavy split with exactly one stage boundary — the invariant the
4036    /// `forward_layers` path relies on. This pins the planner the wiring hands
4037    /// to model construction (#314 → #310 epic).
4038    #[test]
4039    fn engine_multi_device_map_has_one_boundary_per_stage_gap() {
4040        use crate::runtime::device_pool::{DevicePool, LayerDeviceMap};
4041        let pool = DevicePool::from_devices(vec![Device::Cuda(0), Device::Cuda(1)]).unwrap();
4042        // 7 layers over 2 devices → [0,0,0,0, 1,1,1] (prefix gets the extra).
4043        let map = LayerDeviceMap::even_split(&pool, 7).unwrap();
4044        assert_eq!(map.len(), 7);
4045        let mut boundaries = 0;
4046        let mut prev = map.device_for(0);
4047        for g in 1..map.len() {
4048            if map.is_boundary(prev, g) {
4049                boundaries += 1;
4050            }
4051            prev = map.device_for(g);
4052        }
4053        assert_eq!(
4054            boundaries, 1,
4055            "2-device even_split has exactly one cross-device copy per forward"
4056        );
4057        assert!(!map.is_single_device());
4058    }
4059}
4060
4061/// Internal sampling parameters passed to TensorSampler
4062#[derive(Debug, Clone, Copy)]
4063struct SamplingParams {
4064    temperature: f32,
4065    top_p: f32,
4066    top_k: Option<usize>,
4067    repeat_penalty: f32,
4068}
4069
4070use futures::Stream;
4071use std::collections::VecDeque;
4072use std::pin::Pin;
4073use std::task::{Context as TaskContext, Poll};
4074
4075/// Statistics about text generation with separate prefill/inference metrics
4076#[derive(Debug, Clone)]
4077pub struct GenerationStats {
4078    // Overall metrics
4079    pub tokens_generated: usize,
4080    pub generation_time_ms: u64,
4081    pub tokens_per_second: f32,
4082    pub finish_reason: Option<FinishReason>,
4083    /// Quality metrics captured during generation (for self-supervised training)
4084    pub quality_metrics: Option<crate::runtime::generation_metrics::GenerationQualityMetrics>,
4085
4086    // Prefill metrics (processing the prompt)
4087    pub prefill_tokens: usize,
4088    pub prefill_time_ms: u64,
4089    pub prefill_tokens_per_sec: f32,
4090
4091    // Inference metrics (generating new tokens, excluding prefill)
4092    pub inference_tokens: usize,
4093    pub inference_time_ms: u64,
4094    /// Cumulative average: tokens / time (accurate for final reporting)
4095    pub inference_tokens_per_sec: f32,
4096    /// Exponential moving average (responsive for real-time adaptive batching)
4097    pub inference_tokens_per_sec_ema: f32,
4098
4099    // MTP self-speculative decoding outcomes (0 when speculation is off)
4100    /// Draft tokens accepted by greedy exact-match verify this request
4101    pub speculative_accepted: u64,
4102    /// Draft tokens rejected (and rewound) this request
4103    pub speculative_rejected: u64,
4104}
4105
4106/// Stream that yields decoded UTF-8 text chunks during generation.
4107///
4108/// Automatically handles UTF-8 buffering, stop tokens, EOS detection,
4109/// timeout handling, cancellation, and all generation complexities.
4110/// Just iterate and get text!
4111pub struct TextStream<'a> {
4112    engine: &'a TorchEngine,
4113
4114    /// Optional per-tenant delta for delta-aware inference.
4115    /// When present, LoRA corrections are injected at each attention layer.
4116    delta: Option<std::sync::Arc<parking_lot::Mutex<crate::training::TenantDelta>>>,
4117
4118    prompt_tokens: Vec<i64>,
4119    last_generated: Option<i64>,
4120
4121    recent_tokens: VecDeque<i64>,
4122    repeat_last_n: usize,
4123
4124    // PERF: Individual sampling params removed - now stored in sampling_params field below
4125    // This avoids struct recreation per token while keeping backward compatibility
4126    max_tokens: usize,
4127    stop_token_ids: Vec<u32>,
4128
4129    // Store tokenizer as Arc for safe sharing across streams
4130    // IMPORTANT: This field is required for the unsafe transmute in TextStream::new()
4131    // DO NOT REMOVE - it ensures the tokenizer lives long enough for the decode_stream lifetime 'a
4132    #[allow(dead_code)]
4133    tokenizer: Arc<Tokenizer>,
4134    decode_stream: tokenizers::tokenizer::DecodeStream<
4135        'a,
4136        tokenizers::models::ModelWrapper,
4137        tokenizers::normalizers::NormalizerWrapper,
4138        tokenizers::pre_tokenizers::PreTokenizerWrapper,
4139        tokenizers::processors::PostProcessorWrapper,
4140        tokenizers::decoders::DecoderWrapper,
4141    >,
4142
4143    prompt_len: usize,
4144    /// Position from which prefill should start (0 = full prefill, >0 = partial via prefix cache hit)
4145    prefill_start_pos: usize,
4146    /// SSM (conv/rec) state snapshot captured at end-of-prefill (hybrid
4147    /// recurrent models only; `None` for pure-attention models). Saved into
4148    /// the session cache by `save_cached_tokens` so a later prefix hit can
4149    /// rewind the recurrent state alongside the KV truncation
4150    /// (`rewind_session_state`).
4151    prefill_ssm_snapshot: Option<(Vec<Option<Tensor>>, Vec<Option<Tensor>>)>,
4152    /// KV cache position tracking for this stream
4153    /// Each stream has exclusive access via &mut self, so no atomic needed
4154    kv_cache_position: usize,
4155    /// Total tokens generated (including buffered UTF-8) - for statistics only
4156    tokens_generated: usize,
4157    start_time: std::time::Instant,
4158    finished: bool,
4159    finish_reason: Option<FinishReason>,
4160
4161    // Timeout handling
4162    timeout_ms: Option<u64>,
4163
4164    /// Whether to collect per-token quality metrics (expensive - ~10x overhead)
4165    collect_metrics: bool,
4166
4167    /// Metrics accumulator for self-supervised training quality signals
4168    metrics_accumulator: crate::runtime::generation_metrics::GenerationMetricsAccumulator,
4169
4170    // PERF: Cached values to avoid per-token recomputation
4171    /// Pre-created sampling params - avoids struct creation per token
4172    sampling_params: SamplingParams,
4173    /// Cached tokenizer vocab size - avoids lock acquisition per token
4174    vocab_size: usize,
4175    /// Cached model vocab size (from logits shape) - set after first forward
4176    model_vocab_size: usize,
4177    /// Reusable buffer for recent_tokens when VecDeque wraps around
4178    recent_tokens_buffer: Vec<i64>,
4179    /// Token IDs exempt from repeat penalty (single-character tokens like digits 0-9,
4180    /// punctuation, etc.) These tokens appear frequently in number generation and
4181    /// should not be penalized just for reoccurring.
4182    penalty_exempt_tokens: HashSet<i64>,
4183
4184    // Timing for prefill/inference separation
4185    /// Time spent on prefill (processing prompt), set after first forward pass
4186    prefill_time_ms: Option<u64>,
4187    /// Timestamp when first token was sampled (after prefill completes)
4188    first_token_time: Option<std::time::Instant>,
4189
4190    // EMA rate tracking for adaptive batching
4191    /// Timestamp when the last token was generated (for inter-token timing)
4192    last_token_time: Option<std::time::Instant>,
4193    /// Exponential moving average of tokens per second
4194    ema_tokens_per_sec: f32,
4195
4196    /// Model label for OTel token-burn metric attributes (#1261).
4197    model_label: String,
4198    /// Opaque (hashed) tenant id for token-burn metric attributes — never the
4199    /// raw request subject (#1253). `None` when no tenant is in scope.
4200    tenant_hash: Option<u64>,
4201
4202    // MTP self-speculative decoding state (v1: greedy, batch=1, dense Qwen3.5).
4203    /// Whether this stream decodes via MTP draft + verify. Decided once at
4204    /// construction from the engine config, model capability, and request
4205    /// sampling params (see `new_with_delta`).
4206    speculative: bool,
4207    /// Draft token proposed by the MTP head for the position after
4208    /// `last_generated`; always `Some` between speculative decode rounds.
4209    spec_pending_draft: Option<u32>,
4210    /// Accepted tokens from the current speculative round not yet consumed by
4211    /// `poll_next` (1 on reject, up to 2 on accept).
4212    spec_out_queue: VecDeque<u32>,
4213    /// Draft outcome counters for this request (OTel `record_speculative` at
4214    /// drop + `GenerationStats`). Plain fields, never gated on collect_metrics.
4215    spec_accepted: u64,
4216    spec_rejected: u64,
4217}
4218
4219impl<'a> TextStream<'a> {
4220    fn new(engine: &'a TorchEngine, request: GenerationRequest) -> Result<Self> {
4221        // No delta path: no tenant dependency and adapter generation 0 (base
4222        // weights only).
4223        Self::new_with_delta(engine, request, None, None, 0, 0)
4224    }
4225
4226    fn new_with_delta(
4227        engine: &'a TorchEngine,
4228        request: GenerationRequest,
4229        delta: Option<std::sync::Arc<parking_lot::Mutex<crate::training::TenantDelta>>>,
4230        tenant_id: Option<String>,
4231        weight_epoch: u64,
4232        adapter_generation: u64,
4233    ) -> Result<Self> {
4234        let prompt_tokens = engine.tokenize(&request.prompt)?;
4235        let prompt_len = prompt_tokens.len();
4236
4237        // Token-burn metering (#1261): emit the prompt-token count at the
4238        // tokenization boundary as an OTel metric. The raw tenant subject is never
4239        // a metric attribute — it is reduced to an opaque hash (#1253); the
4240        // authoritative spend ledger is #1264.
4241        let tenant_hash = crate::runtime::token_metrics::opaque_tenant_id(&tenant_id);
4242        let model_label = engine.model_label();
4243        crate::runtime::token_metrics::global_meter().record_prompt(
4244            &model_label,
4245            tenant_hash,
4246            prompt_len as u64,
4247        );
4248
4249        let tokenizer = engine.get_tokenizer()?;
4250        let stop_token_ids: Vec<u32> = request.stop_tokens.as_deref().unwrap_or(&[])
4251            .iter()
4252            .filter_map(|stop_str| {
4253                let encoding = tokenizer.encode(stop_str.as_str(), false).ok()?;
4254                let ids = encoding.get_ids();
4255                if ids.len() == 1 {
4256                    Some(ids[0])
4257                } else {
4258                    // The stop sequence is caller-supplied content and is never
4259                    // logged (#1253); only its byte length and resulting token
4260                    // count are retained. The tenant is referenced only by its
4261                    // opaque hash — the raw subject is dropped from logs.
4262                    tracing::warn!(
4263                        stop_bytes = stop_str.len(),
4264                        encoded_tokens = ids.len(),
4265                        tenant_hash = ?tenant_hash,
4266                        "Stop token encodes to multiple tokens, skipping (only single-token stops supported; content redacted)"
4267                    );
4268                    None
4269                }
4270            })
4271            .collect();
4272
4273        // Evict LRU session caches if memory budget is exceeded.
4274        // This must happen before allocating/swapping in a new cache.
4275        engine.evict_session_caches();
4276
4277        // Session cache management with prefix detection:
4278        // If an active session exists, swap its KV cache into the model and check
4279        // if the new prompt shares a prefix with the cached tokens. This avoids
4280        // re-computing attention for unchanged conversation history.
4281        let prefill_start_pos = if engine.swap_session_cache(tenant_id, weight_epoch) {
4282            // Session cache swapped in — check for prefix match
4283            let prefix_len = if let Some(model_arc) = &engine.persistent_model {
4284                let model = model_arc.lock();
4285                if let Some(cache) = model.get_kv_cache() {
4286                    let mut cache_guard = cache.lock();
4287
4288                    // KV-cache compatibility guard (#1277): KV produced under an
4289                    // incompatible (or unproven) model/runtime config is never
4290                    // reused. The policy is **fail-closed** via
4291                    // `decide_cache_reuse`: a stamp mismatch, an unstamped but
4292                    // *populated* cache, or the absence of an authoritative
4293                    // expected identity all discard the KV and recompute. Only a
4294                    // genuinely fresh (empty) cache under an authoritative
4295                    // identity is stamped for reuse. This is the pull-based
4296                    // complement to the registry's push-based delta invalidation.
4297                    let expected = engine.kv_compat_fingerprint_for(adapter_generation);
4298                    cache_guard.apply_reuse_policy(expected);
4299
4300                    let matched = cache_guard.prefix_match_len(&prompt_tokens);
4301                    if matched > 0 {
4302                        tracing::info!(
4303                            "Prefix cache hit: {} of {} tokens reusable (skipping {:.0}% of prefill)",
4304                            matched, prompt_len,
4305                            (matched as f64 / prompt_len as f64) * 100.0
4306                        );
4307                    }
4308                    matched
4309                } else {
4310                    0
4311                }
4312            } else {
4313                0
4314            };
4315
4316            if prefix_len > 0 && prefix_len <= prompt_len {
4317                // Resolve the hit: a strict partial prefix rewinds cached
4318                // state (KV truncation; hybrid recurrent models additionally
4319                // restore the end-of-prefill GDN conv/rec snapshot). An
4320                // exact-prompt repeat (`prefix_len == prompt_len`) must
4321                // recompute from cleared state instead — there is no suffix
4322                // left to prefill, and the fresh forward would double-advance
4323                // restored recurrent state. Both decisions live in
4324                // `resolve_session_prefill_start`.
4325                if let Some(model_arc) = &engine.persistent_model {
4326                    let model = model_arc.lock();
4327                    resolve_session_prefill_start(model.as_ref(), prefix_len, prompt_len)
4328                } else {
4329                    0
4330                }
4331            } else {
4332                // No match — clear and start fresh
4333                engine.clear_kv_cache();
4334                0
4335            }
4336        } else {
4337            engine.clear_kv_cache();
4338            0
4339        };
4340
4341        let repeat_last_n = request
4342            .repeat_last_n
4343            .map(|v| v as usize)
4344            .filter(|&v| v > 0)
4345            .unwrap_or(64);
4346
4347        // Use Arc for safe tokenizer sharing
4348        // This Arc is REQUIRED by the TextStream struct - see the comment on the tokenizer field
4349        let tokenizer_arc = Arc::new(tokenizer);
4350
4351        // Create DecodeStream with proper lifetime management
4352        // The Arc<Tokenizer> is stored in the TextStream, ensuring the tokenizer lives as long as 'a
4353        // DEPENDENCY: The unsafe transmute below depends on tokenizer_arc being stored in the struct
4354        let decode_stream = {
4355            // We need to extend the tokenizer lifetime to match 'a
4356            // Since we have Arc<Tokenizer> stored in the struct, we can safely create a reference
4357            let tokenizer_ref = unsafe {
4358                // SAFETY: This is safe because:
4359                // 1. The tokenizer_arc is stored in the TextStream struct, ensuring it lives as long as 'a
4360                // 2. We're not moving or deallocating the tokenizer while the stream exists
4361                // 3. Arc guarantees thread-safe reference counting and proper lifetime management
4362                std::mem::transmute::<&Tokenizer, &'a Tokenizer>(tokenizer_arc.as_ref())
4363            };
4364
4365            // Use skip_special_tokens=false because <|extra_N|> tokens are special tokens
4366            // that should appear in output (they represent actual model vocabulary)
4367            tokenizer_ref.decode_stream(false) // skip_special_tokens=false
4368        };
4369
4370        // Build set of token IDs exempt from repeat penalty.
4371        // Single-character tokens (digits, punctuation, single letters) appear frequently
4372        // in number generation. Penalizing them causes digit suppression: "1917" → "209" → "41".
4373        // We exempt any token whose decoded form is a single character.
4374        let penalty_exempt_tokens = {
4375            let vocab = tokenizer_arc.get_vocab(true);
4376            let mut exempt = HashSet::new();
4377            for (token_str, &id) in &vocab {
4378                // Check if this token decodes to a single character
4379                // Token strings in the vocab may have special prefixes (like Ġ for space+char)
4380                // We want to exempt raw single characters: digits, punctuation, single letters
4381                let clean = token_str.trim();
4382                if clean.chars().count() == 1 {
4383                    exempt.insert(id as i64);
4384                }
4385            }
4386            tracing::debug!(
4387                "Built penalty-exempt token set: {} single-char tokens",
4388                exempt.len()
4389            );
4390            exempt
4391        };
4392
4393        // PERF: Pre-create sampling params to avoid struct allocation per token.
4394        // Fall back to model's generation_config defaults (from generation_config.json
4395        // or engine defaults) when the request doesn't specify a value.
4396        let gc = &engine.generation_config;
4397        let sampling_params = SamplingParams {
4398            temperature: request.temperature.unwrap_or(gc.temperature),
4399            top_p: request.top_p.unwrap_or(gc.top_p),
4400            top_k: request.top_k.map(|v| v as usize).or(gc.top_k),
4401            repeat_penalty: request.repeat_penalty.unwrap_or(gc.repeat_penalty),
4402        };
4403
4404        // PERF: Cache vocab_size to avoid lock acquisition per token
4405        let vocab_size = engine.get_vocab_size();
4406
4407        // MTP self-speculative decoding (v1): opt-in via config, and only when
4408        // every correctness prerequisite holds. Reasons are logged once here
4409        // (counts/config only — never content, #1253).
4410        let speculative = if !engine.config.speculative_decoding {
4411            false
4412        } else if delta.is_some() {
4413            // The MTP head is never delta-adapted, so acceptance would degrade
4414            // under a tenant delta. Correctness would be preserved (verify runs
4415            // the delta'd model), but v1 keeps the two features disjoint.
4416            info!("Speculative decoding disabled: tenant LoRA delta active (MTP head is not delta-adapted)");
4417            false
4418        } else if engine.config.kv_quant_type != crate::runtime::KVQuantType::None {
4419            // Quantized LayerKVCache::truncate_to falls back to a full clear(),
4420            // which would silently break the reject rewind.
4421            info!("Speculative decoding disabled: quantized KV cache (reject rewind needs O(1) truncate)");
4422            false
4423        } else if sampling_params.temperature > 0.01 {
4424            // Greedy-only v1: exact-match verify is only exact under argmax.
4425            info!("Speculative decoding disabled: non-greedy sampling (v1 is greedy-only)");
4426            false
4427        } else if !engine.speculative_supported() {
4428            info!("Speculative decoding disabled: model has no dense MTP draft head");
4429            false
4430        } else {
4431            info!("Speculative decoding enabled (MTP self-draft, k=1, greedy)");
4432            true
4433        };
4434
4435        Ok(Self {
4436            engine,
4437            delta,
4438            prompt_tokens,
4439            last_generated: None,
4440            recent_tokens: VecDeque::with_capacity(repeat_last_n),
4441            repeat_last_n,
4442            max_tokens: request.max_tokens.unwrap_or(2048) as usize,
4443            stop_token_ids,
4444            tokenizer: tokenizer_arc,
4445            decode_stream,
4446            prompt_len,
4447            prefill_start_pos,
4448            prefill_ssm_snapshot: None, // Captured after successful serial or speculative prefill
4449            // KV cache starts with prompt already in it after first forward
4450            kv_cache_position: prompt_len,
4451            tokens_generated: 0,
4452            start_time: std::time::Instant::now(),
4453            finished: false,
4454            finish_reason: None,
4455            timeout_ms: request.timeout_ms,
4456            collect_metrics: false, // Server-side concern, not per-request
4457            metrics_accumulator:
4458                crate::runtime::generation_metrics::GenerationMetricsAccumulator::new(),
4459            // PERF: Cached values initialized here, avoid per-token recomputation
4460            sampling_params,
4461            vocab_size,
4462            model_vocab_size: 0, // Set after first forward pass from logits shape
4463            recent_tokens_buffer: Vec::with_capacity(repeat_last_n),
4464            penalty_exempt_tokens,
4465            prefill_time_ms: None,
4466            first_token_time: None,
4467            last_token_time: None,
4468            ema_tokens_per_sec: 0.0,
4469            model_label,
4470            tenant_hash,
4471            speculative,
4472            spec_pending_draft: None,
4473            spec_out_queue: VecDeque::new(),
4474            spec_accepted: 0,
4475            spec_rejected: 0,
4476        })
4477    }
4478
4479    /// Get generation statistics (call after stream exhausted)
4480    pub fn stats(&self) -> GenerationStats {
4481        let total_time = self.start_time.elapsed();
4482
4483        // Prefill metrics
4484        let prefill_time_ms = self.prefill_time_ms.unwrap_or(0);
4485        let prefill_tokens = self.prompt_len;
4486        let prefill_tokens_per_sec = if prefill_time_ms > 0 {
4487            (prefill_tokens as f32 * 1000.0) / prefill_time_ms as f32
4488        } else {
4489            0.0
4490        };
4491
4492        // Inference metrics (time since first token was sampled)
4493        let inference_time_ms = self
4494            .first_token_time
4495            .map(|t| t.elapsed().as_millis() as u64)
4496            .unwrap_or(0);
4497        let inference_tokens = self.tokens_generated;
4498
4499        // Cumulative average for final reporting
4500        let inference_tokens_per_sec = if inference_time_ms > 0 {
4501            (inference_tokens as f32 * 1000.0) / inference_time_ms as f32
4502        } else {
4503            0.0
4504        };
4505
4506        // EMA for real-time adaptive batching (falls back to cumulative if not yet initialized)
4507        let inference_tokens_per_sec_ema = if self.ema_tokens_per_sec > 0.0 {
4508            self.ema_tokens_per_sec
4509        } else {
4510            inference_tokens_per_sec
4511        };
4512
4513        // Finalize quality metrics from accumulator
4514        let quality_metrics = if !self.metrics_accumulator.is_empty() {
4515            Some(self.metrics_accumulator.finalize())
4516        } else {
4517            None
4518        };
4519
4520        GenerationStats {
4521            tokens_generated: self.tokens_generated,
4522            generation_time_ms: total_time.as_millis() as u64,
4523            tokens_per_second: if total_time.as_secs_f32() > 0.0 {
4524                self.tokens_generated as f32 / total_time.as_secs_f32()
4525            } else {
4526                0.0
4527            },
4528            finish_reason: self.finish_reason.clone(),
4529            quality_metrics,
4530
4531            // Separate prefill/inference metrics
4532            prefill_tokens,
4533            prefill_time_ms,
4534            prefill_tokens_per_sec,
4535            inference_tokens,
4536            inference_time_ms,
4537            inference_tokens_per_sec,
4538            inference_tokens_per_sec_ema,
4539
4540            speculative_accepted: self.spec_accepted,
4541            speculative_rejected: self.spec_rejected,
4542        }
4543    }
4544
4545    /// Update EMA rate after generating a token.
4546    ///
4547    /// Uses exponential moving average with alpha=0.3 to smooth rate measurements.
4548    /// This handles initial acceleration gracefully without needing warmup hacks.
4549    /// Save the current token sequence (prompt + generated) to the session KV cache.
4550    ///
4551    /// Called when generation finishes so the next turn can detect prefix overlap.
4552    /// For hybrid recurrent models (Qwen3.5) the end-of-prefill SSM snapshot
4553    /// captured by either prefill path is stored alongside the tokens, making
4554    /// the next turn's prefix hit rewindable (`rewind_session_state`).
4555    fn save_cached_tokens(&mut self) {
4556        // Session-cache recording describes the model's live KV/SSM state,
4557        // which exists only once this stream's first prefill completed. A drop
4558        // before the first poll (cancellation, client disconnect) reaches this
4559        // Drop with the cache already cleared — exact-hit/miss resolution runs
4560        // at generation setup, before any forward — and a failed first prefill
4561        // can leave partial forward state matching no token count. Recording
4562        // prompt IDs in either case lets a later extending prompt claim a
4563        // prefix_len over missing or wrong state (a pure-attention partial
4564        // prefill would then resume with no history behind it). `prefill_time_ms`
4565        // is set exactly when the first prefill forward returned successfully
4566        // on either decoding path, so it is the fail-safe recording gate.
4567        if self.prefill_time_ms.is_none() {
4568            return;
4569        }
4570        if let Some(model_arc) = &self.engine.persistent_model {
4571            let model = model_arc.lock();
4572            if let Some(cache) = model.get_kv_cache() {
4573                let mut cache_guard = cache.lock();
4574                // Save the prompt token IDs for prefix matching on the next turn.
4575                // We only save the prompt (not generated tokens) because the next turn's
4576                // prompt will include the assistant's response via the chat template —
4577                // so the entire current prompt becomes a prefix of the next turn's prompt.
4578                cache_guard.set_cached_tokens_with_ssm(
4579                    self.prompt_tokens.clone(),
4580                    self.prefill_ssm_snapshot.take(),
4581                );
4582                tracing::debug!(
4583                    "Saved {} cached tokens for prefix matching on next turn",
4584                    cache_guard.cached_token_count()
4585                );
4586            }
4587        }
4588    }
4589
4590    fn update_ema_rate(&mut self) {
4591        let now = std::time::Instant::now();
4592
4593        if let Some(last_time) = self.last_token_time {
4594            let elapsed_secs = last_time.elapsed().as_secs_f32();
4595            if elapsed_secs > 0.0 {
4596                let instantaneous_rate = 1.0 / elapsed_secs;
4597
4598                // EMA: alpha=0.3 gives good responsiveness while smoothing noise
4599                const ALPHA: f32 = 0.3;
4600                if self.ema_tokens_per_sec > 0.0 {
4601                    self.ema_tokens_per_sec =
4602                        ALPHA * instantaneous_rate + (1.0 - ALPHA) * self.ema_tokens_per_sec;
4603                } else {
4604                    // First measurement - use instantaneous rate
4605                    self.ema_tokens_per_sec = instantaneous_rate;
4606                }
4607            }
4608        }
4609
4610        self.last_token_time = Some(now);
4611    }
4612
4613    fn sample_next_token(&mut self) -> Result<u32> {
4614        // Determine KV position for this forward pass
4615        let current_kv_pos = if self.tokens_generated == 0 {
4616            if self.prefill_start_pos > 0 {
4617                self.prefill_start_pos // Resume from prefix cache hit
4618            } else {
4619                0 // Full prefill from start
4620            }
4621        } else {
4622            self.kv_cache_position
4623        };
4624
4625        // Lock delta once for this token's forward pass (if present)
4626        let delta_guard = self.delta.as_ref().map(|d| d.lock());
4627
4628        let logits = if self.tokens_generated == 0 {
4629            // PREFILL: Process prompt (full or partial based on prefix cache hit)
4630            let prefill_start = std::time::Instant::now();
4631
4632            let result = if self.prefill_start_pos > 0
4633                && self.prefill_start_pos < self.prompt_tokens.len()
4634            {
4635                // PARTIAL PREFILL: prefix is cached, only forward new tokens
4636                let new_tokens = &self.prompt_tokens[self.prefill_start_pos..];
4637                tracing::info!(
4638                    "Partial prefill: processing {} new tokens (skipped {} cached)",
4639                    new_tokens.len(),
4640                    self.prefill_start_pos
4641                );
4642                if delta_guard.is_some() {
4643                    let delta_ref = delta_guard.as_deref();
4644                    self.engine.forward_with_delta_cached(
4645                        new_tokens,
4646                        self.prefill_start_pos,
4647                        delta_ref,
4648                    )?
4649                } else {
4650                    self.engine
4651                        .forward_cached(new_tokens, self.prefill_start_pos)?
4652                }
4653            } else {
4654                // FULL PREFILL: no prefix cache hit
4655                if delta_guard.is_some() {
4656                    let delta_ref = delta_guard.as_deref();
4657                    self.engine
4658                        .forward_with_delta_full(&self.prompt_tokens, delta_ref)?
4659                } else {
4660                    self.engine.forward(&self.prompt_tokens)?
4661                }
4662            };
4663            let prefill_elapsed = prefill_start.elapsed();
4664
4665            // Hybrid recurrent models (Qwen3.5): capture the SSM conv/rec state
4666            // as of end-of-prefill, before decode advances it. `save_cached_tokens`
4667            // stores it with the prompt tokens so a later session prefix hit can
4668            // rewind the recurrent state (KV truncation alone cannot). This is a
4669            // cheap downcast returning `None` for pure-attention models.
4670            self.prefill_ssm_snapshot = self.engine.snapshot_ssm_states();
4671
4672            // Store prefill timing
4673            self.prefill_time_ms = Some(prefill_elapsed.as_millis() as u64);
4674            self.first_token_time = Some(std::time::Instant::now());
4675
4676            let prefill_token_count = if self.prefill_start_pos > 0 {
4677                self.prompt_tokens.len() - self.prefill_start_pos
4678            } else {
4679                self.prompt_tokens.len()
4680            };
4681            tracing::info!(
4682                "PREFILL: {} tokens in {:?} ({:.2} tok/sec){}{}",
4683                prefill_token_count,
4684                prefill_elapsed,
4685                if prefill_elapsed.as_secs_f32() > 0.0 {
4686                    prefill_token_count as f32 / prefill_elapsed.as_secs_f32()
4687                } else {
4688                    0.0
4689                },
4690                if self.prefill_start_pos > 0 {
4691                    format!(" [prefix cached: {} tokens]", self.prefill_start_pos)
4692                } else {
4693                    String::new()
4694                },
4695                if delta_guard.is_some() {
4696                    " [delta-aware]"
4697                } else {
4698                    ""
4699                }
4700            );
4701
4702            result
4703        } else {
4704            let last_token = self.last_generated.ok_or_else(|| {
4705                anyhow::anyhow!(
4706                    "Internal error: last_generated not set after {} tokens",
4707                    self.tokens_generated
4708                )
4709            })?;
4710
4711            if self.tokens_generated.is_multiple_of(50) {
4712                // Counts/positions only — token IDs are reversible to text via the
4713                // loaded tokenizer and must never hit process logs (#1253/#1261).
4714                tracing::debug!(
4715                    "🔵 KV cache position: {}, tokens_generated: {}",
4716                    self.kv_cache_position,
4717                    self.tokens_generated
4718                );
4719            }
4720
4721            // Use kv_cache_position directly - this is where the next token will be written
4722            if delta_guard.is_some() {
4723                let delta_ref = delta_guard.as_deref();
4724                self.engine
4725                    .forward_with_delta_cached(&[last_token], current_kv_pos, delta_ref)?
4726            } else {
4727                self.engine.forward_cached(&[last_token], current_kv_pos)?
4728            }
4729        };
4730
4731        // Release delta lock before sampling (no longer needed)
4732        drop(delta_guard);
4733
4734        // NOTE: Logits truncation has been DISABLED (Nov 6, 2025)
4735        //
4736        // Previous code tried to truncate logits from model vocab (151936) to tokenizer vocab (151669).
4737        // This was causing year tokens to be excluded from sampling, leading to "1 7 7 6" instead of "1776".
4738        //
4739        // vLLM with the same model does NOT have this issue, suggesting the truncation was too aggressive.
4740        //
4741        // Solution: Use model vocab directly. The sampler will only pick valid tokens anyway.
4742        // The tokenizer's get_vocab_size() should match model vocab if properly configured.
4743
4744        // PERF: Use cached vocab_size - avoids lock acquisition per token
4745        let vocab_size = self.vocab_size;
4746
4747        // PERF: Cache model_vocab_size on first call (from logits shape)
4748        let model_vocab_size = if self.model_vocab_size == 0 {
4749            let logits_shape = logits.size();
4750            let size = logits_shape[logits_shape.len() - 1] as usize;
4751            self.model_vocab_size = size;
4752            size
4753        } else {
4754            self.model_vocab_size
4755        };
4756
4757        if vocab_size == 0 {
4758            // Tokenizer not loaded - this should never happen during generation
4759            return Err(anyhow::anyhow!(
4760                "Cannot sample tokens: tokenizer vocabulary size is 0 (tokenizer not loaded)"
4761            ));
4762        }
4763
4764        // Log mismatch for debugging but don't truncate
4765        if model_vocab_size != vocab_size {
4766            tracing::debug!(
4767                "Vocab mismatch: model_vocab_size={}, tokenizer_vocab_size={}. Using model vocab directly.",
4768                model_vocab_size, vocab_size
4769            );
4770        }
4771
4772        // UTF-8 reranking DISABLED - DecodeStream handles UTF-8 correctly
4773        // The reranker was causing number corruption by manipulating logits
4774        // for non-UTF-8 related tokens (like digits)
4775        //
4776        // YEAR CORRUPTION BUG FIX (Nov 6, 2025):
4777        // The real issue is vocab mismatch: model (151936) vs tokenizer (151669).
4778        // Years like "1776" are split into individual digits "1 7 7 6" because:
4779        // 1. Year tokens may be in the truncated range (>151669) if model's vocab
4780        //    extends beyond tokenizer's
4781        // 2. OR the model's logits heavily favor digit tokens over year tokens
4782        // 3. This is NOT a UTF-8 issue - it's a vocab/tokenization issue
4783        //
4784        // Current approach: Keep reranker disabled (it corrupted numbers), but
4785        // we need to investigate why year tokens are disfavored in the logits.
4786        // The truncation to vocab_size is necessary to prevent out-of-bounds,
4787        // but may be filtering out valid year token IDs.
4788
4789        // PERF: Use pre-created sampling params - avoids struct allocation per token
4790        let params = &self.sampling_params;
4791
4792        // PERF: Optimize VecDeque slice handling
4793        // Pass penalty_exempt_tokens to sampler for reduced penalty on single-char tokens
4794        let (slice1, slice2) = self.recent_tokens.as_slices();
4795        let recent_tokens_slice = if slice2.is_empty() {
4796            slice1
4797        } else {
4798            self.recent_tokens_buffer.clear();
4799            self.recent_tokens_buffer
4800                .extend(self.recent_tokens.iter().copied());
4801            &self.recent_tokens_buffer
4802        };
4803        let next_token = self.engine.sample_token_with_params(
4804            &logits,
4805            params,
4806            recent_tokens_slice,
4807            &self.penalty_exempt_tokens,
4808        )?;
4809
4810        // Validate sampled token is within model vocabulary
4811        if model_vocab_size > 0 && next_token >= model_vocab_size {
4812            return Err(anyhow::anyhow!(
4813                "Generated out-of-bounds token {}: exceeds model vocab size {}",
4814                next_token,
4815                model_vocab_size
4816            ));
4817        }
4818
4819        // Warn if token is beyond tokenizer vocabulary (but allow it)
4820        if next_token >= vocab_size {
4821            tracing::warn!(
4822                "⚠️ Sampled token {} is beyond tokenizer vocab ({}) but within model vocab ({}). This may indicate a vocab mismatch.",
4823                next_token, vocab_size, model_vocab_size
4824            );
4825        }
4826
4827        // Capture metrics for self-supervised training (EXPENSIVE - only if explicitly enabled)
4828        // Computes softmax + entropy over full vocabulary (~150K) with GPU->CPU syncs
4829        // This adds ~10x overhead per token, so only enable for training
4830        if self.collect_metrics {
4831            // Get last token's logits (already squeezed to 1D in sampler, but handle 2D case)
4832            let last_logits = if logits.dim() > 1 {
4833                let shape = logits.size();
4834                if shape.len() == 3 {
4835                    // [batch, seq_len, vocab] -> take [-1, -1, :] -> [vocab]
4836                    logits.select(1, shape[1] - 1).squeeze_dim(0)
4837                } else if shape.len() == 2 {
4838                    // [seq_len, vocab] or [batch, vocab] -> squeeze
4839                    logits.squeeze_dim(0)
4840                } else {
4841                    logits.shallow_clone()
4842                }
4843            } else {
4844                logits.shallow_clone()
4845            };
4846
4847            // Compute stable softmax: exp(x - max(x)) / sum(exp(x - max(x)))
4848            let max_logit = last_logits.max();
4849            let shifted = &last_logits - &max_logit;
4850            let exp_logits = shifted.exp();
4851            let sum_exp = exp_logits.sum(tch::Kind::Float);
4852            let probs = &exp_logits / &sum_exp;
4853
4854            // Compute log_prob of sampled token (in nats)
4855            // log_prob = log(probs[next_token])
4856            let log_prob = probs.select(0, next_token as i64).log().double_value(&[]) as f32;
4857
4858            // Compute entropy: -sum(p * log(p)) where p > 0
4859            // Use a small epsilon to avoid log(0)
4860            let eps = 1e-10f64;
4861            let safe_probs = probs.clamp(eps, 1.0 - eps);
4862            let log_probs = safe_probs.log();
4863            let entropy = (-(&probs * &log_probs))
4864                .sum(tch::Kind::Float)
4865                .double_value(&[]) as f32;
4866
4867            // Add to accumulator (O(1) per token)
4868            self.metrics_accumulator
4869                .add_token(log_prob, entropy, next_token as u32);
4870        }
4871
4872        Ok(next_token as u32)
4873    }
4874
4875    /// Sample one token for the speculative path with the standard tiered
4876    /// repeat penalty. `extra_window` holds tokens accepted earlier in the
4877    /// current round but not yet pushed to `recent_tokens` by `poll_next`, so
4878    /// the penalty window matches what serial decoding would see at this
4879    /// position. Mirrors the sampling section of `sample_next_token`.
4880    fn sample_spec_position(&mut self, logits: &Tensor, extra_window: &[i64]) -> Result<u32> {
4881        let vocab_size = self.vocab_size;
4882
4883        // PERF: Cache model_vocab_size on first call (from logits shape)
4884        let model_vocab_size = if self.model_vocab_size == 0 {
4885            let logits_shape = logits.size();
4886            let size = logits_shape[logits_shape.len() - 1] as usize;
4887            self.model_vocab_size = size;
4888            size
4889        } else {
4890            self.model_vocab_size
4891        };
4892
4893        if vocab_size == 0 {
4894            return Err(anyhow::anyhow!(
4895                "Cannot sample tokens: tokenizer vocabulary size is 0 (tokenizer not loaded)"
4896            ));
4897        }
4898
4899        let params = self.sampling_params;
4900        // Reuse the PERF buffer: recent_tokens (penalty window) + round-local extras.
4901        self.recent_tokens_buffer.clear();
4902        self.recent_tokens_buffer
4903            .extend(self.recent_tokens.iter().copied());
4904        self.recent_tokens_buffer
4905            .extend(extra_window.iter().copied());
4906        let excess = self
4907            .recent_tokens_buffer
4908            .len()
4909            .saturating_sub(self.repeat_last_n);
4910        self.recent_tokens_buffer.drain(..excess);
4911        let next_token = self.engine.sample_token_with_params(
4912            logits,
4913            &params,
4914            &self.recent_tokens_buffer,
4915            &self.penalty_exempt_tokens,
4916        )?;
4917
4918        if model_vocab_size > 0 && next_token >= model_vocab_size {
4919            return Err(anyhow::anyhow!(
4920                "Generated out-of-bounds token {}: exceeds model vocab size {}",
4921                next_token,
4922                model_vocab_size
4923            ));
4924        }
4925        if next_token >= vocab_size {
4926            tracing::warn!(
4927                "⚠️ Sampled token {} is beyond tokenizer vocab ({}) but within model vocab ({}). This may indicate a vocab mismatch.",
4928                next_token, vocab_size, model_vocab_size
4929            );
4930        }
4931        Ok(next_token as u32)
4932    }
4933
4934    /// Greedy argmax of draft logits `[1, 1, vocab]` (no repeat penalty — the
4935    /// draft only proposes; the penalty-applied verify decides acceptance).
4936    fn spec_argmax_draft(draft_logits: &Tensor) -> u32 {
4937        draft_logits
4938            .reshape([-1i64])
4939            .argmax(-1, false)
4940            .int64_value(&[]) as u32
4941    }
4942
4943    /// One speculative decode step (MTP self-draft, k=1, greedy exact-match
4944    /// verify). Returns the 1–2 tokens accepted this round, in order.
4945    ///
4946    /// Round structure (decode phase; `last_generated` = x_pos is emitted but
4947    /// not yet forwarded, `kv_cache_position` = pos, and `spec_pending_draft`
4948    /// holds the MTP head's candidate for position pos+1):
4949    ///
4950    /// 1. Snapshot the GDN conv/rec state (before the verify forward mutates it).
4951    /// 2. Verify: one main forward over `[x_pos, draft]` at `pos`. Logits at
4952    ///    position 0 check the draft; logits at position 1 yield the bonus
4953    ///    token when the draft matches.
4954    /// 3. Accept (draft == verifier's token): emit both tokens; the verify
4955    ///    forward already consumed exactly the true sequence, so KV and SSM
4956    ///    state are consistent. Draft the next round from the two new slots.
4957    ///    Reject: emit only the verifier's correction token, restore the SSM
4958    ///    snapshot, truncate KV to `pos + 1` (dropping the rejected draft's
4959    ///    KV), then re-forward `x_pos` alone — the 2-token verify advanced the
4960    ///    GDN state through the rejected draft as well, so the restore alone
4961    ///    would leave SSM one accepted token behind the kept KV. Re-draft from
4962    ///    the correction token's slot.
4963    ///
4964    /// Invariants: `last_generated`/`tokens_generated`/`recent_tokens`/
4965    /// `decode_stream` advance only per accepted token (the rejected draft
4966    /// never enters the penalty window or decode stream);
4967    /// `kv_cache_position` advances by exactly the accepted count.
4968    fn sample_speculative_tokens(&mut self) -> Result<Vec<u32>> {
4969        if self.tokens_generated == 0 {
4970            // ---- PREFILL (full, or partial on a session prefix-cache hit) ----
4971            let prefill_start = std::time::Instant::now();
4972            let prompt_len = self.prompt_tokens.len();
4973            let partial = self.prefill_start_pos > 0 && self.prefill_start_pos < prompt_len;
4974            // Owned copy: the forward + sampling below need `&mut self`.
4975            let (slice, start_pos) = if partial {
4976                tracing::info!(
4977                    "Partial prefill: processing {} new tokens (skipped {} cached)",
4978                    prompt_len - self.prefill_start_pos,
4979                    self.prefill_start_pos
4980                );
4981                (
4982                    self.prompt_tokens[self.prefill_start_pos..].to_vec(),
4983                    self.prefill_start_pos,
4984                )
4985            } else {
4986                (self.prompt_tokens.clone(), 0)
4987            };
4988
4989            let (logits, hidden) = self.engine.forward_speculative(&slice, start_pos, true)?;
4990
4991            // Retain the prompt's recurrent state before verification advances
4992            // it beyond the emitted-token queue. Session Drop persists this
4993            // deep snapshot alongside the prompt IDs, just like serial prefill.
4994            self.prefill_ssm_snapshot = self.engine.snapshot_ssm_states();
4995
4996            let prefill_elapsed = prefill_start.elapsed();
4997            self.prefill_time_ms = Some(prefill_elapsed.as_millis() as u64);
4998            self.first_token_time = Some(std::time::Instant::now());
4999            tracing::info!(
5000                "PREFILL: {} tokens in {:?} ({:.2} tok/sec){} [speculative]",
5001                slice.len(),
5002                prefill_elapsed,
5003                if prefill_elapsed.as_secs_f32() > 0.0 {
5004                    slice.len() as f32 / prefill_elapsed.as_secs_f32()
5005                } else {
5006                    0.0
5007                },
5008                if partial {
5009                    format!(" [prefix cached: {} tokens]", self.prefill_start_pos)
5010                } else {
5011                    String::new()
5012                },
5013            );
5014
5015            let next = self.sample_spec_position(&logits, &[])?;
5016
5017            // MTP prompt prefill + first draft in one call: slot t pairs
5018            // (hidden after x_t, embed(x_{t+1})); the final slot pairs the
5019            // last hidden with the just-sampled token, yielding the draft for
5020            // the following position. On a prefix-cache hit the pre-hit slots
5021            // keep the swapped-in cache's contents (stale or zero entries only
5022            // degrade acceptance, never correctness — the verify is decisive).
5023            let mut mtp_tokens: Vec<i64> = slice[1..].to_vec();
5024            mtp_tokens.push(next as i64);
5025            let draft_logits = self
5026                .engine
5027                .forward_mtp_draft(&mtp_tokens, &hidden, start_pos)?;
5028            self.spec_pending_draft = Some(Self::spec_argmax_draft(&draft_logits));
5029
5030            Ok(vec![next])
5031        } else {
5032            // ---- VERIFY ROUND ----
5033            let pos = self.kv_cache_position;
5034
5035            // Context boundary: the 2-token verify forward writes KV at `pos`
5036            // and `pos + 1`, and LayerKVCache::update hard-errors past
5037            // max_seq_len. When fewer than two slots remain, the serial path
5038            // can still forward `last` in the final slot — fall back to a
5039            // single-token step for THIS round (not a permanent latch; the
5040            // next round re-checks and ends exactly as the serial path would).
5041            if let Some(cap) = self.spec_kv_capacity() {
5042                if pos + 2 > cap {
5043                    let token = self.sample_next_token()?;
5044                    // poll_next's per-token increment is skipped on the
5045                    // speculative path, so advance here like a serial step.
5046                    self.kv_cache_position += 1;
5047                    return Ok(vec![token]);
5048                }
5049            }
5050
5051            let last = self.last_generated.ok_or_else(|| {
5052                anyhow::anyhow!(
5053                    "Internal error: last_generated not set after {} tokens",
5054                    self.tokens_generated
5055                )
5056            })?;
5057            let draft = self.spec_pending_draft.take().ok_or_else(|| {
5058                anyhow::anyhow!("Internal error: speculative round without a pending draft")
5059            })?;
5060
5061            let snapshot = self.engine.snapshot_ssm_states();
5062
5063            let (logits, hidden) =
5064                self.engine
5065                    .forward_speculative(&[last, draft as i64], pos, false)?;
5066            let v1 = self.sample_spec_position(&logits.select(1, 0), &[])?;
5067
5068            if v1 == draft {
5069                // ACCEPT: the draft was the verifier's own greedy choice, so the
5070                // verify forward consumed exactly the true sequence (KV + SSM
5071                // consistent). Emit the verified draft plus the bonus token.
5072                let v2 = self.sample_spec_position(&logits.select(1, 1), &[v1 as i64])?;
5073                let next_draft_logits =
5074                    self.engine
5075                        .forward_mtp_draft(&[v1 as i64, v2 as i64], &hidden, pos)?;
5076                self.spec_pending_draft = Some(Self::spec_argmax_draft(&next_draft_logits));
5077                self.kv_cache_position += 2;
5078                self.spec_accepted += 1;
5079                Ok(vec![v1, v2])
5080            } else {
5081                // REJECT: emit only the verifier's correction token and rewind.
5082                self.engine.restore_ssm_states(snapshot);
5083                // Drop the rejected draft's KV; the +1 keeps the KV the verify
5084                // forward wrote for `last`. The MTP slot lags one position
5085                // behind the main cache (it never consumed the rejected draft),
5086                // so this manager-wide truncate is a no-op for it.
5087                if let Some(model_arc) = &self.engine.persistent_model {
5088                    let model = model_arc.lock();
5089                    if let Some(cache) = model.get_kv_cache() {
5090                        cache.lock().truncate_to(pos + 1);
5091                    }
5092                }
5093                // Re-forward `last` alone to re-sync the GDN conv/rec state
5094                // with the kept KV (see fn docs); rewrites identical KV at pos.
5095                let (_logits, hidden2) = self.engine.forward_speculative(&[last], pos, true)?;
5096                let next_draft_logits =
5097                    self.engine.forward_mtp_draft(&[v1 as i64], &hidden2, pos)?;
5098                self.spec_pending_draft = Some(Self::spec_argmax_draft(&next_draft_logits));
5099                self.kv_cache_position += 1;
5100                self.spec_rejected += 1;
5101                Ok(vec![v1])
5102            }
5103        }
5104    }
5105
5106    /// Next token from the speculative path: drains the current round's
5107    /// accepted-token queue, running a new speculative round when empty.
5108    /// `kv_cache_position` is advanced inside `sample_speculative_tokens`
5109    /// (per accepted token), so unlike the serial path no increment happens here.
5110    fn next_speculative_token(&mut self) -> Result<u32> {
5111        if self.spec_out_queue.is_empty() {
5112            let tokens = self.sample_speculative_tokens()?;
5113            self.spec_out_queue.extend(tokens);
5114        }
5115        self.spec_out_queue
5116            .pop_front()
5117            .ok_or_else(|| anyhow::anyhow!("Speculative sampler produced no tokens"))
5118    }
5119
5120    /// KV capacity (max sequence length) of the model's cache manager — the
5121    /// hard bound `LayerKVCache::update` enforces. `None` when no model/cache
5122    /// is installed (caller then skips the boundary check).
5123    fn spec_kv_capacity(&self) -> Option<usize> {
5124        let model_arc = self.engine.persistent_model.as_ref()?;
5125        let model = model_arc.lock();
5126        let cache = model.get_kv_cache()?;
5127        let cap = cache.lock().max_seq_len();
5128        Some(cap)
5129    }
5130}
5131
5132// SAFETY: TextStream can be Send because:
5133// - tokenizer pointer points to heap data that won't move
5134// - decode_stream is tied to the tokenizer lifetime
5135// - Tokenizer itself is Send
5136unsafe impl<'a> Send for TextStream<'a> {}
5137
5138// No custom Drop needed - Arc handles automatic cleanup
5139
5140impl<'a> Stream for TextStream<'a> {
5141    type Item = Result<String>;
5142
5143    fn poll_next(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {
5144        loop {
5145            if self.finished {
5146                return Poll::Ready(None);
5147            }
5148
5149            // Check timeout
5150            if let Some(timeout_ms) = self.timeout_ms {
5151                let elapsed = self.start_time.elapsed();
5152                if elapsed.as_millis() >= timeout_ms as u128 {
5153                    tracing::debug!("Generation timed out after {}ms", timeout_ms);
5154                    self.finished = true;
5155                    self.finish_reason = Some(FinishReason::Error(format!(
5156                        "Generation timed out after {timeout_ms}ms"
5157                    )));
5158                    return Poll::Ready(Some(Err(anyhow::anyhow!(
5159                        "Generation timed out after {}ms",
5160                        timeout_ms
5161                    ))));
5162                }
5163            }
5164
5165            // Check max tokens
5166            if self.max_tokens > 0 && self.tokens_generated >= self.max_tokens {
5167                tracing::debug!("Reached max tokens: {}", self.max_tokens);
5168                self.finished = true;
5169                self.finish_reason = Some(FinishReason::MaxTokens);
5170                return Poll::Ready(None);
5171            }
5172
5173            // Sample next token
5174            // The `!self.collect_metrics` guard keeps metrics-collecting
5175            // (training-quality) requests on the serial path: the speculative
5176            // sampler does not replicate the per-token logprob/entropy
5177            // accumulator block from `sample_next_token`.
5178            let next_token = if self.speculative && !self.collect_metrics {
5179                match self.next_speculative_token() {
5180                    Ok(token) => token,
5181                    Err(e) => {
5182                        self.finished = true;
5183                        self.finish_reason = Some(FinishReason::Error(e.to_string()));
5184                        return Poll::Ready(Some(Err(e)));
5185                    }
5186                }
5187            } else {
5188                match self.sample_next_token() {
5189                    Ok(token) => {
5190                        // FIX: Increment KV cache position after successful token sampling
5191                        // This ensures KV cache stays synchronized with generation state
5192                        if self.tokens_generated > 0 {
5193                            // Don't increment on initial prompt
5194                            self.kv_cache_position += 1;
5195                        }
5196                        token
5197                    }
5198                    Err(e) => {
5199                        self.finished = true;
5200                        self.finish_reason = Some(FinishReason::Error(e.to_string()));
5201                        return Poll::Ready(Some(Err(e)));
5202                    }
5203                }
5204            };
5205
5206            // Check EOS
5207            if self.engine.is_eos_token(next_token as usize) {
5208                tracing::debug!(
5209                    "EOS token detected at generated position {}",
5210                    self.tokens_generated
5211                );
5212                self.finished = true;
5213                self.finish_reason = Some(FinishReason::EndOfSequence);
5214                return Poll::Ready(None);
5215            }
5216
5217            // Check stop tokens
5218            if self.stop_token_ids.contains(&next_token) {
5219                // Never log the matched stop-token ID: it identifies a
5220                // caller-supplied stop string (#1253/#1261). Position only.
5221                tracing::debug!(
5222                    "Stop token matched at generated position {}",
5223                    self.tokens_generated
5224                );
5225                self.finished = true;
5226                self.finish_reason = Some(FinishReason::StopToken(format!("{next_token}")));
5227                return Poll::Ready(None);
5228            }
5229
5230            // KV cache position is already tracked correctly in sample_next_token
5231            // No need to increment here
5232
5233            // Process decode_stream FIRST, then update state
5234            // This prevents state corruption if decode_stream fails
5235            match self.decode_stream.step(next_token) {
5236                Ok(Some(text)) => {
5237                    // DecodeStream succeeded - now update state
5238                    let token_i64 = next_token as i64;
5239                    self.last_generated = Some(token_i64);
5240                    self.tokens_generated += 1;
5241                    self.update_ema_rate();
5242
5243                    self.recent_tokens.push_back(token_i64);
5244                    if self.recent_tokens.len() > self.repeat_last_n {
5245                        self.recent_tokens.pop_front();
5246                    }
5247
5248                    // DecodeStream returned text - emit it. The chunk text is
5249                    // generated content and is never logged (#1253); generated-
5250                    // token counts are now emitted as OTel metrics at completion
5251                    // (#1261), so the per-chunk count log is dropped.
5252                    return Poll::Ready(Some(Ok(text)));
5253                }
5254                Ok(None) => {
5255                    // DecodeStream is buffering incomplete UTF-8 - update state and continue
5256                    let token_i64 = next_token as i64;
5257                    self.last_generated = Some(token_i64);
5258                    self.tokens_generated += 1;
5259                    self.update_ema_rate();
5260
5261                    self.recent_tokens.push_back(token_i64);
5262                    if self.recent_tokens.len() > self.repeat_last_n {
5263                        self.recent_tokens.pop_front();
5264                    }
5265
5266                    // Simple trace without expensive tokenizer lock or decode.
5267                    // Counts/positions only — token IDs are reversible to text
5268                    // and must never hit process logs (#1253/#1261).
5269                    tracing::trace!(
5270                        "Token at generated position {} buffering (incomplete UTF-8)",
5271                        self.tokens_generated
5272                    );
5273                    continue;
5274                }
5275                Err(e) => {
5276                    // CRITICAL: Do NOT update state on decode_stream error
5277                    // This prevents state corruption and keeps generation consistent
5278                    tracing::error!(
5279                        "DecodeStream error at generated position {}: {}",
5280                        self.tokens_generated,
5281                        e
5282                    );
5283                    self.finished = true;
5284                    self.finish_reason = Some(FinishReason::Error(e.to_string()));
5285                    return Poll::Ready(Some(Err(anyhow::anyhow!("Decode error: {}", e))));
5286                }
5287            }
5288        }
5289    }
5290}
5291
5292impl<'a> Drop for TextStream<'a> {
5293    fn drop(&mut self) {
5294        // Save token IDs to the session cache for prefix matching on the next turn.
5295        // This runs on all exit paths (EOS, stop token, max tokens, timeout, error).
5296        self.save_cached_tokens();
5297
5298        // Token-burn metering (#1261): emit the generated-token count and the
5299        // per-request total at completion. Drop runs on every exit path, so this
5300        // covers normal EOS, stop-token, max-tokens, timeout, and error finishes.
5301        // Only integer counts and the opaque tenant hash are metric attributes (#1253).
5302        let generated = self.tokens_generated as u64;
5303        let meter = crate::runtime::token_metrics::global_meter();
5304        meter.record_generated(&self.model_label, self.tenant_hash, generated);
5305        meter.record_request_total(
5306            &self.model_label,
5307            self.tenant_hash,
5308            self.prompt_len as u64 + generated,
5309        );
5310        // MTP self-speculative draft outcomes (integer counts only, #1253).
5311        meter.record_speculative(
5312            &self.model_label,
5313            self.tenant_hash,
5314            self.spec_accepted,
5315            self.spec_rejected,
5316        );
5317    }
5318}