Skip to main content

hyprstream_inference/runtime/
mod.rs

1//! Runtime abstraction layer for inference engines
2//!
3//! This module provides a unified interface for different inference engines:
4//! - TorchEngine: Primary PyTorch-based engine with tch-rs
5//! - CandleEngine: Legacy Candle engine (DEPRECATED)
6//! - LlamaCppEngine: Legacy engine for reference (DEPRECATED)
7
8use anyhow::Result;
9use async_trait::async_trait;
10use hyprstream_rpc_std::inference_client::{GenerationRequest, ModelInfo};
11use std::path::Path;
12
13// Preserve the public runtime API while sharing the canonical RPC enum.
14pub use hyprstream_rpc_std::model_client::KVQuantType;
15
16// Re-export everything from the unified config
17pub use crate::config::{
18    FinishReason, GenerationConfig, GenerationResult, ModelConfig, RuntimeConfig,
19};
20
21pub(crate) mod channel_fp8;
22mod resolved_spool;
23pub use channel_fp8::Fp8ExecutionError;
24pub mod architectures; // Architecture-specific model implementations (includes Janus placeholder utils)
25pub mod batched_lora;
26pub mod ttn_profile; // TTN analysis pipeline: adaptive layer profiling + embedded profiles // Batched multi-tenant LoRA forward pass
27                     // REMOVED: pub mod conversation_router; // Dead code - VDB TemporalStreamingLayer removed
28pub mod device_pool; // Multi-GPU device abstraction (DevicePool) — Send+Sync, holds only Device values
29pub mod generation_metrics; // Quality metrics for self-supervised training
30pub mod image_utils; // Image loading and preprocessing for multimodal models
31pub mod inference_profile; // Explicit inference isolation/deployment scheduling profile
32pub mod inference_stage; // Thread-local execution boundary for one decoder stage
33pub mod kv_cache; // Key-Value caching for efficient autoregressive generation
34pub mod kv_compat; // KV-cache compatibility fingerprint + rejection policy (#1277)
35pub mod model_config; // Unified model configuration management
36pub mod model_factory; // Single factory for model creation
37pub mod rope; // Rotary Position Embedding (RoPE) implementation
38pub mod template_engine; // Jinja2 template engine for chat templates
39pub mod tensor_helpers; // Helper functions for Tch tensor operations
40pub mod tensor_sampling; // Device-agnostic tensor-based sampling
41pub mod token_metrics; // OpenTelemetry token-burn metering (prompt + generated tokens per request)
42pub mod tokenizer_config; // Trait-based tokenizer configuration for models
43pub mod torch_engine; // PyTorch-based engine with tch-rs
44pub mod torch_utils; // Utilities for safe PyTorch operations with OOM handling
45pub mod weight_provider; // Weight provider for streaming large models
46
47// Primary exports - use TorchEngine as default
48pub use inference_stage::{
49    InferenceStage, StageContract, StageInput, StageOutput, StageSequence, StageTensor,
50};
51pub use torch_engine::{GenerationStats, TextStream, TorchEngine};
52
53// Multi-GPU device abstraction
54pub use device_pool::DevicePool;
55
56// KV cache exports for multi-session support
57pub use kv_cache::{CacheConfig, CacheOwner, KVCacheManager, KVCacheRegistry};
58
59// KV-cache compatibility fingerprint + rejection policy (#1277)
60pub use kv_compat::{
61    check_compatibility, KvCompatDescriptor, KvCompatFingerprint, KvCompatMismatch, KvDtype,
62    KvQuantMode, WeightIdentity, KV_COMPAT_FORMAT_VERSION,
63};
64
65// Generation metrics exports for self-supervised training
66pub use generation_metrics::{
67    GenerationMetricsAccumulator, GenerationQualityMetrics, SessionMetrics,
68};
69
70#[derive(Debug, Clone)]
71pub struct MistralEngine;
72
73#[derive(Debug, Clone)]
74pub enum ModelBuilderConfig {
75    Default,
76}
77
78// REMOVED: Conversation routing exports - dead code
79// pub use conversation_router::{...};
80
81// LoRA and adapter exports
82// LoRA wrapper removed - using direct PyTorch implementation
83
84/// Core runtime engine trait - all engines implement this
85#[async_trait]
86pub trait RuntimeEngine: Send + Sync {
87    /// Load a model from the given path
88    async fn load_model(&mut self, path: &Path) -> Result<()>;
89
90    /// Generate text from a prompt (convenience method)
91    async fn generate(&self, prompt: &str, max_tokens: usize) -> Result<String>;
92
93    /// Generate text with full parameters (main method)
94    async fn generate_with_params(&self, request: GenerationRequest) -> Result<GenerationResult>;
95
96    /// Get loaded model information
97    fn model_info(&self) -> ModelInfo;
98
99    /// Check if model is loaded and ready
100    fn is_loaded(&self) -> bool;
101
102    /// Apply chat template to messages (for template support).
103    ///
104    /// `tools` is an optional JSON array of tool definitions. When provided, it is
105    /// passed to the HF chat template as the `tools` variable so that models with
106    /// native tool-calling support can format them correctly.
107    fn apply_chat_template(
108        &self,
109        messages: &[template_engine::ChatMessage],
110        add_generation_prompt: bool,
111        tools: Option<&serde_json::Value>,
112    ) -> Result<String> {
113        let _ = tools; // default impl ignores tools
114                       // Default implementation with simple formatting
115        let mut formatted = String::new();
116        for msg in messages {
117            formatted.push_str(&format!(
118                "{}: {}\n",
119                msg.role,
120                msg.content.as_deref().unwrap_or("")
121            ));
122        }
123        if add_generation_prompt {
124            formatted.push_str("assistant: ");
125        }
126        Ok(formatted)
127    }
128
129    /// Apply chat template with extended parameters (thinking mode, extra vars).
130    fn apply_chat_template_with_vars(
131        &self,
132        messages: &[template_engine::ChatMessage],
133        add_generation_prompt: bool,
134        tools: Option<&serde_json::Value>,
135        _enable_thinking: Option<bool>,
136        _template_vars_json: Option<&str>,
137    ) -> Result<String> {
138        // Default: delegate to basic apply_chat_template (ignores extended params)
139        self.apply_chat_template(messages, add_generation_prompt, tools)
140    }
141}
142
143/// Create the default runtime engine (now uses PyTorch)
144pub fn create_engine(config: &RuntimeConfig) -> Result<TorchEngine> {
145    TorchEngine::new(config.clone())
146}
147
148/* Commented out - VDB TemporalStreamingLayer removed
149/// Create conversation router with model pool and temporal streaming
150pub async fn create_conversation_router(
151    model_pool: std::sync::Arc<ModelPool>,
152    temporal_streaming: std::sync::Arc<crate::storage::vdb::TemporalStreamingLayer>,
153    config: Option<RoutingConfig>,
154) -> Result<ConversationRouter> {
155    ConversationRouter::new(
156        model_pool,
157        temporal_streaming,
158        config.unwrap_or_default(),
159    ).await
160}
161*/