hyprstream_inference/runtime/mod.rs
1//! Runtime abstraction layer for inference engines
2//!
3//! This module provides a unified interface for different inference engines:
4//! - TorchEngine: Primary PyTorch-based engine with tch-rs
5//! - CandleEngine: Legacy Candle engine (DEPRECATED)
6//! - LlamaCppEngine: Legacy engine for reference (DEPRECATED)
7
8use anyhow::Result;
9use async_trait::async_trait;
10use hyprstream_rpc_std::inference_client::{GenerationRequest, ModelInfo};
11use std::path::Path;
12
13// Preserve the public runtime API while sharing the canonical RPC enum.
14pub use hyprstream_rpc_std::model_client::KVQuantType;
15
16// Re-export everything from the unified config
17pub use crate::config::{
18 FinishReason, GenerationConfig, GenerationResult, ModelConfig, RuntimeConfig,
19};
20
21pub(crate) mod channel_fp8;
22mod resolved_spool;
23pub use channel_fp8::Fp8ExecutionError;
24pub mod architectures; // Architecture-specific model implementations (includes Janus placeholder utils)
25pub mod batched_lora;
26pub mod ttn_profile; // TTN analysis pipeline: adaptive layer profiling + embedded profiles // Batched multi-tenant LoRA forward pass
27 // REMOVED: pub mod conversation_router; // Dead code - VDB TemporalStreamingLayer removed
28pub mod device_pool; // Multi-GPU device abstraction (DevicePool) — Send+Sync, holds only Device values
29pub mod generation_metrics; // Quality metrics for self-supervised training
30pub mod image_utils; // Image loading and preprocessing for multimodal models
31pub mod inference_profile; // Explicit inference isolation/deployment scheduling profile
32pub mod inference_stage; // Thread-local execution boundary for one decoder stage
33pub mod kv_cache; // Key-Value caching for efficient autoregressive generation
34pub mod kv_compat; // KV-cache compatibility fingerprint + rejection policy (#1277)
35pub mod model_config; // Unified model configuration management
36pub mod model_factory; // Single factory for model creation
37pub mod rope; // Rotary Position Embedding (RoPE) implementation
38pub mod template_engine; // Jinja2 template engine for chat templates
39pub mod tensor_helpers; // Helper functions for Tch tensor operations
40pub mod tensor_sampling; // Device-agnostic tensor-based sampling
41pub mod token_metrics; // OpenTelemetry token-burn metering (prompt + generated tokens per request)
42pub mod tokenizer_config; // Trait-based tokenizer configuration for models
43pub mod torch_engine; // PyTorch-based engine with tch-rs
44pub mod torch_utils; // Utilities for safe PyTorch operations with OOM handling
45pub mod weight_provider; // Weight provider for streaming large models
46
47// Primary exports - use TorchEngine as default
48pub use inference_stage::{
49 InferenceStage, StageContract, StageInput, StageOutput, StageSequence, StageTensor,
50};
51pub use torch_engine::{GenerationStats, TextStream, TorchEngine};
52
53// Multi-GPU device abstraction
54pub use device_pool::DevicePool;
55
56// KV cache exports for multi-session support
57pub use kv_cache::{CacheConfig, CacheOwner, KVCacheManager, KVCacheRegistry};
58
59// KV-cache compatibility fingerprint + rejection policy (#1277)
60pub use kv_compat::{
61 check_compatibility, KvCompatDescriptor, KvCompatFingerprint, KvCompatMismatch, KvDtype,
62 KvQuantMode, WeightIdentity, KV_COMPAT_FORMAT_VERSION,
63};
64
65// Generation metrics exports for self-supervised training
66pub use generation_metrics::{
67 GenerationMetricsAccumulator, GenerationQualityMetrics, SessionMetrics,
68};
69
70#[derive(Debug, Clone)]
71pub struct MistralEngine;
72
73#[derive(Debug, Clone)]
74pub enum ModelBuilderConfig {
75 Default,
76}
77
78// REMOVED: Conversation routing exports - dead code
79// pub use conversation_router::{...};
80
81// LoRA and adapter exports
82// LoRA wrapper removed - using direct PyTorch implementation
83
84/// Core runtime engine trait - all engines implement this
85#[async_trait]
86pub trait RuntimeEngine: Send + Sync {
87 /// Load a model from the given path
88 async fn load_model(&mut self, path: &Path) -> Result<()>;
89
90 /// Generate text from a prompt (convenience method)
91 async fn generate(&self, prompt: &str, max_tokens: usize) -> Result<String>;
92
93 /// Generate text with full parameters (main method)
94 async fn generate_with_params(&self, request: GenerationRequest) -> Result<GenerationResult>;
95
96 /// Get loaded model information
97 fn model_info(&self) -> ModelInfo;
98
99 /// Check if model is loaded and ready
100 fn is_loaded(&self) -> bool;
101
102 /// Apply chat template to messages (for template support).
103 ///
104 /// `tools` is an optional JSON array of tool definitions. When provided, it is
105 /// passed to the HF chat template as the `tools` variable so that models with
106 /// native tool-calling support can format them correctly.
107 fn apply_chat_template(
108 &self,
109 messages: &[template_engine::ChatMessage],
110 add_generation_prompt: bool,
111 tools: Option<&serde_json::Value>,
112 ) -> Result<String> {
113 let _ = tools; // default impl ignores tools
114 // Default implementation with simple formatting
115 let mut formatted = String::new();
116 for msg in messages {
117 formatted.push_str(&format!(
118 "{}: {}\n",
119 msg.role,
120 msg.content.as_deref().unwrap_or("")
121 ));
122 }
123 if add_generation_prompt {
124 formatted.push_str("assistant: ");
125 }
126 Ok(formatted)
127 }
128
129 /// Apply chat template with extended parameters (thinking mode, extra vars).
130 fn apply_chat_template_with_vars(
131 &self,
132 messages: &[template_engine::ChatMessage],
133 add_generation_prompt: bool,
134 tools: Option<&serde_json::Value>,
135 _enable_thinking: Option<bool>,
136 _template_vars_json: Option<&str>,
137 ) -> Result<String> {
138 // Default: delegate to basic apply_chat_template (ignores extended params)
139 self.apply_chat_template(messages, add_generation_prompt, tools)
140 }
141}
142
143/// Create the default runtime engine (now uses PyTorch)
144pub fn create_engine(config: &RuntimeConfig) -> Result<TorchEngine> {
145 TorchEngine::new(config.clone())
146}
147
148/* Commented out - VDB TemporalStreamingLayer removed
149/// Create conversation router with model pool and temporal streaming
150pub async fn create_conversation_router(
151 model_pool: std::sync::Arc<ModelPool>,
152 temporal_streaming: std::sync::Arc<crate::storage::vdb::TemporalStreamingLayer>,
153 config: Option<RoutingConfig>,
154) -> Result<ConversationRouter> {
155 ConversationRouter::new(
156 model_pool,
157 temporal_streaming,
158 config.unwrap_or_default(),
159 ).await
160}
161*/