Skip to main content

hyprstream_core/inference/
client.rs

1//! Inference client trait and error types.
2//!
3//! This module defines the transport-agnostic interface for inference services.
4//! Implementations can be in-process (channels) or remote (gRPC, etc.).
5
6use crate::config::GenerationResult;
7use crate::training::TenantDeltaConfig;
8use async_trait::async_trait;
9use hyprstream_rpc_std::inference_client::GenerationRequest;
10use hyprstream_rpc_std::inference_client::ModelInfo;
11use std::path::Path;
12use thiserror::Error;
13
14use super::response::StreamHandle;
15
16/// Transport-agnostic inference client trait.
17///
18/// Implementations can be in-process (channels) or remote (gRPC, etc.).
19/// All methods return owned data to avoid lifetime issues across transports.
20#[async_trait]
21pub trait InferenceClient: Send + Sync {
22    // === Generation ===
23
24    /// Generate text (non-streaming, blocking until complete).
25    async fn generate(
26        &self,
27        request: GenerationRequest,
28    ) -> Result<GenerationResult, InferenceError>;
29
30    /// Start streaming generation, returns handle to receive text chunks.
31    async fn generate_stream(
32        &self,
33        request: GenerationRequest,
34    ) -> Result<StreamHandle, InferenceError>;
35
36    // === Model Info ===
37
38    /// Get information about the loaded model.
39    async fn model_info(&self) -> Result<ModelInfo, InferenceError>;
40
41    /// Check if model is loaded and ready.
42    async fn is_ready(&self) -> Result<bool, InferenceError>;
43
44    /// Apply chat template to format messages for generation.
45    async fn apply_chat_template(
46        &self,
47        messages: &[crate::runtime::template_engine::ChatMessage],
48        add_generation_prompt: bool,
49        tools: Option<serde_json::Value>,
50    ) -> Result<String, InferenceError>;
51
52    // === LoRA Operations ===
53
54    /// Initialize LoRA adapter structure for training.
55    async fn create_lora(&self, config: TenantDeltaConfig) -> Result<(), InferenceError>;
56
57    /// Load LoRA adapter weights from file.
58    async fn load_lora(&self, path: &Path) -> Result<(), InferenceError>;
59
60    /// Save LoRA adapter weights to file.
61    async fn save_lora(&self, path: &Path) -> Result<(), InferenceError>;
62
63    /// Unload current LoRA adapter.
64    async fn unload_lora(&self) -> Result<(), InferenceError>;
65
66    /// Check if LoRA is loaded.
67    async fn has_lora(&self) -> Result<bool, InferenceError>;
68
69    // === Session Management (KV Cache) ===
70
71    /// Set active session for KV cache isolation.
72    async fn set_session(&self, session_id: String) -> Result<(), InferenceError>;
73
74    /// Clear current session's KV cache.
75    async fn clear_session(&self) -> Result<(), InferenceError>;
76
77    /// Release a session's KV cache.
78    async fn release_session(&self, session_id: &str) -> Result<(), InferenceError>;
79
80    // === Embeddings ===
81
82    /// Compute embeddings for one or more images.
83    ///
84    /// Returns one embedding vector per input image. The image bytes can be
85    /// raw RGB pixels, PNG, or JPEG encoded data — the model's preprocessor
86    /// handles decoding.
87    async fn embed(&self, images: &[Vec<u8>]) -> Result<Vec<Vec<f32>>, InferenceError>;
88
89    // === Health ===
90
91    /// Check service health.
92    async fn health_check(&self) -> Result<(), InferenceError>;
93
94    /// Graceful shutdown signal.
95    async fn shutdown(&self) -> Result<(), InferenceError>;
96}
97
98/// Inference service error type.
99#[derive(Debug, Error, PartialEq)]
100pub enum InferenceError {
101    /// Model loading failed.
102    #[error("Model loading failed: {0}")]
103    ModelLoad(String),
104
105    /// Generation failed.
106    #[error("Generation failed: {0}")]
107    Generation(String),
108
109    /// LoRA operation failed.
110    #[error("LoRA operation failed: {0}")]
111    LoRA(String),
112
113    /// Session management failed.
114    #[error("Session operation failed: {0}")]
115    Session(String),
116
117    /// Service communication failed (channel closed, send failed, etc.).
118    #[error("Service communication failed: {0}")]
119    Channel(String),
120
121    /// Service is unavailable (not started, shutdown, etc.).
122    #[error("Service unavailable")]
123    Unavailable,
124
125    /// Configuration error.
126    #[error("Configuration error: {0}")]
127    Config(String),
128
129    /// Internal error.
130    #[error("Internal error: {0}")]
131    Internal(String),
132}
133
134impl InferenceError {
135    /// Create a channel error.
136    pub fn channel<S: Into<String>>(msg: S) -> Self {
137        Self::Channel(msg.into())
138    }
139
140    /// Create an internal error.
141    pub fn internal<S: Into<String>>(msg: S) -> Self {
142        Self::Internal(msg.into())
143    }
144}
145
146impl From<anyhow::Error> for InferenceError {
147    fn from(e: anyhow::Error) -> Self {
148        Self::Internal(e.to_string())
149    }
150}