hyprstream_core/inference/
client.rs1use crate::config::GenerationResult;
7use crate::training::TenantDeltaConfig;
8use async_trait::async_trait;
9use hyprstream_rpc_std::inference_client::GenerationRequest;
10use hyprstream_rpc_std::inference_client::ModelInfo;
11use std::path::Path;
12use thiserror::Error;
13
14use super::response::StreamHandle;
15
16#[async_trait]
21pub trait InferenceClient: Send + Sync {
22 async fn generate(
26 &self,
27 request: GenerationRequest,
28 ) -> Result<GenerationResult, InferenceError>;
29
30 async fn generate_stream(
32 &self,
33 request: GenerationRequest,
34 ) -> Result<StreamHandle, InferenceError>;
35
36 async fn model_info(&self) -> Result<ModelInfo, InferenceError>;
40
41 async fn is_ready(&self) -> Result<bool, InferenceError>;
43
44 async fn apply_chat_template(
46 &self,
47 messages: &[crate::runtime::template_engine::ChatMessage],
48 add_generation_prompt: bool,
49 tools: Option<serde_json::Value>,
50 ) -> Result<String, InferenceError>;
51
52 async fn create_lora(&self, config: TenantDeltaConfig) -> Result<(), InferenceError>;
56
57 async fn load_lora(&self, path: &Path) -> Result<(), InferenceError>;
59
60 async fn save_lora(&self, path: &Path) -> Result<(), InferenceError>;
62
63 async fn unload_lora(&self) -> Result<(), InferenceError>;
65
66 async fn has_lora(&self) -> Result<bool, InferenceError>;
68
69 async fn set_session(&self, session_id: String) -> Result<(), InferenceError>;
73
74 async fn clear_session(&self) -> Result<(), InferenceError>;
76
77 async fn release_session(&self, session_id: &str) -> Result<(), InferenceError>;
79
80 async fn embed(&self, images: &[Vec<u8>]) -> Result<Vec<Vec<f32>>, InferenceError>;
88
89 async fn health_check(&self) -> Result<(), InferenceError>;
93
94 async fn shutdown(&self) -> Result<(), InferenceError>;
96}
97
98#[derive(Debug, Error, PartialEq)]
100pub enum InferenceError {
101 #[error("Model loading failed: {0}")]
103 ModelLoad(String),
104
105 #[error("Generation failed: {0}")]
107 Generation(String),
108
109 #[error("LoRA operation failed: {0}")]
111 LoRA(String),
112
113 #[error("Session operation failed: {0}")]
115 Session(String),
116
117 #[error("Service communication failed: {0}")]
119 Channel(String),
120
121 #[error("Service unavailable")]
123 Unavailable,
124
125 #[error("Configuration error: {0}")]
127 Config(String),
128
129 #[error("Internal error: {0}")]
131 Internal(String),
132}
133
134impl InferenceError {
135 pub fn channel<S: Into<String>>(msg: S) -> Self {
137 Self::Channel(msg.into())
138 }
139
140 pub fn internal<S: Into<String>>(msg: S) -> Self {
142 Self::Internal(msg.into())
143 }
144}
145
146impl From<anyhow::Error> for InferenceError {
147 fn from(e: anyhow::Error) -> Self {
148 Self::Internal(e.to_string())
149 }
150}