hyprstream_core/inference/
request.rs1use crate::config::GenerationResult;
7use crate::training::TenantDeltaConfig;
8use hyprstream_rpc_std::inference_client::GenerationRequest;
9use hyprstream_rpc_std::inference_client::ModelInfo;
10use std::path::PathBuf;
11use tokio::sync::{mpsc, oneshot};
12
13use super::client::InferenceError;
14use super::response::StreamStats;
15
16pub enum InferenceRequest {
21 Generate {
24 request: GenerationRequest,
25 reply: oneshot::Sender<Result<GenerationResult, InferenceError>>,
26 },
27
28 GenerateStream {
33 request: GenerationRequest,
34 chunk_sender: mpsc::Sender<Result<String, InferenceError>>,
36 stats_sender: oneshot::Sender<StreamStats>,
38 reply: oneshot::Sender<Result<(), InferenceError>>,
40 },
41
42 ModelInfo {
45 reply: oneshot::Sender<Result<ModelInfo, InferenceError>>,
46 },
47
48 IsReady {
50 reply: oneshot::Sender<Result<bool, InferenceError>>,
51 },
52
53 ApplyChatTemplate {
55 messages: Vec<crate::runtime::template_engine::ChatMessage>,
56 add_generation_prompt: bool,
57 tools: Option<serde_json::Value>,
58 reply: oneshot::Sender<Result<String, InferenceError>>,
59 },
60
61 CreateLora {
64 config: TenantDeltaConfig,
65 reply: oneshot::Sender<Result<(), InferenceError>>,
66 },
67
68 LoadLora {
70 path: PathBuf,
71 reply: oneshot::Sender<Result<(), InferenceError>>,
72 },
73
74 SaveLora {
76 path: PathBuf,
77 reply: oneshot::Sender<Result<(), InferenceError>>,
78 },
79
80 UnloadLora {
82 reply: oneshot::Sender<Result<(), InferenceError>>,
83 },
84
85 HasLora {
87 reply: oneshot::Sender<Result<bool, InferenceError>>,
88 },
89
90 SetSession {
93 session_id: String,
94 reply: oneshot::Sender<Result<(), InferenceError>>,
95 },
96
97 ClearSession {
99 reply: oneshot::Sender<Result<(), InferenceError>>,
100 },
101
102 ReleaseSession {
104 session_id: String,
105 reply: oneshot::Sender<Result<(), InferenceError>>,
106 },
107
108 Embed {
111 images: Vec<Vec<u8>>,
112 reply: oneshot::Sender<Result<Vec<Vec<f32>>, InferenceError>>,
113 },
114
115 HealthCheck {
118 reply: oneshot::Sender<Result<(), InferenceError>>,
119 },
120
121 Shutdown {
123 reply: oneshot::Sender<Result<(), InferenceError>>,
124 },
125}
126
127impl std::fmt::Debug for InferenceRequest {
128 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
129 match self {
130 Self::Generate { request, .. } => {
131 write!(
132 f,
133 "InferenceRequest::Generate {{ prompt_len: {} }}",
134 request.prompt.len()
135 )
136 }
137 Self::GenerateStream { request, .. } => {
138 write!(
139 f,
140 "InferenceRequest::GenerateStream {{ prompt_len: {} }}",
141 request.prompt.len()
142 )
143 }
144 Self::ModelInfo { .. } => write!(f, "InferenceRequest::ModelInfo"),
145 Self::IsReady { .. } => write!(f, "InferenceRequest::IsReady"),
146 Self::ApplyChatTemplate { messages, .. } => {
147 write!(
148 f,
149 "InferenceRequest::ApplyChatTemplate {{ messages: {} }}",
150 messages.len()
151 )
152 }
153 Self::CreateLora { config, .. } => {
154 write!(
155 f,
156 "InferenceRequest::CreateLora {{ rank: {} }}",
157 config.rank
158 )
159 }
160 Self::LoadLora { path, .. } => {
161 write!(f, "InferenceRequest::LoadLora {{ path: {path:?} }}")
162 }
163 Self::SaveLora { path, .. } => {
164 write!(f, "InferenceRequest::SaveLora {{ path: {path:?} }}")
165 }
166 Self::UnloadLora { .. } => write!(f, "InferenceRequest::UnloadLora"),
167 Self::HasLora { .. } => write!(f, "InferenceRequest::HasLora"),
168 Self::SetSession { session_id, .. } => {
169 write!(
170 f,
171 "InferenceRequest::SetSession {{ session_id: {session_id:?} }}"
172 )
173 }
174 Self::ClearSession { .. } => write!(f, "InferenceRequest::ClearSession"),
175 Self::ReleaseSession { session_id, .. } => {
176 write!(
177 f,
178 "InferenceRequest::ReleaseSession {{ session_id: {session_id:?} }}"
179 )
180 }
181 Self::Embed { images, .. } => {
182 write!(f, "InferenceRequest::Embed {{ images: {} }}", images.len())
183 }
184 Self::HealthCheck { .. } => write!(f, "InferenceRequest::HealthCheck"),
185 Self::Shutdown { .. } => write!(f, "InferenceRequest::Shutdown"),
186 }
187 }
188}