Skip to main content

hyprstream_core/inference/
request.rs

1//! Request types for the inference service.
2//!
3//! These types define the message format for communication between
4//! LocalInferenceClient and LocalInferenceService via channels.
5
6use crate::config::GenerationResult;
7use crate::training::TenantDeltaConfig;
8use hyprstream_rpc_std::inference_client::GenerationRequest;
9use hyprstream_rpc_std::inference_client::ModelInfo;
10use std::path::PathBuf;
11use tokio::sync::{mpsc, oneshot};
12
13use super::client::InferenceError;
14use super::response::StreamStats;
15
16/// Request types for the inference service.
17///
18/// All variants include a oneshot reply channel for returning results.
19/// For streaming generation, we use mpsc channels for chunks.
20pub enum InferenceRequest {
21    // === Generation ===
22    /// Non-streaming generation request.
23    Generate {
24        request: GenerationRequest,
25        reply: oneshot::Sender<Result<GenerationResult, InferenceError>>,
26    },
27
28    /// Streaming generation request.
29    ///
30    /// Unlike other requests, this uses an mpsc channel for streaming chunks
31    /// and a separate oneshot for final stats.
32    GenerateStream {
33        request: GenerationRequest,
34        /// Channel to send text chunks as they're generated.
35        chunk_sender: mpsc::Sender<Result<String, InferenceError>>,
36        /// Channel to send final statistics after generation completes.
37        stats_sender: oneshot::Sender<StreamStats>,
38        /// Reply channel for acknowledging request receipt or early errors.
39        reply: oneshot::Sender<Result<(), InferenceError>>,
40    },
41
42    // === Model Info ===
43    /// Get model information.
44    ModelInfo {
45        reply: oneshot::Sender<Result<ModelInfo, InferenceError>>,
46    },
47
48    /// Check if model is ready.
49    IsReady {
50        reply: oneshot::Sender<Result<bool, InferenceError>>,
51    },
52
53    /// Apply chat template to messages.
54    ApplyChatTemplate {
55        messages: Vec<crate::runtime::template_engine::ChatMessage>,
56        add_generation_prompt: bool,
57        tools: Option<serde_json::Value>,
58        reply: oneshot::Sender<Result<String, InferenceError>>,
59    },
60
61    // === LoRA Operations ===
62    /// Create LoRA adapter structure.
63    CreateLora {
64        config: TenantDeltaConfig,
65        reply: oneshot::Sender<Result<(), InferenceError>>,
66    },
67
68    /// Load LoRA adapter from file.
69    LoadLora {
70        path: PathBuf,
71        reply: oneshot::Sender<Result<(), InferenceError>>,
72    },
73
74    /// Save LoRA adapter to file.
75    SaveLora {
76        path: PathBuf,
77        reply: oneshot::Sender<Result<(), InferenceError>>,
78    },
79
80    /// Unload current LoRA adapter.
81    UnloadLora {
82        reply: oneshot::Sender<Result<(), InferenceError>>,
83    },
84
85    /// Check if LoRA is loaded.
86    HasLora {
87        reply: oneshot::Sender<Result<bool, InferenceError>>,
88    },
89
90    // === Session Management ===
91    /// Set active session for KV cache isolation.
92    SetSession {
93        session_id: String,
94        reply: oneshot::Sender<Result<(), InferenceError>>,
95    },
96
97    /// Clear current session's KV cache.
98    ClearSession {
99        reply: oneshot::Sender<Result<(), InferenceError>>,
100    },
101
102    /// Release a session's KV cache.
103    ReleaseSession {
104        session_id: String,
105        reply: oneshot::Sender<Result<(), InferenceError>>,
106    },
107
108    // === Embeddings ===
109    /// Compute embeddings for images.
110    Embed {
111        images: Vec<Vec<u8>>,
112        reply: oneshot::Sender<Result<Vec<Vec<f32>>, InferenceError>>,
113    },
114
115    // === Health ===
116    /// Health check request.
117    HealthCheck {
118        reply: oneshot::Sender<Result<(), InferenceError>>,
119    },
120
121    /// Shutdown request.
122    Shutdown {
123        reply: oneshot::Sender<Result<(), InferenceError>>,
124    },
125}
126
127impl std::fmt::Debug for InferenceRequest {
128    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
129        match self {
130            Self::Generate { request, .. } => {
131                write!(
132                    f,
133                    "InferenceRequest::Generate {{ prompt_len: {} }}",
134                    request.prompt.len()
135                )
136            }
137            Self::GenerateStream { request, .. } => {
138                write!(
139                    f,
140                    "InferenceRequest::GenerateStream {{ prompt_len: {} }}",
141                    request.prompt.len()
142                )
143            }
144            Self::ModelInfo { .. } => write!(f, "InferenceRequest::ModelInfo"),
145            Self::IsReady { .. } => write!(f, "InferenceRequest::IsReady"),
146            Self::ApplyChatTemplate { messages, .. } => {
147                write!(
148                    f,
149                    "InferenceRequest::ApplyChatTemplate {{ messages: {} }}",
150                    messages.len()
151                )
152            }
153            Self::CreateLora { config, .. } => {
154                write!(
155                    f,
156                    "InferenceRequest::CreateLora {{ rank: {} }}",
157                    config.rank
158                )
159            }
160            Self::LoadLora { path, .. } => {
161                write!(f, "InferenceRequest::LoadLora {{ path: {path:?} }}")
162            }
163            Self::SaveLora { path, .. } => {
164                write!(f, "InferenceRequest::SaveLora {{ path: {path:?} }}")
165            }
166            Self::UnloadLora { .. } => write!(f, "InferenceRequest::UnloadLora"),
167            Self::HasLora { .. } => write!(f, "InferenceRequest::HasLora"),
168            Self::SetSession { session_id, .. } => {
169                write!(
170                    f,
171                    "InferenceRequest::SetSession {{ session_id: {session_id:?} }}"
172                )
173            }
174            Self::ClearSession { .. } => write!(f, "InferenceRequest::ClearSession"),
175            Self::ReleaseSession { session_id, .. } => {
176                write!(
177                    f,
178                    "InferenceRequest::ReleaseSession {{ session_id: {session_id:?} }}"
179                )
180            }
181            Self::Embed { images, .. } => {
182                write!(f, "InferenceRequest::Embed {{ images: {} }}", images.len())
183            }
184            Self::HealthCheck { .. } => write!(f, "InferenceRequest::HealthCheck"),
185            Self::Shutdown { .. } => write!(f, "InferenceRequest::Shutdown"),
186        }
187    }
188}