hyprstream_core/inference/
local.rs1use async_trait::async_trait;
23use std::path::{Path, PathBuf};
24use tokio::sync::{mpsc, oneshot};
25use tracing::{debug, error, info, instrument, warn};
26
27use crate::config::GenerationResult;
28use crate::runtime::kv_cache::CacheOwner;
29use crate::runtime::{create_engine, RuntimeConfig, RuntimeEngine, TorchEngine};
30use crate::training::TenantDeltaConfig;
31use hyprstream_rpc_std::inference_client::GenerationRequest;
32use hyprstream_rpc_std::inference_client::ModelInfo;
33
34use super::client::{InferenceClient, InferenceError};
35use super::request::InferenceRequest;
36use super::response::{StreamHandle, StreamStats};
37
38pub struct LocalInferenceService {
44 engine: TorchEngine,
45 requests: mpsc::UnboundedReceiver<InferenceRequest>,
46 shutdown_requested: bool,
47}
48
49impl LocalInferenceService {
50 #[instrument(skip_all, fields(model_path = %model_path.as_ref().display()))]
66 pub async fn start(
67 model_path: impl AsRef<Path>,
68 config: RuntimeConfig,
69 ) -> Result<LocalInferenceClient, InferenceError> {
70 let model_path = model_path.as_ref().to_path_buf();
71 info!("Starting inference service");
72
73 let (tx, rx) = mpsc::unbounded_channel();
75
76 let (init_tx, init_rx) = oneshot::channel();
78
79 std::thread::spawn(move || {
85 let rt = match tokio::runtime::Builder::new_current_thread()
86 .enable_all()
87 .build()
88 {
89 Ok(rt) => rt,
90 Err(e) => {
91 let _ = init_tx.send(Err(InferenceError::Internal(format!(
92 "Failed to create runtime: {e}"
93 ))));
94 return;
95 }
96 };
97
98 rt.block_on(async move {
99 match Self::initialize(model_path, config, rx).await {
101 Ok(service) => {
102 let _ = init_tx.send(Ok(()));
103 service.run().await;
104 }
105 Err(e) => {
106 let _ = init_tx.send(Err(InferenceError::Internal(e.to_string())));
107 }
108 }
109 });
110 });
111
112 init_rx
114 .await
115 .map_err(|_| InferenceError::channel("Service init channel closed"))?
116 .map_err(|e| InferenceError::ModelLoad(e.to_string()))?;
117
118 info!("Inference service started");
119 Ok(LocalInferenceClient { sender: tx })
120 }
121
122 async fn initialize(
124 model_path: PathBuf,
125 config: RuntimeConfig,
126 requests: mpsc::UnboundedReceiver<InferenceRequest>,
127 ) -> Result<Self, anyhow::Error> {
128 let mut engine = create_engine(&config)?;
129 engine.load_model(&model_path).await?;
130
131 let model_info = engine.model_info();
134 let num_layers = model_info.num_hidden_layers.unwrap_or(32) as usize;
135 let max_seq_len = config.max_context.unwrap_or(model_info.context_length) as usize;
136 let kv_budget = engine.compute_kv_budget();
137 engine.initialize_kv_registry(num_layers, max_seq_len, config.kv_quant_type, kv_budget);
138 info!(
139 "KV cache registry initialized: {} layers, max_seq_len={}",
140 num_layers, max_seq_len
141 );
142
143 Ok(Self {
144 engine,
145 requests,
146 shutdown_requested: false,
147 })
148 }
149
150 async fn run(mut self) {
152 debug!("Service loop started");
153 while let Some(request) = self.requests.recv().await {
154 if self.shutdown_requested {
155 debug!("Shutdown requested, draining remaining requests");
156 }
158 debug!(?request, "Processing request");
159 self.handle_request(request).await;
160
161 if self.shutdown_requested {
162 break;
163 }
164 }
165 info!("Service loop ended");
166 }
167
168 async fn handle_request(&mut self, request: InferenceRequest) {
170 match request {
171 InferenceRequest::Generate { request, reply } => {
172 let result = self.handle_generate(request).await;
173 let _ = reply.send(result);
174 }
175
176 InferenceRequest::GenerateStream {
177 request,
178 chunk_sender,
179 stats_sender,
180 reply,
181 } => {
182 let _ = reply.send(Ok(()));
184 self.handle_generate_stream(request, chunk_sender, stats_sender)
186 .await;
187 }
188
189 InferenceRequest::ModelInfo { reply } => {
190 let info = self.engine.model_info();
191 let _ = reply.send(Ok(info));
192 }
193
194 InferenceRequest::IsReady { reply } => {
195 let ready = !self.engine.model_info().name.is_empty();
197 let _ = reply.send(Ok(ready));
198 }
199
200 InferenceRequest::ApplyChatTemplate {
201 messages,
202 add_generation_prompt,
203 tools,
204 reply,
205 } => {
206 let result = self
207 .engine
208 .apply_chat_template(&messages, add_generation_prompt, tools.as_ref())
209 .map_err(|e| InferenceError::Internal(e.to_string()));
210 let _ = reply.send(result);
211 }
212
213 InferenceRequest::CreateLora { config, reply } => {
214 let result = self
215 .engine
216 .create_lora(config)
217 .map_err(|e| InferenceError::LoRA(e.to_string()));
218 let _ = reply.send(result);
219 }
220
221 InferenceRequest::LoadLora { reply, .. }
222 | InferenceRequest::SaveLora { reply, .. }
223 | InferenceRequest::UnloadLora { reply } => {
224 let _ = reply.send(Err(InferenceError::LoRA(
225 "LoRA operations are handled by InferenceService (RPC mode)".to_owned(),
226 )));
227 }
228
229 InferenceRequest::HasLora { reply } => {
230 let has_lora = false; let _ = reply.send(Ok(has_lora));
232 }
233
234 InferenceRequest::SetSession { session_id, reply } => {
235 let result = self
236 .engine
237 .set_session(CacheOwner::Session(session_id))
238 .map_err(|e| InferenceError::Session(e.to_string()));
239 let _ = reply.send(result);
240 }
241
242 InferenceRequest::ClearSession { reply } => {
243 self.engine.clear_kv_cache();
244 let _ = reply.send(Ok(()));
245 }
246
247 InferenceRequest::ReleaseSession { session_id, reply } => {
248 let result = self
249 .engine
250 .release_session(&CacheOwner::Session(session_id.clone()))
251 .map_err(|e| InferenceError::Session(e.to_string()));
252 let _ = reply.send(result);
253 }
254
255 InferenceRequest::Embed { images, reply } => {
256 let result = self.handle_embed(images).await;
257 let _ = reply.send(result);
258 }
259
260 InferenceRequest::HealthCheck { reply } => {
261 let _ = reply.send(Ok(()));
263 }
264
265 InferenceRequest::Shutdown { reply } => {
266 info!("Shutdown requested");
267 self.shutdown_requested = true;
268
269 self.engine.release_device_resources();
280
281 let _ = reply.send(Ok(()));
282 }
283 }
284 }
285
286 async fn handle_generate(
288 &mut self,
289 request: GenerationRequest,
290 ) -> Result<GenerationResult, InferenceError> {
291 self.engine
292 .generate_with_params(request)
293 .await
294 .map_err(|e| InferenceError::Generation(e.to_string()))
295 }
296
297 async fn handle_generate_stream(
302 &mut self,
303 request: GenerationRequest,
304 chunk_sender: mpsc::Sender<Result<String, InferenceError>>,
305 stats_sender: oneshot::Sender<StreamStats>,
306 ) {
307 use futures::StreamExt;
308
309 let stream_result = self.engine.generate(request);
310
311 match stream_result {
312 Ok(mut stream) => {
313 while let Some(chunk_result) = stream.next().await {
315 let mapped =
316 chunk_result.map_err(|e| InferenceError::Generation(e.to_string()));
317 if chunk_sender.send(mapped).await.is_err() {
318 warn!("Client dropped, stopping generation");
320 break;
321 }
322 }
323
324 let stats = StreamStats::from_generation_stats(&stream.stats());
326 let _ = stats_sender.send(stats);
327 }
328 Err(e) => {
329 error!("Generation failed: {}", e);
330 let _ = chunk_sender
331 .send(Err(InferenceError::Generation(e.to_string())))
332 .await;
333 let _ = stats_sender.send(StreamStats::default());
334 }
335 }
336 }
337
338 async fn handle_embed(&self, images: Vec<Vec<u8>>) -> Result<Vec<Vec<f32>>, InferenceError> {
340 self.engine
341 .embed_images(&images)
342 .map_err(|e| InferenceError::Generation(e.to_string()))
343 }
344}
345
346#[derive(Clone)]
351pub struct LocalInferenceClient {
352 sender: mpsc::UnboundedSender<InferenceRequest>,
353}
354
355#[async_trait]
356impl InferenceClient for LocalInferenceClient {
357 async fn generate(
358 &self,
359 request: GenerationRequest,
360 ) -> Result<GenerationResult, InferenceError> {
361 let (tx, rx) = oneshot::channel();
362 self.sender
363 .send(InferenceRequest::Generate { request, reply: tx })
364 .map_err(|_| InferenceError::Unavailable)?;
365 rx.await
366 .map_err(|_| InferenceError::channel("No response"))?
367 }
368
369 async fn generate_stream(
370 &self,
371 request: GenerationRequest,
372 ) -> Result<StreamHandle, InferenceError> {
373 let (chunk_tx, chunk_rx) = mpsc::channel(32); let (stats_tx, stats_rx) = oneshot::channel();
375 let (reply_tx, reply_rx) = oneshot::channel();
376
377 self.sender
378 .send(InferenceRequest::GenerateStream {
379 request,
380 chunk_sender: chunk_tx,
381 stats_sender: stats_tx,
382 reply: reply_tx,
383 })
384 .map_err(|_| InferenceError::Unavailable)?;
385
386 reply_rx
388 .await
389 .map_err(|_| InferenceError::channel("No response"))??;
390
391 Ok(StreamHandle::new(chunk_rx, stats_rx))
392 }
393
394 async fn model_info(&self) -> Result<ModelInfo, InferenceError> {
395 let (tx, rx) = oneshot::channel();
396 self.sender
397 .send(InferenceRequest::ModelInfo { reply: tx })
398 .map_err(|_| InferenceError::Unavailable)?;
399 rx.await
400 .map_err(|_| InferenceError::channel("No response"))?
401 }
402
403 async fn is_ready(&self) -> Result<bool, InferenceError> {
404 let (tx, rx) = oneshot::channel();
405 self.sender
406 .send(InferenceRequest::IsReady { reply: tx })
407 .map_err(|_| InferenceError::Unavailable)?;
408 rx.await
409 .map_err(|_| InferenceError::channel("No response"))?
410 }
411
412 async fn apply_chat_template(
413 &self,
414 messages: &[crate::runtime::template_engine::ChatMessage],
415 add_generation_prompt: bool,
416 tools: Option<serde_json::Value>,
417 ) -> Result<String, InferenceError> {
418 let (tx, rx) = oneshot::channel();
419 self.sender
420 .send(InferenceRequest::ApplyChatTemplate {
421 messages: messages.to_vec(),
422 add_generation_prompt,
423 tools,
424 reply: tx,
425 })
426 .map_err(|_| InferenceError::Unavailable)?;
427 rx.await
428 .map_err(|_| InferenceError::channel("No response"))?
429 }
430
431 async fn create_lora(&self, config: TenantDeltaConfig) -> Result<(), InferenceError> {
432 let (tx, rx) = oneshot::channel();
433 self.sender
434 .send(InferenceRequest::CreateLora { config, reply: tx })
435 .map_err(|_| InferenceError::Unavailable)?;
436 rx.await
437 .map_err(|_| InferenceError::channel("No response"))?
438 }
439
440 async fn load_lora(&self, path: &Path) -> Result<(), InferenceError> {
441 let (tx, rx) = oneshot::channel();
442 self.sender
443 .send(InferenceRequest::LoadLora {
444 path: path.to_path_buf(),
445 reply: tx,
446 })
447 .map_err(|_| InferenceError::Unavailable)?;
448 rx.await
449 .map_err(|_| InferenceError::channel("No response"))?
450 }
451
452 async fn save_lora(&self, path: &Path) -> Result<(), InferenceError> {
453 let (tx, rx) = oneshot::channel();
454 self.sender
455 .send(InferenceRequest::SaveLora {
456 path: path.to_path_buf(),
457 reply: tx,
458 })
459 .map_err(|_| InferenceError::Unavailable)?;
460 rx.await
461 .map_err(|_| InferenceError::channel("No response"))?
462 }
463
464 async fn unload_lora(&self) -> Result<(), InferenceError> {
465 let (tx, rx) = oneshot::channel();
466 self.sender
467 .send(InferenceRequest::UnloadLora { reply: tx })
468 .map_err(|_| InferenceError::Unavailable)?;
469 rx.await
470 .map_err(|_| InferenceError::channel("No response"))?
471 }
472
473 async fn has_lora(&self) -> Result<bool, InferenceError> {
474 let (tx, rx) = oneshot::channel();
475 self.sender
476 .send(InferenceRequest::HasLora { reply: tx })
477 .map_err(|_| InferenceError::Unavailable)?;
478 rx.await
479 .map_err(|_| InferenceError::channel("No response"))?
480 }
481
482 async fn set_session(&self, session_id: String) -> Result<(), InferenceError> {
483 let (tx, rx) = oneshot::channel();
484 self.sender
485 .send(InferenceRequest::SetSession {
486 session_id,
487 reply: tx,
488 })
489 .map_err(|_| InferenceError::Unavailable)?;
490 rx.await
491 .map_err(|_| InferenceError::channel("No response"))?
492 }
493
494 async fn clear_session(&self) -> Result<(), InferenceError> {
495 let (tx, rx) = oneshot::channel();
496 self.sender
497 .send(InferenceRequest::ClearSession { reply: tx })
498 .map_err(|_| InferenceError::Unavailable)?;
499 rx.await
500 .map_err(|_| InferenceError::channel("No response"))?
501 }
502
503 async fn release_session(&self, session_id: &str) -> Result<(), InferenceError> {
504 let (tx, rx) = oneshot::channel();
505 self.sender
506 .send(InferenceRequest::ReleaseSession {
507 session_id: session_id.to_owned(),
508 reply: tx,
509 })
510 .map_err(|_| InferenceError::Unavailable)?;
511 rx.await
512 .map_err(|_| InferenceError::channel("No response"))?
513 }
514
515 async fn embed(&self, images: &[Vec<u8>]) -> Result<Vec<Vec<f32>>, InferenceError> {
516 let (tx, rx) = oneshot::channel();
517 self.sender
518 .send(InferenceRequest::Embed {
519 images: images.to_vec(),
520 reply: tx,
521 })
522 .map_err(|_| InferenceError::Unavailable)?;
523 rx.await
524 .map_err(|_| InferenceError::channel("No response"))?
525 }
526
527 async fn health_check(&self) -> Result<(), InferenceError> {
528 let (tx, rx) = oneshot::channel();
529 self.sender
530 .send(InferenceRequest::HealthCheck { reply: tx })
531 .map_err(|_| InferenceError::Unavailable)?;
532 rx.await
533 .map_err(|_| InferenceError::channel("No response"))?
534 }
535
536 async fn shutdown(&self) -> Result<(), InferenceError> {
537 let (tx, rx) = oneshot::channel();
538 self.sender
539 .send(InferenceRequest::Shutdown { reply: tx })
540 .map_err(|_| InferenceError::Unavailable)?;
541 rx.await
542 .map_err(|_| InferenceError::channel("No response"))?
543 }
544}