Skip to main content

hyprstream_core/inference/
local.rs

1//! In-process inference service using channels.
2//!
3//! This module provides a local service implementation that runs on a dedicated
4//! thread and communicates with clients via unbounded mpsc channels.
5//!
6//! # Architecture
7//!
8//! ```text
9//! ┌─────────────────────────────────────────────────┐
10//! │         LocalInferenceClient (cloneable)         │
11//! │  - sender: mpsc::UnboundedSender<Request>       │
12//! └───────────────────────┬─────────────────────────┘
13//!                         │ sends request
14//!                         ▼
15//! ┌─────────────────────────────────────────────────┐
16//! │            LocalInferenceService                 │
17//! │  - engine: TorchEngine                          │
18//! │  - runs on dedicated thread                     │
19//! └─────────────────────────────────────────────────┘
20//! ```
21
22use async_trait::async_trait;
23use std::path::{Path, PathBuf};
24use tokio::sync::{mpsc, oneshot};
25use tracing::{debug, error, info, instrument, warn};
26
27use crate::config::GenerationResult;
28use crate::runtime::kv_cache::CacheOwner;
29use crate::runtime::{create_engine, RuntimeConfig, RuntimeEngine, TorchEngine};
30use crate::training::TenantDeltaConfig;
31use hyprstream_rpc_std::inference_client::GenerationRequest;
32use hyprstream_rpc_std::inference_client::ModelInfo;
33
34use super::client::{InferenceClient, InferenceError};
35use super::request::InferenceRequest;
36use super::response::{StreamHandle, StreamStats};
37
38/// In-process inference service that runs on a dedicated thread.
39///
40/// The service owns a TorchEngine and processes requests from clients
41/// via an unbounded mpsc channel. It runs on its own thread with a
42/// single-threaded tokio runtime to isolate GPU operations.
43pub struct LocalInferenceService {
44    engine: TorchEngine,
45    requests: mpsc::UnboundedReceiver<InferenceRequest>,
46    shutdown_requested: bool,
47}
48
49impl LocalInferenceService {
50    /// Start the service and return a client handle.
51    ///
52    /// The service runs on a dedicated thread with its own tokio runtime.
53    /// This is necessary because tch-rs types contain raw pointers that
54    /// are not Send, and GPU operations should be isolated.
55    ///
56    /// # Arguments
57    /// * `model_path` - Path to the model directory
58    /// * `config` - Runtime configuration
59    ///
60    /// # Example
61    /// ```rust,ignore
62    /// let client = LocalInferenceService::start("/path/to/model", config).await?;
63    /// let result = client.generate(request).await?;
64    /// ```
65    #[instrument(skip_all, fields(model_path = %model_path.as_ref().display()))]
66    pub async fn start(
67        model_path: impl AsRef<Path>,
68        config: RuntimeConfig,
69    ) -> Result<LocalInferenceClient, InferenceError> {
70        let model_path = model_path.as_ref().to_path_buf();
71        info!("Starting inference service");
72
73        // Create channel for communication
74        let (tx, rx) = mpsc::unbounded_channel();
75
76        // Use oneshot to get result from spawned thread
77        let (init_tx, init_rx) = oneshot::channel();
78
79        // Spawn service on dedicated thread with its own runtime
80        // This is necessary because:
81        // 1. tch-rs types contain raw pointers (not Send)
82        // 2. GPU operations benefit from thread isolation
83        // 3. Matches git2db's LocalService pattern
84        std::thread::spawn(move || {
85            let rt = match tokio::runtime::Builder::new_current_thread()
86                .enable_all()
87                .build()
88            {
89                Ok(rt) => rt,
90                Err(e) => {
91                    let _ = init_tx.send(Err(InferenceError::Internal(format!(
92                        "Failed to create runtime: {e}"
93                    ))));
94                    return;
95                }
96            };
97
98            rt.block_on(async move {
99                // Load model
100                match Self::initialize(model_path, config, rx).await {
101                    Ok(service) => {
102                        let _ = init_tx.send(Ok(()));
103                        service.run().await;
104                    }
105                    Err(e) => {
106                        let _ = init_tx.send(Err(InferenceError::Internal(e.to_string())));
107                    }
108                }
109            });
110        });
111
112        // Wait for initialization
113        init_rx
114            .await
115            .map_err(|_| InferenceError::channel("Service init channel closed"))?
116            .map_err(|e| InferenceError::ModelLoad(e.to_string()))?;
117
118        info!("Inference service started");
119        Ok(LocalInferenceClient { sender: tx })
120    }
121
122    /// Initialize the service (called on service thread).
123    async fn initialize(
124        model_path: PathBuf,
125        config: RuntimeConfig,
126        requests: mpsc::UnboundedReceiver<InferenceRequest>,
127    ) -> Result<Self, anyhow::Error> {
128        let mut engine = create_engine(&config)?;
129        engine.load_model(&model_path).await?;
130
131        // Initialize KV cache registry for session-based cache isolation
132        // This enables concurrent inference with isolated context per session
133        let model_info = engine.model_info();
134        let num_layers = model_info.num_hidden_layers.unwrap_or(32) as usize;
135        let max_seq_len = config.max_context.unwrap_or(model_info.context_length) as usize;
136        let kv_budget = engine.compute_kv_budget();
137        engine.initialize_kv_registry(num_layers, max_seq_len, config.kv_quant_type, kv_budget);
138        info!(
139            "KV cache registry initialized: {} layers, max_seq_len={}",
140            num_layers, max_seq_len
141        );
142
143        Ok(Self {
144            engine,
145            requests,
146            shutdown_requested: false,
147        })
148    }
149
150    /// Main service loop - processes requests until shutdown or channel closes.
151    async fn run(mut self) {
152        debug!("Service loop started");
153        while let Some(request) = self.requests.recv().await {
154            if self.shutdown_requested {
155                debug!("Shutdown requested, draining remaining requests");
156                // Continue processing to drain queue gracefully
157            }
158            debug!(?request, "Processing request");
159            self.handle_request(request).await;
160
161            if self.shutdown_requested {
162                break;
163            }
164        }
165        info!("Service loop ended");
166    }
167
168    /// Handle a single request.
169    async fn handle_request(&mut self, request: InferenceRequest) {
170        match request {
171            InferenceRequest::Generate { request, reply } => {
172                let result = self.handle_generate(request).await;
173                let _ = reply.send(result);
174            }
175
176            InferenceRequest::GenerateStream {
177                request,
178                chunk_sender,
179                stats_sender,
180                reply,
181            } => {
182                // Acknowledge request receipt
183                let _ = reply.send(Ok(()));
184                // Run streaming generation
185                self.handle_generate_stream(request, chunk_sender, stats_sender)
186                    .await;
187            }
188
189            InferenceRequest::ModelInfo { reply } => {
190                let info = self.engine.model_info();
191                let _ = reply.send(Ok(info));
192            }
193
194            InferenceRequest::IsReady { reply } => {
195                // Engine is ready if model is loaded
196                let ready = !self.engine.model_info().name.is_empty();
197                let _ = reply.send(Ok(ready));
198            }
199
200            InferenceRequest::ApplyChatTemplate {
201                messages,
202                add_generation_prompt,
203                tools,
204                reply,
205            } => {
206                let result = self
207                    .engine
208                    .apply_chat_template(&messages, add_generation_prompt, tools.as_ref())
209                    .map_err(|e| InferenceError::Internal(e.to_string()));
210                let _ = reply.send(result);
211            }
212
213            InferenceRequest::CreateLora { config, reply } => {
214                let result = self
215                    .engine
216                    .create_lora(config)
217                    .map_err(|e| InferenceError::LoRA(e.to_string()));
218                let _ = reply.send(result);
219            }
220
221            InferenceRequest::LoadLora { reply, .. }
222            | InferenceRequest::SaveLora { reply, .. }
223            | InferenceRequest::UnloadLora { reply } => {
224                let _ = reply.send(Err(InferenceError::LoRA(
225                    "LoRA operations are handled by InferenceService (RPC mode)".to_owned(),
226                )));
227            }
228
229            InferenceRequest::HasLora { reply } => {
230                let has_lora = false; // LoRA state managed by InferenceService's base_delta
231                let _ = reply.send(Ok(has_lora));
232            }
233
234            InferenceRequest::SetSession { session_id, reply } => {
235                let result = self
236                    .engine
237                    .set_session(CacheOwner::Session(session_id))
238                    .map_err(|e| InferenceError::Session(e.to_string()));
239                let _ = reply.send(result);
240            }
241
242            InferenceRequest::ClearSession { reply } => {
243                self.engine.clear_kv_cache();
244                let _ = reply.send(Ok(()));
245            }
246
247            InferenceRequest::ReleaseSession { session_id, reply } => {
248                let result = self
249                    .engine
250                    .release_session(&CacheOwner::Session(session_id.clone()))
251                    .map_err(|e| InferenceError::Session(e.to_string()));
252                let _ = reply.send(result);
253            }
254
255            InferenceRequest::Embed { images, reply } => {
256                let result = self.handle_embed(images).await;
257                let _ = reply.send(result);
258            }
259
260            InferenceRequest::HealthCheck { reply } => {
261                // Service is healthy if we can respond
262                let _ = reply.send(Ok(()));
263            }
264
265            InferenceRequest::Shutdown { reply } => {
266                info!("Shutdown requested");
267                self.shutdown_requested = true;
268
269                // #429: drop every CUDA tensor the engine owns *now*, on this
270                // service thread, before we acknowledge shutdown. The service
271                // thread is spawned detached, so if we left the RoPE sin/cos
272                // tables (thread-local) and model weights to drop at thread-exit,
273                // their TLS destructors could fire during process-exit cleanup —
274                // after libtorch's atexit handler has torn down the CUDA context
275                // — aborting with `c10::Error: invalid device pointer`. Doing it
276                // here, while the client is still blocked on `reply`, guarantees
277                // the context is alive when these device tensors free. See
278                // `TorchEngine::release_device_resources`.
279                self.engine.release_device_resources();
280
281                let _ = reply.send(Ok(()));
282            }
283        }
284    }
285
286    /// Handle non-streaming generation.
287    async fn handle_generate(
288        &mut self,
289        request: GenerationRequest,
290    ) -> Result<GenerationResult, InferenceError> {
291        self.engine
292            .generate_with_params(request)
293            .await
294            .map_err(|e| InferenceError::Generation(e.to_string()))
295    }
296
297    /// Handle streaming generation.
298    ///
299    /// This method runs the TextStream internally and pumps chunks
300    /// to the client via the provided channel.
301    async fn handle_generate_stream(
302        &mut self,
303        request: GenerationRequest,
304        chunk_sender: mpsc::Sender<Result<String, InferenceError>>,
305        stats_sender: oneshot::Sender<StreamStats>,
306    ) {
307        use futures::StreamExt;
308
309        let stream_result = self.engine.generate(request);
310
311        match stream_result {
312            Ok(mut stream) => {
313                // Pump stream to channel
314                while let Some(chunk_result) = stream.next().await {
315                    let mapped =
316                        chunk_result.map_err(|e| InferenceError::Generation(e.to_string()));
317                    if chunk_sender.send(mapped).await.is_err() {
318                        // Client dropped, stop generating
319                        warn!("Client dropped, stopping generation");
320                        break;
321                    }
322                }
323
324                // Send final stats
325                let stats = StreamStats::from_generation_stats(&stream.stats());
326                let _ = stats_sender.send(stats);
327            }
328            Err(e) => {
329                error!("Generation failed: {}", e);
330                let _ = chunk_sender
331                    .send(Err(InferenceError::Generation(e.to_string())))
332                    .await;
333                let _ = stats_sender.send(StreamStats::default());
334            }
335        }
336    }
337
338    /// Handle embedding request.
339    async fn handle_embed(&self, images: Vec<Vec<u8>>) -> Result<Vec<Vec<f32>>, InferenceError> {
340        self.engine
341            .embed_images(&images)
342            .map_err(|e| InferenceError::Generation(e.to_string()))
343    }
344}
345
346/// Lightweight, cloneable client handle for the inference service.
347///
348/// Clients can be cloned and shared across tasks. All clients communicate
349/// with the same underlying service instance.
350#[derive(Clone)]
351pub struct LocalInferenceClient {
352    sender: mpsc::UnboundedSender<InferenceRequest>,
353}
354
355#[async_trait]
356impl InferenceClient for LocalInferenceClient {
357    async fn generate(
358        &self,
359        request: GenerationRequest,
360    ) -> Result<GenerationResult, InferenceError> {
361        let (tx, rx) = oneshot::channel();
362        self.sender
363            .send(InferenceRequest::Generate { request, reply: tx })
364            .map_err(|_| InferenceError::Unavailable)?;
365        rx.await
366            .map_err(|_| InferenceError::channel("No response"))?
367    }
368
369    async fn generate_stream(
370        &self,
371        request: GenerationRequest,
372    ) -> Result<StreamHandle, InferenceError> {
373        let (chunk_tx, chunk_rx) = mpsc::channel(32); // Bounded for backpressure
374        let (stats_tx, stats_rx) = oneshot::channel();
375        let (reply_tx, reply_rx) = oneshot::channel();
376
377        self.sender
378            .send(InferenceRequest::GenerateStream {
379                request,
380                chunk_sender: chunk_tx,
381                stats_sender: stats_tx,
382                reply: reply_tx,
383            })
384            .map_err(|_| InferenceError::Unavailable)?;
385
386        // Wait for acknowledgment
387        reply_rx
388            .await
389            .map_err(|_| InferenceError::channel("No response"))??;
390
391        Ok(StreamHandle::new(chunk_rx, stats_rx))
392    }
393
394    async fn model_info(&self) -> Result<ModelInfo, InferenceError> {
395        let (tx, rx) = oneshot::channel();
396        self.sender
397            .send(InferenceRequest::ModelInfo { reply: tx })
398            .map_err(|_| InferenceError::Unavailable)?;
399        rx.await
400            .map_err(|_| InferenceError::channel("No response"))?
401    }
402
403    async fn is_ready(&self) -> Result<bool, InferenceError> {
404        let (tx, rx) = oneshot::channel();
405        self.sender
406            .send(InferenceRequest::IsReady { reply: tx })
407            .map_err(|_| InferenceError::Unavailable)?;
408        rx.await
409            .map_err(|_| InferenceError::channel("No response"))?
410    }
411
412    async fn apply_chat_template(
413        &self,
414        messages: &[crate::runtime::template_engine::ChatMessage],
415        add_generation_prompt: bool,
416        tools: Option<serde_json::Value>,
417    ) -> Result<String, InferenceError> {
418        let (tx, rx) = oneshot::channel();
419        self.sender
420            .send(InferenceRequest::ApplyChatTemplate {
421                messages: messages.to_vec(),
422                add_generation_prompt,
423                tools,
424                reply: tx,
425            })
426            .map_err(|_| InferenceError::Unavailable)?;
427        rx.await
428            .map_err(|_| InferenceError::channel("No response"))?
429    }
430
431    async fn create_lora(&self, config: TenantDeltaConfig) -> Result<(), InferenceError> {
432        let (tx, rx) = oneshot::channel();
433        self.sender
434            .send(InferenceRequest::CreateLora { config, reply: tx })
435            .map_err(|_| InferenceError::Unavailable)?;
436        rx.await
437            .map_err(|_| InferenceError::channel("No response"))?
438    }
439
440    async fn load_lora(&self, path: &Path) -> Result<(), InferenceError> {
441        let (tx, rx) = oneshot::channel();
442        self.sender
443            .send(InferenceRequest::LoadLora {
444                path: path.to_path_buf(),
445                reply: tx,
446            })
447            .map_err(|_| InferenceError::Unavailable)?;
448        rx.await
449            .map_err(|_| InferenceError::channel("No response"))?
450    }
451
452    async fn save_lora(&self, path: &Path) -> Result<(), InferenceError> {
453        let (tx, rx) = oneshot::channel();
454        self.sender
455            .send(InferenceRequest::SaveLora {
456                path: path.to_path_buf(),
457                reply: tx,
458            })
459            .map_err(|_| InferenceError::Unavailable)?;
460        rx.await
461            .map_err(|_| InferenceError::channel("No response"))?
462    }
463
464    async fn unload_lora(&self) -> Result<(), InferenceError> {
465        let (tx, rx) = oneshot::channel();
466        self.sender
467            .send(InferenceRequest::UnloadLora { reply: tx })
468            .map_err(|_| InferenceError::Unavailable)?;
469        rx.await
470            .map_err(|_| InferenceError::channel("No response"))?
471    }
472
473    async fn has_lora(&self) -> Result<bool, InferenceError> {
474        let (tx, rx) = oneshot::channel();
475        self.sender
476            .send(InferenceRequest::HasLora { reply: tx })
477            .map_err(|_| InferenceError::Unavailable)?;
478        rx.await
479            .map_err(|_| InferenceError::channel("No response"))?
480    }
481
482    async fn set_session(&self, session_id: String) -> Result<(), InferenceError> {
483        let (tx, rx) = oneshot::channel();
484        self.sender
485            .send(InferenceRequest::SetSession {
486                session_id,
487                reply: tx,
488            })
489            .map_err(|_| InferenceError::Unavailable)?;
490        rx.await
491            .map_err(|_| InferenceError::channel("No response"))?
492    }
493
494    async fn clear_session(&self) -> Result<(), InferenceError> {
495        let (tx, rx) = oneshot::channel();
496        self.sender
497            .send(InferenceRequest::ClearSession { reply: tx })
498            .map_err(|_| InferenceError::Unavailable)?;
499        rx.await
500            .map_err(|_| InferenceError::channel("No response"))?
501    }
502
503    async fn release_session(&self, session_id: &str) -> Result<(), InferenceError> {
504        let (tx, rx) = oneshot::channel();
505        self.sender
506            .send(InferenceRequest::ReleaseSession {
507                session_id: session_id.to_owned(),
508                reply: tx,
509            })
510            .map_err(|_| InferenceError::Unavailable)?;
511        rx.await
512            .map_err(|_| InferenceError::channel("No response"))?
513    }
514
515    async fn embed(&self, images: &[Vec<u8>]) -> Result<Vec<Vec<f32>>, InferenceError> {
516        let (tx, rx) = oneshot::channel();
517        self.sender
518            .send(InferenceRequest::Embed {
519                images: images.to_vec(),
520                reply: tx,
521            })
522            .map_err(|_| InferenceError::Unavailable)?;
523        rx.await
524            .map_err(|_| InferenceError::channel("No response"))?
525    }
526
527    async fn health_check(&self) -> Result<(), InferenceError> {
528        let (tx, rx) = oneshot::channel();
529        self.sender
530            .send(InferenceRequest::HealthCheck { reply: tx })
531            .map_err(|_| InferenceError::Unavailable)?;
532        rx.await
533            .map_err(|_| InferenceError::channel("No response"))?
534    }
535
536    async fn shutdown(&self) -> Result<(), InferenceError> {
537        let (tx, rx) = oneshot::channel();
538        self.sender
539            .send(InferenceRequest::Shutdown { reply: tx })
540            .map_err(|_| InferenceError::Unavailable)?;
541        rx.await
542            .map_err(|_| InferenceError::channel("No response"))?
543    }
544}