hyprstream_core/inference/mod.rs
1//! Inference service abstraction for in-process and future IPC access.
2//!
3//! This module provides a service-oriented interface to inference operations,
4//! allowing components to share inference capabilities through client handles.
5//!
6//! # Architecture
7//!
8//! ```text
9//! ┌─────────────────────────────────────────────────┐
10//! │ InferenceClient (trait) │
11//! │ - generate() - generate_stream() │
12//! │ - model_info() - load_lora() │
13//! │ - set_session() - health_check() │
14//! └─────────────────────────────────────────────────┘
15//! ▲
16//! │
17//! ┌─────────┴─────────┐
18//! │ LocalInferenceClient │
19//! │ (mpsc channels) │
20//! └───────────────────┘
21//! │
22//! ▼
23//! ┌─────────────────────────────────────────────────┐
24//! │ LocalInferenceService │
25//! │ - engine: TorchEngine │
26//! │ - runs on dedicated thread │
27//! └─────────────────────────────────────────────────┘
28//! ```
29//!
30//! # Example
31//!
32//! ```rust,ignore
33//! use hyprstream::inference::{LocalInferenceService, InferenceClient};
34//! use std::sync::Arc;
35//!
36//! // Start the service (returns client handle)
37//! let client = LocalInferenceService::start("/path/to/model", config).await?;
38//!
39//! // Generate text
40//! let result = client.generate(request).await?;
41//!
42//! // Streaming generation
43//! let mut stream = client.generate_stream(request).await?;
44//! while let Some(chunk) = stream.next().await {
45//! print!("{}", chunk?);
46//! }
47//! let stats = stream.stats().await?;
48//!
49//! // Client can be cloned and shared
50//! let client2 = client.clone();
51//! ```
52//!
53//! # Thread Model
54//!
55//! The service runs on a dedicated thread with its own single-threaded tokio
56//! runtime. This isolates GPU operations and handles tch-rs types that contain
57//! raw pointers (not Send). This matches the pattern used by git2db's
58//! LocalService.
59
60mod client;
61mod local;
62mod request;
63mod response;
64
65pub use client::{InferenceClient, InferenceError};
66pub use local::{LocalInferenceClient, LocalInferenceService};
67pub use request::InferenceRequest;
68pub use response::{StreamHandle, StreamStats};