Skip to main content

InferenceClient

Trait InferenceClient 

Source
pub trait InferenceClient: Send + Sync {
Show 16 methods // Required methods fn generate<'life0, 'async_trait>( &'life0 self, request: GenerationRequest, ) -> Pin<Box<dyn Future<Output = Result<GenerationResult, InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait; fn generate_stream<'life0, 'async_trait>( &'life0 self, request: GenerationRequest, ) -> Pin<Box<dyn Future<Output = Result<StreamHandle, InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait; fn model_info<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<ModelInfo, InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait; fn is_ready<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait; fn apply_chat_template<'life0, 'life1, 'async_trait>( &'life0 self, messages: &'life1 [ChatMessage], add_generation_prompt: bool, tools: Option<Value>, ) -> Pin<Box<dyn Future<Output = Result<String, InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait; fn create_lora<'life0, 'async_trait>( &'life0 self, config: TenantDeltaConfig, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait; fn load_lora<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 Path, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait; fn save_lora<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 Path, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait; fn unload_lora<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait; fn has_lora<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait; fn set_session<'life0, 'async_trait>( &'life0 self, session_id: String, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait; fn clear_session<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait; fn release_session<'life0, 'life1, 'async_trait>( &'life0 self, session_id: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait; fn embed<'life0, 'life1, 'async_trait>( &'life0 self, images: &'life1 [Vec<u8>], ) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>, InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait; fn health_check<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait; fn shutdown<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>> where Self: 'async_trait, 'life0: 'async_trait;
}
Expand description

Transport-agnostic inference client trait.

Implementations can be in-process (channels) or remote (gRPC, etc.). All methods return owned data to avoid lifetime issues across transports.

Required Methods§

Source

fn generate<'life0, 'async_trait>( &'life0 self, request: GenerationRequest, ) -> Pin<Box<dyn Future<Output = Result<GenerationResult, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Generate text (non-streaming, blocking until complete).

Source

fn generate_stream<'life0, 'async_trait>( &'life0 self, request: GenerationRequest, ) -> Pin<Box<dyn Future<Output = Result<StreamHandle, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Start streaming generation, returns handle to receive text chunks.

Source

fn model_info<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<ModelInfo, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Get information about the loaded model.

Source

fn is_ready<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Check if model is loaded and ready.

Source

fn apply_chat_template<'life0, 'life1, 'async_trait>( &'life0 self, messages: &'life1 [ChatMessage], add_generation_prompt: bool, tools: Option<Value>, ) -> Pin<Box<dyn Future<Output = Result<String, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Apply chat template to format messages for generation.

Source

fn create_lora<'life0, 'async_trait>( &'life0 self, config: TenantDeltaConfig, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Initialize LoRA adapter structure for training.

Source

fn load_lora<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 Path, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Load LoRA adapter weights from file.

Source

fn save_lora<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 Path, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Save LoRA adapter weights to file.

Source

fn unload_lora<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Unload current LoRA adapter.

Source

fn has_lora<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Check if LoRA is loaded.

Source

fn set_session<'life0, 'async_trait>( &'life0 self, session_id: String, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Set active session for KV cache isolation.

Source

fn clear_session<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Clear current session’s KV cache.

Source

fn release_session<'life0, 'life1, 'async_trait>( &'life0 self, session_id: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Release a session’s KV cache.

Source

fn embed<'life0, 'life1, 'async_trait>( &'life0 self, images: &'life1 [Vec<u8>], ) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Compute embeddings for one or more images.

Returns one embedding vector per input image. The image bytes can be raw RGB pixels, PNG, or JPEG encoded data — the model’s preprocessor handles decoding.

Source

fn health_check<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Check service health.

Source

fn shutdown<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Graceful shutdown signal.

Implementors§