pub trait InferenceClient: Send + Sync {
Show 16 methods
// Required methods
fn generate<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<GenerationResult, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn generate_stream<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<StreamHandle, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn model_info<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<ModelInfo, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn is_ready<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn apply_chat_template<'life0, 'life1, 'async_trait>(
&'life0 self,
messages: &'life1 [ChatMessage],
add_generation_prompt: bool,
tools: Option<Value>,
) -> Pin<Box<dyn Future<Output = Result<String, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait;
fn create_lora<'life0, 'async_trait>(
&'life0 self,
config: TenantDeltaConfig,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn load_lora<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait;
fn save_lora<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait;
fn unload_lora<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn has_lora<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn set_session<'life0, 'async_trait>(
&'life0 self,
session_id: String,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn clear_session<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn release_session<'life0, 'life1, 'async_trait>(
&'life0 self,
session_id: &'life1 str,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait;
fn embed<'life0, 'life1, 'async_trait>(
&'life0 self,
images: &'life1 [Vec<u8>],
) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait;
fn health_check<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn shutdown<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
}Expand description
Transport-agnostic inference client trait.
Implementations can be in-process (channels) or remote (gRPC, etc.). All methods return owned data to avoid lifetime issues across transports.
Required Methods§
Sourcefn generate<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<GenerationResult, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn generate<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<GenerationResult, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Generate text (non-streaming, blocking until complete).
Sourcefn generate_stream<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<StreamHandle, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn generate_stream<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<StreamHandle, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Start streaming generation, returns handle to receive text chunks.
Sourcefn model_info<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<ModelInfo, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn model_info<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<ModelInfo, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Get information about the loaded model.
Sourcefn is_ready<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn is_ready<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Check if model is loaded and ready.
Sourcefn apply_chat_template<'life0, 'life1, 'async_trait>(
&'life0 self,
messages: &'life1 [ChatMessage],
add_generation_prompt: bool,
tools: Option<Value>,
) -> Pin<Box<dyn Future<Output = Result<String, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn apply_chat_template<'life0, 'life1, 'async_trait>(
&'life0 self,
messages: &'life1 [ChatMessage],
add_generation_prompt: bool,
tools: Option<Value>,
) -> Pin<Box<dyn Future<Output = Result<String, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Apply chat template to format messages for generation.
Sourcefn create_lora<'life0, 'async_trait>(
&'life0 self,
config: TenantDeltaConfig,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn create_lora<'life0, 'async_trait>(
&'life0 self,
config: TenantDeltaConfig,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Initialize LoRA adapter structure for training.
Sourcefn load_lora<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn load_lora<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Load LoRA adapter weights from file.
Sourcefn save_lora<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn save_lora<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Save LoRA adapter weights to file.
Sourcefn unload_lora<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn unload_lora<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Unload current LoRA adapter.
Sourcefn has_lora<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn has_lora<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Check if LoRA is loaded.
Sourcefn set_session<'life0, 'async_trait>(
&'life0 self,
session_id: String,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn set_session<'life0, 'async_trait>(
&'life0 self,
session_id: String,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Set active session for KV cache isolation.
Sourcefn clear_session<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn clear_session<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Clear current session’s KV cache.
Sourcefn release_session<'life0, 'life1, 'async_trait>(
&'life0 self,
session_id: &'life1 str,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn release_session<'life0, 'life1, 'async_trait>(
&'life0 self,
session_id: &'life1 str,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Release a session’s KV cache.
Sourcefn embed<'life0, 'life1, 'async_trait>(
&'life0 self,
images: &'life1 [Vec<u8>],
) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn embed<'life0, 'life1, 'async_trait>(
&'life0 self,
images: &'life1 [Vec<u8>],
) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Compute embeddings for one or more images.
Returns one embedding vector per input image. The image bytes can be raw RGB pixels, PNG, or JPEG encoded data — the model’s preprocessor handles decoding.
Sourcefn health_check<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn health_check<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Check service health.