pub struct LocalInferenceClient { /* private fields */ }Expand description
Lightweight, cloneable client handle for the inference service.
Clients can be cloned and shared across tasks. All clients communicate with the same underlying service instance.
Trait Implementations§
Source§impl Clone for LocalInferenceClient
impl Clone for LocalInferenceClient
Source§fn clone(&self) -> LocalInferenceClient
fn clone(&self) -> LocalInferenceClient
Returns a duplicate of the value. Read more
1.0.0 (const: unstable) · Source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
Performs copy-assignment from
source. Read moreSource§impl InferenceClient for LocalInferenceClient
impl InferenceClient for LocalInferenceClient
Source§fn generate<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<GenerationResult, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn generate<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<GenerationResult, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Generate text (non-streaming, blocking until complete).
Source§fn generate_stream<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<StreamHandle, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn generate_stream<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<StreamHandle, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Start streaming generation, returns handle to receive text chunks.
Source§fn model_info<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<ModelInfo, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn model_info<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<ModelInfo, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Get information about the loaded model.
Source§fn is_ready<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn is_ready<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Check if model is loaded and ready.
Source§fn apply_chat_template<'life0, 'life1, 'async_trait>(
&'life0 self,
messages: &'life1 [ChatMessage],
add_generation_prompt: bool,
tools: Option<Value>,
) -> Pin<Box<dyn Future<Output = Result<String, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn apply_chat_template<'life0, 'life1, 'async_trait>(
&'life0 self,
messages: &'life1 [ChatMessage],
add_generation_prompt: bool,
tools: Option<Value>,
) -> Pin<Box<dyn Future<Output = Result<String, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Apply chat template to format messages for generation.
Source§fn create_lora<'life0, 'async_trait>(
&'life0 self,
config: TenantDeltaConfig,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn create_lora<'life0, 'async_trait>(
&'life0 self,
config: TenantDeltaConfig,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Initialize LoRA adapter structure for training.
Source§fn load_lora<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn load_lora<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Load LoRA adapter weights from file.
Source§fn save_lora<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn save_lora<'life0, 'life1, 'async_trait>(
&'life0 self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Save LoRA adapter weights to file.
Source§fn unload_lora<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn unload_lora<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Unload current LoRA adapter.
Source§fn has_lora<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn has_lora<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Check if LoRA is loaded.
Source§fn set_session<'life0, 'async_trait>(
&'life0 self,
session_id: String,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn set_session<'life0, 'async_trait>(
&'life0 self,
session_id: String,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Set active session for KV cache isolation.
Source§fn clear_session<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn clear_session<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Clear current session’s KV cache.
Source§fn release_session<'life0, 'life1, 'async_trait>(
&'life0 self,
session_id: &'life1 str,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn release_session<'life0, 'life1, 'async_trait>(
&'life0 self,
session_id: &'life1 str,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Release a session’s KV cache.
Source§fn embed<'life0, 'life1, 'async_trait>(
&'life0 self,
images: &'life1 [Vec<u8>],
) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn embed<'life0, 'life1, 'async_trait>(
&'life0 self,
images: &'life1 [Vec<u8>],
) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>, InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Compute embeddings for one or more images. Read more
Source§fn health_check<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn health_check<'life0, 'async_trait>(
&'life0 self,
) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Check service health.
Auto Trait Implementations§
impl Freeze for LocalInferenceClient
impl RefUnwindSafe for LocalInferenceClient
impl Send for LocalInferenceClient
impl Sync for LocalInferenceClient
impl Unpin for LocalInferenceClient
impl UnsafeUnpin for LocalInferenceClient
impl UnwindSafe for LocalInferenceClient
Blanket Implementations§
§impl<T> AnySync for T
impl<T> AnySync for T
§impl<'a, T, E> AsTaggedExplicit<'a, E> for Twhere
T: 'a,
impl<'a, T, E> AsTaggedExplicit<'a, E> for Twhere
T: 'a,
§impl<'a, T, E> AsTaggedImplicit<'a, E> for Twhere
T: 'a,
impl<'a, T, E> AsTaggedImplicit<'a, E> for Twhere
T: 'a,
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
§impl<T> FutureExt for T
impl<T> FutureExt for T
§fn with_context(self, otel_cx: Context) -> WithContext<Self>
fn with_context(self, otel_cx: Context) -> WithContext<Self>
§fn with_current_context(self) -> WithContext<Self>
fn with_current_context(self) -> WithContext<Self>
§impl<T> Identity for Twhere
T: ?Sized,
impl<T> Identity for Twhere
T: ?Sized,
§impl<T> Instrument for T
impl<T> Instrument for T
§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§impl<T> IntoRequest<T> for T
impl<T> IntoRequest<T> for T
Source§fn into_request(self) -> Request<T>
fn into_request(self) -> Request<T>
Wrap the input message
T in a tonic::Request§impl<T> JaggedSlice<T> for T
impl<T> JaggedSlice<T> for T
§impl<T> Pointable for T
impl<T> Pointable for T
§impl<T> PolicyExt for Twhere
T: ?Sized,
impl<T> PolicyExt for Twhere
T: ?Sized,
§impl<T> ServiceExt for T
impl<T> ServiceExt for T
§fn map_response_body<F>(self, f: F) -> MapResponseBody<Self, F>where
Self: Sized,
fn map_response_body<F>(self, f: F) -> MapResponseBody<Self, F>where
Self: Sized,
Apply a transformation to the response body. Read more
§fn trace_for_http(self) -> Trace<Self, SharedClassifier<ServerErrorsAsFailures>>where
Self: Sized,
fn trace_for_http(self) -> Trace<Self, SharedClassifier<ServerErrorsAsFailures>>where
Self: Sized,
High level tracing that classifies responses using HTTP status codes. Read more
§fn trace_for_grpc(self) -> Trace<Self, SharedClassifier<GrpcErrorsAsFailures>>where
Self: Sized,
fn trace_for_grpc(self) -> Trace<Self, SharedClassifier<GrpcErrorsAsFailures>>where
Self: Sized,
High level tracing that classifies responses using gRPC headers. Read more