Skip to main content

LocalInferenceClient

Struct LocalInferenceClient 

Source
pub struct LocalInferenceClient { /* private fields */ }
Expand description

Lightweight, cloneable client handle for the inference service.

Clients can be cloned and shared across tasks. All clients communicate with the same underlying service instance.

Trait Implementations§

Source§

impl Clone for LocalInferenceClient

Source§

fn clone(&self) -> LocalInferenceClient

Returns a duplicate of the value. Read more
1.0.0 (const: unstable) · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more
Source§

impl InferenceClient for LocalInferenceClient

Source§

fn generate<'life0, 'async_trait>( &'life0 self, request: GenerationRequest, ) -> Pin<Box<dyn Future<Output = Result<GenerationResult, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Generate text (non-streaming, blocking until complete).
Source§

fn generate_stream<'life0, 'async_trait>( &'life0 self, request: GenerationRequest, ) -> Pin<Box<dyn Future<Output = Result<StreamHandle, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Start streaming generation, returns handle to receive text chunks.
Source§

fn model_info<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<ModelInfo, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Get information about the loaded model.
Source§

fn is_ready<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Check if model is loaded and ready.
Source§

fn apply_chat_template<'life0, 'life1, 'async_trait>( &'life0 self, messages: &'life1 [ChatMessage], add_generation_prompt: bool, tools: Option<Value>, ) -> Pin<Box<dyn Future<Output = Result<String, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Apply chat template to format messages for generation.
Source§

fn create_lora<'life0, 'async_trait>( &'life0 self, config: TenantDeltaConfig, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Initialize LoRA adapter structure for training.
Source§

fn load_lora<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 Path, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Load LoRA adapter weights from file.
Source§

fn save_lora<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 Path, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Save LoRA adapter weights to file.
Source§

fn unload_lora<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Unload current LoRA adapter.
Source§

fn has_lora<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Check if LoRA is loaded.
Source§

fn set_session<'life0, 'async_trait>( &'life0 self, session_id: String, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Set active session for KV cache isolation.
Source§

fn clear_session<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Clear current session’s KV cache.
Source§

fn release_session<'life0, 'life1, 'async_trait>( &'life0 self, session_id: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Release a session’s KV cache.
Source§

fn embed<'life0, 'life1, 'async_trait>( &'life0 self, images: &'life1 [Vec<u8>], ) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<f32>>, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Compute embeddings for one or more images. Read more
Source§

fn health_check<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Check service health.
Source§

fn shutdown<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Graceful shutdown signal.

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
§

impl<T> Any for T
where T: Any,

§

fn into_any(self: Box<T>) -> Box<dyn Any>

§

fn into_any_rc(self: Rc<T>) -> Rc<dyn Any>

§

fn type_name(&self) -> &'static str

§

impl<T> AnySync for T
where T: Any + Send + Sync,

§

fn into_any_arc(self: Arc<T>) -> Arc<dyn Any + Sync + Send> ⓘ

§

impl<'a, T, E> AsTaggedExplicit<'a, E> for T
where T: 'a,

§

fn explicit(self, class: Class, tag: u32) -> TaggedParser<'a, Explicit, Self, E>

§

impl<'a, T, E> AsTaggedImplicit<'a, E> for T
where T: 'a,

§

fn implicit( self, class: Class, constructed: bool, tag: u32, ) -> TaggedParser<'a, Implicit, Self, E>

Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> CloneToUninit for T
where T: Clone,

Source§

unsafe fn clone_to_uninit(&self, dest: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)
Performs copy-assignment from self to dest. Read more
Source§

impl<T> DynClone for T
where T: Clone,

Source§

fn __clone_box(&self, _: Private) -> *mut ()

Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

§

impl<T> FromRef<T> for T
where T: Clone,

§

fn from_ref(input: &T) -> T

Converts to this type from a reference to the input type.
§

impl<T> FutureExt for T

§

fn with_context(self, otel_cx: Context) -> WithContext<Self>

Attaches the provided Context to this type, returning a WithContext wrapper. Read more
§

fn with_current_context(self) -> WithContext<Self>

Attaches the current Context to this type, returning a WithContext wrapper. Read more
§

impl<T, W> HasTypeWitness<W> for T
where W: MakeTypeWitness<Arg = T>, T: ?Sized,

§

const WITNESS: W = W::MAKE

A constant of the type witness
§

impl<T> Identity for T
where T: ?Sized,

§

const TYPE_EQ: TypeEq<T, <T as Identity>::Type> = TypeEq::NEW

Proof that Self is the same type as Self::Type, provides methods for casting between Self and Self::Type.
§

type Type = T

The same type as Self, used to emulate type equality bounds (T == U) with associated type equality constraints (T: Identity<Type = U>).
§

impl<T> Instrument for T

§

fn instrument(self, span: Span) -> Instrumented<Self>

Instruments this type with the provided [Span], returning an Instrumented wrapper. Read more
§

fn in_current_span(self) -> Instrumented<Self>

Instruments this type with the current Span, returning an Instrumented wrapper. Read more
Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> IntoEither for T

Source§

fn into_either(self, into_left: bool) -> Either<Self, Self>

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

impl<T> IntoRequest<T> for T

Source§

fn into_request(self) -> Request<T>

Wrap the input message T in a tonic::Request
§

impl<T> JaggedSlice<T> for T

§

type Index = Index2

§

fn push_into(self, array: &mut Jagged<T>)

§

fn insert_into(self, index: <T as JaggedSlice<T>>::Index, array: &mut Jagged<T>)

§

impl<T> Pointable for T

§

const ALIGN: usize

The alignment of pointer.
§

type Init = T

The type for initializers.
§

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more
§

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more
§

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more
§

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more
§

impl<T> PolicyExt for T
where T: ?Sized,

§

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns [Action::Follow] only if self and other return Action::Follow. Read more
§

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns [Action::Follow] if either self or other returns Action::Follow. Read more
Source§

impl<T> Same for T

Source§

type Output = T

Should always be Self
§

impl<T> ServiceExt for T

§

fn map_response_body<F>(self, f: F) -> MapResponseBody<Self, F>
where Self: Sized,

Apply a transformation to the response body. Read more
§

fn trace_for_http(self) -> Trace<Self, SharedClassifier<ServerErrorsAsFailures>>
where Self: Sized,

High level tracing that classifies responses using HTTP status codes. Read more
§

fn trace_for_grpc(self) -> Trace<Self, SharedClassifier<GrpcErrorsAsFailures>>
where Self: Sized,

High level tracing that classifies responses using gRPC headers. Read more
§

fn follow_redirects(self) -> FollowRedirect<Self>
where Self: Sized,

Follow redirect resposes using the Standard policy. Read more
Source§

impl<T> ToOwned for T
where T: Clone,

Source§

type Owned = T

The resulting type after obtaining ownership.
Source§

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more
Source§

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = Infallible

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

§

fn vzip(self) -> V

§

impl<T> WithSubscriber for T

§

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a [WithDispatch] wrapper. Read more
§

fn with_current_subscriber(self) -> WithDispatch<Self>

Attaches the current default Subscriber to this type, returning a [WithDispatch] wrapper. Read more
§

impl<T> ErasedDestructor for T
where T: 'static,

§

impl<T> MaybeSend for T
where T: Send,

§

impl<T> MaybeSend for T
where T: Send,

§

impl<T> MaybeSync for T
where T: Sync,

§

impl<T> MaybeSync for T
where T: Sync,

§

impl<T> SendSyncUnwindSafe for T
where T: Send + Sync + UnwindSafe + ?Sized,

Source§

impl<T> TerminalState for T
where T: Clone + Send + Sync,