Skip to main content

InferHandler

Trait InferHandler 

Source
pub trait InferHandler {
    // Required methods
    fn handle_generate_stream<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        ctx: &'life1 EnvelopeContext,
        request_id: u64,
        model_ref: &'life2 str,
        data: &'life3 GenerationRequest,
    ) -> Pin<Box<dyn Future<Output = Result<(StreamInfo, Continuation)>> + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait;
    fn handle_apply_chat_template<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        ctx: &'life1 EnvelopeContext,
        request_id: u64,
        model_ref: &'life2 str,
        data: &'life3 ChatTemplateRequest,
    ) -> Pin<Box<dyn Future<Output = Result<String>> + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait;
    fn handle_status<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        ctx: &'life1 EnvelopeContext,
        request_id: u64,
        model_ref: &'life2 str,
    ) -> Pin<Box<dyn Future<Output = Result<ModelStatusResponse>> + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;
    fn handle_embed<'life0, 'life1, 'life2, 'life3, 'async_trait>(
        &'life0 self,
        ctx: &'life1 EnvelopeContext,
        request_id: u64,
        model_ref: &'life2 str,
        data: &'life3 EmbedImagesRequest,
    ) -> Pin<Box<dyn Future<Output = Result<EmbedImagesResponse>> + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait,
             'life3: 'async_trait;
}
Expand description

Generated handler trait for the infer scope.

Required Methods§

Source

fn handle_generate_stream<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, ctx: &'life1 EnvelopeContext, request_id: u64, model_ref: &'life2 str, data: &'life3 GenerationRequest, ) -> Pin<Box<dyn Future<Output = Result<(StreamInfo, Continuation)>> + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Run inference with automatic domain adaptation. When TTT is enabled, the model adapts to your prompt BEFORE generating — the response is always produced using the adapted weights, even when adaptationStrategy=speculative. Check onlineTrainingMetrics.recommendation in the response, then call ttt.writeback (if true) or ttt.evict (if false) within the pending rollback window (default 60 seconds). If you call generateStream again before resolving a pending adaptation, the new adaptation stacks on top and evict will restore to the state before the first pending adaptation. Pending adaptations auto-evict after the timeout if writeback/evict is not called.

Source

fn handle_apply_chat_template<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, ctx: &'life1 EnvelopeContext, request_id: u64, model_ref: &'life2 str, data: &'life3 ChatTemplateRequest, ) -> Pin<Box<dyn Future<Output = Result<String>> + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Apply chat template to messages for a loaded model

Source

fn handle_status<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, ctx: &'life1 EnvelopeContext, request_id: u64, model_ref: &'life2 str, ) -> Pin<Box<dyn Future<Output = Result<ModelStatusResponse>> + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Get detailed status information about a model including online training configuration

Source

fn handle_embed<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, ctx: &'life1 EnvelopeContext, request_id: u64, model_ref: &'life2 str, data: &'life3 EmbedImagesRequest, ) -> Pin<Box<dyn Future<Output = Result<EmbedImagesResponse>> + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Compute embeddings for one or more images. Returns embedding vectors from the model’s vision encoder (e.g. SigLIP). Synchronous — returns all embeddings in a single response.

Implementors§