pub trait InferHandler {
// Required methods
fn handle_generate_stream<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
data: &'life3 GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<(StreamInfo, Continuation)>> + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait;
fn handle_apply_chat_template<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
data: &'life3 ChatTemplateRequest,
) -> Pin<Box<dyn Future<Output = Result<String>> + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait;
fn handle_status<'life0, 'life1, 'life2, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
) -> Pin<Box<dyn Future<Output = Result<ModelStatusResponse>> + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait;
fn handle_embed<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
data: &'life3 EmbedImagesRequest,
) -> Pin<Box<dyn Future<Output = Result<EmbedImagesResponse>> + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait;
}Expand description
Generated handler trait for the infer scope.
Required Methods§
Sourcefn handle_generate_stream<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
data: &'life3 GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<(StreamInfo, Continuation)>> + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait,
fn handle_generate_stream<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
data: &'life3 GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<(StreamInfo, Continuation)>> + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait,
Run inference with automatic domain adaptation. When TTT is enabled, the model adapts to your prompt BEFORE generating — the response is always produced using the adapted weights, even when adaptationStrategy=speculative. Check onlineTrainingMetrics.recommendation in the response, then call ttt.writeback (if true) or ttt.evict (if false) within the pending rollback window (default 60 seconds). If you call generateStream again before resolving a pending adaptation, the new adaptation stacks on top and evict will restore to the state before the first pending adaptation. Pending adaptations auto-evict after the timeout if writeback/evict is not called.
Sourcefn handle_apply_chat_template<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
data: &'life3 ChatTemplateRequest,
) -> Pin<Box<dyn Future<Output = Result<String>> + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait,
fn handle_apply_chat_template<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
data: &'life3 ChatTemplateRequest,
) -> Pin<Box<dyn Future<Output = Result<String>> + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait,
Apply chat template to messages for a loaded model
Sourcefn handle_status<'life0, 'life1, 'life2, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
) -> Pin<Box<dyn Future<Output = Result<ModelStatusResponse>> + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
fn handle_status<'life0, 'life1, 'life2, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
) -> Pin<Box<dyn Future<Output = Result<ModelStatusResponse>> + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
Get detailed status information about a model including online training configuration
Sourcefn handle_embed<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
data: &'life3 EmbedImagesRequest,
) -> Pin<Box<dyn Future<Output = Result<EmbedImagesResponse>> + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait,
fn handle_embed<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
ctx: &'life1 EnvelopeContext,
request_id: u64,
model_ref: &'life2 str,
data: &'life3 EmbedImagesRequest,
) -> Pin<Box<dyn Future<Output = Result<EmbedImagesResponse>> + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait,
Compute embeddings for one or more images. Returns embedding vectors from the model’s vision encoder (e.g. SigLIP). Synchronous — returns all embeddings in a single response.