Skip to main content

RuntimeEngine

Trait RuntimeEngine 

Source
pub trait RuntimeEngine: Send + Sync {
    // Required methods
    fn load_model<'life0, 'life1, 'async_trait>(
        &'life0 mut self,
        path: &'life1 Path,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn generate<'life0, 'life1, 'async_trait>(
        &'life0 self,
        prompt: &'life1 str,
        max_tokens: usize,
    ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn generate_with_params<'life0, 'async_trait>(
        &'life0 self,
        request: GenerationRequest,
    ) -> Pin<Box<dyn Future<Output = Result<GenerationResult>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn model_info(&self) -> ModelInfo;
    fn is_loaded(&self) -> bool;

    // Provided methods
    fn apply_chat_template(
        &self,
        messages: &[ChatMessage],
        add_generation_prompt: bool,
        tools: Option<&Value>,
    ) -> Result<String> { ... }
    fn apply_chat_template_with_vars(
        &self,
        messages: &[ChatMessage],
        add_generation_prompt: bool,
        tools: Option<&Value>,
        _enable_thinking: Option<bool>,
        _template_vars_json: Option<&str>,
    ) -> Result<String> { ... }
}
Expand description

Core runtime engine trait - all engines implement this

Required Methods§

Source

fn load_model<'life0, 'life1, 'async_trait>( &'life0 mut self, path: &'life1 Path, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Load a model from the given path

Source

fn generate<'life0, 'life1, 'async_trait>( &'life0 self, prompt: &'life1 str, max_tokens: usize, ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Generate text from a prompt (convenience method)

Source

fn generate_with_params<'life0, 'async_trait>( &'life0 self, request: GenerationRequest, ) -> Pin<Box<dyn Future<Output = Result<GenerationResult>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Generate text with full parameters (main method)

Source

fn model_info(&self) -> ModelInfo

Get loaded model information

Source

fn is_loaded(&self) -> bool

Check if model is loaded and ready

Provided Methods§

Source

fn apply_chat_template( &self, messages: &[ChatMessage], add_generation_prompt: bool, tools: Option<&Value>, ) -> Result<String>

Apply chat template to messages (for template support).

tools is an optional JSON array of tool definitions. When provided, it is passed to the HF chat template as the tools variable so that models with native tool-calling support can format them correctly.

Source

fn apply_chat_template_with_vars( &self, messages: &[ChatMessage], add_generation_prompt: bool, tools: Option<&Value>, _enable_thinking: Option<bool>, _template_vars_json: Option<&str>, ) -> Result<String>

Apply chat template with extended parameters (thinking mode, extra vars).

Implementors§