pub trait RuntimeEngine: Send + Sync {
// Required methods
fn load_model<'life0, 'life1, 'async_trait>(
&'life0 mut self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait;
fn generate<'life0, 'life1, 'async_trait>(
&'life0 self,
prompt: &'life1 str,
max_tokens: usize,
) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait;
fn generate_with_params<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<GenerationResult>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn model_info(&self) -> ModelInfo;
fn is_loaded(&self) -> bool;
// Provided methods
fn apply_chat_template(
&self,
messages: &[ChatMessage],
add_generation_prompt: bool,
tools: Option<&Value>,
) -> Result<String> { ... }
fn apply_chat_template_with_vars(
&self,
messages: &[ChatMessage],
add_generation_prompt: bool,
tools: Option<&Value>,
_enable_thinking: Option<bool>,
_template_vars_json: Option<&str>,
) -> Result<String> { ... }
}Expand description
Core runtime engine trait - all engines implement this
Required Methods§
Sourcefn load_model<'life0, 'life1, 'async_trait>(
&'life0 mut self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn load_model<'life0, 'life1, 'async_trait>(
&'life0 mut self,
path: &'life1 Path,
) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Load a model from the given path
Sourcefn generate<'life0, 'life1, 'async_trait>(
&'life0 self,
prompt: &'life1 str,
max_tokens: usize,
) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn generate<'life0, 'life1, 'async_trait>(
&'life0 self,
prompt: &'life1 str,
max_tokens: usize,
) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Generate text from a prompt (convenience method)
Sourcefn generate_with_params<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<GenerationResult>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn generate_with_params<'life0, 'async_trait>(
&'life0 self,
request: GenerationRequest,
) -> Pin<Box<dyn Future<Output = Result<GenerationResult>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Generate text with full parameters (main method)
Sourcefn model_info(&self) -> ModelInfo
fn model_info(&self) -> ModelInfo
Get loaded model information
Provided Methods§
Sourcefn apply_chat_template(
&self,
messages: &[ChatMessage],
add_generation_prompt: bool,
tools: Option<&Value>,
) -> Result<String>
fn apply_chat_template( &self, messages: &[ChatMessage], add_generation_prompt: bool, tools: Option<&Value>, ) -> Result<String>
Apply chat template to messages (for template support).
tools is an optional JSON array of tool definitions. When provided, it is
passed to the HF chat template as the tools variable so that models with
native tool-calling support can format them correctly.