pub struct DefaultTokenizerConfig;Expand description
Default tokenizer configuration for unknown model types
This provides a fallback that doesn’t modify the tokenizer, suitable for models that don’t require special handling.
Trait Implementations§
Source§impl TokenizerConfig for DefaultTokenizerConfig
impl TokenizerConfig for DefaultTokenizerConfig
Source§fn configure_tokenizer(
&self,
_tokenizer: &mut Tokenizer,
_model_vocab_size: usize,
) -> Result<()>
fn configure_tokenizer( &self, _tokenizer: &mut Tokenizer, _model_vocab_size: usize, ) -> Result<()>
Configure the tokenizer for this specific model architecture Read more
Source§fn get_special_tokens(
&self,
_model_vocab_size: usize,
_tokenizer_vocab_size: usize,
) -> Vec<AddedToken>
fn get_special_tokens( &self, _model_vocab_size: usize, _tokenizer_vocab_size: usize, ) -> Vec<AddedToken>
Get special tokens that should be added for this model Read more
Source§fn is_special_token(&self, _token_id: usize, _vocab_size: usize) -> bool
fn is_special_token(&self, _token_id: usize, _vocab_size: usize) -> bool
Check if a token ID represents a special token for this model Read more
Auto Trait Implementations§
impl Freeze for DefaultTokenizerConfig
impl RefUnwindSafe for DefaultTokenizerConfig
impl Send for DefaultTokenizerConfig
impl Sync for DefaultTokenizerConfig
impl Unpin for DefaultTokenizerConfig
impl UnsafeUnpin for DefaultTokenizerConfig
impl UnwindSafe for DefaultTokenizerConfig
Blanket Implementations§
§impl<T> AnySync for T
impl<T> AnySync for T
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
§impl<T> FutureExt for T
impl<T> FutureExt for T
§fn with_context(self, otel_cx: Context) -> WithContext<Self>
fn with_context(self, otel_cx: Context) -> WithContext<Self>
§fn with_current_context(self) -> WithContext<Self>
fn with_current_context(self) -> WithContext<Self>
§impl<T> Identity for Twhere
T: ?Sized,
impl<T> Identity for Twhere
T: ?Sized,
§impl<T> Instrument for T
impl<T> Instrument for T
§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more§impl<T> Pointable for T
impl<T> Pointable for T
§impl<T> PolicyExt for Twhere
T: ?Sized,
impl<T> PolicyExt for Twhere
T: ?Sized,
§impl<T> ServiceExt for T
impl<T> ServiceExt for T
§fn map_response_body<F>(self, f: F) -> MapResponseBody<Self, F>where
Self: Sized,
fn map_response_body<F>(self, f: F) -> MapResponseBody<Self, F>where
Self: Sized,
Apply a transformation to the response body. Read more
§fn trace_for_http(self) -> Trace<Self, SharedClassifier<ServerErrorsAsFailures>>where
Self: Sized,
fn trace_for_http(self) -> Trace<Self, SharedClassifier<ServerErrorsAsFailures>>where
Self: Sized,
High level tracing that classifies responses using HTTP status codes. Read more
§fn trace_for_grpc(self) -> Trace<Self, SharedClassifier<GrpcErrorsAsFailures>>where
Self: Sized,
fn trace_for_grpc(self) -> Trace<Self, SharedClassifier<GrpcErrorsAsFailures>>where
Self: Sized,
High level tracing that classifies responses using gRPC headers. Read more