pub struct GemmaTokenizerConfig;Expand description
Gemma model tokenizer configuration
Trait Implementations§
Source§impl TokenizerConfig for GemmaTokenizerConfig
impl TokenizerConfig for GemmaTokenizerConfig
Source§fn configure_tokenizer(
&self,
tokenizer: &mut Tokenizer,
model_vocab_size: usize,
) -> Result<()>
fn configure_tokenizer( &self, tokenizer: &mut Tokenizer, model_vocab_size: usize, ) -> Result<()>
Configure the tokenizer for this specific model architecture Read more
Source§fn get_special_tokens(
&self,
_model_vocab_size: usize,
_tokenizer_vocab_size: usize,
) -> Vec<AddedToken>
fn get_special_tokens( &self, _model_vocab_size: usize, _tokenizer_vocab_size: usize, ) -> Vec<AddedToken>
Get special tokens that should be added for this model Read more
Source§fn is_special_token(&self, _token_id: usize, _vocab_size: usize) -> bool
fn is_special_token(&self, _token_id: usize, _vocab_size: usize) -> bool
Check if a token ID represents a special token for this model Read more
Auto Trait Implementations§
impl Freeze for GemmaTokenizerConfig
impl RefUnwindSafe for GemmaTokenizerConfig
impl Send for GemmaTokenizerConfig
impl Sync for GemmaTokenizerConfig
impl Unpin for GemmaTokenizerConfig
impl UnsafeUnpin for GemmaTokenizerConfig
impl UnwindSafe for GemmaTokenizerConfig
Blanket Implementations§
§impl<T> AnySync for T
impl<T> AnySync for T
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
§impl<T> FutureExt for T
impl<T> FutureExt for T
§fn with_context(self, otel_cx: Context) -> WithContext<Self>
fn with_context(self, otel_cx: Context) -> WithContext<Self>
§fn with_current_context(self) -> WithContext<Self>
fn with_current_context(self) -> WithContext<Self>
§impl<T> Identity for Twhere
T: ?Sized,
impl<T> Identity for Twhere
T: ?Sized,
§impl<T> Instrument for T
impl<T> Instrument for T
§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more§impl<T> Pointable for T
impl<T> Pointable for T
§impl<T> PolicyExt for Twhere
T: ?Sized,
impl<T> PolicyExt for Twhere
T: ?Sized,
§impl<T> ServiceExt for T
impl<T> ServiceExt for T
§fn map_response_body<F>(self, f: F) -> MapResponseBody<Self, F>where
Self: Sized,
fn map_response_body<F>(self, f: F) -> MapResponseBody<Self, F>where
Self: Sized,
Apply a transformation to the response body. Read more
§fn trace_for_http(self) -> Trace<Self, SharedClassifier<ServerErrorsAsFailures>>where
Self: Sized,
fn trace_for_http(self) -> Trace<Self, SharedClassifier<ServerErrorsAsFailures>>where
Self: Sized,
High level tracing that classifies responses using HTTP status codes. Read more
§fn trace_for_grpc(self) -> Trace<Self, SharedClassifier<GrpcErrorsAsFailures>>where
Self: Sized,
fn trace_for_grpc(self) -> Trace<Self, SharedClassifier<GrpcErrorsAsFailures>>where
Self: Sized,
High level tracing that classifies responses using gRPC headers. Read more