Index

_

__call__() (tokenizers.tools.EncodingVisualizer method)

A

add_special_tokens() (tokenizers.Tokenizer method) add_tokens() (tokenizers.Tokenizer method) AddedToken (class in tokenizers)

alphabet() (tokenizers.pre_tokenizers.ByteLevel static method) Annotation (class in tokenizers.tools) attention_mask (tokenizers.Encoding attribute)

B

BertNormalizer (class in tokenizers.normalizers) BertPreTokenizer (class in tokenizers.pre_tokenizers) BertProcessing (class in tokenizers.processors) BPE (class in tokenizers.models)

BPEDecoder (class in tokenizers.decoders) BpeTrainer (class in tokenizers.trainers) ByteLevel (class in tokenizers.decoders) (class in tokenizers.pre_tokenizers) (class in tokenizers.processors)

C

char_to_token() (tokenizers.Encoding method) char_to_word() (tokenizers.Encoding method)

CharDelimiterSplit (class in tokenizers.pre_tokenizers) content (tokenizers.AddedToken attribute) CTC (class in tokenizers.decoders)

D

decode() (tokenizers.decoders.Decoder method) (tokenizers.Tokenizer method) decode_batch() (tokenizers.Tokenizer method)

Decoder (class in tokenizers.decoders) decoder (tokenizers.Tokenizer attribute) Digits (class in tokenizers.pre_tokenizers)

E

enable_padding() (tokenizers.Tokenizer method) enable_truncation() (tokenizers.Tokenizer method) encode() (tokenizers.Tokenizer method)

encode_batch() (tokenizers.Tokenizer method) EncodeInput (in module tokenizers) Encoding (class in tokenizers) EncodingVisualizer (class in tokenizers.tools)

F

from_buffer() (tokenizers.Tokenizer static method) from_file() (tokenizers.models.BPE method) (tokenizers.models.WordLevel method) (tokenizers.models.WordPiece method) (tokenizers.Tokenizer static method)

from_pretrained() (tokenizers.Tokenizer static method) from_str() (tokenizers.Tokenizer static method)

G

get_trainer() (tokenizers.models.Model method)

get_vocab() (tokenizers.Tokenizer method) get_vocab_size() (tokenizers.Tokenizer method)

I

id_to_token() (tokenizers.models.Model method) (tokenizers.Tokenizer method)

ids (tokenizers.Encoding attribute) InputSequence (in module tokenizers)

L

Lowercase (class in tokenizers.normalizers)

lstrip (tokenizers.AddedToken attribute)

M

merge() (tokenizers.Encoding static method) Metaspace (class in tokenizers.decoders) (class in tokenizers.pre_tokenizers) Model (class in tokenizers.models) model (tokenizers.Tokenizer attribute) module tokenizers.decoders tokenizers.models tokenizers.normalizers tokenizers.pre_tokenizers tokenizers.processors tokenizers.trainers

N

n_sequences (tokenizers.Encoding attribute) NFC (class in tokenizers.normalizers) NFD (class in tokenizers.normalizers) NFKC (class in tokenizers.normalizers) NFKD (class in tokenizers.normalizers) Nmt (class in tokenizers.normalizers) no_padding() (tokenizers.Tokenizer method)

no_truncation() (tokenizers.Tokenizer method) normalize() (tokenizers.normalizers.Normalizer method) normalize_str() (tokenizers.normalizers.Normalizer method) normalized (tokenizers.AddedToken attribute) Normalizer (class in tokenizers.normalizers) normalizer (tokenizers.Tokenizer attribute) num_special_tokens_to_add() (tokenizers.processors.PostProcessor method) (tokenizers.Tokenizer method)

O

offsets (tokenizers.Encoding attribute)

overflowing (tokenizers.Encoding attribute)

P

pad() (tokenizers.Encoding method) padding (tokenizers.Tokenizer attribute) post_process() (tokenizers.Tokenizer method) post_processor (tokenizers.Tokenizer attribute) PostProcessor (class in tokenizers.processors) pre_tokenize() (tokenizers.pre_tokenizers.PreTokenizer method) pre_tokenize_str() (tokenizers.pre_tokenizers.PreTokenizer method)

pre_tokenizer (tokenizers.Tokenizer attribute) Precompiled (class in tokenizers.normalizers) PreTokenizedEncodeInput (in module tokenizers) PreTokenizedInputSequence (in module tokenizers) PreTokenizer (class in tokenizers.pre_tokenizers) process() (tokenizers.processors.PostProcessor method) Punctuation (class in tokenizers.pre_tokenizers)

R

read_file() (tokenizers.models.BPE static method) (tokenizers.models.WordLevel static method) (tokenizers.models.WordPiece static method)

Replace (class in tokenizers.normalizers) RobertaProcessing (class in tokenizers.processors) rstrip (tokenizers.AddedToken attribute)

S

save() (tokenizers.models.Model method) (tokenizers.Tokenizer method) Sequence (class in tokenizers.normalizers) (class in tokenizers.pre_tokenizers) sequence_ids (tokenizers.Encoding attribute)

set_sequence_id() (tokenizers.Encoding method) single_word (tokenizers.AddedToken attribute) special_tokens_mask (tokenizers.Encoding attribute) Split (class in tokenizers.pre_tokenizers) Strip (class in tokenizers.normalizers) StripAccents (class in tokenizers.normalizers)

T

TemplateProcessing (class in tokenizers.processors) TextEncodeInput (in module tokenizers) TextInputSequence (in module tokenizers) to_str() (tokenizers.Tokenizer method) token_to_chars() (tokenizers.Encoding method) token_to_id() (tokenizers.models.Model method) (tokenizers.Tokenizer method) token_to_sequence() (tokenizers.Encoding method) token_to_word() (tokenizers.Encoding method) tokenize() (tokenizers.models.Model method) Tokenizer (class in tokenizers) tokenizers.decoders module tokenizers.models module

tokenizers.normalizers module tokenizers.pre_tokenizers module tokenizers.processors module tokenizers.trainers module tokens (tokenizers.Encoding attribute) train() (tokenizers.Tokenizer method) train_from_iterator() (tokenizers.Tokenizer method) Trainer (class in tokenizers.trainers) truncate() (tokenizers.Encoding method) truncation (tokenizers.Tokenizer attribute) type_ids (tokenizers.Encoding attribute)

U

UnicodeScripts (class in tokenizers.pre_tokenizers)

Unigram (class in tokenizers.models) UnigramTrainer (class in tokenizers.trainers)

W

Whitespace (class in tokenizers.pre_tokenizers) WhitespaceSplit (class in tokenizers.pre_tokenizers) word_ids (tokenizers.Encoding attribute) word_to_chars() (tokenizers.Encoding method) word_to_tokens() (tokenizers.Encoding method)

WordLevel (class in tokenizers.models) WordLevelTrainer (class in tokenizers.trainers) WordPiece (class in tokenizers.decoders) (class in tokenizers.models) WordPieceTrainer (class in tokenizers.trainers) words (tokenizers.Encoding attribute)