tokenizers
Tokenizers are used to prepare textual inputs for a model.
Example: Create an AutoTokenizer
and use it to tokenize a sentence.
This will automatically detect the tokenizer type based on the tokenizer class defined in tokenizer.json
.
import { AutoTokenizer } from '@huggingface/transformers';
const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased');
const { input_ids } = await tokenizer('I love transformers!');
// Tensor {
// data: BigInt64Array(6) [101n, 1045n, 2293n, 19081n, 999n, 102n],
// dims: [1, 6],
// type: 'int64',
// size: 6,
// }
- tokenizers
- static
- .TokenizerModel β
Callable
new TokenizerModel(config)
- instance
.vocab
:Array.<string>
.tokens_to_ids
:Map.<string, number>
.fuse_unk
:boolean
._call(tokens)
βArray.<string>
.encode(tokens)
βArray.<string>
.convert_tokens_to_ids(tokens)
βArray.<number>
.convert_ids_to_tokens(ids)
βArray.<string>
- static
.fromConfig(config, ...args)
βTokenizerModel
- .PreTrainedTokenizer
new PreTrainedTokenizer(tokenizerJSON, tokenizerConfig)
- instance
.added_tokens
:Array.<AddedToken>
.remove_space
:boolean
._call(text, options)
βBatchEncoding
._encode_text(text)
βArray<string>
|null
._tokenize_helper(text, options)
β*
.tokenize(text, options)
βArray.<string>
.encode(text, options)
βArray.<number>
.batch_decode(batch, decode_args)
βArray.<string>
.decode(token_ids, [decode_args])
βstring
.decode_single(token_ids, decode_args)
βstring
.get_chat_template(options)
βstring
.apply_chat_template(conversation, options)
βstring
|Tensor
|Array<number>
|Array<Array<number>>
|BatchEncoding
- static
.from_pretrained(pretrained_model_name_or_path, options)
βPromise.<PreTrainedTokenizer>
- .BertTokenizer β
PreTrainedTokenizer
- .AlbertTokenizer β
PreTrainedTokenizer
- .NllbTokenizer
- .M2M100Tokenizer
- .WhisperTokenizer β
PreTrainedTokenizer
._decode_asr(sequences, options)
β*
.decode()
:*
- .MarianTokenizer
- .AutoTokenizer
.from_pretrained(pretrained_model_name_or_path, options)
βPromise.<PreTrainedTokenizer>
.is_chinese_char(cp)
βboolean
- .TokenizerModel β
- inner
- ~AddedToken
- ~WordPieceTokenizer β
TokenizerModel
new WordPieceTokenizer(config)
.tokens_to_ids
:Map.<string, number>
.unk_token_id
:number
.unk_token
:string
.max_input_chars_per_word
:number
.vocab
:Array.<string>
.encode(tokens)
βArray.<string>
- ~Unigram β
TokenizerModel
new Unigram(config, moreConfig)
.populateNodes(lattice)
.tokenize(normalized)
βArray.<string>
.encode(tokens)
βArray.<string>
- ~BPE β
TokenizerModel
new BPE(config)
.tokens_to_ids
:Map.<string, number>
.merges
:*
.config.merges
:*
.cache
:Map.<string, Array<string>>
.bpe(token)
βArray.<string>
.encode(tokens)
βArray.<string>
- ~LegacyTokenizerModel
new LegacyTokenizerModel(config, moreConfig)
.tokens_to_ids
:Map.<string, number>
- ~Normalizer
new Normalizer(config)
- instance
.normalize(text)
βstring
._call(text)
βstring
- static
.fromConfig(config)
βNormalizer
- ~Replace β
Normalizer
.normalize(text)
βstring
- ~NFC β
Normalizer
.normalize(text)
βstring
- ~NFKC β
Normalizer
.normalize(text)
βstring
- ~NFKD β
Normalizer
.normalize(text)
βstring
- ~StripNormalizer
.normalize(text)
βstring
- ~StripAccents β
Normalizer
.normalize(text)
βstring
- ~Lowercase β
Normalizer
.normalize(text)
βstring
- ~Prepend β
Normalizer
.normalize(text)
βstring
- ~NormalizerSequence β
Normalizer
new NormalizerSequence(config)
.normalize(text)
βstring
- ~BertNormalizer β
Normalizer
._tokenize_chinese_chars(text)
βstring
.stripAccents(text)
βstring
.normalize(text)
βstring
- ~PreTokenizer β
Callable
- instance
.pre_tokenize_text(text, [options])
βArray.<string>
.pre_tokenize(text, [options])
βArray.<string>
._call(text, [options])
βArray.<string>
- static
.fromConfig(config)
βPreTokenizer
- instance
- ~BertPreTokenizer β
PreTokenizer
new BertPreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
- ~ByteLevelPreTokenizer β
PreTokenizer
new ByteLevelPreTokenizer(config)
.add_prefix_space
:boolean
.trim_offsets
:boolean
.use_regex
:boolean
.pre_tokenize_text(text, [options])
βArray.<string>
- ~SplitPreTokenizer β
PreTokenizer
new SplitPreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
- ~PunctuationPreTokenizer β
PreTokenizer
new PunctuationPreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
- ~DigitsPreTokenizer β
PreTokenizer
new DigitsPreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
- ~PostProcessor β
Callable
new PostProcessor(config)
- instance
.post_process(tokens, ...args)
βPostProcessedOutput
._call(tokens, ...args)
βPostProcessedOutput
- static
.fromConfig(config)
βPostProcessor
- ~BertProcessing
new BertProcessing(config)
.post_process(tokens, [tokens_pair])
βPostProcessedOutput
- ~TemplateProcessing β
PostProcessor
new TemplateProcessing(config)
.post_process(tokens, [tokens_pair])
βPostProcessedOutput
- ~ByteLevelPostProcessor β
PostProcessor
.post_process(tokens, [tokens_pair])
βPostProcessedOutput
- ~PostProcessorSequence
new PostProcessorSequence(config)
.post_process(tokens, [tokens_pair])
βPostProcessedOutput
- ~Decoder β
Callable
new Decoder(config)
- instance
.added_tokens
:Array.<AddedToken>
._call(tokens)
βstring
.decode(tokens)
βstring
.decode_chain(tokens)
βArray.<string>
- static
.fromConfig(config)
βDecoder
- ~FuseDecoder
.decode_chain()
:*
- ~WordPieceDecoder β
Decoder
- ~ByteLevelDecoder β
Decoder
- ~CTCDecoder
.convert_tokens_to_string(tokens)
βstring
.decode_chain()
:*
- ~DecoderSequence β
Decoder
- ~MetaspacePreTokenizer β
PreTokenizer
new MetaspacePreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
- ~MetaspaceDecoder β
Decoder
- ~Precompiled β
Normalizer
new Precompiled(config)
.normalize(text)
βstring
- ~PreTokenizerSequence β
PreTokenizer
new PreTokenizerSequence(config)
.pre_tokenize_text(text, [options])
βArray.<string>
- ~WhitespacePreTokenizer
new WhitespacePreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
- ~WhitespaceSplit β
PreTokenizer
new WhitespaceSplit(config)
.pre_tokenize_text(text, [options])
βArray.<string>
- ~ReplacePreTokenizer
new ReplacePreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
~BYTES_TO_UNICODE
βObject
~loadTokenizer(pretrained_model_name_or_path, options)
βPromise.<Array<any>>
~regexSplit(text, regex)
βArray.<string>
~createPattern(pattern, invert)
βRegExp
|null
~objectToMap(obj)
βMap.<string, any>
~prepareTensorForDecode(tensor)
βArray.<number>
~clean_up_tokenization(text)
βstring
~remove_accents(text)
βstring
~lowercase_and_remove_accent(text)
βstring
~whitespace_split(text)
βArray.<string>
~PretrainedTokenizerOptions
:Object
~BPENode
:Object
~SplitDelimiterBehavior
:βremovedβ
|βisolatedβ
|βmergedWithPreviousβ
|βmergedWithNextβ
|βcontiguousβ
~PostProcessedOutput
:Object
~EncodingSingle
:Object
~Message
:Object
~BatchEncoding
:Array<number>
|Array<Array<number>>
|Tensor
- static
tokenizers.TokenizerModel β <code> Callable </code>
Abstract base class for tokenizer models.
Kind: static class of tokenizers
Extends: Callable
- .TokenizerModel β
Callable
new TokenizerModel(config)
- instance
.vocab
:Array.<string>
.tokens_to_ids
:Map.<string, number>
.fuse_unk
:boolean
._call(tokens)
βArray.<string>
.encode(tokens)
βArray.<string>
.convert_tokens_to_ids(tokens)
βArray.<number>
.convert_ids_to_tokens(ids)
βArray.<string>
- static
.fromConfig(config, ...args)
βTokenizerModel
new TokenizerModel(config)
Creates a new instance of TokenizerModel.
Param | Type | Description |
---|---|---|
config | Object | The configuration object for the TokenizerModel. |
tokenizerModel.vocab : <code> Array. < string > </code>
Kind: instance property of TokenizerModel
tokenizerModel.tokens_to_ids : <code> Map. < string, number > </code>
A mapping of tokens to ids.
Kind: instance property of TokenizerModel
tokenizerModel.fuse_unk : <code> boolean </code>
Whether to fuse unknown tokens when encoding. Defaults to false.
Kind: instance property of TokenizerModel
tokenizerModel._call(tokens) β <code> Array. < string > </code>
Internal function to call the TokenizerModel instance.
Kind: instance method of TokenizerModel
Overrides: _call
Returns: Array.<string>
- The encoded tokens.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | The tokens to encode. |
tokenizerModel.encode(tokens) β <code> Array. < string > </code>
Encodes a list of tokens into a list of token IDs.
Kind: instance method of TokenizerModel
Returns: Array.<string>
- The encoded tokens.
Throws:
- Will throw an error if not implemented in a subclass.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | The tokens to encode. |
tokenizerModel.convert_tokens_to_ids(tokens) β <code> Array. < number > </code>
Converts a list of tokens into a list of token IDs.
Kind: instance method of TokenizerModel
Returns: Array.<number>
- The converted token IDs.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | The tokens to convert. |
tokenizerModel.convert_ids_to_tokens(ids) β <code> Array. < string > </code>
Converts a list of token IDs into a list of tokens.
Kind: instance method of TokenizerModel
Returns: Array.<string>
- The converted tokens.
Param | Type | Description |
---|---|---|
ids | Array<number> | Array<bigint> | The token IDs to convert. |
TokenizerModel.fromConfig(config, ...args) β <code> TokenizerModel </code>
Instantiates a new TokenizerModel instance based on the configuration object provided.
Kind: static method of TokenizerModel
Returns: TokenizerModel
- A new instance of a TokenizerModel.
Throws:
- Will throw an error if the TokenizerModel type in the config is not recognized.
Param | Type | Description |
---|---|---|
config | Object | The configuration object for the TokenizerModel. |
...args | * | Optional arguments to pass to the specific TokenizerModel constructor. |
tokenizers.PreTrainedTokenizer
Kind: static class of tokenizers
- .PreTrainedTokenizer
new PreTrainedTokenizer(tokenizerJSON, tokenizerConfig)
- instance
.added_tokens
:Array.<AddedToken>
.remove_space
:boolean
._call(text, options)
βBatchEncoding
._encode_text(text)
βArray<string>
|null
._tokenize_helper(text, options)
β*
.tokenize(text, options)
βArray.<string>
.encode(text, options)
βArray.<number>
.batch_decode(batch, decode_args)
βArray.<string>
.decode(token_ids, [decode_args])
βstring
.decode_single(token_ids, decode_args)
βstring
.get_chat_template(options)
βstring
.apply_chat_template(conversation, options)
βstring
|Tensor
|Array<number>
|Array<Array<number>>
|BatchEncoding
- static
.from_pretrained(pretrained_model_name_or_path, options)
βPromise.<PreTrainedTokenizer>
new PreTrainedTokenizer(tokenizerJSON, tokenizerConfig)
Create a new PreTrainedTokenizer instance.
Param | Type | Description |
---|---|---|
tokenizerJSON | Object | The JSON of the tokenizer. |
tokenizerConfig | Object | The config of the tokenizer. |
preTrainedTokenizer.added_tokens : <code> Array. < AddedToken > </code>
Kind: instance property of PreTrainedTokenizer
preTrainedTokenizer.remove_space : <code> boolean </code>
Whether or not to strip the text when tokenizing (removing excess spaces before and after the string).
Kind: instance property of PreTrainedTokenizer
preTrainedTokenizer._call(text, options) β <code> BatchEncoding </code>
Encode/tokenize the given text(s).
Kind: instance method of PreTrainedTokenizer
Returns: BatchEncoding
- Object to be passed to the model.
Param | Type | Default | Description |
---|---|---|---|
text | string | Array<string> | The text to tokenize. | |
options | Object | An optional object containing the following properties: | |
[options.text_pair] | string | Array<string> | null | Optional second sequence to be encoded. If set, must be the same type as text. |
[options.padding] | boolean | 'max_length' | false | Whether to pad the input sequences. |
[options.add_special_tokens] | boolean | true | Whether or not to add the special tokens associated with the corresponding model. |
[options.truncation] | boolean |
| Whether to truncate the input sequences. |
[options.max_length] | number |
| Maximum length of the returned list and optionally padding length. |
[options.return_tensor] | boolean | true | Whether to return the results as Tensors or arrays. |
[options.return_token_type_ids] | boolean |
| Whether to return the token type ids. |
preTrainedTokenizer._encode_text(text) β <code> Array < string > </code> | <code> null </code>
Encodes a single text using the preprocessor pipeline of the tokenizer.
Kind: instance method of PreTrainedTokenizer
Returns: Array<string>
| null
- The encoded tokens.
Param | Type | Description |
---|---|---|
text | string | null | The text to encode. |
preTrainedTokenizer._tokenize_helper(text, options) β <code> * </code>
Internal helper function to tokenize a text, and optionally a pair of texts.
Kind: instance method of PreTrainedTokenizer
Returns: *
- An object containing the tokens and optionally the token type IDs.
Param | Type | Default | Description |
---|---|---|---|
text | string | The text to tokenize. | |
options | Object | An optional object containing the following properties: | |
[options.pair] | string | null | The optional second text to tokenize. |
[options.add_special_tokens] | boolean | false | Whether or not to add the special tokens associated with the corresponding model. |
preTrainedTokenizer.tokenize(text, options) β <code> Array. < string > </code>
Converts a string into a sequence of tokens.
Kind: instance method of PreTrainedTokenizer
Returns: Array.<string>
- The list of tokens.
Param | Type | Default | Description |
---|---|---|---|
text | string | The sequence to be encoded. | |
options | Object | An optional object containing the following properties: | |
[options.pair] | string | A second sequence to be encoded with the first. | |
[options.add_special_tokens] | boolean | false | Whether or not to add the special tokens associated with the corresponding model. |
preTrainedTokenizer.encode(text, options) β <code> Array. < number > </code>
Encodes a single text or a pair of texts using the modelβs tokenizer.
Kind: instance method of PreTrainedTokenizer
Returns: Array.<number>
- An array of token IDs representing the encoded text(s).
Param | Type | Default | Description |
---|---|---|---|
text | string | The text to encode. | |
options | Object | An optional object containing the following properties: | |
[options.text_pair] | string | null | The optional second text to encode. |
[options.add_special_tokens] | boolean | true | Whether or not to add the special tokens associated with the corresponding model. |
[options.return_token_type_ids] | boolean |
| Whether to return token_type_ids. |
preTrainedTokenizer.batch_decode(batch, decode_args) β <code> Array. < string > </code>
Decode a batch of tokenized sequences.
Kind: instance method of PreTrainedTokenizer
Returns: Array.<string>
- List of decoded sequences.
Param | Type | Description |
---|---|---|
batch | Array<Array<number>> | Tensor | List/Tensor of tokenized input sequences. |
decode_args | Object | (Optional) Object with decoding arguments. |
preTrainedTokenizer.decode(token_ids, [decode_args]) β <code> string </code>
Decodes a sequence of token IDs back to a string.
Kind: instance method of PreTrainedTokenizer
Returns: string
- The decoded string.
Throws:
Error
If `token_ids` is not a non-empty array of integers.
Param | Type | Default | Description |
---|---|---|---|
token_ids | Array<number> | Array<bigint> | Tensor | List/Tensor of token IDs to decode. | |
[decode_args] | Object | {} | |
[decode_args.skip_special_tokens] | boolean | false | If true, special tokens are removed from the output string. |
[decode_args.clean_up_tokenization_spaces] | boolean | true | If true, spaces before punctuations and abbreviated forms are removed. |
preTrainedTokenizer.decode_single(token_ids, decode_args) β <code> string </code>
Decode a single list of token ids to a string.
Kind: instance method of PreTrainedTokenizer
Returns: string
- The decoded string
Param | Type | Default | Description |
---|---|---|---|
token_ids | Array<number> | Array<bigint> | List of token ids to decode | |
decode_args | Object | Optional arguments for decoding | |
[decode_args.skip_special_tokens] | boolean | false | Whether to skip special tokens during decoding |
[decode_args.clean_up_tokenization_spaces] | boolean |
| Whether to clean up tokenization spaces during decoding.
If null, the value is set to |
preTrainedTokenizer.get_chat_template(options) β <code> string </code>
Retrieve the chat template string used for tokenizing chat messages. This template is used
internally by the apply_chat_template
method and can also be used externally to retrieve the modelβs chat
template for better generation tracking.
Kind: instance method of PreTrainedTokenizer
Returns: string
- The chat template string.
Param | Type | Default | Description |
---|---|---|---|
options | Object | An optional object containing the following properties: | |
[options.chat_template] | string | null | A Jinja template or the name of a template to use for this conversion. It is usually not necessary to pass anything to this argument, as the model's template will be used by default. |
[options.tools] | Array.<Object> |
| A list of tools (callable functions) that will be accessible to the model. If the template does not support function calling, this argument will have no effect. Each tool should be passed as a JSON Schema, giving the name, description and argument types for the tool. See our chat templating guide for more information. |
preTrainedTokenizer.apply_chat_template(conversation, options) β <code> string </code> | <code> Tensor </code> | <code> Array < number > </code> | <code> Array < Array < number > > </code> | <code> BatchEncoding </code>
Converts a list of message objects with "role"
and "content"
keys to a list of token
ids. This method is intended for use with chat models, and will read the tokenizerβs chat_template attribute to
determine the format and control tokens to use when converting.
See here for more information.
Example: Applying a chat template to a conversation.
import { AutoTokenizer } from "@huggingface/transformers";
const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1");
const chat = [
{ "role": "user", "content": "Hello, how are you?" },
{ "role": "assistant", "content": "I'm doing great. How can I help you today?" },
{ "role": "user", "content": "I'd like to show off how chat templating works!" },
]
const text = tokenizer.apply_chat_template(chat, { tokenize: false });
// "<s>[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today?</s> [INST] I'd like to show off how chat templating works! [/INST]"
const input_ids = tokenizer.apply_chat_template(chat, { tokenize: true, return_tensor: false });
// [1, 733, 16289, 28793, 22557, 28725, 910, 460, 368, 28804, 733, 28748, 16289, 28793, 28737, 28742, 28719, 2548, 1598, 28723, 1602, 541, 315, 1316, 368, 3154, 28804, 2, 28705, 733, 16289, 28793, 315, 28742, 28715, 737, 298, 1347, 805, 910, 10706, 5752, 1077, 3791, 28808, 733, 28748, 16289, 28793]
Kind: instance method of PreTrainedTokenizer
Returns: string
| Tensor
| Array<number>
| Array<Array<number>>
| BatchEncoding
- The tokenized output.
Param | Type | Default | Description |
---|---|---|---|
conversation | Array.<Message> | A list of message objects with | |
options | Object | An optional object containing the following properties: | |
[options.chat_template] | string | null | A Jinja template to use for this conversion. If this is not passed, the model's chat template will be used instead. |
[options.tools] | Array.<Object> |
| A list of tools (callable functions) that will be accessible to the model. If the template does not support function calling, this argument will have no effect. Each tool should be passed as a JSON Schema, giving the name, description and argument types for the tool. See our chat templating guide for more information. |
[options.documents] | * |
| A list of dicts representing documents that will be accessible to the model if it is performing RAG (retrieval-augmented generation). If the template does not support RAG, this argument will have no effect. We recommend that each document should be a dict containing "title" and "text" keys. Please see the RAG section of the chat templating guide for examples of passing documents with chat templates. |
[options.add_generation_prompt] | boolean | false | Whether to end the prompt with the token(s) that indicate the start of an assistant message. This is useful when you want to generate a response from the model. Note that this argument will be passed to the chat template, and so it must be supported in the template for this argument to have any effect. |
[options.tokenize] | boolean | true | Whether to tokenize the output. If false, the output will be a string. |
[options.padding] | boolean | false | Whether to pad sequences to the maximum length. Has no effect if tokenize is false. |
[options.truncation] | boolean | false | Whether to truncate sequences to the maximum length. Has no effect if tokenize is false. |
[options.max_length] | number |
| Maximum length (in tokens) to use for padding or truncation. Has no effect if tokenize is false.
If not specified, the tokenizer's |
[options.return_tensor] | boolean | true | Whether to return the output as a Tensor or an Array. Has no effect if tokenize is false. |
[options.return_dict] | boolean | true | Whether to return a dictionary with named outputs. Has no effect if tokenize is false. |
[options.tokenizer_kwargs] | Object | {} | Additional options to pass to the tokenizer. |
PreTrainedTokenizer.from_pretrained(pretrained_model_name_or_path, options) β <code> Promise. < PreTrainedTokenizer > </code>
Loads a pre-trained tokenizer from the given pretrained_model_name_or_path
.
Kind: static method of PreTrainedTokenizer
Returns: Promise.<PreTrainedTokenizer>
- A new instance of the PreTrainedTokenizer
class.
Throws:
Error
Throws an error if the tokenizer.json or tokenizer_config.json files are not found in the `pretrained_model_name_or_path`.
Param | Type | Description |
---|---|---|
pretrained_model_name_or_path | string | The path to the pre-trained tokenizer. |
options | PretrainedTokenizerOptions | Additional options for loading the tokenizer. |
tokenizers.BertTokenizer β <code> PreTrainedTokenizer </code>
BertTokenizer is a class used to tokenize text for BERT models.
Kind: static class of tokenizers
Extends: PreTrainedTokenizer
tokenizers.AlbertTokenizer β <code> PreTrainedTokenizer </code>
Albert tokenizer
Kind: static class of tokenizers
Extends: PreTrainedTokenizer
tokenizers.NllbTokenizer
The NllbTokenizer class is used to tokenize text for NLLB (βNo Language Left Behindβ) models.
No Language Left Behind (NLLB) is a first-of-its-kind, AI breakthrough project that open-sources models capable of delivering high-quality translations directly between any pair of 200+ languages β including low-resource languages like Asturian, Luganda, Urdu and more. It aims to help people communicate with anyone, anywhere, regardless of their language preferences. For more information, check out their paper.
For a list of supported languages (along with their language codes),
Kind: static class of tokenizers
See: https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200
nllbTokenizer._build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) β <code> Object </code>
Helper function to build translation inputs for an NllbTokenizer
.
Kind: instance method of NllbTokenizer
Returns: Object
- Object to be passed to the model.
Param | Type | Description |
---|---|---|
raw_inputs | string | Array<string> | The text to tokenize. |
tokenizer_options | Object | Options to be sent to the tokenizer |
generate_kwargs | Object | Generation options. |
tokenizers.M2M100Tokenizer
The M2M100Tokenizer class is used to tokenize text for M2M100 (βMany-to-Manyβ) models.
M2M100 is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many multilingual translation. It was introduced in this paper and first released in this repository.
For a list of supported languages (along with their language codes),
Kind: static class of tokenizers
See: https://huggingface.co/facebook/m2m100_418M#languages-covered
m2M100Tokenizer._build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) β <code> Object </code>
Helper function to build translation inputs for an M2M100Tokenizer
.
Kind: instance method of M2M100Tokenizer
Returns: Object
- Object to be passed to the model.
Param | Type | Description |
---|---|---|
raw_inputs | string | Array<string> | The text to tokenize. |
tokenizer_options | Object | Options to be sent to the tokenizer |
generate_kwargs | Object | Generation options. |
tokenizers.WhisperTokenizer β <code> PreTrainedTokenizer </code>
WhisperTokenizer tokenizer
Kind: static class of tokenizers
Extends: PreTrainedTokenizer
- .WhisperTokenizer β
PreTrainedTokenizer
._decode_asr(sequences, options)
β*
.decode()
:*
whisperTokenizer._decode_asr(sequences, options) β <code> * </code>
Decodes automatic speech recognition (ASR) sequences.
Kind: instance method of WhisperTokenizer
Returns: *
- The decoded sequences.
Param | Type | Description |
---|---|---|
sequences | * | The sequences to decode. |
options | Object | The options to use for decoding. |
whisperTokenizer.decode() : <code> * </code>
Kind: instance method of WhisperTokenizer
tokenizers.MarianTokenizer
Kind: static class of tokenizers
Todo
- This model is not yet supported by Hugging Faceβs βfastβ tokenizers library (https://github.com/huggingface/tokenizers). Therefore, this implementation (which is based on fast tokenizers) may produce slightly inaccurate results.
new MarianTokenizer(tokenizerJSON, tokenizerConfig)
Create a new MarianTokenizer instance.
Param | Type | Description |
---|---|---|
tokenizerJSON | Object | The JSON of the tokenizer. |
tokenizerConfig | Object | The config of the tokenizer. |
marianTokenizer._encode_text(text) β <code> Array </code>
Encodes a single text. Overriding this method is necessary since the language codes must be removed before encoding with sentencepiece model.
Kind: instance method of MarianTokenizer
Returns: Array
- The encoded tokens.
See: https://github.com/huggingface/transformers/blob/12d51db243a00726a548a43cc333390ebae731e3/src/transformers/models/marian/tokenization_marian.py#L204-L213
Param | Type | Description |
---|---|---|
text | string | null | The text to encode. |
tokenizers.AutoTokenizer
Helper class which is used to instantiate pretrained tokenizers with the from_pretrained
function.
The chosen tokenizer class is determined by the type specified in the tokenizer config.
Kind: static class of tokenizers
AutoTokenizer.from_pretrained(pretrained_model_name_or_path, options) β <code> Promise. < PreTrainedTokenizer > </code>
Instantiate one of the tokenizer classes of the library from a pretrained model.
The tokenizer class to instantiate is selected based on the tokenizer_class
property of the config object
(either passed as an argument or loaded from pretrained_model_name_or_path
if possible)
Kind: static method of AutoTokenizer
Returns: Promise.<PreTrainedTokenizer>
- A new instance of the PreTrainedTokenizer class.
Param | Type | Description |
---|---|---|
pretrained_model_name_or_path | string | The name or path of the pretrained model. Can be either:
|
options | PretrainedTokenizerOptions | Additional options for loading the tokenizer. |
tokenizers.is_chinese_char(cp) β <code> boolean </code>
Checks whether the given Unicode codepoint represents a CJK (Chinese, Japanese, or Korean) character.
A βchinese characterβ is defined as anything in the CJK Unicode block: https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
Note that the CJK Unicode block is NOT all Japanese and Korean characters, despite its name. The modern Korean Hangul alphabet is a different block, as is Japanese Hiragana and Katakana. Those alphabets are used to write space-separated words, so they are not treated specially and are handled like all other languages.
Kind: static method of tokenizers
Returns: boolean
- True if the codepoint represents a CJK character, false otherwise.
Param | Type | Description |
---|---|---|
cp | number | bigint | The Unicode codepoint to check. |
tokenizers~AddedToken
Represent a token added by the user on top of the existing Model vocabulary. AddedToken can be configured to specify the behavior they should have in various situations like:
- Whether they should only match single words
- Whether to include any whitespace on its left or right
Kind: inner class of tokenizers
new AddedToken(config)
Creates a new instance of AddedToken.
Param | Type | Default | Description |
---|---|---|---|
config | Object | Added token configuration object. | |
config.content | string | The content of the added token. | |
config.id | number | The id of the added token. | |
[config.single_word] | boolean | false | Whether this token must be a single word or can break words. |
[config.lstrip] | boolean | false | Whether this token should strip whitespaces on its left. |
[config.rstrip] | boolean | false | Whether this token should strip whitespaces on its right. |
[config.normalized] | boolean | false | Whether this token should be normalized. |
[config.special] | boolean | false | Whether this token is special. |
tokenizers~WordPieceTokenizer β <code> TokenizerModel </code>
A subclass of TokenizerModel that uses WordPiece encoding to encode tokens.
Kind: inner class of tokenizers
Extends: TokenizerModel
- ~WordPieceTokenizer β
TokenizerModel
new WordPieceTokenizer(config)
.tokens_to_ids
:Map.<string, number>
.unk_token_id
:number
.unk_token
:string
.max_input_chars_per_word
:number
.vocab
:Array.<string>
.encode(tokens)
βArray.<string>
new WordPieceTokenizer(config)
Param | Type | Default | Description |
---|---|---|---|
config | Object | The configuration object. | |
config.vocab | Object | A mapping of tokens to ids. | |
config.unk_token | string | The unknown token string. | |
config.continuing_subword_prefix | string | The prefix to use for continuing subwords. | |
[config.max_input_chars_per_word] | number | 100 | The maximum number of characters per word. |
wordPieceTokenizer.tokens_to_ids : <code> Map. < string, number > </code>
A mapping of tokens to ids.
Kind: instance property of WordPieceTokenizer
wordPieceTokenizer.unk_token_id : <code> number </code>
The id of the unknown token.
Kind: instance property of WordPieceTokenizer
wordPieceTokenizer.unk_token : <code> string </code>
The unknown token string.
Kind: instance property of WordPieceTokenizer
wordPieceTokenizer.max_input_chars_per_word : <code> number </code>
The maximum number of characters allowed per word.
Kind: instance property of WordPieceTokenizer
wordPieceTokenizer.vocab : <code> Array. < string > </code>
An array of tokens.
Kind: instance property of WordPieceTokenizer
wordPieceTokenizer.encode(tokens) β <code> Array. < string > </code>
Encodes an array of tokens using WordPiece encoding.
Kind: instance method of WordPieceTokenizer
Returns: Array.<string>
- An array of encoded tokens.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | The tokens to encode. |
tokenizers~Unigram β <code> TokenizerModel </code>
Class representing a Unigram tokenizer model.
Kind: inner class of tokenizers
Extends: TokenizerModel
- ~Unigram β
TokenizerModel
new Unigram(config, moreConfig)
.populateNodes(lattice)
.tokenize(normalized)
βArray.<string>
.encode(tokens)
βArray.<string>
new Unigram(config, moreConfig)
Create a new Unigram tokenizer model.
Param | Type | Description |
---|---|---|
config | Object | The configuration object for the Unigram model. |
config.unk_id | number | The ID of the unknown token |
config.vocab | Array.<Array<any>> | A 2D array representing a mapping of tokens to scores. |
moreConfig | Object | Additional configuration object for the Unigram model. |
unigram.populateNodes(lattice)
Populates lattice nodes.
Kind: instance method of Unigram
Param | Type | Description |
---|---|---|
lattice | TokenLattice | The token lattice to populate with nodes. |
unigram.tokenize(normalized) β <code> Array. < string > </code>
Encodes an array of tokens into an array of subtokens using the unigram model.
Kind: instance method of Unigram
Returns: Array.<string>
- An array of subtokens obtained by encoding the input tokens using the unigram model.
Param | Type | Description |
---|---|---|
normalized | string | The normalized string. |
unigram.encode(tokens) β <code> Array. < string > </code>
Encodes an array of tokens using Unigram encoding.
Kind: instance method of Unigram
Returns: Array.<string>
- An array of encoded tokens.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | The tokens to encode. |
tokenizers~BPE β <code> TokenizerModel </code>
BPE class for encoding text into Byte-Pair-Encoding (BPE) tokens.
Kind: inner class of tokenizers
Extends: TokenizerModel
- ~BPE β
TokenizerModel
new BPE(config)
.tokens_to_ids
:Map.<string, number>
.merges
:*
.config.merges
:*
.cache
:Map.<string, Array<string>>
.bpe(token)
βArray.<string>
.encode(tokens)
βArray.<string>
new BPE(config)
Create a BPE instance.
Param | Type | Default | Description |
---|---|---|---|
config | Object | The configuration object for BPE. | |
config.vocab | Object | A mapping of tokens to ids. | |
config.merges | * | An array of BPE merges as strings. | |
config.unk_token | string | The unknown token used for out of vocabulary words. | |
config.end_of_word_suffix | string | The suffix to place at the end of each word. | |
[config.continuing_subword_suffix] | string | The suffix to insert between words. | |
[config.byte_fallback] | boolean | false | Whether to use spm byte-fallback trick (defaults to False) |
[config.ignore_merges] | boolean | false | Whether or not to match tokens with the vocab before using merges. |
bpE.tokens_to_ids : <code> Map. < string, number > </code>
Kind: instance property of BPE
bpE.merges : <code> * </code>
Kind: instance property of BPE
merges.config.merges : <code> * </code>
Kind: static property of merges
bpE.cache : <code> Map. < string, Array < string > > </code>
Kind: instance property of BPE
bpE.bpe(token) β <code> Array. < string > </code>
Apply Byte-Pair-Encoding (BPE) to a given token. Efficient heap-based priority queue implementation adapted from https://github.com/belladoreai/llama-tokenizer-js.
Kind: instance method of BPE
Returns: Array.<string>
- The BPE encoded tokens.
Param | Type | Description |
---|---|---|
token | string | The token to encode. |
bpE.encode(tokens) β <code> Array. < string > </code>
Encodes the input sequence of tokens using the BPE algorithm and returns the resulting subword tokens.
Kind: instance method of BPE
Returns: Array.<string>
- The resulting subword tokens after applying the BPE algorithm to the input sequence of tokens.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | The input sequence of tokens to encode. |
tokenizers~LegacyTokenizerModel
Legacy tokenizer class for tokenizers with only a vocabulary.
Kind: inner class of tokenizers
- ~LegacyTokenizerModel
new LegacyTokenizerModel(config, moreConfig)
.tokens_to_ids
:Map.<string, number>
new LegacyTokenizerModel(config, moreConfig)
Create a LegacyTokenizerModel instance.
Param | Type | Description |
---|---|---|
config | Object | The configuration object for LegacyTokenizerModel. |
config.vocab | Object | A (possibly nested) mapping of tokens to ids. |
moreConfig | Object | Additional configuration object for the LegacyTokenizerModel model. |
legacyTokenizerModel.tokens_to_ids : <code> Map. < string, number > </code>
Kind: instance property of LegacyTokenizerModel
tokenizers~Normalizer
A base class for text normalization.
Kind: inner abstract class of tokenizers
- ~Normalizer
new Normalizer(config)
- instance
.normalize(text)
βstring
._call(text)
βstring
- static
.fromConfig(config)
βNormalizer
new Normalizer(config)
Param | Type | Description |
---|---|---|
config | Object | The configuration object for the normalizer. |
normalizer.normalize(text) β <code> string </code>
Normalize the input text.
Kind: instance abstract method of Normalizer
Returns: string
- The normalized text.
Throws:
Error
If this method is not implemented in a subclass.
Param | Type | Description |
---|---|---|
text | string | The text to normalize. |
normalizer._call(text) β <code> string </code>
Alias for Normalizer#normalize.
Kind: instance method of Normalizer
Returns: string
- The normalized text.
Param | Type | Description |
---|---|---|
text | string | The text to normalize. |
Normalizer.fromConfig(config) β <code> Normalizer </code>
Factory method for creating normalizers from config objects.
Kind: static method of Normalizer
Returns: Normalizer
- A Normalizer object.
Throws:
Error
If an unknown Normalizer type is specified in the config.
Param | Type | Description |
---|---|---|
config | Object | The configuration object for the normalizer. |
tokenizers~Replace β <code> Normalizer </code>
Replace normalizer that replaces occurrences of a pattern with a given string or regular expression.
Kind: inner class of tokenizers
Extends: Normalizer
replace.normalize(text) β <code> string </code>
Normalize the input text by replacing the pattern with the content.
Kind: instance method of Replace
Returns: string
- The normalized text after replacing the pattern with the content.
Param | Type | Description |
---|---|---|
text | string | The input text to be normalized. |
tokenizers~NFC β <code> Normalizer </code>
A normalizer that applies Unicode normalization form C (NFC) to the input text.
Kind: inner class of tokenizers
Extends: Normalizer
nfC.normalize(text) β <code> string </code>
Normalize the input text by applying Unicode normalization form C (NFC).
Kind: instance method of NFC
Returns: string
- The normalized text.
Param | Type | Description |
---|---|---|
text | string | The input text to be normalized. |
tokenizers~NFKC β <code> Normalizer </code>
NFKC Normalizer.
Kind: inner class of tokenizers
Extends: Normalizer
nfkC.normalize(text) β <code> string </code>
Normalize text using NFKC normalization.
Kind: instance method of NFKC
Returns: string
- The normalized text.
Param | Type | Description |
---|---|---|
text | string | The text to be normalized. |
tokenizers~NFKD β <code> Normalizer </code>
NFKD Normalizer.
Kind: inner class of tokenizers
Extends: Normalizer
nfkD.normalize(text) β <code> string </code>
Normalize text using NFKD normalization.
Kind: instance method of NFKD
Returns: string
- The normalized text.
Param | Type | Description |
---|---|---|
text | string | The text to be normalized. |
tokenizers~StripNormalizer
A normalizer that strips leading and/or trailing whitespace from the input text.
Kind: inner class of tokenizers
stripNormalizer.normalize(text) β <code> string </code>
Strip leading and/or trailing whitespace from the input text.
Kind: instance method of StripNormalizer
Returns: string
- The normalized text.
Param | Type | Description |
---|---|---|
text | string | The input text. |
tokenizers~StripAccents β <code> Normalizer </code>
StripAccents normalizer removes all accents from the text.
Kind: inner class of tokenizers
Extends: Normalizer
stripAccents.normalize(text) β <code> string </code>
Remove all accents from the text.
Kind: instance method of StripAccents
Returns: string
- The normalized text without accents.
Param | Type | Description |
---|---|---|
text | string | The input text. |
tokenizers~Lowercase β <code> Normalizer </code>
A Normalizer that lowercases the input string.
Kind: inner class of tokenizers
Extends: Normalizer
lowercase.normalize(text) β <code> string </code>
Lowercases the input string.
Kind: instance method of Lowercase
Returns: string
- The normalized text.
Param | Type | Description |
---|---|---|
text | string | The text to normalize. |
tokenizers~Prepend β <code> Normalizer </code>
A Normalizer that prepends a string to the input string.
Kind: inner class of tokenizers
Extends: Normalizer
prepend.normalize(text) β <code> string </code>
Prepends the input string.
Kind: instance method of Prepend
Returns: string
- The normalized text.
Param | Type | Description |
---|---|---|
text | string | The text to normalize. |
tokenizers~NormalizerSequence β <code> Normalizer </code>
A Normalizer that applies a sequence of Normalizers.
Kind: inner class of tokenizers
Extends: Normalizer
- ~NormalizerSequence β
Normalizer
new NormalizerSequence(config)
.normalize(text)
βstring
new NormalizerSequence(config)
Create a new instance of NormalizerSequence.
Param | Type | Description |
---|---|---|
config | Object | The configuration object. |
config.normalizers | Array.<Object> | An array of Normalizer configuration objects. |
normalizerSequence.normalize(text) β <code> string </code>
Apply a sequence of Normalizers to the input text.
Kind: instance method of NormalizerSequence
Returns: string
- The normalized text.
Param | Type | Description |
---|---|---|
text | string | The text to normalize. |
tokenizers~BertNormalizer β <code> Normalizer </code>
A class representing a normalizer used in BERT tokenization.
Kind: inner class of tokenizers
Extends: Normalizer
- ~BertNormalizer β
Normalizer
._tokenize_chinese_chars(text)
βstring
.stripAccents(text)
βstring
.normalize(text)
βstring
bertNormalizer._tokenize_chinese_chars(text) β <code> string </code>
Adds whitespace around any CJK (Chinese, Japanese, or Korean) character in the input text.
Kind: instance method of BertNormalizer
Returns: string
- The tokenized text with whitespace added around CJK characters.
Param | Type | Description |
---|---|---|
text | string | The input text to tokenize. |
bertNormalizer.stripAccents(text) β <code> string </code>
Strips accents from the given text.
Kind: instance method of BertNormalizer
Returns: string
- The text with accents removed.
Param | Type | Description |
---|---|---|
text | string | The text to strip accents from. |
bertNormalizer.normalize(text) β <code> string </code>
Normalizes the given text based on the configuration.
Kind: instance method of BertNormalizer
Returns: string
- The normalized text.
Param | Type | Description |
---|---|---|
text | string | The text to normalize. |
tokenizers~PreTokenizer β <code> Callable </code>
A callable class representing a pre-tokenizer used in tokenization. Subclasses
should implement the pre_tokenize_text
method to define the specific pre-tokenization logic.
Kind: inner class of tokenizers
Extends: Callable
- ~PreTokenizer β
Callable
- instance
.pre_tokenize_text(text, [options])
βArray.<string>
.pre_tokenize(text, [options])
βArray.<string>
._call(text, [options])
βArray.<string>
- static
.fromConfig(config)
βPreTokenizer
- instance
preTokenizer.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
Method that should be implemented by subclasses to define the specific pre-tokenization logic.
Kind: instance abstract method of PreTokenizer
Returns: Array.<string>
- The pre-tokenized text.
Throws:
Error
If the method is not implemented in the subclass.
Param | Type | Description |
---|---|---|
text | string | The text to pre-tokenize. |
[options] | Object | Additional options for the pre-tokenization logic. |
preTokenizer.pre_tokenize(text, [options]) β <code> Array. < string > </code>
Tokenizes the given text into pre-tokens.
Kind: instance method of PreTokenizer
Returns: Array.<string>
- An array of pre-tokens.
Param | Type | Description |
---|---|---|
text | string | Array<string> | The text or array of texts to pre-tokenize. |
[options] | Object | Additional options for the pre-tokenization logic. |
preTokenizer._call(text, [options]) β <code> Array. < string > </code>
Alias for PreTokenizer#pre_tokenize.
Kind: instance method of PreTokenizer
Overrides: _call
Returns: Array.<string>
- An array of pre-tokens.
Param | Type | Description |
---|---|---|
text | string | Array<string> | The text or array of texts to pre-tokenize. |
[options] | Object | Additional options for the pre-tokenization logic. |
PreTokenizer.fromConfig(config) β <code> PreTokenizer </code>
Factory method that returns an instance of a subclass of PreTokenizer
based on the provided configuration.
Kind: static method of PreTokenizer
Returns: PreTokenizer
- An instance of a subclass of PreTokenizer
.
Throws:
Error
If the provided configuration object does not correspond to any known pre-tokenizer.
Param | Type | Description |
---|---|---|
config | Object | A configuration object for the pre-tokenizer. |
tokenizers~BertPreTokenizer β <code> PreTokenizer </code>
Kind: inner class of tokenizers
Extends: PreTokenizer
- ~BertPreTokenizer β
PreTokenizer
new BertPreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
new BertPreTokenizer(config)
A PreTokenizer that splits text into wordpieces using a basic tokenization scheme similar to that used in the original implementation of BERT.
Param | Type | Description |
---|---|---|
config | Object | The configuration object. |
bertPreTokenizer.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
Tokenizes a single text using the BERT pre-tokenization scheme.
Kind: instance method of BertPreTokenizer
Returns: Array.<string>
- An array of tokens.
Param | Type | Description |
---|---|---|
text | string | The text to tokenize. |
[options] | Object | Additional options for the pre-tokenization logic. |
tokenizers~ByteLevelPreTokenizer β <code> PreTokenizer </code>
A pre-tokenizer that splits text into Byte-Pair-Encoding (BPE) subwords.
Kind: inner class of tokenizers
Extends: PreTokenizer
- ~ByteLevelPreTokenizer β
PreTokenizer
new ByteLevelPreTokenizer(config)
.add_prefix_space
:boolean
.trim_offsets
:boolean
.use_regex
:boolean
.pre_tokenize_text(text, [options])
βArray.<string>
new ByteLevelPreTokenizer(config)
Creates a new instance of the ByteLevelPreTokenizer
class.
Param | Type | Description |
---|---|---|
config | Object | The configuration object. |
byteLevelPreTokenizer.add_prefix_space : <code> boolean </code>
Whether to add a leading space to the first word.This allows to treat the leading word just as any other word.
Kind: instance property of ByteLevelPreTokenizer
byteLevelPreTokenizer.trim_offsets : <code> boolean </code>
Whether the post processing step should trim offsetsto avoid including whitespaces.
Kind: instance property of ByteLevelPreTokenizer
Todo
- Use this in the pretokenization step.
byteLevelPreTokenizer.use_regex : <code> boolean </code>
Whether to use the standard GPT2 regex for whitespace splitting.Set it to False if you want to use your own splitting. Defaults to true.
Kind: instance property of ByteLevelPreTokenizer
byteLevelPreTokenizer.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
Tokenizes a single piece of text using byte-level tokenization.
Kind: instance method of ByteLevelPreTokenizer
Returns: Array.<string>
- An array of tokens.
Param | Type | Description |
---|---|---|
text | string | The text to tokenize. |
[options] | Object | Additional options for the pre-tokenization logic. |
tokenizers~SplitPreTokenizer β <code> PreTokenizer </code>
Splits text using a given pattern.
Kind: inner class of tokenizers
Extends: PreTokenizer
- ~SplitPreTokenizer β
PreTokenizer
new SplitPreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
new SplitPreTokenizer(config)
Param | Type | Description |
---|---|---|
config | Object | The configuration options for the pre-tokenizer. |
config.pattern | Object | The pattern used to split the text. Can be a string or a regex object. |
config.pattern.String | string | undefined | The string to use for splitting. Only defined if the pattern is a string. |
config.pattern.Regex | string | undefined | The regex to use for splitting. Only defined if the pattern is a regex. |
config.behavior | SplitDelimiterBehavior | The behavior to use when splitting. |
config.invert | boolean | Whether to split (invert=false) or match (invert=true) the pattern. |
splitPreTokenizer.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
Tokenizes text by splitting it using the given pattern.
Kind: instance method of SplitPreTokenizer
Returns: Array.<string>
- An array of tokens.
Param | Type | Description |
---|---|---|
text | string | The text to tokenize. |
[options] | Object | Additional options for the pre-tokenization logic. |
tokenizers~PunctuationPreTokenizer β <code> PreTokenizer </code>
Splits text based on punctuation.
Kind: inner class of tokenizers
Extends: PreTokenizer
- ~PunctuationPreTokenizer β
PreTokenizer
new PunctuationPreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
new PunctuationPreTokenizer(config)
Param | Type | Description |
---|---|---|
config | Object | The configuration options for the pre-tokenizer. |
config.behavior | SplitDelimiterBehavior | The behavior to use when splitting. |
punctuationPreTokenizer.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
Tokenizes text by splitting it using the given pattern.
Kind: instance method of PunctuationPreTokenizer
Returns: Array.<string>
- An array of tokens.
Param | Type | Description |
---|---|---|
text | string | The text to tokenize. |
[options] | Object | Additional options for the pre-tokenization logic. |
tokenizers~DigitsPreTokenizer β <code> PreTokenizer </code>
Splits text based on digits.
Kind: inner class of tokenizers
Extends: PreTokenizer
- ~DigitsPreTokenizer β
PreTokenizer
new DigitsPreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
new DigitsPreTokenizer(config)
Param | Type | Description |
---|---|---|
config | Object | The configuration options for the pre-tokenizer. |
config.individual_digits | boolean | Whether to split on individual digits. |
digitsPreTokenizer.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
Tokenizes text by splitting it using the given pattern.
Kind: instance method of DigitsPreTokenizer
Returns: Array.<string>
- An array of tokens.
Param | Type | Description |
---|---|---|
text | string | The text to tokenize. |
[options] | Object | Additional options for the pre-tokenization logic. |
tokenizers~PostProcessor β <code> Callable </code>
Kind: inner class of tokenizers
Extends: Callable
- ~PostProcessor β
Callable
new PostProcessor(config)
- instance
.post_process(tokens, ...args)
βPostProcessedOutput
._call(tokens, ...args)
βPostProcessedOutput
- static
.fromConfig(config)
βPostProcessor
new PostProcessor(config)
Param | Type | Description |
---|---|---|
config | Object | The configuration for the post-processor. |
postProcessor.post_process(tokens, ...args) β <code> PostProcessedOutput </code>
Method to be implemented in subclass to apply post-processing on the given tokens.
Kind: instance method of PostProcessor
Returns: PostProcessedOutput
- The post-processed tokens.
Throws:
Error
If the method is not implemented in subclass.
Param | Type | Description |
---|---|---|
tokens | Array | The input tokens to be post-processed. |
...args | * | Additional arguments required by the post-processing logic. |
postProcessor._call(tokens, ...args) β <code> PostProcessedOutput </code>
Alias for PostProcessor#post_process.
Kind: instance method of PostProcessor
Overrides: _call
Returns: PostProcessedOutput
- The post-processed tokens.
Param | Type | Description |
---|---|---|
tokens | Array | The text or array of texts to post-process. |
...args | * | Additional arguments required by the post-processing logic. |
PostProcessor.fromConfig(config) β <code> PostProcessor </code>
Factory method to create a PostProcessor object from a configuration object.
Kind: static method of PostProcessor
Returns: PostProcessor
- A PostProcessor object created from the given configuration.
Throws:
Error
If an unknown PostProcessor type is encountered.
Param | Type | Description |
---|---|---|
config | Object | Configuration object representing a PostProcessor. |
tokenizers~BertProcessing
A post-processor that adds special tokens to the beginning and end of the input.
Kind: inner class of tokenizers
- ~BertProcessing
new BertProcessing(config)
.post_process(tokens, [tokens_pair])
βPostProcessedOutput
new BertProcessing(config)
Param | Type | Description |
---|---|---|
config | Object | The configuration for the post-processor. |
config.cls | Array.<string> | The special tokens to add to the beginning of the input. |
config.sep | Array.<string> | The special tokens to add to the end of the input. |
bertProcessing.post_process(tokens, [tokens_pair]) β <code> PostProcessedOutput </code>
Adds the special tokens to the beginning and end of the input.
Kind: instance method of BertProcessing
Returns: PostProcessedOutput
- The post-processed tokens with the special tokens added to the beginning and end.
Param | Type | Default | Description |
---|---|---|---|
tokens | Array.<string> | The input tokens. | |
[tokens_pair] | Array.<string> |
| An optional second set of input tokens. |
tokenizers~TemplateProcessing β <code> PostProcessor </code>
Post processor that replaces special tokens in a template with actual tokens.
Kind: inner class of tokenizers
Extends: PostProcessor
- ~TemplateProcessing β
PostProcessor
new TemplateProcessing(config)
.post_process(tokens, [tokens_pair])
βPostProcessedOutput
new TemplateProcessing(config)
Creates a new instance of TemplateProcessing
.
Param | Type | Description |
---|---|---|
config | Object | The configuration options for the post processor. |
config.single | Array | The template for a single sequence of tokens. |
config.pair | Array | The template for a pair of sequences of tokens. |
templateProcessing.post_process(tokens, [tokens_pair]) β <code> PostProcessedOutput </code>
Replaces special tokens in the template with actual tokens.
Kind: instance method of TemplateProcessing
Returns: PostProcessedOutput
- An object containing the list of tokens with the special tokens replaced with actual tokens.
Param | Type | Default | Description |
---|---|---|---|
tokens | Array.<string> | The list of tokens for the first sequence. | |
[tokens_pair] | Array.<string> |
| The list of tokens for the second sequence (optional). |
tokenizers~ByteLevelPostProcessor β <code> PostProcessor </code>
A PostProcessor that returns the given tokens as is.
Kind: inner class of tokenizers
Extends: PostProcessor
byteLevelPostProcessor.post_process(tokens, [tokens_pair]) β <code> PostProcessedOutput </code>
Post process the given tokens.
Kind: instance method of ByteLevelPostProcessor
Returns: PostProcessedOutput
- An object containing the post-processed tokens.
Param | Type | Default | Description |
---|---|---|---|
tokens | Array.<string> | The list of tokens for the first sequence. | |
[tokens_pair] | Array.<string> |
| The list of tokens for the second sequence (optional). |
tokenizers~PostProcessorSequence
A post-processor that applies multiple post-processors in sequence.
Kind: inner class of tokenizers
- ~PostProcessorSequence
new PostProcessorSequence(config)
.post_process(tokens, [tokens_pair])
βPostProcessedOutput
new PostProcessorSequence(config)
Creates a new instance of PostProcessorSequence.
Param | Type | Description |
---|---|---|
config | Object | The configuration object. |
config.processors | Array.<Object> | The list of post-processors to apply. |
postProcessorSequence.post_process(tokens, [tokens_pair]) β <code> PostProcessedOutput </code>
Post process the given tokens.
Kind: instance method of PostProcessorSequence
Returns: PostProcessedOutput
- An object containing the post-processed tokens.
Param | Type | Default | Description |
---|---|---|---|
tokens | Array.<string> | The list of tokens for the first sequence. | |
[tokens_pair] | Array.<string> |
| The list of tokens for the second sequence (optional). |
tokenizers~Decoder β <code> Callable </code>
The base class for token decoders.
Kind: inner class of tokenizers
Extends: Callable
- ~Decoder β
Callable
new Decoder(config)
- instance
.added_tokens
:Array.<AddedToken>
._call(tokens)
βstring
.decode(tokens)
βstring
.decode_chain(tokens)
βArray.<string>
- static
.fromConfig(config)
βDecoder
new Decoder(config)
Creates an instance of Decoder
.
Param | Type | Description |
---|---|---|
config | Object | The configuration object. |
decoder.added_tokens : <code> Array. < AddedToken > </code>
Kind: instance property of Decoder
decoder._call(tokens) β <code> string </code>
Calls the decode
method.
Kind: instance method of Decoder
Overrides: _call
Returns: string
- The decoded string.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | The list of tokens. |
decoder.decode(tokens) β <code> string </code>
Decodes a list of tokens.
Kind: instance method of Decoder
Returns: string
- The decoded string.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | The list of tokens. |
decoder.decode_chain(tokens) β <code> Array. < string > </code>
Apply the decoder to a list of tokens.
Kind: instance method of Decoder
Returns: Array.<string>
- The decoded list of tokens.
Throws:
Error
If the `decode_chain` method is not implemented in the subclass.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | The list of tokens. |
Decoder.fromConfig(config) β <code> Decoder </code>
Creates a decoder instance based on the provided configuration.
Kind: static method of Decoder
Returns: Decoder
- A decoder instance.
Throws:
Error
If an unknown decoder type is provided.
Param | Type | Description |
---|---|---|
config | Object | The configuration object. |
tokenizers~FuseDecoder
Fuse simply fuses all tokens into one big string. Itβs usually the last decoding step anyway, but this decoder exists incase some decoders need to happen after that step
Kind: inner class of tokenizers
fuseDecoder.decode_chain() : <code> * </code>
Kind: instance method of FuseDecoder
tokenizers~WordPieceDecoder β <code> Decoder </code>
A decoder that decodes a list of WordPiece tokens into a single string.
Kind: inner class of tokenizers
Extends: Decoder
- ~WordPieceDecoder β
Decoder
new WordPieceDecoder(config)
Creates a new instance of WordPieceDecoder.
Param | Type | Description |
---|---|---|
config | Object | The configuration object. |
config.prefix | string | The prefix used for WordPiece encoding. |
config.cleanup | boolean | Whether to cleanup the decoded string. |
wordPieceDecoder.decode_chain() : <code> * </code>
Kind: instance method of WordPieceDecoder
tokenizers~ByteLevelDecoder β <code> Decoder </code>
Byte-level decoder for tokenization output. Inherits from the Decoder
class.
Kind: inner class of tokenizers
Extends: Decoder
- ~ByteLevelDecoder β
Decoder
new ByteLevelDecoder(config)
Create a ByteLevelDecoder
object.
Param | Type | Description |
---|---|---|
config | Object | Configuration object. |
byteLevelDecoder.convert_tokens_to_string(tokens) β <code> string </code>
Convert an array of tokens to string by decoding each byte.
Kind: instance method of ByteLevelDecoder
Returns: string
- The decoded string.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | Array of tokens to be decoded. |
byteLevelDecoder.decode_chain() : <code> * </code>
Kind: instance method of ByteLevelDecoder
tokenizers~CTCDecoder
The CTC (Connectionist Temporal Classification) decoder. See https://github.com/huggingface/tokenizers/blob/bb38f390a61883fc2f29d659af696f428d1cda6b/tokenizers/src/decoders/ctc.rs
Kind: inner class of tokenizers
- ~CTCDecoder
.convert_tokens_to_string(tokens)
βstring
.decode_chain()
:*
ctcDecoder.convert_tokens_to_string(tokens) β <code> string </code>
Converts a connectionist-temporal-classification (CTC) output tokens into a single string.
Kind: instance method of CTCDecoder
Returns: string
- The decoded string.
Param | Type | Description |
---|---|---|
tokens | Array.<string> | Array of tokens to be decoded. |
ctcDecoder.decode_chain() : <code> * </code>
Kind: instance method of CTCDecoder
tokenizers~DecoderSequence β <code> Decoder </code>
Apply a sequence of decoders.
Kind: inner class of tokenizers
Extends: Decoder
- ~DecoderSequence β
Decoder
new DecoderSequence(config)
Creates a new instance of DecoderSequence.
Param | Type | Description |
---|---|---|
config | Object | The configuration object. |
config.decoders | Array.<Object> | The list of decoders to apply. |
decoderSequence.decode_chain() : <code> * </code>
Kind: instance method of DecoderSequence
tokenizers~MetaspacePreTokenizer β <code> PreTokenizer </code>
This PreTokenizer replaces spaces with the given replacement character, adds a prefix space if requested, and returns a list of tokens.
Kind: inner class of tokenizers
Extends: PreTokenizer
- ~MetaspacePreTokenizer β
PreTokenizer
new MetaspacePreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
new MetaspacePreTokenizer(config)
Param | Type | Default | Description |
---|---|---|---|
config | Object | The configuration object for the MetaspacePreTokenizer. | |
config.add_prefix_space | boolean | Whether to add a prefix space to the first token. | |
config.replacement | string | The character to replace spaces with. | |
[config.str_rep] | string | "config.replacement" | An optional string representation of the replacement character. |
[config.prepend_scheme] | 'first' | 'never' | 'always' | 'always' | The metaspace prepending scheme. |
metaspacePreTokenizer.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
This method takes a string, replaces spaces with the replacement character, adds a prefix space if requested, and returns a new list of tokens.
Kind: instance method of MetaspacePreTokenizer
Returns: Array.<string>
- A new list of pre-tokenized tokens.
Param | Type | Description |
---|---|---|
text | string | The text to pre-tokenize. |
[options] | Object | The options for the pre-tokenization. |
[options.section_index] | number | The index of the section to pre-tokenize. |
tokenizers~MetaspaceDecoder β <code> Decoder </code>
MetaspaceDecoder class extends the Decoder class and decodes Metaspace tokenization.
Kind: inner class of tokenizers
Extends: Decoder
- ~MetaspaceDecoder β
Decoder
new MetaspaceDecoder(config)
Constructs a new MetaspaceDecoder object.
Param | Type | Description |
---|---|---|
config | Object | The configuration object for the MetaspaceDecoder. |
config.add_prefix_space | boolean | Whether to add a prefix space to the decoded string. |
config.replacement | string | The string to replace spaces with. |
metaspaceDecoder.decode_chain() : <code> * </code>
Kind: instance method of MetaspaceDecoder
tokenizers~Precompiled β <code> Normalizer </code>
A normalizer that applies a precompiled charsmap. This is useful for applying complex normalizations in C++ and exposing them to JavaScript.
Kind: inner class of tokenizers
Extends: Normalizer
- ~Precompiled β
Normalizer
new Precompiled(config)
.normalize(text)
βstring
new Precompiled(config)
Create a new instance of Precompiled normalizer.
Param | Type | Description |
---|---|---|
config | Object | The configuration object for the Precompiled normalizer. |
config.precompiled_charsmap | Object | The precompiled charsmap object. |
precompiled.normalize(text) β <code> string </code>
Normalizes the given text by applying the precompiled charsmap.
Kind: instance method of Precompiled
Returns: string
- The normalized text.
Param | Type | Description |
---|---|---|
text | string | The text to normalize. |
tokenizers~PreTokenizerSequence β <code> PreTokenizer </code>
A pre-tokenizer that applies a sequence of pre-tokenizers to the input text.
Kind: inner class of tokenizers
Extends: PreTokenizer
- ~PreTokenizerSequence β
PreTokenizer
new PreTokenizerSequence(config)
.pre_tokenize_text(text, [options])
βArray.<string>
new PreTokenizerSequence(config)
Creates an instance of PreTokenizerSequence.
Param | Type | Description |
---|---|---|
config | Object | The configuration object for the pre-tokenizer sequence. |
config.pretokenizers | Array.<Object> | An array of pre-tokenizer configurations. |
preTokenizerSequence.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
Applies each pre-tokenizer in the sequence to the input text in turn.
Kind: instance method of PreTokenizerSequence
Returns: Array.<string>
- The pre-tokenized text.
Param | Type | Description |
---|---|---|
text | string | The text to pre-tokenize. |
[options] | Object | Additional options for the pre-tokenization logic. |
tokenizers~WhitespacePreTokenizer
Splits on word boundaries (using the following regular expression: \w+|[^\w\s]+
).
Kind: inner class of tokenizers
- ~WhitespacePreTokenizer
new WhitespacePreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
new WhitespacePreTokenizer(config)
Creates an instance of WhitespacePreTokenizer.
Param | Type | Description |
---|---|---|
config | Object | The configuration object for the pre-tokenizer. |
whitespacePreTokenizer.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
Pre-tokenizes the input text by splitting it on word boundaries.
Kind: instance method of WhitespacePreTokenizer
Returns: Array.<string>
- An array of tokens produced by splitting the input text on whitespace.
Param | Type | Description |
---|---|---|
text | string | The text to be pre-tokenized. |
[options] | Object | Additional options for the pre-tokenization logic. |
tokenizers~WhitespaceSplit β <code> PreTokenizer </code>
Splits a string of text by whitespace characters into individual tokens.
Kind: inner class of tokenizers
Extends: PreTokenizer
- ~WhitespaceSplit β
PreTokenizer
new WhitespaceSplit(config)
.pre_tokenize_text(text, [options])
βArray.<string>
new WhitespaceSplit(config)
Creates an instance of WhitespaceSplit.
Param | Type | Description |
---|---|---|
config | Object | The configuration object for the pre-tokenizer. |
whitespaceSplit.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
Pre-tokenizes the input text by splitting it on whitespace characters.
Kind: instance method of WhitespaceSplit
Returns: Array.<string>
- An array of tokens produced by splitting the input text on whitespace.
Param | Type | Description |
---|---|---|
text | string | The text to be pre-tokenized. |
[options] | Object | Additional options for the pre-tokenization logic. |
tokenizers~ReplacePreTokenizer
Kind: inner class of tokenizers
- ~ReplacePreTokenizer
new ReplacePreTokenizer(config)
.pre_tokenize_text(text, [options])
βArray.<string>
new ReplacePreTokenizer(config)
Param | Type | Description |
---|---|---|
config | Object | The configuration options for the pre-tokenizer. |
config.pattern | Object | The pattern used to split the text. Can be a string or a regex object. |
config.content | string | What to replace the pattern with. |
replacePreTokenizer.pre_tokenize_text(text, [options]) β <code> Array. < string > </code>
Pre-tokenizes the input text by replacing certain characters.
Kind: instance method of ReplacePreTokenizer
Returns: Array.<string>
- An array of tokens produced by replacing certain characters.
Param | Type | Description |
---|---|---|
text | string | The text to be pre-tokenized. |
[options] | Object | Additional options for the pre-tokenization logic. |
tokenizers~BYTES_TO_UNICODE β <code> Object </code>
Returns list of utf-8 byte and a mapping to unicode strings. Specifically avoids mapping to whitespace/control characters the BPE code barfs on.
Kind: inner constant of tokenizers
Returns: Object
- Object with utf-8 byte keys and unicode string values.
tokenizers~loadTokenizer(pretrained_model_name_or_path, options) β <code> Promise. < Array < any > > </code>
Loads a tokenizer from the specified path.
Kind: inner method of tokenizers
Returns: Promise.<Array<any>>
- A promise that resolves with information about the loaded tokenizer.
Param | Type | Description |
---|---|---|
pretrained_model_name_or_path | string | The path to the tokenizer directory. |
options | PretrainedTokenizerOptions | Additional options for loading the tokenizer. |
tokenizers~regexSplit(text, regex) β <code> Array. < string > </code>
Helper function to split a string on a regex, but keep the delimiters.
This is required, because the JavaScript .split()
method does not keep the delimiters,
and wrapping in a capturing group causes issues with existing capturing groups (due to nesting).
Kind: inner method of tokenizers
Returns: Array.<string>
- The split string.
Param | Type | Description |
---|---|---|
text | string | The text to split. |
regex | RegExp | The regex to split on. |
tokenizers~createPattern(pattern, invert) β <code> RegExp </code> | <code> null </code>
Helper method to construct a pattern from a config object.
Kind: inner method of tokenizers
Returns: RegExp
| null
- The compiled pattern.
Param | Type | Default | Description |
---|---|---|---|
pattern | Object | The pattern object. | |
invert | boolean | true | Whether to invert the pattern. |
tokenizers~objectToMap(obj) β <code> Map. < string, any > </code>
Helper function to convert an Object to a Map
Kind: inner method of tokenizers
Returns: Map.<string, any>
- The map.
Param | Type | Description |
---|---|---|
obj | Object | The object to convert. |
tokenizers~prepareTensorForDecode(tensor) β <code> Array. < number > </code>
Helper function to convert a tensor to a list before decoding.
Kind: inner method of tokenizers
Returns: Array.<number>
- The tensor as a list.
Param | Type | Description |
---|---|---|
tensor | Tensor | The tensor to convert. |
tokenizers~clean_up_tokenization(text) β <code> string </code>
Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms
Kind: inner method of tokenizers
Returns: string
- The cleaned up text.
Param | Type | Description |
---|---|---|
text | string | The text to clean up. |
tokenizers~remove_accents(text) β <code> string </code>
Helper function to remove accents from a string.
Kind: inner method of tokenizers
Returns: string
- The text with accents removed.
Param | Type | Description |
---|---|---|
text | string | The text to remove accents from. |
tokenizers~lowercase_and_remove_accent(text) β <code> string </code>
Helper function to lowercase a string and remove accents.
Kind: inner method of tokenizers
Returns: string
- The lowercased text with accents removed.
Param | Type | Description |
---|---|---|
text | string | The text to lowercase and remove accents from. |
tokenizers~whitespace_split(text) β <code> Array. < string > </code>
Split a string on whitespace.
Kind: inner method of tokenizers
Returns: Array.<string>
- The split string.
Param | Type | Description |
---|---|---|
text | string | The text to split. |
tokenizers~PretrainedTokenizerOptions : <code> Object </code>
Additional tokenizer-specific properties.
Kind: inner typedef of tokenizers
Properties
Name | Type | Default | Description |
---|---|---|---|
[legacy] | boolean | false | Whether or not the |
tokenizers~BPENode : <code> Object </code>
Kind: inner typedef of tokenizers
Properties
Name | Type | Description |
---|---|---|
token | string | The token associated with the node |
bias | number | A positional bias for the node. |
[score] | number | The score of the node. |
[prev] | BPENode | The previous node in the linked list. |
[next] | BPENode | The next node in the linked list. |
tokenizers~SplitDelimiterBehavior : <code> β removed β </code> | <code> β isolated β </code> | <code> β mergedWithPrevious β </code> | <code> β mergedWithNext β </code> | <code> β contiguous β </code>
Kind: inner typedef of tokenizers
tokenizers~PostProcessedOutput : <code> Object </code>
Kind: inner typedef of tokenizers
Properties
Name | Type | Description |
---|---|---|
tokens | Array.<string> | List of token produced by the post-processor. |
[token_type_ids] | Array.<number> | List of token type ids produced by the post-processor. |
tokenizers~EncodingSingle : <code> Object </code>
Kind: inner typedef of tokenizers
Properties
Name | Type | Description |
---|---|---|
input_ids | Array.<number> | List of token ids to be fed to a model. |
attention_mask | Array.<number> | List of token type ids to be fed to a model |
[token_type_ids] | Array.<number> | List of indices specifying which tokens should be attended to by the model |
tokenizers~Message : <code> Object </code>
Kind: inner typedef of tokenizers
Properties
Name | Type | Description |
---|---|---|
role | string | The role of the message (e.g., "user" or "assistant" or "system"). |
content | string | The content of the message. |
tokenizers~BatchEncoding : <code> Array < number > </code> | <code> Array < Array < number > > </code> | <code> Tensor </code>
Holds the output of the tokenizerβs call function.
Kind: inner typedef of tokenizers
Properties
Name | Type | Description |
---|---|---|
input_ids | BatchEncodingItem | List of token ids to be fed to a model. |
attention_mask | BatchEncodingItem | List of indices specifying which tokens should be attended to by the model. |
[token_type_ids] | BatchEncodingItem | List of token type ids to be fed to a model. |
< > Update on GitHub