FINDA-FIT
/

llama-r

Text Generation

Inference Endpoints

text-generation-inference

Model card Files Files and versions Community

llama-r / tokenizer_config.json

amphora's picture

Upload tokenizer

6bdde9a 8 months ago

raw history blame contribute delete

No virus

985 Bytes

	{
	"added_tokens_decoder": {
	"0": {
	"content": "<\|endoftext\|>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"1": {
	"content": "<\|sep\|>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"2": {
	"content": "<\|tel\|>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"3": {
	"content": "<\|rrn\|>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	}
	},
	"additional_special_tokens": [],
	"bos_token": "<\|sep\|>",
	"clean_up_tokenization_spaces": true,
	"eos_token": "<\|endoftext\|>",
	"model_max_length": 1000000000000000019884624838656,
	"pad_token": "<\|endoftext\|>",
	"tokenizer_class": "PreTrainedTokenizerFast"
	}