eci-io
/

climategpt-7b

Text Generation

Carbon Emissions

Inference Endpoints

text-generation-inference

Model card Files Files and versions Community

climategpt-7b / tokenizer_config.json

dthulke's picture

adds chat template and update README

87ee610 2 months ago

raw history blame contribute delete

No virus

3.54 kB

	{
	"add_bos_token": true,
	"add_eos_token": false,
	"added_tokens_decoder": {
	"0": {
	"content": "<unk>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"1": {
	"content": "<s>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"2": {
	"content": "</s>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32000": {
	"content": "<CLS>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32001": {
	"content": "<SEP>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32002": {
	"content": "<EOD>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32003": {
	"content": "<MASK>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32004": {
	"content": "<PAD>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32005": {
	"content": "<\|im_start\|>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32006": {
	"content": "<\|im_end\|>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32007": {
	"content": "[[0]]",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32008": {
	"content": "[[1]]",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32009": {
	"content": "[[2]]",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32010": {
	"content": "[[3]]",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	},
	"32011": {
	"content": "[[4]]",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false,
	"special": true
	}
	},
	"additional_special_tokens": [
	"<\|im_start\|>",
	"<\|im_end\|>",
	"[[0]]",
	"[[1]]",
	"[[2]]",
	"[[3]]",
	"[[4]]"
	],
	"bos_token": "<s>",
	"clean_up_tokenization_spaces": false,
	"cls_token": "<CLS>",
	"eos_token": "<\|im_end\|>",
	"legacy": true,
	"mask_token": "<MASK>",
	"model_max_length": 1000000000000000019884624838656,
	"pad_token": "<PAD>",
	"sep_token": "<SEP>",
	"sp_model_kwargs": {},
	"spaces_between_special_tokens": false,
	"tokenizer_class": "LlamaTokenizer",
	"unk_token": "<unk>",
	"use_default_system_prompt": false,
	"chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<\|im_start\|>' + message['role'] + '\n' + message['content'] + '<\|im_end\|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<\|im_start\|>assistant\n' }}{% endif %}"
	}