bs-modeling-metadata
/

checkpoints_all_06_12_html_special_tokens_0.5

Model card Files Files and versions Community

checkpoints_all_06_12_html_special_tokens_0.5 / checkpoint-1250step /tokenizer_config.json

jordiclive's picture

Upload checkpoint-1250step with huggingface_hub

69a6084 about 1 year ago

raw history blame contribute delete

966 Bytes

	{
	"add_prefix_space": false,
	"additional_special_tokens": [
	{
	"__type": "AddedToken",
	"content": "<ENTITY_CHAIN>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	},
	{
	"__type": "AddedToken",
	"content": " </ENTITY_CHAIN> ",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	},
	{
	"__type": "AddedToken",
	"content": "<HTML>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	},
	{
	"__type": "AddedToken",
	"content": "</HTML>",
	"lstrip": false,
	"normalized": false,
	"rstrip": false,
	"single_word": false
	}
	],
	"bos_token": "<\|endoftext\|>",
	"eos_token": "<\|endoftext\|>",
	"model_max_length": 1024,
	"special_tokens_map_file": null,
	"tokenizer_class": "GPT2Tokenizer",
	"unk_token": "<\|endoftext\|>"
	}