{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": " ": 53,
"",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 64000,
"content": "<|startoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 64001,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Prepend",
"prepend": "▁"
},
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {}
},
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "": 1,
"": 2,
"<|Human|>": 3,
"<|Assistant|>": 4,
"<|System|>": 5,
"<|im_start|>": 6,
"<|im_end|>": 7,
"<|im_sep|>": 8,
"<|reserved003|>": 9,
"<|reserved004|>": 10,
"<|reserved005|>": 11,
"<|reserved006|>": 12,
"<|reserved007|>": 13,
"": 32,
"": 33,
"
": 34,
"": 35,
"": 36,
"
": 37,
"": 38,
"": 39,
"
": 40,
"": 41,
"": 42,
"
": 43,
"": 44,
"": 45,
"
": 46,
"
": 47,
"
": 48,
"": 49,
"": 50,
"": 51,
"": 52,
"