|
{ |
|
"version": "1.0", |
|
"truncation": { |
|
"direction": "Right", |
|
"max_length": 128, |
|
"strategy": "LongestFirst", |
|
"stride": 0 |
|
}, |
|
"padding": null, |
|
"added_tokens": [ |
|
{ |
|
"id": 0, |
|
"content": "<|endoftext|>", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
} |
|
], |
|
"normalizer": null, |
|
"pre_tokenizer": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": false, |
|
"trim_offsets": true, |
|
"use_regex": true |
|
}, |
|
"post_processor": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": true, |
|
"trim_offsets": false, |
|
"use_regex": true |
|
}, |
|
"decoder": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": true, |
|
"trim_offsets": true, |
|
"use_regex": true |
|
}, |
|
"model": { |
|
"type": "BPE", |
|
"dropout": null, |
|
"unk_token": null, |
|
"continuing_subword_prefix": null, |
|
"end_of_word_suffix": null, |
|
"fuse_unk": false, |
|
"vocab": { |
|
"<|endoftext|>": 0, |
|
"#": 1, |
|
"-": 2, |
|
".": 3, |
|
"/": 4, |
|
"0": 5, |
|
"1": 6, |
|
"2": 7, |
|
"3": 8, |
|
"4": 9, |
|
"5": 10, |
|
"6": 11, |
|
"7": 12, |
|
"8": 13, |
|
"9": 14, |
|
"A": 15, |
|
"B": 16, |
|
"C": 17, |
|
"D": 18, |
|
"E": 19, |
|
"F": 20, |
|
"G": 21, |
|
"_": 22, |
|
"c": 23, |
|
"n": 24, |
|
"r": 25, |
|
"Ġ": 26, |
|
"25": 27, |
|
"Ġn": 28, |
|
"Ġc": 29, |
|
"Ġr": 30, |
|
"75": 31, |
|
"12": 32, |
|
"11": 33, |
|
"13": 34, |
|
"17": 35, |
|
"23": 36, |
|
"19": 37, |
|
"24": 38, |
|
"10": 39, |
|
"29": 40, |
|
"35": 41, |
|
"37": 42, |
|
"125": 43, |
|
"375": 44, |
|
"20": 45, |
|
"41": 46, |
|
"53": 47, |
|
"49": 48 |
|
}, |
|
"merges": [ |
|
"2 5", |
|
"Ġ n", |
|
"Ġ c", |
|
"Ġ r", |
|
"7 5", |
|
"1 2", |
|
"1 1", |
|
"1 3", |
|
"1 7", |
|
"2 3", |
|
"1 9", |
|
"2 4", |
|
"1 0", |
|
"2 9", |
|
"3 5", |
|
"3 7", |
|
"1 25", |
|
"3 75", |
|
"2 0", |
|
"4 1", |
|
"5 3", |
|
"4 9" |
|
] |
|
} |
|
} |