|
{ |
|
"version": "1.0", |
|
"truncation": { |
|
"direction": "Right", |
|
"max_length": 20, |
|
"strategy": "LongestFirst", |
|
"stride": 0 |
|
}, |
|
"padding": null, |
|
"added_tokens": [ |
|
{ |
|
"id": 0, |
|
"content": "<[|endoftext|>]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 1, |
|
"content": "...", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 2, |
|
"content": "\n", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 270, |
|
"content": "<|endoftext|>", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
} |
|
], |
|
"normalizer": null, |
|
"pre_tokenizer": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": false, |
|
"trim_offsets": true, |
|
"use_regex": true |
|
}, |
|
"post_processor": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": true, |
|
"trim_offsets": false, |
|
"use_regex": true |
|
}, |
|
"decoder": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": true, |
|
"trim_offsets": true, |
|
"use_regex": true |
|
}, |
|
"model": { |
|
"type": "BPE", |
|
"dropout": null, |
|
"unk_token": null, |
|
"continuing_subword_prefix": null, |
|
"end_of_word_suffix": null, |
|
"fuse_unk": false, |
|
"byte_fallback": false, |
|
"vocab": { |
|
"<[|endoftext|>]": 0, |
|
"...": 1, |
|
"\n": 2, |
|
"A": 3, |
|
"B": 4, |
|
"C": 5, |
|
"D": 6, |
|
"E": 7, |
|
"F": 8, |
|
"G": 9, |
|
"H": 10, |
|
"I": 11, |
|
"J": 12, |
|
"K": 13, |
|
"L": 14, |
|
"M": 15, |
|
"N": 16, |
|
"O": 17, |
|
"P": 18, |
|
"Ċ": 19, |
|
"LL": 20, |
|
"HH": 21, |
|
"AA": 22, |
|
"BB": 23, |
|
"FF": 24, |
|
"KK": 25, |
|
"PP": 26, |
|
"GG": 27, |
|
"NN": 28, |
|
"OO": 29, |
|
"MM": 30, |
|
"CC": 31, |
|
"EE": 32, |
|
"II": 33, |
|
"JJ": 34, |
|
"DD": 35, |
|
"HHHH": 36, |
|
"LLLL": 37, |
|
"AAAA": 38, |
|
"FFFF": 39, |
|
"BBBB": 40, |
|
"KKKK": 41, |
|
"PPPP": 42, |
|
"NNNN": 43, |
|
"OOOO": 44, |
|
"MMMM": 45, |
|
"GGGG": 46, |
|
"EEEE": 47, |
|
"CCCC": 48, |
|
"IIII": 49, |
|
"JJJJ": 50, |
|
"DDDD": 51, |
|
"LLL": 52, |
|
"BBB": 53, |
|
"HHH": 54, |
|
"AAA": 55, |
|
"FFF": 56, |
|
"GGG": 57, |
|
"KKK": 58, |
|
"CCC": 59, |
|
"MMM": 60, |
|
"NNN": 61, |
|
"PPP": 62, |
|
"OOO": 63, |
|
"EEE": 64, |
|
"III": 65, |
|
"JJJ": 66, |
|
"DDD": 67, |
|
"HHHHHHHH": 68, |
|
"FFFFFFFF": 69, |
|
"AAAAAAAA": 70, |
|
"BBBBB": 71, |
|
"LLLLLLLL": 72, |
|
"KKKKKKKK": 73, |
|
"NNNNNNNN": 74, |
|
"PPPPP": 75, |
|
"GGGGG": 76, |
|
"CCCCC": 77, |
|
"MMMMMMMM": 78, |
|
"LLLLL": 79, |
|
"OOOOO": 80, |
|
"HHHHH": 81, |
|
"EEEEE": 82, |
|
"AAAAA": 83, |
|
"IIIII": 84, |
|
"DDDDD": 85, |
|
"FFFFF": 86, |
|
"JJJJJ": 87, |
|
"KKKKK": 88, |
|
"BBBBBBBB": 89, |
|
"NNNNN": 90, |
|
"MMMMM": 91, |
|
"OOOOOOOO": 92, |
|
"PPPPPPPP": 93, |
|
"EEEEEEEE": 94, |
|
"JJJJJJJJ": 95, |
|
"IIIIIIII": 96, |
|
"GGGGGGGG": 97, |
|
"CCCCCCCC": 98, |
|
"LLLLLL": 99, |
|
"DDDDDDDD": 100, |
|
"AAAAAA": 101, |
|
"HHHHHH": 102, |
|
"BBBBBB": 103, |
|
"FFFFFF": 104, |
|
"KKKKKK": 105, |
|
"PPPPPP": 106, |
|
"GGGGGG": 107, |
|
"NNNNNN": 108, |
|
"EEEEEE": 109, |
|
"DDDDDD": 110, |
|
"OOOOOO": 111, |
|
"CCCCCC": 112, |
|
"MMMMMM": 113, |
|
"JJJJJJ": 114, |
|
"IIIIII": 115, |
|
"LLLLLLL": 116, |
|
"AAAAAAA": 117, |
|
"HHHHHHH": 118, |
|
"BBBBBBB": 119, |
|
"FFFFFFF": 120, |
|
"KKKKKKK": 121, |
|
"PPPPPPP": 122, |
|
"MMMMMMM": 123, |
|
"OOOOOOO": 124, |
|
"GGGGGGG": 125, |
|
"CCCCCCC": 126, |
|
"NNNNNNN": 127, |
|
"EEEEEEE": 128, |
|
"IIIIIII": 129, |
|
"JJJJJJJ": 130, |
|
"DDDDDDD": 131, |
|
"FFFFFFFFF": 132, |
|
"AAAAAAAAA": 133, |
|
"HHHHHHHHH": 134, |
|
"BBBBBBBBB": 135, |
|
"LLLLLLLLL": 136, |
|
"KKKKKKKKK": 137, |
|
"OOOOOOOOO": 138, |
|
"NNNNNNNNN": 139, |
|
"PPPPPPPPP": 140, |
|
"GGGGGGGGG": 141, |
|
"CCCCCCCCC": 142, |
|
"MMMMMMMMM": 143, |
|
"IIIIIIIII": 144, |
|
"DDDDDDDDD": 145, |
|
"EEEEEEEEE": 146, |
|
"JJJJJJJJJ": 147, |
|
"HHHHHHHHHH": 148, |
|
"FFFFFFFFFF": 149, |
|
"AAAAAAAAAA": 150, |
|
"LLLLLLLLLL": 151, |
|
"KKKKKKKKKK": 152, |
|
"BBBBBBBBBB": 153, |
|
"PPPPPPPPPP": 154, |
|
"OOOOOOOOOO": 155, |
|
"HHHHHHHHHHH": 156, |
|
"NNNNNNNNNN": 157, |
|
"EEEEEEEEEE": 158, |
|
"MMMMMMMMMM": 159, |
|
"FFFFFFFFFFF": 160, |
|
"JJJJJJJJJJ": 161, |
|
"DDDDDDDDDD": 162, |
|
"CCCCCCCCCC": 163, |
|
"IIIIIIIIII": 164, |
|
"GGGGGGGGGG": 165, |
|
"KKKKKKKKKKK": 166, |
|
"AAAAAAAAAAA": 167, |
|
"LLLLLLLLLLL": 168, |
|
"BBBBBBBBBBB": 169, |
|
"FFFFFFFFFFFFFFFF": 170, |
|
"NNNNNNNNNNN": 171, |
|
"MMMMMMMMMMM": 172, |
|
"OOOOOOOOOOO": 173, |
|
"PPPPPPPPPPP": 174, |
|
"HHHHHHHHHHHHHHHH": 175, |
|
"EEEEEEEEEEE": 176, |
|
"JJJJJJJJJJJ": 177, |
|
"IIIIIIIIIII": 178, |
|
"AAAAAAAAAAAAAAAA": 179, |
|
"CCCCCCCCCCC": 180, |
|
"GGGGGGGGGGG": 181, |
|
"LLLLLLLLLLLLLLLL": 182, |
|
"DDDDDDDDDDD": 183, |
|
"HHHHHHHHHHHH": 184, |
|
"KKKKKKKKKKKKKKKK": 185, |
|
"AAAAAAAAAAAA": 186, |
|
"LLLMMM": 187, |
|
"MMMMMMMMMMMMMMMM": 188, |
|
"NNNNNNNNNNNNNNNN": 189, |
|
"BBBBBBBBBBBBBBBB": 190, |
|
"LLLHHH": 191, |
|
"FFFFFFFFFFFF": 192, |
|
"JJJJJJJJJJJJJJJJ": 193, |
|
"LLLCCC": 194, |
|
"LLLLLLLLLLLL": 195, |
|
"LLLBBB": 196, |
|
"KKKKKKKKKKKK": 197, |
|
"IIIIIIIIIIII": 198, |
|
"PPPPPPPPPPPP": 199, |
|
"BBBHHH": 200, |
|
"OOOOOOOOOOOOOOOO": 201, |
|
"BBBBBBBBBBBB": 202, |
|
"EEEEEEEEEEEE": 203, |
|
"AAAOOO": 204, |
|
"NNNNNNNNNNNN": 205, |
|
"DDDDDDDDDDDD": 206, |
|
"MMMMMMMMMMMM": 207, |
|
"CCCCCCCCCCCC": 208, |
|
"MMMCCC": 209, |
|
"GGGGGGGGGGGG": 210, |
|
"BBBLLL": 211, |
|
"FFFFFFFFFFFFF": 212, |
|
"BBBCCC": 213, |
|
"BBBMMM": 214, |
|
"DDDKKK": 215, |
|
"BBBGGG": 216, |
|
"LLLGGG": 217, |
|
"OOOOOOOOOOOO": 218, |
|
"HHHCCC": 219, |
|
"FFFKKK": 220, |
|
"HHHLLL": 221, |
|
"AAAKKK": 222, |
|
"EEEGGG": 223, |
|
"MMMLLL": 224, |
|
"BBBFFF": 225, |
|
"AAAPPP": 226, |
|
"AAAAAAAAAAAAA": 227, |
|
"HHHHHHHHHHHHH": 228, |
|
"AAAHHH": 229, |
|
"JJJJJJJJJJJJ": 230, |
|
"PPPPPPPPPPPPPPPP": 231, |
|
"AAAFFF": 232, |
|
"LLLKKK": 233, |
|
"AAAGGG": 234, |
|
"EEEKKK": 235, |
|
"PPPFFF": 236, |
|
"IIICCC": 237, |
|
"EEEEEEEEEEEEEEEE": 238, |
|
"LLLLCCC": 239, |
|
"KKKKKKKKKKKKK": 240, |
|
"NNNHHH": 241, |
|
"NNNGGG": 242, |
|
"LLLLLLLLLLLLL": 243, |
|
"MMMMMMMMMMMMM": 244, |
|
"BBBKKK": 245, |
|
"AAAJJJ": 246, |
|
"BBBBBBBBBBBBB": 247, |
|
"BBBPPP": 248, |
|
"HHHJJJ": 249, |
|
"NNNKKK": 250, |
|
"IIIGGG": 251, |
|
"LLLLHHHH": 252, |
|
"DDDGGG": 253, |
|
"BBBEEE": 254, |
|
"PPPCCC": 255, |
|
"LLLLHHH": 256, |
|
"IIIFFF": 257, |
|
"BBBOOO": 258, |
|
"BBBJJJ": 259, |
|
"LLLLCCCC": 260, |
|
"GGGJJJ": 261, |
|
"GGGGGGGGGGGGGGGG": 262, |
|
"LLLOOO": 263, |
|
"NNNOOO": 264, |
|
"AAACCC": 265, |
|
"LLLLBBBB": 266, |
|
"DDDCCC": 267, |
|
"MMMHHH": 268, |
|
"IIIIIIIIIIIIIIII": 269 |
|
}, |
|
"merges": [ |
|
"L L", |
|
"H H", |
|
"A A", |
|
"B B", |
|
"F F", |
|
"K K", |
|
"P P", |
|
"G G", |
|
"N N", |
|
"O O", |
|
"M M", |
|
"C C", |
|
"E E", |
|
"I I", |
|
"J J", |
|
"D D", |
|
"HH HH", |
|
"LL LL", |
|
"AA AA", |
|
"FF FF", |
|
"BB BB", |
|
"KK KK", |
|
"PP PP", |
|
"NN NN", |
|
"OO OO", |
|
"MM MM", |
|
"GG GG", |
|
"EE EE", |
|
"CC CC", |
|
"II II", |
|
"JJ JJ", |
|
"DD DD", |
|
"LL L", |
|
"BB B", |
|
"HH H", |
|
"AA A", |
|
"FF F", |
|
"GG G", |
|
"KK K", |
|
"CC C", |
|
"MM M", |
|
"NN N", |
|
"PP P", |
|
"OO O", |
|
"EE E", |
|
"II I", |
|
"JJ J", |
|
"DD D", |
|
"HHHH HHHH", |
|
"FFFF FFFF", |
|
"AAAA AAAA", |
|
"BBBB B", |
|
"LLLL LLLL", |
|
"KKKK KKKK", |
|
"NNNN NNNN", |
|
"PPPP P", |
|
"GGGG G", |
|
"CCCC C", |
|
"MMMM MMMM", |
|
"LLLL L", |
|
"OOOO O", |
|
"HHHH H", |
|
"EEEE E", |
|
"AAAA A", |
|
"IIII I", |
|
"DDDD D", |
|
"FFFF F", |
|
"JJJJ J", |
|
"KKKK K", |
|
"BBBB BBBB", |
|
"NNNN N", |
|
"MMMM M", |
|
"OOOO OOOO", |
|
"PPPP PPPP", |
|
"EEEE EEEE", |
|
"JJJJ JJJJ", |
|
"IIII IIII", |
|
"GGGG GGGG", |
|
"CCCC CCCC", |
|
"LLLL LL", |
|
"DDDD DDDD", |
|
"AAAA AA", |
|
"HHHH HH", |
|
"BBBB BB", |
|
"FFFF FF", |
|
"KKKK KK", |
|
"PPPP PP", |
|
"GGGG GG", |
|
"NNNN NN", |
|
"EEEE EE", |
|
"DDDD DD", |
|
"OOOO OO", |
|
"CCCC CC", |
|
"MMMM MM", |
|
"JJJJ JJ", |
|
"IIII II", |
|
"LLLL LLL", |
|
"AAAA AAA", |
|
"HHHH HHH", |
|
"BBBB BBB", |
|
"FFFF FFF", |
|
"KKKK KKK", |
|
"PPPP PPP", |
|
"MMMM MMM", |
|
"OOOO OOO", |
|
"GGGG GGG", |
|
"CCCC CCC", |
|
"NNNN NNN", |
|
"EEEE EEE", |
|
"IIII III", |
|
"JJJJ JJJ", |
|
"DDDD DDD", |
|
"FFFFFFFF F", |
|
"AAAAAAAA A", |
|
"HHHHHHHH H", |
|
"BBBB BBBBB", |
|
"LLLLLLLL L", |
|
"KKKKKKKK K", |
|
"OOOO OOOOO", |
|
"NNNNNNNN N", |
|
"PPPP PPPPP", |
|
"GGGG GGGGG", |
|
"CCCC CCCCC", |
|
"MMMMMMMM M", |
|
"IIII IIIII", |
|
"DDDD DDDDD", |
|
"EEEE EEEEE", |
|
"JJJJ JJJJJ", |
|
"HHHHHHHH HH", |
|
"FFFFFFFF FF", |
|
"AAAAAAAA AA", |
|
"LLLLLLLL LL", |
|
"KKKKKKKK KK", |
|
"BBBBBBBB BB", |
|
"PPPPPPPP PP", |
|
"OOOOOOOO OO", |
|
"HHHHHHHH HHH", |
|
"NNNNNNNN NN", |
|
"EEEEEEEE EE", |
|
"MMMMMMMM MM", |
|
"FFFFFFFF FFF", |
|
"JJJJJJJJ JJ", |
|
"DDDDDDDD DD", |
|
"CCCCCCCC CC", |
|
"IIIIIIII II", |
|
"GGGGGGGG GG", |
|
"KKKKKKKK KKK", |
|
"AAAAAAAA AAA", |
|
"LLLLLLLL LLL", |
|
"BBBBBBBB BBB", |
|
"FFFFFFFF FFFFFFFF", |
|
"NNNNNNNN NNN", |
|
"MMMMMMMM MMM", |
|
"OOOOOOOO OOO", |
|
"PPPPPPPP PPP", |
|
"HHHHHHHH HHHHHHHH", |
|
"EEEEEEEE EEE", |
|
"JJJJJJJJ JJJ", |
|
"IIIIIIII III", |
|
"AAAAAAAA AAAAAAAA", |
|
"CCCCCCCC CCC", |
|
"GGGGGGGG GGG", |
|
"LLLLLLLL LLLLLLLL", |
|
"DDDDDDDD DDD", |
|
"HHHHHHHH HHHH", |
|
"KKKKKKKK KKKKKKKK", |
|
"AAAAAAAA AAAA", |
|
"LLL MMM", |
|
"MMMMMMMM MMMMMMMM", |
|
"NNNNNNNN NNNNNNNN", |
|
"BBBBBBBB BBBBBBBB", |
|
"LLL HHH", |
|
"FFFFFFFF FFFF", |
|
"JJJJJJJJ JJJJJJJJ", |
|
"LLL CCC", |
|
"LLLLLLLL LLLL", |
|
"LLL BBB", |
|
"KKKKKKKK KKKK", |
|
"IIIIIIII IIII", |
|
"PPPPPPPP PPPP", |
|
"BBB HHH", |
|
"OOOOOOOO OOOOOOOO", |
|
"BBBBBBBB BBBB", |
|
"EEEEEEEE EEEE", |
|
"AAA OOO", |
|
"NNNNNNNN NNNN", |
|
"DDDDDDDD DDDD", |
|
"MMMMMMMM MMMM", |
|
"CCCCCCCC CCCC", |
|
"MMM CCC", |
|
"GGGGGGGG GGGG", |
|
"BBB LLL", |
|
"FFFFFFFF FFFFF", |
|
"BBB CCC", |
|
"BBB MMM", |
|
"DDD KKK", |
|
"BBB GGG", |
|
"LLL GGG", |
|
"OOOOOOOO OOOO", |
|
"HHH CCC", |
|
"FFF KKK", |
|
"HHH LLL", |
|
"AAA KKK", |
|
"EEE GGG", |
|
"MMM LLL", |
|
"BBB FFF", |
|
"AAA PPP", |
|
"AAAAAAAA AAAAA", |
|
"HHHHHHHH HHHHH", |
|
"AAA HHH", |
|
"JJJJJJJJ JJJJ", |
|
"PPPPPPPP PPPPPPPP", |
|
"AAA FFF", |
|
"LLL KKK", |
|
"AAA GGG", |
|
"EEE KKK", |
|
"PPP FFF", |
|
"III CCC", |
|
"EEEEEEEE EEEEEEEE", |
|
"LLLL CCC", |
|
"KKKKKKKK KKKKK", |
|
"NNN HHH", |
|
"NNN GGG", |
|
"LLLLLLLL LLLLL", |
|
"MMMMMMMM MMMMM", |
|
"BBB KKK", |
|
"AAA JJJ", |
|
"BBBBBBBB BBBBB", |
|
"BBB PPP", |
|
"HHH JJJ", |
|
"NNN KKK", |
|
"III GGG", |
|
"LLLL HHHH", |
|
"DDD GGG", |
|
"BBB EEE", |
|
"PPP CCC", |
|
"LLLL HHH", |
|
"III FFF", |
|
"BBB OOO", |
|
"BBB JJJ", |
|
"LLLL CCCC", |
|
"GGG JJJ", |
|
"GGGGGGGG GGGGGGGG", |
|
"LLL OOO", |
|
"NNN OOO", |
|
"AAA CCC", |
|
"LLLL BBBB", |
|
"DDD CCC", |
|
"MMM HHH", |
|
"IIIIIIII IIIIIIII" |
|
] |
|
} |
|
} |