|
{ |
|
"version": "1.0", |
|
"truncation": { |
|
"direction": "Right", |
|
"max_length": 20, |
|
"strategy": "LongestFirst", |
|
"stride": 0 |
|
}, |
|
"padding": null, |
|
"added_tokens": [ |
|
{ |
|
"id": 0, |
|
"content": "<[|endoftext|>]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 1, |
|
"content": "...", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 2, |
|
"content": "\n", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 340, |
|
"content": "<|endoftext|>", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
} |
|
], |
|
"normalizer": null, |
|
"pre_tokenizer": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": false, |
|
"trim_offsets": true, |
|
"use_regex": true |
|
}, |
|
"post_processor": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": true, |
|
"trim_offsets": false, |
|
"use_regex": true |
|
}, |
|
"decoder": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": true, |
|
"trim_offsets": true, |
|
"use_regex": true |
|
}, |
|
"model": { |
|
"type": "BPE", |
|
"dropout": null, |
|
"unk_token": null, |
|
"continuing_subword_prefix": null, |
|
"end_of_word_suffix": null, |
|
"fuse_unk": false, |
|
"byte_fallback": false, |
|
"vocab": { |
|
"<[|endoftext|>]": 0, |
|
"...": 1, |
|
"\n": 2, |
|
"A": 3, |
|
"B": 4, |
|
"C": 5, |
|
"D": 6, |
|
"E": 7, |
|
"F": 8, |
|
"G": 9, |
|
"H": 10, |
|
"I": 11, |
|
"J": 12, |
|
"K": 13, |
|
"L": 14, |
|
"M": 15, |
|
"N": 16, |
|
"O": 17, |
|
"P": 18, |
|
"Q": 19, |
|
"R": 20, |
|
"S": 21, |
|
"T": 22, |
|
"U": 23, |
|
"V": 24, |
|
"W": 25, |
|
"X": 26, |
|
"Y": 27, |
|
"Z": 28, |
|
"[": 29, |
|
"\\": 30, |
|
"]": 31, |
|
"^": 32, |
|
"_": 33, |
|
"`": 34, |
|
"Ċ": 35, |
|
"AA": 36, |
|
"FF": 37, |
|
"HH": 38, |
|
"XX": 39, |
|
"UU": 40, |
|
"BB": 41, |
|
"SS": 42, |
|
"VV": 43, |
|
"OO": 44, |
|
"CC": 45, |
|
"YY": 46, |
|
"LL": 47, |
|
"MM": 48, |
|
"TT": 49, |
|
"NN": 50, |
|
"\\\\": 51, |
|
"]]": 52, |
|
"JJ": 53, |
|
"QQ": 54, |
|
"[[": 55, |
|
"II": 56, |
|
"GG": 57, |
|
"PP": 58, |
|
"``": 59, |
|
"__": 60, |
|
"RR": 61, |
|
"^^": 62, |
|
"DD": 63, |
|
"ZZ": 64, |
|
"KK": 65, |
|
"WW": 66, |
|
"EE": 67, |
|
"AAAA": 68, |
|
"XXXX": 69, |
|
"FFFF": 70, |
|
"HHHH": 71, |
|
"UUUU": 72, |
|
"BBBB": 73, |
|
"VVVV": 74, |
|
"SSSS": 75, |
|
"LLLL": 76, |
|
"MMMM": 77, |
|
"NNNN": 78, |
|
"OOOO": 79, |
|
"CCCC": 80, |
|
"YYYY": 81, |
|
"TTTT": 82, |
|
"\\\\\\\\": 83, |
|
"JJJJ": 84, |
|
"]]]]": 85, |
|
"[[[[": 86, |
|
"QQQQ": 87, |
|
"RRRR": 88, |
|
"IIII": 89, |
|
"GGGG": 90, |
|
"PPPP": 91, |
|
"````": 92, |
|
"____": 93, |
|
"^^^^": 94, |
|
"KKKK": 95, |
|
"ZZZZ": 96, |
|
"DDDD": 97, |
|
"WWWW": 98, |
|
"EEEE": 99, |
|
"FFF": 100, |
|
"HHH": 101, |
|
"AAA": 102, |
|
"UUU": 103, |
|
"YYY": 104, |
|
"SSS": 105, |
|
"OOO": 106, |
|
"XXX": 107, |
|
"TTT": 108, |
|
"BBB": 109, |
|
"]]]": 110, |
|
"\\\\\\": 111, |
|
"CCC": 112, |
|
"GGG": 113, |
|
"QQQ": 114, |
|
"VVV": 115, |
|
"LLL": 116, |
|
"MMM": 117, |
|
"NNN": 118, |
|
"JJJ": 119, |
|
"III": 120, |
|
"[[[": 121, |
|
"DDD": 122, |
|
"___": 123, |
|
"PPP": 124, |
|
"WWW": 125, |
|
"^^^": 126, |
|
"ZZZ": 127, |
|
"EEE": 128, |
|
"```": 129, |
|
"KKK": 130, |
|
"RRR": 131, |
|
"HHHHH": 132, |
|
"FFFFF": 133, |
|
"AAAAA": 134, |
|
"BBBBB": 135, |
|
"XXXXX": 136, |
|
"UUUUU": 137, |
|
"CCCCC": 138, |
|
"OOOOO": 139, |
|
"SSSSS": 140, |
|
"YYYYY": 141, |
|
"VVVVV": 142, |
|
"TTTTT": 143, |
|
"LLLLL": 144, |
|
"\\\\\\\\\\": 145, |
|
"JJJJJ": 146, |
|
"NNNNN": 147, |
|
"]]]]]": 148, |
|
"MMMMM": 149, |
|
"GGGGG": 150, |
|
"QQQQQ": 151, |
|
"[[[[[": 152, |
|
"IIIII": 153, |
|
"`````": 154, |
|
"PPPPP": 155, |
|
"^^^^^": 156, |
|
"_____": 157, |
|
"DDDDD": 158, |
|
"XXXXXXXX": 159, |
|
"KKKKK": 160, |
|
"AAAAAAAA": 161, |
|
"RRRRRRRR": 162, |
|
"ZZZZZ": 163, |
|
"EEEEE": 164, |
|
"WWWWW": 165, |
|
"VVVVVVVV": 166, |
|
"UUUUUUUU": 167, |
|
"HHHHHH": 168, |
|
"FFFFFFFF": 169, |
|
"BBBBBB": 170, |
|
"RRRRR": 171, |
|
"MMMMMM": 172, |
|
"CCCCCC": 173, |
|
"SSSSSS": 174, |
|
"NNNNNNNN": 175, |
|
"AAAAAA": 176, |
|
"LLLLLL": 177, |
|
"OOOOOO": 178, |
|
"FFFFFF": 179, |
|
"]]]]]]": 180, |
|
"YYYYYY": 181, |
|
"JJJJJJ": 182, |
|
"\\\\\\\\\\\\": 183, |
|
"TTTTTT": 184, |
|
"XXXXXX": 185, |
|
"[[[[[[": 186, |
|
"IIIIII": 187, |
|
"BBBBBBBB": 188, |
|
"QQQQQQ": 189, |
|
"UUUUUU": 190, |
|
"GGGGGG": 191, |
|
"HHHHHHHH": 192, |
|
"PPPPPP": 193, |
|
"MMMMMMMM": 194, |
|
"``````": 195, |
|
"KKKKKK": 196, |
|
"VVVVVV": 197, |
|
"^^^^^^": 198, |
|
"NNNNNN": 199, |
|
"______": 200, |
|
"SSSSSSSS": 201, |
|
"ZZZZZZ": 202, |
|
"LLLLLLLL": 203, |
|
"EEEEEE": 204, |
|
"DDDDDD": 205, |
|
"WWWWWW": 206, |
|
"OOOOOOOO": 207, |
|
"\\\\\\\\\\\\\\\\": 208, |
|
"YYYYYYYY": 209, |
|
"TTTTTTTT": 210, |
|
"CCCCCCCC": 211, |
|
"RRRRRR": 212, |
|
"[[[[[[[[": 213, |
|
"JJJJJJJJ": 214, |
|
"AAAAAAA": 215, |
|
"]]]]]]]]": 216, |
|
"IIIIIIII": 217, |
|
"QQQQQQQQ": 218, |
|
"FFFFFFF": 219, |
|
"XXXXXXX": 220, |
|
"HHHHHHH": 221, |
|
"````````": 222, |
|
"PPPPPPPP": 223, |
|
"BBBBBBB": 224, |
|
"UUUUUUU": 225, |
|
"LLLLLLL": 226, |
|
"TTTTTTT": 227, |
|
"VVVVVVV": 228, |
|
"NNNNNNN": 229, |
|
"KKKKKKKK": 230, |
|
"________": 231, |
|
"CCCCCCC": 232, |
|
"\\\\\\\\\\\\\\": 233, |
|
"SSSSSSS": 234, |
|
"^^^^^^^^": 235, |
|
"MMMMMMM": 236, |
|
"OOOOOOO": 237, |
|
"YYYYYYY": 238, |
|
"GGGGGGGG": 239, |
|
"]]]]]]]": 240, |
|
"QQQQQQQ": 241, |
|
"[[[[[[[": 242, |
|
"IIIIIII": 243, |
|
"JJJJJJJ": 244, |
|
"DDDDDDD": 245, |
|
"ZZZZZZZZ": 246, |
|
"WWWWWWWW": 247, |
|
"RRRRRRR": 248, |
|
"EEEEEEEE": 249, |
|
"```````": 250, |
|
"GGGGGGG": 251, |
|
"PPPPPPP": 252, |
|
"^^^^^^^": 253, |
|
"KKKKKKK": 254, |
|
"_______": 255, |
|
"ZZZZZZZ": 256, |
|
"WWWWWWW": 257, |
|
"EEEEEEE": 258, |
|
"DDDDDDDD": 259, |
|
"AAAAAAAAA": 260, |
|
"FFFFFFFFF": 261, |
|
"XXXXXXXXX": 262, |
|
"UUUUUUUUU": 263, |
|
"HHHHHHHHH": 264, |
|
"BBBBBBBBB": 265, |
|
"LLLLLLLLL": 266, |
|
"VVVVVVVVV": 267, |
|
"NNNNNNNNN": 268, |
|
"SSSSSSSSS": 269, |
|
"MMMMMMMMM": 270, |
|
"OOOOOOOOO": 271, |
|
"TTTTTTTTT": 272, |
|
"CCCCCCCCC": 273, |
|
"[[[[[[[[[": 274, |
|
"\\\\\\\\\\\\\\\\\\": 275, |
|
"RRRRRRRRR": 276, |
|
"YYYYYYYYY": 277, |
|
"JJJJJJJJJ": 278, |
|
"]]]]]]]]]": 279, |
|
"QQQQQQQQQ": 280, |
|
"AAAAAAAAAA": 281, |
|
"IIIIIIIII": 282, |
|
"KKKKKKKKK": 283, |
|
"`````````": 284, |
|
"XXXXXXXXXX": 285, |
|
"VVVVVVVVVV": 286, |
|
"_________": 287, |
|
"GGGGGGGGG": 288, |
|
"PPPPPPPPP": 289, |
|
"UUUUUUUUUU": 290, |
|
"HHHHHHHHHH": 291, |
|
"^^^^^^^^^": 292, |
|
"BBBBBBBBBB": 293, |
|
"NNNNNNNNNN": 294, |
|
"WWWWWWWWW": 295, |
|
"LLLLLLLLLL": 296, |
|
"FFFFFFFFFF": 297, |
|
"MMMMMMMMMM": 298, |
|
"EEEEEEEEE": 299, |
|
"ZZZZZZZZZ": 300, |
|
"SSSSSSSSSS": 301, |
|
"DDDDDDDDD": 302, |
|
"CCCCCCCCCC": 303, |
|
"AAAAAAAAAAA": 304, |
|
"XXXXXXXXXXX": 305, |
|
"JJJJJJJJJJ": 306, |
|
"YYYYYYYYYY": 307, |
|
"\\\\\\\\\\\\\\\\\\\\": 308, |
|
"TTTTTTTTTT": 309, |
|
"QQQQQQQQQQ": 310, |
|
"RRRRRRRRRR": 311, |
|
"[[[[[[[[[[": 312, |
|
"]]]]]]]]]]": 313, |
|
"UUUUUUUUUUU": 314, |
|
"OOOOOOOOOO": 315, |
|
"KKKKKKKKKK": 316, |
|
"HHHHHHHHHHH": 317, |
|
"FFFFFFFFFFF": 318, |
|
"IIIIIIIIII": 319, |
|
"VVVVVVVVVVV": 320, |
|
"BBBBBBBBBBB": 321, |
|
"NNNNNNNNNNN": 322, |
|
"RRRRRRRRRRR": 323, |
|
"PPPPPPPPPP": 324, |
|
"GGGGGGGGGG": 325, |
|
"``````````": 326, |
|
"^^^^^^^^^^": 327, |
|
"MMMMMMMMMMM": 328, |
|
"LLLLLLLLLLL": 329, |
|
"__________": 330, |
|
"ZZZZZZZZZZ": 331, |
|
"WWWWWWWWWW": 332, |
|
"JJJJJJJJJJJ": 333, |
|
"AAAAAAAAAAAA": 334, |
|
"XXXXXXXXXXXX": 335, |
|
"DDDDDDDDDD": 336, |
|
"SSSSSSSSSSS": 337, |
|
"RRRRRRRRRRRRRRRR": 338, |
|
"\\\\\\\\\\\\\\\\\\\\\\": 339 |
|
}, |
|
"merges": [ |
|
"A A", |
|
"F F", |
|
"H H", |
|
"X X", |
|
"U U", |
|
"B B", |
|
"S S", |
|
"V V", |
|
"O O", |
|
"C C", |
|
"Y Y", |
|
"L L", |
|
"M M", |
|
"T T", |
|
"N N", |
|
"\\ \\", |
|
"] ]", |
|
"J J", |
|
"Q Q", |
|
"[ [", |
|
"I I", |
|
"G G", |
|
"P P", |
|
"` `", |
|
"_ _", |
|
"R R", |
|
"^ ^", |
|
"D D", |
|
"Z Z", |
|
"K K", |
|
"W W", |
|
"E E", |
|
"AA AA", |
|
"XX XX", |
|
"FF FF", |
|
"HH HH", |
|
"UU UU", |
|
"BB BB", |
|
"VV VV", |
|
"SS SS", |
|
"LL LL", |
|
"MM MM", |
|
"NN NN", |
|
"OO OO", |
|
"CC CC", |
|
"YY YY", |
|
"TT TT", |
|
"\\\\ \\\\", |
|
"JJ JJ", |
|
"]] ]]", |
|
"[[ [[", |
|
"QQ QQ", |
|
"RR RR", |
|
"II II", |
|
"GG GG", |
|
"PP PP", |
|
"`` ``", |
|
"__ __", |
|
"^^ ^^", |
|
"KK KK", |
|
"ZZ ZZ", |
|
"DD DD", |
|
"WW WW", |
|
"EE EE", |
|
"FF F", |
|
"HH H", |
|
"AA A", |
|
"UU U", |
|
"YY Y", |
|
"SS S", |
|
"OO O", |
|
"XX X", |
|
"TT T", |
|
"BB B", |
|
"]] ]", |
|
"\\\\ \\", |
|
"CC C", |
|
"GG G", |
|
"QQ Q", |
|
"VV V", |
|
"LL L", |
|
"MM M", |
|
"NN N", |
|
"JJ J", |
|
"II I", |
|
"[[ [", |
|
"DD D", |
|
"__ _", |
|
"PP P", |
|
"WW W", |
|
"^^ ^", |
|
"ZZ Z", |
|
"EE E", |
|
"`` `", |
|
"KK K", |
|
"RR R", |
|
"HHHH H", |
|
"FFFF F", |
|
"AAAA A", |
|
"BBBB B", |
|
"XXXX X", |
|
"UUUU U", |
|
"CCCC C", |
|
"OOOO O", |
|
"SSSS S", |
|
"YYYY Y", |
|
"VVVV V", |
|
"TTTT T", |
|
"LLLL L", |
|
"\\\\\\\\ \\", |
|
"JJJJ J", |
|
"NNNN N", |
|
"]]]] ]", |
|
"MMMM M", |
|
"GGGG G", |
|
"QQQQ Q", |
|
"[[[[ [", |
|
"IIII I", |
|
"```` `", |
|
"PPPP P", |
|
"^^^^ ^", |
|
"____ _", |
|
"DDDD D", |
|
"XXXX XXXX", |
|
"KKKK K", |
|
"AAAA AAAA", |
|
"RRRR RRRR", |
|
"ZZZZ Z", |
|
"EEEE E", |
|
"WWWW W", |
|
"VVVV VVVV", |
|
"UUUU UUUU", |
|
"HHHH HH", |
|
"FFFF FFFF", |
|
"BBBB BB", |
|
"RRRR R", |
|
"MMMM MM", |
|
"CCCC CC", |
|
"SSSS SS", |
|
"NNNN NNNN", |
|
"AAAA AA", |
|
"LLLL LL", |
|
"OOOO OO", |
|
"FFFF FF", |
|
"]]]] ]]", |
|
"YYYY YY", |
|
"JJJJ JJ", |
|
"\\\\\\\\ \\\\", |
|
"TTTT TT", |
|
"XXXX XX", |
|
"[[[[ [[", |
|
"IIII II", |
|
"BBBB BBBB", |
|
"QQQQ QQ", |
|
"UUUU UU", |
|
"GGGG GG", |
|
"HHHH HHHH", |
|
"PPPP PP", |
|
"MMMM MMMM", |
|
"```` ``", |
|
"KKKK KK", |
|
"VVVV VV", |
|
"^^^^ ^^", |
|
"NNNN NN", |
|
"____ __", |
|
"SSSS SSSS", |
|
"ZZZZ ZZ", |
|
"LLLL LLLL", |
|
"EEEE EE", |
|
"DDDD DD", |
|
"WWWW WW", |
|
"OOOO OOOO", |
|
"\\\\\\\\ \\\\\\\\", |
|
"YYYY YYYY", |
|
"TTTT TTTT", |
|
"CCCC CCCC", |
|
"RRRR RR", |
|
"[[[[ [[[[", |
|
"JJJJ JJJJ", |
|
"AAAA AAA", |
|
"]]]] ]]]]", |
|
"IIII IIII", |
|
"QQQQ QQQQ", |
|
"FFFF FFF", |
|
"XXXX XXX", |
|
"HHHH HHH", |
|
"```` ````", |
|
"PPPP PPPP", |
|
"BBBB BBB", |
|
"UUUU UUU", |
|
"LLLL LLL", |
|
"TTTT TTT", |
|
"VVVV VVV", |
|
"NNNN NNN", |
|
"KKKK KKKK", |
|
"____ ____", |
|
"CCCC CCC", |
|
"\\\\\\\\ \\\\\\", |
|
"SSSS SSS", |
|
"^^^^ ^^^^", |
|
"MMMM MMM", |
|
"OOOO OOO", |
|
"YYYY YYY", |
|
"GGGG GGGG", |
|
"]]]] ]]]", |
|
"QQQQ QQQ", |
|
"[[[[ [[[", |
|
"IIII III", |
|
"JJJJ JJJ", |
|
"DDDD DDD", |
|
"ZZZZ ZZZZ", |
|
"WWWW WWWW", |
|
"RRRR RRR", |
|
"EEEE EEEE", |
|
"```` ```", |
|
"GGGG GGG", |
|
"PPPP PPP", |
|
"^^^^ ^^^", |
|
"KKKK KKK", |
|
"____ ___", |
|
"ZZZZ ZZZ", |
|
"WWWW WWW", |
|
"EEEE EEE", |
|
"DDDD DDDD", |
|
"AAAA AAAAA", |
|
"FFFF FFFFF", |
|
"XXXX XXXXX", |
|
"UUUU UUUUU", |
|
"HHHH HHHHH", |
|
"BBBB BBBBB", |
|
"LLLL LLLLL", |
|
"VVVV VVVVV", |
|
"NNNN NNNNN", |
|
"SSSS SSSSS", |
|
"MMMM MMMMM", |
|
"OOOO OOOOO", |
|
"TTTT TTTTT", |
|
"CCCC CCCCC", |
|
"[[[[ [[[[[", |
|
"\\\\\\\\ \\\\\\\\\\", |
|
"RRRRRRRR R", |
|
"YYYY YYYYY", |
|
"JJJJ JJJJJ", |
|
"]]]] ]]]]]", |
|
"QQQQ QQQQQ", |
|
"AAAAAAAA AA", |
|
"IIII IIIII", |
|
"KKKK KKKKK", |
|
"```` `````", |
|
"XXXXXXXX XX", |
|
"VVVVVVVV VV", |
|
"____ _____", |
|
"GGGG GGGGG", |
|
"PPPP PPPPP", |
|
"UUUUUUUU UU", |
|
"HHHH HHHHHH", |
|
"^^^^ ^^^^^", |
|
"BBBB BBBBBB", |
|
"NNNNNNNN NN", |
|
"WWWW WWWWW", |
|
"LLLL LLLLLL", |
|
"FFFFFFFF FF", |
|
"MMMM MMMMMM", |
|
"EEEE EEEEE", |
|
"ZZZZ ZZZZZ", |
|
"SSSS SSSSSS", |
|
"DDDD DDDDD", |
|
"CCCC CCCCCC", |
|
"AAAAAAAA AAA", |
|
"XXXXXXXX XXX", |
|
"JJJJ JJJJJJ", |
|
"YYYY YYYYYY", |
|
"\\\\\\\\ \\\\\\\\\\\\", |
|
"TTTT TTTTTT", |
|
"QQQQ QQQQQQ", |
|
"RRRRRRRR RR", |
|
"[[[[ [[[[[[", |
|
"]]]] ]]]]]]", |
|
"UUUUUUUU UUU", |
|
"OOOO OOOOOO", |
|
"KKKK KKKKKK", |
|
"HHHHHHHH HHH", |
|
"FFFFFFFF FFF", |
|
"IIII IIIIII", |
|
"VVVVVVVV VVV", |
|
"BBBBBBBB BBB", |
|
"NNNNNNNN NNN", |
|
"RRRRRRRR RRR", |
|
"PPPP PPPPPP", |
|
"GGGG GGGGGG", |
|
"```` ``````", |
|
"^^^^ ^^^^^^", |
|
"MMMMMMMM MMM", |
|
"LLLLLLLL LLL", |
|
"____ ______", |
|
"ZZZZ ZZZZZZ", |
|
"WWWW WWWWWW", |
|
"JJJJJJJJ JJJ", |
|
"AAAAAAAA AAAA", |
|
"XXXXXXXX XXXX", |
|
"DDDD DDDDDD", |
|
"SSSSSSSS SSS", |
|
"RRRRRRRR RRRRRRRR", |
|
"\\\\\\\\\\\\\\\\ \\\\\\" |
|
] |
|
} |
|
} |