{ "version": "1.0", "truncation": { "direction": "Right", "max_length": 20, "strategy": "LongestFirst", "stride": 0 }, "padding": null, "added_tokens": [ { "id": 0, "content": "<[|endoftext|>]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "...", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "\n", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 350, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": false, "use_regex": true }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "vocab": { "<[|endoftext|>]": 0, "...": 1, "\n": 2, "A": 3, "B": 4, "C": 5, "D": 6, "E": 7, "F": 8, "G": 9, "H": 10, "Ċ": 11, "AA": 12, "EE": 13, "GG": 14, "FF": 15, "BB": 16, "CC": 17, "DD": 18, "HH": 19, "AAAA": 20, "EEEE": 21, "GGGG": 22, "BBBB": 23, "FFFF": 24, "CCCC": 25, "DDDD": 26, "HHHH": 27, "AAA": 28, "EEE": 29, "FFF": 30, "AAAAAAAA": 31, "CCC": 32, "GGG": 33, "BBB": 34, "DDD": 35, "EEEEEEEE": 36, "GGGGGGGG": 37, "BBBBBBBB": 38, "HHH": 39, "CCCCCCCC": 40, "FFFFFFFF": 41, "DDDDDDDD": 42, "HHHHHHHH": 43, "AAAAA": 44, "EEEEE": 45, "FFFFF": 46, "GGGGG": 47, "CCCCC": 48, "BBBBB": 49, "DDDDD": 50, "HHHHH": 51, "AAAAAA": 52, "EEEEEE": 53, "GGGGGG": 54, "FFFFFF": 55, "BBBBBB": 56, "CCCCCC": 57, "DDDDDD": 58, "HHHHHH": 59, "AAAAAAA": 60, "EEEEEEE": 61, "GGGGGGG": 62, "FFFFFFF": 63, "CCCCCCC": 64, "BBBBBBB": 65, "DDDDDDD": 66, "HHHHHHH": 67, "AAAAAAAAA": 68, "EEEEEEEEE": 69, "FFFFFFFFF": 70, "GGGGGGGGG": 71, "BBBBBBBBB": 72, "CCCCCCCCC": 73, "AAAAAAAAAAAAAAAA": 74, "DDDDDDDDD": 75, "EEEEEEEEEEEEEEEE": 76, "HHHHHHHHH": 77, "BBBBBBBBBBBBBBBB": 78, "GGGGGGGGGGGGGGGG": 79, "AAAAAAAAAA": 80, "EEEEEEEEEE": 81, "CCCCCCCCCC": 82, "FFFFFFFFFF": 83, "GGGGGGGGGG": 84, "DDDDDDDDDD": 85, "BBBBBBBBBB": 86, "HHHHHHHHHH": 87, "AAAAAAAAAAA": 88, "CCCCCCCCCCCCCCCC": 89, "FFFFFFFFFFF": 90, "HHHHHHHHHHHHHHHH": 91, "GGGGGGGGGGG": 92, "EEEEEEEEEEE": 93, "DDDDDDDDDDDDDDDD": 94, "BBBBBBBBBBB": 95, "CCCCCCCCCCC": 96, "DDDDDDDDDDD": 97, "AAACCC": 98, "AAAAAAAAAAAA": 99, "AAAEEE": 100, "FFFFFFFFFFFFFFFF": 101, "HHHHHHHHHHH": 102, "FFFEEE": 103, "EEEEEEEEEEEE": 104, "GGGGGGGGGGGG": 105, "BBBBBBBBBBBB": 106, "AAAFFF": 107, "CCCCCCCCCCCC": 108, "AAADDD": 109, "CCCFFF": 110, "AAAGGG": 111, "AAAAAAAAAAAAA": 112, "FFFFFFFFFFFF": 113, "CCCEEE": 114, "DDDDDDDDDDDD": 115, "AAABBB": 116, "DDDEEE": 117, "AAAAEEEE": 118, "GGGEEE": 119, "GGGFFF": 120, "GGGDDD": 121, "HHHHHHHHHHHH": 122, "AAAAEEE": 123, "EEEEEEEEEEEEE": 124, "GGGBBB": 125, "AAAEEEE": 126, "AAACCCC": 127, "AAAACCCC": 128, "GGGGGGGGGGGGG": 129, "AAAHHH": 130, "BBBBBBBBBBBBB": 131, "FFFFEEE": 132, "GGGCCC": 133, "AAAAAAAAAAAAAA": 134, "BBBEEE": 135, "AAAACCC": 136, "FFFCCC": 137, "FFFHHH": 138, "CCCCCCCCCCCCC": 139, "AAAAGGGG": 140, "FFFFEEEE": 141, "FFFBBB": 142, "FFFFCCCC": 143, "DDDBBB": 144, "DDDDDDDDDDDDD": 145, "FFFDDD": 146, "GGGHHH": 147, "AAAADDDD": 148, "GGGGEEE": 149, "FFFFFFFFFFFFF": 150, "DDDHHH": 151, "HHHHHHHHHHHHH": 152, "CCCBBB": 153, "EEEEEEEEEEEEEE": 154, "AAAABBBB": 155, "AAAAFFFF": 156, "AAADDDD": 157, "GGGGEEEE": 158, "GGGGGGGGGGGGGG": 159, "FFFEEEE": 160, "CCCHHH": 161, "GGGGDDDD": 162, "BBBHHH": 163, "AAAADDD": 164, "AAAAFFF": 165, "AAACC": 166, "BBBBEEEE": 167, "AAAABBB": 168, "FFFFDDDD": 169, "CCCDDD": 170, "GGGEEEE": 171, "CCCCEEEE": 172, "AAEEE": 173, "AAAEE": 174, "BBBBDDDD": 175, "CCCCEEE": 176, "AAAAHHHH": 177, "AAAFFFF": 178, "AAAAAAAAAAAAAAA": 179, "GGGGDDD": 180, "BBBBBBBBBBBBBB": 181, "GGGDDDD": 182, "AAAAHHH": 183, "BBBBEEE": 184, "CCCCCCCCCCCCCC": 185, "AAAAAEEEE": 186, "FFFFHHH": 187, "EEEEEEEEEEEEEEE": 188, "BBBDDD": 189, "BBBBCCCC": 190, "BBBBGGGG": 191, "GGGGGGGGGGGGGGG": 192, "GGGCCCC": 193, "AAADD": 194, "FFFFFFFFFFFFFF": 195, "FFFFGGGG": 196, "AAAGGGG": 197, "AAAAEEEEE": 198, "HHHHDDDD": 199, "DDDDDDDDDDDDDD": 200, "FFFFCCC": 201, "FFFCCCC": 202, "HHHEEE": 203, "BBBBBBBBBBBBBBB": 204, "GGGHHHH": 205, "AAAFF": 206, "BBBBHHHH": 207, "GGGEE": 208, "AAAHHHH": 209, "BBBCCC": 210, "FFFFHHHH": 211, "DDDDEEE": 212, "BBBBCCC": 213, "BBBBHHH": 214, "BBBBDDD": 215, "BBBFFF": 216, "GGGGCCCC": 217, "FFFFFEEEE": 218, "AACCC": 219, "AAABB": 220, "HHHHHHHHHHHHHH": 221, "FFEEE": 222, "BBBBFFF": 223, "DDDDEEEE": 224, "BBBBFFFF": 225, "AAAEEEEE": 226, "AAACCCCC": 227, "GGGGHHHH": 228, "DDDCCC": 229, "CCCCCCCCCCCCCCC": 230, "AAAACCCCC": 231, "GGGGGEEEE": 232, "FFFFFFFFFFFFFFF": 233, "AAAAACCCC": 234, "FFFEE": 235, "GGGGHHH": 236, "GGGGCCC": 237, "AAAGG": 238, "AAAAFFFFF": 239, "AAAAAEEEEE": 240, "FFFFBBB": 241, "GGGGBBB": 242, "AAAAADDDD": 243, "BBBBBEEEE": 244, "HHHHEEE": 245, "GGGFF": 246, "AAAAACCCCC": 247, "AAAAGGGGG": 248, "CCCEEEE": 249, "DDDFFF": 250, "GGGAA": 251, "AAAAAEEE": 252, "DDDDDDDDDDDDDDD": 253, "AAAADDDDD": 254, "GGGGFFF": 255, "CCCCDDDD": 256, "GGGDD": 257, "GGGFFFF": 258, "CCEEE": 259, "GGGGFFFF": 260, "HHHHEEEE": 261, "HHHHBBB": 262, "CCCCCEEEE": 263, "AAAABBBBB": 264, "AAAHH": 265, "DDDEEEE": 266, "HHHHHHHHHHHHHHH": 267, "GGEEE": 268, "AAAAADDDDD": 269, "HHHBBB": 270, "BBBBGGGGG": 271, "HHHHCCCC": 272, "FFFDDDD": 273, "BBBBFFFFF": 274, "GGGCC": 275, "BBBEEEE": 276, "GGGBB": 277, "AAAAAAAAAAAAAAAAA": 278, "HHHDDD": 279, "FFFAA": 280, "FFFFFCCCC": 281, "AAAAHHHHH": 282, "GGGGEEEEE": 283, "DDDDDEEEE": 284, "BBBBEEEEE": 285, "DDEEE": 286, "EEEEEEEEEEEEEEEEE": 287, "FFFFCCCCC": 288, "HHHFFF": 289, "BBBBBDDDD": 290, "FFFFDDD": 291, "AAAFFFFF": 292, "FFFFEEEEE": 293, "HHHCCC": 294, "BBBDDDD": 295, "AAAAAAEEEE": 296, "BBBBCCCCC": 297, "FFFCC": 298, "BBBBBEEEEE": 299, "DDDCCCC": 300, "GGGHH": 301, "GGGGGEEE": 302, "AAADDDDD": 303, "GGGGGDDDD": 304, "AAABBBB": 305, "GGGGGGGGGGGGGGGGG": 306, "GGGGFFFFF": 307, "AAAAAHHHH": 308, "FFCCC": 309, "DDDEE": 310, "BBBBBCCCC": 311, "FFFFGGGGG": 312, "GGGGDDDDD": 313, "FFFFFHHHH": 314, "AAAAEEEEEE": 315, "CCCCCEEE": 316, "BBEEE": 317, "BBBBBBBBBBBBBBBBB": 318, "FFFFFDDDD": 319, "GGGGGCCCC": 320, "AAEE": 321, "FFFFFEEE": 322, "BBBCCCC": 323, "AAAAGGG": 324, "DDDDDEEEEE": 325, "FFFGG": 326, "CCCDDDD": 327, "AAAAAHHHHH": 328, "GGGGBBBB": 329, "FFFFHHHHH": 330, "EEEECCCC": 331, "AAAAABBBBB": 332, "GGGGGEEEEE": 333, "AADDD": 334, "FFFFFEEEEE": 335, "FFFFFCCCCC": 336, "BBBBBHHHHH": 337, "EEEEEEEEEEEEEEEEEE": 338, "AAAACCCCCC": 339, "GGGGHHHHH": 340, "AAAAAAEEE": 341, "AACC": 342, "BBBBBEEE": 343, "FFFFDDDDD": 344, "GGGDDDDD": 345, "CCCEE": 346, "AAAAAGGGGG": 347, "FFFFFFEEEE": 348, "AAAGGGGG": 349 }, "merges": [ "A A", "E E", "G G", "F F", "B B", "C C", "D D", "H H", "AA AA", "EE EE", "GG GG", "BB BB", "FF FF", "CC CC", "DD DD", "HH HH", "AA A", "EE E", "FF F", "AAAA AAAA", "CC C", "GG G", "BB B", "DD D", "EEEE EEEE", "GGGG GGGG", "BBBB BBBB", "HH H", "CCCC CCCC", "FFFF FFFF", "DDDD DDDD", "HHHH HHHH", "AAAA A", "EEEE E", "FFFF F", "GGGG G", "CCCC C", "BBBB B", "DDDD D", "HHHH H", "AAAA AA", "EEEE EE", "GGGG GG", "FFFF FF", "BBBB BB", "CCCC CC", "DDDD DD", "HHHH HH", "AAAA AAA", "EEEE EEE", "GGGG GGG", "FFFF FFF", "CCCC CCC", "BBBB BBB", "DDDD DDD", "HHHH HHH", "AAAAAAAA A", "EEEEEEEE E", "FFFFFFFF F", "GGGGGGGG G", "BBBBBBBB B", "CCCCCCCC C", "AAAAAAAA AAAAAAAA", "DDDDDDDD D", "EEEEEEEE EEEEEEEE", "HHHHHHHH H", "BBBBBBBB BBBBBBBB", "GGGGGGGG GGGGGGGG", "AAAAAAAA AA", "EEEEEEEE EE", "CCCCCCCC CC", "FFFFFFFF FF", "GGGGGGGG GG", "DDDDDDDD DD", "BBBBBBBB BB", "HHHHHHHH HH", "AAAAAAAA AAA", "CCCCCCCC CCCCCCCC", "FFFFFFFF FFF", "HHHHHHHH HHHHHHHH", "GGGGGGGG GGG", "EEEEEEEE EEE", "DDDDDDDD DDDDDDDD", "BBBBBBBB BBB", "CCCCCCCC CCC", "DDDDDDDD DDD", "AAA CCC", "AAAAAAAA AAAA", "AAA EEE", "FFFFFFFF FFFFFFFF", "HHHHHHHH HHH", "FFF EEE", "EEEEEEEE EEEE", "GGGGGGGG GGGG", "BBBBBBBB BBBB", "AAA FFF", "CCCCCCCC CCCC", "AAA DDD", "CCC FFF", "AAA GGG", "AAAAAAAA AAAAA", "FFFFFFFF FFFF", "CCC EEE", "DDDDDDDD DDDD", "AAA BBB", "DDD EEE", "AAAA EEEE", "GGG EEE", "GGG FFF", "GGG DDD", "HHHHHHHH HHHH", "AAAA EEE", "EEEEEEEE EEEEE", "GGG BBB", "AAA EEEE", "AAA CCCC", "AAAA CCCC", "GGGGGGGG GGGGG", "AAA HHH", "BBBBBBBB BBBBB", "FFFF EEE", "GGG CCC", "AAAAAAAA AAAAAA", "BBB EEE", "AAAA CCC", "FFF CCC", "FFF HHH", "CCCCCCCC CCCCC", "AAAA GGGG", "FFFF EEEE", "FFF BBB", "FFFF CCCC", "DDD BBB", "DDDDDDDD DDDDD", "FFF DDD", "GGG HHH", "AAAA DDDD", "GGGG EEE", "FFFFFFFF FFFFF", "DDD HHH", "HHHHHHHH HHHHH", "CCC BBB", "EEEEEEEE EEEEEE", "AAAA BBBB", "AAAA FFFF", "AAA DDDD", "GGGG EEEE", "GGGGGGGG GGGGGG", "FFF EEEE", "CCC HHH", "GGGG DDDD", "BBB HHH", "AAAA DDD", "AAAA FFF", "AAA CC", "BBBB EEEE", "AAAA BBB", "FFFF DDDD", "CCC DDD", "GGG EEEE", "CCCC EEEE", "AA EEE", "AAA EE", "BBBB DDDD", "CCCC EEE", "AAAA HHHH", "AAA FFFF", "AAAAAAAA AAAAAAA", "GGGG DDD", "BBBBBBBB BBBBBB", "GGG DDDD", "AAAA HHH", "BBBB EEE", "CCCCCCCC CCCCCC", "AAAAA EEEE", "FFFF HHH", "EEEEEEEE EEEEEEE", "BBB DDD", "BBBB CCCC", "BBBB GGGG", "GGGGGGGG GGGGGGG", "GGG CCCC", "AAA DD", "FFFFFFFF FFFFFF", "FFFF GGGG", "AAA GGGG", "AAAA EEEEE", "HHHH DDDD", "DDDDDDDD DDDDDD", "FFFF CCC", "FFF CCCC", "HHH EEE", "BBBBBBBB BBBBBBB", "GGG HHHH", "AAA FF", "BBBB HHHH", "GGG EE", "AAA HHHH", "BBB CCC", "FFFF HHHH", "DDDD EEE", "BBBB CCC", "BBBB HHH", "BBBB DDD", "BBB FFF", "GGGG CCCC", "FFFFF EEEE", "AA CCC", "AAA BB", "HHHHHHHH HHHHHH", "FF EEE", "BBBB FFF", "DDDD EEEE", "BBBB FFFF", "AAA EEEEE", "AAA CCCCC", "GGGG HHHH", "DDD CCC", "CCCCCCCC CCCCCCC", "AAAA CCCCC", "GGGGG EEEE", "FFFFFFFF FFFFFFF", "AAAAA CCCC", "FFF EE", "GGGG HHH", "GGGG CCC", "AAA GG", "AAAA FFFFF", "AAAAA EEEEE", "FFFF BBB", "GGGG BBB", "AAAAA DDDD", "BBBBB EEEE", "HHHH EEE", "GGG FF", "AAAAA CCCCC", "AAAA GGGGG", "CCC EEEE", "DDD FFF", "GGG AA", "AAAAA EEE", "DDDDDDDD DDDDDDD", "AAAA DDDDD", "GGGG FFF", "CCCC DDDD", "GGG DD", "GGG FFFF", "CC EEE", "GGGG FFFF", "HHHH EEEE", "HHHH BBB", "CCCCC EEEE", "AAAA BBBBB", "AAA HH", "DDD EEEE", "HHHHHHHH HHHHHHH", "GG EEE", "AAAAA DDDDD", "HHH BBB", "BBBB GGGGG", "HHHH CCCC", "FFF DDDD", "BBBB FFFFF", "GGG CC", "BBB EEEE", "GGG BB", "AAAAAAAA AAAAAAAAA", "HHH DDD", "FFF AA", "FFFFF CCCC", "AAAA HHHHH", "GGGG EEEEE", "DDDDD EEEE", "BBBB EEEEE", "DD EEE", "EEEEEEEE EEEEEEEEE", "FFFF CCCCC", "HHH FFF", "BBBBB DDDD", "FFFF DDD", "AAA FFFFF", "FFFF EEEEE", "HHH CCC", "BBB DDDD", "AAAAAA EEEE", "BBBB CCCCC", "FFF CC", "BBBBB EEEEE", "DDD CCCC", "GGG HH", "GGGGG EEE", "AAA DDDDD", "GGGGG DDDD", "AAA BBBB", "GGGGGGGG GGGGGGGGG", "GGGG FFFFF", "AAAAA HHHH", "FF CCC", "DDD EE", "BBBBB CCCC", "FFFF GGGGG", "GGGG DDDDD", "FFFFF HHHH", "AAAA EEEEEE", "CCCCC EEE", "BB EEE", "BBBBBBBB BBBBBBBBB", "FFFFF DDDD", "GGGGG CCCC", "AA EE", "FFFFF EEE", "BBB CCCC", "AAAA GGG", "DDDDD EEEEE", "FFF GG", "CCC DDDD", "AAAAA HHHHH", "GGGG BBBB", "FFFF HHHHH", "EEEE CCCC", "AAAAA BBBBB", "GGGGG EEEEE", "AA DDD", "FFFFF EEEEE", "FFFFF CCCCC", "BBBBB HHHHH", "EEEEEEEEEEEEEEEE EE", "AAAA CCCCCC", "GGGG HHHHH", "AAAAAA EEE", "AA CC", "BBBBB EEE", "FFFF DDDDD", "GGG DDDDD", "CCC EE", "AAAAA GGGGG", "FFFFFF EEEE", "AAA GGGGG" ] } }