{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": true }, { "id": 3, "content": "", "single_word": true, "lstrip": false, "rstrip": false, "normalized": true, "special": true } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "Prepend", "prepend": "▁" }, { "type": "Replace", "pattern": { "String": " " }, "content": "▁" } ] }, "pre_tokenizer": null, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } } ], "special_tokens": {} }, "decoder": { "type": "Sequence", "decoders": [ { "type": "Replace", "pattern": { "String": "▁" }, "content": " " }, { "type": "ByteFallback" }, { "type": "Fuse" }, { "type": "Strip", "content": " ", "start": 1, "stop": 0 } ] }, "model": { "type": "BPE", "dropout": null, "unk_token": "", "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": true, "byte_fallback": true, "vocab": { "": 0, "": 1, "": 2, "": 3, "": 4, "": 5, "\n": 6, "\t": 7, "0": 8, "1": 9, "2": 10, "3": 11, "4": 12, "5": 13, "6": 14, "7": 15, "8": 16, "9": 17, "０": 18, "１": 19, "２": 20, "３": 21, "４": 22, "５": 23, "６": 24, "７": 25, "８": 26, "９": 27, "

": 28, "

": 29, "

": 30, "
": 31, "
": 32, "
": 33, "": 34, "": 35, "": 36, "

": 37, "

": 38, "

": 39, "

": 40, "

": 41, "

": 42, "
": 43, "
": 44, "
": 45, "

": 46, "

": 47, "

": 48, "": 49, "
": 50, "
": 51, "
  • ": 52, "
  • ": 53, "
    • ": 54, "": 55, "": 56, "": 57, "

    ": 58, "

    ": 59, "

    ": 60, "": 61, "": 62, "": 63, "": 64, "": 65, "": 66, "": 67, "": 68, "": 69, "": 70, "": 71, "": 72, "": 74, "": 75, "": 76, "