File size: 991 Bytes
39cafdf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"post_processor": null,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"vocab": {
"<|endoftext|>": 0,
" A": 1,
" B": 2,
" C": 3,
" D": 4,
" E": 5,
" F": 6,
" G": 7,
" H": 8,
" I": 9,
" J": 10,
" K": 11,
" L": 12,
" M": 13,
" N": 14,
" O": 15,
" P": 16,
" Q": 17,
" R": 18,
" S": 19,
" T": 20,
" U": 21,
" V": 22,
" W": 23,
" X": 24,
" Y": 25,
" Z": 26,
" a": 27,
" b": 28,
" c": 29,
" d": 30
},
"merges": []
}
} |