vits-cmn / tokenizer.json
hans00's picture
Upload folder using huggingface_hub
6c14801 verified
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 53,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Lowercase"
},
{
"type": "Replace",
"pattern": {
"Regex": "[^_ abcdefghijklmnopqrstuwxyz\u00e0\u00e1\u00e8\u00e9\u00ec\u00ed\u00f2\u00f3\u00f9\u00fa\u00fc\u0101\u0113\u011b\u012b\u0144\u014d\u016b\u01ce\u01d0\u01d2\u01d4\u01d8\u01da\u01dc\u1e3f]"
},
"content": ""
},
{
"type": "Strip",
"strip_left": true,
"strip_right": true
},
{
"type": "Replace",
"pattern": {
"Regex": "(?=.)|(?<!^)$"
},
"content": "_"
}
]
},
"pre_tokenizer": {
"type": "Split",
"pattern": {
"Regex": ""
},
"behavior": "Isolated",
"invert": false
},
"post_processor": null,
"decoder": null,
"model": {
"vocab": {
"_": 0,
" ": 1,
"a": 2,
"b": 3,
"c": 4,
"d": 5,
"e": 6,
"f": 7,
"g": 8,
"h": 9,
"i": 10,
"j": 11,
"k": 12,
"l": 13,
"m": 14,
"n": 15,
"o": 16,
"p": 17,
"q": 18,
"r": 19,
"s": 20,
"t": 21,
"u": 22,
"w": 23,
"x": 24,
"y": 25,
"z": 26,
"\u00e0": 27,
"\u00e1": 28,
"\u00e8": 29,
"\u00e9": 30,
"\u00ec": 31,
"\u00ed": 32,
"\u00f2": 33,
"\u00f3": 34,
"\u00f9": 35,
"\u00fa": 36,
"\u00fc": 37,
"\u0101": 38,
"\u0113": 39,
"\u011b": 40,
"\u012b": 41,
"\u0144": 42,
"\u014d": 43,
"\u016b": 44,
"\u01ce": 45,
"\u01d0": 46,
"\u01d2": 47,
"\u01d4": 48,
"\u01d8": 49,
"\u01da": 50,
"\u01dc": 51,
"\u1e3f": 52,
"<unk>": 53
}
}
}