speecht5_tts / tokenizer.json
Xenova's picture
Xenova HF staff
Upload tokenizer.json
810ba42
raw
history blame
5.27 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "</s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 79,
"content": "<mask>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 80,
"content": "<ctc_blank>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Precompiled",
"precompiled_charsmap": null
},
"pre_tokenizer": {
"type": "Sequence",
"pretokenizers": [
{
"type": "WhitespaceSplit"
},
{
"type": "Metaspace",
"replacement": "\u2581",
"add_prefix_space": true
},
{
"type": "Split",
"pattern": {
"Regex": ""
},
"behavior": "Isolated",
"invert": false
}
]
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</s>",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</s>",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "</s>",
"type_id": 0
}
}
],
"special_tokens": {
"</s>": {
"id": "</s>",
"ids": [
2
],
"tokens": [
"</s>"
]
}
}
},
"decoder": {
"type": "Metaspace",
"replacement": "\u2581",
"add_prefix_space": true
},
"model": {
"unk_id": 2,
"vocab": {
"<s>": 0,
"<pad>": 1,
"</s>": 2,
"<unk>": 3,
"\u2581": 4,
"e": 5,
"t": 6,
"a": 7,
"o": 8,
"n": 9,
"i": 10,
"h": 11,
"s": 12,
"r": 13,
"d": 14,
"l": 15,
"u": 16,
"c": 17,
"m": 18,
"f": 19,
"w": 20,
"g": 21,
"y": 22,
",": 23,
"p": 24,
"b": 25,
".": 26,
"v": 27,
"k": 28,
"\"": 29,
"I": 30,
"'": 31,
"T": 32,
"A": 33,
"S": 34,
"H": 35,
";": 36,
"x": 37,
"W": 38,
"-": 39,
"B": 40,
"?": 41,
"C": 42,
"M": 43,
"!": 44,
"q": 45,
"j": 46,
"E": 47,
"N": 48,
"P": 49,
"O": 50,
"D": 51,
"L": 52,
"G": 53,
"R": 54,
"F": 55,
"Y": 56,
"z": 57,
"J": 58,
":": 59,
"K": 60,
"U": 61,
"V": 62,
")": 63,
"(": 64,
"Q": 65,
"Z": 66,
"]": 67,
"[": 68,
"X": 69,
"\u2014": 70,
"/": 71,
"\u00e6": 72,
"\u00e9": 73,
"{": 74,
"}": 75,
"\u00ea": 76,
"\u0153": 77,
"\u0304": 78,
"<mask>": 79,
"<ctc_blank>": 80
}
}
}