Upload 7 files
50f8ba6
verified
|
{ |
|
"version": "1.0", |
|
"truncation": null, |
|
"padding": null, |
|
"added_tokens": [], |
|
"normalizer": { |
|
"type": "Sequence", |
|
"normalizers": [ |
|
{ |
|
"type": "Lowercase" |
|
}, |
|
{ |
|
"type": "Replace", |
|
"pattern": { |
|
"Regex": "(?=.)|(?<!^)$" |
|
}, |
|
"content": "_" |
|
} |
|
] |
|
}, |
|
"pre_tokenizer": { |
|
"type": "Split", |
|
"pattern": { |
|
"Regex": "" |
|
}, |
|
"behavior": "Isolated", |
|
"invert": false |
|
}, |
|
"post_processor": null, |
|
"decoder": null, |
|
"model": { |
|
"vocab": { |
|
"_": 0, |
|
";": 1, |
|
":": 2, |
|
",": 3, |
|
".": 4, |
|
"!": 5, |
|
"?": 6, |
|
"\u00a1": 7, |
|
"\u00bf": 8, |
|
"\u2014": 9, |
|
"\u2026": 10, |
|
"\"": 11, |
|
"\u00ab": 12, |
|
"\u00bb": 13, |
|
"\u201c": 14, |
|
"\u201d": 15, |
|
" ": 16, |
|
"A": 17, |
|
"B": 18, |
|
"C": 19, |
|
"D": 20, |
|
"E": 21, |
|
"F": 22, |
|
"G": 23, |
|
"H": 24, |
|
"I": 25, |
|
"J": 26, |
|
"K": 27, |
|
"L": 28, |
|
"M": 29, |
|
"N": 30, |
|
"O": 31, |
|
"P": 32, |
|
"Q": 33, |
|
"R": 34, |
|
"S": 35, |
|
"T": 36, |
|
"U": 37, |
|
"V": 38, |
|
"W": 39, |
|
"X": 40, |
|
"Y": 41, |
|
"Z": 42, |
|
"a": 43, |
|
"b": 44, |
|
"c": 45, |
|
"d": 46, |
|
"e": 47, |
|
"f": 48, |
|
"g": 49, |
|
"h": 50, |
|
"i": 51, |
|
"j": 52, |
|
"k": 53, |
|
"l": 54, |
|
"m": 55, |
|
"n": 56, |
|
"o": 57, |
|
"p": 58, |
|
"q": 59, |
|
"r": 60, |
|
"s": 61, |
|
"t": 62, |
|
"u": 63, |
|
"v": 64, |
|
"w": 65, |
|
"x": 66, |
|
"y": 67, |
|
"z": 68, |
|
"\u0251": 69, |
|
"\u0250": 70, |
|
"\u0252": 71, |
|
"\u00e6": 72, |
|
"\u0253": 73, |
|
"\u0299": 74, |
|
"\u03b2": 75, |
|
"\u0254": 76, |
|
"\u0255": 77, |
|
"\u00e7": 78, |
|
"\u0257": 79, |
|
"\u0256": 80, |
|
"\u00f0": 81, |
|
"\u02a4": 82, |
|
"\u0259": 83, |
|
"\u0258": 84, |
|
"\u025a": 85, |
|
"\u025b": 86, |
|
"\u025c": 87, |
|
"\u025d": 88, |
|
"\u025e": 89, |
|
"\u025f": 90, |
|
"\u0284": 91, |
|
"\u0261": 92, |
|
"\u0260": 93, |
|
"\u0262": 94, |
|
"\u029b": 95, |
|
"\u0266": 96, |
|
"\u0267": 97, |
|
"\u0127": 98, |
|
"\u0265": 99, |
|
"\u029c": 100, |
|
"\u0268": 101, |
|
"\u026a": 102, |
|
"\u029d": 103, |
|
"\u026d": 104, |
|
"\u026c": 105, |
|
"\u026b": 106, |
|
"\u026e": 107, |
|
"\u029f": 108, |
|
"\u0271": 109, |
|
"\u026f": 110, |
|
"\u0270": 111, |
|
"\u014b": 112, |
|
"\u0273": 113, |
|
"\u0272": 114, |
|
"\u0274": 115, |
|
"\u00f8": 116, |
|
"\u0275": 117, |
|
"\u0278": 118, |
|
"\u03b8": 119, |
|
"\u0153": 120, |
|
"\u0276": 121, |
|
"\u0298": 122, |
|
"\u0279": 123, |
|
"\u027a": 124, |
|
"\u027e": 125, |
|
"\u027b": 126, |
|
"\u0280": 127, |
|
"\u0281": 128, |
|
"\u027d": 129, |
|
"\u0282": 130, |
|
"\u0283": 131, |
|
"\u0288": 132, |
|
"\u02a7": 133, |
|
"\u0289": 134, |
|
"\u028a": 135, |
|
"\u028b": 136, |
|
"\u2c71": 137, |
|
"\u028c": 138, |
|
"\u0263": 139, |
|
"\u0264": 140, |
|
"\u028d": 141, |
|
"\u03c7": 142, |
|
"\u028e": 143, |
|
"\u028f": 144, |
|
"\u0291": 145, |
|
"\u0290": 146, |
|
"\u0292": 147, |
|
"\u0294": 148, |
|
"\u02a1": 149, |
|
"\u0295": 150, |
|
"\u02a2": 151, |
|
"\u01c0": 152, |
|
"\u01c1": 153, |
|
"\u01c2": 154, |
|
"\u01c3": 155, |
|
"\u02c8": 156, |
|
"\u02cc": 157, |
|
"\u02d0": 158, |
|
"\u02d1": 159, |
|
"\u02bc": 160, |
|
"\u02b4": 161, |
|
"\u02b0": 162, |
|
"\u02b1": 163, |
|
"\u02b2": 164, |
|
"\u02b7": 165, |
|
"\u02e0": 166, |
|
"\u02e4": 167, |
|
"\u02de": 168, |
|
"\u2193": 169, |
|
"\u2191": 170, |
|
"\u2192": 171, |
|
"\u2197": 172, |
|
"\u2198": 173, |
|
"null": 174, |
|
"\u0329": 175, |
|
"'": 176 |
|
} |
|
}, |
|
"phonemize": true |
|
} |
|
|
