low-resource-language-model-adaptation
/
adapted_tokenizers
/added-opt-yor
/opt_500-add_full-yor-opt
/added_tokens.json
{ | |
" ": 50449, | |
" ": 50706, | |
" Aa": 50712, | |
" Ade": 50576, | |
" Ati": 50662, | |
" Awọn": 50326, | |
" Bí": 50525, | |
" Bíbélì": 50645, | |
" Ile": 50585, | |
" Jèhófà": 50438, | |
" Jé": 50587, | |
" Jésù": 50647, | |
" Kí": 50689, | |
" Kẹ": 50634, | |
" Nai": 50520, | |
" Naijiria": 50557, | |
" Ní": 50556, | |
" Ohun": 50733, | |
" Olo": 50751, | |
" Olu": 50519, | |
" Oṣu": 50740, | |
" Yoru": 50753, | |
" aa": 50371, | |
" aaye": 50583, | |
" afi": 50633, | |
" agba": 50388, | |
" agbara": 50548, | |
" agbe": 50456, | |
" agbegbe": 50577, | |
" ai": 50484, | |
" ako": 50407, | |
" akoko": 50517, | |
" akọ": 50454, | |
" akọkọ": 50611, | |
" ala": 50350, | |
" alaye": 50658, | |
" anfani": 50724, | |
" apa": 50527, | |
" ara": 50339, | |
" ará": 50749, | |
" ati": 50275, | |
" awo": 50594, | |
" awon": 50522, | |
" awọn": 50271, | |
" aye": 50387, | |
" ayelujara": 50551, | |
" ayé": 50722, | |
" aṣ": 50442, | |
" aṣa": 50676, | |
" bayi": 50626, | |
" bii": 50643, | |
" bá": 50366, | |
" bé": 50717, | |
" bí": 50428, | |
" bẹ": 50416, | |
" bẹrẹ": 50582, | |
" bẹẹ": 50705, | |
" bọ": 50524, | |
" dara": 50364, | |
" diẹ": 50458, | |
" dá": 50558, | |
" dọ": 50502, | |
" ee": 50468, | |
" eeyan": 50506, | |
" elo": 50606, | |
" eni": 50409, | |
" eniyan": 50445, | |
" ere": 50553, | |
" ero": 50757, | |
" eto": 50473, | |
" eyi": 50363, | |
" fé": 50719, | |
" fún": 50381, | |
" fẹ": 50419, | |
" fọ": 50739, | |
" gan": 50545, | |
" gb": 50274, | |
" gba": 50318, | |
" gbe": 50386, | |
" gbi": 50538, | |
" gbo": 50322, | |
" gbogbo": 50331, | |
" gbà": 50630, | |
" gbé": 50612, | |
" gbọ": 50487, | |
" gi": 50461, | |
" gẹgẹ": 50644, | |
" han": 50726, | |
" iba": 50500, | |
" ibi": 50390, | |
" ida": 50460, | |
" ifi": 50610, | |
" igba": 50410, | |
" igbe": 50476, | |
" ijọba": 50536, | |
" ilana": 50709, | |
" ile": 50319, | |
" ilee": 50692, | |
" ilu": 50481, | |
" ilé": 50670, | |
" ilẹ": 50541, | |
" imọ": 50544, | |
" inu": 50651, | |
" ipa": 50443, | |
" ipe": 50661, | |
" ipin": 50450, | |
" ipinlẹ": 50575, | |
" ipo": 50503, | |
" iran": 50618, | |
" irin": 50613, | |
" iroyin": 50701, | |
" iru": 50603, | |
" ise": 50746, | |
" ita": 50640, | |
" itan": 50759, | |
" itọ": 50702, | |
" iwa": 50513, | |
" iwadi": 50728, | |
" iwe": 50561, | |
" iya": 50498, | |
" iye": 50681, | |
" iṣ": 50367, | |
" iṣe": 50581, | |
" iṣẹ": 50345, | |
" jade": 50595, | |
" ji": 50486, | |
" julọ": 50459, | |
" jé": 50422, | |
" jẹ": 50290, | |
" jọ": 50673, | |
" kii": 50695, | |
" ku": 50600, | |
" kuro": 50697, | |
" ká": 50439, | |
" ké": 50533, | |
" kékeré": 50716, | |
" kí": 50391, | |
" kò": 50437, | |
" kó": 50406, | |
" kẹ": 50523, | |
" kọ": 50355, | |
" laa": 50494, | |
" laarin": 50601, | |
" lai": 50565, | |
" lati": 50282, | |
" lilo": 50727, | |
" lori": 50321, | |
" lá": 50332, | |
" láti": 50411, | |
" lè": 50413, | |
" lé": 50492, | |
" ló": 50335, | |
" lórí": 50671, | |
" lẹ": 50316, | |
" lẹhin": 50696, | |
" lọ": 50296, | |
" lọwọ": 50530, | |
" maa": 50448, | |
" meji": 50560, | |
" mii": 50550, | |
" miiran": 50573, | |
" máa": 50344, | |
" mé": 50657, | |
" mí": 50435, | |
" mó": 50742, | |
" mú": 50562, | |
" mẹ": 50343, | |
" mọ": 50334, | |
" naa": 50297, | |
" nado": 50624, | |
" nibi": 50599, | |
" nigba": 50401, | |
" nikan": 50652, | |
" nilo": 50677, | |
" ninu": 50320, | |
" nipa": 50333, | |
" nipasẹ": 50453, | |
" nitori": 50482, | |
" nkan": 50664, | |
" nla": 50614, | |
" nn": 50543, | |
" náà": 50362, | |
" nì": 50736, | |
" ní": 50289, | |
" nígbà": 50707, | |
" nínú": 50475, | |
" nípa": 50635, | |
" nọ": 50466, | |
" ofin": 50637, | |
" ohun": 50311, | |
" oju": 50415, | |
" olo": 50462, | |
" olu": 50377, | |
" omi": 50619, | |
" oni": 50395, | |
" orilẹ": 50617, | |
" orin": 50729, | |
" orí": 50639, | |
" owo": 50404, | |
" oṣ": 50621, | |
" oṣu": 50659, | |
" paa": 50688, | |
" pada": 50609, | |
" pata": 50474, | |
" pataki": 50509, | |
" pese": 50755, | |
" pupọ": 50674, | |
" pé": 50317, | |
" pẹ": 50308, | |
" pẹlu": 50315, | |
" pọ": 50667, | |
" ri": 50376, | |
" rè": 50483, | |
" rí": 50505, | |
" rẹ": 50292, | |
" rọ": 50602, | |
" sii": 50579, | |
" silẹ": 50655, | |
" sinu": 50721, | |
" sì": 50365, | |
" sí": 50337, | |
" sọ": 50304, | |
" sọrọ": 50758, | |
" tabi": 50360, | |
" tuntun": 50680, | |
" tà": 50679, | |
" tàbí": 50754, | |
" tá": 50528, | |
" tí": 50307, | |
" tó": 50305, | |
" tún": 50678, | |
" tẹ": 50389, | |
" tọ": 50549, | |
" tọn": 50418, | |
" waye": 50698, | |
" wà": 50397, | |
" wá": 50412, | |
" wó": 50369, | |
" wẹ": 50649, | |
" wọ": 50425, | |
" wọn": 50281, | |
" wọnyi": 50665, | |
" yan": 50653, | |
" yi": 50380, | |
" yii": 50341, | |
" yin": 50493, | |
" yoo": 50327, | |
" yé": 50732, | |
" yìí": 50479, | |
" yẹ": 50373, | |
" yẹn": 50597, | |
" yọ": 50731, | |
" À": 50378, | |
" Àwọn": 50620, | |
" È": 50714, | |
" Ì": 50361, | |
" Ò": 50584, | |
" Ó": 50568, | |
" àti": 50393, | |
" àwọn": 50309, | |
" á": 50480, | |
" è": 50325, | |
" èèyàn": 50569, | |
" ì": 50287, | |
" ìgbà": 50539, | |
" í": 50704, | |
" ò": 50313, | |
" òun": 50672, | |
" ó": 50348, | |
" ń": 50312, | |
" Ṣ": 50383, | |
" ṣ": 50273, | |
" ṣa": 50441, | |
" ṣe": 50278, | |
" ṣi": 50465, | |
" ṣiṣẹ": 50574, | |
" ṣugbọn": 50529, | |
" ṣẹ": 50485, | |
" Ẹ": 50398, | |
" ẹ": 50279, | |
" ẹgbẹ": 50403, | |
" ẹni": 50396, | |
" ẹrọ": 50507, | |
" ẹya": 50687, | |
" Ọ": 50301, | |
" Ọló": 50521, | |
" Ọlọ": 50648, | |
" ọ": 50272, | |
" ọdun": 50434, | |
" ọdún": 50589, | |
" ọhun": 50703, | |
" ọja": 50638, | |
" ọjọ": 50405, | |
" ọkan": 50546, | |
" ọkunrin": 50748, | |
" ọkọ": 50444, | |
" ọlọ": 50490, | |
" ọlọpaa": 50650, | |
" ọmọ": 50330, | |
" ọna": 50452, | |
" ọpọ": 50489, | |
" ọpọlọpọ": 50596, | |
" ọrọ": 50431, | |
" ọwọ": 50738, | |
" “": 50351, | |
".”": 50499, | |
"Awọn": 50408, | |
"aaju": 50745, | |
"binrin": 50586, | |
"bogbo": 50737, | |
"bá": 50663, | |
"bè": 50593, | |
"bé": 50511, | |
"bélì": 50641, | |
"bí": 50472, | |
"bò": 50744, | |
"bẹ": 50340, | |
"bọ": 50491, | |
"dani": 50683, | |
"dara": 50554, | |
"dia": 50750, | |
"dio": 50693, | |
"dun": 50338, | |
"duro": 50632, | |
"dà": 50710, | |
"dá": 50713, | |
"dè": 50518, | |
"dé": 50741, | |
"dì": 50570, | |
"dí": 50540, | |
"dún": 50495, | |
"dọ": 50374, | |
"esu": 50743, | |
"fani": 50690, | |
"fà": 50402, | |
"fẹ": 50384, | |
"gba": 50283, | |
"gbara": 50730, | |
"gbe": 50294, | |
"gbi": 50608, | |
"gbo": 50295, | |
"gbon": 50720, | |
"gbà": 50356, | |
"gbè": 50623, | |
"gbé": 50627, | |
"gbó": 50723, | |
"gbẹ": 50336, | |
"gbọ": 50555, | |
"gbọn": 50368, | |
"gẹ": 50471, | |
"hó": 50417, | |
"inu": 50426, | |
"iwe": 50691, | |
"iṣẹ": 50353, | |
"jade": 50666, | |
"jara": 50516, | |
"jiria": 50552, | |
"já": 50566, | |
"jì": 50598, | |
"jó": 50685, | |
"jú": 50382, | |
"jẹ": 50379, | |
"jọ": 50288, | |
"jọba": 50392, | |
"kere": 50684, | |
"keré": 50711, | |
"koso": 50501, | |
"kunrin": 50514, | |
"kàn": 50508, | |
"kò": 50512, | |
"kó": 50488, | |
"kùn": 50669, | |
"kú": 50694, | |
"kẹ": 50400, | |
"kọ": 50286, | |
"lana": 50590, | |
"laye": 50725, | |
"lujara": 50526, | |
"lá": 50467, | |
"lè": 50399, | |
"lì": 50478, | |
"ló": 50385, | |
"lù": 50580, | |
"lú": 50455, | |
"lẹ": 50277, | |
"lọ": 50293, | |
"lọpọ": 50469, | |
"lọwọ": 50668, | |
"mba": 50656, | |
"mí": 50700, | |
"mó": 50537, | |
"mẹ": 50423, | |
"mọ": 50284, | |
"nà": 50646, | |
"nì": 50628, | |
"ní": 50708, | |
"nú": 50375, | |
"oju": 50660, | |
"omina": 50629, | |
"onu": 50424, | |
"owo": 50463, | |
"paa": 50446, | |
"pada": 50625, | |
"pò": 50592, | |
"pẹ": 50457, | |
"pọ": 50298, | |
"royin": 50535, | |
"rukọ": 50642, | |
"ràn": 50675, | |
"rá": 50532, | |
"rè": 50563, | |
"rí": 50310, | |
"rò": 50394, | |
"rùn": 50718, | |
"rú": 50572, | |
"rẹ": 50314, | |
"rọ": 50300, | |
"siko": 50699, | |
"silẹ": 50631, | |
"siwaju": 50616, | |
"soke": 50735, | |
"sti": 50607, | |
"stẹ": 50636, | |
"sè": 50734, | |
"sì": 50682, | |
"sí": 50547, | |
"sù": 50497, | |
"sẹ": 50323, | |
"sọ": 50420, | |
"tori": 50427, | |
"tà": 50564, | |
"tì": 50534, | |
"tí": 50504, | |
"tò": 50571, | |
"tó": 50588, | |
"tẹ": 50358, | |
"tọ": 50324, | |
"tọn": 50496, | |
"ugbọn": 50440, | |
"waju": 50421, | |
"wà": 50747, | |
"wá": 50752, | |
"wé": 50559, | |
"wò": 50605, | |
"wó": 50451, | |
"wọ": 50299, | |
"wọn": 50268, | |
"yin": 50328, | |
"yà": 50604, | |
"yàn": 50430, | |
"yé": 50464, | |
"yìn": 50510, | |
"yí": 50622, | |
"yẹ": 50349, | |
"yẹn": 50531, | |
"yọ": 50447, | |
"àn": 50303, | |
"àá": 50567, | |
"áa": 50329, | |
"áà": 50347, | |
"èdè": 50756, | |
"èhó": 50429, | |
"èhófà": 50433, | |
"èyàn": 50515, | |
"ìn": 50352, | |
"ìí": 50372, | |
"óò": 50654, | |
"ùn": 50357, | |
"ún": 50302, | |
"̀": 50280, | |
"́": 50276, | |
"̣": 50270, | |
"̣,": 50470, | |
"̣.": 50477, | |
"Ṣ": 50354, | |
"ṣ": 50269, | |
"ṣa": 50686, | |
"ṣe": 50306, | |
"ṣi": 50414, | |
"ṣu": 50436, | |
"ṣẹ": 50285, | |
"ṣọ": 50578, | |
"Ẹ": 50346, | |
"ẹ": 50266, | |
"ẹgẹ": 50542, | |
"ẹn": 50591, | |
"ẹẹ": 50370, | |
"ẹẹrẹ": 50715, | |
"Ọ": 50291, | |
"ọ": 50265, | |
"ọn": 50267, | |
"ọna": 50359, | |
"ọwọ": 50615, | |
"’": 50432, | |
"”": 50342 | |
} | |