sage-m2m100-1.2B / added_tokens.json
ai-forever's picture
Upload tokenizer
87eb5bd verified
{
"__af__": 14241,
"__am__": 14242,
"__ar__": 14243,
"__ast__": 14244,
"__az__": 14245,
"__ba__": 14246,
"__be__": 14247,
"__bg__": 14248,
"__bn__": 14249,
"__br__": 14250,
"__bs__": 14251,
"__ca__": 14252,
"__ceb__": 14253,
"__cs__": 14254,
"__cy__": 14255,
"__da__": 14256,
"__de__": 14257,
"__el__": 14258,
"__en__": 14259,
"__es__": 14260,
"__et__": 14261,
"__fa__": 14262,
"__ff__": 14263,
"__fi__": 14264,
"__fr__": 14265,
"__fy__": 14266,
"__ga__": 14267,
"__gd__": 14268,
"__gl__": 14269,
"__gu__": 14270,
"__ha__": 14271,
"__he__": 14272,
"__hi__": 14273,
"__hr__": 14274,
"__ht__": 14275,
"__hu__": 14276,
"__hy__": 14277,
"__id__": 14278,
"__ig__": 14279,
"__ilo__": 14280,
"__is__": 14281,
"__it__": 14282,
"__ja__": 14283,
"__jv__": 14284,
"__ka__": 14285,
"__kk__": 14286,
"__km__": 14287,
"__kn__": 14288,
"__ko__": 14289,
"__lb__": 14290,
"__lg__": 14291,
"__ln__": 14292,
"__lo__": 14293,
"__lt__": 14294,
"__lv__": 14295,
"__mg__": 14296,
"__mk__": 14297,
"__ml__": 14298,
"__mn__": 14299,
"__mr__": 14300,
"__ms__": 14301,
"__my__": 14302,
"__ne__": 14303,
"__nl__": 14304,
"__no__": 14305,
"__ns__": 14306,
"__oc__": 14307,
"__or__": 14308,
"__pa__": 14309,
"__pl__": 14310,
"__ps__": 14311,
"__pt__": 14312,
"__ro__": 14313,
"__ru__": 14314,
"__sd__": 14315,
"__si__": 14316,
"__sk__": 14317,
"__sl__": 14318,
"__so__": 14319,
"__sq__": 14320,
"__sr__": 14321,
"__ss__": 14322,
"__su__": 14323,
"__sv__": 14324,
"__sw__": 14325,
"__ta__": 14326,
"__th__": 14327,
"__tl__": 14328,
"__tn__": 14329,
"__tr__": 14330,
"__uk__": 14331,
"__ur__": 14332,
"__uz__": 14333,
"__vi__": 14334,
"__wo__": 14335,
"__xh__": 14336,
"__yi__": 14337,
"__yo__": 14338,
"__zh__": 14339,
"__zu__": 14340
}