prix-lm / added_tokens.json
wzhouad's picture
add tokenizer
e7b1ef3
{"[te]": 250077, "[ja]": 250040, "[sq]": 250071, "[et]": 250024, "[fa]": 250026, "[zh]": 250085, "[ru]": 250065, "[hi]": 250033, "[la]": 250047, "[is]": 250038, "[el]": 250020, "[he]": 250032, "[yi]": 250084, "[ca]": 250015, "[pt]": 250063, "[sw]": 250075, "[jv]": 250041, "[su]": 250073, "[ur]": 250081, "[fi]": 250027, "[fr]": 250028, "[sco]": 250067, "[th]": 250078, "[S]": 250002, "[sv]": 250074, "[my]": 250056, "[pl]": 250061, "[mg]": 250050, "[uk]": 250080, "[mr]": 250054, "[hy]": 250036, "[eo]": 250022, "[it]": 250039, "[cs]": 250016, "[es]": 250023, "[sa]": 250066, "[mk]": 250051, "[az]": 250009, "[lt]": 250048, "[am]": 250007, "[kk]": 250043, "[or]": 250060, "[zh_min_nan]": 250086, "[ta]": 250076, "[sr]": 250072, "[hu]": 250035, "[sh]": 250068, "[da]": 250018, "[bn]": 250012, "[vi]": 250083, "[sk]": 250069, "[kn]": 250044, "[ka]": 250042, "[hr]": 250034, "[ne]": 250057, "[ku]": 250046, "[ko]": 250045, "[gu]": 250031, "[nl]": 250058, "[ro]": 250064, "[ml]": 250052, "[ms]": 250055, "[pnb]": 250062, "[af]": 250006, "[uz]": 250082, "[gl]": 250030, "[en]": 250021, "[no]": 250059, "[eu]": 250025, "[br]": 250013, "[P]": 250003, "[id]": 250037, "[be]": 250010, "[cy]": 250017, "[sl]": 250070, "[EOS]": 250005, "[ar]": 250008, "[O]": 250004, "[bg]": 250011, "[bs]": 250014, "[lv]": 250049, "[mn]": 250053, "[de]": 250019, "[ga]": 250029, "[tr]": 250079}