|
{ |
|
"added_tokens_decoder": { |
|
"0": { |
|
"content": "<s>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"1": { |
|
"content": "<pad>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"2": { |
|
"content": "</s>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"3": { |
|
"content": "<unk>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40000": { |
|
"content": "<mask>", |
|
"lstrip": true, |
|
"normalized": true, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40001": { |
|
"content": "ace_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40002": { |
|
"content": "ace_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40003": { |
|
"content": "acm_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40004": { |
|
"content": "acq_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40005": { |
|
"content": "aeb_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40006": { |
|
"content": "afr_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40007": { |
|
"content": "ajp_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40008": { |
|
"content": "aka_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40009": { |
|
"content": "als_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40010": { |
|
"content": "amh_Ethi", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40011": { |
|
"content": "apc_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40012": { |
|
"content": "arb_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40013": { |
|
"content": "arg_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40014": { |
|
"content": "arn_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40015": { |
|
"content": "ars_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40016": { |
|
"content": "ary_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40017": { |
|
"content": "arz_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40018": { |
|
"content": "asm_Beng", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40019": { |
|
"content": "ast_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40020": { |
|
"content": "awa_Deva", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40021": { |
|
"content": "ayr_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40022": { |
|
"content": "azb_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40023": { |
|
"content": "azj_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40024": { |
|
"content": "bak_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40025": { |
|
"content": "bam_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40026": { |
|
"content": "ban_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40027": { |
|
"content": "bel_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40028": { |
|
"content": "bem_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40029": { |
|
"content": "ben_Beng", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40030": { |
|
"content": "bho_Deva", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40031": { |
|
"content": "bjn_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40032": { |
|
"content": "bjn_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40033": { |
|
"content": "bod_Tibt", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40034": { |
|
"content": "bos_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40035": { |
|
"content": "bug_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40036": { |
|
"content": "bul_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40037": { |
|
"content": "cat_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40038": { |
|
"content": "ceb_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40039": { |
|
"content": "ces_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40040": { |
|
"content": "cjk_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40041": { |
|
"content": "ckb_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40042": { |
|
"content": "crh_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40043": { |
|
"content": "cym_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40044": { |
|
"content": "dan_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40045": { |
|
"content": "deu_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40046": { |
|
"content": "dik_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40047": { |
|
"content": "dyu_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40048": { |
|
"content": "dzo_Tibt", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40049": { |
|
"content": "ell_Grek", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40050": { |
|
"content": "eng_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40051": { |
|
"content": "epo_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40052": { |
|
"content": "est_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40053": { |
|
"content": "eus_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40054": { |
|
"content": "ewe_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40055": { |
|
"content": "fao_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40056": { |
|
"content": "fij_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40057": { |
|
"content": "fin_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40058": { |
|
"content": "fon_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40059": { |
|
"content": "fra_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40060": { |
|
"content": "fur_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40061": { |
|
"content": "fuv_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40062": { |
|
"content": "gaz_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40063": { |
|
"content": "gla_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40064": { |
|
"content": "gle_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40065": { |
|
"content": "glg_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40066": { |
|
"content": "grn_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40067": { |
|
"content": "guj_Gujr", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40068": { |
|
"content": "hat_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40069": { |
|
"content": "hau_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40070": { |
|
"content": "heb_Hebr", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40071": { |
|
"content": "hin_Deva", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40072": { |
|
"content": "hne_Deva", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40073": { |
|
"content": "hrv_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40074": { |
|
"content": "hun_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40075": { |
|
"content": "hye_Armn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40076": { |
|
"content": "ibo_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40077": { |
|
"content": "ilo_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40078": { |
|
"content": "ind_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40079": { |
|
"content": "isl_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40080": { |
|
"content": "ita_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40081": { |
|
"content": "jav_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40082": { |
|
"content": "jpn_Jpan", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40083": { |
|
"content": "kab_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40084": { |
|
"content": "kac_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40085": { |
|
"content": "kam_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40086": { |
|
"content": "kan_Knda", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40087": { |
|
"content": "kas_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40088": { |
|
"content": "kas_Deva", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40089": { |
|
"content": "kat_Geor", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40090": { |
|
"content": "kaz_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40091": { |
|
"content": "kbp_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40092": { |
|
"content": "kea_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40093": { |
|
"content": "khk_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40094": { |
|
"content": "khm_Khmr", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40095": { |
|
"content": "kik_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40096": { |
|
"content": "kin_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40097": { |
|
"content": "kir_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40098": { |
|
"content": "kmb_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40099": { |
|
"content": "kmr_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40100": { |
|
"content": "knc_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40101": { |
|
"content": "knc_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40102": { |
|
"content": "kon_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40103": { |
|
"content": "kor_Hang", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40104": { |
|
"content": "lao_Laoo", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40105": { |
|
"content": "lij_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40106": { |
|
"content": "lim_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40107": { |
|
"content": "lin_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40108": { |
|
"content": "lit_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40109": { |
|
"content": "lmo_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40110": { |
|
"content": "ltg_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40111": { |
|
"content": "ltz_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40112": { |
|
"content": "lua_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40113": { |
|
"content": "lug_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40114": { |
|
"content": "luo_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40115": { |
|
"content": "lus_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40116": { |
|
"content": "lvs_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40117": { |
|
"content": "mag_Deva", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40118": { |
|
"content": "mai_Deva", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40119": { |
|
"content": "mal_Mlym", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40120": { |
|
"content": "mar_Deva", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40121": { |
|
"content": "min_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40122": { |
|
"content": "mkd_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40123": { |
|
"content": "mlt_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40124": { |
|
"content": "mni_Beng", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40125": { |
|
"content": "mos_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40126": { |
|
"content": "mri_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40127": { |
|
"content": "mya_Mymr", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40128": { |
|
"content": "nld_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40129": { |
|
"content": "nno_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40130": { |
|
"content": "nob_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40131": { |
|
"content": "npi_Deva", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40132": { |
|
"content": "nso_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40133": { |
|
"content": "nus_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40134": { |
|
"content": "nya_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40135": { |
|
"content": "oci_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40136": { |
|
"content": "ory_Orya", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40137": { |
|
"content": "pag_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40138": { |
|
"content": "pan_Guru", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40139": { |
|
"content": "pap_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40140": { |
|
"content": "pbt_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40141": { |
|
"content": "pes_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40142": { |
|
"content": "plt_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40143": { |
|
"content": "pol_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40144": { |
|
"content": "por_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40145": { |
|
"content": "prs_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40146": { |
|
"content": "quy_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40147": { |
|
"content": "ron_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40148": { |
|
"content": "run_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40149": { |
|
"content": "rus_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40150": { |
|
"content": "sag_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40151": { |
|
"content": "san_Deva", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40152": { |
|
"content": "sat_Beng", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40153": { |
|
"content": "scn_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40154": { |
|
"content": "shn_Mymr", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40155": { |
|
"content": "sin_Sinh", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40156": { |
|
"content": "slk_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40157": { |
|
"content": "slv_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40158": { |
|
"content": "smo_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40159": { |
|
"content": "sna_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40160": { |
|
"content": "snd_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40161": { |
|
"content": "som_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40162": { |
|
"content": "sot_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40163": { |
|
"content": "spa_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40164": { |
|
"content": "srd_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40165": { |
|
"content": "srp_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40166": { |
|
"content": "ssw_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40167": { |
|
"content": "sun_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40168": { |
|
"content": "swe_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40169": { |
|
"content": "swh_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40170": { |
|
"content": "szl_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40171": { |
|
"content": "tam_Taml", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40172": { |
|
"content": "taq_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40173": { |
|
"content": "taq_Tfng", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40174": { |
|
"content": "tat_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40175": { |
|
"content": "tel_Telu", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40176": { |
|
"content": "tgk_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40177": { |
|
"content": "tgl_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40178": { |
|
"content": "tha_Thai", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40179": { |
|
"content": "tir_Ethi", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40180": { |
|
"content": "tpi_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40181": { |
|
"content": "tsn_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40182": { |
|
"content": "tso_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40183": { |
|
"content": "tuk_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40184": { |
|
"content": "tum_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40185": { |
|
"content": "tur_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40186": { |
|
"content": "twi_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40187": { |
|
"content": "tzm_Tfng", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40188": { |
|
"content": "uig_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40189": { |
|
"content": "ukr_Cyrl", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40190": { |
|
"content": "umb_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40191": { |
|
"content": "urd_Arab", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40192": { |
|
"content": "uzn_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40193": { |
|
"content": "vec_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40194": { |
|
"content": "vie_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40195": { |
|
"content": "war_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40196": { |
|
"content": "wol_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40197": { |
|
"content": "xho_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40198": { |
|
"content": "ydd_Hebr", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40199": { |
|
"content": "yor_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40200": { |
|
"content": "yue_Hant", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40201": { |
|
"content": "zho_Hans", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40202": { |
|
"content": "zho_Hant", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40203": { |
|
"content": "zsm_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"40204": { |
|
"content": "zul_Latn", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
} |
|
}, |
|
"additional_special_tokens": [ |
|
"ace_Arab", |
|
"ace_Latn", |
|
"acm_Arab", |
|
"acq_Arab", |
|
"aeb_Arab", |
|
"afr_Latn", |
|
"ajp_Arab", |
|
"aka_Latn", |
|
"als_Latn", |
|
"amh_Ethi", |
|
"apc_Arab", |
|
"arb_Arab", |
|
"arg_Latn", |
|
"arn_Latn", |
|
"ars_Arab", |
|
"ary_Arab", |
|
"arz_Arab", |
|
"asm_Beng", |
|
"ast_Latn", |
|
"awa_Deva", |
|
"ayr_Latn", |
|
"azb_Arab", |
|
"azj_Latn", |
|
"bak_Cyrl", |
|
"bam_Latn", |
|
"ban_Latn", |
|
"bel_Cyrl", |
|
"bem_Latn", |
|
"ben_Beng", |
|
"bho_Deva", |
|
"bjn_Arab", |
|
"bjn_Latn", |
|
"bod_Tibt", |
|
"bos_Latn", |
|
"bug_Latn", |
|
"bul_Cyrl", |
|
"cat_Latn", |
|
"ceb_Latn", |
|
"ces_Latn", |
|
"cjk_Latn", |
|
"ckb_Arab", |
|
"crh_Latn", |
|
"cym_Latn", |
|
"dan_Latn", |
|
"deu_Latn", |
|
"dik_Latn", |
|
"dyu_Latn", |
|
"dzo_Tibt", |
|
"ell_Grek", |
|
"eng_Latn", |
|
"epo_Latn", |
|
"est_Latn", |
|
"eus_Latn", |
|
"ewe_Latn", |
|
"fao_Latn", |
|
"fij_Latn", |
|
"fin_Latn", |
|
"fon_Latn", |
|
"fra_Latn", |
|
"fur_Latn", |
|
"fuv_Latn", |
|
"gaz_Latn", |
|
"gla_Latn", |
|
"gle_Latn", |
|
"glg_Latn", |
|
"grn_Latn", |
|
"guj_Gujr", |
|
"hat_Latn", |
|
"hau_Latn", |
|
"heb_Hebr", |
|
"hin_Deva", |
|
"hne_Deva", |
|
"hrv_Latn", |
|
"hun_Latn", |
|
"hye_Armn", |
|
"ibo_Latn", |
|
"ilo_Latn", |
|
"ind_Latn", |
|
"isl_Latn", |
|
"ita_Latn", |
|
"jav_Latn", |
|
"jpn_Jpan", |
|
"kab_Latn", |
|
"kac_Latn", |
|
"kam_Latn", |
|
"kan_Knda", |
|
"kas_Arab", |
|
"kas_Deva", |
|
"kat_Geor", |
|
"kaz_Cyrl", |
|
"kbp_Latn", |
|
"kea_Latn", |
|
"khk_Cyrl", |
|
"khm_Khmr", |
|
"kik_Latn", |
|
"kin_Latn", |
|
"kir_Cyrl", |
|
"kmb_Latn", |
|
"kmr_Latn", |
|
"knc_Arab", |
|
"knc_Latn", |
|
"kon_Latn", |
|
"kor_Hang", |
|
"lao_Laoo", |
|
"lij_Latn", |
|
"lim_Latn", |
|
"lin_Latn", |
|
"lit_Latn", |
|
"lmo_Latn", |
|
"ltg_Latn", |
|
"ltz_Latn", |
|
"lua_Latn", |
|
"lug_Latn", |
|
"luo_Latn", |
|
"lus_Latn", |
|
"lvs_Latn", |
|
"mag_Deva", |
|
"mai_Deva", |
|
"mal_Mlym", |
|
"mar_Deva", |
|
"min_Latn", |
|
"mkd_Cyrl", |
|
"mlt_Latn", |
|
"mni_Beng", |
|
"mos_Latn", |
|
"mri_Latn", |
|
"mya_Mymr", |
|
"nld_Latn", |
|
"nno_Latn", |
|
"nob_Latn", |
|
"npi_Deva", |
|
"nso_Latn", |
|
"nus_Latn", |
|
"nya_Latn", |
|
"oci_Latn", |
|
"ory_Orya", |
|
"pag_Latn", |
|
"pan_Guru", |
|
"pap_Latn", |
|
"pbt_Arab", |
|
"pes_Arab", |
|
"plt_Latn", |
|
"pol_Latn", |
|
"por_Latn", |
|
"prs_Arab", |
|
"quy_Latn", |
|
"ron_Latn", |
|
"run_Latn", |
|
"rus_Cyrl", |
|
"sag_Latn", |
|
"san_Deva", |
|
"sat_Beng", |
|
"scn_Latn", |
|
"shn_Mymr", |
|
"sin_Sinh", |
|
"slk_Latn", |
|
"slv_Latn", |
|
"smo_Latn", |
|
"sna_Latn", |
|
"snd_Arab", |
|
"som_Latn", |
|
"sot_Latn", |
|
"spa_Latn", |
|
"srd_Latn", |
|
"srp_Cyrl", |
|
"ssw_Latn", |
|
"sun_Latn", |
|
"swe_Latn", |
|
"swh_Latn", |
|
"szl_Latn", |
|
"tam_Taml", |
|
"taq_Latn", |
|
"taq_Tfng", |
|
"tat_Cyrl", |
|
"tel_Telu", |
|
"tgk_Cyrl", |
|
"tgl_Latn", |
|
"tha_Thai", |
|
"tir_Ethi", |
|
"tpi_Latn", |
|
"tsn_Latn", |
|
"tso_Latn", |
|
"tuk_Latn", |
|
"tum_Latn", |
|
"tur_Latn", |
|
"twi_Latn", |
|
"tzm_Tfng", |
|
"uig_Arab", |
|
"ukr_Cyrl", |
|
"umb_Latn", |
|
"urd_Arab", |
|
"uzn_Latn", |
|
"vec_Latn", |
|
"vie_Latn", |
|
"war_Latn", |
|
"wol_Latn", |
|
"xho_Latn", |
|
"ydd_Hebr", |
|
"yor_Latn", |
|
"yue_Hant", |
|
"zho_Hans", |
|
"zho_Hant", |
|
"zsm_Latn", |
|
"zul_Latn" |
|
], |
|
"bos_token": "<s>", |
|
"clean_up_tokenization_spaces": true, |
|
"cls_token": "<s>", |
|
"eos_token": "</s>", |
|
"legacy_behaviour": false, |
|
"mask_token": "<mask>", |
|
"model_max_length": 1000000000000000019884624838656, |
|
"pad_token": "<pad>", |
|
"sep_token": "</s>", |
|
"sp_model_kwargs": {}, |
|
"src_lang": "eng_Latn", |
|
"tgt_lang": "eng_Latn", |
|
"tokenizer_class": "NllbTokenizer", |
|
"unk_token": "<unk>" |
|
} |
|
|