Update tiny models for NllbMoeModel
#6
by
hf-transformers-bot
- opened
- .gitattributes +1 -0
- config.json +2 -2
- pytorch_model.bin +2 -2
- tokenizer.json +0 -0
- tokenizer_config.json +2 -204
.gitattributes
CHANGED
@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
32 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
33 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
34 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
32 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
33 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
34 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
35 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
config.json
CHANGED
@@ -25,7 +25,7 @@
|
|
25 |
"init_std": 0.02,
|
26 |
"is_encoder_decoder": true,
|
27 |
"max_position_embeddings": 20,
|
28 |
-
"model_type": "
|
29 |
"moe_eval_capacity_token_fraction": 1.0,
|
30 |
"moe_token_dropout": 0.2,
|
31 |
"normalize_router_prob_before_dropping": false,
|
@@ -44,5 +44,5 @@
|
|
44 |
"torch_dtype": "float32",
|
45 |
"transformers_version": "4.28.0.dev0",
|
46 |
"use_cache": true,
|
47 |
-
"vocab_size":
|
48 |
}
|
|
|
25 |
"init_std": 0.02,
|
26 |
"is_encoder_decoder": true,
|
27 |
"max_position_embeddings": 20,
|
28 |
+
"model_type": "nllb-moe",
|
29 |
"moe_eval_capacity_token_fraction": 1.0,
|
30 |
"moe_token_dropout": 0.2,
|
31 |
"normalize_router_prob_before_dropping": false,
|
|
|
44 |
"torch_dtype": "float32",
|
45 |
"transformers_version": "4.28.0.dev0",
|
46 |
"use_cache": true,
|
47 |
+
"vocab_size": 256204
|
48 |
}
|
pytorch_model.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:dd431a3eac75d2e57d01c4b5ddc52c2c88ff1ec5b1c6fe00f828a5d38b32d0e5
|
3 |
+
size 16552833
|
tokenizer.json
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
CHANGED
@@ -1,209 +1,7 @@
|
|
1 |
{
|
2 |
-
"additional_special_tokens":
|
3 |
-
"ace_Arab",
|
4 |
-
"ace_Latn",
|
5 |
-
"acm_Arab",
|
6 |
-
"acq_Arab",
|
7 |
-
"aeb_Arab",
|
8 |
-
"afr_Latn",
|
9 |
-
"ajp_Arab",
|
10 |
-
"aka_Latn",
|
11 |
-
"amh_Ethi",
|
12 |
-
"apc_Arab",
|
13 |
-
"arb_Arab",
|
14 |
-
"ars_Arab",
|
15 |
-
"ary_Arab",
|
16 |
-
"arz_Arab",
|
17 |
-
"asm_Beng",
|
18 |
-
"ast_Latn",
|
19 |
-
"awa_Deva",
|
20 |
-
"ayr_Latn",
|
21 |
-
"azb_Arab",
|
22 |
-
"azj_Latn",
|
23 |
-
"bak_Cyrl",
|
24 |
-
"bam_Latn",
|
25 |
-
"ban_Latn",
|
26 |
-
"bel_Cyrl",
|
27 |
-
"bem_Latn",
|
28 |
-
"ben_Beng",
|
29 |
-
"bho_Deva",
|
30 |
-
"bjn_Arab",
|
31 |
-
"bjn_Latn",
|
32 |
-
"bod_Tibt",
|
33 |
-
"bos_Latn",
|
34 |
-
"bug_Latn",
|
35 |
-
"bul_Cyrl",
|
36 |
-
"cat_Latn",
|
37 |
-
"ceb_Latn",
|
38 |
-
"ces_Latn",
|
39 |
-
"cjk_Latn",
|
40 |
-
"ckb_Arab",
|
41 |
-
"crh_Latn",
|
42 |
-
"cym_Latn",
|
43 |
-
"dan_Latn",
|
44 |
-
"deu_Latn",
|
45 |
-
"dik_Latn",
|
46 |
-
"dyu_Latn",
|
47 |
-
"dzo_Tibt",
|
48 |
-
"ell_Grek",
|
49 |
-
"eng_Latn",
|
50 |
-
"epo_Latn",
|
51 |
-
"est_Latn",
|
52 |
-
"eus_Latn",
|
53 |
-
"ewe_Latn",
|
54 |
-
"fao_Latn",
|
55 |
-
"pes_Arab",
|
56 |
-
"fij_Latn",
|
57 |
-
"fin_Latn",
|
58 |
-
"fon_Latn",
|
59 |
-
"fra_Latn",
|
60 |
-
"fur_Latn",
|
61 |
-
"fuv_Latn",
|
62 |
-
"gla_Latn",
|
63 |
-
"gle_Latn",
|
64 |
-
"glg_Latn",
|
65 |
-
"grn_Latn",
|
66 |
-
"guj_Gujr",
|
67 |
-
"hat_Latn",
|
68 |
-
"hau_Latn",
|
69 |
-
"heb_Hebr",
|
70 |
-
"hin_Deva",
|
71 |
-
"hne_Deva",
|
72 |
-
"hrv_Latn",
|
73 |
-
"hun_Latn",
|
74 |
-
"hye_Armn",
|
75 |
-
"ibo_Latn",
|
76 |
-
"ilo_Latn",
|
77 |
-
"ind_Latn",
|
78 |
-
"isl_Latn",
|
79 |
-
"ita_Latn",
|
80 |
-
"jav_Latn",
|
81 |
-
"jpn_Jpan",
|
82 |
-
"kab_Latn",
|
83 |
-
"kac_Latn",
|
84 |
-
"kam_Latn",
|
85 |
-
"kan_Knda",
|
86 |
-
"kas_Arab",
|
87 |
-
"kas_Deva",
|
88 |
-
"kat_Geor",
|
89 |
-
"knc_Arab",
|
90 |
-
"knc_Latn",
|
91 |
-
"kaz_Cyrl",
|
92 |
-
"kbp_Latn",
|
93 |
-
"kea_Latn",
|
94 |
-
"khm_Khmr",
|
95 |
-
"kik_Latn",
|
96 |
-
"kin_Latn",
|
97 |
-
"kir_Cyrl",
|
98 |
-
"kmb_Latn",
|
99 |
-
"kon_Latn",
|
100 |
-
"kor_Hang",
|
101 |
-
"kmr_Latn",
|
102 |
-
"lao_Laoo",
|
103 |
-
"lvs_Latn",
|
104 |
-
"lij_Latn",
|
105 |
-
"lim_Latn",
|
106 |
-
"lin_Latn",
|
107 |
-
"lit_Latn",
|
108 |
-
"lmo_Latn",
|
109 |
-
"ltg_Latn",
|
110 |
-
"ltz_Latn",
|
111 |
-
"lua_Latn",
|
112 |
-
"lug_Latn",
|
113 |
-
"luo_Latn",
|
114 |
-
"lus_Latn",
|
115 |
-
"mag_Deva",
|
116 |
-
"mai_Deva",
|
117 |
-
"mal_Mlym",
|
118 |
-
"mar_Deva",
|
119 |
-
"min_Latn",
|
120 |
-
"mkd_Cyrl",
|
121 |
-
"plt_Latn",
|
122 |
-
"mlt_Latn",
|
123 |
-
"mni_Beng",
|
124 |
-
"khk_Cyrl",
|
125 |
-
"mos_Latn",
|
126 |
-
"mri_Latn",
|
127 |
-
"zsm_Latn",
|
128 |
-
"mya_Mymr",
|
129 |
-
"nld_Latn",
|
130 |
-
"nno_Latn",
|
131 |
-
"nob_Latn",
|
132 |
-
"npi_Deva",
|
133 |
-
"nso_Latn",
|
134 |
-
"nus_Latn",
|
135 |
-
"nya_Latn",
|
136 |
-
"oci_Latn",
|
137 |
-
"gaz_Latn",
|
138 |
-
"ory_Orya",
|
139 |
-
"pag_Latn",
|
140 |
-
"pan_Guru",
|
141 |
-
"pap_Latn",
|
142 |
-
"pol_Latn",
|
143 |
-
"por_Latn",
|
144 |
-
"prs_Arab",
|
145 |
-
"pbt_Arab",
|
146 |
-
"quy_Latn",
|
147 |
-
"ron_Latn",
|
148 |
-
"run_Latn",
|
149 |
-
"rus_Cyrl",
|
150 |
-
"sag_Latn",
|
151 |
-
"san_Deva",
|
152 |
-
"sat_Beng",
|
153 |
-
"scn_Latn",
|
154 |
-
"shn_Mymr",
|
155 |
-
"sin_Sinh",
|
156 |
-
"slk_Latn",
|
157 |
-
"slv_Latn",
|
158 |
-
"smo_Latn",
|
159 |
-
"sna_Latn",
|
160 |
-
"snd_Arab",
|
161 |
-
"som_Latn",
|
162 |
-
"sot_Latn",
|
163 |
-
"spa_Latn",
|
164 |
-
"als_Latn",
|
165 |
-
"srd_Latn",
|
166 |
-
"srp_Cyrl",
|
167 |
-
"ssw_Latn",
|
168 |
-
"sun_Latn",
|
169 |
-
"swe_Latn",
|
170 |
-
"swh_Latn",
|
171 |
-
"szl_Latn",
|
172 |
-
"tam_Taml",
|
173 |
-
"tat_Cyrl",
|
174 |
-
"tel_Telu",
|
175 |
-
"tgk_Cyrl",
|
176 |
-
"tgl_Latn",
|
177 |
-
"tha_Thai",
|
178 |
-
"tir_Ethi",
|
179 |
-
"taq_Latn",
|
180 |
-
"taq_Tfng",
|
181 |
-
"tpi_Latn",
|
182 |
-
"tsn_Latn",
|
183 |
-
"tso_Latn",
|
184 |
-
"tuk_Latn",
|
185 |
-
"tum_Latn",
|
186 |
-
"tur_Latn",
|
187 |
-
"twi_Latn",
|
188 |
-
"tzm_Tfng",
|
189 |
-
"uig_Arab",
|
190 |
-
"ukr_Cyrl",
|
191 |
-
"umb_Latn",
|
192 |
-
"urd_Arab",
|
193 |
-
"uzn_Latn",
|
194 |
-
"vec_Latn",
|
195 |
-
"vie_Latn",
|
196 |
-
"war_Latn",
|
197 |
-
"wol_Latn",
|
198 |
-
"xho_Latn",
|
199 |
-
"ydd_Hebr",
|
200 |
-
"yor_Latn",
|
201 |
-
"yue_Hant",
|
202 |
-
"zho_Hans",
|
203 |
-
"zho_Hant",
|
204 |
-
"zul_Latn"
|
205 |
-
],
|
206 |
"bos_token": "<s>",
|
|
|
207 |
"cls_token": "<s>",
|
208 |
"eos_token": "</s>",
|
209 |
"mask_token": {
|
|
|
1 |
{
|
2 |
+
"additional_special_tokens": null,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3 |
"bos_token": "<s>",
|
4 |
+
"clean_up_tokenization_spaces": true,
|
5 |
"cls_token": "<s>",
|
6 |
"eos_token": "</s>",
|
7 |
"mask_token": {
|