RaphaelMourad commited on
Commit
cbb162a
1 Parent(s): 14a56fe

Upload 10 files

Browse files
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "data/models/Mixtral-8x7B-v0.1",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 256,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 256,
13
+ "max_position_embeddings": 512,
14
+ "model_type": "mixtral",
15
+ "num_attention_heads": 8,
16
+ "num_experts_per_tok": 1,
17
+ "num_hidden_layers": 8,
18
+ "num_key_value_heads": 8,
19
+ "num_local_experts": 64,
20
+ "output_router_logits": false,
21
+ "rms_norm_eps": 1e-05,
22
+ "rope_theta": 1000000.0,
23
+ "router_aux_loss_coef": 0.02,
24
+ "sliding_window": null,
25
+ "tie_word_embeddings": false,
26
+ "torch_dtype": "float32",
27
+ "transformers_version": "4.37.0.dev0",
28
+ "use_cache": true,
29
+ "vocab_size": 4096
30
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "transformers_version": "4.37.0.dev0"
6
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e90fd1b6cce1e2134e6f1ca98b8775490f0c9c4778f4476ced1263848545f53
3
+ size 420171272
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0cbca065eb6b80f5d4370cfb563129ef642e3db7ceb00682b342c5a8c1716486
3
+ size 841300302
rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9fe25ef400b7da44284490d7c6549ffafd7b8dedbd6684750f0602e9e7a8915b
3
+ size 14575
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b94ac265b91edf6758a6e0e6ffa8f19c5d987c73d2e5b89c88f79283586a211b
3
+ size 627
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "[UNK]", "cls_token": "[CLS]", "sep_token": "[SEP]", "pad_token": "[PAD]", "mask_token": "[MASK]"}
trainer_state.json ADDED
@@ -0,0 +1,457 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 5.340388298034668,
3
+ "best_model_checkpoint": "./results/models/checkpoint-9325",
4
+ "epoch": 40.99982412944073,
5
+ "eval_steps": 500,
6
+ "global_step": 9325,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 1.0,
13
+ "eval_loss": 6.240943908691406,
14
+ "eval_runtime": 1306.4602,
15
+ "eval_samples_per_second": 2228.206,
16
+ "eval_steps_per_second": 2.176,
17
+ "step": 227
18
+ },
19
+ {
20
+ "epoch": 2.0,
21
+ "eval_loss": 5.840852737426758,
22
+ "eval_runtime": 1315.4788,
23
+ "eval_samples_per_second": 2212.93,
24
+ "eval_steps_per_second": 2.161,
25
+ "step": 454
26
+ },
27
+ {
28
+ "epoch": 2.2,
29
+ "learning_rate": 0.00047797356828193835,
30
+ "loss": 6.2671,
31
+ "step": 500
32
+ },
33
+ {
34
+ "epoch": 3.0,
35
+ "eval_loss": 5.713865756988525,
36
+ "eval_runtime": 1312.4705,
37
+ "eval_samples_per_second": 2218.002,
38
+ "eval_steps_per_second": 2.166,
39
+ "step": 682
40
+ },
41
+ {
42
+ "epoch": 4.0,
43
+ "eval_loss": 5.635574817657471,
44
+ "eval_runtime": 1313.1826,
45
+ "eval_samples_per_second": 2216.799,
46
+ "eval_steps_per_second": 2.165,
47
+ "step": 909
48
+ },
49
+ {
50
+ "epoch": 4.4,
51
+ "learning_rate": 0.00045594713656387664,
52
+ "loss": 5.7007,
53
+ "step": 1000
54
+ },
55
+ {
56
+ "epoch": 5.0,
57
+ "eval_loss": 5.646896839141846,
58
+ "eval_runtime": 1315.5043,
59
+ "eval_samples_per_second": 2212.887,
60
+ "eval_steps_per_second": 2.161,
61
+ "step": 1137
62
+ },
63
+ {
64
+ "epoch": 6.0,
65
+ "eval_loss": 5.588204383850098,
66
+ "eval_runtime": 1308.1379,
67
+ "eval_samples_per_second": 2225.348,
68
+ "eval_steps_per_second": 2.173,
69
+ "step": 1364
70
+ },
71
+ {
72
+ "epoch": 6.6,
73
+ "learning_rate": 0.000433920704845815,
74
+ "loss": 5.577,
75
+ "step": 1500
76
+ },
77
+ {
78
+ "epoch": 7.0,
79
+ "eval_loss": 5.526153564453125,
80
+ "eval_runtime": 1313.5897,
81
+ "eval_samples_per_second": 2216.112,
82
+ "eval_steps_per_second": 2.164,
83
+ "step": 1592
84
+ },
85
+ {
86
+ "epoch": 8.0,
87
+ "eval_loss": 5.503279685974121,
88
+ "eval_runtime": 1319.9731,
89
+ "eval_samples_per_second": 2205.395,
90
+ "eval_steps_per_second": 2.154,
91
+ "step": 1819
92
+ },
93
+ {
94
+ "epoch": 8.79,
95
+ "learning_rate": 0.00041189427312775327,
96
+ "loss": 5.521,
97
+ "step": 2000
98
+ },
99
+ {
100
+ "epoch": 9.0,
101
+ "eval_loss": 5.491898536682129,
102
+ "eval_runtime": 1312.4959,
103
+ "eval_samples_per_second": 2217.959,
104
+ "eval_steps_per_second": 2.166,
105
+ "step": 2046
106
+ },
107
+ {
108
+ "epoch": 10.0,
109
+ "eval_loss": 5.471615314483643,
110
+ "eval_runtime": 1334.8552,
111
+ "eval_samples_per_second": 2180.807,
112
+ "eval_steps_per_second": 2.13,
113
+ "step": 2274
114
+ },
115
+ {
116
+ "epoch": 10.99,
117
+ "learning_rate": 0.0003898678414096916,
118
+ "loss": 5.4662,
119
+ "step": 2500
120
+ },
121
+ {
122
+ "epoch": 11.0,
123
+ "eval_loss": 5.465639114379883,
124
+ "eval_runtime": 1311.5138,
125
+ "eval_samples_per_second": 2219.62,
126
+ "eval_steps_per_second": 2.168,
127
+ "step": 2501
128
+ },
129
+ {
130
+ "epoch": 12.0,
131
+ "eval_loss": 5.454081058502197,
132
+ "eval_runtime": 1331.8262,
133
+ "eval_samples_per_second": 2185.767,
134
+ "eval_steps_per_second": 2.135,
135
+ "step": 2729
136
+ },
137
+ {
138
+ "epoch": 13.0,
139
+ "eval_loss": 5.4356560707092285,
140
+ "eval_runtime": 1314.9113,
141
+ "eval_samples_per_second": 2213.885,
142
+ "eval_steps_per_second": 2.162,
143
+ "step": 2956
144
+ },
145
+ {
146
+ "epoch": 13.19,
147
+ "learning_rate": 0.00036784140969163,
148
+ "loss": 5.4335,
149
+ "step": 3000
150
+ },
151
+ {
152
+ "epoch": 14.0,
153
+ "eval_loss": 5.438130855560303,
154
+ "eval_runtime": 1321.1785,
155
+ "eval_samples_per_second": 2203.383,
156
+ "eval_steps_per_second": 2.152,
157
+ "step": 3184
158
+ },
159
+ {
160
+ "epoch": 15.0,
161
+ "eval_loss": 5.445527076721191,
162
+ "eval_runtime": 1373.6525,
163
+ "eval_samples_per_second": 2119.213,
164
+ "eval_steps_per_second": 2.07,
165
+ "step": 3411
166
+ },
167
+ {
168
+ "epoch": 15.39,
169
+ "learning_rate": 0.0003458149779735683,
170
+ "loss": 5.4113,
171
+ "step": 3500
172
+ },
173
+ {
174
+ "epoch": 16.0,
175
+ "eval_loss": 5.419914722442627,
176
+ "eval_runtime": 1306.18,
177
+ "eval_samples_per_second": 2228.684,
178
+ "eval_steps_per_second": 2.177,
179
+ "step": 3639
180
+ },
181
+ {
182
+ "epoch": 17.0,
183
+ "eval_loss": 5.416874885559082,
184
+ "eval_runtime": 1324.9112,
185
+ "eval_samples_per_second": 2197.175,
186
+ "eval_steps_per_second": 2.146,
187
+ "step": 3866
188
+ },
189
+ {
190
+ "epoch": 17.59,
191
+ "learning_rate": 0.00032378854625550663,
192
+ "loss": 5.3891,
193
+ "step": 4000
194
+ },
195
+ {
196
+ "epoch": 18.0,
197
+ "eval_loss": 5.415682315826416,
198
+ "eval_runtime": 1305.4824,
199
+ "eval_samples_per_second": 2229.875,
200
+ "eval_steps_per_second": 2.178,
201
+ "step": 4093
202
+ },
203
+ {
204
+ "epoch": 19.0,
205
+ "eval_loss": 5.391342639923096,
206
+ "eval_runtime": 1304.433,
207
+ "eval_samples_per_second": 2231.669,
208
+ "eval_steps_per_second": 2.179,
209
+ "step": 4321
210
+ },
211
+ {
212
+ "epoch": 19.79,
213
+ "learning_rate": 0.000301762114537445,
214
+ "loss": 5.3728,
215
+ "step": 4500
216
+ },
217
+ {
218
+ "epoch": 20.0,
219
+ "eval_loss": 5.3856120109558105,
220
+ "eval_runtime": 1316.8368,
221
+ "eval_samples_per_second": 2210.648,
222
+ "eval_steps_per_second": 2.159,
223
+ "step": 4548
224
+ },
225
+ {
226
+ "epoch": 21.0,
227
+ "eval_loss": 5.4083051681518555,
228
+ "eval_runtime": 1315.5537,
229
+ "eval_samples_per_second": 2212.804,
230
+ "eval_steps_per_second": 2.161,
231
+ "step": 4776
232
+ },
233
+ {
234
+ "epoch": 21.98,
235
+ "learning_rate": 0.00027973568281938326,
236
+ "loss": 5.3561,
237
+ "step": 5000
238
+ },
239
+ {
240
+ "epoch": 22.0,
241
+ "eval_loss": 5.3793134689331055,
242
+ "eval_runtime": 1313.7832,
243
+ "eval_samples_per_second": 2215.786,
244
+ "eval_steps_per_second": 2.164,
245
+ "step": 5003
246
+ },
247
+ {
248
+ "epoch": 23.0,
249
+ "eval_loss": 5.421042442321777,
250
+ "eval_runtime": 1337.1348,
251
+ "eval_samples_per_second": 2177.089,
252
+ "eval_steps_per_second": 2.126,
253
+ "step": 5231
254
+ },
255
+ {
256
+ "epoch": 24.0,
257
+ "eval_loss": 5.371836185455322,
258
+ "eval_runtime": 1305.6324,
259
+ "eval_samples_per_second": 2229.618,
260
+ "eval_steps_per_second": 2.177,
261
+ "step": 5458
262
+ },
263
+ {
264
+ "epoch": 24.18,
265
+ "learning_rate": 0.0002577092511013216,
266
+ "loss": 5.3458,
267
+ "step": 5500
268
+ },
269
+ {
270
+ "epoch": 25.0,
271
+ "eval_loss": 5.3682403564453125,
272
+ "eval_runtime": 1309.0037,
273
+ "eval_samples_per_second": 2223.876,
274
+ "eval_steps_per_second": 2.172,
275
+ "step": 5686
276
+ },
277
+ {
278
+ "epoch": 26.0,
279
+ "eval_loss": 5.368312358856201,
280
+ "eval_runtime": 1303.9762,
281
+ "eval_samples_per_second": 2232.45,
282
+ "eval_steps_per_second": 2.18,
283
+ "step": 5913
284
+ },
285
+ {
286
+ "epoch": 26.38,
287
+ "learning_rate": 0.00023568281938325992,
288
+ "loss": 5.3355,
289
+ "step": 6000
290
+ },
291
+ {
292
+ "epoch": 27.0,
293
+ "eval_loss": 5.381237983703613,
294
+ "eval_runtime": 1308.1689,
295
+ "eval_samples_per_second": 2225.295,
296
+ "eval_steps_per_second": 2.173,
297
+ "step": 6140
298
+ },
299
+ {
300
+ "epoch": 28.0,
301
+ "eval_loss": 5.359594821929932,
302
+ "eval_runtime": 1327.0099,
303
+ "eval_samples_per_second": 2193.7,
304
+ "eval_steps_per_second": 2.142,
305
+ "step": 6368
306
+ },
307
+ {
308
+ "epoch": 28.58,
309
+ "learning_rate": 0.00021365638766519823,
310
+ "loss": 5.325,
311
+ "step": 6500
312
+ },
313
+ {
314
+ "epoch": 29.0,
315
+ "eval_loss": 5.356747627258301,
316
+ "eval_runtime": 1307.1198,
317
+ "eval_samples_per_second": 2227.081,
318
+ "eval_steps_per_second": 2.175,
319
+ "step": 6595
320
+ },
321
+ {
322
+ "epoch": 30.0,
323
+ "eval_loss": 5.362793922424316,
324
+ "eval_runtime": 1308.9269,
325
+ "eval_samples_per_second": 2224.007,
326
+ "eval_steps_per_second": 2.172,
327
+ "step": 6823
328
+ },
329
+ {
330
+ "epoch": 30.78,
331
+ "learning_rate": 0.00019162995594713657,
332
+ "loss": 5.3152,
333
+ "step": 7000
334
+ },
335
+ {
336
+ "epoch": 31.0,
337
+ "eval_loss": 5.352448463439941,
338
+ "eval_runtime": 1309.5201,
339
+ "eval_samples_per_second": 2222.999,
340
+ "eval_steps_per_second": 2.171,
341
+ "step": 7050
342
+ },
343
+ {
344
+ "epoch": 32.0,
345
+ "eval_loss": 5.350729942321777,
346
+ "eval_runtime": 1330.3256,
347
+ "eval_samples_per_second": 2188.233,
348
+ "eval_steps_per_second": 2.137,
349
+ "step": 7278
350
+ },
351
+ {
352
+ "epoch": 32.98,
353
+ "learning_rate": 0.0001696035242290749,
354
+ "loss": 5.308,
355
+ "step": 7500
356
+ },
357
+ {
358
+ "epoch": 33.0,
359
+ "eval_loss": 5.350089073181152,
360
+ "eval_runtime": 1308.4994,
361
+ "eval_samples_per_second": 2224.733,
362
+ "eval_steps_per_second": 2.173,
363
+ "step": 7505
364
+ },
365
+ {
366
+ "epoch": 34.0,
367
+ "eval_loss": 5.350754261016846,
368
+ "eval_runtime": 1309.6607,
369
+ "eval_samples_per_second": 2222.76,
370
+ "eval_steps_per_second": 2.171,
371
+ "step": 7732
372
+ },
373
+ {
374
+ "epoch": 35.0,
375
+ "eval_loss": 5.346126556396484,
376
+ "eval_runtime": 1313.4913,
377
+ "eval_samples_per_second": 2216.278,
378
+ "eval_steps_per_second": 2.164,
379
+ "step": 7960
380
+ },
381
+ {
382
+ "epoch": 35.17,
383
+ "learning_rate": 0.00014757709251101323,
384
+ "loss": 5.2995,
385
+ "step": 8000
386
+ },
387
+ {
388
+ "epoch": 36.0,
389
+ "eval_loss": 5.345256805419922,
390
+ "eval_runtime": 1308.5486,
391
+ "eval_samples_per_second": 2224.65,
392
+ "eval_steps_per_second": 2.173,
393
+ "step": 8187
394
+ },
395
+ {
396
+ "epoch": 37.0,
397
+ "eval_loss": 5.342759132385254,
398
+ "eval_runtime": 1310.0897,
399
+ "eval_samples_per_second": 2222.033,
400
+ "eval_steps_per_second": 2.17,
401
+ "step": 8415
402
+ },
403
+ {
404
+ "epoch": 37.37,
405
+ "learning_rate": 0.00012555066079295154,
406
+ "loss": 5.2925,
407
+ "step": 8500
408
+ },
409
+ {
410
+ "epoch": 38.0,
411
+ "eval_loss": 5.342674255371094,
412
+ "eval_runtime": 1309.6934,
413
+ "eval_samples_per_second": 2222.705,
414
+ "eval_steps_per_second": 2.171,
415
+ "step": 8642
416
+ },
417
+ {
418
+ "epoch": 39.0,
419
+ "eval_loss": 5.3418426513671875,
420
+ "eval_runtime": 1305.0501,
421
+ "eval_samples_per_second": 2230.613,
422
+ "eval_steps_per_second": 2.178,
423
+ "step": 8870
424
+ },
425
+ {
426
+ "epoch": 39.57,
427
+ "learning_rate": 0.00010352422907488987,
428
+ "loss": 5.2885,
429
+ "step": 9000
430
+ },
431
+ {
432
+ "epoch": 40.0,
433
+ "eval_loss": 5.341084957122803,
434
+ "eval_runtime": 1304.7409,
435
+ "eval_samples_per_second": 2231.142,
436
+ "eval_steps_per_second": 2.179,
437
+ "step": 9097
438
+ },
439
+ {
440
+ "epoch": 41.0,
441
+ "eval_loss": 5.340388298034668,
442
+ "eval_runtime": 1304.8663,
443
+ "eval_samples_per_second": 2230.927,
444
+ "eval_steps_per_second": 2.179,
445
+ "step": 9325
446
+ }
447
+ ],
448
+ "logging_steps": 500,
449
+ "max_steps": 11350,
450
+ "num_input_tokens_seen": 0,
451
+ "num_train_epochs": 50,
452
+ "save_steps": 500,
453
+ "total_flos": 1.6473372597391884e+19,
454
+ "train_batch_size": 1024,
455
+ "trial_name": null,
456
+ "trial_params": null
457
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3ee188095cc0ad6e726a205eb1235ccc135ed86505ff509f635489e62e4cf3f8
3
+ size 4219