sedrickkeh commited on
Commit
5fc1298
1 Parent(s): 54095ec

End of training

Browse files
README.md CHANGED
@@ -4,6 +4,7 @@ license: apache-2.0
4
  base_model: mistralai/Mistral-7B-v0.1
5
  tags:
6
  - llama-factory
 
7
  - generated_from_trainer
8
  model-index:
9
  - name: hp_ablations_grid_mistral_bsz2048_lr2e-6_scheduler-cosine-warmup0.15-minlr5e-7
@@ -15,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
15
 
16
  # hp_ablations_grid_mistral_bsz2048_lr2e-6_scheduler-cosine-warmup0.15-minlr5e-7
17
 
18
- This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
19
  It achieves the following results on the evaluation set:
20
  - Loss: 0.0599
21
 
 
4
  base_model: mistralai/Mistral-7B-v0.1
5
  tags:
6
  - llama-factory
7
+ - full
8
  - generated_from_trainer
9
  model-index:
10
  - name: hp_ablations_grid_mistral_bsz2048_lr2e-6_scheduler-cosine-warmup0.15-minlr5e-7
 
16
 
17
  # hp_ablations_grid_mistral_bsz2048_lr2e-6_scheduler-cosine-warmup0.15-minlr5e-7
18
 
19
+ This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the mlfoundations-dev/oh-dcft-v3-llama3.1-nemotron-70b_shareGPT_format dataset.
20
  It achieves the following results on the evaluation set:
21
  - Loss: 0.0599
22
 
all_results.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 3.0,
3
+ "eval_loss": 0.05991922318935394,
4
+ "eval_runtime": 115.096,
5
+ "eval_samples_per_second": 157.286,
6
+ "eval_steps_per_second": 0.617,
7
+ "total_flos": 3376037568184320.0,
8
+ "train_loss": 0.5144212938963421,
9
+ "train_runtime": 17119.0234,
10
+ "train_samples_per_second": 60.276,
11
+ "train_steps_per_second": 0.029
12
+ }
eval_results.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 3.0,
3
+ "eval_loss": 0.05991922318935394,
4
+ "eval_runtime": 115.096,
5
+ "eval_samples_per_second": 157.286,
6
+ "eval_steps_per_second": 0.617
7
+ }
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 3.0,
3
+ "total_flos": 3376037568184320.0,
4
+ "train_loss": 0.5144212938963421,
5
+ "train_runtime": 17119.0234,
6
+ "train_samples_per_second": 60.276,
7
+ "train_steps_per_second": 0.029
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 3.0,
5
+ "eval_steps": 500,
6
+ "global_step": 504,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.05952380952380952,
13
+ "grad_norm": 11.642287552521111,
14
+ "learning_rate": 2.631578947368421e-07,
15
+ "loss": 0.8829,
16
+ "step": 10
17
+ },
18
+ {
19
+ "epoch": 0.11904761904761904,
20
+ "grad_norm": 7.106858407842631,
21
+ "learning_rate": 5.263157894736842e-07,
22
+ "loss": 0.8087,
23
+ "step": 20
24
+ },
25
+ {
26
+ "epoch": 0.17857142857142858,
27
+ "grad_norm": 2.093269967612724,
28
+ "learning_rate": 7.894736842105263e-07,
29
+ "loss": 0.7296,
30
+ "step": 30
31
+ },
32
+ {
33
+ "epoch": 0.23809523809523808,
34
+ "grad_norm": 1.2477023318233176,
35
+ "learning_rate": 1.0526315789473683e-06,
36
+ "loss": 0.6712,
37
+ "step": 40
38
+ },
39
+ {
40
+ "epoch": 0.2976190476190476,
41
+ "grad_norm": 1.0512343429003204,
42
+ "learning_rate": 1.3157894736842106e-06,
43
+ "loss": 0.6321,
44
+ "step": 50
45
+ },
46
+ {
47
+ "epoch": 0.35714285714285715,
48
+ "grad_norm": 4.474475922392763,
49
+ "learning_rate": 1.5789473684210526e-06,
50
+ "loss": 0.6026,
51
+ "step": 60
52
+ },
53
+ {
54
+ "epoch": 0.4166666666666667,
55
+ "grad_norm": 1.6529279657405915,
56
+ "learning_rate": 1.8421052631578946e-06,
57
+ "loss": 0.5884,
58
+ "step": 70
59
+ },
60
+ {
61
+ "epoch": 0.47619047619047616,
62
+ "grad_norm": 2.7949986781945366,
63
+ "learning_rate": 1.9996767546702485e-06,
64
+ "loss": 0.5783,
65
+ "step": 80
66
+ },
67
+ {
68
+ "epoch": 0.5357142857142857,
69
+ "grad_norm": 2.118745503355743,
70
+ "learning_rate": 1.996043443883064e-06,
71
+ "loss": 0.5706,
72
+ "step": 90
73
+ },
74
+ {
75
+ "epoch": 0.5952380952380952,
76
+ "grad_norm": 2.0357375426196147,
77
+ "learning_rate": 1.988392397752233e-06,
78
+ "loss": 0.5598,
79
+ "step": 100
80
+ },
81
+ {
82
+ "epoch": 0.6547619047619048,
83
+ "grad_norm": 2.2701188124758,
84
+ "learning_rate": 1.9767648201496052e-06,
85
+ "loss": 0.5569,
86
+ "step": 110
87
+ },
88
+ {
89
+ "epoch": 0.7142857142857143,
90
+ "grad_norm": 2.865482452040676,
91
+ "learning_rate": 1.961223330122206e-06,
92
+ "loss": 0.5477,
93
+ "step": 120
94
+ },
95
+ {
96
+ "epoch": 0.7738095238095238,
97
+ "grad_norm": 2.7451064629160493,
98
+ "learning_rate": 1.941851624664209e-06,
99
+ "loss": 0.5459,
100
+ "step": 130
101
+ },
102
+ {
103
+ "epoch": 0.8333333333333334,
104
+ "grad_norm": 2.0988347378452,
105
+ "learning_rate": 1.9187540279759314e-06,
106
+ "loss": 0.5383,
107
+ "step": 140
108
+ },
109
+ {
110
+ "epoch": 0.8928571428571429,
111
+ "grad_norm": 1.7836154166022118,
112
+ "learning_rate": 1.8920549296372686e-06,
113
+ "loss": 0.5337,
114
+ "step": 150
115
+ },
116
+ {
117
+ "epoch": 0.9523809523809523,
118
+ "grad_norm": 1.0480422517342471,
119
+ "learning_rate": 1.861898114721218e-06,
120
+ "loss": 0.528,
121
+ "step": 160
122
+ },
123
+ {
124
+ "epoch": 1.0,
125
+ "eval_loss": 0.0656253919005394,
126
+ "eval_runtime": 116.504,
127
+ "eval_samples_per_second": 155.385,
128
+ "eval_steps_per_second": 0.609,
129
+ "step": 168
130
+ },
131
+ {
132
+ "epoch": 1.0119047619047619,
133
+ "grad_norm": 1.8121694809796869,
134
+ "learning_rate": 1.8284459894551025e-06,
135
+ "loss": 0.5223,
136
+ "step": 170
137
+ },
138
+ {
139
+ "epoch": 1.0714285714285714,
140
+ "grad_norm": 1.420187622758248,
141
+ "learning_rate": 1.7918787065996015e-06,
142
+ "loss": 0.5032,
143
+ "step": 180
144
+ },
145
+ {
146
+ "epoch": 1.130952380952381,
147
+ "grad_norm": 1.5424317018915787,
148
+ "learning_rate": 1.7523931952557666e-06,
149
+ "loss": 0.5004,
150
+ "step": 190
151
+ },
152
+ {
153
+ "epoch": 1.1904761904761905,
154
+ "grad_norm": 1.850588186757041,
155
+ "learning_rate": 1.7102021003248955e-06,
156
+ "loss": 0.498,
157
+ "step": 200
158
+ },
159
+ {
160
+ "epoch": 1.25,
161
+ "grad_norm": 1.4232612836192142,
162
+ "learning_rate": 1.6655326373326793e-06,
163
+ "loss": 0.4987,
164
+ "step": 210
165
+ },
166
+ {
167
+ "epoch": 1.3095238095238095,
168
+ "grad_norm": 1.230835146194867,
169
+ "learning_rate": 1.6186253687848507e-06,
170
+ "loss": 0.493,
171
+ "step": 220
172
+ },
173
+ {
174
+ "epoch": 1.369047619047619,
175
+ "grad_norm": 1.2193965194507899,
176
+ "learning_rate": 1.569732908644127e-06,
177
+ "loss": 0.4923,
178
+ "step": 230
179
+ },
180
+ {
181
+ "epoch": 1.4285714285714286,
182
+ "grad_norm": 1.5201451501554284,
183
+ "learning_rate": 1.5191185619053519e-06,
184
+ "loss": 0.4902,
185
+ "step": 240
186
+ },
187
+ {
188
+ "epoch": 1.4880952380952381,
189
+ "grad_norm": 1.2106618553239001,
190
+ "learning_rate": 1.4670549065952552e-06,
191
+ "loss": 0.4863,
192
+ "step": 250
193
+ },
194
+ {
195
+ "epoch": 1.5476190476190477,
196
+ "grad_norm": 1.073369144919772,
197
+ "learning_rate": 1.4138223258333096e-06,
198
+ "loss": 0.4845,
199
+ "step": 260
200
+ },
201
+ {
202
+ "epoch": 1.6071428571428572,
203
+ "grad_norm": 1.3569659774066425,
204
+ "learning_rate": 1.3597074978591206e-06,
205
+ "loss": 0.4823,
206
+ "step": 270
207
+ },
208
+ {
209
+ "epoch": 1.6666666666666665,
210
+ "grad_norm": 1.1172719170259655,
211
+ "learning_rate": 1.3050018521581279e-06,
212
+ "loss": 0.4826,
213
+ "step": 280
214
+ },
215
+ {
216
+ "epoch": 1.7261904761904763,
217
+ "grad_norm": 1.057235946491605,
218
+ "learning_rate": 1.2499999999999999e-06,
219
+ "loss": 0.4817,
220
+ "step": 290
221
+ },
222
+ {
223
+ "epoch": 1.7857142857142856,
224
+ "grad_norm": 1.1395484188056737,
225
+ "learning_rate": 1.1949981478418721e-06,
226
+ "loss": 0.4763,
227
+ "step": 300
228
+ },
229
+ {
230
+ "epoch": 1.8452380952380953,
231
+ "grad_norm": 0.9673060331448388,
232
+ "learning_rate": 1.1402925021408796e-06,
233
+ "loss": 0.4751,
234
+ "step": 310
235
+ },
236
+ {
237
+ "epoch": 1.9047619047619047,
238
+ "grad_norm": 1.2716355726282513,
239
+ "learning_rate": 1.0861776741666901e-06,
240
+ "loss": 0.4743,
241
+ "step": 320
242
+ },
243
+ {
244
+ "epoch": 1.9642857142857144,
245
+ "grad_norm": 0.8670777947729602,
246
+ "learning_rate": 1.032945093404745e-06,
247
+ "loss": 0.4738,
248
+ "step": 330
249
+ },
250
+ {
251
+ "epoch": 2.0,
252
+ "eval_loss": 0.061109066009521484,
253
+ "eval_runtime": 116.1265,
254
+ "eval_samples_per_second": 155.89,
255
+ "eval_steps_per_second": 0.611,
256
+ "step": 336
257
+ },
258
+ {
259
+ "epoch": 2.0238095238095237,
260
+ "grad_norm": 1.1781799129689645,
261
+ "learning_rate": 9.80881438094648e-07,
262
+ "loss": 0.4626,
263
+ "step": 340
264
+ },
265
+ {
266
+ "epoch": 2.0833333333333335,
267
+ "grad_norm": 1.196256029896588,
268
+ "learning_rate": 9.302670913558731e-07,
269
+ "loss": 0.4514,
270
+ "step": 350
271
+ },
272
+ {
273
+ "epoch": 2.142857142857143,
274
+ "grad_norm": 1.240120670934915,
275
+ "learning_rate": 8.813746312151494e-07,
276
+ "loss": 0.4466,
277
+ "step": 360
278
+ },
279
+ {
280
+ "epoch": 2.2023809523809526,
281
+ "grad_norm": 1.4515250964073299,
282
+ "learning_rate": 8.344673626673205e-07,
283
+ "loss": 0.4461,
284
+ "step": 370
285
+ },
286
+ {
287
+ "epoch": 2.261904761904762,
288
+ "grad_norm": 0.8373786848430113,
289
+ "learning_rate": 7.897978996751046e-07,
290
+ "loss": 0.4491,
291
+ "step": 380
292
+ },
293
+ {
294
+ "epoch": 2.3214285714285716,
295
+ "grad_norm": 1.0346083625080364,
296
+ "learning_rate": 7.476068047442332e-07,
297
+ "loss": 0.4443,
298
+ "step": 390
299
+ },
300
+ {
301
+ "epoch": 2.380952380952381,
302
+ "grad_norm": 0.9459213157979981,
303
+ "learning_rate": 7.081212934003984e-07,
304
+ "loss": 0.4435,
305
+ "step": 400
306
+ },
307
+ {
308
+ "epoch": 2.4404761904761907,
309
+ "grad_norm": 0.9958744135548997,
310
+ "learning_rate": 6.715540105448972e-07,
311
+ "loss": 0.4428,
312
+ "step": 410
313
+ },
314
+ {
315
+ "epoch": 2.5,
316
+ "grad_norm": 0.9392635640773219,
317
+ "learning_rate": 6.381018852787821e-07,
318
+ "loss": 0.4427,
319
+ "step": 420
320
+ },
321
+ {
322
+ "epoch": 2.5595238095238093,
323
+ "grad_norm": 0.7294108780936411,
324
+ "learning_rate": 6.079450703627314e-07,
325
+ "loss": 0.4443,
326
+ "step": 430
327
+ },
328
+ {
329
+ "epoch": 2.619047619047619,
330
+ "grad_norm": 0.8142282440596496,
331
+ "learning_rate": 5.812459720240681e-07,
332
+ "loss": 0.4448,
333
+ "step": 440
334
+ },
335
+ {
336
+ "epoch": 2.678571428571429,
337
+ "grad_norm": 0.8207822906148434,
338
+ "learning_rate": 5.581483753357905e-07,
339
+ "loss": 0.4425,
340
+ "step": 450
341
+ },
342
+ {
343
+ "epoch": 2.738095238095238,
344
+ "grad_norm": 0.9245896216419172,
345
+ "learning_rate": 5.387766698777935e-07,
346
+ "loss": 0.4438,
347
+ "step": 460
348
+ },
349
+ {
350
+ "epoch": 2.7976190476190474,
351
+ "grad_norm": 0.7453831990866285,
352
+ "learning_rate": 5.232351798503945e-07,
353
+ "loss": 0.4423,
354
+ "step": 470
355
+ },
356
+ {
357
+ "epoch": 2.857142857142857,
358
+ "grad_norm": 0.8498775475082037,
359
+ "learning_rate": 5.116076022477671e-07,
360
+ "loss": 0.4416,
361
+ "step": 480
362
+ },
363
+ {
364
+ "epoch": 2.9166666666666665,
365
+ "grad_norm": 0.8873720999371016,
366
+ "learning_rate": 5.039565561169362e-07,
367
+ "loss": 0.4385,
368
+ "step": 490
369
+ },
370
+ {
371
+ "epoch": 2.9761904761904763,
372
+ "grad_norm": 0.8671697549026851,
373
+ "learning_rate": 5.003232453297512e-07,
374
+ "loss": 0.4405,
375
+ "step": 500
376
+ },
377
+ {
378
+ "epoch": 3.0,
379
+ "eval_loss": 0.05991922318935394,
380
+ "eval_runtime": 116.0997,
381
+ "eval_samples_per_second": 155.926,
382
+ "eval_steps_per_second": 0.612,
383
+ "step": 504
384
+ },
385
+ {
386
+ "epoch": 3.0,
387
+ "step": 504,
388
+ "total_flos": 3376037568184320.0,
389
+ "train_loss": 0.5144212938963421,
390
+ "train_runtime": 17119.0234,
391
+ "train_samples_per_second": 60.276,
392
+ "train_steps_per_second": 0.029
393
+ }
394
+ ],
395
+ "logging_steps": 10,
396
+ "max_steps": 504,
397
+ "num_input_tokens_seen": 0,
398
+ "num_train_epochs": 3,
399
+ "save_steps": 500,
400
+ "stateful_callbacks": {
401
+ "TrainerControl": {
402
+ "args": {
403
+ "should_epoch_stop": false,
404
+ "should_evaluate": false,
405
+ "should_log": false,
406
+ "should_save": true,
407
+ "should_training_stop": true
408
+ },
409
+ "attributes": {}
410
+ }
411
+ },
412
+ "total_flos": 3376037568184320.0,
413
+ "train_batch_size": 8,
414
+ "trial_name": null,
415
+ "trial_params": null
416
+ }
training_eval_loss.png ADDED
training_loss.png ADDED