emilykang commited on
Commit
6046657
1 Parent(s): 0655972

Training in progress, epoch 0

Browse files
adapter_config.json CHANGED
@@ -20,10 +20,13 @@
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
23
- "k_proj",
24
- "q_proj",
25
  "o_proj",
26
- "v_proj"
 
 
 
 
 
27
  ],
28
  "task_type": "CAUSAL_LM",
29
  "use_dora": false,
 
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
 
 
23
  "o_proj",
24
+ "gate_proj",
25
+ "v_proj",
26
+ "down_proj",
27
+ "up_proj",
28
+ "q_proj",
29
+ "k_proj"
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f6dd92530a50a4acbbab81b082d48890f2ae9670ea4aea5f79148cd87d6e6fde
3
- size 7391832
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f92c6d0d2b76669a54671f747540128274cdbff0d803fd3b107998209d4db29e
3
+ size 39256704
trainer_state.json ADDED
@@ -0,0 +1,506 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 9.963369963369964,
5
+ "eval_steps": 500,
6
+ "global_step": 680,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.14652014652014653,
13
+ "grad_norm": 0.301513671875,
14
+ "learning_rate": 0.00019989329748023725,
15
+ "loss": 0.8924,
16
+ "step": 10
17
+ },
18
+ {
19
+ "epoch": 0.29304029304029305,
20
+ "grad_norm": 0.32275390625,
21
+ "learning_rate": 0.00019957341762950344,
22
+ "loss": 0.8199,
23
+ "step": 20
24
+ },
25
+ {
26
+ "epoch": 0.43956043956043955,
27
+ "grad_norm": 0.353515625,
28
+ "learning_rate": 0.0001990410430875205,
29
+ "loss": 0.8061,
30
+ "step": 30
31
+ },
32
+ {
33
+ "epoch": 0.5860805860805861,
34
+ "grad_norm": 0.445068359375,
35
+ "learning_rate": 0.0001982973099683902,
36
+ "loss": 0.8483,
37
+ "step": 40
38
+ },
39
+ {
40
+ "epoch": 0.7326007326007326,
41
+ "grad_norm": 0.396728515625,
42
+ "learning_rate": 0.0001973438054360693,
43
+ "loss": 0.8471,
44
+ "step": 50
45
+ },
46
+ {
47
+ "epoch": 0.8791208791208791,
48
+ "grad_norm": 0.35693359375,
49
+ "learning_rate": 0.00019618256431728194,
50
+ "loss": 0.8433,
51
+ "step": 60
52
+ },
53
+ {
54
+ "epoch": 1.0256410256410255,
55
+ "grad_norm": 0.360595703125,
56
+ "learning_rate": 0.0001948160647590966,
57
+ "loss": 0.8381,
58
+ "step": 70
59
+ },
60
+ {
61
+ "epoch": 1.1721611721611722,
62
+ "grad_norm": 0.37109375,
63
+ "learning_rate": 0.00019324722294043558,
64
+ "loss": 0.7955,
65
+ "step": 80
66
+ },
67
+ {
68
+ "epoch": 1.3186813186813187,
69
+ "grad_norm": 0.388671875,
70
+ "learning_rate": 0.0001914793868488021,
71
+ "loss": 0.8182,
72
+ "step": 90
73
+ },
74
+ {
75
+ "epoch": 1.4652014652014653,
76
+ "grad_norm": 0.45458984375,
77
+ "learning_rate": 0.00018951632913550626,
78
+ "loss": 0.7914,
79
+ "step": 100
80
+ },
81
+ {
82
+ "epoch": 1.6117216117216118,
83
+ "grad_norm": 0.51025390625,
84
+ "learning_rate": 0.00018736223906463696,
85
+ "loss": 0.8114,
86
+ "step": 110
87
+ },
88
+ {
89
+ "epoch": 1.7582417582417582,
90
+ "grad_norm": 0.52001953125,
91
+ "learning_rate": 0.00018502171357296144,
92
+ "loss": 0.8434,
93
+ "step": 120
94
+ },
95
+ {
96
+ "epoch": 1.9047619047619047,
97
+ "grad_norm": 0.433349609375,
98
+ "learning_rate": 0.00018249974745983023,
99
+ "loss": 0.7869,
100
+ "step": 130
101
+ },
102
+ {
103
+ "epoch": 2.051282051282051,
104
+ "grad_norm": 0.62255859375,
105
+ "learning_rate": 0.000179801722728024,
106
+ "loss": 0.7767,
107
+ "step": 140
108
+ },
109
+ {
110
+ "epoch": 2.197802197802198,
111
+ "grad_norm": 0.469970703125,
112
+ "learning_rate": 0.00017693339709828792,
113
+ "loss": 0.7711,
114
+ "step": 150
115
+ },
116
+ {
117
+ "epoch": 2.3443223443223444,
118
+ "grad_norm": 0.447998046875,
119
+ "learning_rate": 0.00017390089172206592,
120
+ "loss": 0.7823,
121
+ "step": 160
122
+ },
123
+ {
124
+ "epoch": 2.490842490842491,
125
+ "grad_norm": 0.41015625,
126
+ "learning_rate": 0.00017071067811865476,
127
+ "loss": 0.7747,
128
+ "step": 170
129
+ },
130
+ {
131
+ "epoch": 2.6373626373626373,
132
+ "grad_norm": 0.466796875,
133
+ "learning_rate": 0.00016736956436465573,
134
+ "loss": 0.7877,
135
+ "step": 180
136
+ },
137
+ {
138
+ "epoch": 2.7838827838827838,
139
+ "grad_norm": 0.48046875,
140
+ "learning_rate": 0.00016388468056519612,
141
+ "loss": 0.7778,
142
+ "step": 190
143
+ },
144
+ {
145
+ "epoch": 2.9304029304029307,
146
+ "grad_norm": 0.47998046875,
147
+ "learning_rate": 0.00016026346363792567,
148
+ "loss": 0.754,
149
+ "step": 200
150
+ },
151
+ {
152
+ "epoch": 3.076923076923077,
153
+ "grad_norm": 0.50244140625,
154
+ "learning_rate": 0.0001565136414422592,
155
+ "loss": 0.8089,
156
+ "step": 210
157
+ },
158
+ {
159
+ "epoch": 3.2234432234432235,
160
+ "grad_norm": 0.475830078125,
161
+ "learning_rate": 0.0001526432162877356,
162
+ "loss": 0.75,
163
+ "step": 220
164
+ },
165
+ {
166
+ "epoch": 3.36996336996337,
167
+ "grad_norm": 0.464111328125,
168
+ "learning_rate": 0.00014866044785668563,
169
+ "loss": 0.762,
170
+ "step": 230
171
+ },
172
+ {
173
+ "epoch": 3.5164835164835164,
174
+ "grad_norm": 0.61474609375,
175
+ "learning_rate": 0.00014457383557765386,
176
+ "loss": 0.7423,
177
+ "step": 240
178
+ },
179
+ {
180
+ "epoch": 3.663003663003663,
181
+ "grad_norm": 0.4873046875,
182
+ "learning_rate": 0.00014039210048718949,
183
+ "loss": 0.764,
184
+ "step": 250
185
+ },
186
+ {
187
+ "epoch": 3.8095238095238093,
188
+ "grad_norm": 0.5546875,
189
+ "learning_rate": 0.00013612416661871533,
190
+ "loss": 0.7422,
191
+ "step": 260
192
+ },
193
+ {
194
+ "epoch": 3.956043956043956,
195
+ "grad_norm": 0.473388671875,
196
+ "learning_rate": 0.00013177914195819016,
197
+ "loss": 0.7702,
198
+ "step": 270
199
+ },
200
+ {
201
+ "epoch": 4.102564102564102,
202
+ "grad_norm": 0.476318359375,
203
+ "learning_rate": 0.0001273662990072083,
204
+ "loss": 0.7237,
205
+ "step": 280
206
+ },
207
+ {
208
+ "epoch": 4.249084249084249,
209
+ "grad_norm": 0.471923828125,
210
+ "learning_rate": 0.0001228950549950134,
211
+ "loss": 0.7334,
212
+ "step": 290
213
+ },
214
+ {
215
+ "epoch": 4.395604395604396,
216
+ "grad_norm": 0.50537109375,
217
+ "learning_rate": 0.00011837495178165706,
218
+ "loss": 0.7274,
219
+ "step": 300
220
+ },
221
+ {
222
+ "epoch": 4.542124542124542,
223
+ "grad_norm": 0.5,
224
+ "learning_rate": 0.00011381563549518823,
225
+ "loss": 0.7142,
226
+ "step": 310
227
+ },
228
+ {
229
+ "epoch": 4.688644688644689,
230
+ "grad_norm": 0.48779296875,
231
+ "learning_rate": 0.00010922683594633021,
232
+ "loss": 0.7166,
233
+ "step": 320
234
+ },
235
+ {
236
+ "epoch": 4.835164835164835,
237
+ "grad_norm": 0.513671875,
238
+ "learning_rate": 0.00010461834586457398,
239
+ "loss": 0.76,
240
+ "step": 330
241
+ },
242
+ {
243
+ "epoch": 4.981684981684982,
244
+ "grad_norm": 0.5048828125,
245
+ "learning_rate": 0.0001,
246
+ "loss": 0.7571,
247
+ "step": 340
248
+ },
249
+ {
250
+ "epoch": 5.128205128205128,
251
+ "grad_norm": 0.5166015625,
252
+ "learning_rate": 9.538165413542607e-05,
253
+ "loss": 0.7046,
254
+ "step": 350
255
+ },
256
+ {
257
+ "epoch": 5.274725274725275,
258
+ "grad_norm": 0.51416015625,
259
+ "learning_rate": 9.077316405366981e-05,
260
+ "loss": 0.6916,
261
+ "step": 360
262
+ },
263
+ {
264
+ "epoch": 5.4212454212454215,
265
+ "grad_norm": 0.61279296875,
266
+ "learning_rate": 8.61843645048118e-05,
267
+ "loss": 0.7281,
268
+ "step": 370
269
+ },
270
+ {
271
+ "epoch": 5.5677655677655675,
272
+ "grad_norm": 0.54736328125,
273
+ "learning_rate": 8.162504821834295e-05,
274
+ "loss": 0.6963,
275
+ "step": 380
276
+ },
277
+ {
278
+ "epoch": 5.714285714285714,
279
+ "grad_norm": 0.6103515625,
280
+ "learning_rate": 7.710494500498662e-05,
281
+ "loss": 0.7323,
282
+ "step": 390
283
+ },
284
+ {
285
+ "epoch": 5.860805860805861,
286
+ "grad_norm": 0.60595703125,
287
+ "learning_rate": 7.263370099279172e-05,
288
+ "loss": 0.7342,
289
+ "step": 400
290
+ },
291
+ {
292
+ "epoch": 6.007326007326007,
293
+ "grad_norm": 0.55078125,
294
+ "learning_rate": 6.822085804180984e-05,
295
+ "loss": 0.7018,
296
+ "step": 410
297
+ },
298
+ {
299
+ "epoch": 6.153846153846154,
300
+ "grad_norm": 0.55078125,
301
+ "learning_rate": 6.387583338128471e-05,
302
+ "loss": 0.6998,
303
+ "step": 420
304
+ },
305
+ {
306
+ "epoch": 6.3003663003663,
307
+ "grad_norm": 0.591796875,
308
+ "learning_rate": 5.960789951281052e-05,
309
+ "loss": 0.6639,
310
+ "step": 430
311
+ },
312
+ {
313
+ "epoch": 6.446886446886447,
314
+ "grad_norm": 0.60107421875,
315
+ "learning_rate": 5.542616442234618e-05,
316
+ "loss": 0.7004,
317
+ "step": 440
318
+ },
319
+ {
320
+ "epoch": 6.593406593406593,
321
+ "grad_norm": 0.55810546875,
322
+ "learning_rate": 5.1339552143314384e-05,
323
+ "loss": 0.7155,
324
+ "step": 450
325
+ },
326
+ {
327
+ "epoch": 6.73992673992674,
328
+ "grad_norm": 0.609375,
329
+ "learning_rate": 4.735678371226441e-05,
330
+ "loss": 0.7091,
331
+ "step": 460
332
+ },
333
+ {
334
+ "epoch": 6.886446886446887,
335
+ "grad_norm": 0.53662109375,
336
+ "learning_rate": 4.3486358557740814e-05,
337
+ "loss": 0.6888,
338
+ "step": 470
339
+ },
340
+ {
341
+ "epoch": 7.032967032967033,
342
+ "grad_norm": 0.5029296875,
343
+ "learning_rate": 3.973653636207437e-05,
344
+ "loss": 0.6888,
345
+ "step": 480
346
+ },
347
+ {
348
+ "epoch": 7.17948717948718,
349
+ "grad_norm": 0.53662109375,
350
+ "learning_rate": 3.6115319434803894e-05,
351
+ "loss": 0.6845,
352
+ "step": 490
353
+ },
354
+ {
355
+ "epoch": 7.326007326007326,
356
+ "grad_norm": 0.59130859375,
357
+ "learning_rate": 3.263043563534428e-05,
358
+ "loss": 0.6731,
359
+ "step": 500
360
+ },
361
+ {
362
+ "epoch": 7.472527472527473,
363
+ "grad_norm": 0.564453125,
364
+ "learning_rate": 2.9289321881345254e-05,
365
+ "loss": 0.6708,
366
+ "step": 510
367
+ },
368
+ {
369
+ "epoch": 7.619047619047619,
370
+ "grad_norm": 0.62158203125,
371
+ "learning_rate": 2.6099108277934103e-05,
372
+ "loss": 0.6906,
373
+ "step": 520
374
+ },
375
+ {
376
+ "epoch": 7.7655677655677655,
377
+ "grad_norm": 0.572265625,
378
+ "learning_rate": 2.3066602901712108e-05,
379
+ "loss": 0.6841,
380
+ "step": 530
381
+ },
382
+ {
383
+ "epoch": 7.912087912087912,
384
+ "grad_norm": 0.5771484375,
385
+ "learning_rate": 2.0198277271976052e-05,
386
+ "loss": 0.6664,
387
+ "step": 540
388
+ },
389
+ {
390
+ "epoch": 8.058608058608058,
391
+ "grad_norm": 0.587890625,
392
+ "learning_rate": 1.750025254016978e-05,
393
+ "loss": 0.6899,
394
+ "step": 550
395
+ },
396
+ {
397
+ "epoch": 8.205128205128204,
398
+ "grad_norm": 0.55810546875,
399
+ "learning_rate": 1.4978286427038601e-05,
400
+ "loss": 0.6723,
401
+ "step": 560
402
+ },
403
+ {
404
+ "epoch": 8.351648351648352,
405
+ "grad_norm": 0.560546875,
406
+ "learning_rate": 1.2637760935363053e-05,
407
+ "loss": 0.6494,
408
+ "step": 570
409
+ },
410
+ {
411
+ "epoch": 8.498168498168498,
412
+ "grad_norm": 0.568359375,
413
+ "learning_rate": 1.0483670864493778e-05,
414
+ "loss": 0.6958,
415
+ "step": 580
416
+ },
417
+ {
418
+ "epoch": 8.644688644688644,
419
+ "grad_norm": 0.5947265625,
420
+ "learning_rate": 8.520613151197898e-06,
421
+ "loss": 0.6637,
422
+ "step": 590
423
+ },
424
+ {
425
+ "epoch": 8.791208791208792,
426
+ "grad_norm": 0.5556640625,
427
+ "learning_rate": 6.75277705956443e-06,
428
+ "loss": 0.6876,
429
+ "step": 600
430
+ },
431
+ {
432
+ "epoch": 8.937728937728938,
433
+ "grad_norm": 0.60791015625,
434
+ "learning_rate": 5.183935240903414e-06,
435
+ "loss": 0.6656,
436
+ "step": 610
437
+ },
438
+ {
439
+ "epoch": 9.084249084249084,
440
+ "grad_norm": 0.51416015625,
441
+ "learning_rate": 3.817435682718096e-06,
442
+ "loss": 0.6786,
443
+ "step": 620
444
+ },
445
+ {
446
+ "epoch": 9.23076923076923,
447
+ "grad_norm": 0.56689453125,
448
+ "learning_rate": 2.656194563930714e-06,
449
+ "loss": 0.668,
450
+ "step": 630
451
+ },
452
+ {
453
+ "epoch": 9.377289377289378,
454
+ "grad_norm": 0.5615234375,
455
+ "learning_rate": 1.7026900316098215e-06,
456
+ "loss": 0.6681,
457
+ "step": 640
458
+ },
459
+ {
460
+ "epoch": 9.523809523809524,
461
+ "grad_norm": 0.587890625,
462
+ "learning_rate": 9.589569124794916e-07,
463
+ "loss": 0.6496,
464
+ "step": 650
465
+ },
466
+ {
467
+ "epoch": 9.67032967032967,
468
+ "grad_norm": 0.57373046875,
469
+ "learning_rate": 4.2658237049655323e-07,
470
+ "loss": 0.6891,
471
+ "step": 660
472
+ },
473
+ {
474
+ "epoch": 9.816849816849818,
475
+ "grad_norm": 0.572265625,
476
+ "learning_rate": 1.0670251976275803e-07,
477
+ "loss": 0.6689,
478
+ "step": 670
479
+ },
480
+ {
481
+ "epoch": 9.963369963369964,
482
+ "grad_norm": 0.57470703125,
483
+ "learning_rate": 0.0,
484
+ "loss": 0.6729,
485
+ "step": 680
486
+ },
487
+ {
488
+ "epoch": 9.963369963369964,
489
+ "step": 680,
490
+ "total_flos": 3.318222409433088e+16,
491
+ "train_loss": 0.735039118458243,
492
+ "train_runtime": 593.4308,
493
+ "train_samples_per_second": 4.6,
494
+ "train_steps_per_second": 1.146
495
+ }
496
+ ],
497
+ "logging_steps": 10,
498
+ "max_steps": 680,
499
+ "num_input_tokens_seen": 0,
500
+ "num_train_epochs": 10,
501
+ "save_steps": 500,
502
+ "total_flos": 3.318222409433088e+16,
503
+ "train_batch_size": 1,
504
+ "trial_name": null,
505
+ "trial_params": null
506
+ }
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:bb611cdd078dd227617ee5d507d6becf4765696694e646333ed941825fc14695
3
  size 4984
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:95d9b9804befc7d2e7cf1aac8f4923915fcb7840ff2f288c997a3fa8da17f484
3
  size 4984