mikhail-panzo commited on
Commit
d1f6fbb
1 Parent(s): e56ab03

Training in progress, step 2000, checkpoint

Browse files
checkpoint-2000/model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5167e45a53420757be2ae833de0691ad71020c500fd2b63d06fa8609a0489fd5
3
  size 577789320
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79b8c2ef6559fda7a0c631d81d0cf9e2e01ce736e9bbb012ea95ba139acf922a
3
  size 577789320
checkpoint-2000/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3682bc4ace9b79832b5d6559e7e145bfe1f89be764244b42fe4e99a83e226ee8
3
  size 1155772233
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b1ffa02a9c80aeb83759259e8d797eb100be2f1b6cec13fd289724f0b86ff0b
3
  size 1155772233
checkpoint-2000/rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f8fee66ccce35c05ce0e3c71525b4f8220f17962160d4a0c25bd1576311110b0
3
  size 14244
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d397cd4ca016fe3edacb623ad658b8a4b20cdf7ab9cfb204322ddb40417ca91f
3
  size 14244
checkpoint-2000/trainer_state.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
- "best_metric": 0.43240800499916077,
3
- "best_model_checkpoint": "mikhail_panzo/zlm_b64_le5_s12000/checkpoint-2000",
4
- "epoch": 1.675392670157068,
5
  "eval_steps": 500,
6
  "global_step": 2000,
7
  "is_hyper_param_search": false,
@@ -9,315 +9,315 @@
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
- "epoch": 0.041884816753926704,
13
- "grad_norm": 12.821951866149902,
14
- "learning_rate": 2.4000000000000003e-07,
15
- "loss": 1.1152,
16
  "step": 50
17
  },
18
  {
19
- "epoch": 0.08376963350785341,
20
- "grad_norm": 6.186221122741699,
21
- "learning_rate": 4.900000000000001e-07,
22
- "loss": 1.0584,
23
  "step": 100
24
  },
25
  {
26
- "epoch": 0.1256544502617801,
27
- "grad_norm": 13.08674430847168,
28
  "learning_rate": 7.4e-07,
29
- "loss": 0.9996,
30
  "step": 150
31
  },
32
  {
33
- "epoch": 0.16753926701570682,
34
- "grad_norm": 4.10632848739624,
35
- "learning_rate": 9.9e-07,
36
- "loss": 0.9355,
37
  "step": 200
38
  },
39
  {
40
- "epoch": 0.2094240837696335,
41
- "grad_norm": 3.2428064346313477,
42
- "learning_rate": 1.2400000000000002e-06,
43
- "loss": 0.852,
44
  "step": 250
45
  },
46
  {
47
- "epoch": 0.2513089005235602,
48
- "grad_norm": 2.0861401557922363,
49
- "learning_rate": 1.4900000000000001e-06,
50
- "loss": 0.8241,
51
  "step": 300
52
  },
53
  {
54
- "epoch": 0.2931937172774869,
55
- "grad_norm": 8.602317810058594,
56
- "learning_rate": 1.74e-06,
57
- "loss": 0.7629,
58
  "step": 350
59
  },
60
  {
61
- "epoch": 0.33507853403141363,
62
- "grad_norm": 2.7594990730285645,
63
- "learning_rate": 1.9900000000000004e-06,
64
- "loss": 0.7418,
65
  "step": 400
66
  },
67
  {
68
- "epoch": 0.3769633507853403,
69
- "grad_norm": 2.7047348022460938,
70
- "learning_rate": 2.24e-06,
71
- "loss": 0.7166,
72
  "step": 450
73
  },
74
  {
75
- "epoch": 0.418848167539267,
76
- "grad_norm": 2.0666158199310303,
77
- "learning_rate": 2.4900000000000003e-06,
78
- "loss": 0.6876,
79
  "step": 500
80
  },
81
  {
82
- "epoch": 0.418848167539267,
83
- "eval_loss": 0.5937426090240479,
84
- "eval_runtime": 263.0955,
85
- "eval_samples_per_second": 32.266,
86
- "eval_steps_per_second": 4.037,
87
  "step": 500
88
  },
89
  {
90
- "epoch": 0.4607329842931937,
91
- "grad_norm": 2.955193519592285,
92
- "learning_rate": 2.7400000000000004e-06,
93
- "loss": 0.673,
94
  "step": 550
95
  },
96
  {
97
- "epoch": 0.5026178010471204,
98
- "grad_norm": 4.856322765350342,
99
- "learning_rate": 2.99e-06,
100
- "loss": 0.6539,
101
  "step": 600
102
  },
103
  {
104
- "epoch": 0.5445026178010471,
105
- "grad_norm": 7.631103992462158,
106
- "learning_rate": 3.2400000000000003e-06,
107
- "loss": 0.6588,
108
  "step": 650
109
  },
110
  {
111
- "epoch": 0.5863874345549738,
112
- "grad_norm": 2.715771436691284,
113
- "learning_rate": 3.49e-06,
114
- "loss": 0.624,
115
  "step": 700
116
  },
117
  {
118
- "epoch": 0.6282722513089005,
119
- "grad_norm": 2.7684273719787598,
120
- "learning_rate": 3.74e-06,
121
- "loss": 0.6116,
122
  "step": 750
123
  },
124
  {
125
- "epoch": 0.6701570680628273,
126
- "grad_norm": 2.219599723815918,
127
- "learning_rate": 3.990000000000001e-06,
128
- "loss": 0.6097,
129
  "step": 800
130
  },
131
  {
132
- "epoch": 0.7120418848167539,
133
- "grad_norm": 3.749188184738159,
134
- "learning_rate": 4.24e-06,
135
- "loss": 0.614,
136
  "step": 850
137
  },
138
  {
139
- "epoch": 0.7539267015706806,
140
- "grad_norm": 2.120619773864746,
141
- "learning_rate": 4.49e-06,
142
- "loss": 0.5814,
143
  "step": 900
144
  },
145
  {
146
- "epoch": 0.7958115183246073,
147
- "grad_norm": 2.7725327014923096,
148
- "learning_rate": 4.74e-06,
149
- "loss": 0.5895,
150
  "step": 950
151
  },
152
  {
153
- "epoch": 0.837696335078534,
154
- "grad_norm": 9.851961135864258,
155
- "learning_rate": 4.9900000000000005e-06,
156
- "loss": 0.5623,
157
  "step": 1000
158
  },
159
  {
160
- "epoch": 0.837696335078534,
161
- "eval_loss": 0.4933945834636688,
162
- "eval_runtime": 258.739,
163
- "eval_samples_per_second": 32.809,
164
- "eval_steps_per_second": 4.105,
165
  "step": 1000
166
  },
167
  {
168
- "epoch": 0.8795811518324608,
169
- "grad_norm": 5.074089527130127,
170
- "learning_rate": 5.240000000000001e-06,
171
- "loss": 0.5748,
172
  "step": 1050
173
  },
174
  {
175
- "epoch": 0.9214659685863874,
176
- "grad_norm": 5.884639739990234,
177
  "learning_rate": 5.485e-06,
178
- "loss": 0.5395,
179
  "step": 1100
180
  },
181
  {
182
- "epoch": 0.9633507853403142,
183
- "grad_norm": 3.1588447093963623,
184
  "learning_rate": 5.735e-06,
185
- "loss": 0.5478,
186
  "step": 1150
187
  },
188
  {
189
- "epoch": 1.0052356020942408,
190
- "grad_norm": 2.4452970027923584,
191
  "learning_rate": 5.985000000000001e-06,
192
- "loss": 0.538,
193
  "step": 1200
194
  },
195
  {
196
- "epoch": 1.0471204188481675,
197
- "grad_norm": 8.290769577026367,
198
  "learning_rate": 6.235000000000001e-06,
199
- "loss": 0.5411,
200
  "step": 1250
201
  },
202
  {
203
- "epoch": 1.0890052356020943,
204
- "grad_norm": 4.080046653747559,
205
  "learning_rate": 6.485000000000001e-06,
206
- "loss": 0.5387,
207
  "step": 1300
208
  },
209
  {
210
- "epoch": 1.130890052356021,
211
- "grad_norm": 2.7520201206207275,
212
  "learning_rate": 6.735000000000001e-06,
213
- "loss": 0.5426,
214
  "step": 1350
215
  },
216
  {
217
- "epoch": 1.1727748691099475,
218
- "grad_norm": 3.8192107677459717,
219
  "learning_rate": 6.985000000000001e-06,
220
- "loss": 0.5315,
221
  "step": 1400
222
  },
223
  {
224
- "epoch": 1.2146596858638743,
225
- "grad_norm": 9.535676956176758,
226
  "learning_rate": 7.235000000000001e-06,
227
- "loss": 0.5159,
228
  "step": 1450
229
  },
230
  {
231
- "epoch": 1.256544502617801,
232
- "grad_norm": 2.5731022357940674,
233
  "learning_rate": 7.485000000000001e-06,
234
- "loss": 0.5165,
235
  "step": 1500
236
  },
237
  {
238
- "epoch": 1.256544502617801,
239
- "eval_loss": 0.4628298878669739,
240
- "eval_runtime": 257.0265,
241
- "eval_samples_per_second": 33.028,
242
- "eval_steps_per_second": 4.132,
243
  "step": 1500
244
  },
245
  {
246
- "epoch": 1.2984293193717278,
247
- "grad_norm": 3.662090539932251,
248
  "learning_rate": 7.735e-06,
249
- "loss": 0.5188,
250
  "step": 1550
251
  },
252
  {
253
- "epoch": 1.3403141361256545,
254
- "grad_norm": 3.1680572032928467,
255
  "learning_rate": 7.985e-06,
256
- "loss": 0.5148,
257
  "step": 1600
258
  },
259
  {
260
- "epoch": 1.3821989528795813,
261
- "grad_norm": 3.601471185684204,
262
  "learning_rate": 8.235e-06,
263
- "loss": 0.5127,
264
  "step": 1650
265
  },
266
  {
267
- "epoch": 1.4240837696335078,
268
- "grad_norm": 4.497046947479248,
269
  "learning_rate": 8.485000000000001e-06,
270
- "loss": 0.5039,
271
  "step": 1700
272
  },
273
  {
274
- "epoch": 1.4659685863874345,
275
- "grad_norm": 3.6496424674987793,
276
  "learning_rate": 8.735000000000002e-06,
277
- "loss": 0.5114,
278
  "step": 1750
279
  },
280
  {
281
- "epoch": 1.5078534031413613,
282
- "grad_norm": 2.7080061435699463,
283
  "learning_rate": 8.985000000000001e-06,
284
- "loss": 0.4953,
285
  "step": 1800
286
  },
287
  {
288
- "epoch": 1.5497382198952878,
289
- "grad_norm": 2.5439274311065674,
290
  "learning_rate": 9.235e-06,
291
- "loss": 0.497,
292
  "step": 1850
293
  },
294
  {
295
- "epoch": 1.5916230366492146,
296
- "grad_norm": 2.004573106765747,
297
  "learning_rate": 9.485000000000002e-06,
298
- "loss": 0.4889,
299
  "step": 1900
300
  },
301
  {
302
- "epoch": 1.6335078534031413,
303
- "grad_norm": 4.699184894561768,
304
  "learning_rate": 9.735e-06,
305
- "loss": 0.4925,
306
  "step": 1950
307
  },
308
  {
309
- "epoch": 1.675392670157068,
310
- "grad_norm": 3.2080323696136475,
311
  "learning_rate": 9.985000000000002e-06,
312
- "loss": 0.4851,
313
  "step": 2000
314
  },
315
  {
316
- "epoch": 1.675392670157068,
317
- "eval_loss": 0.43240800499916077,
318
- "eval_runtime": 256.4745,
319
- "eval_samples_per_second": 33.099,
320
- "eval_steps_per_second": 4.141,
321
  "step": 2000
322
  }
323
  ],
@@ -338,8 +338,8 @@
338
  "attributes": {}
339
  }
340
  },
341
- "total_flos": 1.7986388390898048e+16,
342
- "train_batch_size": 16,
343
  "trial_name": null,
344
  "trial_params": null
345
  }
 
1
  {
2
+ "best_metric": 0.43827974796295166,
3
+ "best_model_checkpoint": "mikhail-panzo/zlm_b64_le5_s12000/checkpoint-2000",
4
+ "epoch": 1.675041876046901,
5
  "eval_steps": 500,
6
  "global_step": 2000,
7
  "is_hyper_param_search": false,
 
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
+ "epoch": 0.04187604690117253,
13
+ "grad_norm": 4.788218975067139,
14
+ "learning_rate": 2.4500000000000004e-07,
15
+ "loss": 1.1819,
16
  "step": 50
17
  },
18
  {
19
+ "epoch": 0.08375209380234507,
20
+ "grad_norm": 15.618935585021973,
21
+ "learning_rate": 4.95e-07,
22
+ "loss": 1.0042,
23
  "step": 100
24
  },
25
  {
26
+ "epoch": 0.12562814070351758,
27
+ "grad_norm": 2.7821555137634277,
28
  "learning_rate": 7.4e-07,
29
+ "loss": 0.9676,
30
  "step": 150
31
  },
32
  {
33
+ "epoch": 0.16750418760469013,
34
+ "grad_norm": 3.759263038635254,
35
+ "learning_rate": 9.85e-07,
36
+ "loss": 0.9362,
37
  "step": 200
38
  },
39
  {
40
+ "epoch": 0.20938023450586266,
41
+ "grad_norm": 3.906439781188965,
42
+ "learning_rate": 1.235e-06,
43
+ "loss": 0.8667,
44
  "step": 250
45
  },
46
  {
47
+ "epoch": 0.25125628140703515,
48
+ "grad_norm": 4.397643566131592,
49
+ "learning_rate": 1.485e-06,
50
+ "loss": 0.8455,
51
  "step": 300
52
  },
53
  {
54
+ "epoch": 0.2931323283082077,
55
+ "grad_norm": 2.94077467918396,
56
+ "learning_rate": 1.7350000000000001e-06,
57
+ "loss": 0.8376,
58
  "step": 350
59
  },
60
  {
61
+ "epoch": 0.33500837520938026,
62
+ "grad_norm": 4.161074161529541,
63
+ "learning_rate": 1.985e-06,
64
+ "loss": 0.7521,
65
  "step": 400
66
  },
67
  {
68
+ "epoch": 0.3768844221105528,
69
+ "grad_norm": 2.8088552951812744,
70
+ "learning_rate": 2.235e-06,
71
+ "loss": 0.7444,
72
  "step": 450
73
  },
74
  {
75
+ "epoch": 0.4187604690117253,
76
+ "grad_norm": 2.62283992767334,
77
+ "learning_rate": 2.4850000000000003e-06,
78
+ "loss": 0.7129,
79
  "step": 500
80
  },
81
  {
82
+ "epoch": 0.4187604690117253,
83
+ "eval_loss": 0.6066474914550781,
84
+ "eval_runtime": 213.4911,
85
+ "eval_samples_per_second": 39.763,
86
+ "eval_steps_per_second": 4.974,
87
  "step": 500
88
  },
89
  {
90
+ "epoch": 0.46063651591289784,
91
+ "grad_norm": 3.654665946960449,
92
+ "learning_rate": 2.7350000000000005e-06,
93
+ "loss": 0.6772,
94
  "step": 550
95
  },
96
  {
97
+ "epoch": 0.5025125628140703,
98
+ "grad_norm": 5.1851887702941895,
99
+ "learning_rate": 2.9850000000000002e-06,
100
+ "loss": 0.6552,
101
  "step": 600
102
  },
103
  {
104
+ "epoch": 0.5443886097152428,
105
+ "grad_norm": 4.017187595367432,
106
+ "learning_rate": 3.2350000000000004e-06,
107
+ "loss": 0.6506,
108
  "step": 650
109
  },
110
  {
111
+ "epoch": 0.5862646566164154,
112
+ "grad_norm": 2.1161563396453857,
113
+ "learning_rate": 3.485e-06,
114
+ "loss": 0.6301,
115
  "step": 700
116
  },
117
  {
118
+ "epoch": 0.628140703517588,
119
+ "grad_norm": 1.857313871383667,
120
+ "learning_rate": 3.7350000000000002e-06,
121
+ "loss": 0.6373,
122
  "step": 750
123
  },
124
  {
125
+ "epoch": 0.6700167504187605,
126
+ "grad_norm": 3.6841330528259277,
127
+ "learning_rate": 3.985000000000001e-06,
128
+ "loss": 0.6305,
129
  "step": 800
130
  },
131
  {
132
+ "epoch": 0.711892797319933,
133
+ "grad_norm": 2.6450912952423096,
134
+ "learning_rate": 4.235e-06,
135
+ "loss": 0.6169,
136
  "step": 850
137
  },
138
  {
139
+ "epoch": 0.7537688442211056,
140
+ "grad_norm": 1.7241184711456299,
141
+ "learning_rate": 4.485e-06,
142
+ "loss": 0.5933,
143
  "step": 900
144
  },
145
  {
146
+ "epoch": 0.7956448911222781,
147
+ "grad_norm": 2.602735996246338,
148
+ "learning_rate": 4.735e-06,
149
+ "loss": 0.583,
150
  "step": 950
151
  },
152
  {
153
+ "epoch": 0.8375209380234506,
154
+ "grad_norm": 12.419028282165527,
155
+ "learning_rate": 4.9850000000000006e-06,
156
+ "loss": 0.5922,
157
  "step": 1000
158
  },
159
  {
160
+ "epoch": 0.8375209380234506,
161
+ "eval_loss": 0.4979843497276306,
162
+ "eval_runtime": 214.0502,
163
+ "eval_samples_per_second": 39.659,
164
+ "eval_steps_per_second": 4.961,
165
  "step": 1000
166
  },
167
  {
168
+ "epoch": 0.8793969849246231,
169
+ "grad_norm": 3.200178861618042,
170
+ "learning_rate": 5.235e-06,
171
+ "loss": 0.5678,
172
  "step": 1050
173
  },
174
  {
175
+ "epoch": 0.9212730318257957,
176
+ "grad_norm": 2.458869218826294,
177
  "learning_rate": 5.485e-06,
178
+ "loss": 0.5558,
179
  "step": 1100
180
  },
181
  {
182
+ "epoch": 0.9631490787269682,
183
+ "grad_norm": 2.341752290725708,
184
  "learning_rate": 5.735e-06,
185
+ "loss": 0.5666,
186
  "step": 1150
187
  },
188
  {
189
+ "epoch": 1.0050251256281406,
190
+ "grad_norm": 2.9191603660583496,
191
  "learning_rate": 5.985000000000001e-06,
192
+ "loss": 0.5345,
193
  "step": 1200
194
  },
195
  {
196
+ "epoch": 1.0469011725293131,
197
+ "grad_norm": 2.68806529045105,
198
  "learning_rate": 6.235000000000001e-06,
199
+ "loss": 0.5653,
200
  "step": 1250
201
  },
202
  {
203
+ "epoch": 1.0887772194304857,
204
+ "grad_norm": 3.278820276260376,
205
  "learning_rate": 6.485000000000001e-06,
206
+ "loss": 0.5565,
207
  "step": 1300
208
  },
209
  {
210
+ "epoch": 1.1306532663316582,
211
+ "grad_norm": 3.066511869430542,
212
  "learning_rate": 6.735000000000001e-06,
213
+ "loss": 0.5644,
214
  "step": 1350
215
  },
216
  {
217
+ "epoch": 1.1725293132328307,
218
+ "grad_norm": 4.269862651824951,
219
  "learning_rate": 6.985000000000001e-06,
220
+ "loss": 0.5356,
221
  "step": 1400
222
  },
223
  {
224
+ "epoch": 1.2144053601340032,
225
+ "grad_norm": 3.6681156158447266,
226
  "learning_rate": 7.235000000000001e-06,
227
+ "loss": 0.5246,
228
  "step": 1450
229
  },
230
  {
231
+ "epoch": 1.2562814070351758,
232
+ "grad_norm": 3.1227505207061768,
233
  "learning_rate": 7.485000000000001e-06,
234
+ "loss": 0.5168,
235
  "step": 1500
236
  },
237
  {
238
+ "epoch": 1.2562814070351758,
239
+ "eval_loss": 0.45785218477249146,
240
+ "eval_runtime": 211.6749,
241
+ "eval_samples_per_second": 40.104,
242
+ "eval_steps_per_second": 5.017,
243
  "step": 1500
244
  },
245
  {
246
+ "epoch": 1.2981574539363483,
247
+ "grad_norm": 2.6239662170410156,
248
  "learning_rate": 7.735e-06,
249
+ "loss": 0.5153,
250
  "step": 1550
251
  },
252
  {
253
+ "epoch": 1.3400335008375208,
254
+ "grad_norm": 2.881901979446411,
255
  "learning_rate": 7.985e-06,
256
+ "loss": 0.518,
257
  "step": 1600
258
  },
259
  {
260
+ "epoch": 1.3819095477386933,
261
+ "grad_norm": 2.8052310943603516,
262
  "learning_rate": 8.235e-06,
263
+ "loss": 0.5272,
264
  "step": 1650
265
  },
266
  {
267
+ "epoch": 1.4237855946398659,
268
+ "grad_norm": 2.6334052085876465,
269
  "learning_rate": 8.485000000000001e-06,
270
+ "loss": 0.5188,
271
  "step": 1700
272
  },
273
  {
274
+ "epoch": 1.4656616415410384,
275
+ "grad_norm": 3.038274049758911,
276
  "learning_rate": 8.735000000000002e-06,
277
+ "loss": 0.5139,
278
  "step": 1750
279
  },
280
  {
281
+ "epoch": 1.507537688442211,
282
+ "grad_norm": 3.2438852787017822,
283
  "learning_rate": 8.985000000000001e-06,
284
+ "loss": 0.5092,
285
  "step": 1800
286
  },
287
  {
288
+ "epoch": 1.5494137353433834,
289
+ "grad_norm": 4.168883800506592,
290
  "learning_rate": 9.235e-06,
291
+ "loss": 0.5103,
292
  "step": 1850
293
  },
294
  {
295
+ "epoch": 1.591289782244556,
296
+ "grad_norm": 2.6399967670440674,
297
  "learning_rate": 9.485000000000002e-06,
298
+ "loss": 0.5025,
299
  "step": 1900
300
  },
301
  {
302
+ "epoch": 1.6331658291457285,
303
+ "grad_norm": 1.6828950643539429,
304
  "learning_rate": 9.735e-06,
305
+ "loss": 0.4963,
306
  "step": 1950
307
  },
308
  {
309
+ "epoch": 1.675041876046901,
310
+ "grad_norm": 1.8955535888671875,
311
  "learning_rate": 9.985000000000002e-06,
312
+ "loss": 0.4954,
313
  "step": 2000
314
  },
315
  {
316
+ "epoch": 1.675041876046901,
317
+ "eval_loss": 0.43827974796295166,
318
+ "eval_runtime": 212.1455,
319
+ "eval_samples_per_second": 40.015,
320
+ "eval_steps_per_second": 5.006,
321
  "step": 2000
322
  }
323
  ],
 
338
  "attributes": {}
339
  }
340
  },
341
+ "total_flos": 1.946262825111053e+16,
342
+ "train_batch_size": 32,
343
  "trial_name": null,
344
  "trial_params": null
345
  }
checkpoint-2000/training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b20f3b06aa67136878a19fd6f17a63e91db785ca54662fa7debf3d46f1fe910d
3
  size 5304
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f4687cdb2b9124db8f1d7d7525bb9403c3df24fb82ca7d74640c10781e1e94be
3
  size 5304