DrNicefellow commited on
Commit
d5efbe7
1 Parent(s): ca8f628

Upload 12 files

Browse files
config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MistralForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": 1,
7
+ "eos_token_id": 2,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 2048,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 3072,
12
+ "max_position_embeddings": 512,
13
+ "model_type": "mistral",
14
+ "num_attention_heads": 32,
15
+ "num_hidden_layers": 24,
16
+ "num_key_value_heads": 4,
17
+ "rms_norm_eps": 1e-06,
18
+ "rope_theta": 10000.0,
19
+ "sliding_window": 4096,
20
+ "tie_word_embeddings": false,
21
+ "torch_dtype": "float32",
22
+ "transformers_version": "4.41.0",
23
+ "use_cache": true,
24
+ "vocab_size": 32000
25
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "transformers_version": "4.41.0"
6
+ }
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b5c5033017be5b6be25ae6127e16665b074a283a1226a91e64703f37f87cf316
3
+ size 6485383922
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdee91576b37d82f15fda552a7fe094144a5a91e9c81e827bebe6f182f2a6c10
3
+ size 3242672438
rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ff264f99d31b522cc7e2a4eac9d38606d0c58a34c0adc74d71e0ca8b371dc36
3
+ size 14244
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:60cb5bcbd5853ab7ae07cac79c2cd0dbfb65907c91c1a7e8b2691330ed5e21ef
3
+ size 1064
special_tokens_map.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "unk_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
+ size 493443
tokenizer_config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ }
29
+ },
30
+ "additional_special_tokens": [],
31
+ "bos_token": "<s>",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "</s>",
34
+ "legacy": true,
35
+ "model_max_length": 1000000000000000019884624838656,
36
+ "pad_token": null,
37
+ "sp_model_kwargs": {},
38
+ "spaces_between_special_tokens": false,
39
+ "tokenizer_class": "LlamaTokenizer",
40
+ "unk_token": "<unk>",
41
+ "use_default_system_prompt": false
42
+ }
trainer_state.json ADDED
@@ -0,0 +1,1047 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.656944252197384,
5
+ "eval_steps": 100000000,
6
+ "global_step": 81000,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0,
13
+ "learning_rate": 1.0000000000000001e-07,
14
+ "loss": 10.7586,
15
+ "step": 1
16
+ },
17
+ {
18
+ "epoch": 0.0,
19
+ "learning_rate": 5e-05,
20
+ "loss": 6.5292,
21
+ "step": 500
22
+ },
23
+ {
24
+ "epoch": 0.01,
25
+ "learning_rate": 4.99998214600225e-05,
26
+ "loss": 4.7262,
27
+ "step": 1000
28
+ },
29
+ {
30
+ "epoch": 0.01,
31
+ "learning_rate": 4.9999285842640105e-05,
32
+ "loss": 4.1997,
33
+ "step": 1500
34
+ },
35
+ {
36
+ "epoch": 0.01,
37
+ "learning_rate": 4.999839315550315e-05,
38
+ "loss": 3.8892,
39
+ "step": 2000
40
+ },
41
+ {
42
+ "epoch": 0.02,
43
+ "learning_rate": 4.9997143411362066e-05,
44
+ "loss": 3.6618,
45
+ "step": 2500
46
+ },
47
+ {
48
+ "epoch": 0.02,
49
+ "learning_rate": 4.9995536628067196e-05,
50
+ "loss": 3.5132,
51
+ "step": 3000
52
+ },
53
+ {
54
+ "epoch": 0.03,
55
+ "learning_rate": 4.9993572828568536e-05,
56
+ "loss": 3.4092,
57
+ "step": 3500
58
+ },
59
+ {
60
+ "epoch": 0.03,
61
+ "learning_rate": 4.999125204091544e-05,
62
+ "loss": 3.3295,
63
+ "step": 4000
64
+ },
65
+ {
66
+ "epoch": 0.03,
67
+ "learning_rate": 4.998857429825615e-05,
68
+ "loss": 3.2596,
69
+ "step": 4500
70
+ },
71
+ {
72
+ "epoch": 0.04,
73
+ "learning_rate": 4.9985539638837424e-05,
74
+ "loss": 3.2068,
75
+ "step": 5000
76
+ },
77
+ {
78
+ "epoch": 0.04,
79
+ "learning_rate": 4.998214810600389e-05,
80
+ "loss": 3.163,
81
+ "step": 5500
82
+ },
83
+ {
84
+ "epoch": 0.04,
85
+ "learning_rate": 4.997839974819749e-05,
86
+ "loss": 3.1256,
87
+ "step": 6000
88
+ },
89
+ {
90
+ "epoch": 0.05,
91
+ "learning_rate": 4.997429461895675e-05,
92
+ "loss": 3.0844,
93
+ "step": 6500
94
+ },
95
+ {
96
+ "epoch": 0.05,
97
+ "learning_rate": 4.996983277691606e-05,
98
+ "loss": 3.0541,
99
+ "step": 7000
100
+ },
101
+ {
102
+ "epoch": 0.05,
103
+ "learning_rate": 4.996501428580478e-05,
104
+ "loss": 3.0293,
105
+ "step": 7500
106
+ },
107
+ {
108
+ "epoch": 0.06,
109
+ "learning_rate": 4.995983921444638e-05,
110
+ "loss": 2.999,
111
+ "step": 8000
112
+ },
113
+ {
114
+ "epoch": 0.06,
115
+ "learning_rate": 4.9954307636757434e-05,
116
+ "loss": 2.9744,
117
+ "step": 8500
118
+ },
119
+ {
120
+ "epoch": 0.06,
121
+ "learning_rate": 4.994841963174656e-05,
122
+ "loss": 2.9525,
123
+ "step": 9000
124
+ },
125
+ {
126
+ "epoch": 0.07,
127
+ "learning_rate": 4.9942175283513294e-05,
128
+ "loss": 2.9321,
129
+ "step": 9500
130
+ },
131
+ {
132
+ "epoch": 0.07,
133
+ "learning_rate": 4.993557468124691e-05,
134
+ "loss": 2.9237,
135
+ "step": 10000
136
+ },
137
+ {
138
+ "epoch": 0.08,
139
+ "learning_rate": 4.9928617919225106e-05,
140
+ "loss": 2.9021,
141
+ "step": 10500
142
+ },
143
+ {
144
+ "epoch": 0.08,
145
+ "learning_rate": 4.99213050968127e-05,
146
+ "loss": 2.8766,
147
+ "step": 11000
148
+ },
149
+ {
150
+ "epoch": 0.08,
151
+ "learning_rate": 4.991363631846019e-05,
152
+ "loss": 2.8691,
153
+ "step": 11500
154
+ },
155
+ {
156
+ "epoch": 0.09,
157
+ "learning_rate": 4.990561169370224e-05,
158
+ "loss": 2.8531,
159
+ "step": 12000
160
+ },
161
+ {
162
+ "epoch": 0.09,
163
+ "learning_rate": 4.989723133715618e-05,
164
+ "loss": 2.8366,
165
+ "step": 12500
166
+ },
167
+ {
168
+ "epoch": 0.09,
169
+ "learning_rate": 4.988849536852028e-05,
170
+ "loss": 2.8265,
171
+ "step": 13000
172
+ },
173
+ {
174
+ "epoch": 0.1,
175
+ "learning_rate": 4.987940391257212e-05,
176
+ "loss": 2.8099,
177
+ "step": 13500
178
+ },
179
+ {
180
+ "epoch": 0.1,
181
+ "learning_rate": 4.986995709916677e-05,
182
+ "loss": 2.8009,
183
+ "step": 14000
184
+ },
185
+ {
186
+ "epoch": 0.1,
187
+ "learning_rate": 4.9860155063234944e-05,
188
+ "loss": 2.7881,
189
+ "step": 14500
190
+ },
191
+ {
192
+ "epoch": 0.11,
193
+ "learning_rate": 4.9849997944781055e-05,
194
+ "loss": 2.7794,
195
+ "step": 15000
196
+ },
197
+ {
198
+ "epoch": 0.11,
199
+ "learning_rate": 4.9839485888881235e-05,
200
+ "loss": 2.7674,
201
+ "step": 15500
202
+ },
203
+ {
204
+ "epoch": 0.12,
205
+ "learning_rate": 4.982861904568127e-05,
206
+ "loss": 2.7559,
207
+ "step": 16000
208
+ },
209
+ {
210
+ "epoch": 0.12,
211
+ "learning_rate": 4.981739757039443e-05,
212
+ "loss": 2.7455,
213
+ "step": 16500
214
+ },
215
+ {
216
+ "epoch": 0.12,
217
+ "learning_rate": 4.9805821623299285e-05,
218
+ "loss": 2.7351,
219
+ "step": 17000
220
+ },
221
+ {
222
+ "epoch": 0.13,
223
+ "learning_rate": 4.979389136973737e-05,
224
+ "loss": 2.7317,
225
+ "step": 17500
226
+ },
227
+ {
228
+ "epoch": 0.13,
229
+ "learning_rate": 4.978160698011085e-05,
230
+ "loss": 2.7192,
231
+ "step": 18000
232
+ },
233
+ {
234
+ "epoch": 0.13,
235
+ "learning_rate": 4.976896862988012e-05,
236
+ "loss": 2.709,
237
+ "step": 18500
238
+ },
239
+ {
240
+ "epoch": 0.14,
241
+ "learning_rate": 4.975597649956122e-05,
242
+ "loss": 2.7043,
243
+ "step": 19000
244
+ },
245
+ {
246
+ "epoch": 0.14,
247
+ "learning_rate": 4.974263077472334e-05,
248
+ "loss": 2.6954,
249
+ "step": 19500
250
+ },
251
+ {
252
+ "epoch": 0.14,
253
+ "learning_rate": 4.97289316459861e-05,
254
+ "loss": 2.6905,
255
+ "step": 20000
256
+ },
257
+ {
258
+ "epoch": 0.15,
259
+ "learning_rate": 4.9714879309016874e-05,
260
+ "loss": 2.6782,
261
+ "step": 20500
262
+ },
263
+ {
264
+ "epoch": 0.15,
265
+ "learning_rate": 4.970047396452798e-05,
266
+ "loss": 2.6751,
267
+ "step": 21000
268
+ },
269
+ {
270
+ "epoch": 0.15,
271
+ "learning_rate": 4.968571581827381e-05,
272
+ "loss": 2.6659,
273
+ "step": 21500
274
+ },
275
+ {
276
+ "epoch": 0.16,
277
+ "learning_rate": 4.9670605081047886e-05,
278
+ "loss": 2.6594,
279
+ "step": 22000
280
+ },
281
+ {
282
+ "epoch": 0.16,
283
+ "learning_rate": 4.9655141968679866e-05,
284
+ "loss": 2.6553,
285
+ "step": 22500
286
+ },
287
+ {
288
+ "epoch": 0.17,
289
+ "learning_rate": 4.963932670203245e-05,
290
+ "loss": 2.6499,
291
+ "step": 23000
292
+ },
293
+ {
294
+ "epoch": 0.17,
295
+ "learning_rate": 4.962315950699822e-05,
296
+ "loss": 2.6391,
297
+ "step": 23500
298
+ },
299
+ {
300
+ "epoch": 0.17,
301
+ "learning_rate": 4.960664061449643e-05,
302
+ "loss": 2.6313,
303
+ "step": 24000
304
+ },
305
+ {
306
+ "epoch": 0.18,
307
+ "learning_rate": 4.958977026046971e-05,
308
+ "loss": 2.6318,
309
+ "step": 24500
310
+ },
311
+ {
312
+ "epoch": 0.18,
313
+ "learning_rate": 4.957254868588065e-05,
314
+ "loss": 2.6275,
315
+ "step": 25000
316
+ },
317
+ {
318
+ "epoch": 0.18,
319
+ "learning_rate": 4.955497613670842e-05,
320
+ "loss": 2.6192,
321
+ "step": 25500
322
+ },
323
+ {
324
+ "epoch": 0.19,
325
+ "learning_rate": 4.953705286394522e-05,
326
+ "loss": 2.6153,
327
+ "step": 26000
328
+ },
329
+ {
330
+ "epoch": 0.19,
331
+ "learning_rate": 4.9518779123592705e-05,
332
+ "loss": 2.6102,
333
+ "step": 26500
334
+ },
335
+ {
336
+ "epoch": 0.19,
337
+ "learning_rate": 4.9500155176658345e-05,
338
+ "loss": 2.601,
339
+ "step": 27000
340
+ },
341
+ {
342
+ "epoch": 0.2,
343
+ "learning_rate": 4.9481181289151655e-05,
344
+ "loss": 2.5991,
345
+ "step": 27500
346
+ },
347
+ {
348
+ "epoch": 0.2,
349
+ "learning_rate": 4.9461857732080436e-05,
350
+ "loss": 2.5946,
351
+ "step": 28000
352
+ },
353
+ {
354
+ "epoch": 0.21,
355
+ "learning_rate": 4.9442184781446876e-05,
356
+ "loss": 2.5911,
357
+ "step": 28500
358
+ },
359
+ {
360
+ "epoch": 0.21,
361
+ "learning_rate": 4.9422162718243626e-05,
362
+ "loss": 2.5827,
363
+ "step": 29000
364
+ },
365
+ {
366
+ "epoch": 0.21,
367
+ "learning_rate": 4.94017918284498e-05,
368
+ "loss": 2.5785,
369
+ "step": 29500
370
+ },
371
+ {
372
+ "epoch": 0.22,
373
+ "learning_rate": 4.938107240302683e-05,
374
+ "loss": 2.5757,
375
+ "step": 30000
376
+ },
377
+ {
378
+ "epoch": 0.22,
379
+ "learning_rate": 4.93600047379144e-05,
380
+ "loss": 2.5705,
381
+ "step": 30500
382
+ },
383
+ {
384
+ "epoch": 0.22,
385
+ "learning_rate": 4.9338589134026124e-05,
386
+ "loss": 2.5703,
387
+ "step": 31000
388
+ },
389
+ {
390
+ "epoch": 0.23,
391
+ "learning_rate": 4.931682589724534e-05,
392
+ "loss": 2.5593,
393
+ "step": 31500
394
+ },
395
+ {
396
+ "epoch": 0.23,
397
+ "learning_rate": 4.929471533842065e-05,
398
+ "loss": 2.5647,
399
+ "step": 32000
400
+ },
401
+ {
402
+ "epoch": 0.23,
403
+ "learning_rate": 4.9272257773361574e-05,
404
+ "loss": 2.5564,
405
+ "step": 32500
406
+ },
407
+ {
408
+ "epoch": 0.24,
409
+ "learning_rate": 4.924945352283394e-05,
410
+ "loss": 2.5492,
411
+ "step": 33000
412
+ },
413
+ {
414
+ "epoch": 0.24,
415
+ "learning_rate": 4.922630291255539e-05,
416
+ "loss": 2.5509,
417
+ "step": 33500
418
+ },
419
+ {
420
+ "epoch": 0.25,
421
+ "learning_rate": 4.920280627319067e-05,
422
+ "loss": 2.5466,
423
+ "step": 34000
424
+ },
425
+ {
426
+ "epoch": 0.25,
427
+ "learning_rate": 4.917896394034696e-05,
428
+ "loss": 2.5412,
429
+ "step": 34500
430
+ },
431
+ {
432
+ "epoch": 0.25,
433
+ "learning_rate": 4.915477625456899e-05,
434
+ "loss": 2.5383,
435
+ "step": 35000
436
+ },
437
+ {
438
+ "epoch": 0.26,
439
+ "learning_rate": 4.91302435613343e-05,
440
+ "loss": 2.5364,
441
+ "step": 35500
442
+ },
443
+ {
444
+ "epoch": 0.26,
445
+ "learning_rate": 4.91053662110482e-05,
446
+ "loss": 2.5292,
447
+ "step": 36000
448
+ },
449
+ {
450
+ "epoch": 0.26,
451
+ "learning_rate": 4.908014455903881e-05,
452
+ "loss": 2.5304,
453
+ "step": 36500
454
+ },
455
+ {
456
+ "epoch": 0.27,
457
+ "learning_rate": 4.905457896555199e-05,
458
+ "loss": 2.5257,
459
+ "step": 37000
460
+ },
461
+ {
462
+ "epoch": 0.27,
463
+ "learning_rate": 4.902866979574618e-05,
464
+ "loss": 2.5189,
465
+ "step": 37500
466
+ },
467
+ {
468
+ "epoch": 0.27,
469
+ "learning_rate": 4.9002417419687183e-05,
470
+ "loss": 2.522,
471
+ "step": 38000
472
+ },
473
+ {
474
+ "epoch": 0.28,
475
+ "learning_rate": 4.897582221234289e-05,
476
+ "loss": 2.5141,
477
+ "step": 38500
478
+ },
479
+ {
480
+ "epoch": 0.28,
481
+ "learning_rate": 4.8948884553577926e-05,
482
+ "loss": 2.517,
483
+ "step": 39000
484
+ },
485
+ {
486
+ "epoch": 0.28,
487
+ "learning_rate": 4.8921604828148206e-05,
488
+ "loss": 2.5077,
489
+ "step": 39500
490
+ },
491
+ {
492
+ "epoch": 0.29,
493
+ "learning_rate": 4.8893983425695445e-05,
494
+ "loss": 2.5068,
495
+ "step": 40000
496
+ },
497
+ {
498
+ "epoch": 0.29,
499
+ "learning_rate": 4.8866020740741626e-05,
500
+ "loss": 2.5048,
501
+ "step": 40500
502
+ },
503
+ {
504
+ "epoch": 0.3,
505
+ "learning_rate": 4.883771717268331e-05,
506
+ "loss": 2.4972,
507
+ "step": 41000
508
+ },
509
+ {
510
+ "epoch": 0.3,
511
+ "learning_rate": 4.8809073125785964e-05,
512
+ "loss": 2.4999,
513
+ "step": 41500
514
+ },
515
+ {
516
+ "epoch": 0.3,
517
+ "learning_rate": 4.8780089009178206e-05,
518
+ "loss": 2.4941,
519
+ "step": 42000
520
+ },
521
+ {
522
+ "epoch": 0.31,
523
+ "learning_rate": 4.8750765236845894e-05,
524
+ "loss": 2.4891,
525
+ "step": 42500
526
+ },
527
+ {
528
+ "epoch": 0.31,
529
+ "learning_rate": 4.8721102227626304e-05,
530
+ "loss": 2.4876,
531
+ "step": 43000
532
+ },
533
+ {
534
+ "epoch": 0.31,
535
+ "learning_rate": 4.869110040520206e-05,
536
+ "loss": 2.4859,
537
+ "step": 43500
538
+ },
539
+ {
540
+ "epoch": 0.32,
541
+ "learning_rate": 4.8660760198095145e-05,
542
+ "loss": 2.4833,
543
+ "step": 44000
544
+ },
545
+ {
546
+ "epoch": 0.32,
547
+ "learning_rate": 4.8630082039660744e-05,
548
+ "loss": 2.4825,
549
+ "step": 44500
550
+ },
551
+ {
552
+ "epoch": 0.32,
553
+ "learning_rate": 4.859906636808108e-05,
554
+ "loss": 2.4801,
555
+ "step": 45000
556
+ },
557
+ {
558
+ "epoch": 0.33,
559
+ "learning_rate": 4.8567713626359134e-05,
560
+ "loss": 2.4763,
561
+ "step": 45500
562
+ },
563
+ {
564
+ "epoch": 0.33,
565
+ "learning_rate": 4.853602426231234e-05,
566
+ "loss": 2.4742,
567
+ "step": 46000
568
+ },
569
+ {
570
+ "epoch": 0.34,
571
+ "learning_rate": 4.850399872856615e-05,
572
+ "loss": 2.4702,
573
+ "step": 46500
574
+ },
575
+ {
576
+ "epoch": 0.34,
577
+ "learning_rate": 4.847163748254762e-05,
578
+ "loss": 2.4679,
579
+ "step": 47000
580
+ },
581
+ {
582
+ "epoch": 0.34,
583
+ "learning_rate": 4.843894098647884e-05,
584
+ "loss": 2.4676,
585
+ "step": 47500
586
+ },
587
+ {
588
+ "epoch": 0.35,
589
+ "learning_rate": 4.840590970737035e-05,
590
+ "loss": 2.4679,
591
+ "step": 48000
592
+ },
593
+ {
594
+ "epoch": 0.35,
595
+ "learning_rate": 4.8372544117014436e-05,
596
+ "loss": 2.4644,
597
+ "step": 48500
598
+ },
599
+ {
600
+ "epoch": 0.35,
601
+ "learning_rate": 4.8338844691978456e-05,
602
+ "loss": 2.4622,
603
+ "step": 49000
604
+ },
605
+ {
606
+ "epoch": 0.36,
607
+ "learning_rate": 4.830481191359797e-05,
608
+ "loss": 2.4605,
609
+ "step": 49500
610
+ },
611
+ {
612
+ "epoch": 0.36,
613
+ "learning_rate": 4.82704462679699e-05,
614
+ "loss": 2.4552,
615
+ "step": 50000
616
+ },
617
+ {
618
+ "epoch": 0.36,
619
+ "learning_rate": 4.823574824594558e-05,
620
+ "loss": 2.4501,
621
+ "step": 50500
622
+ },
623
+ {
624
+ "epoch": 0.37,
625
+ "learning_rate": 4.820071834312372e-05,
626
+ "loss": 2.4563,
627
+ "step": 51000
628
+ },
629
+ {
630
+ "epoch": 0.37,
631
+ "learning_rate": 4.816535705984337e-05,
632
+ "loss": 2.4467,
633
+ "step": 51500
634
+ },
635
+ {
636
+ "epoch": 0.37,
637
+ "learning_rate": 4.812966490117676e-05,
638
+ "loss": 2.4434,
639
+ "step": 52000
640
+ },
641
+ {
642
+ "epoch": 0.38,
643
+ "learning_rate": 4.8093642376922057e-05,
644
+ "loss": 2.4422,
645
+ "step": 52500
646
+ },
647
+ {
648
+ "epoch": 0.38,
649
+ "learning_rate": 4.8057290001596106e-05,
650
+ "loss": 2.4462,
651
+ "step": 53000
652
+ },
653
+ {
654
+ "epoch": 0.39,
655
+ "learning_rate": 4.802060829442711e-05,
656
+ "loss": 2.4424,
657
+ "step": 53500
658
+ },
659
+ {
660
+ "epoch": 0.39,
661
+ "learning_rate": 4.798359777934714e-05,
662
+ "loss": 2.4373,
663
+ "step": 54000
664
+ },
665
+ {
666
+ "epoch": 0.39,
667
+ "learning_rate": 4.7946258984984735e-05,
668
+ "loss": 2.4381,
669
+ "step": 54500
670
+ },
671
+ {
672
+ "epoch": 0.4,
673
+ "learning_rate": 4.7908592444657287e-05,
674
+ "loss": 2.4334,
675
+ "step": 55000
676
+ },
677
+ {
678
+ "epoch": 0.4,
679
+ "learning_rate": 4.787059869636346e-05,
680
+ "loss": 2.4328,
681
+ "step": 55500
682
+ },
683
+ {
684
+ "epoch": 0.4,
685
+ "learning_rate": 4.783227828277549e-05,
686
+ "loss": 2.4284,
687
+ "step": 56000
688
+ },
689
+ {
690
+ "epoch": 0.41,
691
+ "learning_rate": 4.779363175123144e-05,
692
+ "loss": 2.4278,
693
+ "step": 56500
694
+ },
695
+ {
696
+ "epoch": 0.41,
697
+ "learning_rate": 4.7754659653727384e-05,
698
+ "loss": 2.4231,
699
+ "step": 57000
700
+ },
701
+ {
702
+ "epoch": 0.41,
703
+ "learning_rate": 4.771536254690951e-05,
704
+ "loss": 2.4212,
705
+ "step": 57500
706
+ },
707
+ {
708
+ "epoch": 0.42,
709
+ "learning_rate": 4.767574099206619e-05,
710
+ "loss": 2.4247,
711
+ "step": 58000
712
+ },
713
+ {
714
+ "epoch": 0.42,
715
+ "learning_rate": 4.763579555511993e-05,
716
+ "loss": 2.4209,
717
+ "step": 58500
718
+ },
719
+ {
720
+ "epoch": 0.43,
721
+ "learning_rate": 4.759552680661933e-05,
722
+ "loss": 2.4224,
723
+ "step": 59000
724
+ },
725
+ {
726
+ "epoch": 0.43,
727
+ "learning_rate": 4.755493532173092e-05,
728
+ "loss": 2.4204,
729
+ "step": 59500
730
+ },
731
+ {
732
+ "epoch": 0.43,
733
+ "learning_rate": 4.75140216802309e-05,
734
+ "loss": 2.4201,
735
+ "step": 60000
736
+ },
737
+ {
738
+ "epoch": 0.44,
739
+ "learning_rate": 4.7472786466496944e-05,
740
+ "loss": 2.4127,
741
+ "step": 60500
742
+ },
743
+ {
744
+ "epoch": 0.44,
745
+ "learning_rate": 4.7431230269499764e-05,
746
+ "loss": 2.4146,
747
+ "step": 61000
748
+ },
749
+ {
750
+ "epoch": 0.44,
751
+ "learning_rate": 4.7389353682794765e-05,
752
+ "loss": 2.4074,
753
+ "step": 61500
754
+ },
755
+ {
756
+ "epoch": 0.45,
757
+ "learning_rate": 4.7347157304513544e-05,
758
+ "loss": 2.411,
759
+ "step": 62000
760
+ },
761
+ {
762
+ "epoch": 0.45,
763
+ "learning_rate": 4.7304641737355326e-05,
764
+ "loss": 2.411,
765
+ "step": 62500
766
+ },
767
+ {
768
+ "epoch": 0.45,
769
+ "learning_rate": 4.7261807588578375e-05,
770
+ "loss": 2.4068,
771
+ "step": 63000
772
+ },
773
+ {
774
+ "epoch": 0.45778831291669553,
775
+ "grad_norm": 0.4898933470249176,
776
+ "learning_rate": 4.721865546999135e-05,
777
+ "loss": 2.3865,
778
+ "step": 63500
779
+ },
780
+ {
781
+ "epoch": 0.4613929453018663,
782
+ "grad_norm": 0.5031083226203918,
783
+ "learning_rate": 4.7175185997944485e-05,
784
+ "loss": 2.3853,
785
+ "step": 64000
786
+ },
787
+ {
788
+ "epoch": 0.46499757768703714,
789
+ "grad_norm": 0.5099691152572632,
790
+ "learning_rate": 4.713139979332089e-05,
791
+ "loss": 2.3843,
792
+ "step": 64500
793
+ },
794
+ {
795
+ "epoch": 0.468602210072208,
796
+ "grad_norm": 0.528971791267395,
797
+ "learning_rate": 4.708729748152759e-05,
798
+ "loss": 2.3867,
799
+ "step": 65000
800
+ },
801
+ {
802
+ "epoch": 0.4722068424573788,
803
+ "grad_norm": 0.5447410941123962,
804
+ "learning_rate": 4.7042879692486655e-05,
805
+ "loss": 2.3871,
806
+ "step": 65500
807
+ },
808
+ {
809
+ "epoch": 0.47581147484254965,
810
+ "grad_norm": 0.5187677145004272,
811
+ "learning_rate": 4.699814706062616e-05,
812
+ "loss": 2.3821,
813
+ "step": 66000
814
+ },
815
+ {
816
+ "epoch": 0.4794161072277205,
817
+ "grad_norm": 0.5079197883605957,
818
+ "learning_rate": 4.6953100224871163e-05,
819
+ "loss": 2.3846,
820
+ "step": 66500
821
+ },
822
+ {
823
+ "epoch": 0.4830207396128913,
824
+ "grad_norm": 0.5204797387123108,
825
+ "learning_rate": 4.6907739828634536e-05,
826
+ "loss": 2.3867,
827
+ "step": 67000
828
+ },
829
+ {
830
+ "epoch": 0.48662537199806216,
831
+ "grad_norm": 0.51589435338974,
832
+ "learning_rate": 4.6862066519807816e-05,
833
+ "loss": 2.3792,
834
+ "step": 67500
835
+ },
836
+ {
837
+ "epoch": 0.490230004383233,
838
+ "grad_norm": 0.5342502593994141,
839
+ "learning_rate": 4.681608095075192e-05,
840
+ "loss": 2.3767,
841
+ "step": 68000
842
+ },
843
+ {
844
+ "epoch": 0.49383463676840383,
845
+ "grad_norm": 0.5265562534332275,
846
+ "learning_rate": 4.676978377828785e-05,
847
+ "loss": 2.3839,
848
+ "step": 68500
849
+ },
850
+ {
851
+ "epoch": 0.4974392691535746,
852
+ "grad_norm": 0.5328825116157532,
853
+ "learning_rate": 4.67231756636873e-05,
854
+ "loss": 2.3813,
855
+ "step": 69000
856
+ },
857
+ {
858
+ "epoch": 0.5010439015387455,
859
+ "grad_norm": 0.4992935061454773,
860
+ "learning_rate": 4.6676257272663194e-05,
861
+ "loss": 2.377,
862
+ "step": 69500
863
+ },
864
+ {
865
+ "epoch": 0.5046485339239163,
866
+ "grad_norm": 0.5137242674827576,
867
+ "learning_rate": 4.662902927536022e-05,
868
+ "loss": 2.3798,
869
+ "step": 70000
870
+ },
871
+ {
872
+ "epoch": 0.5082531663090871,
873
+ "grad_norm": 0.5188072323799133,
874
+ "learning_rate": 4.658149234634523e-05,
875
+ "loss": 2.3783,
876
+ "step": 70500
877
+ },
878
+ {
879
+ "epoch": 0.511857798694258,
880
+ "grad_norm": 0.5286343693733215,
881
+ "learning_rate": 4.65336471645976e-05,
882
+ "loss": 2.3719,
883
+ "step": 71000
884
+ },
885
+ {
886
+ "epoch": 0.5154624310794288,
887
+ "grad_norm": 0.5098246335983276,
888
+ "learning_rate": 4.648549441349953e-05,
889
+ "loss": 2.3712,
890
+ "step": 71500
891
+ },
892
+ {
893
+ "epoch": 0.5190670634645996,
894
+ "grad_norm": 0.528311014175415,
895
+ "learning_rate": 4.643703478082632e-05,
896
+ "loss": 2.37,
897
+ "step": 72000
898
+ },
899
+ {
900
+ "epoch": 0.5226716958497705,
901
+ "grad_norm": 0.5106325745582581,
902
+ "learning_rate": 4.638826895873651e-05,
903
+ "loss": 2.3717,
904
+ "step": 72500
905
+ },
906
+ {
907
+ "epoch": 0.5262763282349413,
908
+ "grad_norm": 0.5130262970924377,
909
+ "learning_rate": 4.6339197643762e-05,
910
+ "loss": 2.3669,
911
+ "step": 73000
912
+ },
913
+ {
914
+ "epoch": 0.5298809606201121,
915
+ "grad_norm": 0.5350182056427002,
916
+ "learning_rate": 4.628982153679811e-05,
917
+ "loss": 2.3675,
918
+ "step": 73500
919
+ },
920
+ {
921
+ "epoch": 0.533485593005283,
922
+ "grad_norm": 0.5258482098579407,
923
+ "learning_rate": 4.6240141343093545e-05,
924
+ "loss": 2.3673,
925
+ "step": 74000
926
+ },
927
+ {
928
+ "epoch": 0.5370902253904538,
929
+ "grad_norm": 0.5219452977180481,
930
+ "learning_rate": 4.6190157772240385e-05,
931
+ "loss": 2.3644,
932
+ "step": 74500
933
+ },
934
+ {
935
+ "epoch": 0.5406948577756246,
936
+ "grad_norm": 0.531566858291626,
937
+ "learning_rate": 4.6139871538163856e-05,
938
+ "loss": 2.366,
939
+ "step": 75000
940
+ },
941
+ {
942
+ "epoch": 0.5442994901607955,
943
+ "grad_norm": 0.5164352655410767,
944
+ "learning_rate": 4.608928335911222e-05,
945
+ "loss": 2.3651,
946
+ "step": 75500
947
+ },
948
+ {
949
+ "epoch": 0.5479041225459663,
950
+ "grad_norm": 0.558618426322937,
951
+ "learning_rate": 4.603839395764646e-05,
952
+ "loss": 2.3618,
953
+ "step": 76000
954
+ },
955
+ {
956
+ "epoch": 0.5515087549311372,
957
+ "grad_norm": 0.5367295145988464,
958
+ "learning_rate": 4.598720406062999e-05,
959
+ "loss": 2.3581,
960
+ "step": 76500
961
+ },
962
+ {
963
+ "epoch": 0.5551133873163079,
964
+ "grad_norm": 0.5240716934204102,
965
+ "learning_rate": 4.593571439921825e-05,
966
+ "loss": 2.3609,
967
+ "step": 77000
968
+ },
969
+ {
970
+ "epoch": 0.5587180197014787,
971
+ "grad_norm": 0.5218440890312195,
972
+ "learning_rate": 4.5883925708848275e-05,
973
+ "loss": 2.3583,
974
+ "step": 77500
975
+ },
976
+ {
977
+ "epoch": 0.5623226520866496,
978
+ "grad_norm": 0.5101985335350037,
979
+ "learning_rate": 4.58318387292282e-05,
980
+ "loss": 2.3587,
981
+ "step": 78000
982
+ },
983
+ {
984
+ "epoch": 0.636668195030798,
985
+ "grad_norm": 0.527410626411438,
986
+ "learning_rate": 4.469806659761896e-05,
987
+ "loss": 2.3498,
988
+ "step": 78500
989
+ },
990
+ {
991
+ "epoch": 0.6407234064641152,
992
+ "grad_norm": 0.4921613335609436,
993
+ "learning_rate": 4.4632426356679503e-05,
994
+ "loss": 2.3469,
995
+ "step": 79000
996
+ },
997
+ {
998
+ "epoch": 0.6447786178974323,
999
+ "grad_norm": 0.4988476037979126,
1000
+ "learning_rate": 4.456643111161236e-05,
1001
+ "loss": 2.348,
1002
+ "step": 79500
1003
+ },
1004
+ {
1005
+ "epoch": 0.6488338293307495,
1006
+ "grad_norm": 0.48752740025520325,
1007
+ "learning_rate": 4.450008205577918e-05,
1008
+ "loss": 2.3469,
1009
+ "step": 80000
1010
+ },
1011
+ {
1012
+ "epoch": 0.6528890407640667,
1013
+ "grad_norm": 0.5155342817306519,
1014
+ "learning_rate": 4.443338038893938e-05,
1015
+ "loss": 2.346,
1016
+ "step": 80500
1017
+ },
1018
+ {
1019
+ "epoch": 0.656944252197384,
1020
+ "grad_norm": 0.4966714680194855,
1021
+ "learning_rate": 4.4366327317228506e-05,
1022
+ "loss": 2.346,
1023
+ "step": 81000
1024
+ }
1025
+ ],
1026
+ "logging_steps": 500,
1027
+ "max_steps": 369894,
1028
+ "num_input_tokens_seen": 0,
1029
+ "num_train_epochs": 3,
1030
+ "save_steps": 3000,
1031
+ "stateful_callbacks": {
1032
+ "TrainerControl": {
1033
+ "args": {
1034
+ "should_epoch_stop": false,
1035
+ "should_evaluate": false,
1036
+ "should_log": false,
1037
+ "should_save": true,
1038
+ "should_training_stop": false
1039
+ },
1040
+ "attributes": {}
1041
+ }
1042
+ },
1043
+ "total_flos": 2.9802636509184e+19,
1044
+ "train_batch_size": 12,
1045
+ "trial_name": null,
1046
+ "trial_params": null
1047
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:065ebb10f7f1fa4e4d5e3bf0f21f456da189b06345fcfc9a69be6e3a85f3e79e
3
+ size 5112