Hugging Face's logo

cadaeic
/
llama2-7b-recipe-lora

Text Generation
Transformers
llama
generated_from_trainer
Inference Endpoints
text-generation-inference
8-bit precision
bitsandbytes
Model card Files Community
1

Upload folder using huggingface_hub

#1
by cadaeic - opened
base: refs/heads/main
←
from: refs/pr/1
Discussion Files changed
+2966
-0
Files changed (40) hide show
  1. README.md +64 -0
  2. adapter_config.json +28 -0
  3. adapter_model.bin +3 -0
  4. added_tokens.json +5 -0
  5. checkpoint-100/README.md +21 -0
  6. checkpoint-100/adapter_config.json +28 -0
  7. checkpoint-100/adapter_model.bin +3 -0
  8. checkpoint-100/optimizer.pt +3 -0
  9. checkpoint-100/rng_state.pth +3 -0
  10. checkpoint-100/scheduler.pt +3 -0
  11. checkpoint-100/trainer_state.json +659 -0
  12. checkpoint-100/training_args.bin +3 -0
  13. checkpoint-92/README.md +21 -0
  14. checkpoint-92/adapter_config.json +28 -0
  15. checkpoint-92/adapter_model.bin +3 -0
  16. checkpoint-92/optimizer.pt +3 -0
  17. checkpoint-92/rng_state.pth +3 -0
  18. checkpoint-92/scheduler.pt +3 -0
  19. checkpoint-92/trainer_state.json +603 -0
  20. checkpoint-92/training_args.bin +3 -0
  21. checkpoint-95/README.md +21 -0
  22. checkpoint-95/adapter_config.json +28 -0
  23. checkpoint-95/adapter_model.bin +3 -0
  24. checkpoint-95/optimizer.pt +3 -0
  25. checkpoint-95/rng_state.pth +3 -0
  26. checkpoint-95/scheduler.pt +3 -0
  27. checkpoint-95/trainer_state.json +621 -0
  28. checkpoint-95/training_args.bin +3 -0
  29. checkpoint-97/README.md +21 -0
  30. checkpoint-97/adapter_config.json +28 -0
  31. checkpoint-97/adapter_model.bin +3 -0
  32. checkpoint-97/optimizer.pt +3 -0
  33. checkpoint-97/rng_state.pth +3 -0
  34. checkpoint-97/scheduler.pt +3 -0
  35. checkpoint-97/trainer_state.json +633 -0
  36. checkpoint-97/training_args.bin +3 -0
  37. config.json +40 -0
  38. special_tokens_map.json +6 -0
  39. tokenizer.model +3 -0
  40. tokenizer_config.json +45 -0
README.md ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: NousResearch/Llama-2-7b-hf
3
+ tags:
4
+ - generated_from_trainer
5
+ model-index:
6
+ - name: lora-out
7
+ results: []
8
+ ---
9
+
10
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
11
+ should probably proofread and complete it, then remove this comment. -->
12
+
13
+ [<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
14
+ # lora-out
15
+
16
+ This model is a fine-tuned version of [NousResearch/Llama-2-7b-hf](https://huggingface.co/NousResearch/Llama-2-7b-hf) on the None dataset.
17
+ It achieves the following results on the evaluation set:
18
+ - Loss: 0.8666
19
+
20
+ ## Model description
21
+
22
+ More information needed
23
+
24
+ ## Intended uses & limitations
25
+
26
+ More information needed
27
+
28
+ ## Training and evaluation data
29
+
30
+ More information needed
31
+
32
+ ## Training procedure
33
+
34
+ ### Training hyperparameters
35
+
36
+ The following hyperparameters were used during training:
37
+ - learning_rate: 0.0002
38
+ - train_batch_size: 6
39
+ - eval_batch_size: 6
40
+ - seed: 42
41
+ - gradient_accumulation_steps: 8
42
+ - total_train_batch_size: 48
43
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
44
+ - lr_scheduler_type: cosine
45
+ - lr_scheduler_warmup_steps: 10
46
+ - num_epochs: 50
47
+
48
+ ### Training results
49
+
50
+ | Training Loss | Epoch | Step | Validation Loss |
51
+ |:-------------:|:-----:|:----:|:---------------:|
52
+ | 0.9548 | 8.0 | 20 | 0.9240 |
53
+ | 0.8514 | 16.0 | 40 | 0.8523 |
54
+ | 0.7774 | 24.0 | 60 | 0.8498 |
55
+ | 0.7178 | 32.0 | 80 | 0.8597 |
56
+ | 0.7103 | 40.0 | 100 | 0.8666 |
57
+
58
+
59
+ ### Framework versions
60
+
61
+ - Transformers 4.34.0.dev0
62
+ - Pytorch 2.0.1+cu118
63
+ - Datasets 2.14.5
64
+ - Tokenizers 0.14.0
adapter_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "NousResearch/Llama-2-7b-hf",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 16,
12
+ "lora_dropout": 0.05,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 32,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "q_proj",
20
+ "down_proj",
21
+ "v_proj",
22
+ "o_proj",
23
+ "up_proj",
24
+ "gate_proj",
25
+ "k_proj"
26
+ ],
27
+ "task_type": "CAUSAL_LM"
28
+ }
adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc89aa04ee209e32eddca7899848d7d981a9fd8e085e60586cf21d482eb92b0e
3
+ size 319977229
added_tokens.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "</s>": 2,
3
+ "<s>": 1,
4
+ "<unk>": 0
5
+ }
checkpoint-100/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: True
10
+ - load_in_4bit: False
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: fp4
16
+ - bnb_4bit_use_double_quant: False
17
+ - bnb_4bit_compute_dtype: float32
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
checkpoint-100/adapter_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "NousResearch/Llama-2-7b-hf",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 16,
12
+ "lora_dropout": 0.05,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 32,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "q_proj",
20
+ "down_proj",
21
+ "v_proj",
22
+ "o_proj",
23
+ "up_proj",
24
+ "gate_proj",
25
+ "k_proj"
26
+ ],
27
+ "task_type": "CAUSAL_LM"
28
+ }
checkpoint-100/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc89aa04ee209e32eddca7899848d7d981a9fd8e085e60586cf21d482eb92b0e
3
+ size 319977229
checkpoint-100/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84393b02e1f6ed434e663dfdec14d02d4a9bb8a808d2aaa0cd2e88209c7d5bc6
3
+ size 160735647
checkpoint-100/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7483d8ad7bc3e44454c055137e336d425c616846f6a83eb7cced7546acd49b50
3
+ size 14575
checkpoint-100/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a09c5a80c301db5abcd0bc19859cc7c5a2d4cbc2f18140aeda130762ab36bc79
3
+ size 627
checkpoint-100/trainer_state.json ADDED
@@ -0,0 +1,659 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 40.0,
5
+ "eval_steps": 20,
6
+ "global_step": 100,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.4,
13
+ "learning_rate": 2e-05,
14
+ "loss": 1.2516,
15
+ "step": 1
16
+ },
17
+ {
18
+ "epoch": 0.8,
19
+ "learning_rate": 4e-05,
20
+ "loss": 1.2581,
21
+ "step": 2
22
+ },
23
+ {
24
+ "epoch": 1.2,
25
+ "learning_rate": 6e-05,
26
+ "loss": 1.2684,
27
+ "step": 3
28
+ },
29
+ {
30
+ "epoch": 1.6,
31
+ "learning_rate": 8e-05,
32
+ "loss": 1.2529,
33
+ "step": 4
34
+ },
35
+ {
36
+ "epoch": 2.0,
37
+ "learning_rate": 0.0001,
38
+ "loss": 1.2441,
39
+ "step": 5
40
+ },
41
+ {
42
+ "epoch": 2.4,
43
+ "learning_rate": 0.00012,
44
+ "loss": 1.2166,
45
+ "step": 6
46
+ },
47
+ {
48
+ "epoch": 2.8,
49
+ "learning_rate": 0.00014,
50
+ "loss": 1.225,
51
+ "step": 7
52
+ },
53
+ {
54
+ "epoch": 3.2,
55
+ "learning_rate": 0.00016,
56
+ "loss": 1.1785,
57
+ "step": 8
58
+ },
59
+ {
60
+ "epoch": 3.6,
61
+ "learning_rate": 0.00018,
62
+ "loss": 1.1504,
63
+ "step": 9
64
+ },
65
+ {
66
+ "epoch": 4.0,
67
+ "learning_rate": 0.0002,
68
+ "loss": 1.1148,
69
+ "step": 10
70
+ },
71
+ {
72
+ "epoch": 4.4,
73
+ "learning_rate": 0.0001999390827019096,
74
+ "loss": 1.0771,
75
+ "step": 11
76
+ },
77
+ {
78
+ "epoch": 4.8,
79
+ "learning_rate": 0.00019975640502598244,
80
+ "loss": 1.0768,
81
+ "step": 12
82
+ },
83
+ {
84
+ "epoch": 5.2,
85
+ "learning_rate": 0.00019945218953682734,
86
+ "loss": 1.0422,
87
+ "step": 13
88
+ },
89
+ {
90
+ "epoch": 5.6,
91
+ "learning_rate": 0.00019902680687415705,
92
+ "loss": 1.0305,
93
+ "step": 14
94
+ },
95
+ {
96
+ "epoch": 6.0,
97
+ "learning_rate": 0.00019848077530122083,
98
+ "loss": 1.0148,
99
+ "step": 15
100
+ },
101
+ {
102
+ "epoch": 6.4,
103
+ "learning_rate": 0.00019781476007338058,
104
+ "loss": 1.0024,
105
+ "step": 16
106
+ },
107
+ {
108
+ "epoch": 6.8,
109
+ "learning_rate": 0.00019702957262759965,
110
+ "loss": 0.9914,
111
+ "step": 17
112
+ },
113
+ {
114
+ "epoch": 7.2,
115
+ "learning_rate": 0.0001961261695938319,
116
+ "loss": 0.9746,
117
+ "step": 18
118
+ },
119
+ {
120
+ "epoch": 7.6,
121
+ "learning_rate": 0.00019510565162951537,
122
+ "loss": 0.9764,
123
+ "step": 19
124
+ },
125
+ {
126
+ "epoch": 8.0,
127
+ "learning_rate": 0.00019396926207859084,
128
+ "loss": 0.9548,
129
+ "step": 20
130
+ },
131
+ {
132
+ "epoch": 8.0,
133
+ "eval_loss": 0.9240104556083679,
134
+ "eval_runtime": 1.9642,
135
+ "eval_samples_per_second": 10.182,
136
+ "eval_steps_per_second": 2.036,
137
+ "step": 20
138
+ },
139
+ {
140
+ "epoch": 8.4,
141
+ "learning_rate": 0.00019271838545667876,
142
+ "loss": 0.9726,
143
+ "step": 21
144
+ },
145
+ {
146
+ "epoch": 8.8,
147
+ "learning_rate": 0.0001913545457642601,
148
+ "loss": 0.9381,
149
+ "step": 22
150
+ },
151
+ {
152
+ "epoch": 9.2,
153
+ "learning_rate": 0.0001898794046299167,
154
+ "loss": 0.9339,
155
+ "step": 23
156
+ },
157
+ {
158
+ "epoch": 9.6,
159
+ "learning_rate": 0.00018829475928589271,
160
+ "loss": 0.9256,
161
+ "step": 24
162
+ },
163
+ {
164
+ "epoch": 10.0,
165
+ "learning_rate": 0.00018660254037844388,
166
+ "loss": 0.9266,
167
+ "step": 25
168
+ },
169
+ {
170
+ "epoch": 10.4,
171
+ "learning_rate": 0.0001848048096156426,
172
+ "loss": 0.9146,
173
+ "step": 26
174
+ },
175
+ {
176
+ "epoch": 10.8,
177
+ "learning_rate": 0.00018290375725550417,
178
+ "loss": 0.9155,
179
+ "step": 27
180
+ },
181
+ {
182
+ "epoch": 11.2,
183
+ "learning_rate": 0.00018090169943749476,
184
+ "loss": 0.8992,
185
+ "step": 28
186
+ },
187
+ {
188
+ "epoch": 11.6,
189
+ "learning_rate": 0.00017880107536067218,
190
+ "loss": 0.8951,
191
+ "step": 29
192
+ },
193
+ {
194
+ "epoch": 12.0,
195
+ "learning_rate": 0.0001766044443118978,
196
+ "loss": 0.8855,
197
+ "step": 30
198
+ },
199
+ {
200
+ "epoch": 12.4,
201
+ "learning_rate": 0.00017431448254773944,
202
+ "loss": 0.8901,
203
+ "step": 31
204
+ },
205
+ {
206
+ "epoch": 12.8,
207
+ "learning_rate": 0.0001719339800338651,
208
+ "loss": 0.8704,
209
+ "step": 32
210
+ },
211
+ {
212
+ "epoch": 13.2,
213
+ "learning_rate": 0.00016946583704589973,
214
+ "loss": 0.869,
215
+ "step": 33
216
+ },
217
+ {
218
+ "epoch": 13.6,
219
+ "learning_rate": 0.00016691306063588583,
220
+ "loss": 0.864,
221
+ "step": 34
222
+ },
223
+ {
224
+ "epoch": 14.0,
225
+ "learning_rate": 0.00016427876096865394,
226
+ "loss": 0.862,
227
+ "step": 35
228
+ },
229
+ {
230
+ "epoch": 14.4,
231
+ "learning_rate": 0.0001615661475325658,
232
+ "loss": 0.8577,
233
+ "step": 36
234
+ },
235
+ {
236
+ "epoch": 14.8,
237
+ "learning_rate": 0.00015877852522924732,
238
+ "loss": 0.8544,
239
+ "step": 37
240
+ },
241
+ {
242
+ "epoch": 15.2,
243
+ "learning_rate": 0.0001559192903470747,
244
+ "loss": 0.8467,
245
+ "step": 38
246
+ },
247
+ {
248
+ "epoch": 15.6,
249
+ "learning_rate": 0.0001529919264233205,
250
+ "loss": 0.8379,
251
+ "step": 39
252
+ },
253
+ {
254
+ "epoch": 16.0,
255
+ "learning_rate": 0.00015000000000000001,
256
+ "loss": 0.8514,
257
+ "step": 40
258
+ },
259
+ {
260
+ "epoch": 16.0,
261
+ "eval_loss": 0.8522964119911194,
262
+ "eval_runtime": 1.9623,
263
+ "eval_samples_per_second": 10.192,
264
+ "eval_steps_per_second": 2.038,
265
+ "step": 40
266
+ },
267
+ {
268
+ "epoch": 16.4,
269
+ "learning_rate": 0.00014694715627858908,
270
+ "loss": 0.8333,
271
+ "step": 41
272
+ },
273
+ {
274
+ "epoch": 16.8,
275
+ "learning_rate": 0.00014383711467890774,
276
+ "loss": 0.8379,
277
+ "step": 42
278
+ },
279
+ {
280
+ "epoch": 17.2,
281
+ "learning_rate": 0.00014067366430758004,
282
+ "loss": 0.8278,
283
+ "step": 43
284
+ },
285
+ {
286
+ "epoch": 17.6,
287
+ "learning_rate": 0.00013746065934159123,
288
+ "loss": 0.8273,
289
+ "step": 44
290
+ },
291
+ {
292
+ "epoch": 18.0,
293
+ "learning_rate": 0.00013420201433256689,
294
+ "loss": 0.8382,
295
+ "step": 45
296
+ },
297
+ {
298
+ "epoch": 18.4,
299
+ "learning_rate": 0.00013090169943749476,
300
+ "loss": 0.8208,
301
+ "step": 46
302
+ },
303
+ {
304
+ "epoch": 18.8,
305
+ "learning_rate": 0.0001275637355816999,
306
+ "loss": 0.8203,
307
+ "step": 47
308
+ },
309
+ {
310
+ "epoch": 19.2,
311
+ "learning_rate": 0.00012419218955996676,
312
+ "loss": 0.8118,
313
+ "step": 48
314
+ },
315
+ {
316
+ "epoch": 19.6,
317
+ "learning_rate": 0.00012079116908177593,
318
+ "loss": 0.8175,
319
+ "step": 49
320
+ },
321
+ {
322
+ "epoch": 20.0,
323
+ "learning_rate": 0.00011736481776669306,
324
+ "loss": 0.8059,
325
+ "step": 50
326
+ },
327
+ {
328
+ "epoch": 20.4,
329
+ "learning_rate": 0.00011391731009600654,
330
+ "loss": 0.8114,
331
+ "step": 51
332
+ },
333
+ {
334
+ "epoch": 20.8,
335
+ "learning_rate": 0.00011045284632676536,
336
+ "loss": 0.8004,
337
+ "step": 52
338
+ },
339
+ {
340
+ "epoch": 21.2,
341
+ "learning_rate": 0.00010697564737441252,
342
+ "loss": 0.8026,
343
+ "step": 53
344
+ },
345
+ {
346
+ "epoch": 21.6,
347
+ "learning_rate": 0.00010348994967025012,
348
+ "loss": 0.7943,
349
+ "step": 54
350
+ },
351
+ {
352
+ "epoch": 22.0,
353
+ "learning_rate": 0.0001,
354
+ "loss": 0.79,
355
+ "step": 55
356
+ },
357
+ {
358
+ "epoch": 22.4,
359
+ "learning_rate": 9.651005032974994e-05,
360
+ "loss": 0.7952,
361
+ "step": 56
362
+ },
363
+ {
364
+ "epoch": 22.8,
365
+ "learning_rate": 9.302435262558747e-05,
366
+ "loss": 0.7898,
367
+ "step": 57
368
+ },
369
+ {
370
+ "epoch": 23.2,
371
+ "learning_rate": 8.954715367323468e-05,
372
+ "loss": 0.7769,
373
+ "step": 58
374
+ },
375
+ {
376
+ "epoch": 23.6,
377
+ "learning_rate": 8.608268990399349e-05,
378
+ "loss": 0.7812,
379
+ "step": 59
380
+ },
381
+ {
382
+ "epoch": 24.0,
383
+ "learning_rate": 8.263518223330697e-05,
384
+ "loss": 0.7774,
385
+ "step": 60
386
+ },
387
+ {
388
+ "epoch": 24.0,
389
+ "eval_loss": 0.8498085141181946,
390
+ "eval_runtime": 1.9653,
391
+ "eval_samples_per_second": 10.177,
392
+ "eval_steps_per_second": 2.035,
393
+ "step": 60
394
+ },
395
+ {
396
+ "epoch": 24.4,
397
+ "learning_rate": 7.920883091822408e-05,
398
+ "loss": 0.7651,
399
+ "step": 61
400
+ },
401
+ {
402
+ "epoch": 24.8,
403
+ "learning_rate": 7.580781044003324e-05,
404
+ "loss": 0.778,
405
+ "step": 62
406
+ },
407
+ {
408
+ "epoch": 25.2,
409
+ "learning_rate": 7.243626441830009e-05,
410
+ "loss": 0.772,
411
+ "step": 63
412
+ },
413
+ {
414
+ "epoch": 25.6,
415
+ "learning_rate": 6.909830056250527e-05,
416
+ "loss": 0.767,
417
+ "step": 64
418
+ },
419
+ {
420
+ "epoch": 26.0,
421
+ "learning_rate": 6.579798566743314e-05,
422
+ "loss": 0.7569,
423
+ "step": 65
424
+ },
425
+ {
426
+ "epoch": 26.4,
427
+ "learning_rate": 6.25393406584088e-05,
428
+ "loss": 0.7554,
429
+ "step": 66
430
+ },
431
+ {
432
+ "epoch": 26.8,
433
+ "learning_rate": 5.9326335692419995e-05,
434
+ "loss": 0.76,
435
+ "step": 67
436
+ },
437
+ {
438
+ "epoch": 27.2,
439
+ "learning_rate": 5.616288532109225e-05,
440
+ "loss": 0.7554,
441
+ "step": 68
442
+ },
443
+ {
444
+ "epoch": 27.6,
445
+ "learning_rate": 5.305284372141095e-05,
446
+ "loss": 0.748,
447
+ "step": 69
448
+ },
449
+ {
450
+ "epoch": 28.0,
451
+ "learning_rate": 5.000000000000002e-05,
452
+ "loss": 0.7527,
453
+ "step": 70
454
+ },
455
+ {
456
+ "epoch": 28.4,
457
+ "learning_rate": 4.700807357667952e-05,
458
+ "loss": 0.7595,
459
+ "step": 71
460
+ },
461
+ {
462
+ "epoch": 28.8,
463
+ "learning_rate": 4.4080709652925336e-05,
464
+ "loss": 0.7394,
465
+ "step": 72
466
+ },
467
+ {
468
+ "epoch": 29.2,
469
+ "learning_rate": 4.12214747707527e-05,
470
+ "loss": 0.7348,
471
+ "step": 73
472
+ },
473
+ {
474
+ "epoch": 29.6,
475
+ "learning_rate": 3.843385246743417e-05,
476
+ "loss": 0.7497,
477
+ "step": 74
478
+ },
479
+ {
480
+ "epoch": 30.0,
481
+ "learning_rate": 3.5721239031346066e-05,
482
+ "loss": 0.7305,
483
+ "step": 75
484
+ },
485
+ {
486
+ "epoch": 30.4,
487
+ "learning_rate": 3.308693936411421e-05,
488
+ "loss": 0.7317,
489
+ "step": 76
490
+ },
491
+ {
492
+ "epoch": 30.8,
493
+ "learning_rate": 3.053416295410026e-05,
494
+ "loss": 0.7329,
495
+ "step": 77
496
+ },
497
+ {
498
+ "epoch": 31.2,
499
+ "learning_rate": 2.8066019966134904e-05,
500
+ "loss": 0.7494,
501
+ "step": 78
502
+ },
503
+ {
504
+ "epoch": 31.6,
505
+ "learning_rate": 2.5685517452260567e-05,
506
+ "loss": 0.7306,
507
+ "step": 79
508
+ },
509
+ {
510
+ "epoch": 32.0,
511
+ "learning_rate": 2.339555568810221e-05,
512
+ "loss": 0.7178,
513
+ "step": 80
514
+ },
515
+ {
516
+ "epoch": 32.0,
517
+ "eval_loss": 0.8597297072410583,
518
+ "eval_runtime": 1.9653,
519
+ "eval_samples_per_second": 10.177,
520
+ "eval_steps_per_second": 2.035,
521
+ "step": 80
522
+ },
523
+ {
524
+ "epoch": 32.4,
525
+ "learning_rate": 2.119892463932781e-05,
526
+ "loss": 0.7243,
527
+ "step": 81
528
+ },
529
+ {
530
+ "epoch": 32.8,
531
+ "learning_rate": 1.9098300562505266e-05,
532
+ "loss": 0.7393,
533
+ "step": 82
534
+ },
535
+ {
536
+ "epoch": 33.2,
537
+ "learning_rate": 1.7096242744495837e-05,
538
+ "loss": 0.7148,
539
+ "step": 83
540
+ },
541
+ {
542
+ "epoch": 33.6,
543
+ "learning_rate": 1.5195190384357404e-05,
544
+ "loss": 0.7261,
545
+ "step": 84
546
+ },
547
+ {
548
+ "epoch": 34.0,
549
+ "learning_rate": 1.339745962155613e-05,
550
+ "loss": 0.7228,
551
+ "step": 85
552
+ },
553
+ {
554
+ "epoch": 34.4,
555
+ "learning_rate": 1.1705240714107302e-05,
556
+ "loss": 0.7194,
557
+ "step": 86
558
+ },
559
+ {
560
+ "epoch": 34.8,
561
+ "learning_rate": 1.0120595370083318e-05,
562
+ "loss": 0.7252,
563
+ "step": 87
564
+ },
565
+ {
566
+ "epoch": 35.2,
567
+ "learning_rate": 8.645454235739903e-06,
568
+ "loss": 0.7188,
569
+ "step": 88
570
+ },
571
+ {
572
+ "epoch": 35.6,
573
+ "learning_rate": 7.281614543321269e-06,
574
+ "loss": 0.7199,
575
+ "step": 89
576
+ },
577
+ {
578
+ "epoch": 36.0,
579
+ "learning_rate": 6.030737921409169e-06,
580
+ "loss": 0.7239,
581
+ "step": 90
582
+ },
583
+ {
584
+ "epoch": 36.4,
585
+ "learning_rate": 4.8943483704846475e-06,
586
+ "loss": 0.7099,
587
+ "step": 91
588
+ },
589
+ {
590
+ "epoch": 36.8,
591
+ "learning_rate": 3.873830406168111e-06,
592
+ "loss": 0.7265,
593
+ "step": 92
594
+ },
595
+ {
596
+ "epoch": 37.2,
597
+ "learning_rate": 2.970427372400353e-06,
598
+ "loss": 0.7105,
599
+ "step": 93
600
+ },
601
+ {
602
+ "epoch": 37.6,
603
+ "learning_rate": 2.1852399266194314e-06,
604
+ "loss": 0.7149,
605
+ "step": 94
606
+ },
607
+ {
608
+ "epoch": 38.0,
609
+ "learning_rate": 1.5192246987791981e-06,
610
+ "loss": 0.7257,
611
+ "step": 95
612
+ },
613
+ {
614
+ "epoch": 38.4,
615
+ "learning_rate": 9.731931258429638e-07,
616
+ "loss": 0.7233,
617
+ "step": 96
618
+ },
619
+ {
620
+ "epoch": 38.8,
621
+ "learning_rate": 5.478104631726711e-07,
622
+ "loss": 0.7078,
623
+ "step": 97
624
+ },
625
+ {
626
+ "epoch": 39.2,
627
+ "learning_rate": 2.4359497401758024e-07,
628
+ "loss": 0.7192,
629
+ "step": 98
630
+ },
631
+ {
632
+ "epoch": 39.6,
633
+ "learning_rate": 6.09172980904238e-08,
634
+ "loss": 0.7239,
635
+ "step": 99
636
+ },
637
+ {
638
+ "epoch": 40.0,
639
+ "learning_rate": 0.0,
640
+ "loss": 0.7103,
641
+ "step": 100
642
+ },
643
+ {
644
+ "epoch": 40.0,
645
+ "eval_loss": 0.8666132092475891,
646
+ "eval_runtime": 1.9648,
647
+ "eval_samples_per_second": 10.179,
648
+ "eval_steps_per_second": 2.036,
649
+ "step": 100
650
+ }
651
+ ],
652
+ "logging_steps": 1,
653
+ "max_steps": 100,
654
+ "num_train_epochs": 50,
655
+ "save_steps": 500,
656
+ "total_flos": 7.888657163747328e+17,
657
+ "trial_name": null,
658
+ "trial_params": null
659
+ }
checkpoint-100/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b87ab0a71d9571469037bd31ca464f2d5cba068cb664754aff5b20bbe14bd6a1
3
+ size 4411
checkpoint-92/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: True
10
+ - load_in_4bit: False
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: fp4
16
+ - bnb_4bit_use_double_quant: False
17
+ - bnb_4bit_compute_dtype: float32
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
checkpoint-92/adapter_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "NousResearch/Llama-2-7b-hf",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 16,
12
+ "lora_dropout": 0.05,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 32,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "q_proj",
20
+ "down_proj",
21
+ "v_proj",
22
+ "o_proj",
23
+ "up_proj",
24
+ "gate_proj",
25
+ "k_proj"
26
+ ],
27
+ "task_type": "CAUSAL_LM"
28
+ }
checkpoint-92/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:118f7f4d0b69c377cbc344cf886a6f401590941ffb2e33acfd022d8268cd2741
3
+ size 319977229
checkpoint-92/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2bd07df8b39d049c691682024201963d6b36a1b9930f182c259acf8c97d254a7
3
+ size 160735647
checkpoint-92/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f24dd4a31082b708f688f21b98eaa8b549065382721956fb7daaab92e5117d9
3
+ size 14575
checkpoint-92/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4a3ecb9bbeaf2aec53e01df8b3f7b804ed88116e57365bc8cf797d311720412
3
+ size 627
checkpoint-92/trainer_state.json ADDED
@@ -0,0 +1,603 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 36.8,
5
+ "eval_steps": 20,
6
+ "global_step": 92,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.4,
13
+ "learning_rate": 2e-05,
14
+ "loss": 1.2516,
15
+ "step": 1
16
+ },
17
+ {
18
+ "epoch": 0.8,
19
+ "learning_rate": 4e-05,
20
+ "loss": 1.2581,
21
+ "step": 2
22
+ },
23
+ {
24
+ "epoch": 1.2,
25
+ "learning_rate": 6e-05,
26
+ "loss": 1.2684,
27
+ "step": 3
28
+ },
29
+ {
30
+ "epoch": 1.6,
31
+ "learning_rate": 8e-05,
32
+ "loss": 1.2529,
33
+ "step": 4
34
+ },
35
+ {
36
+ "epoch": 2.0,
37
+ "learning_rate": 0.0001,
38
+ "loss": 1.2441,
39
+ "step": 5
40
+ },
41
+ {
42
+ "epoch": 2.4,
43
+ "learning_rate": 0.00012,
44
+ "loss": 1.2166,
45
+ "step": 6
46
+ },
47
+ {
48
+ "epoch": 2.8,
49
+ "learning_rate": 0.00014,
50
+ "loss": 1.225,
51
+ "step": 7
52
+ },
53
+ {
54
+ "epoch": 3.2,
55
+ "learning_rate": 0.00016,
56
+ "loss": 1.1785,
57
+ "step": 8
58
+ },
59
+ {
60
+ "epoch": 3.6,
61
+ "learning_rate": 0.00018,
62
+ "loss": 1.1504,
63
+ "step": 9
64
+ },
65
+ {
66
+ "epoch": 4.0,
67
+ "learning_rate": 0.0002,
68
+ "loss": 1.1148,
69
+ "step": 10
70
+ },
71
+ {
72
+ "epoch": 4.4,
73
+ "learning_rate": 0.0001999390827019096,
74
+ "loss": 1.0771,
75
+ "step": 11
76
+ },
77
+ {
78
+ "epoch": 4.8,
79
+ "learning_rate": 0.00019975640502598244,
80
+ "loss": 1.0768,
81
+ "step": 12
82
+ },
83
+ {
84
+ "epoch": 5.2,
85
+ "learning_rate": 0.00019945218953682734,
86
+ "loss": 1.0422,
87
+ "step": 13
88
+ },
89
+ {
90
+ "epoch": 5.6,
91
+ "learning_rate": 0.00019902680687415705,
92
+ "loss": 1.0305,
93
+ "step": 14
94
+ },
95
+ {
96
+ "epoch": 6.0,
97
+ "learning_rate": 0.00019848077530122083,
98
+ "loss": 1.0148,
99
+ "step": 15
100
+ },
101
+ {
102
+ "epoch": 6.4,
103
+ "learning_rate": 0.00019781476007338058,
104
+ "loss": 1.0024,
105
+ "step": 16
106
+ },
107
+ {
108
+ "epoch": 6.8,
109
+ "learning_rate": 0.00019702957262759965,
110
+ "loss": 0.9914,
111
+ "step": 17
112
+ },
113
+ {
114
+ "epoch": 7.2,
115
+ "learning_rate": 0.0001961261695938319,
116
+ "loss": 0.9746,
117
+ "step": 18
118
+ },
119
+ {
120
+ "epoch": 7.6,
121
+ "learning_rate": 0.00019510565162951537,
122
+ "loss": 0.9764,
123
+ "step": 19
124
+ },
125
+ {
126
+ "epoch": 8.0,
127
+ "learning_rate": 0.00019396926207859084,
128
+ "loss": 0.9548,
129
+ "step": 20
130
+ },
131
+ {
132
+ "epoch": 8.0,
133
+ "eval_loss": 0.9240104556083679,
134
+ "eval_runtime": 1.9642,
135
+ "eval_samples_per_second": 10.182,
136
+ "eval_steps_per_second": 2.036,
137
+ "step": 20
138
+ },
139
+ {
140
+ "epoch": 8.4,
141
+ "learning_rate": 0.00019271838545667876,
142
+ "loss": 0.9726,
143
+ "step": 21
144
+ },
145
+ {
146
+ "epoch": 8.8,
147
+ "learning_rate": 0.0001913545457642601,
148
+ "loss": 0.9381,
149
+ "step": 22
150
+ },
151
+ {
152
+ "epoch": 9.2,
153
+ "learning_rate": 0.0001898794046299167,
154
+ "loss": 0.9339,
155
+ "step": 23
156
+ },
157
+ {
158
+ "epoch": 9.6,
159
+ "learning_rate": 0.00018829475928589271,
160
+ "loss": 0.9256,
161
+ "step": 24
162
+ },
163
+ {
164
+ "epoch": 10.0,
165
+ "learning_rate": 0.00018660254037844388,
166
+ "loss": 0.9266,
167
+ "step": 25
168
+ },
169
+ {
170
+ "epoch": 10.4,
171
+ "learning_rate": 0.0001848048096156426,
172
+ "loss": 0.9146,
173
+ "step": 26
174
+ },
175
+ {
176
+ "epoch": 10.8,
177
+ "learning_rate": 0.00018290375725550417,
178
+ "loss": 0.9155,
179
+ "step": 27
180
+ },
181
+ {
182
+ "epoch": 11.2,
183
+ "learning_rate": 0.00018090169943749476,
184
+ "loss": 0.8992,
185
+ "step": 28
186
+ },
187
+ {
188
+ "epoch": 11.6,
189
+ "learning_rate": 0.00017880107536067218,
190
+ "loss": 0.8951,
191
+ "step": 29
192
+ },
193
+ {
194
+ "epoch": 12.0,
195
+ "learning_rate": 0.0001766044443118978,
196
+ "loss": 0.8855,
197
+ "step": 30
198
+ },
199
+ {
200
+ "epoch": 12.4,
201
+ "learning_rate": 0.00017431448254773944,
202
+ "loss": 0.8901,
203
+ "step": 31
204
+ },
205
+ {
206
+ "epoch": 12.8,
207
+ "learning_rate": 0.0001719339800338651,
208
+ "loss": 0.8704,
209
+ "step": 32
210
+ },
211
+ {
212
+ "epoch": 13.2,
213
+ "learning_rate": 0.00016946583704589973,
214
+ "loss": 0.869,
215
+ "step": 33
216
+ },
217
+ {
218
+ "epoch": 13.6,
219
+ "learning_rate": 0.00016691306063588583,
220
+ "loss": 0.864,
221
+ "step": 34
222
+ },
223
+ {
224
+ "epoch": 14.0,
225
+ "learning_rate": 0.00016427876096865394,
226
+ "loss": 0.862,
227
+ "step": 35
228
+ },
229
+ {
230
+ "epoch": 14.4,
231
+ "learning_rate": 0.0001615661475325658,
232
+ "loss": 0.8577,
233
+ "step": 36
234
+ },
235
+ {
236
+ "epoch": 14.8,
237
+ "learning_rate": 0.00015877852522924732,
238
+ "loss": 0.8544,
239
+ "step": 37
240
+ },
241
+ {
242
+ "epoch": 15.2,
243
+ "learning_rate": 0.0001559192903470747,
244
+ "loss": 0.8467,
245
+ "step": 38
246
+ },
247
+ {
248
+ "epoch": 15.6,
249
+ "learning_rate": 0.0001529919264233205,
250
+ "loss": 0.8379,
251
+ "step": 39
252
+ },
253
+ {
254
+ "epoch": 16.0,
255
+ "learning_rate": 0.00015000000000000001,
256
+ "loss": 0.8514,
257
+ "step": 40
258
+ },
259
+ {
260
+ "epoch": 16.0,
261
+ "eval_loss": 0.8522964119911194,
262
+ "eval_runtime": 1.9623,
263
+ "eval_samples_per_second": 10.192,
264
+ "eval_steps_per_second": 2.038,
265
+ "step": 40
266
+ },
267
+ {
268
+ "epoch": 16.4,
269
+ "learning_rate": 0.00014694715627858908,
270
+ "loss": 0.8333,
271
+ "step": 41
272
+ },
273
+ {
274
+ "epoch": 16.8,
275
+ "learning_rate": 0.00014383711467890774,
276
+ "loss": 0.8379,
277
+ "step": 42
278
+ },
279
+ {
280
+ "epoch": 17.2,
281
+ "learning_rate": 0.00014067366430758004,
282
+ "loss": 0.8278,
283
+ "step": 43
284
+ },
285
+ {
286
+ "epoch": 17.6,
287
+ "learning_rate": 0.00013746065934159123,
288
+ "loss": 0.8273,
289
+ "step": 44
290
+ },
291
+ {
292
+ "epoch": 18.0,
293
+ "learning_rate": 0.00013420201433256689,
294
+ "loss": 0.8382,
295
+ "step": 45
296
+ },
297
+ {
298
+ "epoch": 18.4,
299
+ "learning_rate": 0.00013090169943749476,
300
+ "loss": 0.8208,
301
+ "step": 46
302
+ },
303
+ {
304
+ "epoch": 18.8,
305
+ "learning_rate": 0.0001275637355816999,
306
+ "loss": 0.8203,
307
+ "step": 47
308
+ },
309
+ {
310
+ "epoch": 19.2,
311
+ "learning_rate": 0.00012419218955996676,
312
+ "loss": 0.8118,
313
+ "step": 48
314
+ },
315
+ {
316
+ "epoch": 19.6,
317
+ "learning_rate": 0.00012079116908177593,
318
+ "loss": 0.8175,
319
+ "step": 49
320
+ },
321
+ {
322
+ "epoch": 20.0,
323
+ "learning_rate": 0.00011736481776669306,
324
+ "loss": 0.8059,
325
+ "step": 50
326
+ },
327
+ {
328
+ "epoch": 20.4,
329
+ "learning_rate": 0.00011391731009600654,
330
+ "loss": 0.8114,
331
+ "step": 51
332
+ },
333
+ {
334
+ "epoch": 20.8,
335
+ "learning_rate": 0.00011045284632676536,
336
+ "loss": 0.8004,
337
+ "step": 52
338
+ },
339
+ {
340
+ "epoch": 21.2,
341
+ "learning_rate": 0.00010697564737441252,
342
+ "loss": 0.8026,
343
+ "step": 53
344
+ },
345
+ {
346
+ "epoch": 21.6,
347
+ "learning_rate": 0.00010348994967025012,
348
+ "loss": 0.7943,
349
+ "step": 54
350
+ },
351
+ {
352
+ "epoch": 22.0,
353
+ "learning_rate": 0.0001,
354
+ "loss": 0.79,
355
+ "step": 55
356
+ },
357
+ {
358
+ "epoch": 22.4,
359
+ "learning_rate": 9.651005032974994e-05,
360
+ "loss": 0.7952,
361
+ "step": 56
362
+ },
363
+ {
364
+ "epoch": 22.8,
365
+ "learning_rate": 9.302435262558747e-05,
366
+ "loss": 0.7898,
367
+ "step": 57
368
+ },
369
+ {
370
+ "epoch": 23.2,
371
+ "learning_rate": 8.954715367323468e-05,
372
+ "loss": 0.7769,
373
+ "step": 58
374
+ },
375
+ {
376
+ "epoch": 23.6,
377
+ "learning_rate": 8.608268990399349e-05,
378
+ "loss": 0.7812,
379
+ "step": 59
380
+ },
381
+ {
382
+ "epoch": 24.0,
383
+ "learning_rate": 8.263518223330697e-05,
384
+ "loss": 0.7774,
385
+ "step": 60
386
+ },
387
+ {
388
+ "epoch": 24.0,
389
+ "eval_loss": 0.8498085141181946,
390
+ "eval_runtime": 1.9653,
391
+ "eval_samples_per_second": 10.177,
392
+ "eval_steps_per_second": 2.035,
393
+ "step": 60
394
+ },
395
+ {
396
+ "epoch": 24.4,
397
+ "learning_rate": 7.920883091822408e-05,
398
+ "loss": 0.7651,
399
+ "step": 61
400
+ },
401
+ {
402
+ "epoch": 24.8,
403
+ "learning_rate": 7.580781044003324e-05,
404
+ "loss": 0.778,
405
+ "step": 62
406
+ },
407
+ {
408
+ "epoch": 25.2,
409
+ "learning_rate": 7.243626441830009e-05,
410
+ "loss": 0.772,
411
+ "step": 63
412
+ },
413
+ {
414
+ "epoch": 25.6,
415
+ "learning_rate": 6.909830056250527e-05,
416
+ "loss": 0.767,
417
+ "step": 64
418
+ },
419
+ {
420
+ "epoch": 26.0,
421
+ "learning_rate": 6.579798566743314e-05,
422
+ "loss": 0.7569,
423
+ "step": 65
424
+ },
425
+ {
426
+ "epoch": 26.4,
427
+ "learning_rate": 6.25393406584088e-05,
428
+ "loss": 0.7554,
429
+ "step": 66
430
+ },
431
+ {
432
+ "epoch": 26.8,
433
+ "learning_rate": 5.9326335692419995e-05,
434
+ "loss": 0.76,
435
+ "step": 67
436
+ },
437
+ {
438
+ "epoch": 27.2,
439
+ "learning_rate": 5.616288532109225e-05,
440
+ "loss": 0.7554,
441
+ "step": 68
442
+ },
443
+ {
444
+ "epoch": 27.6,
445
+ "learning_rate": 5.305284372141095e-05,
446
+ "loss": 0.748,
447
+ "step": 69
448
+ },
449
+ {
450
+ "epoch": 28.0,
451
+ "learning_rate": 5.000000000000002e-05,
452
+ "loss": 0.7527,
453
+ "step": 70
454
+ },
455
+ {
456
+ "epoch": 28.4,
457
+ "learning_rate": 4.700807357667952e-05,
458
+ "loss": 0.7595,
459
+ "step": 71
460
+ },
461
+ {
462
+ "epoch": 28.8,
463
+ "learning_rate": 4.4080709652925336e-05,
464
+ "loss": 0.7394,
465
+ "step": 72
466
+ },
467
+ {
468
+ "epoch": 29.2,
469
+ "learning_rate": 4.12214747707527e-05,
470
+ "loss": 0.7348,
471
+ "step": 73
472
+ },
473
+ {
474
+ "epoch": 29.6,
475
+ "learning_rate": 3.843385246743417e-05,
476
+ "loss": 0.7497,
477
+ "step": 74
478
+ },
479
+ {
480
+ "epoch": 30.0,
481
+ "learning_rate": 3.5721239031346066e-05,
482
+ "loss": 0.7305,
483
+ "step": 75
484
+ },
485
+ {
486
+ "epoch": 30.4,
487
+ "learning_rate": 3.308693936411421e-05,
488
+ "loss": 0.7317,
489
+ "step": 76
490
+ },
491
+ {
492
+ "epoch": 30.8,
493
+ "learning_rate": 3.053416295410026e-05,
494
+ "loss": 0.7329,
495
+ "step": 77
496
+ },
497
+ {
498
+ "epoch": 31.2,
499
+ "learning_rate": 2.8066019966134904e-05,
500
+ "loss": 0.7494,
501
+ "step": 78
502
+ },
503
+ {
504
+ "epoch": 31.6,
505
+ "learning_rate": 2.5685517452260567e-05,
506
+ "loss": 0.7306,
507
+ "step": 79
508
+ },
509
+ {
510
+ "epoch": 32.0,
511
+ "learning_rate": 2.339555568810221e-05,
512
+ "loss": 0.7178,
513
+ "step": 80
514
+ },
515
+ {
516
+ "epoch": 32.0,
517
+ "eval_loss": 0.8597297072410583,
518
+ "eval_runtime": 1.9653,
519
+ "eval_samples_per_second": 10.177,
520
+ "eval_steps_per_second": 2.035,
521
+ "step": 80
522
+ },
523
+ {
524
+ "epoch": 32.4,
525
+ "learning_rate": 2.119892463932781e-05,
526
+ "loss": 0.7243,
527
+ "step": 81
528
+ },
529
+ {
530
+ "epoch": 32.8,
531
+ "learning_rate": 1.9098300562505266e-05,
532
+ "loss": 0.7393,
533
+ "step": 82
534
+ },
535
+ {
536
+ "epoch": 33.2,
537
+ "learning_rate": 1.7096242744495837e-05,
538
+ "loss": 0.7148,
539
+ "step": 83
540
+ },
541
+ {
542
+ "epoch": 33.6,
543
+ "learning_rate": 1.5195190384357404e-05,
544
+ "loss": 0.7261,
545
+ "step": 84
546
+ },
547
+ {
548
+ "epoch": 34.0,
549
+ "learning_rate": 1.339745962155613e-05,
550
+ "loss": 0.7228,
551
+ "step": 85
552
+ },
553
+ {
554
+ "epoch": 34.4,
555
+ "learning_rate": 1.1705240714107302e-05,
556
+ "loss": 0.7194,
557
+ "step": 86
558
+ },
559
+ {
560
+ "epoch": 34.8,
561
+ "learning_rate": 1.0120595370083318e-05,
562
+ "loss": 0.7252,
563
+ "step": 87
564
+ },
565
+ {
566
+ "epoch": 35.2,
567
+ "learning_rate": 8.645454235739903e-06,
568
+ "loss": 0.7188,
569
+ "step": 88
570
+ },
571
+ {
572
+ "epoch": 35.6,
573
+ "learning_rate": 7.281614543321269e-06,
574
+ "loss": 0.7199,
575
+ "step": 89
576
+ },
577
+ {
578
+ "epoch": 36.0,
579
+ "learning_rate": 6.030737921409169e-06,
580
+ "loss": 0.7239,
581
+ "step": 90
582
+ },
583
+ {
584
+ "epoch": 36.4,
585
+ "learning_rate": 4.8943483704846475e-06,
586
+ "loss": 0.7099,
587
+ "step": 91
588
+ },
589
+ {
590
+ "epoch": 36.8,
591
+ "learning_rate": 3.873830406168111e-06,
592
+ "loss": 0.7265,
593
+ "step": 92
594
+ }
595
+ ],
596
+ "logging_steps": 1,
597
+ "max_steps": 100,
598
+ "num_train_epochs": 50,
599
+ "save_steps": 500,
600
+ "total_flos": 7.297007876466278e+17,
601
+ "trial_name": null,
602
+ "trial_params": null
603
+ }
checkpoint-92/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b87ab0a71d9571469037bd31ca464f2d5cba068cb664754aff5b20bbe14bd6a1
3
+ size 4411
checkpoint-95/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: True
10
+ - load_in_4bit: False
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: fp4
16
+ - bnb_4bit_use_double_quant: False
17
+ - bnb_4bit_compute_dtype: float32
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
checkpoint-95/adapter_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "NousResearch/Llama-2-7b-hf",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 16,
12
+ "lora_dropout": 0.05,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 32,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "q_proj",
20
+ "down_proj",
21
+ "v_proj",
22
+ "o_proj",
23
+ "up_proj",
24
+ "gate_proj",
25
+ "k_proj"
26
+ ],
27
+ "task_type": "CAUSAL_LM"
28
+ }
checkpoint-95/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8da4d89b8d7a70b860125af6e9b3f5cdb45f74406a46966a79e6db739d089e96
3
+ size 319977229
checkpoint-95/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da68e6fc5323f68a31f5e886d04e56109e3c3fb8f2db797b63aca2141f37cdf7
3
+ size 160735647
checkpoint-95/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ccad5d51fd92c1c7045bbacf237c71c7e440dcf2cd0bac81d80bc83593dbc8ca
3
+ size 14575
checkpoint-95/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecc609af1e17db8f434bbef1b78f24f26d1d6f3bbd2c1091254245422218dc8f
3
+ size 627
checkpoint-95/trainer_state.json ADDED
@@ -0,0 +1,621 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 38.0,
5
+ "eval_steps": 20,
6
+ "global_step": 95,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.4,
13
+ "learning_rate": 2e-05,
14
+ "loss": 1.2516,
15
+ "step": 1
16
+ },
17
+ {
18
+ "epoch": 0.8,
19
+ "learning_rate": 4e-05,
20
+ "loss": 1.2581,
21
+ "step": 2
22
+ },
23
+ {
24
+ "epoch": 1.2,
25
+ "learning_rate": 6e-05,
26
+ "loss": 1.2684,
27
+ "step": 3
28
+ },
29
+ {
30
+ "epoch": 1.6,
31
+ "learning_rate": 8e-05,
32
+ "loss": 1.2529,
33
+ "step": 4
34
+ },
35
+ {
36
+ "epoch": 2.0,
37
+ "learning_rate": 0.0001,
38
+ "loss": 1.2441,
39
+ "step": 5
40
+ },
41
+ {
42
+ "epoch": 2.4,
43
+ "learning_rate": 0.00012,
44
+ "loss": 1.2166,
45
+ "step": 6
46
+ },
47
+ {
48
+ "epoch": 2.8,
49
+ "learning_rate": 0.00014,
50
+ "loss": 1.225,
51
+ "step": 7
52
+ },
53
+ {
54
+ "epoch": 3.2,
55
+ "learning_rate": 0.00016,
56
+ "loss": 1.1785,
57
+ "step": 8
58
+ },
59
+ {
60
+ "epoch": 3.6,
61
+ "learning_rate": 0.00018,
62
+ "loss": 1.1504,
63
+ "step": 9
64
+ },
65
+ {
66
+ "epoch": 4.0,
67
+ "learning_rate": 0.0002,
68
+ "loss": 1.1148,
69
+ "step": 10
70
+ },
71
+ {
72
+ "epoch": 4.4,
73
+ "learning_rate": 0.0001999390827019096,
74
+ "loss": 1.0771,
75
+ "step": 11
76
+ },
77
+ {
78
+ "epoch": 4.8,
79
+ "learning_rate": 0.00019975640502598244,
80
+ "loss": 1.0768,
81
+ "step": 12
82
+ },
83
+ {
84
+ "epoch": 5.2,
85
+ "learning_rate": 0.00019945218953682734,
86
+ "loss": 1.0422,
87
+ "step": 13
88
+ },
89
+ {
90
+ "epoch": 5.6,
91
+ "learning_rate": 0.00019902680687415705,
92
+ "loss": 1.0305,
93
+ "step": 14
94
+ },
95
+ {
96
+ "epoch": 6.0,
97
+ "learning_rate": 0.00019848077530122083,
98
+ "loss": 1.0148,
99
+ "step": 15
100
+ },
101
+ {
102
+ "epoch": 6.4,
103
+ "learning_rate": 0.00019781476007338058,
104
+ "loss": 1.0024,
105
+ "step": 16
106
+ },
107
+ {
108
+ "epoch": 6.8,
109
+ "learning_rate": 0.00019702957262759965,
110
+ "loss": 0.9914,
111
+ "step": 17
112
+ },
113
+ {
114
+ "epoch": 7.2,
115
+ "learning_rate": 0.0001961261695938319,
116
+ "loss": 0.9746,
117
+ "step": 18
118
+ },
119
+ {
120
+ "epoch": 7.6,
121
+ "learning_rate": 0.00019510565162951537,
122
+ "loss": 0.9764,
123
+ "step": 19
124
+ },
125
+ {
126
+ "epoch": 8.0,
127
+ "learning_rate": 0.00019396926207859084,
128
+ "loss": 0.9548,
129
+ "step": 20
130
+ },
131
+ {
132
+ "epoch": 8.0,
133
+ "eval_loss": 0.9240104556083679,
134
+ "eval_runtime": 1.9642,
135
+ "eval_samples_per_second": 10.182,
136
+ "eval_steps_per_second": 2.036,
137
+ "step": 20
138
+ },
139
+ {
140
+ "epoch": 8.4,
141
+ "learning_rate": 0.00019271838545667876,
142
+ "loss": 0.9726,
143
+ "step": 21
144
+ },
145
+ {
146
+ "epoch": 8.8,
147
+ "learning_rate": 0.0001913545457642601,
148
+ "loss": 0.9381,
149
+ "step": 22
150
+ },
151
+ {
152
+ "epoch": 9.2,
153
+ "learning_rate": 0.0001898794046299167,
154
+ "loss": 0.9339,
155
+ "step": 23
156
+ },
157
+ {
158
+ "epoch": 9.6,
159
+ "learning_rate": 0.00018829475928589271,
160
+ "loss": 0.9256,
161
+ "step": 24
162
+ },
163
+ {
164
+ "epoch": 10.0,
165
+ "learning_rate": 0.00018660254037844388,
166
+ "loss": 0.9266,
167
+ "step": 25
168
+ },
169
+ {
170
+ "epoch": 10.4,
171
+ "learning_rate": 0.0001848048096156426,
172
+ "loss": 0.9146,
173
+ "step": 26
174
+ },
175
+ {
176
+ "epoch": 10.8,
177
+ "learning_rate": 0.00018290375725550417,
178
+ "loss": 0.9155,
179
+ "step": 27
180
+ },
181
+ {
182
+ "epoch": 11.2,
183
+ "learning_rate": 0.00018090169943749476,
184
+ "loss": 0.8992,
185
+ "step": 28
186
+ },
187
+ {
188
+ "epoch": 11.6,
189
+ "learning_rate": 0.00017880107536067218,
190
+ "loss": 0.8951,
191
+ "step": 29
192
+ },
193
+ {
194
+ "epoch": 12.0,
195
+ "learning_rate": 0.0001766044443118978,
196
+ "loss": 0.8855,
197
+ "step": 30
198
+ },
199
+ {
200
+ "epoch": 12.4,
201
+ "learning_rate": 0.00017431448254773944,
202
+ "loss": 0.8901,
203
+ "step": 31
204
+ },
205
+ {
206
+ "epoch": 12.8,
207
+ "learning_rate": 0.0001719339800338651,
208
+ "loss": 0.8704,
209
+ "step": 32
210
+ },
211
+ {
212
+ "epoch": 13.2,
213
+ "learning_rate": 0.00016946583704589973,
214
+ "loss": 0.869,
215
+ "step": 33
216
+ },
217
+ {
218
+ "epoch": 13.6,
219
+ "learning_rate": 0.00016691306063588583,
220
+ "loss": 0.864,
221
+ "step": 34
222
+ },
223
+ {
224
+ "epoch": 14.0,
225
+ "learning_rate": 0.00016427876096865394,
226
+ "loss": 0.862,
227
+ "step": 35
228
+ },
229
+ {
230
+ "epoch": 14.4,
231
+ "learning_rate": 0.0001615661475325658,
232
+ "loss": 0.8577,
233
+ "step": 36
234
+ },
235
+ {
236
+ "epoch": 14.8,
237
+ "learning_rate": 0.00015877852522924732,
238
+ "loss": 0.8544,
239
+ "step": 37
240
+ },
241
+ {
242
+ "epoch": 15.2,
243
+ "learning_rate": 0.0001559192903470747,
244
+ "loss": 0.8467,
245
+ "step": 38
246
+ },
247
+ {
248
+ "epoch": 15.6,
249
+ "learning_rate": 0.0001529919264233205,
250
+ "loss": 0.8379,
251
+ "step": 39
252
+ },
253
+ {
254
+ "epoch": 16.0,
255
+ "learning_rate": 0.00015000000000000001,
256
+ "loss": 0.8514,
257
+ "step": 40
258
+ },
259
+ {
260
+ "epoch": 16.0,
261
+ "eval_loss": 0.8522964119911194,
262
+ "eval_runtime": 1.9623,
263
+ "eval_samples_per_second": 10.192,
264
+ "eval_steps_per_second": 2.038,
265
+ "step": 40
266
+ },
267
+ {
268
+ "epoch": 16.4,
269
+ "learning_rate": 0.00014694715627858908,
270
+ "loss": 0.8333,
271
+ "step": 41
272
+ },
273
+ {
274
+ "epoch": 16.8,
275
+ "learning_rate": 0.00014383711467890774,
276
+ "loss": 0.8379,
277
+ "step": 42
278
+ },
279
+ {
280
+ "epoch": 17.2,
281
+ "learning_rate": 0.00014067366430758004,
282
+ "loss": 0.8278,
283
+ "step": 43
284
+ },
285
+ {
286
+ "epoch": 17.6,
287
+ "learning_rate": 0.00013746065934159123,
288
+ "loss": 0.8273,
289
+ "step": 44
290
+ },
291
+ {
292
+ "epoch": 18.0,
293
+ "learning_rate": 0.00013420201433256689,
294
+ "loss": 0.8382,
295
+ "step": 45
296
+ },
297
+ {
298
+ "epoch": 18.4,
299
+ "learning_rate": 0.00013090169943749476,
300
+ "loss": 0.8208,
301
+ "step": 46
302
+ },
303
+ {
304
+ "epoch": 18.8,
305
+ "learning_rate": 0.0001275637355816999,
306
+ "loss": 0.8203,
307
+ "step": 47
308
+ },
309
+ {
310
+ "epoch": 19.2,
311
+ "learning_rate": 0.00012419218955996676,
312
+ "loss": 0.8118,
313
+ "step": 48
314
+ },
315
+ {
316
+ "epoch": 19.6,
317
+ "learning_rate": 0.00012079116908177593,
318
+ "loss": 0.8175,
319
+ "step": 49
320
+ },
321
+ {
322
+ "epoch": 20.0,
323
+ "learning_rate": 0.00011736481776669306,
324
+ "loss": 0.8059,
325
+ "step": 50
326
+ },
327
+ {
328
+ "epoch": 20.4,
329
+ "learning_rate": 0.00011391731009600654,
330
+ "loss": 0.8114,
331
+ "step": 51
332
+ },
333
+ {
334
+ "epoch": 20.8,
335
+ "learning_rate": 0.00011045284632676536,
336
+ "loss": 0.8004,
337
+ "step": 52
338
+ },
339
+ {
340
+ "epoch": 21.2,
341
+ "learning_rate": 0.00010697564737441252,
342
+ "loss": 0.8026,
343
+ "step": 53
344
+ },
345
+ {
346
+ "epoch": 21.6,
347
+ "learning_rate": 0.00010348994967025012,
348
+ "loss": 0.7943,
349
+ "step": 54
350
+ },
351
+ {
352
+ "epoch": 22.0,
353
+ "learning_rate": 0.0001,
354
+ "loss": 0.79,
355
+ "step": 55
356
+ },
357
+ {
358
+ "epoch": 22.4,
359
+ "learning_rate": 9.651005032974994e-05,
360
+ "loss": 0.7952,
361
+ "step": 56
362
+ },
363
+ {
364
+ "epoch": 22.8,
365
+ "learning_rate": 9.302435262558747e-05,
366
+ "loss": 0.7898,
367
+ "step": 57
368
+ },
369
+ {
370
+ "epoch": 23.2,
371
+ "learning_rate": 8.954715367323468e-05,
372
+ "loss": 0.7769,
373
+ "step": 58
374
+ },
375
+ {
376
+ "epoch": 23.6,
377
+ "learning_rate": 8.608268990399349e-05,
378
+ "loss": 0.7812,
379
+ "step": 59
380
+ },
381
+ {
382
+ "epoch": 24.0,
383
+ "learning_rate": 8.263518223330697e-05,
384
+ "loss": 0.7774,
385
+ "step": 60
386
+ },
387
+ {
388
+ "epoch": 24.0,
389
+ "eval_loss": 0.8498085141181946,
390
+ "eval_runtime": 1.9653,
391
+ "eval_samples_per_second": 10.177,
392
+ "eval_steps_per_second": 2.035,
393
+ "step": 60
394
+ },
395
+ {
396
+ "epoch": 24.4,
397
+ "learning_rate": 7.920883091822408e-05,
398
+ "loss": 0.7651,
399
+ "step": 61
400
+ },
401
+ {
402
+ "epoch": 24.8,
403
+ "learning_rate": 7.580781044003324e-05,
404
+ "loss": 0.778,
405
+ "step": 62
406
+ },
407
+ {
408
+ "epoch": 25.2,
409
+ "learning_rate": 7.243626441830009e-05,
410
+ "loss": 0.772,
411
+ "step": 63
412
+ },
413
+ {
414
+ "epoch": 25.6,
415
+ "learning_rate": 6.909830056250527e-05,
416
+ "loss": 0.767,
417
+ "step": 64
418
+ },
419
+ {
420
+ "epoch": 26.0,
421
+ "learning_rate": 6.579798566743314e-05,
422
+ "loss": 0.7569,
423
+ "step": 65
424
+ },
425
+ {
426
+ "epoch": 26.4,
427
+ "learning_rate": 6.25393406584088e-05,
428
+ "loss": 0.7554,
429
+ "step": 66
430
+ },
431
+ {
432
+ "epoch": 26.8,
433
+ "learning_rate": 5.9326335692419995e-05,
434
+ "loss": 0.76,
435
+ "step": 67
436
+ },
437
+ {
438
+ "epoch": 27.2,
439
+ "learning_rate": 5.616288532109225e-05,
440
+ "loss": 0.7554,
441
+ "step": 68
442
+ },
443
+ {
444
+ "epoch": 27.6,
445
+ "learning_rate": 5.305284372141095e-05,
446
+ "loss": 0.748,
447
+ "step": 69
448
+ },
449
+ {
450
+ "epoch": 28.0,
451
+ "learning_rate": 5.000000000000002e-05,
452
+ "loss": 0.7527,
453
+ "step": 70
454
+ },
455
+ {
456
+ "epoch": 28.4,
457
+ "learning_rate": 4.700807357667952e-05,
458
+ "loss": 0.7595,
459
+ "step": 71
460
+ },
461
+ {
462
+ "epoch": 28.8,
463
+ "learning_rate": 4.4080709652925336e-05,
464
+ "loss": 0.7394,
465
+ "step": 72
466
+ },
467
+ {
468
+ "epoch": 29.2,
469
+ "learning_rate": 4.12214747707527e-05,
470
+ "loss": 0.7348,
471
+ "step": 73
472
+ },
473
+ {
474
+ "epoch": 29.6,
475
+ "learning_rate": 3.843385246743417e-05,
476
+ "loss": 0.7497,
477
+ "step": 74
478
+ },
479
+ {
480
+ "epoch": 30.0,
481
+ "learning_rate": 3.5721239031346066e-05,
482
+ "loss": 0.7305,
483
+ "step": 75
484
+ },
485
+ {
486
+ "epoch": 30.4,
487
+ "learning_rate": 3.308693936411421e-05,
488
+ "loss": 0.7317,
489
+ "step": 76
490
+ },
491
+ {
492
+ "epoch": 30.8,
493
+ "learning_rate": 3.053416295410026e-05,
494
+ "loss": 0.7329,
495
+ "step": 77
496
+ },
497
+ {
498
+ "epoch": 31.2,
499
+ "learning_rate": 2.8066019966134904e-05,
500
+ "loss": 0.7494,
501
+ "step": 78
502
+ },
503
+ {
504
+ "epoch": 31.6,
505
+ "learning_rate": 2.5685517452260567e-05,
506
+ "loss": 0.7306,
507
+ "step": 79
508
+ },
509
+ {
510
+ "epoch": 32.0,
511
+ "learning_rate": 2.339555568810221e-05,
512
+ "loss": 0.7178,
513
+ "step": 80
514
+ },
515
+ {
516
+ "epoch": 32.0,
517
+ "eval_loss": 0.8597297072410583,
518
+ "eval_runtime": 1.9653,
519
+ "eval_samples_per_second": 10.177,
520
+ "eval_steps_per_second": 2.035,
521
+ "step": 80
522
+ },
523
+ {
524
+ "epoch": 32.4,
525
+ "learning_rate": 2.119892463932781e-05,
526
+ "loss": 0.7243,
527
+ "step": 81
528
+ },
529
+ {
530
+ "epoch": 32.8,
531
+ "learning_rate": 1.9098300562505266e-05,
532
+ "loss": 0.7393,
533
+ "step": 82
534
+ },
535
+ {
536
+ "epoch": 33.2,
537
+ "learning_rate": 1.7096242744495837e-05,
538
+ "loss": 0.7148,
539
+ "step": 83
540
+ },
541
+ {
542
+ "epoch": 33.6,
543
+ "learning_rate": 1.5195190384357404e-05,
544
+ "loss": 0.7261,
545
+ "step": 84
546
+ },
547
+ {
548
+ "epoch": 34.0,
549
+ "learning_rate": 1.339745962155613e-05,
550
+ "loss": 0.7228,
551
+ "step": 85
552
+ },
553
+ {
554
+ "epoch": 34.4,
555
+ "learning_rate": 1.1705240714107302e-05,
556
+ "loss": 0.7194,
557
+ "step": 86
558
+ },
559
+ {
560
+ "epoch": 34.8,
561
+ "learning_rate": 1.0120595370083318e-05,
562
+ "loss": 0.7252,
563
+ "step": 87
564
+ },
565
+ {
566
+ "epoch": 35.2,
567
+ "learning_rate": 8.645454235739903e-06,
568
+ "loss": 0.7188,
569
+ "step": 88
570
+ },
571
+ {
572
+ "epoch": 35.6,
573
+ "learning_rate": 7.281614543321269e-06,
574
+ "loss": 0.7199,
575
+ "step": 89
576
+ },
577
+ {
578
+ "epoch": 36.0,
579
+ "learning_rate": 6.030737921409169e-06,
580
+ "loss": 0.7239,
581
+ "step": 90
582
+ },
583
+ {
584
+ "epoch": 36.4,
585
+ "learning_rate": 4.8943483704846475e-06,
586
+ "loss": 0.7099,
587
+ "step": 91
588
+ },
589
+ {
590
+ "epoch": 36.8,
591
+ "learning_rate": 3.873830406168111e-06,
592
+ "loss": 0.7265,
593
+ "step": 92
594
+ },
595
+ {
596
+ "epoch": 37.2,
597
+ "learning_rate": 2.970427372400353e-06,
598
+ "loss": 0.7105,
599
+ "step": 93
600
+ },
601
+ {
602
+ "epoch": 37.6,
603
+ "learning_rate": 2.1852399266194314e-06,
604
+ "loss": 0.7149,
605
+ "step": 94
606
+ },
607
+ {
608
+ "epoch": 38.0,
609
+ "learning_rate": 1.5192246987791981e-06,
610
+ "loss": 0.7257,
611
+ "step": 95
612
+ }
613
+ ],
614
+ "logging_steps": 1,
615
+ "max_steps": 100,
616
+ "num_train_epochs": 50,
617
+ "save_steps": 500,
618
+ "total_flos": 7.494224305559962e+17,
619
+ "trial_name": null,
620
+ "trial_params": null
621
+ }
checkpoint-95/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b87ab0a71d9571469037bd31ca464f2d5cba068cb664754aff5b20bbe14bd6a1
3
+ size 4411
checkpoint-97/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: True
10
+ - load_in_4bit: False
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: fp4
16
+ - bnb_4bit_use_double_quant: False
17
+ - bnb_4bit_compute_dtype: float32
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
checkpoint-97/adapter_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "NousResearch/Llama-2-7b-hf",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 16,
12
+ "lora_dropout": 0.05,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 32,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "q_proj",
20
+ "down_proj",
21
+ "v_proj",
22
+ "o_proj",
23
+ "up_proj",
24
+ "gate_proj",
25
+ "k_proj"
26
+ ],
27
+ "task_type": "CAUSAL_LM"
28
+ }
checkpoint-97/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9f3a0447bbc7effc1a6d473a8f3cc7952b8aff7096813f6e2a865ed47a28963
3
+ size 319977229
checkpoint-97/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac813ee2eb13c022de940107ce11522c71fea6f282b28cb6f5069dca41f38378
3
+ size 160735647
checkpoint-97/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8eb19eb2f51e7aeb2f68e51ef8bb9aadf26070b63f2b8a10cc2fc21a36ca76f9
3
+ size 14575
checkpoint-97/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9cbbd944e5dbe7515f1753c2f4c92420b68cedadb9aa655c6c65ace7cc2afe23
3
+ size 627
checkpoint-97/trainer_state.json ADDED
@@ -0,0 +1,633 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 38.8,
5
+ "eval_steps": 20,
6
+ "global_step": 97,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.4,
13
+ "learning_rate": 2e-05,
14
+ "loss": 1.2516,
15
+ "step": 1
16
+ },
17
+ {
18
+ "epoch": 0.8,
19
+ "learning_rate": 4e-05,
20
+ "loss": 1.2581,
21
+ "step": 2
22
+ },
23
+ {
24
+ "epoch": 1.2,
25
+ "learning_rate": 6e-05,
26
+ "loss": 1.2684,
27
+ "step": 3
28
+ },
29
+ {
30
+ "epoch": 1.6,
31
+ "learning_rate": 8e-05,
32
+ "loss": 1.2529,
33
+ "step": 4
34
+ },
35
+ {
36
+ "epoch": 2.0,
37
+ "learning_rate": 0.0001,
38
+ "loss": 1.2441,
39
+ "step": 5
40
+ },
41
+ {
42
+ "epoch": 2.4,
43
+ "learning_rate": 0.00012,
44
+ "loss": 1.2166,
45
+ "step": 6
46
+ },
47
+ {
48
+ "epoch": 2.8,
49
+ "learning_rate": 0.00014,
50
+ "loss": 1.225,
51
+ "step": 7
52
+ },
53
+ {
54
+ "epoch": 3.2,
55
+ "learning_rate": 0.00016,
56
+ "loss": 1.1785,
57
+ "step": 8
58
+ },
59
+ {
60
+ "epoch": 3.6,
61
+ "learning_rate": 0.00018,
62
+ "loss": 1.1504,
63
+ "step": 9
64
+ },
65
+ {
66
+ "epoch": 4.0,
67
+ "learning_rate": 0.0002,
68
+ "loss": 1.1148,
69
+ "step": 10
70
+ },
71
+ {
72
+ "epoch": 4.4,
73
+ "learning_rate": 0.0001999390827019096,
74
+ "loss": 1.0771,
75
+ "step": 11
76
+ },
77
+ {
78
+ "epoch": 4.8,
79
+ "learning_rate": 0.00019975640502598244,
80
+ "loss": 1.0768,
81
+ "step": 12
82
+ },
83
+ {
84
+ "epoch": 5.2,
85
+ "learning_rate": 0.00019945218953682734,
86
+ "loss": 1.0422,
87
+ "step": 13
88
+ },
89
+ {
90
+ "epoch": 5.6,
91
+ "learning_rate": 0.00019902680687415705,
92
+ "loss": 1.0305,
93
+ "step": 14
94
+ },
95
+ {
96
+ "epoch": 6.0,
97
+ "learning_rate": 0.00019848077530122083,
98
+ "loss": 1.0148,
99
+ "step": 15
100
+ },
101
+ {
102
+ "epoch": 6.4,
103
+ "learning_rate": 0.00019781476007338058,
104
+ "loss": 1.0024,
105
+ "step": 16
106
+ },
107
+ {
108
+ "epoch": 6.8,
109
+ "learning_rate": 0.00019702957262759965,
110
+ "loss": 0.9914,
111
+ "step": 17
112
+ },
113
+ {
114
+ "epoch": 7.2,
115
+ "learning_rate": 0.0001961261695938319,
116
+ "loss": 0.9746,
117
+ "step": 18
118
+ },
119
+ {
120
+ "epoch": 7.6,
121
+ "learning_rate": 0.00019510565162951537,
122
+ "loss": 0.9764,
123
+ "step": 19
124
+ },
125
+ {
126
+ "epoch": 8.0,
127
+ "learning_rate": 0.00019396926207859084,
128
+ "loss": 0.9548,
129
+ "step": 20
130
+ },
131
+ {
132
+ "epoch": 8.0,
133
+ "eval_loss": 0.9240104556083679,
134
+ "eval_runtime": 1.9642,
135
+ "eval_samples_per_second": 10.182,
136
+ "eval_steps_per_second": 2.036,
137
+ "step": 20
138
+ },
139
+ {
140
+ "epoch": 8.4,
141
+ "learning_rate": 0.00019271838545667876,
142
+ "loss": 0.9726,
143
+ "step": 21
144
+ },
145
+ {
146
+ "epoch": 8.8,
147
+ "learning_rate": 0.0001913545457642601,
148
+ "loss": 0.9381,
149
+ "step": 22
150
+ },
151
+ {
152
+ "epoch": 9.2,
153
+ "learning_rate": 0.0001898794046299167,
154
+ "loss": 0.9339,
155
+ "step": 23
156
+ },
157
+ {
158
+ "epoch": 9.6,
159
+ "learning_rate": 0.00018829475928589271,
160
+ "loss": 0.9256,
161
+ "step": 24
162
+ },
163
+ {
164
+ "epoch": 10.0,
165
+ "learning_rate": 0.00018660254037844388,
166
+ "loss": 0.9266,
167
+ "step": 25
168
+ },
169
+ {
170
+ "epoch": 10.4,
171
+ "learning_rate": 0.0001848048096156426,
172
+ "loss": 0.9146,
173
+ "step": 26
174
+ },
175
+ {
176
+ "epoch": 10.8,
177
+ "learning_rate": 0.00018290375725550417,
178
+ "loss": 0.9155,
179
+ "step": 27
180
+ },
181
+ {
182
+ "epoch": 11.2,
183
+ "learning_rate": 0.00018090169943749476,
184
+ "loss": 0.8992,
185
+ "step": 28
186
+ },
187
+ {
188
+ "epoch": 11.6,
189
+ "learning_rate": 0.00017880107536067218,
190
+ "loss": 0.8951,
191
+ "step": 29
192
+ },
193
+ {
194
+ "epoch": 12.0,
195
+ "learning_rate": 0.0001766044443118978,
196
+ "loss": 0.8855,
197
+ "step": 30
198
+ },
199
+ {
200
+ "epoch": 12.4,
201
+ "learning_rate": 0.00017431448254773944,
202
+ "loss": 0.8901,
203
+ "step": 31
204
+ },
205
+ {
206
+ "epoch": 12.8,
207
+ "learning_rate": 0.0001719339800338651,
208
+ "loss": 0.8704,
209
+ "step": 32
210
+ },
211
+ {
212
+ "epoch": 13.2,
213
+ "learning_rate": 0.00016946583704589973,
214
+ "loss": 0.869,
215
+ "step": 33
216
+ },
217
+ {
218
+ "epoch": 13.6,
219
+ "learning_rate": 0.00016691306063588583,
220
+ "loss": 0.864,
221
+ "step": 34
222
+ },
223
+ {
224
+ "epoch": 14.0,
225
+ "learning_rate": 0.00016427876096865394,
226
+ "loss": 0.862,
227
+ "step": 35
228
+ },
229
+ {
230
+ "epoch": 14.4,
231
+ "learning_rate": 0.0001615661475325658,
232
+ "loss": 0.8577,
233
+ "step": 36
234
+ },
235
+ {
236
+ "epoch": 14.8,
237
+ "learning_rate": 0.00015877852522924732,
238
+ "loss": 0.8544,
239
+ "step": 37
240
+ },
241
+ {
242
+ "epoch": 15.2,
243
+ "learning_rate": 0.0001559192903470747,
244
+ "loss": 0.8467,
245
+ "step": 38
246
+ },
247
+ {
248
+ "epoch": 15.6,
249
+ "learning_rate": 0.0001529919264233205,
250
+ "loss": 0.8379,
251
+ "step": 39
252
+ },
253
+ {
254
+ "epoch": 16.0,
255
+ "learning_rate": 0.00015000000000000001,
256
+ "loss": 0.8514,
257
+ "step": 40
258
+ },
259
+ {
260
+ "epoch": 16.0,
261
+ "eval_loss": 0.8522964119911194,
262
+ "eval_runtime": 1.9623,
263
+ "eval_samples_per_second": 10.192,
264
+ "eval_steps_per_second": 2.038,
265
+ "step": 40
266
+ },
267
+ {
268
+ "epoch": 16.4,
269
+ "learning_rate": 0.00014694715627858908,
270
+ "loss": 0.8333,
271
+ "step": 41
272
+ },
273
+ {
274
+ "epoch": 16.8,
275
+ "learning_rate": 0.00014383711467890774,
276
+ "loss": 0.8379,
277
+ "step": 42
278
+ },
279
+ {
280
+ "epoch": 17.2,
281
+ "learning_rate": 0.00014067366430758004,
282
+ "loss": 0.8278,
283
+ "step": 43
284
+ },
285
+ {
286
+ "epoch": 17.6,
287
+ "learning_rate": 0.00013746065934159123,
288
+ "loss": 0.8273,
289
+ "step": 44
290
+ },
291
+ {
292
+ "epoch": 18.0,
293
+ "learning_rate": 0.00013420201433256689,
294
+ "loss": 0.8382,
295
+ "step": 45
296
+ },
297
+ {
298
+ "epoch": 18.4,
299
+ "learning_rate": 0.00013090169943749476,
300
+ "loss": 0.8208,
301
+ "step": 46
302
+ },
303
+ {
304
+ "epoch": 18.8,
305
+ "learning_rate": 0.0001275637355816999,
306
+ "loss": 0.8203,
307
+ "step": 47
308
+ },
309
+ {
310
+ "epoch": 19.2,
311
+ "learning_rate": 0.00012419218955996676,
312
+ "loss": 0.8118,
313
+ "step": 48
314
+ },
315
+ {
316
+ "epoch": 19.6,
317
+ "learning_rate": 0.00012079116908177593,
318
+ "loss": 0.8175,
319
+ "step": 49
320
+ },
321
+ {
322
+ "epoch": 20.0,
323
+ "learning_rate": 0.00011736481776669306,
324
+ "loss": 0.8059,
325
+ "step": 50
326
+ },
327
+ {
328
+ "epoch": 20.4,
329
+ "learning_rate": 0.00011391731009600654,
330
+ "loss": 0.8114,
331
+ "step": 51
332
+ },
333
+ {
334
+ "epoch": 20.8,
335
+ "learning_rate": 0.00011045284632676536,
336
+ "loss": 0.8004,
337
+ "step": 52
338
+ },
339
+ {
340
+ "epoch": 21.2,
341
+ "learning_rate": 0.00010697564737441252,
342
+ "loss": 0.8026,
343
+ "step": 53
344
+ },
345
+ {
346
+ "epoch": 21.6,
347
+ "learning_rate": 0.00010348994967025012,
348
+ "loss": 0.7943,
349
+ "step": 54
350
+ },
351
+ {
352
+ "epoch": 22.0,
353
+ "learning_rate": 0.0001,
354
+ "loss": 0.79,
355
+ "step": 55
356
+ },
357
+ {
358
+ "epoch": 22.4,
359
+ "learning_rate": 9.651005032974994e-05,
360
+ "loss": 0.7952,
361
+ "step": 56
362
+ },
363
+ {
364
+ "epoch": 22.8,
365
+ "learning_rate": 9.302435262558747e-05,
366
+ "loss": 0.7898,
367
+ "step": 57
368
+ },
369
+ {
370
+ "epoch": 23.2,
371
+ "learning_rate": 8.954715367323468e-05,
372
+ "loss": 0.7769,
373
+ "step": 58
374
+ },
375
+ {
376
+ "epoch": 23.6,
377
+ "learning_rate": 8.608268990399349e-05,
378
+ "loss": 0.7812,
379
+ "step": 59
380
+ },
381
+ {
382
+ "epoch": 24.0,
383
+ "learning_rate": 8.263518223330697e-05,
384
+ "loss": 0.7774,
385
+ "step": 60
386
+ },
387
+ {
388
+ "epoch": 24.0,
389
+ "eval_loss": 0.8498085141181946,
390
+ "eval_runtime": 1.9653,
391
+ "eval_samples_per_second": 10.177,
392
+ "eval_steps_per_second": 2.035,
393
+ "step": 60
394
+ },
395
+ {
396
+ "epoch": 24.4,
397
+ "learning_rate": 7.920883091822408e-05,
398
+ "loss": 0.7651,
399
+ "step": 61
400
+ },
401
+ {
402
+ "epoch": 24.8,
403
+ "learning_rate": 7.580781044003324e-05,
404
+ "loss": 0.778,
405
+ "step": 62
406
+ },
407
+ {
408
+ "epoch": 25.2,
409
+ "learning_rate": 7.243626441830009e-05,
410
+ "loss": 0.772,
411
+ "step": 63
412
+ },
413
+ {
414
+ "epoch": 25.6,
415
+ "learning_rate": 6.909830056250527e-05,
416
+ "loss": 0.767,
417
+ "step": 64
418
+ },
419
+ {
420
+ "epoch": 26.0,
421
+ "learning_rate": 6.579798566743314e-05,
422
+ "loss": 0.7569,
423
+ "step": 65
424
+ },
425
+ {
426
+ "epoch": 26.4,
427
+ "learning_rate": 6.25393406584088e-05,
428
+ "loss": 0.7554,
429
+ "step": 66
430
+ },
431
+ {
432
+ "epoch": 26.8,
433
+ "learning_rate": 5.9326335692419995e-05,
434
+ "loss": 0.76,
435
+ "step": 67
436
+ },
437
+ {
438
+ "epoch": 27.2,
439
+ "learning_rate": 5.616288532109225e-05,
440
+ "loss": 0.7554,
441
+ "step": 68
442
+ },
443
+ {
444
+ "epoch": 27.6,
445
+ "learning_rate": 5.305284372141095e-05,
446
+ "loss": 0.748,
447
+ "step": 69
448
+ },
449
+ {
450
+ "epoch": 28.0,
451
+ "learning_rate": 5.000000000000002e-05,
452
+ "loss": 0.7527,
453
+ "step": 70
454
+ },
455
+ {
456
+ "epoch": 28.4,
457
+ "learning_rate": 4.700807357667952e-05,
458
+ "loss": 0.7595,
459
+ "step": 71
460
+ },
461
+ {
462
+ "epoch": 28.8,
463
+ "learning_rate": 4.4080709652925336e-05,
464
+ "loss": 0.7394,
465
+ "step": 72
466
+ },
467
+ {
468
+ "epoch": 29.2,
469
+ "learning_rate": 4.12214747707527e-05,
470
+ "loss": 0.7348,
471
+ "step": 73
472
+ },
473
+ {
474
+ "epoch": 29.6,
475
+ "learning_rate": 3.843385246743417e-05,
476
+ "loss": 0.7497,
477
+ "step": 74
478
+ },
479
+ {
480
+ "epoch": 30.0,
481
+ "learning_rate": 3.5721239031346066e-05,
482
+ "loss": 0.7305,
483
+ "step": 75
484
+ },
485
+ {
486
+ "epoch": 30.4,
487
+ "learning_rate": 3.308693936411421e-05,
488
+ "loss": 0.7317,
489
+ "step": 76
490
+ },
491
+ {
492
+ "epoch": 30.8,
493
+ "learning_rate": 3.053416295410026e-05,
494
+ "loss": 0.7329,
495
+ "step": 77
496
+ },
497
+ {
498
+ "epoch": 31.2,
499
+ "learning_rate": 2.8066019966134904e-05,
500
+ "loss": 0.7494,
501
+ "step": 78
502
+ },
503
+ {
504
+ "epoch": 31.6,
505
+ "learning_rate": 2.5685517452260567e-05,
506
+ "loss": 0.7306,
507
+ "step": 79
508
+ },
509
+ {
510
+ "epoch": 32.0,
511
+ "learning_rate": 2.339555568810221e-05,
512
+ "loss": 0.7178,
513
+ "step": 80
514
+ },
515
+ {
516
+ "epoch": 32.0,
517
+ "eval_loss": 0.8597297072410583,
518
+ "eval_runtime": 1.9653,
519
+ "eval_samples_per_second": 10.177,
520
+ "eval_steps_per_second": 2.035,
521
+ "step": 80
522
+ },
523
+ {
524
+ "epoch": 32.4,
525
+ "learning_rate": 2.119892463932781e-05,
526
+ "loss": 0.7243,
527
+ "step": 81
528
+ },
529
+ {
530
+ "epoch": 32.8,
531
+ "learning_rate": 1.9098300562505266e-05,
532
+ "loss": 0.7393,
533
+ "step": 82
534
+ },
535
+ {
536
+ "epoch": 33.2,
537
+ "learning_rate": 1.7096242744495837e-05,
538
+ "loss": 0.7148,
539
+ "step": 83
540
+ },
541
+ {
542
+ "epoch": 33.6,
543
+ "learning_rate": 1.5195190384357404e-05,
544
+ "loss": 0.7261,
545
+ "step": 84
546
+ },
547
+ {
548
+ "epoch": 34.0,
549
+ "learning_rate": 1.339745962155613e-05,
550
+ "loss": 0.7228,
551
+ "step": 85
552
+ },
553
+ {
554
+ "epoch": 34.4,
555
+ "learning_rate": 1.1705240714107302e-05,
556
+ "loss": 0.7194,
557
+ "step": 86
558
+ },
559
+ {
560
+ "epoch": 34.8,
561
+ "learning_rate": 1.0120595370083318e-05,
562
+ "loss": 0.7252,
563
+ "step": 87
564
+ },
565
+ {
566
+ "epoch": 35.2,
567
+ "learning_rate": 8.645454235739903e-06,
568
+ "loss": 0.7188,
569
+ "step": 88
570
+ },
571
+ {
572
+ "epoch": 35.6,
573
+ "learning_rate": 7.281614543321269e-06,
574
+ "loss": 0.7199,
575
+ "step": 89
576
+ },
577
+ {
578
+ "epoch": 36.0,
579
+ "learning_rate": 6.030737921409169e-06,
580
+ "loss": 0.7239,
581
+ "step": 90
582
+ },
583
+ {
584
+ "epoch": 36.4,
585
+ "learning_rate": 4.8943483704846475e-06,
586
+ "loss": 0.7099,
587
+ "step": 91
588
+ },
589
+ {
590
+ "epoch": 36.8,
591
+ "learning_rate": 3.873830406168111e-06,
592
+ "loss": 0.7265,
593
+ "step": 92
594
+ },
595
+ {
596
+ "epoch": 37.2,
597
+ "learning_rate": 2.970427372400353e-06,
598
+ "loss": 0.7105,
599
+ "step": 93
600
+ },
601
+ {
602
+ "epoch": 37.6,
603
+ "learning_rate": 2.1852399266194314e-06,
604
+ "loss": 0.7149,
605
+ "step": 94
606
+ },
607
+ {
608
+ "epoch": 38.0,
609
+ "learning_rate": 1.5192246987791981e-06,
610
+ "loss": 0.7257,
611
+ "step": 95
612
+ },
613
+ {
614
+ "epoch": 38.4,
615
+ "learning_rate": 9.731931258429638e-07,
616
+ "loss": 0.7233,
617
+ "step": 96
618
+ },
619
+ {
620
+ "epoch": 38.8,
621
+ "learning_rate": 5.478104631726711e-07,
622
+ "loss": 0.7078,
623
+ "step": 97
624
+ }
625
+ ],
626
+ "logging_steps": 1,
627
+ "max_steps": 100,
628
+ "num_train_epochs": 50,
629
+ "save_steps": 500,
630
+ "total_flos": 7.691440734653645e+17,
631
+ "trial_name": null,
632
+ "trial_params": null
633
+ }
checkpoint-97/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b87ab0a71d9571469037bd31ca464f2d5cba068cb664754aff5b20bbe14bd6a1
3
+ size 4411
config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "NousResearch/Llama-2-7b-hf",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 4096,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 11008,
13
+ "max_position_embeddings": 4096,
14
+ "model_type": "llama",
15
+ "num_attention_heads": 32,
16
+ "num_hidden_layers": 32,
17
+ "num_key_value_heads": 32,
18
+ "pad_token_id": 0,
19
+ "pretraining_tp": 1,
20
+ "quantization_config": {
21
+ "bnb_4bit_compute_dtype": "float32",
22
+ "bnb_4bit_quant_type": "fp4",
23
+ "bnb_4bit_use_double_quant": false,
24
+ "llm_int8_enable_fp32_cpu_offload": false,
25
+ "llm_int8_has_fp16_weight": false,
26
+ "llm_int8_skip_modules": null,
27
+ "llm_int8_threshold": 6.0,
28
+ "load_in_4bit": false,
29
+ "load_in_8bit": true,
30
+ "quant_method": "bitsandbytes"
31
+ },
32
+ "rms_norm_eps": 1e-05,
33
+ "rope_scaling": null,
34
+ "rope_theta": 10000.0,
35
+ "tie_word_embeddings": false,
36
+ "torch_dtype": "float16",
37
+ "transformers_version": "4.34.0.dev0",
38
+ "use_cache": false,
39
+ "vocab_size": 32000
40
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "eos_token": "</s>",
4
+ "pad_token": "<unk>",
5
+ "unk_token": "<unk>"
6
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": true,
8
+ "normalized": false,
9
+ "rstrip": true,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": true,
16
+ "normalized": false,
17
+ "rstrip": true,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": true,
24
+ "normalized": false,
25
+ "rstrip": true,
26
+ "single_word": false,
27
+ "special": true
28
+ }
29
+ },
30
+ "additional_special_tokens": [],
31
+ "bos_token": "<s>",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "</s>",
34
+ "legacy": false,
35
+ "model_max_length": 1000000000000000019884624838656,
36
+ "pad_token": "<unk>",
37
+ "sp_model_kwargs": {},
38
+ "spaces_between_special_tokens": false,
39
+ "tokenizer_class": "LlamaTokenizer",
40
+ "tokenizer_file": "/root/.cache/huggingface/hub/models--NousResearch--Llama-2-7b-hf/snapshots/dacdfcde31297e34b19ee0e7532f29586d2c17bc/tokenizer.json",
41
+ "trust_remote_code": false,
42
+ "unk_token": "<unk>",
43
+ "use_default_system_prompt": true,
44
+ "use_fast": true
45
+ }