Commit
•
51b439d
1
Parent(s):
f86dd0b
Training in progress, epoch 1
Browse files- README.md +36 -12
- config.json +0 -12
- generation_config.json +7 -0
- pytorch_model.bin +0 -0
- training_args.bin +2 -2
README.md
CHANGED
@@ -1,10 +1,9 @@
|
|
1 |
---
|
2 |
-
|
3 |
-
base_model: cyberagent/calm2-7b-chat
|
4 |
tags:
|
5 |
- generated_from_trainer
|
6 |
model-index:
|
7 |
-
- name:
|
8 |
results: []
|
9 |
---
|
10 |
|
@@ -12,9 +11,11 @@ model-index:
|
|
12 |
should probably proofread and complete it, then remove this comment. -->
|
13 |
|
14 |
[<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
|
15 |
-
#
|
16 |
|
17 |
-
This model is a fine-tuned version of [
|
|
|
|
|
18 |
|
19 |
## Model description
|
20 |
|
@@ -34,18 +35,41 @@ More information needed
|
|
34 |
|
35 |
The following hyperparameters were used during training:
|
36 |
- learning_rate: 0.0002
|
37 |
-
- train_batch_size:
|
38 |
-
- eval_batch_size:
|
39 |
- seed: 42
|
40 |
- distributed_type: multi-GPU
|
41 |
- num_devices: 3
|
42 |
-
-
|
43 |
-
-
|
44 |
-
- total_eval_batch_size: 96
|
45 |
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
|
46 |
- lr_scheduler_type: cosine
|
47 |
-
- lr_scheduler_warmup_steps:
|
48 |
-
- num_epochs:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
49 |
|
50 |
### Framework versions
|
51 |
|
|
|
1 |
---
|
2 |
+
base_model: NousResearch/Llama-2-7b-hf
|
|
|
3 |
tags:
|
4 |
- generated_from_trainer
|
5 |
model-index:
|
6 |
+
- name: out
|
7 |
results: []
|
8 |
---
|
9 |
|
|
|
11 |
should probably proofread and complete it, then remove this comment. -->
|
12 |
|
13 |
[<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
|
14 |
+
# out
|
15 |
|
16 |
+
This model is a fine-tuned version of [NousResearch/Llama-2-7b-hf](https://huggingface.co/NousResearch/Llama-2-7b-hf) on the None dataset.
|
17 |
+
It achieves the following results on the evaluation set:
|
18 |
+
- Loss: 0.9443
|
19 |
|
20 |
## Model description
|
21 |
|
|
|
35 |
|
36 |
The following hyperparameters were used during training:
|
37 |
- learning_rate: 0.0002
|
38 |
+
- train_batch_size: 1
|
39 |
+
- eval_batch_size: 1
|
40 |
- seed: 42
|
41 |
- distributed_type: multi-GPU
|
42 |
- num_devices: 3
|
43 |
+
- total_train_batch_size: 3
|
44 |
+
- total_eval_batch_size: 3
|
|
|
45 |
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
|
46 |
- lr_scheduler_type: cosine
|
47 |
+
- lr_scheduler_warmup_steps: 100
|
48 |
+
- num_epochs: 1
|
49 |
+
|
50 |
+
### Training results
|
51 |
+
|
52 |
+
| Training Loss | Epoch | Step | Validation Loss |
|
53 |
+
|:-------------:|:-----:|:----:|:---------------:|
|
54 |
+
| 1.0254 | 0.03 | 1 | 3.0959 |
|
55 |
+
| 3.2648 | 0.06 | 2 | 3.0959 |
|
56 |
+
| 3.0345 | 0.12 | 4 | 1.6018 |
|
57 |
+
| 1.4912 | 0.18 | 6 | 1.4104 |
|
58 |
+
| 1.4298 | 0.24 | 8 | 1.2483 |
|
59 |
+
| 1.2217 | 0.29 | 10 | 1.1785 |
|
60 |
+
| 1.1975 | 0.35 | 12 | 1.1200 |
|
61 |
+
| 1.1377 | 0.41 | 14 | 1.0922 |
|
62 |
+
| 1.0991 | 0.47 | 16 | 1.0625 |
|
63 |
+
| 0.9783 | 0.53 | 18 | 1.0422 |
|
64 |
+
| 1.0558 | 0.59 | 20 | 1.0100 |
|
65 |
+
| 0.9894 | 0.65 | 22 | 0.9902 |
|
66 |
+
| 0.9677 | 0.71 | 24 | 0.9780 |
|
67 |
+
| 0.9782 | 0.76 | 26 | 0.9679 |
|
68 |
+
| 0.9944 | 0.82 | 28 | 0.9595 |
|
69 |
+
| 0.9245 | 0.88 | 30 | 0.9509 |
|
70 |
+
| 0.9676 | 0.94 | 32 | 0.9468 |
|
71 |
+
| 1.0653 | 1.0 | 34 | 0.9443 |
|
72 |
+
|
73 |
|
74 |
### Framework versions
|
75 |
|
config.json
CHANGED
@@ -17,18 +17,6 @@
|
|
17 |
"num_key_value_heads": 32,
|
18 |
"pad_token_id": 1,
|
19 |
"pretraining_tp": 1,
|
20 |
-
"quantization_config": {
|
21 |
-
"bnb_4bit_compute_dtype": "bfloat16",
|
22 |
-
"bnb_4bit_quant_type": "nf4",
|
23 |
-
"bnb_4bit_use_double_quant": true,
|
24 |
-
"llm_int8_enable_fp32_cpu_offload": false,
|
25 |
-
"llm_int8_has_fp16_weight": false,
|
26 |
-
"llm_int8_skip_modules": null,
|
27 |
-
"llm_int8_threshold": 6.0,
|
28 |
-
"load_in_4bit": true,
|
29 |
-
"load_in_8bit": false,
|
30 |
-
"quant_method": "bitsandbytes"
|
31 |
-
},
|
32 |
"rms_norm_eps": 1e-06,
|
33 |
"rope_scaling": null,
|
34 |
"rope_theta": 500000,
|
|
|
17 |
"num_key_value_heads": 32,
|
18 |
"pad_token_id": 1,
|
19 |
"pretraining_tp": 1,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
20 |
"rms_norm_eps": 1e-06,
|
21 |
"rope_scaling": null,
|
22 |
"rope_theta": 500000,
|
generation_config.json
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_from_model_config": true,
|
3 |
+
"bos_token_id": 0,
|
4 |
+
"eos_token_id": 0,
|
5 |
+
"pad_token_id": 1,
|
6 |
+
"transformers_version": "4.34.1"
|
7 |
+
}
|
pytorch_model.bin
ADDED
File without changes
|
training_args.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:765d7bd291e5773599dd1a122b775bfa93cc0dec79797d7accc64010c9d9dd0d
|
3 |
+
size 6011
|