Upload folder using huggingface_hub
Browse files- README.md +62 -0
- adapter_config.json +34 -0
- adapter_model.safetensors +3 -0
- added_tokens.json +5 -0
- all_results.json +9 -0
- checkpoint-100/README.md +202 -0
- checkpoint-100/adapter_config.json +34 -0
- checkpoint-100/adapter_model.safetensors +3 -0
- checkpoint-100/added_tokens.json +5 -0
- checkpoint-100/merges.txt +0 -0
- checkpoint-100/optimizer.pt +3 -0
- checkpoint-100/rng_state_0.pth +3 -0
- checkpoint-100/rng_state_1.pth +3 -0
- checkpoint-100/scheduler.pt +3 -0
- checkpoint-100/special_tokens_map.json +20 -0
- checkpoint-100/tokenizer.json +0 -0
- checkpoint-100/tokenizer_config.json +44 -0
- checkpoint-100/trainer_state.json +193 -0
- checkpoint-100/training_args.bin +3 -0
- checkpoint-100/vocab.json +0 -0
- llamaboard_config.yaml +65 -0
- merges.txt +0 -0
- running_log.txt +334 -0
- special_tokens_map.json +20 -0
- tokenizer.json +0 -0
- tokenizer_config.json +44 -0
- train_results.json +9 -0
- trainer_log.jsonl +38 -0
- trainer_state.json +339 -0
- training_args.bin +3 -0
- training_args.yaml +33 -0
- training_loss.png +0 -0
- vocab.json +0 -0
README.md
ADDED
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
base_model: Qwen/Qwen2-1.5B-Instruct
|
3 |
+
library_name: peft
|
4 |
+
license: other
|
5 |
+
tags:
|
6 |
+
- llama-factory
|
7 |
+
- lora
|
8 |
+
- generated_from_trainer
|
9 |
+
model-index:
|
10 |
+
- name: train_2024-06-17-19-49-05
|
11 |
+
results: []
|
12 |
+
---
|
13 |
+
|
14 |
+
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
15 |
+
should probably proofread and complete it, then remove this comment. -->
|
16 |
+
|
17 |
+
# train_2024-06-17-19-49-05
|
18 |
+
|
19 |
+
This model is a fine-tuned version of [Qwen/Qwen2-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2-1.5B-Instruct) on the glaive_toolcall_zh and the glaive_toolcall_en datasets.
|
20 |
+
|
21 |
+
## Model description
|
22 |
+
|
23 |
+
More information needed
|
24 |
+
|
25 |
+
## Intended uses & limitations
|
26 |
+
|
27 |
+
More information needed
|
28 |
+
|
29 |
+
## Training and evaluation data
|
30 |
+
|
31 |
+
More information needed
|
32 |
+
|
33 |
+
## Training procedure
|
34 |
+
|
35 |
+
### Training hyperparameters
|
36 |
+
|
37 |
+
The following hyperparameters were used during training:
|
38 |
+
- learning_rate: 5e-05
|
39 |
+
- train_batch_size: 2
|
40 |
+
- eval_batch_size: 8
|
41 |
+
- seed: 42
|
42 |
+
- distributed_type: multi-GPU
|
43 |
+
- num_devices: 2
|
44 |
+
- gradient_accumulation_steps: 8
|
45 |
+
- total_train_batch_size: 32
|
46 |
+
- total_eval_batch_size: 16
|
47 |
+
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
|
48 |
+
- lr_scheduler_type: cosine
|
49 |
+
- num_epochs: 3.0
|
50 |
+
- mixed_precision_training: Native AMP
|
51 |
+
|
52 |
+
### Training results
|
53 |
+
|
54 |
+
|
55 |
+
|
56 |
+
### Framework versions
|
57 |
+
|
58 |
+
- PEFT 0.11.1
|
59 |
+
- Transformers 4.41.2
|
60 |
+
- Pytorch 2.3.1+cu121
|
61 |
+
- Datasets 2.20.0
|
62 |
+
- Tokenizers 0.19.1
|
adapter_config.json
ADDED
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"alpha_pattern": {},
|
3 |
+
"auto_mapping": null,
|
4 |
+
"base_model_name_or_path": "Qwen/Qwen2-1.5B-Instruct",
|
5 |
+
"bias": "none",
|
6 |
+
"fan_in_fan_out": false,
|
7 |
+
"inference_mode": true,
|
8 |
+
"init_lora_weights": true,
|
9 |
+
"layer_replication": null,
|
10 |
+
"layers_pattern": null,
|
11 |
+
"layers_to_transform": null,
|
12 |
+
"loftq_config": {},
|
13 |
+
"lora_alpha": 16,
|
14 |
+
"lora_dropout": 0,
|
15 |
+
"megatron_config": null,
|
16 |
+
"megatron_core": "megatron.core",
|
17 |
+
"modules_to_save": null,
|
18 |
+
"peft_type": "LORA",
|
19 |
+
"r": 8,
|
20 |
+
"rank_pattern": {},
|
21 |
+
"revision": null,
|
22 |
+
"target_modules": [
|
23 |
+
"down_proj",
|
24 |
+
"k_proj",
|
25 |
+
"gate_proj",
|
26 |
+
"up_proj",
|
27 |
+
"o_proj",
|
28 |
+
"v_proj",
|
29 |
+
"q_proj"
|
30 |
+
],
|
31 |
+
"task_type": "CAUSAL_LM",
|
32 |
+
"use_dora": false,
|
33 |
+
"use_rslora": false
|
34 |
+
}
|
adapter_model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1ee7344c30e4a82b5d39ec5658f22cd1b2276c90ecba3888af07aaba4d5d8baa
|
3 |
+
size 36981072
|
added_tokens.json
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"<|endoftext|>": 151643,
|
3 |
+
"<|im_end|>": 151645,
|
4 |
+
"<|im_start|>": 151644
|
5 |
+
}
|
all_results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"epoch": 2.976,
|
3 |
+
"num_input_tokens_seen": 3930992,
|
4 |
+
"total_flos": 3.112338516554547e+16,
|
5 |
+
"train_loss": 0.7158093615885703,
|
6 |
+
"train_runtime": 1518.3826,
|
7 |
+
"train_samples_per_second": 3.952,
|
8 |
+
"train_steps_per_second": 0.122
|
9 |
+
}
|
checkpoint-100/README.md
ADDED
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
base_model: Qwen/Qwen2-1.5B-Instruct
|
3 |
+
library_name: peft
|
4 |
+
---
|
5 |
+
|
6 |
+
# Model Card for Model ID
|
7 |
+
|
8 |
+
<!-- Provide a quick summary of what the model is/does. -->
|
9 |
+
|
10 |
+
|
11 |
+
|
12 |
+
## Model Details
|
13 |
+
|
14 |
+
### Model Description
|
15 |
+
|
16 |
+
<!-- Provide a longer summary of what this model is. -->
|
17 |
+
|
18 |
+
|
19 |
+
|
20 |
+
- **Developed by:** [More Information Needed]
|
21 |
+
- **Funded by [optional]:** [More Information Needed]
|
22 |
+
- **Shared by [optional]:** [More Information Needed]
|
23 |
+
- **Model type:** [More Information Needed]
|
24 |
+
- **Language(s) (NLP):** [More Information Needed]
|
25 |
+
- **License:** [More Information Needed]
|
26 |
+
- **Finetuned from model [optional]:** [More Information Needed]
|
27 |
+
|
28 |
+
### Model Sources [optional]
|
29 |
+
|
30 |
+
<!-- Provide the basic links for the model. -->
|
31 |
+
|
32 |
+
- **Repository:** [More Information Needed]
|
33 |
+
- **Paper [optional]:** [More Information Needed]
|
34 |
+
- **Demo [optional]:** [More Information Needed]
|
35 |
+
|
36 |
+
## Uses
|
37 |
+
|
38 |
+
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
39 |
+
|
40 |
+
### Direct Use
|
41 |
+
|
42 |
+
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
43 |
+
|
44 |
+
[More Information Needed]
|
45 |
+
|
46 |
+
### Downstream Use [optional]
|
47 |
+
|
48 |
+
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
49 |
+
|
50 |
+
[More Information Needed]
|
51 |
+
|
52 |
+
### Out-of-Scope Use
|
53 |
+
|
54 |
+
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
55 |
+
|
56 |
+
[More Information Needed]
|
57 |
+
|
58 |
+
## Bias, Risks, and Limitations
|
59 |
+
|
60 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
61 |
+
|
62 |
+
[More Information Needed]
|
63 |
+
|
64 |
+
### Recommendations
|
65 |
+
|
66 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
67 |
+
|
68 |
+
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
69 |
+
|
70 |
+
## How to Get Started with the Model
|
71 |
+
|
72 |
+
Use the code below to get started with the model.
|
73 |
+
|
74 |
+
[More Information Needed]
|
75 |
+
|
76 |
+
## Training Details
|
77 |
+
|
78 |
+
### Training Data
|
79 |
+
|
80 |
+
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
81 |
+
|
82 |
+
[More Information Needed]
|
83 |
+
|
84 |
+
### Training Procedure
|
85 |
+
|
86 |
+
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
87 |
+
|
88 |
+
#### Preprocessing [optional]
|
89 |
+
|
90 |
+
[More Information Needed]
|
91 |
+
|
92 |
+
|
93 |
+
#### Training Hyperparameters
|
94 |
+
|
95 |
+
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
96 |
+
|
97 |
+
#### Speeds, Sizes, Times [optional]
|
98 |
+
|
99 |
+
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
100 |
+
|
101 |
+
[More Information Needed]
|
102 |
+
|
103 |
+
## Evaluation
|
104 |
+
|
105 |
+
<!-- This section describes the evaluation protocols and provides the results. -->
|
106 |
+
|
107 |
+
### Testing Data, Factors & Metrics
|
108 |
+
|
109 |
+
#### Testing Data
|
110 |
+
|
111 |
+
<!-- This should link to a Dataset Card if possible. -->
|
112 |
+
|
113 |
+
[More Information Needed]
|
114 |
+
|
115 |
+
#### Factors
|
116 |
+
|
117 |
+
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
118 |
+
|
119 |
+
[More Information Needed]
|
120 |
+
|
121 |
+
#### Metrics
|
122 |
+
|
123 |
+
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
124 |
+
|
125 |
+
[More Information Needed]
|
126 |
+
|
127 |
+
### Results
|
128 |
+
|
129 |
+
[More Information Needed]
|
130 |
+
|
131 |
+
#### Summary
|
132 |
+
|
133 |
+
|
134 |
+
|
135 |
+
## Model Examination [optional]
|
136 |
+
|
137 |
+
<!-- Relevant interpretability work for the model goes here -->
|
138 |
+
|
139 |
+
[More Information Needed]
|
140 |
+
|
141 |
+
## Environmental Impact
|
142 |
+
|
143 |
+
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
144 |
+
|
145 |
+
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
146 |
+
|
147 |
+
- **Hardware Type:** [More Information Needed]
|
148 |
+
- **Hours used:** [More Information Needed]
|
149 |
+
- **Cloud Provider:** [More Information Needed]
|
150 |
+
- **Compute Region:** [More Information Needed]
|
151 |
+
- **Carbon Emitted:** [More Information Needed]
|
152 |
+
|
153 |
+
## Technical Specifications [optional]
|
154 |
+
|
155 |
+
### Model Architecture and Objective
|
156 |
+
|
157 |
+
[More Information Needed]
|
158 |
+
|
159 |
+
### Compute Infrastructure
|
160 |
+
|
161 |
+
[More Information Needed]
|
162 |
+
|
163 |
+
#### Hardware
|
164 |
+
|
165 |
+
[More Information Needed]
|
166 |
+
|
167 |
+
#### Software
|
168 |
+
|
169 |
+
[More Information Needed]
|
170 |
+
|
171 |
+
## Citation [optional]
|
172 |
+
|
173 |
+
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
174 |
+
|
175 |
+
**BibTeX:**
|
176 |
+
|
177 |
+
[More Information Needed]
|
178 |
+
|
179 |
+
**APA:**
|
180 |
+
|
181 |
+
[More Information Needed]
|
182 |
+
|
183 |
+
## Glossary [optional]
|
184 |
+
|
185 |
+
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
186 |
+
|
187 |
+
[More Information Needed]
|
188 |
+
|
189 |
+
## More Information [optional]
|
190 |
+
|
191 |
+
[More Information Needed]
|
192 |
+
|
193 |
+
## Model Card Authors [optional]
|
194 |
+
|
195 |
+
[More Information Needed]
|
196 |
+
|
197 |
+
## Model Card Contact
|
198 |
+
|
199 |
+
[More Information Needed]
|
200 |
+
### Framework versions
|
201 |
+
|
202 |
+
- PEFT 0.11.1
|
checkpoint-100/adapter_config.json
ADDED
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"alpha_pattern": {},
|
3 |
+
"auto_mapping": null,
|
4 |
+
"base_model_name_or_path": "Qwen/Qwen2-1.5B-Instruct",
|
5 |
+
"bias": "none",
|
6 |
+
"fan_in_fan_out": false,
|
7 |
+
"inference_mode": true,
|
8 |
+
"init_lora_weights": true,
|
9 |
+
"layer_replication": null,
|
10 |
+
"layers_pattern": null,
|
11 |
+
"layers_to_transform": null,
|
12 |
+
"loftq_config": {},
|
13 |
+
"lora_alpha": 16,
|
14 |
+
"lora_dropout": 0,
|
15 |
+
"megatron_config": null,
|
16 |
+
"megatron_core": "megatron.core",
|
17 |
+
"modules_to_save": null,
|
18 |
+
"peft_type": "LORA",
|
19 |
+
"r": 8,
|
20 |
+
"rank_pattern": {},
|
21 |
+
"revision": null,
|
22 |
+
"target_modules": [
|
23 |
+
"down_proj",
|
24 |
+
"k_proj",
|
25 |
+
"gate_proj",
|
26 |
+
"up_proj",
|
27 |
+
"o_proj",
|
28 |
+
"v_proj",
|
29 |
+
"q_proj"
|
30 |
+
],
|
31 |
+
"task_type": "CAUSAL_LM",
|
32 |
+
"use_dora": false,
|
33 |
+
"use_rslora": false
|
34 |
+
}
|
checkpoint-100/adapter_model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:142b541804154cdf55535cba3bf359060655705c04b2c70424439acab5776e94
|
3 |
+
size 36981072
|
checkpoint-100/added_tokens.json
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"<|endoftext|>": 151643,
|
3 |
+
"<|im_end|>": 151645,
|
4 |
+
"<|im_start|>": 151644
|
5 |
+
}
|
checkpoint-100/merges.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
checkpoint-100/optimizer.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:896f35e3a63391ce73f91fc816e88b284f2576bc9ae78f947efe30b98075b4b5
|
3 |
+
size 74188650
|
checkpoint-100/rng_state_0.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:6cb795a5cea0baa625c50007a6c9da09c6bbb5c16b560424070384a479e7d8a6
|
3 |
+
size 14512
|
checkpoint-100/rng_state_1.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:5f19604377bd828eb366c68946ad997a4ff4d69beaeea93ee58915135768ec63
|
3 |
+
size 14512
|
checkpoint-100/scheduler.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:55724697010011e64bbf43c3defc76460bd60cacef335b6b65467214bd93cd54
|
3 |
+
size 1064
|
checkpoint-100/special_tokens_map.json
ADDED
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
"<|im_start|>",
|
4 |
+
"<|im_end|>"
|
5 |
+
],
|
6 |
+
"eos_token": {
|
7 |
+
"content": "<|im_end|>",
|
8 |
+
"lstrip": false,
|
9 |
+
"normalized": false,
|
10 |
+
"rstrip": false,
|
11 |
+
"single_word": false
|
12 |
+
},
|
13 |
+
"pad_token": {
|
14 |
+
"content": "<|endoftext|>",
|
15 |
+
"lstrip": false,
|
16 |
+
"normalized": false,
|
17 |
+
"rstrip": false,
|
18 |
+
"single_word": false
|
19 |
+
}
|
20 |
+
}
|
checkpoint-100/tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
checkpoint-100/tokenizer_config.json
ADDED
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_prefix_space": false,
|
3 |
+
"added_tokens_decoder": {
|
4 |
+
"151643": {
|
5 |
+
"content": "<|endoftext|>",
|
6 |
+
"lstrip": false,
|
7 |
+
"normalized": false,
|
8 |
+
"rstrip": false,
|
9 |
+
"single_word": false,
|
10 |
+
"special": true
|
11 |
+
},
|
12 |
+
"151644": {
|
13 |
+
"content": "<|im_start|>",
|
14 |
+
"lstrip": false,
|
15 |
+
"normalized": false,
|
16 |
+
"rstrip": false,
|
17 |
+
"single_word": false,
|
18 |
+
"special": true
|
19 |
+
},
|
20 |
+
"151645": {
|
21 |
+
"content": "<|im_end|>",
|
22 |
+
"lstrip": false,
|
23 |
+
"normalized": false,
|
24 |
+
"rstrip": false,
|
25 |
+
"single_word": false,
|
26 |
+
"special": true
|
27 |
+
}
|
28 |
+
},
|
29 |
+
"additional_special_tokens": [
|
30 |
+
"<|im_start|>",
|
31 |
+
"<|im_end|>"
|
32 |
+
],
|
33 |
+
"bos_token": null,
|
34 |
+
"chat_template": "{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\n' + system_message + '<|im_end|>\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\n' + content + '<|im_end|>\n<|im_start|>assistant\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\n' }}{% endif %}{% endfor %}",
|
35 |
+
"clean_up_tokenization_spaces": false,
|
36 |
+
"eos_token": "<|im_end|>",
|
37 |
+
"errors": "replace",
|
38 |
+
"model_max_length": 32768,
|
39 |
+
"pad_token": "<|endoftext|>",
|
40 |
+
"padding_side": "right",
|
41 |
+
"split_special_tokens": false,
|
42 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
43 |
+
"unk_token": null
|
44 |
+
}
|
checkpoint-100/trainer_state.json
ADDED
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"best_metric": null,
|
3 |
+
"best_model_checkpoint": null,
|
4 |
+
"epoch": 1.6,
|
5 |
+
"eval_steps": 500,
|
6 |
+
"global_step": 100,
|
7 |
+
"is_hyper_param_search": false,
|
8 |
+
"is_local_process_zero": true,
|
9 |
+
"is_world_process_zero": true,
|
10 |
+
"log_history": [
|
11 |
+
{
|
12 |
+
"epoch": 0.08,
|
13 |
+
"grad_norm": 0.19458943605422974,
|
14 |
+
"learning_rate": 4.9910902453260824e-05,
|
15 |
+
"loss": 0.8099,
|
16 |
+
"num_input_tokens_seen": 114944,
|
17 |
+
"step": 5
|
18 |
+
},
|
19 |
+
{
|
20 |
+
"epoch": 0.16,
|
21 |
+
"grad_norm": 0.23063905537128448,
|
22 |
+
"learning_rate": 4.964424488287009e-05,
|
23 |
+
"loss": 0.958,
|
24 |
+
"num_input_tokens_seen": 227728,
|
25 |
+
"step": 10
|
26 |
+
},
|
27 |
+
{
|
28 |
+
"epoch": 0.24,
|
29 |
+
"grad_norm": 0.22587424516677856,
|
30 |
+
"learning_rate": 4.920192797165511e-05,
|
31 |
+
"loss": 0.715,
|
32 |
+
"num_input_tokens_seen": 326640,
|
33 |
+
"step": 15
|
34 |
+
},
|
35 |
+
{
|
36 |
+
"epoch": 0.32,
|
37 |
+
"grad_norm": 0.17264235019683838,
|
38 |
+
"learning_rate": 4.858710446774951e-05,
|
39 |
+
"loss": 0.7585,
|
40 |
+
"num_input_tokens_seen": 437280,
|
41 |
+
"step": 20
|
42 |
+
},
|
43 |
+
{
|
44 |
+
"epoch": 0.4,
|
45 |
+
"grad_norm": 0.19875957071781158,
|
46 |
+
"learning_rate": 4.780415671242334e-05,
|
47 |
+
"loss": 0.7342,
|
48 |
+
"num_input_tokens_seen": 530192,
|
49 |
+
"step": 25
|
50 |
+
},
|
51 |
+
{
|
52 |
+
"epoch": 0.48,
|
53 |
+
"grad_norm": 0.36600446701049805,
|
54 |
+
"learning_rate": 4.685866540361456e-05,
|
55 |
+
"loss": 0.6904,
|
56 |
+
"num_input_tokens_seen": 634960,
|
57 |
+
"step": 30
|
58 |
+
},
|
59 |
+
{
|
60 |
+
"epoch": 0.56,
|
61 |
+
"grad_norm": 0.20120520889759064,
|
62 |
+
"learning_rate": 4.5757369817809415e-05,
|
63 |
+
"loss": 0.8254,
|
64 |
+
"num_input_tokens_seen": 740704,
|
65 |
+
"step": 35
|
66 |
+
},
|
67 |
+
{
|
68 |
+
"epoch": 0.64,
|
69 |
+
"grad_norm": 0.22245633602142334,
|
70 |
+
"learning_rate": 4.45081197738023e-05,
|
71 |
+
"loss": 0.7551,
|
72 |
+
"num_input_tokens_seen": 841600,
|
73 |
+
"step": 40
|
74 |
+
},
|
75 |
+
{
|
76 |
+
"epoch": 0.72,
|
77 |
+
"grad_norm": 0.27814680337905884,
|
78 |
+
"learning_rate": 4.3119819680728e-05,
|
79 |
+
"loss": 0.7747,
|
80 |
+
"num_input_tokens_seen": 947648,
|
81 |
+
"step": 45
|
82 |
+
},
|
83 |
+
{
|
84 |
+
"epoch": 0.8,
|
85 |
+
"grad_norm": 0.27052441239356995,
|
86 |
+
"learning_rate": 4.160236506918098e-05,
|
87 |
+
"loss": 0.7027,
|
88 |
+
"num_input_tokens_seen": 1047520,
|
89 |
+
"step": 50
|
90 |
+
},
|
91 |
+
{
|
92 |
+
"epoch": 0.88,
|
93 |
+
"grad_norm": 0.38937926292419434,
|
94 |
+
"learning_rate": 3.9966572057815373e-05,
|
95 |
+
"loss": 0.7581,
|
96 |
+
"num_input_tokens_seen": 1148080,
|
97 |
+
"step": 55
|
98 |
+
},
|
99 |
+
{
|
100 |
+
"epoch": 0.96,
|
101 |
+
"grad_norm": 0.37379005551338196,
|
102 |
+
"learning_rate": 3.822410025817406e-05,
|
103 |
+
"loss": 0.7221,
|
104 |
+
"num_input_tokens_seen": 1252880,
|
105 |
+
"step": 60
|
106 |
+
},
|
107 |
+
{
|
108 |
+
"epoch": 1.04,
|
109 |
+
"grad_norm": 0.34213003516197205,
|
110 |
+
"learning_rate": 3.638736966726585e-05,
|
111 |
+
"loss": 0.8214,
|
112 |
+
"num_input_tokens_seen": 1364304,
|
113 |
+
"step": 65
|
114 |
+
},
|
115 |
+
{
|
116 |
+
"epoch": 1.12,
|
117 |
+
"grad_norm": 0.33407434821128845,
|
118 |
+
"learning_rate": 3.44694721402644e-05,
|
119 |
+
"loss": 0.6304,
|
120 |
+
"num_input_tokens_seen": 1466240,
|
121 |
+
"step": 70
|
122 |
+
},
|
123 |
+
{
|
124 |
+
"epoch": 1.2,
|
125 |
+
"grad_norm": 0.33909159898757935,
|
126 |
+
"learning_rate": 3.2484078074333954e-05,
|
127 |
+
"loss": 0.6434,
|
128 |
+
"num_input_tokens_seen": 1558752,
|
129 |
+
"step": 75
|
130 |
+
},
|
131 |
+
{
|
132 |
+
"epoch": 1.28,
|
133 |
+
"grad_norm": 0.4054814577102661,
|
134 |
+
"learning_rate": 3.0445338968721287e-05,
|
135 |
+
"loss": 0.6796,
|
136 |
+
"num_input_tokens_seen": 1668480,
|
137 |
+
"step": 80
|
138 |
+
},
|
139 |
+
{
|
140 |
+
"epoch": 1.3599999999999999,
|
141 |
+
"grad_norm": 0.2723485231399536,
|
142 |
+
"learning_rate": 2.836778655564653e-05,
|
143 |
+
"loss": 0.6651,
|
144 |
+
"num_input_tokens_seen": 1773264,
|
145 |
+
"step": 85
|
146 |
+
},
|
147 |
+
{
|
148 |
+
"epoch": 1.44,
|
149 |
+
"grad_norm": 0.3467393219470978,
|
150 |
+
"learning_rate": 2.6266229220967818e-05,
|
151 |
+
"loss": 0.7844,
|
152 |
+
"num_input_tokens_seen": 1885920,
|
153 |
+
"step": 90
|
154 |
+
},
|
155 |
+
{
|
156 |
+
"epoch": 1.52,
|
157 |
+
"grad_norm": 0.4250239431858063,
|
158 |
+
"learning_rate": 2.4155646452913296e-05,
|
159 |
+
"loss": 0.8139,
|
160 |
+
"num_input_tokens_seen": 1996656,
|
161 |
+
"step": 95
|
162 |
+
},
|
163 |
+
{
|
164 |
+
"epoch": 1.6,
|
165 |
+
"grad_norm": 0.30656036734580994,
|
166 |
+
"learning_rate": 2.2051082071228854e-05,
|
167 |
+
"loss": 0.6717,
|
168 |
+
"num_input_tokens_seen": 2102896,
|
169 |
+
"step": 100
|
170 |
+
}
|
171 |
+
],
|
172 |
+
"logging_steps": 5,
|
173 |
+
"max_steps": 186,
|
174 |
+
"num_input_tokens_seen": 2102896,
|
175 |
+
"num_train_epochs": 3,
|
176 |
+
"save_steps": 100,
|
177 |
+
"stateful_callbacks": {
|
178 |
+
"TrainerControl": {
|
179 |
+
"args": {
|
180 |
+
"should_epoch_stop": false,
|
181 |
+
"should_evaluate": false,
|
182 |
+
"should_log": false,
|
183 |
+
"should_save": true,
|
184 |
+
"should_training_stop": false
|
185 |
+
},
|
186 |
+
"attributes": {}
|
187 |
+
}
|
188 |
+
},
|
189 |
+
"total_flos": 1.664954843267072e+16,
|
190 |
+
"train_batch_size": 2,
|
191 |
+
"trial_name": null,
|
192 |
+
"trial_params": null
|
193 |
+
}
|
checkpoint-100/training_args.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:25fff679805feed7cb125ffaa9aa318c0b1ee7ef433e1a91fc410bc3dfcec03b
|
3 |
+
size 5304
|
checkpoint-100/vocab.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
llamaboard_config.yaml
ADDED
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
top.booster: none
|
2 |
+
top.checkpoint_path: null
|
3 |
+
top.finetuning_type: lora
|
4 |
+
top.model_name: Qwen2-1.5B-Chat
|
5 |
+
top.quantization_bit: '4'
|
6 |
+
top.rope_scaling: none
|
7 |
+
top.template: qwen
|
8 |
+
top.visual_inputs: false
|
9 |
+
train.additional_target: ''
|
10 |
+
train.badam_mode: layer
|
11 |
+
train.badam_switch_interval: 50
|
12 |
+
train.badam_switch_mode: ascending
|
13 |
+
train.badam_update_ratio: 0.05
|
14 |
+
train.batch_size: 2
|
15 |
+
train.compute_type: fp16
|
16 |
+
train.create_new_adapter: false
|
17 |
+
train.cutoff_len: 3024
|
18 |
+
train.dataset:
|
19 |
+
- glaive_toolcall_zh
|
20 |
+
- glaive_toolcall_en
|
21 |
+
train.dataset_dir: data
|
22 |
+
train.ds_offload: false
|
23 |
+
train.ds_stage: none
|
24 |
+
train.freeze_extra_modules: ''
|
25 |
+
train.freeze_trainable_layers: 2
|
26 |
+
train.freeze_trainable_modules: all
|
27 |
+
train.galore_rank: 16
|
28 |
+
train.galore_scale: 0.25
|
29 |
+
train.galore_target: all
|
30 |
+
train.galore_update_interval: 200
|
31 |
+
train.gradient_accumulation_steps: 8
|
32 |
+
train.learning_rate: 5e-5
|
33 |
+
train.logging_steps: 5
|
34 |
+
train.lora_alpha: 16
|
35 |
+
train.lora_dropout: 0
|
36 |
+
train.lora_rank: 8
|
37 |
+
train.lora_target: ''
|
38 |
+
train.loraplus_lr_ratio: 0
|
39 |
+
train.lr_scheduler_type: cosine
|
40 |
+
train.max_grad_norm: '1.0'
|
41 |
+
train.max_samples: '100000'
|
42 |
+
train.neftune_alpha: 0
|
43 |
+
train.num_train_epochs: '3.0'
|
44 |
+
train.optim: adamw_torch
|
45 |
+
train.packing: false
|
46 |
+
train.ppo_score_norm: false
|
47 |
+
train.ppo_whiten_rewards: false
|
48 |
+
train.pref_beta: 0.1
|
49 |
+
train.pref_ftx: 0
|
50 |
+
train.pref_loss: sigmoid
|
51 |
+
train.report_to: false
|
52 |
+
train.resize_vocab: false
|
53 |
+
train.reward_model: null
|
54 |
+
train.save_steps: 100
|
55 |
+
train.shift_attn: false
|
56 |
+
train.training_stage: Supervised Fine-Tuning
|
57 |
+
train.upcast_layernorm: false
|
58 |
+
train.use_badam: false
|
59 |
+
train.use_dora: false
|
60 |
+
train.use_galore: false
|
61 |
+
train.use_llama_pro: false
|
62 |
+
train.use_pissa: false
|
63 |
+
train.use_rslora: false
|
64 |
+
train.val_size: 0
|
65 |
+
train.warmup_steps: 0
|
merges.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
running_log.txt
ADDED
@@ -0,0 +1,334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
06/17/2024 19:50:24 - INFO - transformers.models.auto.tokenization_auto - Could not locate the tokenizer configuration file, will try to use the model config instead.
|
2 |
+
|
3 |
+
06/17/2024 19:50:25 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /home/featurize/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json
|
4 |
+
|
5 |
+
06/17/2024 19:50:25 - INFO - transformers.configuration_utils - Model config Qwen2Config {
|
6 |
+
"_name_or_path": "Qwen/Qwen2-1.5B-Instruct",
|
7 |
+
"architectures": [
|
8 |
+
"Qwen2ForCausalLM"
|
9 |
+
],
|
10 |
+
"attention_dropout": 0.0,
|
11 |
+
"bos_token_id": 151643,
|
12 |
+
"eos_token_id": 151645,
|
13 |
+
"hidden_act": "silu",
|
14 |
+
"hidden_size": 1536,
|
15 |
+
"initializer_range": 0.02,
|
16 |
+
"intermediate_size": 8960,
|
17 |
+
"max_position_embeddings": 32768,
|
18 |
+
"max_window_layers": 28,
|
19 |
+
"model_type": "qwen2",
|
20 |
+
"num_attention_heads": 12,
|
21 |
+
"num_hidden_layers": 28,
|
22 |
+
"num_key_value_heads": 2,
|
23 |
+
"rms_norm_eps": 1e-06,
|
24 |
+
"rope_theta": 1000000.0,
|
25 |
+
"sliding_window": 32768,
|
26 |
+
"tie_word_embeddings": true,
|
27 |
+
"torch_dtype": "bfloat16",
|
28 |
+
"transformers_version": "4.41.2",
|
29 |
+
"use_cache": true,
|
30 |
+
"use_sliding_window": false,
|
31 |
+
"vocab_size": 151936
|
32 |
+
}
|
33 |
+
|
34 |
+
|
35 |
+
06/17/2024 19:50:30 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /home/featurize/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/vocab.json
|
36 |
+
|
37 |
+
06/17/2024 19:50:30 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at /home/featurize/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/merges.txt
|
38 |
+
|
39 |
+
06/17/2024 19:50:30 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /home/featurize/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer.json
|
40 |
+
|
41 |
+
06/17/2024 19:50:30 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None
|
42 |
+
|
43 |
+
06/17/2024 19:50:30 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at None
|
44 |
+
|
45 |
+
06/17/2024 19:50:30 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /home/featurize/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/tokenizer_config.json
|
46 |
+
|
47 |
+
06/17/2024 19:50:31 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
|
48 |
+
|
49 |
+
06/17/2024 19:50:31 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>
|
50 |
+
|
51 |
+
06/17/2024 19:50:31 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
|
52 |
+
|
53 |
+
06/17/2024 19:50:31 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>
|
54 |
+
|
55 |
+
06/17/2024 19:50:31 - INFO - llamafactory.data.loader - Loading dataset llamafactory/glaive_toolcall_zh...
|
56 |
+
|
57 |
+
06/17/2024 19:50:37 - INFO - llamafactory.data.loader - Loading dataset llamafactory/glaive_toolcall_en...
|
58 |
+
|
59 |
+
06/17/2024 19:50:43 - INFO - llamafactory.data.loader - Loading dataset llamafactory/glaive_toolcall_zh...
|
60 |
+
|
61 |
+
06/17/2024 19:50:47 - INFO - llamafactory.data.loader - Loading dataset llamafactory/glaive_toolcall_en...
|
62 |
+
|
63 |
+
06/17/2024 19:50:52 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /home/featurize/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json
|
64 |
+
|
65 |
+
06/17/2024 19:50:52 - INFO - transformers.configuration_utils - Model config Qwen2Config {
|
66 |
+
"_name_or_path": "Qwen/Qwen2-1.5B-Instruct",
|
67 |
+
"architectures": [
|
68 |
+
"Qwen2ForCausalLM"
|
69 |
+
],
|
70 |
+
"attention_dropout": 0.0,
|
71 |
+
"bos_token_id": 151643,
|
72 |
+
"eos_token_id": 151645,
|
73 |
+
"hidden_act": "silu",
|
74 |
+
"hidden_size": 1536,
|
75 |
+
"initializer_range": 0.02,
|
76 |
+
"intermediate_size": 8960,
|
77 |
+
"max_position_embeddings": 32768,
|
78 |
+
"max_window_layers": 28,
|
79 |
+
"model_type": "qwen2",
|
80 |
+
"num_attention_heads": 12,
|
81 |
+
"num_hidden_layers": 28,
|
82 |
+
"num_key_value_heads": 2,
|
83 |
+
"rms_norm_eps": 1e-06,
|
84 |
+
"rope_theta": 1000000.0,
|
85 |
+
"sliding_window": 32768,
|
86 |
+
"tie_word_embeddings": true,
|
87 |
+
"torch_dtype": "bfloat16",
|
88 |
+
"transformers_version": "4.41.2",
|
89 |
+
"use_cache": true,
|
90 |
+
"use_sliding_window": false,
|
91 |
+
"vocab_size": 151936
|
92 |
+
}
|
93 |
+
|
94 |
+
|
95 |
+
06/17/2024 19:50:52 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit.
|
96 |
+
|
97 |
+
06/17/2024 19:50:52 - INFO - llamafactory.model.model_utils.quantization - Quantizing model to 4 bit.
|
98 |
+
|
99 |
+
06/17/2024 19:50:59 - INFO - transformers.modeling_utils - loading weights file model.safetensors from cache at /home/featurize/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/model.safetensors
|
100 |
+
|
101 |
+
06/17/2024 19:50:59 - INFO - transformers.modeling_utils - Instantiating Qwen2ForCausalLM model under default dtype torch.float16.
|
102 |
+
|
103 |
+
06/17/2024 19:50:59 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig {
|
104 |
+
"bos_token_id": 151643,
|
105 |
+
"eos_token_id": 151645
|
106 |
+
}
|
107 |
+
|
108 |
+
|
109 |
+
06/17/2024 19:51:06 - INFO - transformers.modeling_utils - All model checkpoint weights were used when initializing Qwen2ForCausalLM.
|
110 |
+
|
111 |
+
|
112 |
+
06/17/2024 19:51:06 - INFO - transformers.modeling_utils - All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2-1.5B-Instruct.
|
113 |
+
If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.
|
114 |
+
|
115 |
+
06/17/2024 19:51:06 - INFO - transformers.generation.configuration_utils - loading configuration file generation_config.json from cache at /home/featurize/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/generation_config.json
|
116 |
+
|
117 |
+
06/17/2024 19:51:06 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig {
|
118 |
+
"bos_token_id": 151643,
|
119 |
+
"do_sample": true,
|
120 |
+
"eos_token_id": [
|
121 |
+
151645,
|
122 |
+
151643
|
123 |
+
],
|
124 |
+
"pad_token_id": 151643,
|
125 |
+
"repetition_penalty": 1.1,
|
126 |
+
"temperature": 0.7,
|
127 |
+
"top_k": 20,
|
128 |
+
"top_p": 0.8
|
129 |
+
}
|
130 |
+
|
131 |
+
|
132 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
|
133 |
+
|
134 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
|
135 |
+
|
136 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
|
137 |
+
|
138 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA
|
139 |
+
|
140 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.model_utils.misc - Found linear modules: o_proj,v_proj,down_proj,gate_proj,k_proj,up_proj,q_proj
|
141 |
+
|
142 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
|
143 |
+
|
144 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
|
145 |
+
|
146 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
|
147 |
+
|
148 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA
|
149 |
+
|
150 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.model_utils.misc - Found linear modules: down_proj,k_proj,gate_proj,up_proj,o_proj,v_proj,q_proj
|
151 |
+
|
152 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.loader - trainable params: 9232384 || all params: 1552946688 || trainable%: 0.5945
|
153 |
+
|
154 |
+
06/17/2024 19:51:07 - INFO - llamafactory.model.loader - trainable params: 9232384 || all params: 1552946688 || trainable%: 0.5945
|
155 |
+
|
156 |
+
06/17/2024 19:51:07 - WARNING - accelerate.utils.other - Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
|
157 |
+
|
158 |
+
06/17/2024 19:51:07 - INFO - transformers.trainer - Using auto half precision backend
|
159 |
+
|
160 |
+
06/17/2024 19:51:07 - INFO - transformers.trainer - ***** Running training *****
|
161 |
+
|
162 |
+
06/17/2024 19:51:07 - INFO - transformers.trainer - Num examples = 2,000
|
163 |
+
|
164 |
+
06/17/2024 19:51:07 - INFO - transformers.trainer - Num Epochs = 3
|
165 |
+
|
166 |
+
06/17/2024 19:51:07 - INFO - transformers.trainer - Instantaneous batch size per device = 2
|
167 |
+
|
168 |
+
06/17/2024 19:51:07 - INFO - transformers.trainer - Total train batch size (w. parallel, distributed & accumulation) = 32
|
169 |
+
|
170 |
+
06/17/2024 19:51:07 - INFO - transformers.trainer - Gradient Accumulation steps = 8
|
171 |
+
|
172 |
+
06/17/2024 19:51:07 - INFO - transformers.trainer - Total optimization steps = 186
|
173 |
+
|
174 |
+
06/17/2024 19:51:07 - INFO - transformers.trainer - Number of trainable parameters = 9,232,384
|
175 |
+
|
176 |
+
06/17/2024 19:51:52 - INFO - llamafactory.extras.callbacks - {'loss': 0.8099, 'learning_rate': 4.9911e-05, 'epoch': 0.08, 'throughput': 2566.78}
|
177 |
+
|
178 |
+
06/17/2024 19:52:37 - INFO - llamafactory.extras.callbacks - {'loss': 0.9580, 'learning_rate': 4.9644e-05, 'epoch': 0.16, 'throughput': 2542.66}
|
179 |
+
|
180 |
+
06/17/2024 19:53:18 - INFO - llamafactory.extras.callbacks - {'loss': 0.7150, 'learning_rate': 4.9202e-05, 'epoch': 0.24, 'throughput': 2505.70}
|
181 |
+
|
182 |
+
06/17/2024 19:54:02 - INFO - llamafactory.extras.callbacks - {'loss': 0.7585, 'learning_rate': 4.8587e-05, 'epoch': 0.32, 'throughput': 2511.00}
|
183 |
+
|
184 |
+
06/17/2024 19:54:37 - INFO - llamafactory.extras.callbacks - {'loss': 0.7342, 'learning_rate': 4.7804e-05, 'epoch': 0.40, 'throughput': 2533.17}
|
185 |
+
|
186 |
+
06/17/2024 19:55:16 - INFO - llamafactory.extras.callbacks - {'loss': 0.6904, 'learning_rate': 4.6859e-05, 'epoch': 0.48, 'throughput': 2557.91}
|
187 |
+
|
188 |
+
06/17/2024 19:55:54 - INFO - llamafactory.extras.callbacks - {'loss': 0.8254, 'learning_rate': 4.5757e-05, 'epoch': 0.56, 'throughput': 2587.35}
|
189 |
+
|
190 |
+
06/17/2024 19:56:34 - INFO - llamafactory.extras.callbacks - {'loss': 0.7551, 'learning_rate': 4.4508e-05, 'epoch': 0.64, 'throughput': 2578.43}
|
191 |
+
|
192 |
+
06/17/2024 19:57:15 - INFO - llamafactory.extras.callbacks - {'loss': 0.7747, 'learning_rate': 4.3120e-05, 'epoch': 0.72, 'throughput': 2580.98}
|
193 |
+
|
194 |
+
06/17/2024 19:57:54 - INFO - llamafactory.extras.callbacks - {'loss': 0.7027, 'learning_rate': 4.1602e-05, 'epoch': 0.80, 'throughput': 2578.88}
|
195 |
+
|
196 |
+
06/17/2024 19:58:32 - INFO - llamafactory.extras.callbacks - {'loss': 0.7581, 'learning_rate': 3.9967e-05, 'epoch': 0.88, 'throughput': 2580.17}
|
197 |
+
|
198 |
+
06/17/2024 19:59:13 - INFO - llamafactory.extras.callbacks - {'loss': 0.7221, 'learning_rate': 3.8224e-05, 'epoch': 0.96, 'throughput': 2583.14}
|
199 |
+
|
200 |
+
06/17/2024 19:59:55 - INFO - llamafactory.extras.callbacks - {'loss': 0.8214, 'learning_rate': 3.6387e-05, 'epoch': 1.04, 'throughput': 2587.56}
|
201 |
+
|
202 |
+
06/17/2024 20:00:36 - INFO - llamafactory.extras.callbacks - {'loss': 0.6304, 'learning_rate': 3.4469e-05, 'epoch': 1.12, 'throughput': 2580.91}
|
203 |
+
|
204 |
+
06/17/2024 20:01:12 - INFO - llamafactory.extras.callbacks - {'loss': 0.6434, 'learning_rate': 3.2484e-05, 'epoch': 1.20, 'throughput': 2577.50}
|
205 |
+
|
206 |
+
06/17/2024 20:01:55 - INFO - llamafactory.extras.callbacks - {'loss': 0.6796, 'learning_rate': 3.0445e-05, 'epoch': 1.28, 'throughput': 2575.34}
|
207 |
+
|
208 |
+
06/17/2024 20:02:35 - INFO - llamafactory.extras.callbacks - {'loss': 0.6651, 'learning_rate': 2.8368e-05, 'epoch': 1.36, 'throughput': 2580.63}
|
209 |
+
|
210 |
+
06/17/2024 20:03:17 - INFO - llamafactory.extras.callbacks - {'loss': 0.7844, 'learning_rate': 2.6266e-05, 'epoch': 1.44, 'throughput': 2586.23}
|
211 |
+
|
212 |
+
06/17/2024 20:04:01 - INFO - llamafactory.extras.callbacks - {'loss': 0.8139, 'learning_rate': 2.4156e-05, 'epoch': 1.52, 'throughput': 2581.43}
|
213 |
+
|
214 |
+
06/17/2024 20:04:43 - INFO - llamafactory.extras.callbacks - {'loss': 0.6717, 'learning_rate': 2.2051e-05, 'epoch': 1.60, 'throughput': 2578.63}
|
215 |
+
|
216 |
+
06/17/2024 20:04:43 - INFO - transformers.trainer - Saving model checkpoint to saves/Qwen2-1.5B-Chat/lora/train_2024-06-17-19-49-05/checkpoint-100
|
217 |
+
|
218 |
+
06/17/2024 20:04:44 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /home/featurize/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json
|
219 |
+
|
220 |
+
06/17/2024 20:04:44 - INFO - transformers.configuration_utils - Model config Qwen2Config {
|
221 |
+
"architectures": [
|
222 |
+
"Qwen2ForCausalLM"
|
223 |
+
],
|
224 |
+
"attention_dropout": 0.0,
|
225 |
+
"bos_token_id": 151643,
|
226 |
+
"eos_token_id": 151645,
|
227 |
+
"hidden_act": "silu",
|
228 |
+
"hidden_size": 1536,
|
229 |
+
"initializer_range": 0.02,
|
230 |
+
"intermediate_size": 8960,
|
231 |
+
"max_position_embeddings": 32768,
|
232 |
+
"max_window_layers": 28,
|
233 |
+
"model_type": "qwen2",
|
234 |
+
"num_attention_heads": 12,
|
235 |
+
"num_hidden_layers": 28,
|
236 |
+
"num_key_value_heads": 2,
|
237 |
+
"rms_norm_eps": 1e-06,
|
238 |
+
"rope_theta": 1000000.0,
|
239 |
+
"sliding_window": 32768,
|
240 |
+
"tie_word_embeddings": true,
|
241 |
+
"torch_dtype": "bfloat16",
|
242 |
+
"transformers_version": "4.41.2",
|
243 |
+
"use_cache": true,
|
244 |
+
"use_sliding_window": false,
|
245 |
+
"vocab_size": 151936
|
246 |
+
}
|
247 |
+
|
248 |
+
|
249 |
+
06/17/2024 20:04:44 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in saves/Qwen2-1.5B-Chat/lora/train_2024-06-17-19-49-05/checkpoint-100/tokenizer_config.json
|
250 |
+
|
251 |
+
06/17/2024 20:04:44 - INFO - transformers.tokenization_utils_base - Special tokens file saved in saves/Qwen2-1.5B-Chat/lora/train_2024-06-17-19-49-05/checkpoint-100/special_tokens_map.json
|
252 |
+
|
253 |
+
06/17/2024 20:05:27 - INFO - llamafactory.extras.callbacks - {'loss': 0.7524, 'learning_rate': 1.9968e-05, 'epoch': 1.68, 'throughput': 2577.77}
|
254 |
+
|
255 |
+
06/17/2024 20:06:06 - INFO - llamafactory.extras.callbacks - {'loss': 0.6310, 'learning_rate': 1.7920e-05, 'epoch': 1.76, 'throughput': 2578.49}
|
256 |
+
|
257 |
+
06/17/2024 20:06:48 - INFO - llamafactory.extras.callbacks - {'loss': 0.7462, 'learning_rate': 1.5923e-05, 'epoch': 1.84, 'throughput': 2578.25}
|
258 |
+
|
259 |
+
06/17/2024 20:07:32 - INFO - llamafactory.extras.callbacks - {'loss': 0.6148, 'learning_rate': 1.3990e-05, 'epoch': 1.92, 'throughput': 2578.95}
|
260 |
+
|
261 |
+
06/17/2024 20:08:10 - INFO - llamafactory.extras.callbacks - {'loss': 0.7145, 'learning_rate': 1.2136e-05, 'epoch': 2.00, 'throughput': 2582.83}
|
262 |
+
|
263 |
+
06/17/2024 20:08:52 - INFO - llamafactory.extras.callbacks - {'loss': 0.6798, 'learning_rate': 1.0374e-05, 'epoch': 2.08, 'throughput': 2583.35}
|
264 |
+
|
265 |
+
06/17/2024 20:09:31 - INFO - llamafactory.extras.callbacks - {'loss': 0.6754, 'learning_rate': 8.7157e-06, 'epoch': 2.16, 'throughput': 2584.30}
|
266 |
+
|
267 |
+
06/17/2024 20:10:10 - INFO - llamafactory.extras.callbacks - {'loss': 0.6708, 'learning_rate': 7.1737e-06, 'epoch': 2.24, 'throughput': 2584.52}
|
268 |
+
|
269 |
+
06/17/2024 20:10:46 - INFO - llamafactory.extras.callbacks - {'loss': 0.6386, 'learning_rate': 5.7587e-06, 'epoch': 2.32, 'throughput': 2587.95}
|
270 |
+
|
271 |
+
06/17/2024 20:11:24 - INFO - llamafactory.extras.callbacks - {'loss': 0.6995, 'learning_rate': 4.4809e-06, 'epoch': 2.40, 'throughput': 2590.06}
|
272 |
+
|
273 |
+
06/17/2024 20:12:06 - INFO - llamafactory.extras.callbacks - {'loss': 0.6691, 'learning_rate': 3.3494e-06, 'epoch': 2.48, 'throughput': 2593.15}
|
274 |
+
|
275 |
+
06/17/2024 20:12:50 - INFO - llamafactory.extras.callbacks - {'loss': 0.6024, 'learning_rate': 2.3721e-06, 'epoch': 2.56, 'throughput': 2588.49}
|
276 |
+
|
277 |
+
06/17/2024 20:13:28 - INFO - llamafactory.extras.callbacks - {'loss': 0.6484, 'learning_rate': 1.5562e-06, 'epoch': 2.64, 'throughput': 2591.26}
|
278 |
+
|
279 |
+
06/17/2024 20:14:10 - INFO - llamafactory.extras.callbacks - {'loss': 0.7137, 'learning_rate': 9.0736e-07, 'epoch': 2.72, 'throughput': 2585.69}
|
280 |
+
|
281 |
+
06/17/2024 20:14:56 - INFO - llamafactory.extras.callbacks - {'loss': 0.7770, 'learning_rate': 4.3025e-07, 'epoch': 2.80, 'throughput': 2589.29}
|
282 |
+
|
283 |
+
06/17/2024 20:15:36 - INFO - llamafactory.extras.callbacks - {'loss': 0.6529, 'learning_rate': 1.2827e-07, 'epoch': 2.88, 'throughput': 2590.50}
|
284 |
+
|
285 |
+
06/17/2024 20:16:16 - INFO - llamafactory.extras.callbacks - {'loss': 0.6996, 'learning_rate': 3.5659e-09, 'epoch': 2.96, 'throughput': 2590.41}
|
286 |
+
|
287 |
+
06/17/2024 20:16:26 - INFO - transformers.trainer -
|
288 |
+
|
289 |
+
Training completed. Do not forget to share your model on huggingface.co/models =)
|
290 |
+
|
291 |
+
|
292 |
+
|
293 |
+
06/17/2024 20:16:26 - INFO - transformers.trainer - Saving model checkpoint to saves/Qwen2-1.5B-Chat/lora/train_2024-06-17-19-49-05
|
294 |
+
|
295 |
+
06/17/2024 20:16:27 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /home/featurize/.cache/huggingface/hub/models--Qwen--Qwen2-1.5B-Instruct/snapshots/ba1cf1846d7df0a0591d6c00649f57e798519da8/config.json
|
296 |
+
|
297 |
+
06/17/2024 20:16:27 - INFO - transformers.configuration_utils - Model config Qwen2Config {
|
298 |
+
"architectures": [
|
299 |
+
"Qwen2ForCausalLM"
|
300 |
+
],
|
301 |
+
"attention_dropout": 0.0,
|
302 |
+
"bos_token_id": 151643,
|
303 |
+
"eos_token_id": 151645,
|
304 |
+
"hidden_act": "silu",
|
305 |
+
"hidden_size": 1536,
|
306 |
+
"initializer_range": 0.02,
|
307 |
+
"intermediate_size": 8960,
|
308 |
+
"max_position_embeddings": 32768,
|
309 |
+
"max_window_layers": 28,
|
310 |
+
"model_type": "qwen2",
|
311 |
+
"num_attention_heads": 12,
|
312 |
+
"num_hidden_layers": 28,
|
313 |
+
"num_key_value_heads": 2,
|
314 |
+
"rms_norm_eps": 1e-06,
|
315 |
+
"rope_theta": 1000000.0,
|
316 |
+
"sliding_window": 32768,
|
317 |
+
"tie_word_embeddings": true,
|
318 |
+
"torch_dtype": "bfloat16",
|
319 |
+
"transformers_version": "4.41.2",
|
320 |
+
"use_cache": true,
|
321 |
+
"use_sliding_window": false,
|
322 |
+
"vocab_size": 151936
|
323 |
+
}
|
324 |
+
|
325 |
+
|
326 |
+
06/17/2024 20:16:27 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in saves/Qwen2-1.5B-Chat/lora/train_2024-06-17-19-49-05/tokenizer_config.json
|
327 |
+
|
328 |
+
06/17/2024 20:16:27 - INFO - transformers.tokenization_utils_base - Special tokens file saved in saves/Qwen2-1.5B-Chat/lora/train_2024-06-17-19-49-05/special_tokens_map.json
|
329 |
+
|
330 |
+
06/17/2024 20:16:27 - WARNING - llamafactory.extras.ploting - No metric eval_loss to plot.
|
331 |
+
|
332 |
+
06/17/2024 20:16:27 - INFO - transformers.modelcard - Dropping the following result as it does not have all the necessary fields:
|
333 |
+
{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
|
334 |
+
|
special_tokens_map.json
ADDED
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
"<|im_start|>",
|
4 |
+
"<|im_end|>"
|
5 |
+
],
|
6 |
+
"eos_token": {
|
7 |
+
"content": "<|im_end|>",
|
8 |
+
"lstrip": false,
|
9 |
+
"normalized": false,
|
10 |
+
"rstrip": false,
|
11 |
+
"single_word": false
|
12 |
+
},
|
13 |
+
"pad_token": {
|
14 |
+
"content": "<|endoftext|>",
|
15 |
+
"lstrip": false,
|
16 |
+
"normalized": false,
|
17 |
+
"rstrip": false,
|
18 |
+
"single_word": false
|
19 |
+
}
|
20 |
+
}
|
tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
ADDED
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_prefix_space": false,
|
3 |
+
"added_tokens_decoder": {
|
4 |
+
"151643": {
|
5 |
+
"content": "<|endoftext|>",
|
6 |
+
"lstrip": false,
|
7 |
+
"normalized": false,
|
8 |
+
"rstrip": false,
|
9 |
+
"single_word": false,
|
10 |
+
"special": true
|
11 |
+
},
|
12 |
+
"151644": {
|
13 |
+
"content": "<|im_start|>",
|
14 |
+
"lstrip": false,
|
15 |
+
"normalized": false,
|
16 |
+
"rstrip": false,
|
17 |
+
"single_word": false,
|
18 |
+
"special": true
|
19 |
+
},
|
20 |
+
"151645": {
|
21 |
+
"content": "<|im_end|>",
|
22 |
+
"lstrip": false,
|
23 |
+
"normalized": false,
|
24 |
+
"rstrip": false,
|
25 |
+
"single_word": false,
|
26 |
+
"special": true
|
27 |
+
}
|
28 |
+
},
|
29 |
+
"additional_special_tokens": [
|
30 |
+
"<|im_start|>",
|
31 |
+
"<|im_end|>"
|
32 |
+
],
|
33 |
+
"bos_token": null,
|
34 |
+
"chat_template": "{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\n' + system_message + '<|im_end|>\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\n' + content + '<|im_end|>\n<|im_start|>assistant\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\n' }}{% endif %}{% endfor %}",
|
35 |
+
"clean_up_tokenization_spaces": false,
|
36 |
+
"eos_token": "<|im_end|>",
|
37 |
+
"errors": "replace",
|
38 |
+
"model_max_length": 32768,
|
39 |
+
"pad_token": "<|endoftext|>",
|
40 |
+
"padding_side": "right",
|
41 |
+
"split_special_tokens": false,
|
42 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
43 |
+
"unk_token": null
|
44 |
+
}
|
train_results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"epoch": 2.976,
|
3 |
+
"num_input_tokens_seen": 3930992,
|
4 |
+
"total_flos": 3.112338516554547e+16,
|
5 |
+
"train_loss": 0.7158093615885703,
|
6 |
+
"train_runtime": 1518.3826,
|
7 |
+
"train_samples_per_second": 3.952,
|
8 |
+
"train_steps_per_second": 0.122
|
9 |
+
}
|
trainer_log.jsonl
ADDED
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{"current_steps": 5, "total_steps": 186, "loss": 0.8099, "learning_rate": 4.9910902453260824e-05, "epoch": 0.08, "percentage": 2.69, "elapsed_time": "0:00:44", "remaining_time": "0:27:01", "throughput": "2566.78", "total_tokens": 114944}
|
2 |
+
{"current_steps": 10, "total_steps": 186, "loss": 0.958, "learning_rate": 4.964424488287009e-05, "epoch": 0.16, "percentage": 5.38, "elapsed_time": "0:01:29", "remaining_time": "0:26:16", "throughput": "2542.66", "total_tokens": 227728}
|
3 |
+
{"current_steps": 15, "total_steps": 186, "loss": 0.715, "learning_rate": 4.920192797165511e-05, "epoch": 0.24, "percentage": 8.06, "elapsed_time": "0:02:10", "remaining_time": "0:24:46", "throughput": "2505.70", "total_tokens": 326640}
|
4 |
+
{"current_steps": 20, "total_steps": 186, "loss": 0.7585, "learning_rate": 4.858710446774951e-05, "epoch": 0.32, "percentage": 10.75, "elapsed_time": "0:02:54", "remaining_time": "0:24:05", "throughput": "2511.00", "total_tokens": 437280}
|
5 |
+
{"current_steps": 25, "total_steps": 186, "loss": 0.7342, "learning_rate": 4.780415671242334e-05, "epoch": 0.4, "percentage": 13.44, "elapsed_time": "0:03:29", "remaining_time": "0:22:27", "throughput": "2533.17", "total_tokens": 530192}
|
6 |
+
{"current_steps": 30, "total_steps": 186, "loss": 0.6904, "learning_rate": 4.685866540361456e-05, "epoch": 0.48, "percentage": 16.13, "elapsed_time": "0:04:08", "remaining_time": "0:21:30", "throughput": "2557.91", "total_tokens": 634960}
|
7 |
+
{"current_steps": 35, "total_steps": 186, "loss": 0.8254, "learning_rate": 4.5757369817809415e-05, "epoch": 0.56, "percentage": 18.82, "elapsed_time": "0:04:46", "remaining_time": "0:20:35", "throughput": "2587.35", "total_tokens": 740704}
|
8 |
+
{"current_steps": 40, "total_steps": 186, "loss": 0.7551, "learning_rate": 4.45081197738023e-05, "epoch": 0.64, "percentage": 21.51, "elapsed_time": "0:05:26", "remaining_time": "0:19:51", "throughput": "2578.43", "total_tokens": 841600}
|
9 |
+
{"current_steps": 45, "total_steps": 186, "loss": 0.7747, "learning_rate": 4.3119819680728e-05, "epoch": 0.72, "percentage": 24.19, "elapsed_time": "0:06:07", "remaining_time": "0:19:10", "throughput": "2580.98", "total_tokens": 947648}
|
10 |
+
{"current_steps": 50, "total_steps": 186, "loss": 0.7027, "learning_rate": 4.160236506918098e-05, "epoch": 0.8, "percentage": 26.88, "elapsed_time": "0:06:46", "remaining_time": "0:18:24", "throughput": "2578.88", "total_tokens": 1047520}
|
11 |
+
{"current_steps": 55, "total_steps": 186, "loss": 0.7581, "learning_rate": 3.9966572057815373e-05, "epoch": 0.88, "percentage": 29.57, "elapsed_time": "0:07:24", "remaining_time": "0:17:39", "throughput": "2580.17", "total_tokens": 1148080}
|
12 |
+
{"current_steps": 60, "total_steps": 186, "loss": 0.7221, "learning_rate": 3.822410025817406e-05, "epoch": 0.96, "percentage": 32.26, "elapsed_time": "0:08:05", "remaining_time": "0:16:58", "throughput": "2583.14", "total_tokens": 1252880}
|
13 |
+
{"current_steps": 65, "total_steps": 186, "loss": 0.8214, "learning_rate": 3.638736966726585e-05, "epoch": 1.04, "percentage": 34.95, "elapsed_time": "0:08:47", "remaining_time": "0:16:21", "throughput": "2587.56", "total_tokens": 1364304}
|
14 |
+
{"current_steps": 70, "total_steps": 186, "loss": 0.6304, "learning_rate": 3.44694721402644e-05, "epoch": 1.12, "percentage": 37.63, "elapsed_time": "0:09:28", "remaining_time": "0:15:41", "throughput": "2580.91", "total_tokens": 1466240}
|
15 |
+
{"current_steps": 75, "total_steps": 186, "loss": 0.6434, "learning_rate": 3.2484078074333954e-05, "epoch": 1.2, "percentage": 40.32, "elapsed_time": "0:10:04", "remaining_time": "0:14:55", "throughput": "2577.50", "total_tokens": 1558752}
|
16 |
+
{"current_steps": 80, "total_steps": 186, "loss": 0.6796, "learning_rate": 3.0445338968721287e-05, "epoch": 1.28, "percentage": 43.01, "elapsed_time": "0:10:47", "remaining_time": "0:14:18", "throughput": "2575.34", "total_tokens": 1668480}
|
17 |
+
{"current_steps": 85, "total_steps": 186, "loss": 0.6651, "learning_rate": 2.836778655564653e-05, "epoch": 1.3599999999999999, "percentage": 45.7, "elapsed_time": "0:11:27", "remaining_time": "0:13:36", "throughput": "2580.63", "total_tokens": 1773264}
|
18 |
+
{"current_steps": 90, "total_steps": 186, "loss": 0.7844, "learning_rate": 2.6266229220967818e-05, "epoch": 1.44, "percentage": 48.39, "elapsed_time": "0:12:09", "remaining_time": "0:12:57", "throughput": "2586.23", "total_tokens": 1885920}
|
19 |
+
{"current_steps": 95, "total_steps": 186, "loss": 0.8139, "learning_rate": 2.4155646452913296e-05, "epoch": 1.52, "percentage": 51.08, "elapsed_time": "0:12:53", "remaining_time": "0:12:20", "throughput": "2581.43", "total_tokens": 1996656}
|
20 |
+
{"current_steps": 100, "total_steps": 186, "loss": 0.6717, "learning_rate": 2.2051082071228854e-05, "epoch": 1.6, "percentage": 53.76, "elapsed_time": "0:13:35", "remaining_time": "0:11:41", "throughput": "2578.63", "total_tokens": 2102896}
|
21 |
+
{"current_steps": 105, "total_steps": 186, "loss": 0.7524, "learning_rate": 1.9967536997783494e-05, "epoch": 1.6800000000000002, "percentage": 56.45, "elapsed_time": "0:14:18", "remaining_time": "0:11:02", "throughput": "2577.77", "total_tokens": 2214272}
|
22 |
+
{"current_steps": 110, "total_steps": 186, "loss": 0.631, "learning_rate": 1.79198623329424e-05, "epoch": 1.76, "percentage": 59.14, "elapsed_time": "0:14:58", "remaining_time": "0:10:20", "throughput": "2578.49", "total_tokens": 2316448}
|
23 |
+
{"current_steps": 115, "total_steps": 186, "loss": 0.7462, "learning_rate": 1.5922653499838137e-05, "epoch": 1.8399999999999999, "percentage": 61.83, "elapsed_time": "0:15:40", "remaining_time": "0:09:40", "throughput": "2578.25", "total_tokens": 2425248}
|
24 |
+
{"current_steps": 120, "total_steps": 186, "loss": 0.6148, "learning_rate": 1.399014621105914e-05, "epoch": 1.92, "percentage": 64.52, "elapsed_time": "0:16:23", "remaining_time": "0:09:01", "throughput": "2578.95", "total_tokens": 2537680}
|
25 |
+
{"current_steps": 125, "total_steps": 186, "loss": 0.7145, "learning_rate": 1.2136114999284288e-05, "epoch": 2.0, "percentage": 67.2, "elapsed_time": "0:17:02", "remaining_time": "0:08:19", "throughput": "2582.83", "total_tokens": 2642080}
|
26 |
+
{"current_steps": 130, "total_steps": 186, "loss": 0.6798, "learning_rate": 1.0373775035117305e-05, "epoch": 2.08, "percentage": 69.89, "elapsed_time": "0:17:44", "remaining_time": "0:07:38", "throughput": "2583.35", "total_tokens": 2750752}
|
27 |
+
{"current_steps": 135, "total_steps": 186, "loss": 0.6754, "learning_rate": 8.715687931944449e-06, "epoch": 2.16, "percentage": 72.58, "elapsed_time": "0:18:23", "remaining_time": "0:06:56", "throughput": "2584.30", "total_tokens": 2852096}
|
28 |
+
{"current_steps": 140, "total_steps": 186, "loss": 0.6708, "learning_rate": 7.173672209219495e-06, "epoch": 2.24, "percentage": 75.27, "elapsed_time": "0:19:02", "remaining_time": "0:06:15", "throughput": "2584.52", "total_tokens": 2951888}
|
29 |
+
{"current_steps": 145, "total_steps": 186, "loss": 0.6386, "learning_rate": 5.758719052376693e-06, "epoch": 2.32, "percentage": 77.96, "elapsed_time": "0:19:38", "remaining_time": "0:05:33", "throughput": "2587.95", "total_tokens": 3049616}
|
30 |
+
{"current_steps": 150, "total_steps": 186, "loss": 0.6995, "learning_rate": 4.480913969818098e-06, "epoch": 2.4, "percentage": 80.65, "elapsed_time": "0:20:16", "remaining_time": "0:04:51", "throughput": "2590.06", "total_tokens": 3150784}
|
31 |
+
{"current_steps": 155, "total_steps": 186, "loss": 0.6691, "learning_rate": 3.3493649053890326e-06, "epoch": 2.48, "percentage": 83.33, "elapsed_time": "0:20:58", "remaining_time": "0:04:11", "throughput": "2593.15", "total_tokens": 3262992}
|
32 |
+
{"current_steps": 160, "total_steps": 186, "loss": 0.6024, "learning_rate": 2.372137318741968e-06, "epoch": 2.56, "percentage": 86.02, "elapsed_time": "0:21:42", "remaining_time": "0:03:31", "throughput": "2588.49", "total_tokens": 3371440}
|
33 |
+
{"current_steps": 165, "total_steps": 186, "loss": 0.6484, "learning_rate": 1.5561966963229924e-06, "epoch": 2.64, "percentage": 88.71, "elapsed_time": "0:22:20", "remaining_time": "0:02:50", "throughput": "2591.26", "total_tokens": 3473680}
|
34 |
+
{"current_steps": 170, "total_steps": 186, "loss": 0.7137, "learning_rate": 9.073589027514789e-07, "epoch": 2.7199999999999998, "percentage": 91.4, "elapsed_time": "0:23:02", "remaining_time": "0:02:10", "throughput": "2585.69", "total_tokens": 3574736}
|
35 |
+
{"current_steps": 175, "total_steps": 186, "loss": 0.777, "learning_rate": 4.302487264785521e-07, "epoch": 2.8, "percentage": 94.09, "elapsed_time": "0:23:48", "remaining_time": "0:01:29", "throughput": "2589.29", "total_tokens": 3697920}
|
36 |
+
{"current_steps": 180, "total_steps": 186, "loss": 0.6529, "learning_rate": 1.2826691520262114e-07, "epoch": 2.88, "percentage": 96.77, "elapsed_time": "0:24:28", "remaining_time": "0:00:48", "throughput": "2590.50", "total_tokens": 3803552}
|
37 |
+
{"current_steps": 185, "total_steps": 186, "loss": 0.6996, "learning_rate": 3.565936007254855e-09, "epoch": 2.96, "percentage": 99.46, "elapsed_time": "0:25:08", "remaining_time": "0:00:08", "throughput": "2590.41", "total_tokens": 3907456}
|
38 |
+
{"current_steps": 186, "total_steps": 186, "epoch": 2.976, "percentage": 100.0, "elapsed_time": "0:25:18", "remaining_time": "0:00:00", "throughput": "2589.06", "total_tokens": 3930992}
|
trainer_state.json
ADDED
@@ -0,0 +1,339 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"best_metric": null,
|
3 |
+
"best_model_checkpoint": null,
|
4 |
+
"epoch": 2.976,
|
5 |
+
"eval_steps": 500,
|
6 |
+
"global_step": 186,
|
7 |
+
"is_hyper_param_search": false,
|
8 |
+
"is_local_process_zero": true,
|
9 |
+
"is_world_process_zero": true,
|
10 |
+
"log_history": [
|
11 |
+
{
|
12 |
+
"epoch": 0.08,
|
13 |
+
"grad_norm": 0.19458943605422974,
|
14 |
+
"learning_rate": 4.9910902453260824e-05,
|
15 |
+
"loss": 0.8099,
|
16 |
+
"num_input_tokens_seen": 114944,
|
17 |
+
"step": 5
|
18 |
+
},
|
19 |
+
{
|
20 |
+
"epoch": 0.16,
|
21 |
+
"grad_norm": 0.23063905537128448,
|
22 |
+
"learning_rate": 4.964424488287009e-05,
|
23 |
+
"loss": 0.958,
|
24 |
+
"num_input_tokens_seen": 227728,
|
25 |
+
"step": 10
|
26 |
+
},
|
27 |
+
{
|
28 |
+
"epoch": 0.24,
|
29 |
+
"grad_norm": 0.22587424516677856,
|
30 |
+
"learning_rate": 4.920192797165511e-05,
|
31 |
+
"loss": 0.715,
|
32 |
+
"num_input_tokens_seen": 326640,
|
33 |
+
"step": 15
|
34 |
+
},
|
35 |
+
{
|
36 |
+
"epoch": 0.32,
|
37 |
+
"grad_norm": 0.17264235019683838,
|
38 |
+
"learning_rate": 4.858710446774951e-05,
|
39 |
+
"loss": 0.7585,
|
40 |
+
"num_input_tokens_seen": 437280,
|
41 |
+
"step": 20
|
42 |
+
},
|
43 |
+
{
|
44 |
+
"epoch": 0.4,
|
45 |
+
"grad_norm": 0.19875957071781158,
|
46 |
+
"learning_rate": 4.780415671242334e-05,
|
47 |
+
"loss": 0.7342,
|
48 |
+
"num_input_tokens_seen": 530192,
|
49 |
+
"step": 25
|
50 |
+
},
|
51 |
+
{
|
52 |
+
"epoch": 0.48,
|
53 |
+
"grad_norm": 0.36600446701049805,
|
54 |
+
"learning_rate": 4.685866540361456e-05,
|
55 |
+
"loss": 0.6904,
|
56 |
+
"num_input_tokens_seen": 634960,
|
57 |
+
"step": 30
|
58 |
+
},
|
59 |
+
{
|
60 |
+
"epoch": 0.56,
|
61 |
+
"grad_norm": 0.20120520889759064,
|
62 |
+
"learning_rate": 4.5757369817809415e-05,
|
63 |
+
"loss": 0.8254,
|
64 |
+
"num_input_tokens_seen": 740704,
|
65 |
+
"step": 35
|
66 |
+
},
|
67 |
+
{
|
68 |
+
"epoch": 0.64,
|
69 |
+
"grad_norm": 0.22245633602142334,
|
70 |
+
"learning_rate": 4.45081197738023e-05,
|
71 |
+
"loss": 0.7551,
|
72 |
+
"num_input_tokens_seen": 841600,
|
73 |
+
"step": 40
|
74 |
+
},
|
75 |
+
{
|
76 |
+
"epoch": 0.72,
|
77 |
+
"grad_norm": 0.27814680337905884,
|
78 |
+
"learning_rate": 4.3119819680728e-05,
|
79 |
+
"loss": 0.7747,
|
80 |
+
"num_input_tokens_seen": 947648,
|
81 |
+
"step": 45
|
82 |
+
},
|
83 |
+
{
|
84 |
+
"epoch": 0.8,
|
85 |
+
"grad_norm": 0.27052441239356995,
|
86 |
+
"learning_rate": 4.160236506918098e-05,
|
87 |
+
"loss": 0.7027,
|
88 |
+
"num_input_tokens_seen": 1047520,
|
89 |
+
"step": 50
|
90 |
+
},
|
91 |
+
{
|
92 |
+
"epoch": 0.88,
|
93 |
+
"grad_norm": 0.38937926292419434,
|
94 |
+
"learning_rate": 3.9966572057815373e-05,
|
95 |
+
"loss": 0.7581,
|
96 |
+
"num_input_tokens_seen": 1148080,
|
97 |
+
"step": 55
|
98 |
+
},
|
99 |
+
{
|
100 |
+
"epoch": 0.96,
|
101 |
+
"grad_norm": 0.37379005551338196,
|
102 |
+
"learning_rate": 3.822410025817406e-05,
|
103 |
+
"loss": 0.7221,
|
104 |
+
"num_input_tokens_seen": 1252880,
|
105 |
+
"step": 60
|
106 |
+
},
|
107 |
+
{
|
108 |
+
"epoch": 1.04,
|
109 |
+
"grad_norm": 0.34213003516197205,
|
110 |
+
"learning_rate": 3.638736966726585e-05,
|
111 |
+
"loss": 0.8214,
|
112 |
+
"num_input_tokens_seen": 1364304,
|
113 |
+
"step": 65
|
114 |
+
},
|
115 |
+
{
|
116 |
+
"epoch": 1.12,
|
117 |
+
"grad_norm": 0.33407434821128845,
|
118 |
+
"learning_rate": 3.44694721402644e-05,
|
119 |
+
"loss": 0.6304,
|
120 |
+
"num_input_tokens_seen": 1466240,
|
121 |
+
"step": 70
|
122 |
+
},
|
123 |
+
{
|
124 |
+
"epoch": 1.2,
|
125 |
+
"grad_norm": 0.33909159898757935,
|
126 |
+
"learning_rate": 3.2484078074333954e-05,
|
127 |
+
"loss": 0.6434,
|
128 |
+
"num_input_tokens_seen": 1558752,
|
129 |
+
"step": 75
|
130 |
+
},
|
131 |
+
{
|
132 |
+
"epoch": 1.28,
|
133 |
+
"grad_norm": 0.4054814577102661,
|
134 |
+
"learning_rate": 3.0445338968721287e-05,
|
135 |
+
"loss": 0.6796,
|
136 |
+
"num_input_tokens_seen": 1668480,
|
137 |
+
"step": 80
|
138 |
+
},
|
139 |
+
{
|
140 |
+
"epoch": 1.3599999999999999,
|
141 |
+
"grad_norm": 0.2723485231399536,
|
142 |
+
"learning_rate": 2.836778655564653e-05,
|
143 |
+
"loss": 0.6651,
|
144 |
+
"num_input_tokens_seen": 1773264,
|
145 |
+
"step": 85
|
146 |
+
},
|
147 |
+
{
|
148 |
+
"epoch": 1.44,
|
149 |
+
"grad_norm": 0.3467393219470978,
|
150 |
+
"learning_rate": 2.6266229220967818e-05,
|
151 |
+
"loss": 0.7844,
|
152 |
+
"num_input_tokens_seen": 1885920,
|
153 |
+
"step": 90
|
154 |
+
},
|
155 |
+
{
|
156 |
+
"epoch": 1.52,
|
157 |
+
"grad_norm": 0.4250239431858063,
|
158 |
+
"learning_rate": 2.4155646452913296e-05,
|
159 |
+
"loss": 0.8139,
|
160 |
+
"num_input_tokens_seen": 1996656,
|
161 |
+
"step": 95
|
162 |
+
},
|
163 |
+
{
|
164 |
+
"epoch": 1.6,
|
165 |
+
"grad_norm": 0.30656036734580994,
|
166 |
+
"learning_rate": 2.2051082071228854e-05,
|
167 |
+
"loss": 0.6717,
|
168 |
+
"num_input_tokens_seen": 2102896,
|
169 |
+
"step": 100
|
170 |
+
},
|
171 |
+
{
|
172 |
+
"epoch": 1.6800000000000002,
|
173 |
+
"grad_norm": 0.49129152297973633,
|
174 |
+
"learning_rate": 1.9967536997783494e-05,
|
175 |
+
"loss": 0.7524,
|
176 |
+
"num_input_tokens_seen": 2214272,
|
177 |
+
"step": 105
|
178 |
+
},
|
179 |
+
{
|
180 |
+
"epoch": 1.76,
|
181 |
+
"grad_norm": 0.3904927670955658,
|
182 |
+
"learning_rate": 1.79198623329424e-05,
|
183 |
+
"loss": 0.631,
|
184 |
+
"num_input_tokens_seen": 2316448,
|
185 |
+
"step": 110
|
186 |
+
},
|
187 |
+
{
|
188 |
+
"epoch": 1.8399999999999999,
|
189 |
+
"grad_norm": 0.2829418182373047,
|
190 |
+
"learning_rate": 1.5922653499838137e-05,
|
191 |
+
"loss": 0.7462,
|
192 |
+
"num_input_tokens_seen": 2425248,
|
193 |
+
"step": 115
|
194 |
+
},
|
195 |
+
{
|
196 |
+
"epoch": 1.92,
|
197 |
+
"grad_norm": 0.29749077558517456,
|
198 |
+
"learning_rate": 1.399014621105914e-05,
|
199 |
+
"loss": 0.6148,
|
200 |
+
"num_input_tokens_seen": 2537680,
|
201 |
+
"step": 120
|
202 |
+
},
|
203 |
+
{
|
204 |
+
"epoch": 2.0,
|
205 |
+
"grad_norm": 0.38928714394569397,
|
206 |
+
"learning_rate": 1.2136114999284288e-05,
|
207 |
+
"loss": 0.7145,
|
208 |
+
"num_input_tokens_seen": 2642080,
|
209 |
+
"step": 125
|
210 |
+
},
|
211 |
+
{
|
212 |
+
"epoch": 2.08,
|
213 |
+
"grad_norm": 0.3223271369934082,
|
214 |
+
"learning_rate": 1.0373775035117305e-05,
|
215 |
+
"loss": 0.6798,
|
216 |
+
"num_input_tokens_seen": 2750752,
|
217 |
+
"step": 130
|
218 |
+
},
|
219 |
+
{
|
220 |
+
"epoch": 2.16,
|
221 |
+
"grad_norm": 0.29905465245246887,
|
222 |
+
"learning_rate": 8.715687931944449e-06,
|
223 |
+
"loss": 0.6754,
|
224 |
+
"num_input_tokens_seen": 2852096,
|
225 |
+
"step": 135
|
226 |
+
},
|
227 |
+
{
|
228 |
+
"epoch": 2.24,
|
229 |
+
"grad_norm": 0.3506278991699219,
|
230 |
+
"learning_rate": 7.173672209219495e-06,
|
231 |
+
"loss": 0.6708,
|
232 |
+
"num_input_tokens_seen": 2951888,
|
233 |
+
"step": 140
|
234 |
+
},
|
235 |
+
{
|
236 |
+
"epoch": 2.32,
|
237 |
+
"grad_norm": 0.3896699547767639,
|
238 |
+
"learning_rate": 5.758719052376693e-06,
|
239 |
+
"loss": 0.6386,
|
240 |
+
"num_input_tokens_seen": 3049616,
|
241 |
+
"step": 145
|
242 |
+
},
|
243 |
+
{
|
244 |
+
"epoch": 2.4,
|
245 |
+
"grad_norm": 0.5055662989616394,
|
246 |
+
"learning_rate": 4.480913969818098e-06,
|
247 |
+
"loss": 0.6995,
|
248 |
+
"num_input_tokens_seen": 3150784,
|
249 |
+
"step": 150
|
250 |
+
},
|
251 |
+
{
|
252 |
+
"epoch": 2.48,
|
253 |
+
"grad_norm": 0.30896279215812683,
|
254 |
+
"learning_rate": 3.3493649053890326e-06,
|
255 |
+
"loss": 0.6691,
|
256 |
+
"num_input_tokens_seen": 3262992,
|
257 |
+
"step": 155
|
258 |
+
},
|
259 |
+
{
|
260 |
+
"epoch": 2.56,
|
261 |
+
"grad_norm": 0.30288368463516235,
|
262 |
+
"learning_rate": 2.372137318741968e-06,
|
263 |
+
"loss": 0.6024,
|
264 |
+
"num_input_tokens_seen": 3371440,
|
265 |
+
"step": 160
|
266 |
+
},
|
267 |
+
{
|
268 |
+
"epoch": 2.64,
|
269 |
+
"grad_norm": 0.3581477999687195,
|
270 |
+
"learning_rate": 1.5561966963229924e-06,
|
271 |
+
"loss": 0.6484,
|
272 |
+
"num_input_tokens_seen": 3473680,
|
273 |
+
"step": 165
|
274 |
+
},
|
275 |
+
{
|
276 |
+
"epoch": 2.7199999999999998,
|
277 |
+
"grad_norm": 0.43826210498809814,
|
278 |
+
"learning_rate": 9.073589027514789e-07,
|
279 |
+
"loss": 0.7137,
|
280 |
+
"num_input_tokens_seen": 3574736,
|
281 |
+
"step": 170
|
282 |
+
},
|
283 |
+
{
|
284 |
+
"epoch": 2.8,
|
285 |
+
"grad_norm": 0.22761593759059906,
|
286 |
+
"learning_rate": 4.302487264785521e-07,
|
287 |
+
"loss": 0.777,
|
288 |
+
"num_input_tokens_seen": 3697920,
|
289 |
+
"step": 175
|
290 |
+
},
|
291 |
+
{
|
292 |
+
"epoch": 2.88,
|
293 |
+
"grad_norm": 0.3505564033985138,
|
294 |
+
"learning_rate": 1.2826691520262114e-07,
|
295 |
+
"loss": 0.6529,
|
296 |
+
"num_input_tokens_seen": 3803552,
|
297 |
+
"step": 180
|
298 |
+
},
|
299 |
+
{
|
300 |
+
"epoch": 2.96,
|
301 |
+
"grad_norm": 0.48998603224754333,
|
302 |
+
"learning_rate": 3.565936007254855e-09,
|
303 |
+
"loss": 0.6996,
|
304 |
+
"num_input_tokens_seen": 3907456,
|
305 |
+
"step": 185
|
306 |
+
},
|
307 |
+
{
|
308 |
+
"epoch": 2.976,
|
309 |
+
"num_input_tokens_seen": 3930992,
|
310 |
+
"step": 186,
|
311 |
+
"total_flos": 3.112338516554547e+16,
|
312 |
+
"train_loss": 0.7158093615885703,
|
313 |
+
"train_runtime": 1518.3826,
|
314 |
+
"train_samples_per_second": 3.952,
|
315 |
+
"train_steps_per_second": 0.122
|
316 |
+
}
|
317 |
+
],
|
318 |
+
"logging_steps": 5,
|
319 |
+
"max_steps": 186,
|
320 |
+
"num_input_tokens_seen": 3930992,
|
321 |
+
"num_train_epochs": 3,
|
322 |
+
"save_steps": 100,
|
323 |
+
"stateful_callbacks": {
|
324 |
+
"TrainerControl": {
|
325 |
+
"args": {
|
326 |
+
"should_epoch_stop": false,
|
327 |
+
"should_evaluate": false,
|
328 |
+
"should_log": false,
|
329 |
+
"should_save": true,
|
330 |
+
"should_training_stop": false
|
331 |
+
},
|
332 |
+
"attributes": {}
|
333 |
+
}
|
334 |
+
},
|
335 |
+
"total_flos": 3.112338516554547e+16,
|
336 |
+
"train_batch_size": 2,
|
337 |
+
"trial_name": null,
|
338 |
+
"trial_params": null
|
339 |
+
}
|
training_args.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:25fff679805feed7cb125ffaa9aa318c0b1ee7ef433e1a91fc410bc3dfcec03b
|
3 |
+
size 5304
|
training_args.yaml
ADDED
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
cutoff_len: 3024
|
2 |
+
dataset: glaive_toolcall_zh,glaive_toolcall_en
|
3 |
+
dataset_dir: data
|
4 |
+
ddp_timeout: 180000000
|
5 |
+
do_train: true
|
6 |
+
finetuning_type: lora
|
7 |
+
flash_attn: auto
|
8 |
+
fp16: true
|
9 |
+
gradient_accumulation_steps: 8
|
10 |
+
include_num_input_tokens_seen: true
|
11 |
+
learning_rate: 5.0e-05
|
12 |
+
logging_steps: 5
|
13 |
+
lora_alpha: 16
|
14 |
+
lora_dropout: 0
|
15 |
+
lora_rank: 8
|
16 |
+
lora_target: all
|
17 |
+
lr_scheduler_type: cosine
|
18 |
+
max_grad_norm: 1.0
|
19 |
+
max_samples: 100000
|
20 |
+
model_name_or_path: Qwen/Qwen2-1.5B-Instruct
|
21 |
+
num_train_epochs: 3.0
|
22 |
+
optim: adamw_torch
|
23 |
+
output_dir: saves/Qwen2-1.5B-Chat/lora/train_2024-06-17-19-49-05
|
24 |
+
packing: false
|
25 |
+
per_device_train_batch_size: 2
|
26 |
+
plot_loss: true
|
27 |
+
preprocessing_num_workers: 16
|
28 |
+
quantization_bit: 4
|
29 |
+
report_to: none
|
30 |
+
save_steps: 100
|
31 |
+
stage: sft
|
32 |
+
template: qwen
|
33 |
+
warmup_steps: 0
|
training_loss.png
ADDED
![]() |
vocab.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|