Jonathanmann commited on
Commit
65f3e4b
1 Parent(s): 15df590

Upload 15 files

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,3 +1,202 @@
1
- ---
2
- license: mit
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: Qwen/Qwen2.5-0.5B-Instruct
3
+ library_name: peft
4
+ ---
5
+
6
+ # Model Card for Model ID
7
+
8
+ <!-- Provide a quick summary of what the model is/does. -->
9
+
10
+
11
+
12
+ ## Model Details
13
+
14
+ ### Model Description
15
+
16
+ <!-- Provide a longer summary of what this model is. -->
17
+
18
+
19
+
20
+ - **Developed by:** [More Information Needed]
21
+ - **Funded by [optional]:** [More Information Needed]
22
+ - **Shared by [optional]:** [More Information Needed]
23
+ - **Model type:** [More Information Needed]
24
+ - **Language(s) (NLP):** [More Information Needed]
25
+ - **License:** [More Information Needed]
26
+ - **Finetuned from model [optional]:** [More Information Needed]
27
+
28
+ ### Model Sources [optional]
29
+
30
+ <!-- Provide the basic links for the model. -->
31
+
32
+ - **Repository:** [More Information Needed]
33
+ - **Paper [optional]:** [More Information Needed]
34
+ - **Demo [optional]:** [More Information Needed]
35
+
36
+ ## Uses
37
+
38
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
39
+
40
+ ### Direct Use
41
+
42
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
43
+
44
+ [More Information Needed]
45
+
46
+ ### Downstream Use [optional]
47
+
48
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
49
+
50
+ [More Information Needed]
51
+
52
+ ### Out-of-Scope Use
53
+
54
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
55
+
56
+ [More Information Needed]
57
+
58
+ ## Bias, Risks, and Limitations
59
+
60
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
61
+
62
+ [More Information Needed]
63
+
64
+ ### Recommendations
65
+
66
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
67
+
68
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
69
+
70
+ ## How to Get Started with the Model
71
+
72
+ Use the code below to get started with the model.
73
+
74
+ [More Information Needed]
75
+
76
+ ## Training Details
77
+
78
+ ### Training Data
79
+
80
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
81
+
82
+ [More Information Needed]
83
+
84
+ ### Training Procedure
85
+
86
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
87
+
88
+ #### Preprocessing [optional]
89
+
90
+ [More Information Needed]
91
+
92
+
93
+ #### Training Hyperparameters
94
+
95
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
96
+
97
+ #### Speeds, Sizes, Times [optional]
98
+
99
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
100
+
101
+ [More Information Needed]
102
+
103
+ ## Evaluation
104
+
105
+ <!-- This section describes the evaluation protocols and provides the results. -->
106
+
107
+ ### Testing Data, Factors & Metrics
108
+
109
+ #### Testing Data
110
+
111
+ <!-- This should link to a Dataset Card if possible. -->
112
+
113
+ [More Information Needed]
114
+
115
+ #### Factors
116
+
117
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
118
+
119
+ [More Information Needed]
120
+
121
+ #### Metrics
122
+
123
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
124
+
125
+ [More Information Needed]
126
+
127
+ ### Results
128
+
129
+ [More Information Needed]
130
+
131
+ #### Summary
132
+
133
+
134
+
135
+ ## Model Examination [optional]
136
+
137
+ <!-- Relevant interpretability work for the model goes here -->
138
+
139
+ [More Information Needed]
140
+
141
+ ## Environmental Impact
142
+
143
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
144
+
145
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
146
+
147
+ - **Hardware Type:** [More Information Needed]
148
+ - **Hours used:** [More Information Needed]
149
+ - **Cloud Provider:** [More Information Needed]
150
+ - **Compute Region:** [More Information Needed]
151
+ - **Carbon Emitted:** [More Information Needed]
152
+
153
+ ## Technical Specifications [optional]
154
+
155
+ ### Model Architecture and Objective
156
+
157
+ [More Information Needed]
158
+
159
+ ### Compute Infrastructure
160
+
161
+ [More Information Needed]
162
+
163
+ #### Hardware
164
+
165
+ [More Information Needed]
166
+
167
+ #### Software
168
+
169
+ [More Information Needed]
170
+
171
+ ## Citation [optional]
172
+
173
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
174
+
175
+ **BibTeX:**
176
+
177
+ [More Information Needed]
178
+
179
+ **APA:**
180
+
181
+ [More Information Needed]
182
+
183
+ ## Glossary [optional]
184
+
185
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
186
+
187
+ [More Information Needed]
188
+
189
+ ## More Information [optional]
190
+
191
+ [More Information Needed]
192
+
193
+ ## Model Card Authors [optional]
194
+
195
+ [More Information Needed]
196
+
197
+ ## Model Card Contact
198
+
199
+ [More Information Needed]
200
+ ### Framework versions
201
+
202
+ - PEFT 0.13.2
adapter_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 32,
14
+ "lora_dropout": 0.05,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 16,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "q_proj",
24
+ "v_proj"
25
+ ],
26
+ "task_type": "CAUSAL_LM",
27
+ "use_dora": false,
28
+ "use_rslora": false
29
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3dcab8abc876b999610c4aadc77d0dd9b4c4da17dcbf5122f884598f9770e303
3
+ size 4338000
added_tokens.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</tool_call>": 151658,
3
+ "<tool_call>": 151657,
4
+ "<|box_end|>": 151649,
5
+ "<|box_start|>": 151648,
6
+ "<|endoftext|>": 151643,
7
+ "<|file_sep|>": 151664,
8
+ "<|fim_middle|>": 151660,
9
+ "<|fim_pad|>": 151662,
10
+ "<|fim_prefix|>": 151659,
11
+ "<|fim_suffix|>": 151661,
12
+ "<|im_end|>": 151645,
13
+ "<|im_start|>": 151644,
14
+ "<|image_pad|>": 151655,
15
+ "<|object_ref_end|>": 151647,
16
+ "<|object_ref_start|>": 151646,
17
+ "<|quad_end|>": 151651,
18
+ "<|quad_start|>": 151650,
19
+ "<|repo_name|>": 151663,
20
+ "<|video_pad|>": 151656,
21
+ "<|vision_end|>": 151653,
22
+ "<|vision_pad|>": 151654,
23
+ "<|vision_start|>": 151652
24
+ }
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_attn_implementation_autoset": true,
3
+ "_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct",
4
+ "architectures": [
5
+ "Qwen2ForCausalLM"
6
+ ],
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 151643,
9
+ "eos_token_id": 151645,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 896,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 4864,
14
+ "max_position_embeddings": 32768,
15
+ "max_window_layers": 21,
16
+ "model_type": "qwen2",
17
+ "num_attention_heads": 14,
18
+ "num_hidden_layers": 24,
19
+ "num_key_value_heads": 2,
20
+ "rms_norm_eps": 1e-06,
21
+ "rope_scaling": null,
22
+ "rope_theta": 1000000.0,
23
+ "sliding_window": null,
24
+ "tie_word_embeddings": true,
25
+ "torch_dtype": "float16",
26
+ "transformers_version": "4.46.2",
27
+ "use_cache": true,
28
+ "use_sliding_window": false,
29
+ "vocab_size": 151936
30
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:83cbd6826a25012aca6b3fc2198715c0d7d845cae8c096c465b62d891fd74f4c
3
+ size 8731578
rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab10ba4ccdc006f4c1bd0f55e550e6e7d5edd15b106bacff0ae88cfc475a7cdd
3
+ size 14308
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f29577abc3e2ab5cf838c25315d40a1eadbdd259bd1f83c8f11d1d273b2dde37
3
+ size 1064
special_tokens_map.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "pad_token": "<|im_end|>",
3
+ "eos_token": "<|im_end|>",
4
+ "unk_token": null
5
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
3
+ size 11421896
tokenizer_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_file": "/root/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/vocab.json",
3
+ "merges_file": "/root/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/merges.txt",
4
+ "unk_token": null,
5
+ "bos_token": null,
6
+ "eos_token": "<|im_end|>",
7
+ "pad_token": "<|im_end|>",
8
+ "add_bos_token": false,
9
+ "add_prefix_space": false,
10
+ "additional_special_tokens": [
11
+ "<|im_start|>",
12
+ "<|im_end|>",
13
+ "<|object_ref_start|>",
14
+ "<|object_ref_end|>",
15
+ "<|box_start|>",
16
+ "<|box_end|>",
17
+ "<|quad_start|>",
18
+ "<|quad_end|>",
19
+ "<|vision_start|>",
20
+ "<|vision_end|>",
21
+ "<|vision_pad|>",
22
+ "<|image_pad|>",
23
+ "<|video_pad|>"
24
+ ],
25
+ "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
26
+ "clean_up_tokenization_spaces": false,
27
+ "errors": "replace",
28
+ "model_max_length": 131072,
29
+ "split_special_tokens": false,
30
+ "name_or_path": "Qwen/Qwen2.5-0.5B-Instruct",
31
+ "padding_side": "right"
32
+ }
trainer_state.json ADDED
@@ -0,0 +1,622 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 5.0,
5
+ "eval_steps": 50,
6
+ "global_step": 685,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.072992700729927,
13
+ "grad_norm": 1.2194373607635498,
14
+ "learning_rate": 5.000000000000001e-07,
15
+ "loss": 3.5968,
16
+ "step": 10
17
+ },
18
+ {
19
+ "epoch": 0.145985401459854,
20
+ "grad_norm": 1.1968055963516235,
21
+ "learning_rate": 1.0000000000000002e-06,
22
+ "loss": 3.6726,
23
+ "step": 20
24
+ },
25
+ {
26
+ "epoch": 0.21897810218978103,
27
+ "grad_norm": 1.1814980506896973,
28
+ "learning_rate": 1.5e-06,
29
+ "loss": 3.5953,
30
+ "step": 30
31
+ },
32
+ {
33
+ "epoch": 0.291970802919708,
34
+ "grad_norm": 1.1611206531524658,
35
+ "learning_rate": 2.0000000000000003e-06,
36
+ "loss": 3.5831,
37
+ "step": 40
38
+ },
39
+ {
40
+ "epoch": 0.36496350364963503,
41
+ "grad_norm": 1.0345042943954468,
42
+ "learning_rate": 2.5e-06,
43
+ "loss": 3.6277,
44
+ "step": 50
45
+ },
46
+ {
47
+ "epoch": 0.36496350364963503,
48
+ "eval_loss": 3.6583080291748047,
49
+ "eval_runtime": 4.5719,
50
+ "eval_samples_per_second": 106.74,
51
+ "eval_steps_per_second": 13.342,
52
+ "step": 50
53
+ },
54
+ {
55
+ "epoch": 0.43795620437956206,
56
+ "grad_norm": 1.1489139795303345,
57
+ "learning_rate": 3e-06,
58
+ "loss": 3.5835,
59
+ "step": 60
60
+ },
61
+ {
62
+ "epoch": 0.5109489051094891,
63
+ "grad_norm": 1.3421690464019775,
64
+ "learning_rate": 3.5e-06,
65
+ "loss": 3.5992,
66
+ "step": 70
67
+ },
68
+ {
69
+ "epoch": 0.583941605839416,
70
+ "grad_norm": 1.0778014659881592,
71
+ "learning_rate": 4.000000000000001e-06,
72
+ "loss": 3.5575,
73
+ "step": 80
74
+ },
75
+ {
76
+ "epoch": 0.656934306569343,
77
+ "grad_norm": 0.9279437065124512,
78
+ "learning_rate": 4.5e-06,
79
+ "loss": 3.6968,
80
+ "step": 90
81
+ },
82
+ {
83
+ "epoch": 0.7299270072992701,
84
+ "grad_norm": 0.8781314492225647,
85
+ "learning_rate": 5e-06,
86
+ "loss": 3.5835,
87
+ "step": 100
88
+ },
89
+ {
90
+ "epoch": 0.7299270072992701,
91
+ "eval_loss": 3.6277682781219482,
92
+ "eval_runtime": 4.5628,
93
+ "eval_samples_per_second": 106.953,
94
+ "eval_steps_per_second": 13.369,
95
+ "step": 100
96
+ },
97
+ {
98
+ "epoch": 0.8029197080291971,
99
+ "grad_norm": 1.237952709197998,
100
+ "learning_rate": 5.500000000000001e-06,
101
+ "loss": 3.6226,
102
+ "step": 110
103
+ },
104
+ {
105
+ "epoch": 0.8759124087591241,
106
+ "grad_norm": 1.1926833391189575,
107
+ "learning_rate": 6e-06,
108
+ "loss": 3.6014,
109
+ "step": 120
110
+ },
111
+ {
112
+ "epoch": 0.948905109489051,
113
+ "grad_norm": 0.8673391938209534,
114
+ "learning_rate": 6.5000000000000004e-06,
115
+ "loss": 3.6303,
116
+ "step": 130
117
+ },
118
+ {
119
+ "epoch": 1.0218978102189782,
120
+ "grad_norm": 1.0185987949371338,
121
+ "learning_rate": 7e-06,
122
+ "loss": 3.5962,
123
+ "step": 140
124
+ },
125
+ {
126
+ "epoch": 1.094890510948905,
127
+ "grad_norm": 0.8746767044067383,
128
+ "learning_rate": 7.500000000000001e-06,
129
+ "loss": 3.5365,
130
+ "step": 150
131
+ },
132
+ {
133
+ "epoch": 1.094890510948905,
134
+ "eval_loss": 3.578197956085205,
135
+ "eval_runtime": 4.5695,
136
+ "eval_samples_per_second": 106.796,
137
+ "eval_steps_per_second": 13.349,
138
+ "step": 150
139
+ },
140
+ {
141
+ "epoch": 1.167883211678832,
142
+ "grad_norm": 0.9256734848022461,
143
+ "learning_rate": 8.000000000000001e-06,
144
+ "loss": 3.5504,
145
+ "step": 160
146
+ },
147
+ {
148
+ "epoch": 1.2408759124087592,
149
+ "grad_norm": 0.7662177681922913,
150
+ "learning_rate": 8.5e-06,
151
+ "loss": 3.5484,
152
+ "step": 170
153
+ },
154
+ {
155
+ "epoch": 1.313868613138686,
156
+ "grad_norm": 0.7936022877693176,
157
+ "learning_rate": 9e-06,
158
+ "loss": 3.547,
159
+ "step": 180
160
+ },
161
+ {
162
+ "epoch": 1.3868613138686132,
163
+ "grad_norm": 0.9237962961196899,
164
+ "learning_rate": 9.5e-06,
165
+ "loss": 3.5648,
166
+ "step": 190
167
+ },
168
+ {
169
+ "epoch": 1.4598540145985401,
170
+ "grad_norm": 0.7705855369567871,
171
+ "learning_rate": 1e-05,
172
+ "loss": 3.4902,
173
+ "step": 200
174
+ },
175
+ {
176
+ "epoch": 1.4598540145985401,
177
+ "eval_loss": 3.515073299407959,
178
+ "eval_runtime": 4.5781,
179
+ "eval_samples_per_second": 106.595,
180
+ "eval_steps_per_second": 13.324,
181
+ "step": 200
182
+ },
183
+ {
184
+ "epoch": 1.5328467153284673,
185
+ "grad_norm": 0.7808473110198975,
186
+ "learning_rate": 9.793814432989691e-06,
187
+ "loss": 3.4803,
188
+ "step": 210
189
+ },
190
+ {
191
+ "epoch": 1.6058394160583942,
192
+ "grad_norm": 0.8358160853385925,
193
+ "learning_rate": 9.587628865979383e-06,
194
+ "loss": 3.4905,
195
+ "step": 220
196
+ },
197
+ {
198
+ "epoch": 1.6788321167883211,
199
+ "grad_norm": 0.8808528184890747,
200
+ "learning_rate": 9.381443298969073e-06,
201
+ "loss": 3.4518,
202
+ "step": 230
203
+ },
204
+ {
205
+ "epoch": 1.7518248175182483,
206
+ "grad_norm": 0.6673774719238281,
207
+ "learning_rate": 9.175257731958764e-06,
208
+ "loss": 3.4631,
209
+ "step": 240
210
+ },
211
+ {
212
+ "epoch": 1.8248175182481752,
213
+ "grad_norm": 0.7009378671646118,
214
+ "learning_rate": 8.969072164948455e-06,
215
+ "loss": 3.4264,
216
+ "step": 250
217
+ },
218
+ {
219
+ "epoch": 1.8248175182481752,
220
+ "eval_loss": 3.4590413570404053,
221
+ "eval_runtime": 4.6618,
222
+ "eval_samples_per_second": 104.68,
223
+ "eval_steps_per_second": 13.085,
224
+ "step": 250
225
+ },
226
+ {
227
+ "epoch": 1.897810218978102,
228
+ "grad_norm": 0.6876445412635803,
229
+ "learning_rate": 8.762886597938146e-06,
230
+ "loss": 3.398,
231
+ "step": 260
232
+ },
233
+ {
234
+ "epoch": 1.9708029197080292,
235
+ "grad_norm": 0.5596441626548767,
236
+ "learning_rate": 8.556701030927836e-06,
237
+ "loss": 3.4173,
238
+ "step": 270
239
+ },
240
+ {
241
+ "epoch": 2.0437956204379564,
242
+ "grad_norm": 0.5952299237251282,
243
+ "learning_rate": 8.350515463917526e-06,
244
+ "loss": 3.3986,
245
+ "step": 280
246
+ },
247
+ {
248
+ "epoch": 2.116788321167883,
249
+ "grad_norm": 0.6578339338302612,
250
+ "learning_rate": 8.144329896907216e-06,
251
+ "loss": 3.3949,
252
+ "step": 290
253
+ },
254
+ {
255
+ "epoch": 2.18978102189781,
256
+ "grad_norm": 0.7148367166519165,
257
+ "learning_rate": 7.938144329896907e-06,
258
+ "loss": 3.3845,
259
+ "step": 300
260
+ },
261
+ {
262
+ "epoch": 2.18978102189781,
263
+ "eval_loss": 3.4218263626098633,
264
+ "eval_runtime": 4.6207,
265
+ "eval_samples_per_second": 105.611,
266
+ "eval_steps_per_second": 13.201,
267
+ "step": 300
268
+ },
269
+ {
270
+ "epoch": 2.2627737226277373,
271
+ "grad_norm": 0.5892521142959595,
272
+ "learning_rate": 7.731958762886599e-06,
273
+ "loss": 3.3821,
274
+ "step": 310
275
+ },
276
+ {
277
+ "epoch": 2.335766423357664,
278
+ "grad_norm": 0.6282438635826111,
279
+ "learning_rate": 7.525773195876289e-06,
280
+ "loss": 3.3522,
281
+ "step": 320
282
+ },
283
+ {
284
+ "epoch": 2.408759124087591,
285
+ "grad_norm": 0.5816407799720764,
286
+ "learning_rate": 7.319587628865979e-06,
287
+ "loss": 3.3891,
288
+ "step": 330
289
+ },
290
+ {
291
+ "epoch": 2.4817518248175183,
292
+ "grad_norm": 0.7334665656089783,
293
+ "learning_rate": 7.113402061855671e-06,
294
+ "loss": 3.3682,
295
+ "step": 340
296
+ },
297
+ {
298
+ "epoch": 2.554744525547445,
299
+ "grad_norm": 0.667533278465271,
300
+ "learning_rate": 6.907216494845361e-06,
301
+ "loss": 3.4053,
302
+ "step": 350
303
+ },
304
+ {
305
+ "epoch": 2.554744525547445,
306
+ "eval_loss": 3.3971924781799316,
307
+ "eval_runtime": 4.558,
308
+ "eval_samples_per_second": 107.065,
309
+ "eval_steps_per_second": 13.383,
310
+ "step": 350
311
+ },
312
+ {
313
+ "epoch": 2.627737226277372,
314
+ "grad_norm": 0.7920149564743042,
315
+ "learning_rate": 6.701030927835052e-06,
316
+ "loss": 3.3418,
317
+ "step": 360
318
+ },
319
+ {
320
+ "epoch": 2.7007299270072993,
321
+ "grad_norm": 0.6718395352363586,
322
+ "learning_rate": 6.494845360824743e-06,
323
+ "loss": 3.379,
324
+ "step": 370
325
+ },
326
+ {
327
+ "epoch": 2.7737226277372264,
328
+ "grad_norm": 0.6475045680999756,
329
+ "learning_rate": 6.288659793814433e-06,
330
+ "loss": 3.3975,
331
+ "step": 380
332
+ },
333
+ {
334
+ "epoch": 2.846715328467153,
335
+ "grad_norm": 0.7203289270401001,
336
+ "learning_rate": 6.082474226804124e-06,
337
+ "loss": 3.4352,
338
+ "step": 390
339
+ },
340
+ {
341
+ "epoch": 2.9197080291970803,
342
+ "grad_norm": 0.5514132976531982,
343
+ "learning_rate": 5.876288659793815e-06,
344
+ "loss": 3.3763,
345
+ "step": 400
346
+ },
347
+ {
348
+ "epoch": 2.9197080291970803,
349
+ "eval_loss": 3.3808703422546387,
350
+ "eval_runtime": 4.5435,
351
+ "eval_samples_per_second": 107.407,
352
+ "eval_steps_per_second": 13.426,
353
+ "step": 400
354
+ },
355
+ {
356
+ "epoch": 2.9927007299270074,
357
+ "grad_norm": 0.5394614934921265,
358
+ "learning_rate": 5.670103092783505e-06,
359
+ "loss": 3.3935,
360
+ "step": 410
361
+ },
362
+ {
363
+ "epoch": 3.065693430656934,
364
+ "grad_norm": 0.6785652041435242,
365
+ "learning_rate": 5.463917525773196e-06,
366
+ "loss": 3.3154,
367
+ "step": 420
368
+ },
369
+ {
370
+ "epoch": 3.1386861313868613,
371
+ "grad_norm": 0.546085774898529,
372
+ "learning_rate": 5.257731958762888e-06,
373
+ "loss": 3.323,
374
+ "step": 430
375
+ },
376
+ {
377
+ "epoch": 3.2116788321167884,
378
+ "grad_norm": 0.7874196171760559,
379
+ "learning_rate": 5.051546391752578e-06,
380
+ "loss": 3.3889,
381
+ "step": 440
382
+ },
383
+ {
384
+ "epoch": 3.2846715328467155,
385
+ "grad_norm": 0.5360156297683716,
386
+ "learning_rate": 4.845360824742268e-06,
387
+ "loss": 3.3871,
388
+ "step": 450
389
+ },
390
+ {
391
+ "epoch": 3.2846715328467155,
392
+ "eval_loss": 3.3709511756896973,
393
+ "eval_runtime": 4.5554,
394
+ "eval_samples_per_second": 107.126,
395
+ "eval_steps_per_second": 13.391,
396
+ "step": 450
397
+ },
398
+ {
399
+ "epoch": 3.3576642335766422,
400
+ "grad_norm": 0.6126905083656311,
401
+ "learning_rate": 4.639175257731959e-06,
402
+ "loss": 3.4215,
403
+ "step": 460
404
+ },
405
+ {
406
+ "epoch": 3.4306569343065694,
407
+ "grad_norm": 0.7156729102134705,
408
+ "learning_rate": 4.4329896907216494e-06,
409
+ "loss": 3.3448,
410
+ "step": 470
411
+ },
412
+ {
413
+ "epoch": 3.5036496350364965,
414
+ "grad_norm": 0.5565685629844666,
415
+ "learning_rate": 4.2268041237113405e-06,
416
+ "loss": 3.3041,
417
+ "step": 480
418
+ },
419
+ {
420
+ "epoch": 3.576642335766423,
421
+ "grad_norm": 0.6424775123596191,
422
+ "learning_rate": 4.020618556701032e-06,
423
+ "loss": 3.3398,
424
+ "step": 490
425
+ },
426
+ {
427
+ "epoch": 3.6496350364963503,
428
+ "grad_norm": 0.5503811836242676,
429
+ "learning_rate": 3.814432989690722e-06,
430
+ "loss": 3.3639,
431
+ "step": 500
432
+ },
433
+ {
434
+ "epoch": 3.6496350364963503,
435
+ "eval_loss": 3.364408493041992,
436
+ "eval_runtime": 4.5774,
437
+ "eval_samples_per_second": 106.611,
438
+ "eval_steps_per_second": 13.326,
439
+ "step": 500
440
+ },
441
+ {
442
+ "epoch": 3.7226277372262775,
443
+ "grad_norm": 0.8552574515342712,
444
+ "learning_rate": 3.6082474226804126e-06,
445
+ "loss": 3.3706,
446
+ "step": 510
447
+ },
448
+ {
449
+ "epoch": 3.795620437956204,
450
+ "grad_norm": 0.6950764060020447,
451
+ "learning_rate": 3.4020618556701037e-06,
452
+ "loss": 3.3846,
453
+ "step": 520
454
+ },
455
+ {
456
+ "epoch": 3.8686131386861313,
457
+ "grad_norm": 0.5460401177406311,
458
+ "learning_rate": 3.195876288659794e-06,
459
+ "loss": 3.3343,
460
+ "step": 530
461
+ },
462
+ {
463
+ "epoch": 3.9416058394160585,
464
+ "grad_norm": 0.5183790326118469,
465
+ "learning_rate": 2.9896907216494846e-06,
466
+ "loss": 3.3234,
467
+ "step": 540
468
+ },
469
+ {
470
+ "epoch": 4.014598540145985,
471
+ "grad_norm": 0.6364532113075256,
472
+ "learning_rate": 2.7835051546391757e-06,
473
+ "loss": 3.2898,
474
+ "step": 550
475
+ },
476
+ {
477
+ "epoch": 4.014598540145985,
478
+ "eval_loss": 3.36006236076355,
479
+ "eval_runtime": 4.5449,
480
+ "eval_samples_per_second": 107.373,
481
+ "eval_steps_per_second": 13.422,
482
+ "step": 550
483
+ },
484
+ {
485
+ "epoch": 4.087591240875913,
486
+ "grad_norm": 0.6407883763313293,
487
+ "learning_rate": 2.577319587628866e-06,
488
+ "loss": 3.3432,
489
+ "step": 560
490
+ },
491
+ {
492
+ "epoch": 4.160583941605839,
493
+ "grad_norm": 0.7432695031166077,
494
+ "learning_rate": 2.3711340206185566e-06,
495
+ "loss": 3.3128,
496
+ "step": 570
497
+ },
498
+ {
499
+ "epoch": 4.233576642335766,
500
+ "grad_norm": 0.5630599856376648,
501
+ "learning_rate": 2.1649484536082477e-06,
502
+ "loss": 3.3308,
503
+ "step": 580
504
+ },
505
+ {
506
+ "epoch": 4.306569343065694,
507
+ "grad_norm": 0.691064178943634,
508
+ "learning_rate": 1.9587628865979384e-06,
509
+ "loss": 3.3607,
510
+ "step": 590
511
+ },
512
+ {
513
+ "epoch": 4.37956204379562,
514
+ "grad_norm": 0.6786036491394043,
515
+ "learning_rate": 1.7525773195876288e-06,
516
+ "loss": 3.3413,
517
+ "step": 600
518
+ },
519
+ {
520
+ "epoch": 4.37956204379562,
521
+ "eval_loss": 3.3572518825531006,
522
+ "eval_runtime": 4.5354,
523
+ "eval_samples_per_second": 107.597,
524
+ "eval_steps_per_second": 13.45,
525
+ "step": 600
526
+ },
527
+ {
528
+ "epoch": 4.452554744525547,
529
+ "grad_norm": 0.5392524600028992,
530
+ "learning_rate": 1.5463917525773197e-06,
531
+ "loss": 3.3262,
532
+ "step": 610
533
+ },
534
+ {
535
+ "epoch": 4.525547445255475,
536
+ "grad_norm": 0.6452904343605042,
537
+ "learning_rate": 1.3402061855670104e-06,
538
+ "loss": 3.3672,
539
+ "step": 620
540
+ },
541
+ {
542
+ "epoch": 4.598540145985401,
543
+ "grad_norm": 0.6458753347396851,
544
+ "learning_rate": 1.134020618556701e-06,
545
+ "loss": 3.3539,
546
+ "step": 630
547
+ },
548
+ {
549
+ "epoch": 4.671532846715328,
550
+ "grad_norm": 0.579394519329071,
551
+ "learning_rate": 9.278350515463919e-07,
552
+ "loss": 3.3627,
553
+ "step": 640
554
+ },
555
+ {
556
+ "epoch": 4.744525547445256,
557
+ "grad_norm": 0.5088974237442017,
558
+ "learning_rate": 7.216494845360824e-07,
559
+ "loss": 3.3574,
560
+ "step": 650
561
+ },
562
+ {
563
+ "epoch": 4.744525547445256,
564
+ "eval_loss": 3.3557839393615723,
565
+ "eval_runtime": 4.5731,
566
+ "eval_samples_per_second": 106.712,
567
+ "eval_steps_per_second": 13.339,
568
+ "step": 650
569
+ },
570
+ {
571
+ "epoch": 4.817518248175182,
572
+ "grad_norm": 0.5745378136634827,
573
+ "learning_rate": 5.154639175257732e-07,
574
+ "loss": 3.3426,
575
+ "step": 660
576
+ },
577
+ {
578
+ "epoch": 4.89051094890511,
579
+ "grad_norm": 0.5284786820411682,
580
+ "learning_rate": 3.0927835051546394e-07,
581
+ "loss": 3.3406,
582
+ "step": 670
583
+ },
584
+ {
585
+ "epoch": 4.963503649635037,
586
+ "grad_norm": 0.5814775824546814,
587
+ "learning_rate": 1.0309278350515465e-07,
588
+ "loss": 3.3041,
589
+ "step": 680
590
+ },
591
+ {
592
+ "epoch": 5.0,
593
+ "step": 685,
594
+ "total_flos": 2.417304476319744e+16,
595
+ "train_loss": 3.435254694249508,
596
+ "train_runtime": 659.3834,
597
+ "train_samples_per_second": 33.243,
598
+ "train_steps_per_second": 1.039
599
+ }
600
+ ],
601
+ "logging_steps": 10,
602
+ "max_steps": 685,
603
+ "num_input_tokens_seen": 0,
604
+ "num_train_epochs": 5,
605
+ "save_steps": 1000,
606
+ "stateful_callbacks": {
607
+ "TrainerControl": {
608
+ "args": {
609
+ "should_epoch_stop": false,
610
+ "should_evaluate": false,
611
+ "should_log": false,
612
+ "should_save": true,
613
+ "should_training_stop": true
614
+ },
615
+ "attributes": {}
616
+ }
617
+ },
618
+ "total_flos": 2.417304476319744e+16,
619
+ "train_batch_size": 8,
620
+ "trial_name": null,
621
+ "trial_params": null
622
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a575db8ad49dd5b77a127cc3f20536efe2623aa63788ef389deef7b8298dd6a6
3
+ size 5368
vocab.json ADDED
The diff for this file is too large to render. See raw diff