MaziyarPanahi commited on
Commit
38987b3
1 Parent(s): 9d37689

61811f7651c2ad5d8b23f167ed531f5a0c969bfd7c31f342a76c3988e4ba9d95

Browse files
README.md ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ base_model: mistralai/Mistral-7B-Instruct-v0.2
4
+ tags:
5
+ - axolotl
6
+ - generated_from_trainer
7
+ model-index:
8
+ - name: Mistral-7B-Instruct-KhanAcademy-v0.2
9
+ results: []
10
+ ---
11
+
12
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
13
+ should probably proofread and complete it, then remove this comment. -->
14
+
15
+ [<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
16
+ <details><summary>See axolotl config</summary>
17
+
18
+ axolotl version: `0.4.0`
19
+ ```yaml
20
+ base_model: mistralai/Mistral-7B-Instruct-v0.2
21
+ model_type: MistralForCausalLM
22
+ tokenizer_type: LlamaTokenizer
23
+ is_mistral_derived_model: true
24
+
25
+
26
+ hub_model_id: MaziyarPanahi/Mistral-7B-Instruct-KhanAcademy-v0.2
27
+ hf_use_auth_token: true
28
+
29
+ load_in_8bit: false
30
+ load_in_4bit: false
31
+ strict: false
32
+
33
+ datasets:
34
+ - path: CohereForAI/aya_dataset
35
+ type:
36
+ system_prompt: ""
37
+ field_instruction: inputs
38
+ field_output: targets
39
+ format: "[INST] {instruction} [/INST]"
40
+ no_input_format: "[INST] {instruction} [/INST]"
41
+
42
+ dataset_prepared_path:
43
+ val_set_size: 0.05
44
+ output_dir: ./models/MaziyarPanahi/Mistral-7B-Instruct-Aya-101
45
+
46
+ sequence_len: 8192
47
+ sample_packing: true
48
+ pad_to_sequence_len: true
49
+ eval_sample_packing: false
50
+
51
+ wandb_project:
52
+ wandb_entity:
53
+ wandb_watch:
54
+ wandb_name:
55
+ wandb_log_model:
56
+
57
+ gradient_accumulation_steps: 4
58
+ micro_batch_size: 2
59
+ num_epochs: 1
60
+ optimizer: adamw_bnb_8bit
61
+ lr_scheduler: cosine
62
+ learning_rate: 0.000005
63
+
64
+ train_on_inputs: false
65
+ group_by_length: false
66
+ bf16: auto
67
+ fp16:
68
+ tf32: false
69
+
70
+ gradient_checkpointing: true
71
+ early_stopping_patience:
72
+ resume_from_checkpoint:
73
+ local_rank:
74
+ logging_steps: 1
75
+ xformers_attention:
76
+ flash_attention: true
77
+
78
+ warmup_steps: 10
79
+ evals_per_epoch: 4
80
+ eval_table_size:
81
+ eval_max_new_tokens: 128
82
+ saves_per_epoch: 1
83
+ debug:
84
+ deepspeed:
85
+ weight_decay: 0.0
86
+ fsdp:
87
+ fsdp_config:
88
+ special_tokens:
89
+ bos_token: "<s>"
90
+ eos_token: "</s>"
91
+ unk_token: "<unk>"
92
+ ```
93
+
94
+ </details><br>
95
+
96
+ # Mistral-7B-Instruct-KhanAcademy-v0.2
97
+
98
+ This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on the None dataset.
99
+ It achieves the following results on the evaluation set:
100
+ - Loss: 1.1502
101
+
102
+ ## Model description
103
+
104
+ More information needed
105
+
106
+ ## Intended uses & limitations
107
+
108
+ More information needed
109
+
110
+ ## Training and evaluation data
111
+
112
+ More information needed
113
+
114
+ ## Training procedure
115
+
116
+ ### Training hyperparameters
117
+
118
+ The following hyperparameters were used during training:
119
+ - learning_rate: 5e-06
120
+ - train_batch_size: 2
121
+ - eval_batch_size: 2
122
+ - seed: 42
123
+ - distributed_type: multi-GPU
124
+ - num_devices: 4
125
+ - gradient_accumulation_steps: 4
126
+ - total_train_batch_size: 32
127
+ - total_eval_batch_size: 8
128
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
129
+ - lr_scheduler_type: cosine
130
+ - lr_scheduler_warmup_steps: 10
131
+ - num_epochs: 1
132
+
133
+ ### Training results
134
+
135
+ | Training Loss | Epoch | Step | Validation Loss |
136
+ |:-------------:|:-----:|:----:|:---------------:|
137
+ | 1.9039 | 0.0 | 1 | 3.1495 |
138
+ | 0.9933 | 0.25 | 101 | 1.2402 |
139
+ | 0.9439 | 0.5 | 202 | 1.1683 |
140
+ | 0.9762 | 0.75 | 303 | 1.1502 |
141
+
142
+
143
+ ### Framework versions
144
+
145
+ - Transformers 4.39.0.dev0
146
+ - Pytorch 2.2.0+cu121
147
+ - Datasets 2.17.0
148
+ - Tokenizers 0.15.0
checkpoint-401/config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
3
+ "architectures": [
4
+ "MistralForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 4096,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 14336,
13
+ "max_position_embeddings": 32768,
14
+ "model_type": "mistral",
15
+ "num_attention_heads": 32,
16
+ "num_hidden_layers": 32,
17
+ "num_key_value_heads": 8,
18
+ "rms_norm_eps": 1e-05,
19
+ "rope_theta": 1000000.0,
20
+ "sliding_window": null,
21
+ "tie_word_embeddings": false,
22
+ "torch_dtype": "bfloat16",
23
+ "transformers_version": "4.39.0.dev0",
24
+ "use_cache": false,
25
+ "vocab_size": 32000
26
+ }
checkpoint-401/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "do_sample": true,
5
+ "eos_token_id": 2,
6
+ "transformers_version": "4.39.0.dev0"
7
+ }
checkpoint-401/model.safetensors.index.json ADDED
@@ -0,0 +1,298 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 14483464192
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "model-00003-of-00003.safetensors",
7
+ "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
8
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
9
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
10
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
11
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
12
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
13
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
14
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
15
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
16
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
17
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
18
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
19
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
20
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
21
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
22
+ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
23
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
24
+ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
25
+ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
26
+ "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
27
+ "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
28
+ "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
29
+ "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
30
+ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
31
+ "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
32
+ "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
33
+ "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
34
+ "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
35
+ "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
36
+ "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
37
+ "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
38
+ "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
39
+ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
40
+ "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
41
+ "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
42
+ "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
43
+ "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
44
+ "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
45
+ "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
46
+ "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
47
+ "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
48
+ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
49
+ "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
50
+ "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
51
+ "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
52
+ "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
53
+ "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
54
+ "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
55
+ "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
56
+ "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
57
+ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
58
+ "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
59
+ "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
60
+ "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
61
+ "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
62
+ "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
63
+ "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
64
+ "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
65
+ "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
66
+ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
67
+ "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
68
+ "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
69
+ "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
70
+ "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
71
+ "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
72
+ "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
73
+ "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
74
+ "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
75
+ "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
76
+ "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
77
+ "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
78
+ "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
79
+ "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
80
+ "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
81
+ "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
82
+ "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
83
+ "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
84
+ "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
85
+ "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
86
+ "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
87
+ "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
88
+ "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
89
+ "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
90
+ "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
91
+ "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
92
+ "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
93
+ "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
94
+ "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
95
+ "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
96
+ "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
97
+ "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
98
+ "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
99
+ "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
100
+ "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
101
+ "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
102
+ "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
103
+ "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
104
+ "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
105
+ "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
106
+ "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
107
+ "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
108
+ "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
109
+ "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
110
+ "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
111
+ "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
112
+ "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
113
+ "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
114
+ "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
115
+ "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
116
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
117
+ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
118
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
119
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
120
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
121
+ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
122
+ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
123
+ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
124
+ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
125
+ "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
126
+ "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
127
+ "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
128
+ "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
129
+ "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
130
+ "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
131
+ "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
132
+ "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
133
+ "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
134
+ "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
135
+ "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
136
+ "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
137
+ "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
138
+ "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
139
+ "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
140
+ "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
141
+ "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
142
+ "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
143
+ "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
144
+ "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
145
+ "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
146
+ "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
147
+ "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
148
+ "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
149
+ "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
150
+ "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
151
+ "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
152
+ "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
153
+ "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
154
+ "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
155
+ "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
156
+ "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
157
+ "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
158
+ "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
159
+ "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
160
+ "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
161
+ "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
162
+ "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
163
+ "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
164
+ "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
165
+ "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
166
+ "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
167
+ "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
168
+ "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
169
+ "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
170
+ "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
171
+ "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
172
+ "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
173
+ "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
174
+ "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
175
+ "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
176
+ "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
177
+ "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
178
+ "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
179
+ "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
180
+ "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
181
+ "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
182
+ "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
183
+ "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
184
+ "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
185
+ "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
186
+ "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
187
+ "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
188
+ "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
189
+ "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
190
+ "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
191
+ "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
192
+ "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
193
+ "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
194
+ "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
195
+ "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
196
+ "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
197
+ "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
198
+ "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
199
+ "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
200
+ "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
201
+ "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
202
+ "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
203
+ "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
204
+ "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
205
+ "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
206
+ "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
207
+ "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
208
+ "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
209
+ "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
210
+ "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
211
+ "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
212
+ "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
213
+ "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
214
+ "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
215
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
216
+ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
217
+ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
218
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
219
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
220
+ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
221
+ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
222
+ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
223
+ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
224
+ "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
225
+ "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
226
+ "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
227
+ "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
228
+ "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
229
+ "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
230
+ "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
231
+ "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
232
+ "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
233
+ "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
234
+ "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
235
+ "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
236
+ "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
237
+ "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
238
+ "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
239
+ "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
240
+ "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
241
+ "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
242
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
243
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
244
+ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
245
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
246
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
247
+ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
248
+ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
249
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
250
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
251
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
252
+ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
253
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
254
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
255
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
256
+ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
257
+ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
258
+ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
259
+ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
260
+ "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
261
+ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
262
+ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
263
+ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
264
+ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
265
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
266
+ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
267
+ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
268
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
269
+ "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
270
+ "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
271
+ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
272
+ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
273
+ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
274
+ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
275
+ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
276
+ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
277
+ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
278
+ "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
279
+ "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
280
+ "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
281
+ "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
282
+ "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
283
+ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
284
+ "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
285
+ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
286
+ "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
287
+ "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
288
+ "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
289
+ "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
290
+ "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
291
+ "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
292
+ "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
293
+ "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
294
+ "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
295
+ "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
296
+ "model.norm.weight": "model-00003-of-00003.safetensors"
297
+ }
298
+ }
checkpoint-401/rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c5e18f922d0af74d820247ae97bee506ab412554a58345ddf2558abc94ee3e3
3
+ size 15024
checkpoint-401/rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a2dcca6d9741f46592359768ea2212b9321da6408d1fd7d3a80b017bf37f434
3
+ size 15024
checkpoint-401/rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69420ece2c255923c5cbb3c6c9c4a6b9cb38fb57e5d3033c8b7d436a1faf6f13
3
+ size 15024
checkpoint-401/rng_state_3.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:66f278b40a1e23b88a657c4e5d03afa8dbbbe14dfeb16f6b4beedaece6cdd0b9
3
+ size 15024
checkpoint-401/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3786850cd5038038536913e263fba36a46fc209c59de74586162eba92bb42962
3
+ size 1064
checkpoint-401/trainer_state.json ADDED
@@ -0,0 +1,2860 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.9981331673926571,
5
+ "eval_steps": 101,
6
+ "global_step": 401,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0,
13
+ "grad_norm": 53.25,
14
+ "learning_rate": 5.000000000000001e-07,
15
+ "loss": 1.9039,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.0,
20
+ "eval_loss": 3.1495237350463867,
21
+ "eval_runtime": 1687.5261,
22
+ "eval_samples_per_second": 5.968,
23
+ "eval_steps_per_second": 0.746,
24
+ "step": 1
25
+ },
26
+ {
27
+ "epoch": 0.0,
28
+ "grad_norm": 57.75,
29
+ "learning_rate": 1.0000000000000002e-06,
30
+ "loss": 2.1549,
31
+ "step": 2
32
+ },
33
+ {
34
+ "epoch": 0.01,
35
+ "grad_norm": 45.75,
36
+ "learning_rate": 1.5e-06,
37
+ "loss": 1.85,
38
+ "step": 3
39
+ },
40
+ {
41
+ "epoch": 0.01,
42
+ "grad_norm": 43.75,
43
+ "learning_rate": 2.0000000000000003e-06,
44
+ "loss": 1.8815,
45
+ "step": 4
46
+ },
47
+ {
48
+ "epoch": 0.01,
49
+ "grad_norm": 26.875,
50
+ "learning_rate": 2.5e-06,
51
+ "loss": 1.7001,
52
+ "step": 5
53
+ },
54
+ {
55
+ "epoch": 0.01,
56
+ "grad_norm": 15.8125,
57
+ "learning_rate": 3e-06,
58
+ "loss": 1.4792,
59
+ "step": 6
60
+ },
61
+ {
62
+ "epoch": 0.02,
63
+ "grad_norm": 12.25,
64
+ "learning_rate": 3.5e-06,
65
+ "loss": 1.6061,
66
+ "step": 7
67
+ },
68
+ {
69
+ "epoch": 0.02,
70
+ "grad_norm": 10.3125,
71
+ "learning_rate": 4.000000000000001e-06,
72
+ "loss": 1.4366,
73
+ "step": 8
74
+ },
75
+ {
76
+ "epoch": 0.02,
77
+ "grad_norm": 8.0,
78
+ "learning_rate": 4.5e-06,
79
+ "loss": 1.4531,
80
+ "step": 9
81
+ },
82
+ {
83
+ "epoch": 0.02,
84
+ "grad_norm": 7.0,
85
+ "learning_rate": 5e-06,
86
+ "loss": 1.5216,
87
+ "step": 10
88
+ },
89
+ {
90
+ "epoch": 0.03,
91
+ "grad_norm": 6.96875,
92
+ "learning_rate": 4.9999193036470786e-06,
93
+ "loss": 1.2768,
94
+ "step": 11
95
+ },
96
+ {
97
+ "epoch": 0.03,
98
+ "grad_norm": 7.09375,
99
+ "learning_rate": 4.999677219797834e-06,
100
+ "loss": 1.3153,
101
+ "step": 12
102
+ },
103
+ {
104
+ "epoch": 0.03,
105
+ "grad_norm": 5.875,
106
+ "learning_rate": 4.999273764080493e-06,
107
+ "loss": 1.425,
108
+ "step": 13
109
+ },
110
+ {
111
+ "epoch": 0.03,
112
+ "grad_norm": 6.40625,
113
+ "learning_rate": 4.9987089625409815e-06,
114
+ "loss": 1.3616,
115
+ "step": 14
116
+ },
117
+ {
118
+ "epoch": 0.04,
119
+ "grad_norm": 5.40625,
120
+ "learning_rate": 4.997982851641236e-06,
121
+ "loss": 1.3301,
122
+ "step": 15
123
+ },
124
+ {
125
+ "epoch": 0.04,
126
+ "grad_norm": 5.5625,
127
+ "learning_rate": 4.99709547825686e-06,
128
+ "loss": 1.4655,
129
+ "step": 16
130
+ },
131
+ {
132
+ "epoch": 0.04,
133
+ "grad_norm": 5.6875,
134
+ "learning_rate": 4.996046899674091e-06,
135
+ "loss": 1.3226,
136
+ "step": 17
137
+ },
138
+ {
139
+ "epoch": 0.04,
140
+ "grad_norm": 4.75,
141
+ "learning_rate": 4.9948371835861e-06,
142
+ "loss": 1.3406,
143
+ "step": 18
144
+ },
145
+ {
146
+ "epoch": 0.05,
147
+ "grad_norm": 4.4375,
148
+ "learning_rate": 4.993466408088629e-06,
149
+ "loss": 1.304,
150
+ "step": 19
151
+ },
152
+ {
153
+ "epoch": 0.05,
154
+ "grad_norm": 4.75,
155
+ "learning_rate": 4.991934661674945e-06,
156
+ "loss": 1.2712,
157
+ "step": 20
158
+ },
159
+ {
160
+ "epoch": 0.05,
161
+ "grad_norm": 4.53125,
162
+ "learning_rate": 4.990242043230129e-06,
163
+ "loss": 1.3149,
164
+ "step": 21
165
+ },
166
+ {
167
+ "epoch": 0.05,
168
+ "grad_norm": 4.46875,
169
+ "learning_rate": 4.988388662024687e-06,
170
+ "loss": 1.2938,
171
+ "step": 22
172
+ },
173
+ {
174
+ "epoch": 0.06,
175
+ "grad_norm": 4.34375,
176
+ "learning_rate": 4.9863746377075025e-06,
177
+ "loss": 1.242,
178
+ "step": 23
179
+ },
180
+ {
181
+ "epoch": 0.06,
182
+ "grad_norm": 4.46875,
183
+ "learning_rate": 4.98420010029811e-06,
184
+ "loss": 1.2161,
185
+ "step": 24
186
+ },
187
+ {
188
+ "epoch": 0.06,
189
+ "grad_norm": 4.15625,
190
+ "learning_rate": 4.981865190178299e-06,
191
+ "loss": 1.294,
192
+ "step": 25
193
+ },
194
+ {
195
+ "epoch": 0.06,
196
+ "grad_norm": 4.125,
197
+ "learning_rate": 4.979370058083056e-06,
198
+ "loss": 1.1972,
199
+ "step": 26
200
+ },
201
+ {
202
+ "epoch": 0.07,
203
+ "grad_norm": 3.890625,
204
+ "learning_rate": 4.976714865090827e-06,
205
+ "loss": 1.2295,
206
+ "step": 27
207
+ },
208
+ {
209
+ "epoch": 0.07,
210
+ "grad_norm": 4.09375,
211
+ "learning_rate": 4.973899782613127e-06,
212
+ "loss": 1.1852,
213
+ "step": 28
214
+ },
215
+ {
216
+ "epoch": 0.07,
217
+ "grad_norm": 4.28125,
218
+ "learning_rate": 4.970924992383465e-06,
219
+ "loss": 1.323,
220
+ "step": 29
221
+ },
222
+ {
223
+ "epoch": 0.07,
224
+ "grad_norm": 4.03125,
225
+ "learning_rate": 4.96779068644562e-06,
226
+ "loss": 1.3008,
227
+ "step": 30
228
+ },
229
+ {
230
+ "epoch": 0.08,
231
+ "grad_norm": 3.875,
232
+ "learning_rate": 4.964497067141239e-06,
233
+ "loss": 1.2526,
234
+ "step": 31
235
+ },
236
+ {
237
+ "epoch": 0.08,
238
+ "grad_norm": 3.921875,
239
+ "learning_rate": 4.961044347096773e-06,
240
+ "loss": 1.3551,
241
+ "step": 32
242
+ },
243
+ {
244
+ "epoch": 0.08,
245
+ "grad_norm": 3.84375,
246
+ "learning_rate": 4.957432749209755e-06,
247
+ "loss": 1.1906,
248
+ "step": 33
249
+ },
250
+ {
251
+ "epoch": 0.08,
252
+ "grad_norm": 3.796875,
253
+ "learning_rate": 4.953662506634408e-06,
254
+ "loss": 1.2391,
255
+ "step": 34
256
+ },
257
+ {
258
+ "epoch": 0.09,
259
+ "grad_norm": 4.0,
260
+ "learning_rate": 4.949733862766591e-06,
261
+ "loss": 1.1611,
262
+ "step": 35
263
+ },
264
+ {
265
+ "epoch": 0.09,
266
+ "grad_norm": 4.03125,
267
+ "learning_rate": 4.94564707122809e-06,
268
+ "loss": 1.213,
269
+ "step": 36
270
+ },
271
+ {
272
+ "epoch": 0.09,
273
+ "grad_norm": 3.84375,
274
+ "learning_rate": 4.941402395850244e-06,
275
+ "loss": 1.1394,
276
+ "step": 37
277
+ },
278
+ {
279
+ "epoch": 0.09,
280
+ "grad_norm": 3.796875,
281
+ "learning_rate": 4.93700011065691e-06,
282
+ "loss": 1.2633,
283
+ "step": 38
284
+ },
285
+ {
286
+ "epoch": 0.1,
287
+ "grad_norm": 3.6875,
288
+ "learning_rate": 4.9324404998467755e-06,
289
+ "loss": 1.1892,
290
+ "step": 39
291
+ },
292
+ {
293
+ "epoch": 0.1,
294
+ "grad_norm": 3.734375,
295
+ "learning_rate": 4.927723857775012e-06,
296
+ "loss": 1.2087,
297
+ "step": 40
298
+ },
299
+ {
300
+ "epoch": 0.1,
301
+ "grad_norm": 3.84375,
302
+ "learning_rate": 4.922850488934268e-06,
303
+ "loss": 1.2898,
304
+ "step": 41
305
+ },
306
+ {
307
+ "epoch": 0.1,
308
+ "grad_norm": 3.765625,
309
+ "learning_rate": 4.917820707935019e-06,
310
+ "loss": 1.2064,
311
+ "step": 42
312
+ },
313
+ {
314
+ "epoch": 0.11,
315
+ "grad_norm": 3.71875,
316
+ "learning_rate": 4.912634839485251e-06,
317
+ "loss": 1.1967,
318
+ "step": 43
319
+ },
320
+ {
321
+ "epoch": 0.11,
322
+ "grad_norm": 3.625,
323
+ "learning_rate": 4.907293218369499e-06,
324
+ "loss": 1.215,
325
+ "step": 44
326
+ },
327
+ {
328
+ "epoch": 0.11,
329
+ "grad_norm": 4.0,
330
+ "learning_rate": 4.901796189427238e-06,
331
+ "loss": 1.1548,
332
+ "step": 45
333
+ },
334
+ {
335
+ "epoch": 0.11,
336
+ "grad_norm": 3.53125,
337
+ "learning_rate": 4.896144107530618e-06,
338
+ "loss": 1.1007,
339
+ "step": 46
340
+ },
341
+ {
342
+ "epoch": 0.12,
343
+ "grad_norm": 3.53125,
344
+ "learning_rate": 4.890337337561556e-06,
345
+ "loss": 1.0993,
346
+ "step": 47
347
+ },
348
+ {
349
+ "epoch": 0.12,
350
+ "grad_norm": 4.25,
351
+ "learning_rate": 4.884376254388179e-06,
352
+ "loss": 1.0616,
353
+ "step": 48
354
+ },
355
+ {
356
+ "epoch": 0.12,
357
+ "grad_norm": 3.984375,
358
+ "learning_rate": 4.878261242840624e-06,
359
+ "loss": 1.1689,
360
+ "step": 49
361
+ },
362
+ {
363
+ "epoch": 0.12,
364
+ "grad_norm": 3.90625,
365
+ "learning_rate": 4.8719926976861934e-06,
366
+ "loss": 1.1147,
367
+ "step": 50
368
+ },
369
+ {
370
+ "epoch": 0.13,
371
+ "grad_norm": 3.890625,
372
+ "learning_rate": 4.865571023603875e-06,
373
+ "loss": 1.0578,
374
+ "step": 51
375
+ },
376
+ {
377
+ "epoch": 0.13,
378
+ "grad_norm": 4.21875,
379
+ "learning_rate": 4.858996635158211e-06,
380
+ "loss": 1.1676,
381
+ "step": 52
382
+ },
383
+ {
384
+ "epoch": 0.13,
385
+ "grad_norm": 4.03125,
386
+ "learning_rate": 4.852269956772536e-06,
387
+ "loss": 1.0989,
388
+ "step": 53
389
+ },
390
+ {
391
+ "epoch": 0.13,
392
+ "grad_norm": 3.6875,
393
+ "learning_rate": 4.845391422701582e-06,
394
+ "loss": 1.2259,
395
+ "step": 54
396
+ },
397
+ {
398
+ "epoch": 0.14,
399
+ "grad_norm": 4.03125,
400
+ "learning_rate": 4.838361477003439e-06,
401
+ "loss": 1.1906,
402
+ "step": 55
403
+ },
404
+ {
405
+ "epoch": 0.14,
406
+ "grad_norm": 3.828125,
407
+ "learning_rate": 4.83118057351089e-06,
408
+ "loss": 1.0726,
409
+ "step": 56
410
+ },
411
+ {
412
+ "epoch": 0.14,
413
+ "grad_norm": 3.828125,
414
+ "learning_rate": 4.823849175802114e-06,
415
+ "loss": 1.1111,
416
+ "step": 57
417
+ },
418
+ {
419
+ "epoch": 0.14,
420
+ "grad_norm": 3.640625,
421
+ "learning_rate": 4.816367757170754e-06,
422
+ "loss": 1.3362,
423
+ "step": 58
424
+ },
425
+ {
426
+ "epoch": 0.15,
427
+ "grad_norm": 3.671875,
428
+ "learning_rate": 4.808736800595372e-06,
429
+ "loss": 1.1101,
430
+ "step": 59
431
+ },
432
+ {
433
+ "epoch": 0.15,
434
+ "grad_norm": 3.796875,
435
+ "learning_rate": 4.800956798708256e-06,
436
+ "loss": 1.131,
437
+ "step": 60
438
+ },
439
+ {
440
+ "epoch": 0.15,
441
+ "grad_norm": 3.890625,
442
+ "learning_rate": 4.793028253763633e-06,
443
+ "loss": 1.0771,
444
+ "step": 61
445
+ },
446
+ {
447
+ "epoch": 0.15,
448
+ "grad_norm": 4.3125,
449
+ "learning_rate": 4.784951677605229e-06,
450
+ "loss": 1.1331,
451
+ "step": 62
452
+ },
453
+ {
454
+ "epoch": 0.16,
455
+ "grad_norm": 4.03125,
456
+ "learning_rate": 4.776727591633235e-06,
457
+ "loss": 1.1704,
458
+ "step": 63
459
+ },
460
+ {
461
+ "epoch": 0.16,
462
+ "grad_norm": 3.875,
463
+ "learning_rate": 4.76835652677065e-06,
464
+ "loss": 1.1436,
465
+ "step": 64
466
+ },
467
+ {
468
+ "epoch": 0.16,
469
+ "grad_norm": 3.65625,
470
+ "learning_rate": 4.759839023428994e-06,
471
+ "loss": 1.1931,
472
+ "step": 65
473
+ },
474
+ {
475
+ "epoch": 0.16,
476
+ "grad_norm": 4.15625,
477
+ "learning_rate": 4.751175631473433e-06,
478
+ "loss": 1.2202,
479
+ "step": 66
480
+ },
481
+ {
482
+ "epoch": 0.17,
483
+ "grad_norm": 3.84375,
484
+ "learning_rate": 4.742366910187275e-06,
485
+ "loss": 1.2113,
486
+ "step": 67
487
+ },
488
+ {
489
+ "epoch": 0.17,
490
+ "grad_norm": 3.859375,
491
+ "learning_rate": 4.733413428235864e-06,
492
+ "loss": 1.1091,
493
+ "step": 68
494
+ },
495
+ {
496
+ "epoch": 0.17,
497
+ "grad_norm": 3.859375,
498
+ "learning_rate": 4.724315763629874e-06,
499
+ "loss": 1.0952,
500
+ "step": 69
501
+ },
502
+ {
503
+ "epoch": 0.17,
504
+ "grad_norm": 3.828125,
505
+ "learning_rate": 4.715074503687987e-06,
506
+ "loss": 1.2604,
507
+ "step": 70
508
+ },
509
+ {
510
+ "epoch": 0.18,
511
+ "grad_norm": 4.1875,
512
+ "learning_rate": 4.705690244998981e-06,
513
+ "loss": 1.177,
514
+ "step": 71
515
+ },
516
+ {
517
+ "epoch": 0.18,
518
+ "grad_norm": 3.78125,
519
+ "learning_rate": 4.696163593383217e-06,
520
+ "loss": 1.1761,
521
+ "step": 72
522
+ },
523
+ {
524
+ "epoch": 0.18,
525
+ "grad_norm": 3.9375,
526
+ "learning_rate": 4.686495163853528e-06,
527
+ "loss": 1.2345,
528
+ "step": 73
529
+ },
530
+ {
531
+ "epoch": 0.18,
532
+ "grad_norm": 3.859375,
533
+ "learning_rate": 4.676685580575516e-06,
534
+ "loss": 1.1289,
535
+ "step": 74
536
+ },
537
+ {
538
+ "epoch": 0.19,
539
+ "grad_norm": 3.71875,
540
+ "learning_rate": 4.6667354768272565e-06,
541
+ "loss": 1.1536,
542
+ "step": 75
543
+ },
544
+ {
545
+ "epoch": 0.19,
546
+ "grad_norm": 3.53125,
547
+ "learning_rate": 4.656645494958415e-06,
548
+ "loss": 1.1625,
549
+ "step": 76
550
+ },
551
+ {
552
+ "epoch": 0.19,
553
+ "grad_norm": 3.875,
554
+ "learning_rate": 4.646416286348783e-06,
555
+ "loss": 1.1441,
556
+ "step": 77
557
+ },
558
+ {
559
+ "epoch": 0.19,
560
+ "grad_norm": 3.71875,
561
+ "learning_rate": 4.636048511366222e-06,
562
+ "loss": 1.0877,
563
+ "step": 78
564
+ },
565
+ {
566
+ "epoch": 0.2,
567
+ "grad_norm": 3.96875,
568
+ "learning_rate": 4.625542839324036e-06,
569
+ "loss": 0.998,
570
+ "step": 79
571
+ },
572
+ {
573
+ "epoch": 0.2,
574
+ "grad_norm": 3.6875,
575
+ "learning_rate": 4.61489994843776e-06,
576
+ "loss": 1.1651,
577
+ "step": 80
578
+ },
579
+ {
580
+ "epoch": 0.2,
581
+ "grad_norm": 4.03125,
582
+ "learning_rate": 4.604120525781376e-06,
583
+ "loss": 1.1657,
584
+ "step": 81
585
+ },
586
+ {
587
+ "epoch": 0.2,
588
+ "grad_norm": 4.15625,
589
+ "learning_rate": 4.593205267242962e-06,
590
+ "loss": 1.1362,
591
+ "step": 82
592
+ },
593
+ {
594
+ "epoch": 0.21,
595
+ "grad_norm": 3.9375,
596
+ "learning_rate": 4.582154877479762e-06,
597
+ "loss": 1.1169,
598
+ "step": 83
599
+ },
600
+ {
601
+ "epoch": 0.21,
602
+ "grad_norm": 3.90625,
603
+ "learning_rate": 4.570970069872695e-06,
604
+ "loss": 1.1175,
605
+ "step": 84
606
+ },
607
+ {
608
+ "epoch": 0.21,
609
+ "grad_norm": 3.78125,
610
+ "learning_rate": 4.55965156648031e-06,
611
+ "loss": 1.0583,
612
+ "step": 85
613
+ },
614
+ {
615
+ "epoch": 0.21,
616
+ "grad_norm": 3.921875,
617
+ "learning_rate": 4.548200097992161e-06,
618
+ "loss": 1.0436,
619
+ "step": 86
620
+ },
621
+ {
622
+ "epoch": 0.22,
623
+ "grad_norm": 4.0625,
624
+ "learning_rate": 4.5366164036816415e-06,
625
+ "loss": 1.036,
626
+ "step": 87
627
+ },
628
+ {
629
+ "epoch": 0.22,
630
+ "grad_norm": 3.859375,
631
+ "learning_rate": 4.524901231358261e-06,
632
+ "loss": 1.2137,
633
+ "step": 88
634
+ },
635
+ {
636
+ "epoch": 0.22,
637
+ "grad_norm": 4.375,
638
+ "learning_rate": 4.513055337319361e-06,
639
+ "loss": 1.1137,
640
+ "step": 89
641
+ },
642
+ {
643
+ "epoch": 0.22,
644
+ "grad_norm": 3.984375,
645
+ "learning_rate": 4.501079486301299e-06,
646
+ "loss": 1.1229,
647
+ "step": 90
648
+ },
649
+ {
650
+ "epoch": 0.23,
651
+ "grad_norm": 3.734375,
652
+ "learning_rate": 4.488974451430077e-06,
653
+ "loss": 1.0657,
654
+ "step": 91
655
+ },
656
+ {
657
+ "epoch": 0.23,
658
+ "grad_norm": 3.671875,
659
+ "learning_rate": 4.4767410141714265e-06,
660
+ "loss": 1.1326,
661
+ "step": 92
662
+ },
663
+ {
664
+ "epoch": 0.23,
665
+ "grad_norm": 3.921875,
666
+ "learning_rate": 4.464379964280365e-06,
667
+ "loss": 1.1957,
668
+ "step": 93
669
+ },
670
+ {
671
+ "epoch": 0.23,
672
+ "grad_norm": 3.75,
673
+ "learning_rate": 4.451892099750206e-06,
674
+ "loss": 1.1726,
675
+ "step": 94
676
+ },
677
+ {
678
+ "epoch": 0.24,
679
+ "grad_norm": 3.96875,
680
+ "learning_rate": 4.43927822676105e-06,
681
+ "loss": 1.0951,
682
+ "step": 95
683
+ },
684
+ {
685
+ "epoch": 0.24,
686
+ "grad_norm": 3.921875,
687
+ "learning_rate": 4.426539159627733e-06,
688
+ "loss": 1.1512,
689
+ "step": 96
690
+ },
691
+ {
692
+ "epoch": 0.24,
693
+ "grad_norm": 4.03125,
694
+ "learning_rate": 4.413675720747262e-06,
695
+ "loss": 1.0428,
696
+ "step": 97
697
+ },
698
+ {
699
+ "epoch": 0.24,
700
+ "grad_norm": 3.875,
701
+ "learning_rate": 4.40068874054572e-06,
702
+ "loss": 1.1005,
703
+ "step": 98
704
+ },
705
+ {
706
+ "epoch": 0.25,
707
+ "grad_norm": 3.65625,
708
+ "learning_rate": 4.387579057424655e-06,
709
+ "loss": 1.0811,
710
+ "step": 99
711
+ },
712
+ {
713
+ "epoch": 0.25,
714
+ "grad_norm": 3.96875,
715
+ "learning_rate": 4.374347517706963e-06,
716
+ "loss": 1.1457,
717
+ "step": 100
718
+ },
719
+ {
720
+ "epoch": 0.25,
721
+ "grad_norm": 3.796875,
722
+ "learning_rate": 4.36099497558224e-06,
723
+ "loss": 0.9933,
724
+ "step": 101
725
+ },
726
+ {
727
+ "epoch": 0.25,
728
+ "eval_loss": 1.2402405738830566,
729
+ "eval_runtime": 1697.4894,
730
+ "eval_samples_per_second": 5.933,
731
+ "eval_steps_per_second": 0.742,
732
+ "step": 101
733
+ },
734
+ {
735
+ "epoch": 0.25,
736
+ "grad_norm": 3.96875,
737
+ "learning_rate": 4.3475222930516484e-06,
738
+ "loss": 1.0559,
739
+ "step": 102
740
+ },
741
+ {
742
+ "epoch": 0.26,
743
+ "grad_norm": 4.0625,
744
+ "learning_rate": 4.3339303398722645e-06,
745
+ "loss": 1.1506,
746
+ "step": 103
747
+ },
748
+ {
749
+ "epoch": 0.26,
750
+ "grad_norm": 3.671875,
751
+ "learning_rate": 4.320219993500927e-06,
752
+ "loss": 1.0673,
753
+ "step": 104
754
+ },
755
+ {
756
+ "epoch": 0.26,
757
+ "grad_norm": 3.828125,
758
+ "learning_rate": 4.306392139037598e-06,
759
+ "loss": 1.0358,
760
+ "step": 105
761
+ },
762
+ {
763
+ "epoch": 0.26,
764
+ "grad_norm": 4.03125,
765
+ "learning_rate": 4.292447669168215e-06,
766
+ "loss": 1.2526,
767
+ "step": 106
768
+ },
769
+ {
770
+ "epoch": 0.27,
771
+ "grad_norm": 3.8125,
772
+ "learning_rate": 4.278387484107067e-06,
773
+ "loss": 1.1494,
774
+ "step": 107
775
+ },
776
+ {
777
+ "epoch": 0.27,
778
+ "grad_norm": 4.15625,
779
+ "learning_rate": 4.26421249153868e-06,
780
+ "loss": 1.1066,
781
+ "step": 108
782
+ },
783
+ {
784
+ "epoch": 0.27,
785
+ "grad_norm": 3.734375,
786
+ "learning_rate": 4.2499236065592154e-06,
787
+ "loss": 1.129,
788
+ "step": 109
789
+ },
790
+ {
791
+ "epoch": 0.27,
792
+ "grad_norm": 3.875,
793
+ "learning_rate": 4.235521751617398e-06,
794
+ "loss": 0.9961,
795
+ "step": 110
796
+ },
797
+ {
798
+ "epoch": 0.28,
799
+ "grad_norm": 3.953125,
800
+ "learning_rate": 4.221007856454963e-06,
801
+ "loss": 1.1145,
802
+ "step": 111
803
+ },
804
+ {
805
+ "epoch": 0.28,
806
+ "grad_norm": 3.96875,
807
+ "learning_rate": 4.206382858046636e-06,
808
+ "loss": 1.1684,
809
+ "step": 112
810
+ },
811
+ {
812
+ "epoch": 0.28,
813
+ "grad_norm": 3.984375,
814
+ "learning_rate": 4.191647700539642e-06,
815
+ "loss": 1.1654,
816
+ "step": 113
817
+ },
818
+ {
819
+ "epoch": 0.28,
820
+ "grad_norm": 3.640625,
821
+ "learning_rate": 4.176803335192757e-06,
822
+ "loss": 1.0714,
823
+ "step": 114
824
+ },
825
+ {
826
+ "epoch": 0.29,
827
+ "grad_norm": 3.53125,
828
+ "learning_rate": 4.1618507203149e-06,
829
+ "loss": 1.0937,
830
+ "step": 115
831
+ },
832
+ {
833
+ "epoch": 0.29,
834
+ "grad_norm": 3.859375,
835
+ "learning_rate": 4.1467908212032586e-06,
836
+ "loss": 1.0441,
837
+ "step": 116
838
+ },
839
+ {
840
+ "epoch": 0.29,
841
+ "grad_norm": 3.8125,
842
+ "learning_rate": 4.1316246100809786e-06,
843
+ "loss": 1.0788,
844
+ "step": 117
845
+ },
846
+ {
847
+ "epoch": 0.29,
848
+ "grad_norm": 4.0625,
849
+ "learning_rate": 4.116353066034403e-06,
850
+ "loss": 1.0276,
851
+ "step": 118
852
+ },
853
+ {
854
+ "epoch": 0.3,
855
+ "grad_norm": 3.78125,
856
+ "learning_rate": 4.100977174949856e-06,
857
+ "loss": 1.0827,
858
+ "step": 119
859
+ },
860
+ {
861
+ "epoch": 0.3,
862
+ "grad_norm": 3.90625,
863
+ "learning_rate": 4.085497929450005e-06,
864
+ "loss": 1.0748,
865
+ "step": 120
866
+ },
867
+ {
868
+ "epoch": 0.3,
869
+ "grad_norm": 3.78125,
870
+ "learning_rate": 4.069916328829779e-06,
871
+ "loss": 1.0433,
872
+ "step": 121
873
+ },
874
+ {
875
+ "epoch": 0.3,
876
+ "grad_norm": 4.0,
877
+ "learning_rate": 4.0542333789918474e-06,
878
+ "loss": 1.0983,
879
+ "step": 122
880
+ },
881
+ {
882
+ "epoch": 0.31,
883
+ "grad_norm": 4.03125,
884
+ "learning_rate": 4.038450092381697e-06,
885
+ "loss": 1.1221,
886
+ "step": 123
887
+ },
888
+ {
889
+ "epoch": 0.31,
890
+ "grad_norm": 4.125,
891
+ "learning_rate": 4.02256748792226e-06,
892
+ "loss": 1.0123,
893
+ "step": 124
894
+ },
895
+ {
896
+ "epoch": 0.31,
897
+ "grad_norm": 3.609375,
898
+ "learning_rate": 4.006586590948141e-06,
899
+ "loss": 1.0935,
900
+ "step": 125
901
+ },
902
+ {
903
+ "epoch": 0.31,
904
+ "grad_norm": 4.15625,
905
+ "learning_rate": 3.990508433139421e-06,
906
+ "loss": 1.0307,
907
+ "step": 126
908
+ },
909
+ {
910
+ "epoch": 0.32,
911
+ "grad_norm": 3.9375,
912
+ "learning_rate": 3.97433405245506e-06,
913
+ "loss": 1.0694,
914
+ "step": 127
915
+ },
916
+ {
917
+ "epoch": 0.32,
918
+ "grad_norm": 3.84375,
919
+ "learning_rate": 3.958064493065881e-06,
920
+ "loss": 0.9683,
921
+ "step": 128
922
+ },
923
+ {
924
+ "epoch": 0.32,
925
+ "grad_norm": 3.953125,
926
+ "learning_rate": 3.941700805287169e-06,
927
+ "loss": 1.1073,
928
+ "step": 129
929
+ },
930
+ {
931
+ "epoch": 0.32,
932
+ "grad_norm": 3.84375,
933
+ "learning_rate": 3.925244045510863e-06,
934
+ "loss": 0.9876,
935
+ "step": 130
936
+ },
937
+ {
938
+ "epoch": 0.33,
939
+ "grad_norm": 3.875,
940
+ "learning_rate": 3.9086952761373605e-06,
941
+ "loss": 1.0082,
942
+ "step": 131
943
+ },
944
+ {
945
+ "epoch": 0.33,
946
+ "grad_norm": 3.734375,
947
+ "learning_rate": 3.892055565506929e-06,
948
+ "loss": 0.9779,
949
+ "step": 132
950
+ },
951
+ {
952
+ "epoch": 0.33,
953
+ "grad_norm": 3.6875,
954
+ "learning_rate": 3.8753259878307355e-06,
955
+ "loss": 1.0722,
956
+ "step": 133
957
+ },
958
+ {
959
+ "epoch": 0.33,
960
+ "grad_norm": 3.703125,
961
+ "learning_rate": 3.858507623121506e-06,
962
+ "loss": 1.0229,
963
+ "step": 134
964
+ },
965
+ {
966
+ "epoch": 0.34,
967
+ "grad_norm": 3.703125,
968
+ "learning_rate": 3.841601557123793e-06,
969
+ "loss": 1.081,
970
+ "step": 135
971
+ },
972
+ {
973
+ "epoch": 0.34,
974
+ "grad_norm": 3.8125,
975
+ "learning_rate": 3.824608881243893e-06,
976
+ "loss": 1.2165,
977
+ "step": 136
978
+ },
979
+ {
980
+ "epoch": 0.34,
981
+ "grad_norm": 3.765625,
982
+ "learning_rate": 3.807530692479382e-06,
983
+ "loss": 1.1001,
984
+ "step": 137
985
+ },
986
+ {
987
+ "epoch": 0.34,
988
+ "grad_norm": 3.671875,
989
+ "learning_rate": 3.7903680933482965e-06,
990
+ "loss": 1.0578,
991
+ "step": 138
992
+ },
993
+ {
994
+ "epoch": 0.35,
995
+ "grad_norm": 3.828125,
996
+ "learning_rate": 3.7731221918179645e-06,
997
+ "loss": 0.9931,
998
+ "step": 139
999
+ },
1000
+ {
1001
+ "epoch": 0.35,
1002
+ "grad_norm": 3.890625,
1003
+ "learning_rate": 3.7557941012334687e-06,
1004
+ "loss": 1.0884,
1005
+ "step": 140
1006
+ },
1007
+ {
1008
+ "epoch": 0.35,
1009
+ "grad_norm": 3.984375,
1010
+ "learning_rate": 3.7383849402457807e-06,
1011
+ "loss": 1.0741,
1012
+ "step": 141
1013
+ },
1014
+ {
1015
+ "epoch": 0.35,
1016
+ "grad_norm": 3.71875,
1017
+ "learning_rate": 3.72089583273954e-06,
1018
+ "loss": 1.1207,
1019
+ "step": 142
1020
+ },
1021
+ {
1022
+ "epoch": 0.36,
1023
+ "grad_norm": 3.890625,
1024
+ "learning_rate": 3.703327907760499e-06,
1025
+ "loss": 1.057,
1026
+ "step": 143
1027
+ },
1028
+ {
1029
+ "epoch": 0.36,
1030
+ "grad_norm": 3.734375,
1031
+ "learning_rate": 3.685682299442638e-06,
1032
+ "loss": 1.0551,
1033
+ "step": 144
1034
+ },
1035
+ {
1036
+ "epoch": 0.36,
1037
+ "grad_norm": 3.78125,
1038
+ "learning_rate": 3.6679601469349456e-06,
1039
+ "loss": 0.9792,
1040
+ "step": 145
1041
+ },
1042
+ {
1043
+ "epoch": 0.36,
1044
+ "grad_norm": 3.703125,
1045
+ "learning_rate": 3.650162594327881e-06,
1046
+ "loss": 0.967,
1047
+ "step": 146
1048
+ },
1049
+ {
1050
+ "epoch": 0.37,
1051
+ "grad_norm": 4.0625,
1052
+ "learning_rate": 3.632290790579512e-06,
1053
+ "loss": 1.1393,
1054
+ "step": 147
1055
+ },
1056
+ {
1057
+ "epoch": 0.37,
1058
+ "grad_norm": 3.953125,
1059
+ "learning_rate": 3.6143458894413463e-06,
1060
+ "loss": 1.0892,
1061
+ "step": 148
1062
+ },
1063
+ {
1064
+ "epoch": 0.37,
1065
+ "grad_norm": 3.75,
1066
+ "learning_rate": 3.596329049383843e-06,
1067
+ "loss": 1.0478,
1068
+ "step": 149
1069
+ },
1070
+ {
1071
+ "epoch": 0.37,
1072
+ "grad_norm": 3.765625,
1073
+ "learning_rate": 3.5782414335216297e-06,
1074
+ "loss": 1.0014,
1075
+ "step": 150
1076
+ },
1077
+ {
1078
+ "epoch": 0.38,
1079
+ "grad_norm": 3.84375,
1080
+ "learning_rate": 3.5600842095384125e-06,
1081
+ "loss": 1.0601,
1082
+ "step": 151
1083
+ },
1084
+ {
1085
+ "epoch": 0.38,
1086
+ "grad_norm": 3.90625,
1087
+ "learning_rate": 3.541858549611596e-06,
1088
+ "loss": 1.1093,
1089
+ "step": 152
1090
+ },
1091
+ {
1092
+ "epoch": 0.38,
1093
+ "grad_norm": 3.84375,
1094
+ "learning_rate": 3.523565630336607e-06,
1095
+ "loss": 0.9811,
1096
+ "step": 153
1097
+ },
1098
+ {
1099
+ "epoch": 0.38,
1100
+ "grad_norm": 3.8125,
1101
+ "learning_rate": 3.505206632650944e-06,
1102
+ "loss": 1.1626,
1103
+ "step": 154
1104
+ },
1105
+ {
1106
+ "epoch": 0.39,
1107
+ "grad_norm": 3.875,
1108
+ "learning_rate": 3.48678274175793e-06,
1109
+ "loss": 1.0518,
1110
+ "step": 155
1111
+ },
1112
+ {
1113
+ "epoch": 0.39,
1114
+ "grad_norm": 3.75,
1115
+ "learning_rate": 3.468295147050207e-06,
1116
+ "loss": 1.0028,
1117
+ "step": 156
1118
+ },
1119
+ {
1120
+ "epoch": 0.39,
1121
+ "grad_norm": 3.953125,
1122
+ "learning_rate": 3.4497450420329486e-06,
1123
+ "loss": 1.0596,
1124
+ "step": 157
1125
+ },
1126
+ {
1127
+ "epoch": 0.39,
1128
+ "grad_norm": 3.8125,
1129
+ "learning_rate": 3.431133624246812e-06,
1130
+ "loss": 1.08,
1131
+ "step": 158
1132
+ },
1133
+ {
1134
+ "epoch": 0.4,
1135
+ "grad_norm": 3.90625,
1136
+ "learning_rate": 3.4124620951906283e-06,
1137
+ "loss": 1.02,
1138
+ "step": 159
1139
+ },
1140
+ {
1141
+ "epoch": 0.4,
1142
+ "grad_norm": 3.828125,
1143
+ "learning_rate": 3.3937316602438352e-06,
1144
+ "loss": 1.0798,
1145
+ "step": 160
1146
+ },
1147
+ {
1148
+ "epoch": 0.4,
1149
+ "grad_norm": 3.953125,
1150
+ "learning_rate": 3.3749435285886638e-06,
1151
+ "loss": 1.1169,
1152
+ "step": 161
1153
+ },
1154
+ {
1155
+ "epoch": 0.4,
1156
+ "grad_norm": 3.75,
1157
+ "learning_rate": 3.356098913132077e-06,
1158
+ "loss": 1.0584,
1159
+ "step": 162
1160
+ },
1161
+ {
1162
+ "epoch": 0.41,
1163
+ "grad_norm": 3.859375,
1164
+ "learning_rate": 3.3371990304274654e-06,
1165
+ "loss": 1.0337,
1166
+ "step": 163
1167
+ },
1168
+ {
1169
+ "epoch": 0.41,
1170
+ "grad_norm": 4.03125,
1171
+ "learning_rate": 3.3182451005961146e-06,
1172
+ "loss": 1.1088,
1173
+ "step": 164
1174
+ },
1175
+ {
1176
+ "epoch": 0.41,
1177
+ "grad_norm": 4.0,
1178
+ "learning_rate": 3.2992383472484318e-06,
1179
+ "loss": 0.9837,
1180
+ "step": 165
1181
+ },
1182
+ {
1183
+ "epoch": 0.41,
1184
+ "grad_norm": 3.609375,
1185
+ "learning_rate": 3.280179997404958e-06,
1186
+ "loss": 1.0732,
1187
+ "step": 166
1188
+ },
1189
+ {
1190
+ "epoch": 0.42,
1191
+ "grad_norm": 3.765625,
1192
+ "learning_rate": 3.2610712814171535e-06,
1193
+ "loss": 1.1187,
1194
+ "step": 167
1195
+ },
1196
+ {
1197
+ "epoch": 0.42,
1198
+ "grad_norm": 4.125,
1199
+ "learning_rate": 3.2419134328879703e-06,
1200
+ "loss": 1.104,
1201
+ "step": 168
1202
+ },
1203
+ {
1204
+ "epoch": 0.42,
1205
+ "grad_norm": 4.09375,
1206
+ "learning_rate": 3.2227076885922137e-06,
1207
+ "loss": 0.9941,
1208
+ "step": 169
1209
+ },
1210
+ {
1211
+ "epoch": 0.42,
1212
+ "grad_norm": 3.890625,
1213
+ "learning_rate": 3.2034552883966972e-06,
1214
+ "loss": 1.1258,
1215
+ "step": 170
1216
+ },
1217
+ {
1218
+ "epoch": 0.43,
1219
+ "grad_norm": 3.515625,
1220
+ "learning_rate": 3.184157475180208e-06,
1221
+ "loss": 0.9484,
1222
+ "step": 171
1223
+ },
1224
+ {
1225
+ "epoch": 0.43,
1226
+ "grad_norm": 3.71875,
1227
+ "learning_rate": 3.1648154947532607e-06,
1228
+ "loss": 1.0749,
1229
+ "step": 172
1230
+ },
1231
+ {
1232
+ "epoch": 0.43,
1233
+ "grad_norm": 3.96875,
1234
+ "learning_rate": 3.1454305957776797e-06,
1235
+ "loss": 0.9447,
1236
+ "step": 173
1237
+ },
1238
+ {
1239
+ "epoch": 0.43,
1240
+ "grad_norm": 3.8125,
1241
+ "learning_rate": 3.126004029685984e-06,
1242
+ "loss": 1.1009,
1243
+ "step": 174
1244
+ },
1245
+ {
1246
+ "epoch": 0.44,
1247
+ "grad_norm": 3.90625,
1248
+ "learning_rate": 3.106537050600601e-06,
1249
+ "loss": 1.0355,
1250
+ "step": 175
1251
+ },
1252
+ {
1253
+ "epoch": 0.44,
1254
+ "grad_norm": 3.9375,
1255
+ "learning_rate": 3.0870309152529006e-06,
1256
+ "loss": 1.0504,
1257
+ "step": 176
1258
+ },
1259
+ {
1260
+ "epoch": 0.44,
1261
+ "grad_norm": 3.734375,
1262
+ "learning_rate": 3.0674868829020692e-06,
1263
+ "loss": 1.0575,
1264
+ "step": 177
1265
+ },
1266
+ {
1267
+ "epoch": 0.44,
1268
+ "grad_norm": 3.71875,
1269
+ "learning_rate": 3.0479062152538137e-06,
1270
+ "loss": 1.0417,
1271
+ "step": 178
1272
+ },
1273
+ {
1274
+ "epoch": 0.45,
1275
+ "grad_norm": 3.78125,
1276
+ "learning_rate": 3.0282901763789064e-06,
1277
+ "loss": 0.962,
1278
+ "step": 179
1279
+ },
1280
+ {
1281
+ "epoch": 0.45,
1282
+ "grad_norm": 3.890625,
1283
+ "learning_rate": 3.0086400326315853e-06,
1284
+ "loss": 1.0601,
1285
+ "step": 180
1286
+ },
1287
+ {
1288
+ "epoch": 0.45,
1289
+ "grad_norm": 3.796875,
1290
+ "learning_rate": 2.988957052567797e-06,
1291
+ "loss": 1.0731,
1292
+ "step": 181
1293
+ },
1294
+ {
1295
+ "epoch": 0.45,
1296
+ "grad_norm": 3.8125,
1297
+ "learning_rate": 2.969242506863306e-06,
1298
+ "loss": 1.0532,
1299
+ "step": 182
1300
+ },
1301
+ {
1302
+ "epoch": 0.46,
1303
+ "grad_norm": 3.890625,
1304
+ "learning_rate": 2.949497668231663e-06,
1305
+ "loss": 0.995,
1306
+ "step": 183
1307
+ },
1308
+ {
1309
+ "epoch": 0.46,
1310
+ "grad_norm": 4.09375,
1311
+ "learning_rate": 2.929723811342042e-06,
1312
+ "loss": 0.9706,
1313
+ "step": 184
1314
+ },
1315
+ {
1316
+ "epoch": 0.46,
1317
+ "grad_norm": 3.5625,
1318
+ "learning_rate": 2.9099222127369496e-06,
1319
+ "loss": 1.0618,
1320
+ "step": 185
1321
+ },
1322
+ {
1323
+ "epoch": 0.46,
1324
+ "grad_norm": 3.890625,
1325
+ "learning_rate": 2.890094150749818e-06,
1326
+ "loss": 1.0437,
1327
+ "step": 186
1328
+ },
1329
+ {
1330
+ "epoch": 0.47,
1331
+ "grad_norm": 3.859375,
1332
+ "learning_rate": 2.870240905422476e-06,
1333
+ "loss": 1.0314,
1334
+ "step": 187
1335
+ },
1336
+ {
1337
+ "epoch": 0.47,
1338
+ "grad_norm": 3.75,
1339
+ "learning_rate": 2.850363758422519e-06,
1340
+ "loss": 0.9562,
1341
+ "step": 188
1342
+ },
1343
+ {
1344
+ "epoch": 0.47,
1345
+ "grad_norm": 3.84375,
1346
+ "learning_rate": 2.830463992960561e-06,
1347
+ "loss": 1.1011,
1348
+ "step": 189
1349
+ },
1350
+ {
1351
+ "epoch": 0.47,
1352
+ "grad_norm": 4.125,
1353
+ "learning_rate": 2.8105428937073997e-06,
1354
+ "loss": 0.9929,
1355
+ "step": 190
1356
+ },
1357
+ {
1358
+ "epoch": 0.48,
1359
+ "grad_norm": 3.78125,
1360
+ "learning_rate": 2.7906017467110806e-06,
1361
+ "loss": 1.0573,
1362
+ "step": 191
1363
+ },
1364
+ {
1365
+ "epoch": 0.48,
1366
+ "grad_norm": 3.671875,
1367
+ "learning_rate": 2.770641839313871e-06,
1368
+ "loss": 1.0478,
1369
+ "step": 192
1370
+ },
1371
+ {
1372
+ "epoch": 0.48,
1373
+ "grad_norm": 3.953125,
1374
+ "learning_rate": 2.750664460069157e-06,
1375
+ "loss": 1.047,
1376
+ "step": 193
1377
+ },
1378
+ {
1379
+ "epoch": 0.48,
1380
+ "grad_norm": 3.8125,
1381
+ "learning_rate": 2.730670898658255e-06,
1382
+ "loss": 1.0945,
1383
+ "step": 194
1384
+ },
1385
+ {
1386
+ "epoch": 0.49,
1387
+ "grad_norm": 3.84375,
1388
+ "learning_rate": 2.710662445807156e-06,
1389
+ "loss": 0.9587,
1390
+ "step": 195
1391
+ },
1392
+ {
1393
+ "epoch": 0.49,
1394
+ "grad_norm": 3.84375,
1395
+ "learning_rate": 2.6906403932031973e-06,
1396
+ "loss": 1.0648,
1397
+ "step": 196
1398
+ },
1399
+ {
1400
+ "epoch": 0.49,
1401
+ "grad_norm": 3.90625,
1402
+ "learning_rate": 2.670606033411678e-06,
1403
+ "loss": 1.1022,
1404
+ "step": 197
1405
+ },
1406
+ {
1407
+ "epoch": 0.49,
1408
+ "grad_norm": 3.8125,
1409
+ "learning_rate": 2.6505606597924122e-06,
1410
+ "loss": 1.1515,
1411
+ "step": 198
1412
+ },
1413
+ {
1414
+ "epoch": 0.5,
1415
+ "grad_norm": 3.53125,
1416
+ "learning_rate": 2.630505566416235e-06,
1417
+ "loss": 1.0288,
1418
+ "step": 199
1419
+ },
1420
+ {
1421
+ "epoch": 0.5,
1422
+ "grad_norm": 4.125,
1423
+ "learning_rate": 2.610442047981462e-06,
1424
+ "loss": 0.9577,
1425
+ "step": 200
1426
+ },
1427
+ {
1428
+ "epoch": 0.5,
1429
+ "grad_norm": 3.765625,
1430
+ "learning_rate": 2.5903713997303033e-06,
1431
+ "loss": 0.9455,
1432
+ "step": 201
1433
+ },
1434
+ {
1435
+ "epoch": 0.5,
1436
+ "grad_norm": 3.75,
1437
+ "learning_rate": 2.5702949173652515e-06,
1438
+ "loss": 0.9439,
1439
+ "step": 202
1440
+ },
1441
+ {
1442
+ "epoch": 0.5,
1443
+ "eval_loss": 1.1682730913162231,
1444
+ "eval_runtime": 1697.6225,
1445
+ "eval_samples_per_second": 5.932,
1446
+ "eval_steps_per_second": 0.742,
1447
+ "step": 202
1448
+ },
1449
+ {
1450
+ "epoch": 0.51,
1451
+ "grad_norm": 3.8125,
1452
+ "learning_rate": 2.550213896965431e-06,
1453
+ "loss": 1.0939,
1454
+ "step": 203
1455
+ },
1456
+ {
1457
+ "epoch": 0.51,
1458
+ "grad_norm": 3.5625,
1459
+ "learning_rate": 2.53012963490293e-06,
1460
+ "loss": 1.0256,
1461
+ "step": 204
1462
+ },
1463
+ {
1464
+ "epoch": 0.51,
1465
+ "grad_norm": 3.8125,
1466
+ "learning_rate": 2.5100434277591077e-06,
1467
+ "loss": 1.0595,
1468
+ "step": 205
1469
+ },
1470
+ {
1471
+ "epoch": 0.51,
1472
+ "grad_norm": 3.890625,
1473
+ "learning_rate": 2.4899565722408927e-06,
1474
+ "loss": 1.1182,
1475
+ "step": 206
1476
+ },
1477
+ {
1478
+ "epoch": 0.52,
1479
+ "grad_norm": 3.6875,
1480
+ "learning_rate": 2.4698703650970706e-06,
1481
+ "loss": 1.0328,
1482
+ "step": 207
1483
+ },
1484
+ {
1485
+ "epoch": 0.52,
1486
+ "grad_norm": 3.65625,
1487
+ "learning_rate": 2.449786103034569e-06,
1488
+ "loss": 0.9872,
1489
+ "step": 208
1490
+ },
1491
+ {
1492
+ "epoch": 0.52,
1493
+ "grad_norm": 3.796875,
1494
+ "learning_rate": 2.4297050826347498e-06,
1495
+ "loss": 1.0855,
1496
+ "step": 209
1497
+ },
1498
+ {
1499
+ "epoch": 0.52,
1500
+ "grad_norm": 3.984375,
1501
+ "learning_rate": 2.409628600269697e-06,
1502
+ "loss": 1.0982,
1503
+ "step": 210
1504
+ },
1505
+ {
1506
+ "epoch": 0.53,
1507
+ "grad_norm": 3.8125,
1508
+ "learning_rate": 2.3895579520185393e-06,
1509
+ "loss": 1.1318,
1510
+ "step": 211
1511
+ },
1512
+ {
1513
+ "epoch": 0.53,
1514
+ "grad_norm": 3.640625,
1515
+ "learning_rate": 2.3694944335837653e-06,
1516
+ "loss": 0.9962,
1517
+ "step": 212
1518
+ },
1519
+ {
1520
+ "epoch": 0.53,
1521
+ "grad_norm": 3.765625,
1522
+ "learning_rate": 2.349439340207588e-06,
1523
+ "loss": 0.968,
1524
+ "step": 213
1525
+ },
1526
+ {
1527
+ "epoch": 0.53,
1528
+ "grad_norm": 3.96875,
1529
+ "learning_rate": 2.3293939665883233e-06,
1530
+ "loss": 0.9732,
1531
+ "step": 214
1532
+ },
1533
+ {
1534
+ "epoch": 0.54,
1535
+ "grad_norm": 3.71875,
1536
+ "learning_rate": 2.3093596067968027e-06,
1537
+ "loss": 0.9373,
1538
+ "step": 215
1539
+ },
1540
+ {
1541
+ "epoch": 0.54,
1542
+ "grad_norm": 3.609375,
1543
+ "learning_rate": 2.2893375541928447e-06,
1544
+ "loss": 1.0306,
1545
+ "step": 216
1546
+ },
1547
+ {
1548
+ "epoch": 0.54,
1549
+ "grad_norm": 3.84375,
1550
+ "learning_rate": 2.269329101341745e-06,
1551
+ "loss": 1.1107,
1552
+ "step": 217
1553
+ },
1554
+ {
1555
+ "epoch": 0.54,
1556
+ "grad_norm": 3.875,
1557
+ "learning_rate": 2.249335539930843e-06,
1558
+ "loss": 1.0235,
1559
+ "step": 218
1560
+ },
1561
+ {
1562
+ "epoch": 0.55,
1563
+ "grad_norm": 3.546875,
1564
+ "learning_rate": 2.2293581606861298e-06,
1565
+ "loss": 1.0638,
1566
+ "step": 219
1567
+ },
1568
+ {
1569
+ "epoch": 0.55,
1570
+ "grad_norm": 3.640625,
1571
+ "learning_rate": 2.20939825328892e-06,
1572
+ "loss": 1.0233,
1573
+ "step": 220
1574
+ },
1575
+ {
1576
+ "epoch": 0.55,
1577
+ "grad_norm": 3.6875,
1578
+ "learning_rate": 2.1894571062926008e-06,
1579
+ "loss": 0.9618,
1580
+ "step": 221
1581
+ },
1582
+ {
1583
+ "epoch": 0.55,
1584
+ "grad_norm": 3.84375,
1585
+ "learning_rate": 2.1695360070394396e-06,
1586
+ "loss": 0.9921,
1587
+ "step": 222
1588
+ },
1589
+ {
1590
+ "epoch": 0.56,
1591
+ "grad_norm": 3.90625,
1592
+ "learning_rate": 2.1496362415774814e-06,
1593
+ "loss": 1.0143,
1594
+ "step": 223
1595
+ },
1596
+ {
1597
+ "epoch": 0.56,
1598
+ "grad_norm": 3.59375,
1599
+ "learning_rate": 2.1297590945775244e-06,
1600
+ "loss": 0.9744,
1601
+ "step": 224
1602
+ },
1603
+ {
1604
+ "epoch": 0.56,
1605
+ "grad_norm": 3.59375,
1606
+ "learning_rate": 2.1099058492501825e-06,
1607
+ "loss": 1.0126,
1608
+ "step": 225
1609
+ },
1610
+ {
1611
+ "epoch": 0.56,
1612
+ "grad_norm": 3.96875,
1613
+ "learning_rate": 2.0900777872630513e-06,
1614
+ "loss": 1.1166,
1615
+ "step": 226
1616
+ },
1617
+ {
1618
+ "epoch": 0.57,
1619
+ "grad_norm": 3.515625,
1620
+ "learning_rate": 2.070276188657959e-06,
1621
+ "loss": 0.9741,
1622
+ "step": 227
1623
+ },
1624
+ {
1625
+ "epoch": 0.57,
1626
+ "grad_norm": 3.546875,
1627
+ "learning_rate": 2.0505023317683374e-06,
1628
+ "loss": 0.9359,
1629
+ "step": 228
1630
+ },
1631
+ {
1632
+ "epoch": 0.57,
1633
+ "grad_norm": 3.734375,
1634
+ "learning_rate": 2.0307574931366954e-06,
1635
+ "loss": 1.0426,
1636
+ "step": 229
1637
+ },
1638
+ {
1639
+ "epoch": 0.57,
1640
+ "grad_norm": 3.765625,
1641
+ "learning_rate": 2.011042947432204e-06,
1642
+ "loss": 1.0191,
1643
+ "step": 230
1644
+ },
1645
+ {
1646
+ "epoch": 0.57,
1647
+ "grad_norm": 3.734375,
1648
+ "learning_rate": 1.991359967368416e-06,
1649
+ "loss": 1.0692,
1650
+ "step": 231
1651
+ },
1652
+ {
1653
+ "epoch": 0.58,
1654
+ "grad_norm": 3.875,
1655
+ "learning_rate": 1.9717098236210944e-06,
1656
+ "loss": 0.9858,
1657
+ "step": 232
1658
+ },
1659
+ {
1660
+ "epoch": 0.58,
1661
+ "grad_norm": 3.90625,
1662
+ "learning_rate": 1.9520937847461867e-06,
1663
+ "loss": 0.9709,
1664
+ "step": 233
1665
+ },
1666
+ {
1667
+ "epoch": 0.58,
1668
+ "grad_norm": 3.90625,
1669
+ "learning_rate": 1.9325131170979316e-06,
1670
+ "loss": 1.1824,
1671
+ "step": 234
1672
+ },
1673
+ {
1674
+ "epoch": 0.58,
1675
+ "grad_norm": 3.859375,
1676
+ "learning_rate": 1.9129690847471e-06,
1677
+ "loss": 0.9967,
1678
+ "step": 235
1679
+ },
1680
+ {
1681
+ "epoch": 0.59,
1682
+ "grad_norm": 4.0625,
1683
+ "learning_rate": 1.8934629493994005e-06,
1684
+ "loss": 0.9896,
1685
+ "step": 236
1686
+ },
1687
+ {
1688
+ "epoch": 0.59,
1689
+ "grad_norm": 4.0625,
1690
+ "learning_rate": 1.8739959703140167e-06,
1691
+ "loss": 1.0194,
1692
+ "step": 237
1693
+ },
1694
+ {
1695
+ "epoch": 0.59,
1696
+ "grad_norm": 3.765625,
1697
+ "learning_rate": 1.8545694042223205e-06,
1698
+ "loss": 0.9915,
1699
+ "step": 238
1700
+ },
1701
+ {
1702
+ "epoch": 0.59,
1703
+ "grad_norm": 4.0625,
1704
+ "learning_rate": 1.8351845052467403e-06,
1705
+ "loss": 1.0117,
1706
+ "step": 239
1707
+ },
1708
+ {
1709
+ "epoch": 0.6,
1710
+ "grad_norm": 3.625,
1711
+ "learning_rate": 1.8158425248197931e-06,
1712
+ "loss": 0.9527,
1713
+ "step": 240
1714
+ },
1715
+ {
1716
+ "epoch": 0.6,
1717
+ "grad_norm": 3.375,
1718
+ "learning_rate": 1.7965447116033036e-06,
1719
+ "loss": 0.9097,
1720
+ "step": 241
1721
+ },
1722
+ {
1723
+ "epoch": 0.6,
1724
+ "grad_norm": 3.765625,
1725
+ "learning_rate": 1.7772923114077871e-06,
1726
+ "loss": 0.9966,
1727
+ "step": 242
1728
+ },
1729
+ {
1730
+ "epoch": 0.6,
1731
+ "grad_norm": 3.96875,
1732
+ "learning_rate": 1.7580865671120295e-06,
1733
+ "loss": 1.0222,
1734
+ "step": 243
1735
+ },
1736
+ {
1737
+ "epoch": 0.61,
1738
+ "grad_norm": 3.6875,
1739
+ "learning_rate": 1.738928718582847e-06,
1740
+ "loss": 0.9598,
1741
+ "step": 244
1742
+ },
1743
+ {
1744
+ "epoch": 0.61,
1745
+ "grad_norm": 3.625,
1746
+ "learning_rate": 1.7198200025950423e-06,
1747
+ "loss": 1.0023,
1748
+ "step": 245
1749
+ },
1750
+ {
1751
+ "epoch": 0.61,
1752
+ "grad_norm": 3.84375,
1753
+ "learning_rate": 1.7007616527515695e-06,
1754
+ "loss": 1.0632,
1755
+ "step": 246
1756
+ },
1757
+ {
1758
+ "epoch": 0.61,
1759
+ "grad_norm": 3.765625,
1760
+ "learning_rate": 1.6817548994038862e-06,
1761
+ "loss": 1.054,
1762
+ "step": 247
1763
+ },
1764
+ {
1765
+ "epoch": 0.62,
1766
+ "grad_norm": 4.03125,
1767
+ "learning_rate": 1.6628009695725348e-06,
1768
+ "loss": 1.0083,
1769
+ "step": 248
1770
+ },
1771
+ {
1772
+ "epoch": 0.62,
1773
+ "grad_norm": 3.84375,
1774
+ "learning_rate": 1.643901086867924e-06,
1775
+ "loss": 1.0685,
1776
+ "step": 249
1777
+ },
1778
+ {
1779
+ "epoch": 0.62,
1780
+ "grad_norm": 3.734375,
1781
+ "learning_rate": 1.6250564714113364e-06,
1782
+ "loss": 0.9871,
1783
+ "step": 250
1784
+ },
1785
+ {
1786
+ "epoch": 0.62,
1787
+ "grad_norm": 3.6875,
1788
+ "learning_rate": 1.606268339756166e-06,
1789
+ "loss": 1.0499,
1790
+ "step": 251
1791
+ },
1792
+ {
1793
+ "epoch": 0.63,
1794
+ "grad_norm": 3.75,
1795
+ "learning_rate": 1.5875379048093723e-06,
1796
+ "loss": 1.0568,
1797
+ "step": 252
1798
+ },
1799
+ {
1800
+ "epoch": 0.63,
1801
+ "grad_norm": 3.921875,
1802
+ "learning_rate": 1.568866375753188e-06,
1803
+ "loss": 1.0213,
1804
+ "step": 253
1805
+ },
1806
+ {
1807
+ "epoch": 0.63,
1808
+ "grad_norm": 3.59375,
1809
+ "learning_rate": 1.5502549579670522e-06,
1810
+ "loss": 1.0704,
1811
+ "step": 254
1812
+ },
1813
+ {
1814
+ "epoch": 0.63,
1815
+ "grad_norm": 3.96875,
1816
+ "learning_rate": 1.5317048529497938e-06,
1817
+ "loss": 1.055,
1818
+ "step": 255
1819
+ },
1820
+ {
1821
+ "epoch": 0.64,
1822
+ "grad_norm": 3.4375,
1823
+ "learning_rate": 1.5132172582420713e-06,
1824
+ "loss": 0.9842,
1825
+ "step": 256
1826
+ },
1827
+ {
1828
+ "epoch": 0.64,
1829
+ "grad_norm": 3.75,
1830
+ "learning_rate": 1.4947933673490566e-06,
1831
+ "loss": 1.0064,
1832
+ "step": 257
1833
+ },
1834
+ {
1835
+ "epoch": 0.64,
1836
+ "grad_norm": 3.84375,
1837
+ "learning_rate": 1.4764343696633933e-06,
1838
+ "loss": 1.0286,
1839
+ "step": 258
1840
+ },
1841
+ {
1842
+ "epoch": 0.64,
1843
+ "grad_norm": 3.921875,
1844
+ "learning_rate": 1.4581414503884051e-06,
1845
+ "loss": 0.9656,
1846
+ "step": 259
1847
+ },
1848
+ {
1849
+ "epoch": 0.65,
1850
+ "grad_norm": 3.734375,
1851
+ "learning_rate": 1.439915790461588e-06,
1852
+ "loss": 1.0944,
1853
+ "step": 260
1854
+ },
1855
+ {
1856
+ "epoch": 0.65,
1857
+ "grad_norm": 3.625,
1858
+ "learning_rate": 1.421758566478371e-06,
1859
+ "loss": 1.0269,
1860
+ "step": 261
1861
+ },
1862
+ {
1863
+ "epoch": 0.65,
1864
+ "grad_norm": 3.875,
1865
+ "learning_rate": 1.4036709506161577e-06,
1866
+ "loss": 0.9061,
1867
+ "step": 262
1868
+ },
1869
+ {
1870
+ "epoch": 0.65,
1871
+ "grad_norm": 3.734375,
1872
+ "learning_rate": 1.3856541105586545e-06,
1873
+ "loss": 0.9597,
1874
+ "step": 263
1875
+ },
1876
+ {
1877
+ "epoch": 0.66,
1878
+ "grad_norm": 3.671875,
1879
+ "learning_rate": 1.3677092094204886e-06,
1880
+ "loss": 1.0282,
1881
+ "step": 264
1882
+ },
1883
+ {
1884
+ "epoch": 0.66,
1885
+ "grad_norm": 3.671875,
1886
+ "learning_rate": 1.3498374056721198e-06,
1887
+ "loss": 1.0582,
1888
+ "step": 265
1889
+ },
1890
+ {
1891
+ "epoch": 0.66,
1892
+ "grad_norm": 3.734375,
1893
+ "learning_rate": 1.332039853065055e-06,
1894
+ "loss": 1.0319,
1895
+ "step": 266
1896
+ },
1897
+ {
1898
+ "epoch": 0.66,
1899
+ "grad_norm": 3.71875,
1900
+ "learning_rate": 1.3143177005573626e-06,
1901
+ "loss": 1.0384,
1902
+ "step": 267
1903
+ },
1904
+ {
1905
+ "epoch": 0.67,
1906
+ "grad_norm": 3.75,
1907
+ "learning_rate": 1.2966720922395015e-06,
1908
+ "loss": 0.9468,
1909
+ "step": 268
1910
+ },
1911
+ {
1912
+ "epoch": 0.67,
1913
+ "grad_norm": 3.59375,
1914
+ "learning_rate": 1.2791041672604609e-06,
1915
+ "loss": 1.0335,
1916
+ "step": 269
1917
+ },
1918
+ {
1919
+ "epoch": 0.67,
1920
+ "grad_norm": 3.8125,
1921
+ "learning_rate": 1.2616150597542197e-06,
1922
+ "loss": 0.9515,
1923
+ "step": 270
1924
+ },
1925
+ {
1926
+ "epoch": 0.67,
1927
+ "grad_norm": 3.609375,
1928
+ "learning_rate": 1.244205898766532e-06,
1929
+ "loss": 1.0266,
1930
+ "step": 271
1931
+ },
1932
+ {
1933
+ "epoch": 0.68,
1934
+ "grad_norm": 3.625,
1935
+ "learning_rate": 1.2268778081820363e-06,
1936
+ "loss": 1.0266,
1937
+ "step": 272
1938
+ },
1939
+ {
1940
+ "epoch": 0.68,
1941
+ "grad_norm": 3.59375,
1942
+ "learning_rate": 1.2096319066517037e-06,
1943
+ "loss": 0.8807,
1944
+ "step": 273
1945
+ },
1946
+ {
1947
+ "epoch": 0.68,
1948
+ "grad_norm": 4.0,
1949
+ "learning_rate": 1.192469307520619e-06,
1950
+ "loss": 1.066,
1951
+ "step": 274
1952
+ },
1953
+ {
1954
+ "epoch": 0.68,
1955
+ "grad_norm": 3.453125,
1956
+ "learning_rate": 1.1753911187561075e-06,
1957
+ "loss": 1.002,
1958
+ "step": 275
1959
+ },
1960
+ {
1961
+ "epoch": 0.69,
1962
+ "grad_norm": 3.71875,
1963
+ "learning_rate": 1.1583984428762076e-06,
1964
+ "loss": 1.006,
1965
+ "step": 276
1966
+ },
1967
+ {
1968
+ "epoch": 0.69,
1969
+ "grad_norm": 3.859375,
1970
+ "learning_rate": 1.1414923768784952e-06,
1971
+ "loss": 1.0167,
1972
+ "step": 277
1973
+ },
1974
+ {
1975
+ "epoch": 0.69,
1976
+ "grad_norm": 3.921875,
1977
+ "learning_rate": 1.1246740121692649e-06,
1978
+ "loss": 1.0791,
1979
+ "step": 278
1980
+ },
1981
+ {
1982
+ "epoch": 0.69,
1983
+ "grad_norm": 3.984375,
1984
+ "learning_rate": 1.1079444344930717e-06,
1985
+ "loss": 1.013,
1986
+ "step": 279
1987
+ },
1988
+ {
1989
+ "epoch": 0.7,
1990
+ "grad_norm": 3.75,
1991
+ "learning_rate": 1.0913047238626394e-06,
1992
+ "loss": 1.0024,
1993
+ "step": 280
1994
+ },
1995
+ {
1996
+ "epoch": 0.7,
1997
+ "grad_norm": 4.0,
1998
+ "learning_rate": 1.0747559544891376e-06,
1999
+ "loss": 1.0406,
2000
+ "step": 281
2001
+ },
2002
+ {
2003
+ "epoch": 0.7,
2004
+ "grad_norm": 3.578125,
2005
+ "learning_rate": 1.0582991947128324e-06,
2006
+ "loss": 1.0518,
2007
+ "step": 282
2008
+ },
2009
+ {
2010
+ "epoch": 0.7,
2011
+ "grad_norm": 3.75,
2012
+ "learning_rate": 1.0419355069341206e-06,
2013
+ "loss": 1.0486,
2014
+ "step": 283
2015
+ },
2016
+ {
2017
+ "epoch": 0.71,
2018
+ "grad_norm": 3.65625,
2019
+ "learning_rate": 1.025665947544941e-06,
2020
+ "loss": 0.9368,
2021
+ "step": 284
2022
+ },
2023
+ {
2024
+ "epoch": 0.71,
2025
+ "grad_norm": 3.515625,
2026
+ "learning_rate": 1.0094915668605786e-06,
2027
+ "loss": 0.958,
2028
+ "step": 285
2029
+ },
2030
+ {
2031
+ "epoch": 0.71,
2032
+ "grad_norm": 3.765625,
2033
+ "learning_rate": 9.934134090518593e-07,
2034
+ "loss": 1.0873,
2035
+ "step": 286
2036
+ },
2037
+ {
2038
+ "epoch": 0.71,
2039
+ "grad_norm": 3.34375,
2040
+ "learning_rate": 9.774325120777406e-07,
2041
+ "loss": 1.0053,
2042
+ "step": 287
2043
+ },
2044
+ {
2045
+ "epoch": 0.72,
2046
+ "grad_norm": 3.859375,
2047
+ "learning_rate": 9.61549907618304e-07,
2048
+ "loss": 1.1148,
2049
+ "step": 288
2050
+ },
2051
+ {
2052
+ "epoch": 0.72,
2053
+ "grad_norm": 3.546875,
2054
+ "learning_rate": 9.457666210081537e-07,
2055
+ "loss": 1.1638,
2056
+ "step": 289
2057
+ },
2058
+ {
2059
+ "epoch": 0.72,
2060
+ "grad_norm": 3.6875,
2061
+ "learning_rate": 9.300836711702224e-07,
2062
+ "loss": 1.017,
2063
+ "step": 290
2064
+ },
2065
+ {
2066
+ "epoch": 0.72,
2067
+ "grad_norm": 3.609375,
2068
+ "learning_rate": 9.145020705499947e-07,
2069
+ "loss": 1.0333,
2070
+ "step": 291
2071
+ },
2072
+ {
2073
+ "epoch": 0.73,
2074
+ "grad_norm": 3.953125,
2075
+ "learning_rate": 8.990228250501446e-07,
2076
+ "loss": 1.099,
2077
+ "step": 292
2078
+ },
2079
+ {
2080
+ "epoch": 0.73,
2081
+ "grad_norm": 3.671875,
2082
+ "learning_rate": 8.836469339655979e-07,
2083
+ "loss": 0.9876,
2084
+ "step": 293
2085
+ },
2086
+ {
2087
+ "epoch": 0.73,
2088
+ "grad_norm": 3.578125,
2089
+ "learning_rate": 8.68375389919022e-07,
2090
+ "loss": 1.0734,
2091
+ "step": 294
2092
+ },
2093
+ {
2094
+ "epoch": 0.73,
2095
+ "grad_norm": 3.8125,
2096
+ "learning_rate": 8.532091787967428e-07,
2097
+ "loss": 1.1269,
2098
+ "step": 295
2099
+ },
2100
+ {
2101
+ "epoch": 0.74,
2102
+ "grad_norm": 3.65625,
2103
+ "learning_rate": 8.38149279685101e-07,
2104
+ "loss": 0.9446,
2105
+ "step": 296
2106
+ },
2107
+ {
2108
+ "epoch": 0.74,
2109
+ "grad_norm": 4.0625,
2110
+ "learning_rate": 8.231966648072431e-07,
2111
+ "loss": 1.0824,
2112
+ "step": 297
2113
+ },
2114
+ {
2115
+ "epoch": 0.74,
2116
+ "grad_norm": 3.765625,
2117
+ "learning_rate": 8.083522994603593e-07,
2118
+ "loss": 1.0151,
2119
+ "step": 298
2120
+ },
2121
+ {
2122
+ "epoch": 0.74,
2123
+ "grad_norm": 3.703125,
2124
+ "learning_rate": 7.936171419533653e-07,
2125
+ "loss": 1.0191,
2126
+ "step": 299
2127
+ },
2128
+ {
2129
+ "epoch": 0.75,
2130
+ "grad_norm": 3.46875,
2131
+ "learning_rate": 7.789921435450373e-07,
2132
+ "loss": 1.0482,
2133
+ "step": 300
2134
+ },
2135
+ {
2136
+ "epoch": 0.75,
2137
+ "grad_norm": 3.8125,
2138
+ "learning_rate": 7.644782483826016e-07,
2139
+ "loss": 1.0616,
2140
+ "step": 301
2141
+ },
2142
+ {
2143
+ "epoch": 0.75,
2144
+ "grad_norm": 3.859375,
2145
+ "learning_rate": 7.500763934407851e-07,
2146
+ "loss": 0.958,
2147
+ "step": 302
2148
+ },
2149
+ {
2150
+ "epoch": 0.75,
2151
+ "grad_norm": 3.65625,
2152
+ "learning_rate": 7.357875084613208e-07,
2153
+ "loss": 0.9762,
2154
+ "step": 303
2155
+ },
2156
+ {
2157
+ "epoch": 0.75,
2158
+ "eval_loss": 1.1502219438552856,
2159
+ "eval_runtime": 1698.3617,
2160
+ "eval_samples_per_second": 5.93,
2161
+ "eval_steps_per_second": 0.741,
2162
+ "step": 303
2163
+ },
2164
+ {
2165
+ "epoch": 0.76,
2166
+ "grad_norm": 3.484375,
2167
+ "learning_rate": 7.216125158929337e-07,
2168
+ "loss": 1.0086,
2169
+ "step": 304
2170
+ },
2171
+ {
2172
+ "epoch": 0.76,
2173
+ "grad_norm": 3.734375,
2174
+ "learning_rate": 7.075523308317863e-07,
2175
+ "loss": 0.9599,
2176
+ "step": 305
2177
+ },
2178
+ {
2179
+ "epoch": 0.76,
2180
+ "grad_norm": 3.53125,
2181
+ "learning_rate": 6.936078609624023e-07,
2182
+ "loss": 0.9755,
2183
+ "step": 306
2184
+ },
2185
+ {
2186
+ "epoch": 0.76,
2187
+ "grad_norm": 3.65625,
2188
+ "learning_rate": 6.797800064990734e-07,
2189
+ "loss": 0.8239,
2190
+ "step": 307
2191
+ },
2192
+ {
2193
+ "epoch": 0.77,
2194
+ "grad_norm": 3.671875,
2195
+ "learning_rate": 6.660696601277369e-07,
2196
+ "loss": 1.0655,
2197
+ "step": 308
2198
+ },
2199
+ {
2200
+ "epoch": 0.77,
2201
+ "grad_norm": 3.609375,
2202
+ "learning_rate": 6.524777069483526e-07,
2203
+ "loss": 1.0598,
2204
+ "step": 309
2205
+ },
2206
+ {
2207
+ "epoch": 0.77,
2208
+ "grad_norm": 3.609375,
2209
+ "learning_rate": 6.390050244177615e-07,
2210
+ "loss": 1.0022,
2211
+ "step": 310
2212
+ },
2213
+ {
2214
+ "epoch": 0.77,
2215
+ "grad_norm": 3.546875,
2216
+ "learning_rate": 6.256524822930379e-07,
2217
+ "loss": 1.0651,
2218
+ "step": 311
2219
+ },
2220
+ {
2221
+ "epoch": 0.78,
2222
+ "grad_norm": 3.984375,
2223
+ "learning_rate": 6.124209425753455e-07,
2224
+ "loss": 0.9499,
2225
+ "step": 312
2226
+ },
2227
+ {
2228
+ "epoch": 0.78,
2229
+ "grad_norm": 3.546875,
2230
+ "learning_rate": 5.993112594542813e-07,
2231
+ "loss": 0.9701,
2232
+ "step": 313
2233
+ },
2234
+ {
2235
+ "epoch": 0.78,
2236
+ "grad_norm": 3.734375,
2237
+ "learning_rate": 5.863242792527385e-07,
2238
+ "loss": 1.0464,
2239
+ "step": 314
2240
+ },
2241
+ {
2242
+ "epoch": 0.78,
2243
+ "grad_norm": 3.6875,
2244
+ "learning_rate": 5.734608403722674e-07,
2245
+ "loss": 1.0288,
2246
+ "step": 315
2247
+ },
2248
+ {
2249
+ "epoch": 0.79,
2250
+ "grad_norm": 4.1875,
2251
+ "learning_rate": 5.607217732389503e-07,
2252
+ "loss": 1.0578,
2253
+ "step": 316
2254
+ },
2255
+ {
2256
+ "epoch": 0.79,
2257
+ "grad_norm": 3.921875,
2258
+ "learning_rate": 5.481079002497946e-07,
2259
+ "loss": 1.0421,
2260
+ "step": 317
2261
+ },
2262
+ {
2263
+ "epoch": 0.79,
2264
+ "grad_norm": 4.25,
2265
+ "learning_rate": 5.356200357196362e-07,
2266
+ "loss": 1.0306,
2267
+ "step": 318
2268
+ },
2269
+ {
2270
+ "epoch": 0.79,
2271
+ "grad_norm": 3.71875,
2272
+ "learning_rate": 5.232589858285734e-07,
2273
+ "loss": 1.0028,
2274
+ "step": 319
2275
+ },
2276
+ {
2277
+ "epoch": 0.8,
2278
+ "grad_norm": 4.25,
2279
+ "learning_rate": 5.110255485699237e-07,
2280
+ "loss": 0.9517,
2281
+ "step": 320
2282
+ },
2283
+ {
2284
+ "epoch": 0.8,
2285
+ "grad_norm": 3.625,
2286
+ "learning_rate": 4.989205136987007e-07,
2287
+ "loss": 0.9468,
2288
+ "step": 321
2289
+ },
2290
+ {
2291
+ "epoch": 0.8,
2292
+ "grad_norm": 3.703125,
2293
+ "learning_rate": 4.869446626806404e-07,
2294
+ "loss": 1.0133,
2295
+ "step": 322
2296
+ },
2297
+ {
2298
+ "epoch": 0.8,
2299
+ "grad_norm": 3.578125,
2300
+ "learning_rate": 4.750987686417405e-07,
2301
+ "loss": 0.9474,
2302
+ "step": 323
2303
+ },
2304
+ {
2305
+ "epoch": 0.81,
2306
+ "grad_norm": 3.625,
2307
+ "learning_rate": 4.633835963183583e-07,
2308
+ "loss": 0.9537,
2309
+ "step": 324
2310
+ },
2311
+ {
2312
+ "epoch": 0.81,
2313
+ "grad_norm": 3.578125,
2314
+ "learning_rate": 4.5179990200784006e-07,
2315
+ "loss": 0.9295,
2316
+ "step": 325
2317
+ },
2318
+ {
2319
+ "epoch": 0.81,
2320
+ "grad_norm": 3.53125,
2321
+ "learning_rate": 4.4034843351969005e-07,
2322
+ "loss": 1.0759,
2323
+ "step": 326
2324
+ },
2325
+ {
2326
+ "epoch": 0.81,
2327
+ "grad_norm": 3.546875,
2328
+ "learning_rate": 4.2902993012730543e-07,
2329
+ "loss": 0.9895,
2330
+ "step": 327
2331
+ },
2332
+ {
2333
+ "epoch": 0.82,
2334
+ "grad_norm": 3.609375,
2335
+ "learning_rate": 4.178451225202396e-07,
2336
+ "loss": 1.0568,
2337
+ "step": 328
2338
+ },
2339
+ {
2340
+ "epoch": 0.82,
2341
+ "grad_norm": 3.578125,
2342
+ "learning_rate": 4.067947327570379e-07,
2343
+ "loss": 1.0846,
2344
+ "step": 329
2345
+ },
2346
+ {
2347
+ "epoch": 0.82,
2348
+ "grad_norm": 3.875,
2349
+ "learning_rate": 3.958794742186245e-07,
2350
+ "loss": 1.0305,
2351
+ "step": 330
2352
+ },
2353
+ {
2354
+ "epoch": 0.82,
2355
+ "grad_norm": 3.625,
2356
+ "learning_rate": 3.851000515622408e-07,
2357
+ "loss": 0.9988,
2358
+ "step": 331
2359
+ },
2360
+ {
2361
+ "epoch": 0.83,
2362
+ "grad_norm": 3.6875,
2363
+ "learning_rate": 3.7445716067596506e-07,
2364
+ "loss": 1.0955,
2365
+ "step": 332
2366
+ },
2367
+ {
2368
+ "epoch": 0.83,
2369
+ "grad_norm": 3.796875,
2370
+ "learning_rate": 3.639514886337786e-07,
2371
+ "loss": 1.0371,
2372
+ "step": 333
2373
+ },
2374
+ {
2375
+ "epoch": 0.83,
2376
+ "grad_norm": 3.796875,
2377
+ "learning_rate": 3.5358371365121766e-07,
2378
+ "loss": 1.0718,
2379
+ "step": 334
2380
+ },
2381
+ {
2382
+ "epoch": 0.83,
2383
+ "grad_norm": 3.6875,
2384
+ "learning_rate": 3.43354505041586e-07,
2385
+ "loss": 0.9978,
2386
+ "step": 335
2387
+ },
2388
+ {
2389
+ "epoch": 0.84,
2390
+ "grad_norm": 3.6875,
2391
+ "learning_rate": 3.332645231727441e-07,
2392
+ "loss": 1.0067,
2393
+ "step": 336
2394
+ },
2395
+ {
2396
+ "epoch": 0.84,
2397
+ "grad_norm": 3.5625,
2398
+ "learning_rate": 3.2331441942448446e-07,
2399
+ "loss": 0.9888,
2400
+ "step": 337
2401
+ },
2402
+ {
2403
+ "epoch": 0.84,
2404
+ "grad_norm": 3.484375,
2405
+ "learning_rate": 3.135048361464721e-07,
2406
+ "loss": 0.9309,
2407
+ "step": 338
2408
+ },
2409
+ {
2410
+ "epoch": 0.84,
2411
+ "grad_norm": 3.859375,
2412
+ "learning_rate": 3.0383640661678344e-07,
2413
+ "loss": 1.0667,
2414
+ "step": 339
2415
+ },
2416
+ {
2417
+ "epoch": 0.85,
2418
+ "grad_norm": 3.71875,
2419
+ "learning_rate": 2.943097550010204e-07,
2420
+ "loss": 1.0255,
2421
+ "step": 340
2422
+ },
2423
+ {
2424
+ "epoch": 0.85,
2425
+ "grad_norm": 3.6875,
2426
+ "learning_rate": 2.8492549631201376e-07,
2427
+ "loss": 1.03,
2428
+ "step": 341
2429
+ },
2430
+ {
2431
+ "epoch": 0.85,
2432
+ "grad_norm": 3.703125,
2433
+ "learning_rate": 2.7568423637012646e-07,
2434
+ "loss": 1.019,
2435
+ "step": 342
2436
+ },
2437
+ {
2438
+ "epoch": 0.85,
2439
+ "grad_norm": 3.703125,
2440
+ "learning_rate": 2.6658657176413527e-07,
2441
+ "loss": 1.017,
2442
+ "step": 343
2443
+ },
2444
+ {
2445
+ "epoch": 0.86,
2446
+ "grad_norm": 3.703125,
2447
+ "learning_rate": 2.5763308981272525e-07,
2448
+ "loss": 1.0516,
2449
+ "step": 344
2450
+ },
2451
+ {
2452
+ "epoch": 0.86,
2453
+ "grad_norm": 3.34375,
2454
+ "learning_rate": 2.488243685265676e-07,
2455
+ "loss": 0.9736,
2456
+ "step": 345
2457
+ },
2458
+ {
2459
+ "epoch": 0.86,
2460
+ "grad_norm": 3.828125,
2461
+ "learning_rate": 2.401609765710064e-07,
2462
+ "loss": 0.9969,
2463
+ "step": 346
2464
+ },
2465
+ {
2466
+ "epoch": 0.86,
2467
+ "grad_norm": 3.609375,
2468
+ "learning_rate": 2.3164347322935127e-07,
2469
+ "loss": 0.9944,
2470
+ "step": 347
2471
+ },
2472
+ {
2473
+ "epoch": 0.87,
2474
+ "grad_norm": 3.53125,
2475
+ "learning_rate": 2.232724083667645e-07,
2476
+ "loss": 1.0454,
2477
+ "step": 348
2478
+ },
2479
+ {
2480
+ "epoch": 0.87,
2481
+ "grad_norm": 3.453125,
2482
+ "learning_rate": 2.1504832239477192e-07,
2483
+ "loss": 0.8774,
2484
+ "step": 349
2485
+ },
2486
+ {
2487
+ "epoch": 0.87,
2488
+ "grad_norm": 3.71875,
2489
+ "learning_rate": 2.0697174623636795e-07,
2490
+ "loss": 1.0629,
2491
+ "step": 350
2492
+ },
2493
+ {
2494
+ "epoch": 0.87,
2495
+ "grad_norm": 3.671875,
2496
+ "learning_rate": 1.9904320129174343e-07,
2497
+ "loss": 0.993,
2498
+ "step": 351
2499
+ },
2500
+ {
2501
+ "epoch": 0.88,
2502
+ "grad_norm": 3.828125,
2503
+ "learning_rate": 1.9126319940462935e-07,
2504
+ "loss": 1.0039,
2505
+ "step": 352
2506
+ },
2507
+ {
2508
+ "epoch": 0.88,
2509
+ "grad_norm": 3.53125,
2510
+ "learning_rate": 1.836322428292464e-07,
2511
+ "loss": 0.9873,
2512
+ "step": 353
2513
+ },
2514
+ {
2515
+ "epoch": 0.88,
2516
+ "grad_norm": 3.71875,
2517
+ "learning_rate": 1.761508241978871e-07,
2518
+ "loss": 1.1026,
2519
+ "step": 354
2520
+ },
2521
+ {
2522
+ "epoch": 0.88,
2523
+ "grad_norm": 3.765625,
2524
+ "learning_rate": 1.6881942648911077e-07,
2525
+ "loss": 1.0244,
2526
+ "step": 355
2527
+ },
2528
+ {
2529
+ "epoch": 0.89,
2530
+ "grad_norm": 3.75,
2531
+ "learning_rate": 1.6163852299656142e-07,
2532
+ "loss": 1.0486,
2533
+ "step": 356
2534
+ },
2535
+ {
2536
+ "epoch": 0.89,
2537
+ "grad_norm": 3.671875,
2538
+ "learning_rate": 1.5460857729841877e-07,
2539
+ "loss": 0.9962,
2540
+ "step": 357
2541
+ },
2542
+ {
2543
+ "epoch": 0.89,
2544
+ "grad_norm": 3.375,
2545
+ "learning_rate": 1.477300432274645e-07,
2546
+ "loss": 1.0021,
2547
+ "step": 358
2548
+ },
2549
+ {
2550
+ "epoch": 0.89,
2551
+ "grad_norm": 3.546875,
2552
+ "learning_rate": 1.4100336484178995e-07,
2553
+ "loss": 0.9262,
2554
+ "step": 359
2555
+ },
2556
+ {
2557
+ "epoch": 0.9,
2558
+ "grad_norm": 3.71875,
2559
+ "learning_rate": 1.344289763961257e-07,
2560
+ "loss": 1.1243,
2561
+ "step": 360
2562
+ },
2563
+ {
2564
+ "epoch": 0.9,
2565
+ "grad_norm": 3.671875,
2566
+ "learning_rate": 1.2800730231380675e-07,
2567
+ "loss": 1.0262,
2568
+ "step": 361
2569
+ },
2570
+ {
2571
+ "epoch": 0.9,
2572
+ "grad_norm": 3.59375,
2573
+ "learning_rate": 1.2173875715937726e-07,
2574
+ "loss": 0.9252,
2575
+ "step": 362
2576
+ },
2577
+ {
2578
+ "epoch": 0.9,
2579
+ "grad_norm": 3.5,
2580
+ "learning_rate": 1.1562374561182143e-07,
2581
+ "loss": 0.9864,
2582
+ "step": 363
2583
+ },
2584
+ {
2585
+ "epoch": 0.91,
2586
+ "grad_norm": 3.90625,
2587
+ "learning_rate": 1.0966266243844392e-07,
2588
+ "loss": 1.0454,
2589
+ "step": 364
2590
+ },
2591
+ {
2592
+ "epoch": 0.91,
2593
+ "grad_norm": 3.75,
2594
+ "learning_rate": 1.0385589246938194e-07,
2595
+ "loss": 0.9809,
2596
+ "step": 365
2597
+ },
2598
+ {
2599
+ "epoch": 0.91,
2600
+ "grad_norm": 3.75,
2601
+ "learning_rate": 9.82038105727623e-08,
2602
+ "loss": 1.0173,
2603
+ "step": 366
2604
+ },
2605
+ {
2606
+ "epoch": 0.91,
2607
+ "grad_norm": 3.609375,
2608
+ "learning_rate": 9.270678163050218e-08,
2609
+ "loss": 1.0639,
2610
+ "step": 367
2611
+ },
2612
+ {
2613
+ "epoch": 0.92,
2614
+ "grad_norm": 3.984375,
2615
+ "learning_rate": 8.736516051474992e-08,
2616
+ "loss": 0.9877,
2617
+ "step": 368
2618
+ },
2619
+ {
2620
+ "epoch": 0.92,
2621
+ "grad_norm": 3.625,
2622
+ "learning_rate": 8.217929206498137e-08,
2623
+ "loss": 0.9758,
2624
+ "step": 369
2625
+ },
2626
+ {
2627
+ "epoch": 0.92,
2628
+ "grad_norm": 3.453125,
2629
+ "learning_rate": 7.71495110657322e-08,
2630
+ "loss": 1.0034,
2631
+ "step": 370
2632
+ },
2633
+ {
2634
+ "epoch": 0.92,
2635
+ "grad_norm": 3.953125,
2636
+ "learning_rate": 7.227614222498875e-08,
2637
+ "loss": 1.1404,
2638
+ "step": 371
2639
+ },
2640
+ {
2641
+ "epoch": 0.93,
2642
+ "grad_norm": 3.609375,
2643
+ "learning_rate": 6.755950015322477e-08,
2644
+ "loss": 1.0372,
2645
+ "step": 372
2646
+ },
2647
+ {
2648
+ "epoch": 0.93,
2649
+ "grad_norm": 3.5625,
2650
+ "learning_rate": 6.299988934309026e-08,
2651
+ "loss": 0.975,
2652
+ "step": 373
2653
+ },
2654
+ {
2655
+ "epoch": 0.93,
2656
+ "grad_norm": 3.640625,
2657
+ "learning_rate": 5.859760414975601e-08,
2658
+ "loss": 0.9614,
2659
+ "step": 374
2660
+ },
2661
+ {
2662
+ "epoch": 0.93,
2663
+ "grad_norm": 3.640625,
2664
+ "learning_rate": 5.435292877190995e-08,
2665
+ "loss": 1.0394,
2666
+ "step": 375
2667
+ },
2668
+ {
2669
+ "epoch": 0.94,
2670
+ "grad_norm": 3.640625,
2671
+ "learning_rate": 5.026613723340956e-08,
2672
+ "loss": 0.9794,
2673
+ "step": 376
2674
+ },
2675
+ {
2676
+ "epoch": 0.94,
2677
+ "grad_norm": 3.890625,
2678
+ "learning_rate": 4.6337493365592736e-08,
2679
+ "loss": 1.015,
2680
+ "step": 377
2681
+ },
2682
+ {
2683
+ "epoch": 0.94,
2684
+ "grad_norm": 3.71875,
2685
+ "learning_rate": 4.256725079024554e-08,
2686
+ "loss": 1.087,
2687
+ "step": 378
2688
+ },
2689
+ {
2690
+ "epoch": 0.94,
2691
+ "grad_norm": 3.625,
2692
+ "learning_rate": 3.8955652903228114e-08,
2693
+ "loss": 0.9599,
2694
+ "step": 379
2695
+ },
2696
+ {
2697
+ "epoch": 0.95,
2698
+ "grad_norm": 3.875,
2699
+ "learning_rate": 3.550293285876222e-08,
2700
+ "loss": 0.964,
2701
+ "step": 380
2702
+ },
2703
+ {
2704
+ "epoch": 0.95,
2705
+ "grad_norm": 3.515625,
2706
+ "learning_rate": 3.220931355438051e-08,
2707
+ "loss": 0.9747,
2708
+ "step": 381
2709
+ },
2710
+ {
2711
+ "epoch": 0.95,
2712
+ "grad_norm": 3.515625,
2713
+ "learning_rate": 2.907500761653581e-08,
2714
+ "loss": 1.0822,
2715
+ "step": 382
2716
+ },
2717
+ {
2718
+ "epoch": 0.95,
2719
+ "grad_norm": 3.609375,
2720
+ "learning_rate": 2.6100217386873794e-08,
2721
+ "loss": 1.0306,
2722
+ "step": 383
2723
+ },
2724
+ {
2725
+ "epoch": 0.96,
2726
+ "grad_norm": 3.546875,
2727
+ "learning_rate": 2.3285134909173113e-08,
2728
+ "loss": 1.0334,
2729
+ "step": 384
2730
+ },
2731
+ {
2732
+ "epoch": 0.96,
2733
+ "grad_norm": 3.71875,
2734
+ "learning_rate": 2.0629941916944785e-08,
2735
+ "loss": 1.0246,
2736
+ "step": 385
2737
+ },
2738
+ {
2739
+ "epoch": 0.96,
2740
+ "grad_norm": 6.8125,
2741
+ "learning_rate": 1.8134809821701016e-08,
2742
+ "loss": 0.9324,
2743
+ "step": 386
2744
+ },
2745
+ {
2746
+ "epoch": 0.96,
2747
+ "grad_norm": 3.8125,
2748
+ "learning_rate": 1.579989970189044e-08,
2749
+ "loss": 1.1022,
2750
+ "step": 387
2751
+ },
2752
+ {
2753
+ "epoch": 0.97,
2754
+ "grad_norm": 3.9375,
2755
+ "learning_rate": 1.3625362292497835e-08,
2756
+ "loss": 1.0338,
2757
+ "step": 388
2758
+ },
2759
+ {
2760
+ "epoch": 0.97,
2761
+ "grad_norm": 4.09375,
2762
+ "learning_rate": 1.1611337975313553e-08,
2763
+ "loss": 0.9934,
2764
+ "step": 389
2765
+ },
2766
+ {
2767
+ "epoch": 0.97,
2768
+ "grad_norm": 3.53125,
2769
+ "learning_rate": 9.757956769871624e-09,
2770
+ "loss": 0.9822,
2771
+ "step": 390
2772
+ },
2773
+ {
2774
+ "epoch": 0.97,
2775
+ "grad_norm": 3.65625,
2776
+ "learning_rate": 8.065338325054795e-09,
2777
+ "loss": 1.0552,
2778
+ "step": 391
2779
+ },
2780
+ {
2781
+ "epoch": 0.98,
2782
+ "grad_norm": 3.53125,
2783
+ "learning_rate": 6.533591911371262e-09,
2784
+ "loss": 0.9611,
2785
+ "step": 392
2786
+ },
2787
+ {
2788
+ "epoch": 0.98,
2789
+ "grad_norm": 3.6875,
2790
+ "learning_rate": 5.162816413900873e-09,
2791
+ "loss": 0.954,
2792
+ "step": 393
2793
+ },
2794
+ {
2795
+ "epoch": 0.98,
2796
+ "grad_norm": 3.75,
2797
+ "learning_rate": 3.953100325909953e-09,
2798
+ "loss": 1.0505,
2799
+ "step": 394
2800
+ },
2801
+ {
2802
+ "epoch": 0.98,
2803
+ "grad_norm": 3.765625,
2804
+ "learning_rate": 2.9045217431397655e-09,
2805
+ "loss": 1.0135,
2806
+ "step": 395
2807
+ },
2808
+ {
2809
+ "epoch": 0.99,
2810
+ "grad_norm": 3.59375,
2811
+ "learning_rate": 2.017148358763876e-09,
2812
+ "loss": 1.0444,
2813
+ "step": 396
2814
+ },
2815
+ {
2816
+ "epoch": 0.99,
2817
+ "grad_norm": 3.65625,
2818
+ "learning_rate": 1.2910374590194286e-09,
2819
+ "loss": 1.0298,
2820
+ "step": 397
2821
+ },
2822
+ {
2823
+ "epoch": 0.99,
2824
+ "grad_norm": 3.625,
2825
+ "learning_rate": 7.262359195070456e-10,
2826
+ "loss": 1.1188,
2827
+ "step": 398
2828
+ },
2829
+ {
2830
+ "epoch": 0.99,
2831
+ "grad_norm": 3.765625,
2832
+ "learning_rate": 3.227802021665838e-10,
2833
+ "loss": 0.9863,
2834
+ "step": 399
2835
+ },
2836
+ {
2837
+ "epoch": 1.0,
2838
+ "grad_norm": 3.6875,
2839
+ "learning_rate": 8.069635292179412e-11,
2840
+ "loss": 1.0449,
2841
+ "step": 400
2842
+ },
2843
+ {
2844
+ "epoch": 1.0,
2845
+ "grad_norm": 3.4375,
2846
+ "learning_rate": 0.0,
2847
+ "loss": 0.9839,
2848
+ "step": 401
2849
+ }
2850
+ ],
2851
+ "logging_steps": 1,
2852
+ "max_steps": 401,
2853
+ "num_input_tokens_seen": 0,
2854
+ "num_train_epochs": 1,
2855
+ "save_steps": 500,
2856
+ "total_flos": 4.4848246137752125e+18,
2857
+ "train_batch_size": 2,
2858
+ "trial_name": null,
2859
+ "trial_params": null
2860
+ }
checkpoint-401/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:362ced6da0d7898d52ab2020fd33ac38835b0ffbc42b9542d2f86d5dcaedabb4
3
+ size 5688
config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
3
+ "architectures": [
4
+ "MistralForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 4096,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 14336,
13
+ "max_position_embeddings": 32768,
14
+ "model_type": "mistral",
15
+ "num_attention_heads": 32,
16
+ "num_hidden_layers": 32,
17
+ "num_key_value_heads": 8,
18
+ "rms_norm_eps": 1e-05,
19
+ "rope_theta": 1000000.0,
20
+ "sliding_window": null,
21
+ "tie_word_embeddings": false,
22
+ "torch_dtype": "bfloat16",
23
+ "transformers_version": "4.39.0.dev0",
24
+ "use_cache": false,
25
+ "vocab_size": 32000
26
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "do_sample": true,
5
+ "eos_token_id": 2,
6
+ "transformers_version": "4.39.0.dev0"
7
+ }
model.safetensors.index.json ADDED
@@ -0,0 +1,298 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 14483464192
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "model-00003-of-00003.safetensors",
7
+ "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
8
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
9
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
10
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
11
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
12
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
13
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
14
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
15
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
16
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
17
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
18
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
19
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
20
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
21
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
22
+ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
23
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
24
+ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
25
+ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
26
+ "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
27
+ "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
28
+ "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
29
+ "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
30
+ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
31
+ "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
32
+ "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
33
+ "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
34
+ "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
35
+ "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
36
+ "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
37
+ "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
38
+ "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
39
+ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
40
+ "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
41
+ "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
42
+ "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
43
+ "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
44
+ "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
45
+ "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
46
+ "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
47
+ "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
48
+ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
49
+ "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
50
+ "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
51
+ "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
52
+ "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
53
+ "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
54
+ "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
55
+ "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
56
+ "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
57
+ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
58
+ "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
59
+ "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
60
+ "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
61
+ "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
62
+ "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
63
+ "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
64
+ "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
65
+ "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
66
+ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
67
+ "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
68
+ "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
69
+ "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
70
+ "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
71
+ "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
72
+ "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
73
+ "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
74
+ "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
75
+ "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
76
+ "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
77
+ "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
78
+ "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
79
+ "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
80
+ "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
81
+ "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
82
+ "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
83
+ "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
84
+ "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
85
+ "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
86
+ "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
87
+ "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
88
+ "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
89
+ "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
90
+ "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
91
+ "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
92
+ "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
93
+ "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
94
+ "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
95
+ "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
96
+ "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
97
+ "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
98
+ "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
99
+ "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
100
+ "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
101
+ "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
102
+ "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
103
+ "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
104
+ "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
105
+ "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
106
+ "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
107
+ "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
108
+ "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
109
+ "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
110
+ "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
111
+ "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
112
+ "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
113
+ "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
114
+ "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
115
+ "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
116
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
117
+ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
118
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
119
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
120
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
121
+ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
122
+ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
123
+ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
124
+ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
125
+ "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
126
+ "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
127
+ "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
128
+ "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
129
+ "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
130
+ "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
131
+ "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
132
+ "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
133
+ "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
134
+ "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
135
+ "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
136
+ "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
137
+ "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
138
+ "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
139
+ "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
140
+ "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
141
+ "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
142
+ "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
143
+ "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
144
+ "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
145
+ "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
146
+ "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
147
+ "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
148
+ "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
149
+ "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
150
+ "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
151
+ "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
152
+ "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
153
+ "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
154
+ "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
155
+ "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
156
+ "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
157
+ "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
158
+ "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
159
+ "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
160
+ "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
161
+ "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
162
+ "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
163
+ "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
164
+ "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
165
+ "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
166
+ "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
167
+ "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
168
+ "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
169
+ "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
170
+ "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
171
+ "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
172
+ "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
173
+ "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
174
+ "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
175
+ "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
176
+ "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
177
+ "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
178
+ "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
179
+ "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
180
+ "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
181
+ "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
182
+ "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
183
+ "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
184
+ "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
185
+ "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
186
+ "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
187
+ "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
188
+ "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
189
+ "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
190
+ "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
191
+ "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
192
+ "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
193
+ "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
194
+ "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
195
+ "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
196
+ "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
197
+ "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
198
+ "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
199
+ "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
200
+ "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
201
+ "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
202
+ "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
203
+ "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
204
+ "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
205
+ "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
206
+ "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
207
+ "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
208
+ "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
209
+ "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
210
+ "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
211
+ "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
212
+ "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
213
+ "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
214
+ "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
215
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
216
+ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
217
+ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
218
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
219
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
220
+ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
221
+ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
222
+ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
223
+ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
224
+ "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
225
+ "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
226
+ "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
227
+ "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
228
+ "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
229
+ "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
230
+ "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
231
+ "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
232
+ "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
233
+ "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
234
+ "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
235
+ "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
236
+ "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
237
+ "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
238
+ "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
239
+ "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
240
+ "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
241
+ "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
242
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
243
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
244
+ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
245
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
246
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
247
+ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
248
+ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
249
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
250
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
251
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
252
+ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
253
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
254
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
255
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
256
+ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
257
+ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
258
+ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
259
+ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
260
+ "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
261
+ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
262
+ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
263
+ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
264
+ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
265
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
266
+ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
267
+ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
268
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
269
+ "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
270
+ "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
271
+ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
272
+ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
273
+ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
274
+ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
275
+ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
276
+ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
277
+ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
278
+ "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
279
+ "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
280
+ "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
281
+ "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
282
+ "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
283
+ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
284
+ "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
285
+ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
286
+ "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
287
+ "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
288
+ "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
289
+ "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
290
+ "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
291
+ "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
292
+ "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
293
+ "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
294
+ "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
295
+ "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
296
+ "model.norm.weight": "model-00003-of-00003.safetensors"
297
+ }
298
+ }
pytorch_model-00001-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1fc48b9bd38dfab31354873b7d9ffea79fe57580f8c169135a96770978c612da
3
+ size 772370432
pytorch_model.bin.index.json ADDED
@@ -0,0 +1,298 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 14483464192
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "pytorch_model-00003-of-00003.bin",
7
+ "model.embed_tokens.weight": "pytorch_model-00001-of-00003.bin",
8
+ "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
9
+ "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
10
+ "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
11
+ "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
12
+ "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
13
+ "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
14
+ "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
15
+ "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
16
+ "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
17
+ "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
18
+ "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
19
+ "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
20
+ "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
21
+ "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
22
+ "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
23
+ "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
24
+ "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
25
+ "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
26
+ "model.layers.10.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
27
+ "model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
28
+ "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
29
+ "model.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
30
+ "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
31
+ "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
32
+ "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
33
+ "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
34
+ "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
35
+ "model.layers.11.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
36
+ "model.layers.11.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
37
+ "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
38
+ "model.layers.11.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
39
+ "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
40
+ "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
41
+ "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
42
+ "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
43
+ "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
44
+ "model.layers.12.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
45
+ "model.layers.12.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
46
+ "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
47
+ "model.layers.12.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
48
+ "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
49
+ "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
50
+ "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
51
+ "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
52
+ "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
53
+ "model.layers.13.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
54
+ "model.layers.13.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
55
+ "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
56
+ "model.layers.13.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
57
+ "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
58
+ "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
59
+ "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
60
+ "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
61
+ "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
62
+ "model.layers.14.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
63
+ "model.layers.14.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
64
+ "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
65
+ "model.layers.14.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
66
+ "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
67
+ "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
68
+ "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
69
+ "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
70
+ "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
71
+ "model.layers.15.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
72
+ "model.layers.15.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
73
+ "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
74
+ "model.layers.15.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
75
+ "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
76
+ "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
77
+ "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
78
+ "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
79
+ "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
80
+ "model.layers.16.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
81
+ "model.layers.16.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
82
+ "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
83
+ "model.layers.16.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
84
+ "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
85
+ "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
86
+ "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
87
+ "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
88
+ "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
89
+ "model.layers.17.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
90
+ "model.layers.17.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
91
+ "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
92
+ "model.layers.17.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
93
+ "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
94
+ "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
95
+ "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
96
+ "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
97
+ "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
98
+ "model.layers.18.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
99
+ "model.layers.18.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
100
+ "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
101
+ "model.layers.18.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
102
+ "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
103
+ "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
104
+ "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
105
+ "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
106
+ "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
107
+ "model.layers.19.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
108
+ "model.layers.19.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
109
+ "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
110
+ "model.layers.19.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
111
+ "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
112
+ "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
113
+ "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
114
+ "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
115
+ "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
116
+ "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
117
+ "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
118
+ "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
119
+ "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
120
+ "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
121
+ "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
122
+ "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
123
+ "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
124
+ "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
125
+ "model.layers.20.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
126
+ "model.layers.20.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
127
+ "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
128
+ "model.layers.20.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
129
+ "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
130
+ "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
131
+ "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
132
+ "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
133
+ "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
134
+ "model.layers.21.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
135
+ "model.layers.21.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
136
+ "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
137
+ "model.layers.21.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
138
+ "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
139
+ "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
140
+ "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
141
+ "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
142
+ "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
143
+ "model.layers.22.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
144
+ "model.layers.22.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
145
+ "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
146
+ "model.layers.22.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
147
+ "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
148
+ "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
149
+ "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
150
+ "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
151
+ "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
152
+ "model.layers.23.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
153
+ "model.layers.23.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
154
+ "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
155
+ "model.layers.23.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
156
+ "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
157
+ "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
158
+ "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
159
+ "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
160
+ "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
161
+ "model.layers.24.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
162
+ "model.layers.24.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
163
+ "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
164
+ "model.layers.24.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
165
+ "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
166
+ "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
167
+ "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
168
+ "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
169
+ "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
170
+ "model.layers.25.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
171
+ "model.layers.25.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
172
+ "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
173
+ "model.layers.25.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
174
+ "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
175
+ "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
176
+ "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
177
+ "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
178
+ "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
179
+ "model.layers.26.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
180
+ "model.layers.26.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
181
+ "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
182
+ "model.layers.26.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
183
+ "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
184
+ "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
185
+ "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
186
+ "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
187
+ "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
188
+ "model.layers.27.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
189
+ "model.layers.27.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
190
+ "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
191
+ "model.layers.27.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
192
+ "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
193
+ "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
194
+ "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
195
+ "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
196
+ "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
197
+ "model.layers.28.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
198
+ "model.layers.28.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
199
+ "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
200
+ "model.layers.28.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
201
+ "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
202
+ "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
203
+ "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
204
+ "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
205
+ "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
206
+ "model.layers.29.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
207
+ "model.layers.29.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
208
+ "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
209
+ "model.layers.29.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
210
+ "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
211
+ "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
212
+ "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
213
+ "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
214
+ "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
215
+ "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
216
+ "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
217
+ "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
218
+ "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
219
+ "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
220
+ "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
221
+ "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
222
+ "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
223
+ "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
224
+ "model.layers.30.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
225
+ "model.layers.30.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
226
+ "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
227
+ "model.layers.30.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
228
+ "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
229
+ "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
230
+ "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
231
+ "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
232
+ "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
233
+ "model.layers.31.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
234
+ "model.layers.31.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
235
+ "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
236
+ "model.layers.31.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
237
+ "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
238
+ "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
239
+ "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
240
+ "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
241
+ "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
242
+ "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
243
+ "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
244
+ "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
245
+ "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
246
+ "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
247
+ "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
248
+ "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
249
+ "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
250
+ "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
251
+ "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
252
+ "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
253
+ "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
254
+ "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
255
+ "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
256
+ "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
257
+ "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
258
+ "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
259
+ "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
260
+ "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
261
+ "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
262
+ "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
263
+ "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
264
+ "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
265
+ "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
266
+ "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
267
+ "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
268
+ "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
269
+ "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
270
+ "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
271
+ "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
272
+ "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
273
+ "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
274
+ "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
275
+ "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
276
+ "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
277
+ "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
278
+ "model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
279
+ "model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
280
+ "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
281
+ "model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
282
+ "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
283
+ "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
284
+ "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
285
+ "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
286
+ "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
287
+ "model.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
288
+ "model.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
289
+ "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
290
+ "model.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
291
+ "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
292
+ "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
293
+ "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
294
+ "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
295
+ "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
296
+ "model.norm.weight": "pytorch_model-00003-of-00003.bin"
297
+ }
298
+ }
runs/Feb27_12-22-56_zen/events.out.tfevents.1709036578.zen.3377863.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ad338e52505f858013217ca40dee95726831c1a04d40a3cbaee25b61ffbb9054
3
+ size 90945
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
+ size 493443
tokenizer_config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ }
30
+ },
31
+ "additional_special_tokens": [],
32
+ "bos_token": "<s>",
33
+ "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
34
+ "clean_up_tokenization_spaces": false,
35
+ "eos_token": "</s>",
36
+ "legacy": true,
37
+ "model_max_length": 1000000000000000019884624838656,
38
+ "pad_token": "</s>",
39
+ "sp_model_kwargs": {},
40
+ "spaces_between_special_tokens": false,
41
+ "tokenizer_class": "LlamaTokenizer",
42
+ "unk_token": "<unk>",
43
+ "use_default_system_prompt": false,
44
+ "use_fast": true
45
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:362ced6da0d7898d52ab2020fd33ac38835b0ffbc42b9542d2f86d5dcaedabb4
3
+ size 5688