Upload folder using huggingface_hub

README.md

@@ -1,3 +1,138 @@
----
-license: apache-2.0
----
+# README
+
+## Model Summary
+
+This is a instruction-tuned version of the [Starcoder2-3B model](https://huggingface.co/bigcode/starcoder2-3b). It has been trained using the same [repository](https://github.com/bigcode-project/starcoder2-self-align) and [dataset](https://huggingface.co/datasets/bigcode/self-oss-instruct-sc2-exec-filter-50k) used for Starcoder2-15B. It uses the same prompt generation technique as the Starcoder2-15B mode. So, it can be used as a drop in replacement by just changing the model path. 
+
+* [Paper](https://arxiv.org/abs/2402.19173)
+
+## Intended Use
+
+Running code language models locally. This model can easily run on:
+
+* 8 GB and 10 GB VRAM machines with FP16
+* 6 GB VRAM machines with INT8
+* 4 GB VRAM machines with INT4
+
+## Example
+
+**Using FP16**
+
+```python
+import transformers
+import torch
+
+pipeline = transformers.pipeline(
+    model="outputs_starcoder3b_4e",
+    task="text-generation",
+    torch_dtype=torch.bfloat16,
+    device_map="auto",
+)
+
+def respond(instruction: str, response_prefix: str) -> str:
+    messages = [{"role": "user", "content": instruction}]
+    prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False)
+    prompt += response_prefix
+
+    teminators = [
+        pipeline.tokenizer.eos_token_id,
+        pipeline.tokenizer.convert_tokens_to_ids("###"),
+    ]
+
+    result = pipeline(
+        prompt,
+        max_length=1024,
+        num_return_sequences=1,
+        do_sample=False,
+        eos_token_id=teminators,
+        pad_token_id=pipeline.tokenizer.eos_token_id,
+        truncation=True,
+    )
+    response = response_prefix + result[0]["generated_text"][len(prompt) :].split("###")[0].rstrip()
+    return response
+
+
+instruction = "Write the Transformer encoder in PyTorch."
+response_prefix = ""
+
+print(respond(instruction, response_prefix))
+```
+
+*Output:*
+
+````
+```python
+import torch
+import torch.nn as nn
+
+class TransformerEncoder(nn.Module):
+    def __init__(self, d_model, nhead, num_layers, dim_feedforward=2048, dropout=0.1):
+        super(TransformerEncoder, self).__init__()
+        self.encoder_layer = nn.TransformerEncoderLayer(d_model, nhead, dim_feedforward, dropout)
+        self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers)
+
+    def forward(self, src):
+        return self.transformer_encoder(src)
+```
+````
+
+## Training
+
+* 4 epochs
+* Training type: Full fine tuning
+* Training time: ~4 hours
+* Batch size: 2
+* Gradient accumulation step: 256
+* Sequence length: 1280
+
+### Exact Training Command Used
+
+**See the [repository](https://github.com/bigcode-project/starcoder2-self-align) for setup details.**
+
+```
+MODEL_KEY=bigcode/starcoder2-3b
+LR=1e-5
+EPOCH=4
+SEQ_LEN=1280
+WARMUP_RATIO=0.05
+OUTPUT_DIR=outputs_starcoder3b_4e
+DATASET_FILE=train_data.jsonl
+accelerate launch -m star_align.train \
+    --model_key $MODEL_KEY \
+    --model_name_or_path $MODEL_KEY \
+    --use_flash_attention True \
+    --datafile_paths $DATASET_FILE \
+    --output_dir $OUTPUT_DIR \
+    --bf16 True \
+    --num_train_epochs $EPOCH \
+    --max_training_seq_length $SEQ_LEN \
+    --pad_to_max_length False \
+    --per_device_train_batch_size 2 \
+    --gradient_accumulation_steps 256 \
+    --group_by_length False \
+    --ddp_find_unused_parameters False \
+    --logging_steps 1 \
+    --log_level info \
+    --optim adafactor \
+    --max_grad_norm -1 \
+    --warmup_ratio $WARMUP_RATIO \
+    --learning_rate $LR \
+    --lr_scheduler_type linear \
+    --attention_dropout 0.0 \
+    --residual_dropout 0.0 \
+    --embedding_dropout 0.0
+```
+
+### Hardware
+
+* 40 GB NVIDIA A100 
+
+## Attributions
+
+* [Starcoder2 Self Align codebase](https://github.com/bigcode-project/starcoder2-self-align)
+* [Starcoder2 Self Align dataset](https://huggingface.co/datasets/bigcode/self-oss-instruct-sc2-exec-filter-50k)
+* [Starcoder2 paper](https://arxiv.org/abs/2402.19173)
+
+## License
+
+The model is licensed under the BigCode OpenRAIL-M v1 license agreement. You can find the full agreement [here](https://huggingface.co/spaces/bigcode/bigcode-model-license-agreement).

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "bigcode/starcoder2-3b",
+  "architectures": [
+    "Starcoder2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "embedding_dropout": 0.0,
+  "eos_token_id": 0,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_size": 3072,
+  "initializer_range": 0.018042,
+  "intermediate_size": 12288,
+  "max_position_embeddings": 16384,
+  "mlp_type": "default",
+  "model_type": "starcoder2",
+  "norm_epsilon": 1e-05,
+  "norm_type": "layer_norm",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 2,
+  "residual_dropout": 0.0,
+  "rope_theta": 999999.4420358813,
+  "sliding_window": 4096,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.0",
+  "use_bias": true,
+  "use_cache": true,
+  "vocab_size": 49152
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.39.0"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50ccd3e336b5da6cc64941191deba06e4fd131c4b8d3a3bf97dc127a8611825e
+size 4949934200

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07e318b21a1f6742fc880ef7de8df7855574acaf24bb3c3a668b5e467e24fb43
+size 1110862568

model.safetensors.index.json

@@ -0,0 +1,490 @@
+{
+  "metadata": {
+    "total_size": 6060742656
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.input_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.c_fc.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.c_fc.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.c_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.c_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.input_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.c_fc.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.c_fc.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.c_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.c_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.c_fc.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.c_fc.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.c_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.c_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.c_fc.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.c_fc.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.c_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.c_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.c_fc.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.c_fc.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.c_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.c_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.c_fc.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.c_fc.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.c_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.c_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.c_fc.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.c_fc.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.c_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.bias": "model-00002-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,63 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,357 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<fim_prefix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<fim_middle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<fim_suffix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<fim_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<code_to_intermediate>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<intermediate_to_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<pr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "19": {
+      "content": "<pr_status>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "20": {
+      "content": "<pr_is_merged>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "21": {
+      "content": "<pr_base>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "22": {
+      "content": "<pr_file>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "23": {
+      "content": "<pr_base_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "24": {
+      "content": "<pr_diff>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25": {
+      "content": "<pr_diff_hunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "26": {
+      "content": "<pr_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "27": {
+      "content": "<pr_event_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "<pr_review>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "29": {
+      "content": "<pr_review_state>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30": {
+      "content": "<pr_review_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "31": {
+      "content": "<pr_in_reply_to_review_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<pr_in_reply_to_comment_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "33": {
+      "content": "<pr_diff_hunk_comment_line>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "34": {
+      "content": "<NAME>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "35": {
+      "content": "<EMAIL>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "36": {
+      "content": "<KEY>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "37": {
+      "content": "<PASSWORD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<repo_name>",
+    "<file_sep>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>",
+    "<code_to_intermediate>",
+    "<intermediate_to_code>",
+    "<pr>",
+    "<pr_status>",
+    "<pr_is_merged>",
+    "<pr_base>",
+    "<pr_file>",
+    "<pr_base_code>",
+    "<pr_diff>",
+    "<pr_diff_hunk>",
+    "<pr_comment>",
+    "<pr_event_id>",
+    "<pr_review>",
+    "<pr_review_state>",
+    "<pr_review_comment>",
+    "<pr_in_reply_to_review_id>",
+    "<pr_in_reply_to_comment_id>",
+    "<pr_diff_hunk_comment_line>",
+    "<NAME>",
+    "<EMAIL>",
+    "<KEY>",
+    "<PASSWORD>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "chat_template": "{{bos_token}}{{'You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.\n\n'}}\n{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n        {{ raise_exception('System messages are not allowed in this template.') }}\n    {%- else %}\n        {%- if message['role'] == 'user' %}\n{{'### Instruction\n' + message['content'] + '\n\n'}}\n        {%- else %}\n{{'### Response\n' + message['content'] + eos_token + '\n\n'}}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{{'### Response\n'}}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

trainer_state.json

@@ -0,0 +1,2382 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.9616280446883265,
+  "eval_steps": 500,
+  "global_step": 392,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 0.5718,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.5621,
+      "step": 2
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5e-06,
+      "loss": 0.5689,
+      "step": 3
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.5552,
+      "step": 4
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.5e-06,
+      "loss": 0.5456,
+      "step": 5
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3e-06,
+      "loss": 0.5306,
+      "step": 6
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5e-06,
+      "loss": 0.5115,
+      "step": 7
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.4996,
+      "step": 8
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.5e-06,
+      "loss": 0.4974,
+      "step": 9
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 5e-06,
+      "loss": 0.4635,
+      "step": 10
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 5.500000000000001e-06,
+      "loss": 0.4578,
+      "step": 11
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 6e-06,
+      "loss": 0.4462,
+      "step": 12
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 6.5000000000000004e-06,
+      "loss": 0.4375,
+      "step": 13
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 7e-06,
+      "loss": 0.4371,
+      "step": 14
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 7.500000000000001e-06,
+      "loss": 0.4315,
+      "step": 15
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.4279,
+      "step": 16
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.5e-06,
+      "loss": 0.4321,
+      "step": 17
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 9e-06,
+      "loss": 0.4356,
+      "step": 18
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 9.5e-06,
+      "loss": 0.5092,
+      "step": 19
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1e-05,
+      "loss": 0.43,
+      "step": 20
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 9.973118279569894e-06,
+      "loss": 0.4254,
+      "step": 21
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 9.946236559139786e-06,
+      "loss": 0.4319,
+      "step": 22
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 9.919354838709679e-06,
+      "loss": 0.4532,
+      "step": 23
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 9.89247311827957e-06,
+      "loss": 0.4783,
+      "step": 24
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 9.865591397849464e-06,
+      "loss": 0.4254,
+      "step": 25
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 9.838709677419356e-06,
+      "loss": 0.4172,
+      "step": 26
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 9.811827956989249e-06,
+      "loss": 0.405,
+      "step": 27
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 9.78494623655914e-06,
+      "loss": 0.3986,
+      "step": 28
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 9.758064516129034e-06,
+      "loss": 0.3972,
+      "step": 29
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 9.731182795698925e-06,
+      "loss": 0.3998,
+      "step": 30
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 9.704301075268819e-06,
+      "loss": 0.4102,
+      "step": 31
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 9.67741935483871e-06,
+      "loss": 0.3963,
+      "step": 32
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 9.650537634408604e-06,
+      "loss": 0.4039,
+      "step": 33
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 9.623655913978495e-06,
+      "loss": 0.4044,
+      "step": 34
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 9.596774193548389e-06,
+      "loss": 0.4045,
+      "step": 35
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 9.56989247311828e-06,
+      "loss": 0.405,
+      "step": 36
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 9.543010752688174e-06,
+      "loss": 0.4071,
+      "step": 37
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 9.516129032258065e-06,
+      "loss": 0.3956,
+      "step": 38
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 9.489247311827959e-06,
+      "loss": 0.398,
+      "step": 39
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 9.46236559139785e-06,
+      "loss": 0.3915,
+      "step": 40
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 9.435483870967743e-06,
+      "loss": 0.3963,
+      "step": 41
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 9.408602150537635e-06,
+      "loss": 0.3887,
+      "step": 42
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 9.381720430107528e-06,
+      "loss": 0.3952,
+      "step": 43
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 9.35483870967742e-06,
+      "loss": 0.4082,
+      "step": 44
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 9.327956989247312e-06,
+      "loss": 0.3968,
+      "step": 45
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 9.301075268817205e-06,
+      "loss": 0.3943,
+      "step": 46
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 9.274193548387097e-06,
+      "loss": 0.4045,
+      "step": 47
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 9.24731182795699e-06,
+      "loss": 0.4169,
+      "step": 48
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 9.220430107526881e-06,
+      "loss": 0.4458,
+      "step": 49
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 9.193548387096775e-06,
+      "loss": 0.4166,
+      "step": 50
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.166666666666666e-06,
+      "loss": 0.4306,
+      "step": 51
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.13978494623656e-06,
+      "loss": 0.4071,
+      "step": 52
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.112903225806451e-06,
+      "loss": 0.4128,
+      "step": 53
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 9.086021505376345e-06,
+      "loss": 0.3954,
+      "step": 54
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 9.059139784946236e-06,
+      "loss": 0.3913,
+      "step": 55
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 9.03225806451613e-06,
+      "loss": 0.4003,
+      "step": 56
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 9.005376344086021e-06,
+      "loss": 0.4044,
+      "step": 57
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 8.978494623655915e-06,
+      "loss": 0.3938,
+      "step": 58
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 8.951612903225806e-06,
+      "loss": 0.3905,
+      "step": 59
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 8.9247311827957e-06,
+      "loss": 0.3962,
+      "step": 60
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 8.897849462365593e-06,
+      "loss": 0.3853,
+      "step": 61
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 8.870967741935484e-06,
+      "loss": 0.3853,
+      "step": 62
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 8.844086021505378e-06,
+      "loss": 0.3949,
+      "step": 63
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 8.81720430107527e-06,
+      "loss": 0.3927,
+      "step": 64
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 8.790322580645163e-06,
+      "loss": 0.3874,
+      "step": 65
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 8.763440860215054e-06,
+      "loss": 0.3953,
+      "step": 66
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 8.736559139784948e-06,
+      "loss": 0.388,
+      "step": 67
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 8.70967741935484e-06,
+      "loss": 0.385,
+      "step": 68
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 8.682795698924733e-06,
+      "loss": 0.389,
+      "step": 69
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 8.655913978494624e-06,
+      "loss": 0.3834,
+      "step": 70
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 8.629032258064517e-06,
+      "loss": 0.3893,
+      "step": 71
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 8.602150537634409e-06,
+      "loss": 0.3874,
+      "step": 72
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 8.575268817204302e-06,
+      "loss": 0.3912,
+      "step": 73
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.548387096774194e-06,
+      "loss": 0.3914,
+      "step": 74
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.521505376344087e-06,
+      "loss": 0.3942,
+      "step": 75
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 8.494623655913979e-06,
+      "loss": 0.3997,
+      "step": 76
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 8.467741935483872e-06,
+      "loss": 0.3965,
+      "step": 77
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 8.440860215053764e-06,
+      "loss": 0.4026,
+      "step": 78
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 8.413978494623657e-06,
+      "loss": 0.398,
+      "step": 79
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 8.387096774193549e-06,
+      "loss": 0.3973,
+      "step": 80
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.360215053763442e-06,
+      "loss": 0.3932,
+      "step": 81
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 0.3852,
+      "step": 82
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.306451612903227e-06,
+      "loss": 0.3776,
+      "step": 83
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.279569892473119e-06,
+      "loss": 0.3829,
+      "step": 84
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 8.252688172043012e-06,
+      "loss": 0.3852,
+      "step": 85
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 8.225806451612904e-06,
+      "loss": 0.3923,
+      "step": 86
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 8.198924731182797e-06,
+      "loss": 0.3862,
+      "step": 87
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 8.172043010752689e-06,
+      "loss": 0.3785,
+      "step": 88
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 8.145161290322582e-06,
+      "loss": 0.3785,
+      "step": 89
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.118279569892473e-06,
+      "loss": 0.3832,
+      "step": 90
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.091397849462365e-06,
+      "loss": 0.3914,
+      "step": 91
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.064516129032258e-06,
+      "loss": 0.3785,
+      "step": 92
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 8.03763440860215e-06,
+      "loss": 0.3871,
+      "step": 93
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 8.010752688172043e-06,
+      "loss": 0.3801,
+      "step": 94
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 7.983870967741935e-06,
+      "loss": 0.3721,
+      "step": 95
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 7.956989247311828e-06,
+      "loss": 0.382,
+      "step": 96
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.93010752688172e-06,
+      "loss": 0.3798,
+      "step": 97
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.903225806451613e-06,
+      "loss": 0.3762,
+      "step": 98
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 7.876344086021507e-06,
+      "loss": 0.3864,
+      "step": 99
+    },
+    {
+      "epoch": 1.01,
+      "learning_rate": 7.849462365591398e-06,
+      "loss": 0.385,
+      "step": 100
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 7.822580645161291e-06,
+      "loss": 0.3659,
+      "step": 101
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 7.795698924731183e-06,
+      "loss": 0.3664,
+      "step": 102
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 7.768817204301076e-06,
+      "loss": 0.3668,
+      "step": 103
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 7.741935483870968e-06,
+      "loss": 0.3714,
+      "step": 104
+    },
+    {
+      "epoch": 1.06,
+      "learning_rate": 7.715053763440861e-06,
+      "loss": 0.3766,
+      "step": 105
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 7.688172043010753e-06,
+      "loss": 0.3753,
+      "step": 106
+    },
+    {
+      "epoch": 1.08,
+      "learning_rate": 7.661290322580646e-06,
+      "loss": 0.3715,
+      "step": 107
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 7.634408602150538e-06,
+      "loss": 0.37,
+      "step": 108
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 7.60752688172043e-06,
+      "loss": 0.3753,
+      "step": 109
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 7.580645161290323e-06,
+      "loss": 0.374,
+      "step": 110
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 7.553763440860215e-06,
+      "loss": 0.3748,
+      "step": 111
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 7.526881720430108e-06,
+      "loss": 0.3776,
+      "step": 112
+    },
+    {
+      "epoch": 1.14,
+      "learning_rate": 7.500000000000001e-06,
+      "loss": 0.3957,
+      "step": 113
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 7.4731182795698935e-06,
+      "loss": 0.3742,
+      "step": 114
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 7.446236559139786e-06,
+      "loss": 0.375,
+      "step": 115
+    },
+    {
+      "epoch": 1.17,
+      "learning_rate": 7.4193548387096784e-06,
+      "loss": 0.3784,
+      "step": 116
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 7.392473118279571e-06,
+      "loss": 0.3754,
+      "step": 117
+    },
+    {
+      "epoch": 1.19,
+      "learning_rate": 7.365591397849463e-06,
+      "loss": 0.383,
+      "step": 118
+    },
+    {
+      "epoch": 1.2,
+      "learning_rate": 7.338709677419356e-06,
+      "loss": 0.3761,
+      "step": 119
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 7.311827956989248e-06,
+      "loss": 0.3736,
+      "step": 120
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 7.284946236559141e-06,
+      "loss": 0.3708,
+      "step": 121
+    },
+    {
+      "epoch": 1.23,
+      "learning_rate": 7.258064516129033e-06,
+      "loss": 0.3734,
+      "step": 122
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 7.231182795698926e-06,
+      "loss": 0.3684,
+      "step": 123
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 7.204301075268818e-06,
+      "loss": 0.3669,
+      "step": 124
+    },
+    {
+      "epoch": 1.26,
+      "learning_rate": 7.177419354838711e-06,
+      "loss": 0.3764,
+      "step": 125
+    },
+    {
+      "epoch": 1.27,
+      "learning_rate": 7.150537634408603e-06,
+      "loss": 0.358,
+      "step": 126
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 7.1236559139784956e-06,
+      "loss": 0.3757,
+      "step": 127
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 7.096774193548388e-06,
+      "loss": 0.3713,
+      "step": 128
+    },
+    {
+      "epoch": 1.3,
+      "learning_rate": 7.0698924731182805e-06,
+      "loss": 0.3538,
+      "step": 129
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 7.043010752688173e-06,
+      "loss": 0.3789,
+      "step": 130
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 7.0161290322580654e-06,
+      "loss": 0.377,
+      "step": 131
+    },
+    {
+      "epoch": 1.33,
+      "learning_rate": 6.989247311827958e-06,
+      "loss": 0.3762,
+      "step": 132
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 6.96236559139785e-06,
+      "loss": 0.3583,
+      "step": 133
+    },
+    {
+      "epoch": 1.35,
+      "learning_rate": 6.935483870967743e-06,
+      "loss": 0.3658,
+      "step": 134
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 6.908602150537635e-06,
+      "loss": 0.3683,
+      "step": 135
+    },
+    {
+      "epoch": 1.37,
+      "learning_rate": 6.881720430107528e-06,
+      "loss": 0.3717,
+      "step": 136
+    },
+    {
+      "epoch": 1.38,
+      "learning_rate": 6.854838709677419e-06,
+      "loss": 0.3663,
+      "step": 137
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 6.827956989247312e-06,
+      "loss": 0.3688,
+      "step": 138
+    },
+    {
+      "epoch": 1.4,
+      "learning_rate": 6.801075268817204e-06,
+      "loss": 0.3717,
+      "step": 139
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 6.774193548387097e-06,
+      "loss": 0.3646,
+      "step": 140
+    },
+    {
+      "epoch": 1.42,
+      "learning_rate": 6.747311827956989e-06,
+      "loss": 0.3655,
+      "step": 141
+    },
+    {
+      "epoch": 1.44,
+      "learning_rate": 6.720430107526882e-06,
+      "loss": 0.3642,
+      "step": 142
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 6.693548387096774e-06,
+      "loss": 0.3695,
+      "step": 143
+    },
+    {
+      "epoch": 1.46,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.3801,
+      "step": 144
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 6.639784946236559e-06,
+      "loss": 0.3648,
+      "step": 145
+    },
+    {
+      "epoch": 1.48,
+      "learning_rate": 6.612903225806452e-06,
+      "loss": 0.3668,
+      "step": 146
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 6.586021505376344e-06,
+      "loss": 0.3621,
+      "step": 147
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 6.5591397849462365e-06,
+      "loss": 0.373,
+      "step": 148
+    },
+    {
+      "epoch": 1.51,
+      "learning_rate": 6.532258064516129e-06,
+      "loss": 0.3767,
+      "step": 149
+    },
+    {
+      "epoch": 1.52,
+      "learning_rate": 6.5053763440860214e-06,
+      "loss": 0.3712,
+      "step": 150
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 6.478494623655914e-06,
+      "loss": 0.3725,
+      "step": 151
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 6.451612903225806e-06,
+      "loss": 0.366,
+      "step": 152
+    },
+    {
+      "epoch": 1.55,
+      "learning_rate": 6.4247311827957e-06,
+      "loss": 0.362,
+      "step": 153
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 6.397849462365592e-06,
+      "loss": 0.3575,
+      "step": 154
+    },
+    {
+      "epoch": 1.57,
+      "learning_rate": 6.370967741935485e-06,
+      "loss": 0.3579,
+      "step": 155
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 6.344086021505377e-06,
+      "loss": 0.3624,
+      "step": 156
+    },
+    {
+      "epoch": 1.59,
+      "learning_rate": 6.3172043010752696e-06,
+      "loss": 0.3747,
+      "step": 157
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 6.290322580645162e-06,
+      "loss": 0.3598,
+      "step": 158
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 6.2634408602150545e-06,
+      "loss": 0.3582,
+      "step": 159
+    },
+    {
+      "epoch": 1.62,
+      "learning_rate": 6.236559139784947e-06,
+      "loss": 0.3601,
+      "step": 160
+    },
+    {
+      "epoch": 1.63,
+      "learning_rate": 6.209677419354839e-06,
+      "loss": 0.3637,
+      "step": 161
+    },
+    {
+      "epoch": 1.64,
+      "learning_rate": 6.182795698924732e-06,
+      "loss": 0.377,
+      "step": 162
+    },
+    {
+      "epoch": 1.65,
+      "learning_rate": 6.155913978494624e-06,
+      "loss": 0.361,
+      "step": 163
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 6.129032258064517e-06,
+      "loss": 0.3687,
+      "step": 164
+    },
+    {
+      "epoch": 1.67,
+      "learning_rate": 6.102150537634409e-06,
+      "loss": 0.3633,
+      "step": 165
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 6.075268817204302e-06,
+      "loss": 0.365,
+      "step": 166
+    },
+    {
+      "epoch": 1.69,
+      "learning_rate": 6.048387096774194e-06,
+      "loss": 0.3637,
+      "step": 167
+    },
+    {
+      "epoch": 1.7,
+      "learning_rate": 6.021505376344087e-06,
+      "loss": 0.3725,
+      "step": 168
+    },
+    {
+      "epoch": 1.71,
+      "learning_rate": 5.994623655913979e-06,
+      "loss": 0.3625,
+      "step": 169
+    },
+    {
+      "epoch": 1.72,
+      "learning_rate": 5.967741935483872e-06,
+      "loss": 0.3657,
+      "step": 170
+    },
+    {
+      "epoch": 1.73,
+      "learning_rate": 5.940860215053764e-06,
+      "loss": 0.3573,
+      "step": 171
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 5.9139784946236566e-06,
+      "loss": 0.3656,
+      "step": 172
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 5.887096774193549e-06,
+      "loss": 0.3671,
+      "step": 173
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 5.8602150537634415e-06,
+      "loss": 0.359,
+      "step": 174
+    },
+    {
+      "epoch": 1.77,
+      "learning_rate": 5.833333333333334e-06,
+      "loss": 0.3638,
+      "step": 175
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 5.806451612903226e-06,
+      "loss": 0.3633,
+      "step": 176
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 5.779569892473119e-06,
+      "loss": 0.3645,
+      "step": 177
+    },
+    {
+      "epoch": 1.8,
+      "learning_rate": 5.752688172043011e-06,
+      "loss": 0.3691,
+      "step": 178
+    },
+    {
+      "epoch": 1.81,
+      "learning_rate": 5.725806451612904e-06,
+      "loss": 0.3685,
+      "step": 179
+    },
+    {
+      "epoch": 1.82,
+      "learning_rate": 5.698924731182796e-06,
+      "loss": 0.3697,
+      "step": 180
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 5.672043010752689e-06,
+      "loss": 0.356,
+      "step": 181
+    },
+    {
+      "epoch": 1.84,
+      "learning_rate": 5.645161290322582e-06,
+      "loss": 0.3624,
+      "step": 182
+    },
+    {
+      "epoch": 1.85,
+      "learning_rate": 5.618279569892473e-06,
+      "loss": 0.3556,
+      "step": 183
+    },
+    {
+      "epoch": 1.86,
+      "learning_rate": 5.591397849462365e-06,
+      "loss": 0.3634,
+      "step": 184
+    },
+    {
+      "epoch": 1.87,
+      "learning_rate": 5.564516129032258e-06,
+      "loss": 0.3536,
+      "step": 185
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 5.53763440860215e-06,
+      "loss": 0.3649,
+      "step": 186
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 5.510752688172043e-06,
+      "loss": 0.3758,
+      "step": 187
+    },
+    {
+      "epoch": 1.9,
+      "learning_rate": 5.483870967741935e-06,
+      "loss": 0.3517,
+      "step": 188
+    },
+    {
+      "epoch": 1.91,
+      "learning_rate": 5.456989247311828e-06,
+      "loss": 0.359,
+      "step": 189
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 5.43010752688172e-06,
+      "loss": 0.3663,
+      "step": 190
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 5.4032258064516126e-06,
+      "loss": 0.373,
+      "step": 191
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 5.376344086021506e-06,
+      "loss": 0.363,
+      "step": 192
+    },
+    {
+      "epoch": 1.95,
+      "learning_rate": 5.349462365591398e-06,
+      "loss": 0.3699,
+      "step": 193
+    },
+    {
+      "epoch": 1.96,
+      "learning_rate": 5.322580645161291e-06,
+      "loss": 0.3784,
+      "step": 194
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 5.295698924731183e-06,
+      "loss": 0.3725,
+      "step": 195
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 5.268817204301076e-06,
+      "loss": 0.3611,
+      "step": 196
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 5.241935483870968e-06,
+      "loss": 0.3683,
+      "step": 197
+    },
+    {
+      "epoch": 2.0,
+      "learning_rate": 5.215053763440861e-06,
+      "loss": 0.3588,
+      "step": 198
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 5.188172043010753e-06,
+      "loss": 0.3478,
+      "step": 199
+    },
+    {
+      "epoch": 2.02,
+      "learning_rate": 5.161290322580646e-06,
+      "loss": 0.3459,
+      "step": 200
+    },
+    {
+      "epoch": 2.03,
+      "learning_rate": 5.134408602150538e-06,
+      "loss": 0.3545,
+      "step": 201
+    },
+    {
+      "epoch": 2.04,
+      "learning_rate": 5.1075268817204305e-06,
+      "loss": 0.3487,
+      "step": 202
+    },
+    {
+      "epoch": 2.05,
+      "learning_rate": 5.080645161290323e-06,
+      "loss": 0.3457,
+      "step": 203
+    },
+    {
+      "epoch": 2.06,
+      "learning_rate": 5.0537634408602155e-06,
+      "loss": 0.351,
+      "step": 204
+    },
+    {
+      "epoch": 2.07,
+      "learning_rate": 5.026881720430108e-06,
+      "loss": 0.3531,
+      "step": 205
+    },
+    {
+      "epoch": 2.08,
+      "learning_rate": 5e-06,
+      "loss": 0.344,
+      "step": 206
+    },
+    {
+      "epoch": 2.09,
+      "learning_rate": 4.973118279569893e-06,
+      "loss": 0.3534,
+      "step": 207
+    },
+    {
+      "epoch": 2.1,
+      "learning_rate": 4.946236559139785e-06,
+      "loss": 0.3524,
+      "step": 208
+    },
+    {
+      "epoch": 2.11,
+      "learning_rate": 4.919354838709678e-06,
+      "loss": 0.3504,
+      "step": 209
+    },
+    {
+      "epoch": 2.12,
+      "learning_rate": 4.89247311827957e-06,
+      "loss": 0.3551,
+      "step": 210
+    },
+    {
+      "epoch": 2.13,
+      "learning_rate": 4.865591397849463e-06,
+      "loss": 0.3494,
+      "step": 211
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 4.838709677419355e-06,
+      "loss": 0.3446,
+      "step": 212
+    },
+    {
+      "epoch": 2.15,
+      "learning_rate": 4.811827956989248e-06,
+      "loss": 0.3548,
+      "step": 213
+    },
+    {
+      "epoch": 2.16,
+      "learning_rate": 4.78494623655914e-06,
+      "loss": 0.3623,
+      "step": 214
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 4.758064516129033e-06,
+      "loss": 0.3615,
+      "step": 215
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 4.731182795698925e-06,
+      "loss": 0.3549,
+      "step": 216
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 4.7043010752688175e-06,
+      "loss": 0.3433,
+      "step": 217
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 4.67741935483871e-06,
+      "loss": 0.3484,
+      "step": 218
+    },
+    {
+      "epoch": 2.21,
+      "learning_rate": 4.6505376344086025e-06,
+      "loss": 0.3525,
+      "step": 219
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 4.623655913978495e-06,
+      "loss": 0.3495,
+      "step": 220
+    },
+    {
+      "epoch": 2.23,
+      "learning_rate": 4.596774193548387e-06,
+      "loss": 0.3612,
+      "step": 221
+    },
+    {
+      "epoch": 2.24,
+      "learning_rate": 4.56989247311828e-06,
+      "loss": 0.3446,
+      "step": 222
+    },
+    {
+      "epoch": 2.25,
+      "learning_rate": 4.543010752688172e-06,
+      "loss": 0.3506,
+      "step": 223
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 4.516129032258065e-06,
+      "loss": 0.3525,
+      "step": 224
+    },
+    {
+      "epoch": 2.27,
+      "learning_rate": 4.489247311827957e-06,
+      "loss": 0.3545,
+      "step": 225
+    },
+    {
+      "epoch": 2.28,
+      "learning_rate": 4.46236559139785e-06,
+      "loss": 0.3525,
+      "step": 226
+    },
+    {
+      "epoch": 2.29,
+      "learning_rate": 4.435483870967742e-06,
+      "loss": 0.3509,
+      "step": 227
+    },
+    {
+      "epoch": 2.3,
+      "learning_rate": 4.408602150537635e-06,
+      "loss": 0.3549,
+      "step": 228
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 4.381720430107527e-06,
+      "loss": 0.3477,
+      "step": 229
+    },
+    {
+      "epoch": 2.32,
+      "learning_rate": 4.35483870967742e-06,
+      "loss": 0.3654,
+      "step": 230
+    },
+    {
+      "epoch": 2.33,
+      "learning_rate": 4.327956989247312e-06,
+      "loss": 0.3439,
+      "step": 231
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 4.3010752688172045e-06,
+      "loss": 0.3462,
+      "step": 232
+    },
+    {
+      "epoch": 2.35,
+      "learning_rate": 4.274193548387097e-06,
+      "loss": 0.3387,
+      "step": 233
+    },
+    {
+      "epoch": 2.36,
+      "learning_rate": 4.2473118279569895e-06,
+      "loss": 0.3458,
+      "step": 234
+    },
+    {
+      "epoch": 2.37,
+      "learning_rate": 4.220430107526882e-06,
+      "loss": 0.3482,
+      "step": 235
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 4.193548387096774e-06,
+      "loss": 0.3614,
+      "step": 236
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 4.166666666666667e-06,
+      "loss": 0.3511,
+      "step": 237
+    },
+    {
+      "epoch": 2.41,
+      "learning_rate": 4.139784946236559e-06,
+      "loss": 0.349,
+      "step": 238
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 4.112903225806452e-06,
+      "loss": 0.354,
+      "step": 239
+    },
+    {
+      "epoch": 2.43,
+      "learning_rate": 4.086021505376344e-06,
+      "loss": 0.3539,
+      "step": 240
+    },
+    {
+      "epoch": 2.44,
+      "learning_rate": 4.059139784946237e-06,
+      "loss": 0.3566,
+      "step": 241
+    },
+    {
+      "epoch": 2.45,
+      "learning_rate": 4.032258064516129e-06,
+      "loss": 0.3441,
+      "step": 242
+    },
+    {
+      "epoch": 2.46,
+      "learning_rate": 4.005376344086022e-06,
+      "loss": 0.3476,
+      "step": 243
+    },
+    {
+      "epoch": 2.47,
+      "learning_rate": 3.978494623655914e-06,
+      "loss": 0.3533,
+      "step": 244
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 3.951612903225807e-06,
+      "loss": 0.3419,
+      "step": 245
+    },
+    {
+      "epoch": 2.49,
+      "learning_rate": 3.924731182795699e-06,
+      "loss": 0.353,
+      "step": 246
+    },
+    {
+      "epoch": 2.5,
+      "learning_rate": 3.8978494623655915e-06,
+      "loss": 0.3533,
+      "step": 247
+    },
+    {
+      "epoch": 2.51,
+      "learning_rate": 3.870967741935484e-06,
+      "loss": 0.3529,
+      "step": 248
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 3.8440860215053765e-06,
+      "loss": 0.3491,
+      "step": 249
+    },
+    {
+      "epoch": 2.53,
+      "learning_rate": 3.817204301075269e-06,
+      "loss": 0.3509,
+      "step": 250
+    },
+    {
+      "epoch": 2.54,
+      "learning_rate": 3.7903225806451614e-06,
+      "loss": 0.3533,
+      "step": 251
+    },
+    {
+      "epoch": 2.55,
+      "learning_rate": 3.763440860215054e-06,
+      "loss": 0.3548,
+      "step": 252
+    },
+    {
+      "epoch": 2.56,
+      "learning_rate": 3.7365591397849468e-06,
+      "loss": 0.3567,
+      "step": 253
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 3.7096774193548392e-06,
+      "loss": 0.3439,
+      "step": 254
+    },
+    {
+      "epoch": 2.58,
+      "learning_rate": 3.6827956989247317e-06,
+      "loss": 0.3439,
+      "step": 255
+    },
+    {
+      "epoch": 2.59,
+      "learning_rate": 3.655913978494624e-06,
+      "loss": 0.3479,
+      "step": 256
+    },
+    {
+      "epoch": 2.6,
+      "learning_rate": 3.6290322580645166e-06,
+      "loss": 0.3487,
+      "step": 257
+    },
+    {
+      "epoch": 2.61,
+      "learning_rate": 3.602150537634409e-06,
+      "loss": 0.3538,
+      "step": 258
+    },
+    {
+      "epoch": 2.62,
+      "learning_rate": 3.5752688172043015e-06,
+      "loss": 0.3481,
+      "step": 259
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 3.548387096774194e-06,
+      "loss": 0.3414,
+      "step": 260
+    },
+    {
+      "epoch": 2.64,
+      "learning_rate": 3.5215053763440865e-06,
+      "loss": 0.3496,
+      "step": 261
+    },
+    {
+      "epoch": 2.65,
+      "learning_rate": 3.494623655913979e-06,
+      "loss": 0.3546,
+      "step": 262
+    },
+    {
+      "epoch": 2.66,
+      "learning_rate": 3.4677419354838714e-06,
+      "loss": 0.3431,
+      "step": 263
+    },
+    {
+      "epoch": 2.67,
+      "learning_rate": 3.440860215053764e-06,
+      "loss": 0.3547,
+      "step": 264
+    },
+    {
+      "epoch": 2.68,
+      "learning_rate": 3.413978494623656e-06,
+      "loss": 0.3471,
+      "step": 265
+    },
+    {
+      "epoch": 2.69,
+      "learning_rate": 3.3870967741935484e-06,
+      "loss": 0.3435,
+      "step": 266
+    },
+    {
+      "epoch": 2.7,
+      "learning_rate": 3.360215053763441e-06,
+      "loss": 0.3397,
+      "step": 267
+    },
+    {
+      "epoch": 2.71,
+      "learning_rate": 3.3333333333333333e-06,
+      "loss": 0.3571,
+      "step": 268
+    },
+    {
+      "epoch": 2.72,
+      "learning_rate": 3.306451612903226e-06,
+      "loss": 0.3401,
+      "step": 269
+    },
+    {
+      "epoch": 2.73,
+      "learning_rate": 3.2795698924731183e-06,
+      "loss": 0.3489,
+      "step": 270
+    },
+    {
+      "epoch": 2.74,
+      "learning_rate": 3.2526881720430107e-06,
+      "loss": 0.3416,
+      "step": 271
+    },
+    {
+      "epoch": 2.75,
+      "learning_rate": 3.225806451612903e-06,
+      "loss": 0.3506,
+      "step": 272
+    },
+    {
+      "epoch": 2.76,
+      "learning_rate": 3.198924731182796e-06,
+      "loss": 0.3612,
+      "step": 273
+    },
+    {
+      "epoch": 2.77,
+      "learning_rate": 3.1720430107526885e-06,
+      "loss": 0.3559,
+      "step": 274
+    },
+    {
+      "epoch": 2.78,
+      "learning_rate": 3.145161290322581e-06,
+      "loss": 0.3398,
+      "step": 275
+    },
+    {
+      "epoch": 2.79,
+      "learning_rate": 3.1182795698924735e-06,
+      "loss": 0.3587,
+      "step": 276
+    },
+    {
+      "epoch": 2.8,
+      "learning_rate": 3.091397849462366e-06,
+      "loss": 0.3677,
+      "step": 277
+    },
+    {
+      "epoch": 2.81,
+      "learning_rate": 3.0645161290322584e-06,
+      "loss": 0.3595,
+      "step": 278
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 3.037634408602151e-06,
+      "loss": 0.3512,
+      "step": 279
+    },
+    {
+      "epoch": 2.83,
+      "learning_rate": 3.0107526881720433e-06,
+      "loss": 0.3429,
+      "step": 280
+    },
+    {
+      "epoch": 2.84,
+      "learning_rate": 2.983870967741936e-06,
+      "loss": 0.3471,
+      "step": 281
+    },
+    {
+      "epoch": 2.85,
+      "learning_rate": 2.9569892473118283e-06,
+      "loss": 0.3448,
+      "step": 282
+    },
+    {
+      "epoch": 2.86,
+      "learning_rate": 2.9301075268817207e-06,
+      "loss": 0.3608,
+      "step": 283
+    },
+    {
+      "epoch": 2.87,
+      "learning_rate": 2.903225806451613e-06,
+      "loss": 0.3401,
+      "step": 284
+    },
+    {
+      "epoch": 2.88,
+      "learning_rate": 2.8763440860215057e-06,
+      "loss": 0.3403,
+      "step": 285
+    },
+    {
+      "epoch": 2.89,
+      "learning_rate": 2.849462365591398e-06,
+      "loss": 0.3497,
+      "step": 286
+    },
+    {
+      "epoch": 2.9,
+      "learning_rate": 2.822580645161291e-06,
+      "loss": 0.3541,
+      "step": 287
+    },
+    {
+      "epoch": 2.91,
+      "learning_rate": 2.7956989247311827e-06,
+      "loss": 0.3565,
+      "step": 288
+    },
+    {
+      "epoch": 2.92,
+      "learning_rate": 2.768817204301075e-06,
+      "loss": 0.351,
+      "step": 289
+    },
+    {
+      "epoch": 2.93,
+      "learning_rate": 2.7419354838709676e-06,
+      "loss": 0.3542,
+      "step": 290
+    },
+    {
+      "epoch": 2.94,
+      "learning_rate": 2.71505376344086e-06,
+      "loss": 0.3387,
+      "step": 291
+    },
+    {
+      "epoch": 2.95,
+      "learning_rate": 2.688172043010753e-06,
+      "loss": 0.3541,
+      "step": 292
+    },
+    {
+      "epoch": 2.96,
+      "learning_rate": 2.6612903225806454e-06,
+      "loss": 0.349,
+      "step": 293
+    },
+    {
+      "epoch": 2.97,
+      "learning_rate": 2.634408602150538e-06,
+      "loss": 0.3569,
+      "step": 294
+    },
+    {
+      "epoch": 2.98,
+      "learning_rate": 2.6075268817204303e-06,
+      "loss": 0.3474,
+      "step": 295
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 2.580645161290323e-06,
+      "loss": 0.3668,
+      "step": 296
+    },
+    {
+      "epoch": 3.0,
+      "learning_rate": 2.5537634408602153e-06,
+      "loss": 0.3428,
+      "step": 297
+    },
+    {
+      "epoch": 3.01,
+      "learning_rate": 2.5268817204301077e-06,
+      "loss": 0.3323,
+      "step": 298
+    },
+    {
+      "epoch": 3.02,
+      "learning_rate": 2.5e-06,
+      "loss": 0.343,
+      "step": 299
+    },
+    {
+      "epoch": 3.03,
+      "learning_rate": 2.4731182795698927e-06,
+      "loss": 0.3446,
+      "step": 300
+    },
+    {
+      "epoch": 3.04,
+      "learning_rate": 2.446236559139785e-06,
+      "loss": 0.3439,
+      "step": 301
+    },
+    {
+      "epoch": 3.05,
+      "learning_rate": 2.4193548387096776e-06,
+      "loss": 0.3535,
+      "step": 302
+    },
+    {
+      "epoch": 3.06,
+      "learning_rate": 2.39247311827957e-06,
+      "loss": 0.3318,
+      "step": 303
+    },
+    {
+      "epoch": 3.07,
+      "learning_rate": 2.3655913978494625e-06,
+      "loss": 0.3304,
+      "step": 304
+    },
+    {
+      "epoch": 3.08,
+      "learning_rate": 2.338709677419355e-06,
+      "loss": 0.3361,
+      "step": 305
+    },
+    {
+      "epoch": 3.09,
+      "learning_rate": 2.3118279569892475e-06,
+      "loss": 0.3508,
+      "step": 306
+    },
+    {
+      "epoch": 3.1,
+      "learning_rate": 2.28494623655914e-06,
+      "loss": 0.3443,
+      "step": 307
+    },
+    {
+      "epoch": 3.11,
+      "learning_rate": 2.2580645161290324e-06,
+      "loss": 0.3461,
+      "step": 308
+    },
+    {
+      "epoch": 3.12,
+      "learning_rate": 2.231182795698925e-06,
+      "loss": 0.3448,
+      "step": 309
+    },
+    {
+      "epoch": 3.13,
+      "learning_rate": 2.2043010752688173e-06,
+      "loss": 0.3423,
+      "step": 310
+    },
+    {
+      "epoch": 3.14,
+      "learning_rate": 2.17741935483871e-06,
+      "loss": 0.3463,
+      "step": 311
+    },
+    {
+      "epoch": 3.15,
+      "learning_rate": 2.1505376344086023e-06,
+      "loss": 0.3342,
+      "step": 312
+    },
+    {
+      "epoch": 3.16,
+      "learning_rate": 2.1236559139784947e-06,
+      "loss": 0.349,
+      "step": 313
+    },
+    {
+      "epoch": 3.17,
+      "learning_rate": 2.096774193548387e-06,
+      "loss": 0.3436,
+      "step": 314
+    },
+    {
+      "epoch": 3.18,
+      "learning_rate": 2.0698924731182797e-06,
+      "loss": 0.3451,
+      "step": 315
+    },
+    {
+      "epoch": 3.19,
+      "learning_rate": 2.043010752688172e-06,
+      "loss": 0.3428,
+      "step": 316
+    },
+    {
+      "epoch": 3.2,
+      "learning_rate": 2.0161290322580646e-06,
+      "loss": 0.3476,
+      "step": 317
+    },
+    {
+      "epoch": 3.21,
+      "learning_rate": 1.989247311827957e-06,
+      "loss": 0.3395,
+      "step": 318
+    },
+    {
+      "epoch": 3.22,
+      "learning_rate": 1.9623655913978495e-06,
+      "loss": 0.3423,
+      "step": 319
+    },
+    {
+      "epoch": 3.23,
+      "learning_rate": 1.935483870967742e-06,
+      "loss": 0.3444,
+      "step": 320
+    },
+    {
+      "epoch": 3.24,
+      "learning_rate": 1.9086021505376345e-06,
+      "loss": 0.3511,
+      "step": 321
+    },
+    {
+      "epoch": 3.25,
+      "learning_rate": 1.881720430107527e-06,
+      "loss": 0.3459,
+      "step": 322
+    },
+    {
+      "epoch": 3.26,
+      "learning_rate": 1.8548387096774196e-06,
+      "loss": 0.335,
+      "step": 323
+    },
+    {
+      "epoch": 3.27,
+      "learning_rate": 1.827956989247312e-06,
+      "loss": 0.3533,
+      "step": 324
+    },
+    {
+      "epoch": 3.28,
+      "learning_rate": 1.8010752688172045e-06,
+      "loss": 0.3455,
+      "step": 325
+    },
+    {
+      "epoch": 3.29,
+      "learning_rate": 1.774193548387097e-06,
+      "loss": 0.3526,
+      "step": 326
+    },
+    {
+      "epoch": 3.3,
+      "learning_rate": 1.7473118279569895e-06,
+      "loss": 0.3448,
+      "step": 327
+    },
+    {
+      "epoch": 3.31,
+      "learning_rate": 1.720430107526882e-06,
+      "loss": 0.3446,
+      "step": 328
+    },
+    {
+      "epoch": 3.32,
+      "learning_rate": 1.6935483870967742e-06,
+      "loss": 0.3464,
+      "step": 329
+    },
+    {
+      "epoch": 3.34,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 0.3455,
+      "step": 330
+    },
+    {
+      "epoch": 3.35,
+      "learning_rate": 1.6397849462365591e-06,
+      "loss": 0.3434,
+      "step": 331
+    },
+    {
+      "epoch": 3.36,
+      "learning_rate": 1.6129032258064516e-06,
+      "loss": 0.3451,
+      "step": 332
+    },
+    {
+      "epoch": 3.37,
+      "learning_rate": 1.5860215053763443e-06,
+      "loss": 0.3409,
+      "step": 333
+    },
+    {
+      "epoch": 3.38,
+      "learning_rate": 1.5591397849462367e-06,
+      "loss": 0.342,
+      "step": 334
+    },
+    {
+      "epoch": 3.39,
+      "learning_rate": 1.5322580645161292e-06,
+      "loss": 0.3383,
+      "step": 335
+    },
+    {
+      "epoch": 3.4,
+      "learning_rate": 1.5053763440860217e-06,
+      "loss": 0.3534,
+      "step": 336
+    },
+    {
+      "epoch": 3.41,
+      "learning_rate": 1.4784946236559141e-06,
+      "loss": 0.3452,
+      "step": 337
+    },
+    {
+      "epoch": 3.42,
+      "learning_rate": 1.4516129032258066e-06,
+      "loss": 0.3414,
+      "step": 338
+    },
+    {
+      "epoch": 3.43,
+      "learning_rate": 1.424731182795699e-06,
+      "loss": 0.3459,
+      "step": 339
+    },
+    {
+      "epoch": 3.44,
+      "learning_rate": 1.3978494623655913e-06,
+      "loss": 0.3457,
+      "step": 340
+    },
+    {
+      "epoch": 3.45,
+      "learning_rate": 1.3709677419354838e-06,
+      "loss": 0.3426,
+      "step": 341
+    },
+    {
+      "epoch": 3.46,
+      "learning_rate": 1.3440860215053765e-06,
+      "loss": 0.3413,
+      "step": 342
+    },
+    {
+      "epoch": 3.47,
+      "learning_rate": 1.317204301075269e-06,
+      "loss": 0.3454,
+      "step": 343
+    },
+    {
+      "epoch": 3.48,
+      "learning_rate": 1.2903225806451614e-06,
+      "loss": 0.3354,
+      "step": 344
+    },
+    {
+      "epoch": 3.49,
+      "learning_rate": 1.2634408602150539e-06,
+      "loss": 0.3504,
+      "step": 345
+    },
+    {
+      "epoch": 3.5,
+      "learning_rate": 1.2365591397849463e-06,
+      "loss": 0.3423,
+      "step": 346
+    },
+    {
+      "epoch": 3.51,
+      "learning_rate": 1.2096774193548388e-06,
+      "loss": 0.338,
+      "step": 347
+    },
+    {
+      "epoch": 3.52,
+      "learning_rate": 1.1827956989247313e-06,
+      "loss": 0.3366,
+      "step": 348
+    },
+    {
+      "epoch": 3.53,
+      "learning_rate": 1.1559139784946237e-06,
+      "loss": 0.34,
+      "step": 349
+    },
+    {
+      "epoch": 3.54,
+      "learning_rate": 1.1290322580645162e-06,
+      "loss": 0.3369,
+      "step": 350
+    },
+    {
+      "epoch": 3.55,
+      "learning_rate": 1.1021505376344087e-06,
+      "loss": 0.346,
+      "step": 351
+    },
+    {
+      "epoch": 3.56,
+      "learning_rate": 1.0752688172043011e-06,
+      "loss": 0.3334,
+      "step": 352
+    },
+    {
+      "epoch": 3.57,
+      "learning_rate": 1.0483870967741936e-06,
+      "loss": 0.3401,
+      "step": 353
+    },
+    {
+      "epoch": 3.58,
+      "learning_rate": 1.021505376344086e-06,
+      "loss": 0.3432,
+      "step": 354
+    },
+    {
+      "epoch": 3.59,
+      "learning_rate": 9.946236559139785e-07,
+      "loss": 0.3401,
+      "step": 355
+    },
+    {
+      "epoch": 3.6,
+      "learning_rate": 9.67741935483871e-07,
+      "loss": 0.3399,
+      "step": 356
+    },
+    {
+      "epoch": 3.61,
+      "learning_rate": 9.408602150537635e-07,
+      "loss": 0.3482,
+      "step": 357
+    },
+    {
+      "epoch": 3.62,
+      "learning_rate": 9.13978494623656e-07,
+      "loss": 0.3398,
+      "step": 358
+    },
+    {
+      "epoch": 3.63,
+      "learning_rate": 8.870967741935485e-07,
+      "loss": 0.3384,
+      "step": 359
+    },
+    {
+      "epoch": 3.64,
+      "learning_rate": 8.60215053763441e-07,
+      "loss": 0.3338,
+      "step": 360
+    },
+    {
+      "epoch": 3.65,
+      "learning_rate": 8.333333333333333e-07,
+      "loss": 0.3433,
+      "step": 361
+    },
+    {
+      "epoch": 3.66,
+      "learning_rate": 8.064516129032258e-07,
+      "loss": 0.3445,
+      "step": 362
+    },
+    {
+      "epoch": 3.67,
+      "learning_rate": 7.795698924731184e-07,
+      "loss": 0.3514,
+      "step": 363
+    },
+    {
+      "epoch": 3.68,
+      "learning_rate": 7.526881720430108e-07,
+      "loss": 0.3402,
+      "step": 364
+    },
+    {
+      "epoch": 3.69,
+      "learning_rate": 7.258064516129033e-07,
+      "loss": 0.3449,
+      "step": 365
+    },
+    {
+      "epoch": 3.7,
+      "learning_rate": 6.989247311827957e-07,
+      "loss": 0.348,
+      "step": 366
+    },
+    {
+      "epoch": 3.71,
+      "learning_rate": 6.720430107526882e-07,
+      "loss": 0.3457,
+      "step": 367
+    },
+    {
+      "epoch": 3.72,
+      "learning_rate": 6.451612903225807e-07,
+      "loss": 0.343,
+      "step": 368
+    },
+    {
+      "epoch": 3.73,
+      "learning_rate": 6.182795698924732e-07,
+      "loss": 0.3351,
+      "step": 369
+    },
+    {
+      "epoch": 3.74,
+      "learning_rate": 5.913978494623656e-07,
+      "loss": 0.3391,
+      "step": 370
+    },
+    {
+      "epoch": 3.75,
+      "learning_rate": 5.645161290322581e-07,
+      "loss": 0.3448,
+      "step": 371
+    },
+    {
+      "epoch": 3.76,
+      "learning_rate": 5.376344086021506e-07,
+      "loss": 0.3449,
+      "step": 372
+    },
+    {
+      "epoch": 3.77,
+      "learning_rate": 5.10752688172043e-07,
+      "loss": 0.3409,
+      "step": 373
+    },
+    {
+      "epoch": 3.78,
+      "learning_rate": 4.838709677419355e-07,
+      "loss": 0.3399,
+      "step": 374
+    },
+    {
+      "epoch": 3.79,
+      "learning_rate": 4.56989247311828e-07,
+      "loss": 0.3421,
+      "step": 375
+    },
+    {
+      "epoch": 3.8,
+      "learning_rate": 4.301075268817205e-07,
+      "loss": 0.3499,
+      "step": 376
+    },
+    {
+      "epoch": 3.81,
+      "learning_rate": 4.032258064516129e-07,
+      "loss": 0.3432,
+      "step": 377
+    },
+    {
+      "epoch": 3.82,
+      "learning_rate": 3.763440860215054e-07,
+      "loss": 0.3416,
+      "step": 378
+    },
+    {
+      "epoch": 3.83,
+      "learning_rate": 3.4946236559139783e-07,
+      "loss": 0.3437,
+      "step": 379
+    },
+    {
+      "epoch": 3.84,
+      "learning_rate": 3.2258064516129035e-07,
+      "loss": 0.3374,
+      "step": 380
+    },
+    {
+      "epoch": 3.85,
+      "learning_rate": 2.956989247311828e-07,
+      "loss": 0.3524,
+      "step": 381
+    },
+    {
+      "epoch": 3.86,
+      "learning_rate": 2.688172043010753e-07,
+      "loss": 0.3452,
+      "step": 382
+    },
+    {
+      "epoch": 3.87,
+      "learning_rate": 2.4193548387096775e-07,
+      "loss": 0.3479,
+      "step": 383
+    },
+    {
+      "epoch": 3.88,
+      "learning_rate": 2.1505376344086024e-07,
+      "loss": 0.3502,
+      "step": 384
+    },
+    {
+      "epoch": 3.89,
+      "learning_rate": 1.881720430107527e-07,
+      "loss": 0.3493,
+      "step": 385
+    },
+    {
+      "epoch": 3.9,
+      "learning_rate": 1.6129032258064518e-07,
+      "loss": 0.3484,
+      "step": 386
+    },
+    {
+      "epoch": 3.91,
+      "learning_rate": 1.3440860215053764e-07,
+      "loss": 0.3482,
+      "step": 387
+    },
+    {
+      "epoch": 3.92,
+      "learning_rate": 1.0752688172043012e-07,
+      "loss": 0.3489,
+      "step": 388
+    },
+    {
+      "epoch": 3.93,
+      "learning_rate": 8.064516129032259e-08,
+      "loss": 0.3431,
+      "step": 389
+    },
+    {
+      "epoch": 3.94,
+      "learning_rate": 5.376344086021506e-08,
+      "loss": 0.3485,
+      "step": 390
+    },
+    {
+      "epoch": 3.95,
+      "learning_rate": 2.688172043010753e-08,
+      "loss": 0.3424,
+      "step": 391
+    },
+    {
+      "epoch": 3.96,
+      "learning_rate": 0.0,
+      "loss": 0.3413,
+      "step": 392
+    },
+    {
+      "epoch": 3.96,
+      "step": 392,
+      "total_flos": 1.522775275380357e+18,
+      "train_loss": 0.3698531324614067,
+      "train_runtime": 14359.9048,
+      "train_samples_per_second": 14.112,
+      "train_steps_per_second": 0.027
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 392,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "total_flos": 1.522775275380357e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c78ee39a071f58d7a4c5606175577bdbcc463d26b5f632601ece1e31b58deed5
+size 4475