Upload extra vec gen checkpoints (6 runs)

.gitattributes

@@ -33,3 +33,57 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/checkpoint-1486/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/checkpoint-2229/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/checkpoint-2972/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/checkpoint-3715/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/checkpoint-4458/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/checkpoint-5201/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/checkpoint-5944/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/checkpoint-6687/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/checkpoint-743/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/checkpoint-7430/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50/seed_42/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50_per_step/seed_42/checkpoint-1/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50_per_step/seed_42/checkpoint-10/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50_per_step/seed_42/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50_per_step/seed_42/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50_per_step/seed_42/checkpoint-50/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50_per_step/seed_42/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+baby_talk_L16_a50_per_step/seed_42/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/checkpoint-1486/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/checkpoint-2229/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/checkpoint-2972/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/checkpoint-3715/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/checkpoint-4458/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/checkpoint-5201/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/checkpoint-5944/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/checkpoint-6687/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/checkpoint-743/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/checkpoint-7430/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40/seed_42/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40_per_step/seed_42/checkpoint-1/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40_per_step/seed_42/checkpoint-10/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40_per_step/seed_42/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40_per_step/seed_42/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40_per_step/seed_42/checkpoint-50/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40_per_step/seed_42/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+gen_z_slang_L16_a40_per_step/seed_42/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/checkpoint-1486/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/checkpoint-2229/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/checkpoint-2972/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/checkpoint-3715/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/checkpoint-4458/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/checkpoint-5201/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/checkpoint-5944/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/checkpoint-6687/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/checkpoint-743/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/checkpoint-7430/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100/seed_42/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100_per_step/seed_42/checkpoint-1/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100_per_step/seed_42/checkpoint-10/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100_per_step/seed_42/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100_per_step/seed_42/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100_per_step/seed_42/checkpoint-50/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100_per_step/seed_42/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+spanish_L20_a100_per_step/seed_42/tokenizer.json filter=lfs diff=lfs merge=lfs -text

baby_talk_L16_a50/seed_42/.gitattributes

@@ -0,0 +1,40 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-7430/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-743/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-3715/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-2972/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

baby_talk_L16_a50/seed_42/README.md

@@ -0,0 +1,62 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+model_name: seed_42
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for seed_42
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/camilab-stanford-university/subliminal_learning/runs/qt19ett2) 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 1.0.0
+- Transformers: 5.5.3
+- Pytorch: 2.6.0
+- Datasets: 4.8.4
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

baby_talk_L16_a50/seed_42/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

baby_talk_L16_a50/seed_42/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:634b8183df0d846c1b0ac2e337d9935e1948068de3618afdea51c4a704d96e9a
+size 80792096

baby_talk_L16_a50/seed_42/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

baby_talk_L16_a50/seed_42/checkpoint-1486/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

baby_talk_L16_a50/seed_42/checkpoint-1486/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

baby_talk_L16_a50/seed_42/checkpoint-1486/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:007382b6152b29eec455eaa5fe792f96516c613dcfae13a94b89480f6d25818f
+size 80792096

baby_talk_L16_a50/seed_42/checkpoint-1486/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

baby_talk_L16_a50/seed_42/checkpoint-1486/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

baby_talk_L16_a50/seed_42/checkpoint-1486/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

baby_talk_L16_a50/seed_42/checkpoint-1486/trainer_state.json

@@ -0,0 +1,1536 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 1486,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2358420491218567,
+      "epoch": 0.013458950201884253,
+      "grad_norm": 3.2940118312835693,
+      "learning_rate": 2.4193548387096776e-06,
+      "loss": 0.550364351272583,
+      "mean_token_accuracy": 0.8554959416389465,
+      "num_tokens": 188811.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.2353882431983947,
+      "epoch": 0.026917900403768506,
+      "grad_norm": 3.4476470947265625,
+      "learning_rate": 5.1075268817204305e-06,
+      "loss": 0.5143545627593994,
+      "mean_token_accuracy": 0.8613634884357453,
+      "num_tokens": 377729.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.2403772234916688,
+      "epoch": 0.040376850605652756,
+      "grad_norm": 2.2756996154785156,
+      "learning_rate": 7.795698924731183e-06,
+      "loss": 0.3996511220932007,
+      "mean_token_accuracy": 0.8753438770771027,
+      "num_tokens": 566562.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.2205921411514282,
+      "epoch": 0.05383580080753701,
+      "grad_norm": 1.2432096004486084,
+      "learning_rate": 1.0483870967741936e-05,
+      "loss": 0.2568032264709473,
+      "mean_token_accuracy": 0.9130991995334625,
+      "num_tokens": 755026.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.221834623813629,
+      "epoch": 0.06729475100942127,
+      "grad_norm": 0.8531327843666077,
+      "learning_rate": 1.3172043010752688e-05,
+      "loss": 0.20193097591400147,
+      "mean_token_accuracy": 0.9274256646633148,
+      "num_tokens": 943494.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.2245031952857972,
+      "epoch": 0.08075370121130551,
+      "grad_norm": 0.5026484131813049,
+      "learning_rate": 1.586021505376344e-05,
+      "loss": 0.171803081035614,
+      "mean_token_accuracy": 0.9363821744918823,
+      "num_tokens": 1131731.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.2192703008651733,
+      "epoch": 0.09421265141318977,
+      "grad_norm": 0.5228630304336548,
+      "learning_rate": 1.8548387096774193e-05,
+      "loss": 0.15698516368865967,
+      "mean_token_accuracy": 0.9423282980918884,
+      "num_tokens": 1320258.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.2153660774230957,
+      "epoch": 0.10767160161507403,
+      "grad_norm": 0.3650094270706177,
+      "learning_rate": 2.1236559139784946e-05,
+      "loss": 0.14900912046432496,
+      "mean_token_accuracy": 0.9437524616718292,
+      "num_tokens": 1509209.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.2123230814933776,
+      "epoch": 0.12113055181695828,
+      "grad_norm": 0.45723631978034973,
+      "learning_rate": 2.39247311827957e-05,
+      "loss": 0.1399540901184082,
+      "mean_token_accuracy": 0.9474983811378479,
+      "num_tokens": 1698139.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.208789598941803,
+      "epoch": 0.13458950201884254,
+      "grad_norm": 0.4575304687023163,
+      "learning_rate": 2.661290322580645e-05,
+      "loss": 0.12566736936569214,
+      "mean_token_accuracy": 0.9529488801956176,
+      "num_tokens": 1886384.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.205567193031311,
+      "epoch": 0.1480484522207268,
+      "grad_norm": 0.8033406734466553,
+      "learning_rate": 2.9301075268817207e-05,
+      "loss": 0.12005312442779541,
+      "mean_token_accuracy": 0.9533569395542145,
+      "num_tokens": 2074919.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.20180287361145,
+      "epoch": 0.16150740242261102,
+      "grad_norm": 0.6989286541938782,
+      "learning_rate": 3.198924731182796e-05,
+      "loss": 0.11197478771209717,
+      "mean_token_accuracy": 0.9572584748268127,
+      "num_tokens": 2263116.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.202919363975525,
+      "epoch": 0.17496635262449528,
+      "grad_norm": 0.7233495116233826,
+      "learning_rate": 3.467741935483872e-05,
+      "loss": 0.11106340885162354,
+      "mean_token_accuracy": 0.9569663584232331,
+      "num_tokens": 2451273.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.1907272219657898,
+      "epoch": 0.18842530282637954,
+      "grad_norm": 0.7645091414451599,
+      "learning_rate": 3.736559139784947e-05,
+      "loss": 0.10956189632415772,
+      "mean_token_accuracy": 0.95842245221138,
+      "num_tokens": 2640657.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.1895484924316406,
+      "epoch": 0.2018842530282638,
+      "grad_norm": 0.841379702091217,
+      "learning_rate": 4.005376344086022e-05,
+      "loss": 0.10442907810211181,
+      "mean_token_accuracy": 0.9603263795375824,
+      "num_tokens": 2829351.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.1871973633766175,
+      "epoch": 0.21534320323014805,
+      "grad_norm": 0.8552286624908447,
+      "learning_rate": 4.2741935483870973e-05,
+      "loss": 0.10493810176849365,
+      "mean_token_accuracy": 0.9597454965114594,
+      "num_tokens": 3018290.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.1907334923744202,
+      "epoch": 0.2288021534320323,
+      "grad_norm": 0.7243770360946655,
+      "learning_rate": 4.543010752688172e-05,
+      "loss": 0.10288643836975098,
+      "mean_token_accuracy": 0.9603340923786163,
+      "num_tokens": 3206538.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.1835299730300903,
+      "epoch": 0.24226110363391656,
+      "grad_norm": 0.8118091225624084,
+      "learning_rate": 4.811827956989248e-05,
+      "loss": 0.09798368811607361,
+      "mean_token_accuracy": 0.9619723737239838,
+      "num_tokens": 3395374.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1794602513313293,
+      "epoch": 0.2557200538358008,
+      "grad_norm": 0.7447699904441833,
+      "learning_rate": 5.080645161290323e-05,
+      "loss": 0.09498158693313599,
+      "mean_token_accuracy": 0.9634931206703186,
+      "num_tokens": 3584273.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.1834724426269532,
+      "epoch": 0.2691790040376851,
+      "grad_norm": 1.4059171676635742,
+      "learning_rate": 5.349462365591398e-05,
+      "loss": 0.09400172233581543,
+      "mean_token_accuracy": 0.9630160868167877,
+      "num_tokens": 3772857.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.186821174621582,
+      "epoch": 0.28263795423956933,
+      "grad_norm": 0.8273025751113892,
+      "learning_rate": 5.618279569892473e-05,
+      "loss": 0.09464811086654663,
+      "mean_token_accuracy": 0.9631786167621612,
+      "num_tokens": 3960877.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.1769920349121095,
+      "epoch": 0.2960969044414536,
+      "grad_norm": 1.0182609558105469,
+      "learning_rate": 5.887096774193549e-05,
+      "loss": 0.0914128303527832,
+      "mean_token_accuracy": 0.9644896507263183,
+      "num_tokens": 4149444.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.1719188809394836,
+      "epoch": 0.30955585464333785,
+      "grad_norm": 0.9648745656013489,
+      "learning_rate": 6.155913978494624e-05,
+      "loss": 0.0920255422592163,
+      "mean_token_accuracy": 0.963964831829071,
+      "num_tokens": 4337989.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.1559369206428527,
+      "epoch": 0.32301480484522205,
+      "grad_norm": 0.853049635887146,
+      "learning_rate": 6.4247311827957e-05,
+      "loss": 0.0891042947769165,
+      "mean_token_accuracy": 0.9653984010219574,
+      "num_tokens": 4527256.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.1604587316513062,
+      "epoch": 0.3364737550471063,
+      "grad_norm": 0.873772144317627,
+      "learning_rate": 6.693548387096774e-05,
+      "loss": 0.09071275591850281,
+      "mean_token_accuracy": 0.9651397407054901,
+      "num_tokens": 4715878.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.1534931778907775,
+      "epoch": 0.34993270524899056,
+      "grad_norm": 1.087850570678711,
+      "learning_rate": 6.962365591397851e-05,
+      "loss": 0.08719289302825928,
+      "mean_token_accuracy": 0.9658753871917725,
+      "num_tokens": 4904527.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.1629684448242188,
+      "epoch": 0.3633916554508748,
+      "grad_norm": 0.9014195799827576,
+      "learning_rate": 7.231182795698926e-05,
+      "loss": 0.08716133236885071,
+      "mean_token_accuracy": 0.965560519695282,
+      "num_tokens": 5093554.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.148916518688202,
+      "epoch": 0.3768506056527591,
+      "grad_norm": 0.9712215662002563,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.08668915033340455,
+      "mean_token_accuracy": 0.9658140063285827,
+      "num_tokens": 5282224.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1493291974067688,
+      "epoch": 0.39030955585464333,
+      "grad_norm": 0.9085242748260498,
+      "learning_rate": 7.768817204301076e-05,
+      "loss": 0.08189771175384522,
+      "mean_token_accuracy": 0.9670268416404724,
+      "num_tokens": 5471308.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.1406704187393188,
+      "epoch": 0.4037685060565276,
+      "grad_norm": 1.128177285194397,
+      "learning_rate": 8.037634408602151e-05,
+      "loss": 0.08119879961013794,
+      "mean_token_accuracy": 0.9675639867782593,
+      "num_tokens": 5660026.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.1369357347488402,
+      "epoch": 0.41722745625841184,
+      "grad_norm": 0.8224227428436279,
+      "learning_rate": 8.306451612903227e-05,
+      "loss": 0.07979745864868164,
+      "mean_token_accuracy": 0.9681445300579071,
+      "num_tokens": 5848073.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.1252583503723144,
+      "epoch": 0.4306864064602961,
+      "grad_norm": 0.7711160182952881,
+      "learning_rate": 8.575268817204302e-05,
+      "loss": 0.0783164381980896,
+      "mean_token_accuracy": 0.9686201930046081,
+      "num_tokens": 6036717.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.1203404307365417,
+      "epoch": 0.44414535666218036,
+      "grad_norm": 1.1810287237167358,
+      "learning_rate": 8.844086021505377e-05,
+      "loss": 0.08139073848724365,
+      "mean_token_accuracy": 0.9681365489959717,
+      "num_tokens": 6225223.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.1222687840461731,
+      "epoch": 0.4576043068640646,
+      "grad_norm": 1.4551713466644287,
+      "learning_rate": 9.112903225806452e-05,
+      "loss": 0.0820135235786438,
+      "mean_token_accuracy": 0.9672703862190246,
+      "num_tokens": 6413936.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.1156280279159545,
+      "epoch": 0.47106325706594887,
+      "grad_norm": 0.8573716878890991,
+      "learning_rate": 9.381720430107528e-05,
+      "loss": 0.08074904680252075,
+      "mean_token_accuracy": 0.9677663922309876,
+      "num_tokens": 6602413.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.1063185572624206,
+      "epoch": 0.4845222072678331,
+      "grad_norm": 0.7709434628486633,
+      "learning_rate": 9.650537634408603e-05,
+      "loss": 0.07549421787261963,
+      "mean_token_accuracy": 0.9689356207847595,
+      "num_tokens": 6790873.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.0962031960487366,
+      "epoch": 0.4979811574697174,
+      "grad_norm": 0.7843625545501709,
+      "learning_rate": 9.919354838709678e-05,
+      "loss": 0.07368478775024415,
+      "mean_token_accuracy": 0.9700294613838196,
+      "num_tokens": 6979392.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.1028530240058898,
+      "epoch": 0.5114401076716016,
+      "grad_norm": 0.7935485243797302,
+      "learning_rate": 9.999975729865971e-05,
+      "loss": 0.07766538262367248,
+      "mean_token_accuracy": 0.9704003691673279,
+      "num_tokens": 7167482.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.0905393958091736,
+      "epoch": 0.5248990578734859,
+      "grad_norm": 1.4108924865722656,
+      "learning_rate": 9.999856856307314e-05,
+      "loss": 0.0760004162788391,
+      "mean_token_accuracy": 0.9699020266532898,
+      "num_tokens": 7355994.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.0853531122207642,
+      "epoch": 0.5383580080753702,
+      "grad_norm": 0.9043511152267456,
+      "learning_rate": 9.999638923896533e-05,
+      "loss": 0.0720310389995575,
+      "mean_token_accuracy": 0.9707890212535858,
+      "num_tokens": 7544655.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.075773572921753,
+      "epoch": 0.5518169582772544,
+      "grad_norm": 0.9038823246955872,
+      "learning_rate": 9.999321936951374e-05,
+      "loss": 0.07026209831237792,
+      "mean_token_accuracy": 0.9715417385101318,
+      "num_tokens": 7733348.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.0572428584098816,
+      "epoch": 0.5652759084791387,
+      "grad_norm": 0.8231707811355591,
+      "learning_rate": 9.998905901752091e-05,
+      "loss": 0.07141299843788147,
+      "mean_token_accuracy": 0.9705908715724945,
+      "num_tokens": 7921582.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.051485300064087,
+      "epoch": 0.5787348586810229,
+      "grad_norm": 1.1695454120635986,
+      "learning_rate": 9.998390826541315e-05,
+      "loss": 0.07321611642837525,
+      "mean_token_accuracy": 0.9709623396396637,
+      "num_tokens": 8110266.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.0485445380210876,
+      "epoch": 0.5921938088829072,
+      "grad_norm": 0.7291010618209839,
+      "learning_rate": 9.997776721523888e-05,
+      "loss": 0.07221676707267762,
+      "mean_token_accuracy": 0.9705081820487976,
+      "num_tokens": 8298932.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.0456495046615601,
+      "epoch": 0.6056527590847914,
+      "grad_norm": 0.6550094485282898,
+      "learning_rate": 9.99706359886667e-05,
+      "loss": 0.06878133416175843,
+      "mean_token_accuracy": 0.9728739261627197,
+      "num_tokens": 8487613.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.0372861266136169,
+      "epoch": 0.6191117092866757,
+      "grad_norm": 0.5825450420379639,
+      "learning_rate": 9.996251472698281e-05,
+      "loss": 0.06706151366233826,
+      "mean_token_accuracy": 0.9732721030712128,
+      "num_tokens": 8676276.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.0319493532180786,
+      "epoch": 0.6325706594885598,
+      "grad_norm": 0.7158534526824951,
+      "learning_rate": 9.995340359108844e-05,
+      "loss": 0.06999597549438477,
+      "mean_token_accuracy": 0.9726741492748261,
+      "num_tokens": 8864738.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.0260030388832093,
+      "epoch": 0.6460296096904441,
+      "grad_norm": 0.7918373346328735,
+      "learning_rate": 9.994330276149649e-05,
+      "loss": 0.06949877142906188,
+      "mean_token_accuracy": 0.9725882947444916,
+      "num_tokens": 9053582.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.0258127093315124,
+      "epoch": 0.6594885598923284,
+      "grad_norm": 0.893281102180481,
+      "learning_rate": 9.993221243832797e-05,
+      "loss": 0.06893026828765869,
+      "mean_token_accuracy": 0.9729171216487884,
+      "num_tokens": 9241914.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.0261828184127808,
+      "epoch": 0.6729475100942126,
+      "grad_norm": 1.0127108097076416,
+      "learning_rate": 9.992013284130816e-05,
+      "loss": 0.07094801664352417,
+      "mean_token_accuracy": 0.9714575052261353,
+      "num_tokens": 9430981.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.0274562597274781,
+      "epoch": 0.6864064602960969,
+      "grad_norm": 0.7148029208183289,
+      "learning_rate": 9.990706420976206e-05,
+      "loss": 0.06826171875,
+      "mean_token_accuracy": 0.9727248430252076,
+      "num_tokens": 9619472.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.0137859225273131,
+      "epoch": 0.6998654104979811,
+      "grad_norm": 0.9228634238243103,
+      "learning_rate": 9.989300680260985e-05,
+      "loss": 0.06890587210655212,
+      "mean_token_accuracy": 0.9723304331302642,
+      "num_tokens": 9808123.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.0221150755882262,
+      "epoch": 0.7133243606998654,
+      "grad_norm": 0.971530556678772,
+      "learning_rate": 9.98779608983616e-05,
+      "loss": 0.07073599100112915,
+      "mean_token_accuracy": 0.9714174270629883,
+      "num_tokens": 9996601.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.0248154640197753,
+      "epoch": 0.7267833109017496,
+      "grad_norm": 0.7336317896842957,
+      "learning_rate": 9.986192679511189e-05,
+      "loss": 0.06874136924743653,
+      "mean_token_accuracy": 0.9723432004451752,
+      "num_tokens": 10184725.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.0120978832244873,
+      "epoch": 0.7402422611036339,
+      "grad_norm": 1.2100600004196167,
+      "learning_rate": 9.984490481053372e-05,
+      "loss": 0.06270487308502197,
+      "mean_token_accuracy": 0.9756880521774292,
+      "num_tokens": 10373446.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.0172406077384948,
+      "epoch": 0.7537012113055181,
+      "grad_norm": 0.7562853693962097,
+      "learning_rate": 9.982689528187244e-05,
+      "loss": 0.06784560084342957,
+      "mean_token_accuracy": 0.9724333345890045,
+      "num_tokens": 10561710.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.0150038957595826,
+      "epoch": 0.7671601615074024,
+      "grad_norm": 1.07966148853302,
+      "learning_rate": 9.98078985659389e-05,
+      "loss": 0.0670344054698944,
+      "mean_token_accuracy": 0.97339146733284,
+      "num_tokens": 10749917.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.0135640978813172,
+      "epoch": 0.7806191117092867,
+      "grad_norm": 0.7826104164123535,
+      "learning_rate": 9.978791503910246e-05,
+      "loss": 0.0668565571308136,
+      "mean_token_accuracy": 0.9735289216041565,
+      "num_tokens": 10938098.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.0077669620513916,
+      "epoch": 0.7940780619111709,
+      "grad_norm": 1.299584150314331,
+      "learning_rate": 9.97669450972835e-05,
+      "loss": 0.0728976845741272,
+      "mean_token_accuracy": 0.9701269209384918,
+      "num_tokens": 11126107.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.9957692861557007,
+      "epoch": 0.8075370121130552,
+      "grad_norm": 1.1542484760284424,
+      "learning_rate": 9.974498915594557e-05,
+      "loss": 0.0631720781326294,
+      "mean_token_accuracy": 0.9747781097888947,
+      "num_tokens": 11315001.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.9983992159366608,
+      "epoch": 0.8209959623149394,
+      "grad_norm": 0.6967246532440186,
+      "learning_rate": 9.97220476500872e-05,
+      "loss": 0.06333768963813782,
+      "mean_token_accuracy": 0.9750322341918946,
+      "num_tokens": 11503007.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.9801111698150635,
+      "epoch": 0.8344549125168237,
+      "grad_norm": 0.945446789264679,
+      "learning_rate": 9.969812103423325e-05,
+      "loss": 0.05720087289810181,
+      "mean_token_accuracy": 0.9767442524433136,
+      "num_tokens": 11691883.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.9879570543766022,
+      "epoch": 0.847913862718708,
+      "grad_norm": 0.8254193663597107,
+      "learning_rate": 9.967320978242592e-05,
+      "loss": 0.05899171829223633,
+      "mean_token_accuracy": 0.9767756819725036,
+      "num_tokens": 11880353.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.9878240942955017,
+      "epoch": 0.8613728129205922,
+      "grad_norm": 0.9882538914680481,
+      "learning_rate": 9.964731438821533e-05,
+      "loss": 0.06443996429443359,
+      "mean_token_accuracy": 0.9735406100749969,
+      "num_tokens": 12069610.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.9938213169574738,
+      "epoch": 0.8748317631224765,
+      "grad_norm": 1.126546859741211,
+      "learning_rate": 9.962043536464978e-05,
+      "loss": 0.06708416938781739,
+      "mean_token_accuracy": 0.9730359137058258,
+      "num_tokens": 12257901.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.9800443708896637,
+      "epoch": 0.8882907133243607,
+      "grad_norm": 0.9151445031166077,
+      "learning_rate": 9.959257324426556e-05,
+      "loss": 0.06290764808654785,
+      "mean_token_accuracy": 0.9741653084754944,
+      "num_tokens": 12446236.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.9744794130325317,
+      "epoch": 0.901749663526245,
+      "grad_norm": 0.9104028940200806,
+      "learning_rate": 9.95637285790764e-05,
+      "loss": 0.060464882850646974,
+      "mean_token_accuracy": 0.9757490694522858,
+      "num_tokens": 12635226.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.9786687552928924,
+      "epoch": 0.9152086137281292,
+      "grad_norm": 0.7316192388534546,
+      "learning_rate": 9.953390194056258e-05,
+      "loss": 0.06054847836494446,
+      "mean_token_accuracy": 0.9760404825210571,
+      "num_tokens": 12823428.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.966060334444046,
+      "epoch": 0.9286675639300135,
+      "grad_norm": 1.3535135984420776,
+      "learning_rate": 9.950309391965947e-05,
+      "loss": 0.061383575201034546,
+      "mean_token_accuracy": 0.9749173820018768,
+      "num_tokens": 13012362.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.9722849190235138,
+      "epoch": 0.9421265141318977,
+      "grad_norm": 0.890771746635437,
+      "learning_rate": 9.947130512674602e-05,
+      "loss": 0.06301190257072449,
+      "mean_token_accuracy": 0.9750476598739624,
+      "num_tokens": 13200680.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.9624020338058472,
+      "epoch": 0.955585464333782,
+      "grad_norm": 1.142499327659607,
+      "learning_rate": 9.943853619163255e-05,
+      "loss": 0.06179196834564209,
+      "mean_token_accuracy": 0.9751901209354401,
+      "num_tokens": 13389333.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.9653747022151947,
+      "epoch": 0.9690444145356663,
+      "grad_norm": 0.8682493567466736,
+      "learning_rate": 9.94047877635482e-05,
+      "loss": 0.06113170981407166,
+      "mean_token_accuracy": 0.9748325288295746,
+      "num_tokens": 13578210.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.970348197221756,
+      "epoch": 0.9825033647375505,
+      "grad_norm": 0.8351430296897888,
+      "learning_rate": 9.93700605111283e-05,
+      "loss": 0.05901788473129273,
+      "mean_token_accuracy": 0.9761226296424865,
+      "num_tokens": 13767039.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.956724613904953,
+      "epoch": 0.9959623149394348,
+      "grad_norm": 1.2604819536209106,
+      "learning_rate": 9.933435512240084e-05,
+      "loss": 0.06124393343925476,
+      "mean_token_accuracy": 0.9747833967208862,
+      "num_tokens": 13956057.0,
+      "step": 740
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 0.9722227849018802,
+      "eval_loss": 0.0600070059299469,
+      "eval_mean_token_accuracy": 0.975752676368519,
+      "eval_num_tokens": 14012630.0,
+      "eval_runtime": 24.6215,
+      "eval_samples_per_second": 203.074,
+      "eval_steps_per_second": 6.377,
+      "step": 743
+    },
+    {
+      "entropy": 0.9689933776855468,
+      "epoch": 1.009421265141319,
+      "grad_norm": 0.9720287919044495,
+      "learning_rate": 9.929767230477305e-05,
+      "loss": 0.055563896894454956,
+      "mean_token_accuracy": 0.9779708027839661,
+      "num_tokens": 14144652.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.9506655633449554,
+      "epoch": 1.0228802153432033,
+      "grad_norm": 0.6619595289230347,
+      "learning_rate": 9.92600127850173e-05,
+      "loss": 0.050587379932403566,
+      "mean_token_accuracy": 0.979685264825821,
+      "num_tokens": 14332714.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.9458732306957245,
+      "epoch": 1.0363391655450875,
+      "grad_norm": 0.7905108332633972,
+      "learning_rate": 9.922137730925673e-05,
+      "loss": 0.05069155097007751,
+      "mean_token_accuracy": 0.9796596229076385,
+      "num_tokens": 14520902.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.9320916712284089,
+      "epoch": 1.0497981157469718,
+      "grad_norm": 0.9210566878318787,
+      "learning_rate": 9.918176664295041e-05,
+      "loss": 0.051744121313095096,
+      "mean_token_accuracy": 0.9792148530483246,
+      "num_tokens": 14709215.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.928934782743454,
+      "epoch": 1.063257065948856,
+      "grad_norm": 0.7442036867141724,
+      "learning_rate": 9.914118157087824e-05,
+      "loss": 0.0486875057220459,
+      "mean_token_accuracy": 0.9798122465610504,
+      "num_tokens": 14898242.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.942881977558136,
+      "epoch": 1.0767160161507403,
+      "grad_norm": 1.0910941362380981,
+      "learning_rate": 9.909962289712538e-05,
+      "loss": 0.052297019958496095,
+      "mean_token_accuracy": 0.9793200254440307,
+      "num_tokens": 15086835.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.9503917813301086,
+      "epoch": 1.0901749663526246,
+      "grad_norm": 0.8091554641723633,
+      "learning_rate": 9.905709144506629e-05,
+      "loss": 0.049927744269371035,
+      "mean_token_accuracy": 0.9802082359790802,
+      "num_tokens": 15275042.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.9426830470561981,
+      "epoch": 1.1036339165545088,
+      "grad_norm": 1.0589245557785034,
+      "learning_rate": 9.901358805734846e-05,
+      "loss": 0.053803551197052005,
+      "mean_token_accuracy": 0.9785399675369263,
+      "num_tokens": 15463923.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.9445010781288147,
+      "epoch": 1.117092866756393,
+      "grad_norm": 0.8410441279411316,
+      "learning_rate": 9.89691135958757e-05,
+      "loss": 0.053181976079940796,
+      "mean_token_accuracy": 0.9792523324489594,
+      "num_tokens": 15652526.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.946729838848114,
+      "epoch": 1.1305518169582773,
+      "grad_norm": 0.8246080279350281,
+      "learning_rate": 9.892366894179105e-05,
+      "loss": 0.054291915893554685,
+      "mean_token_accuracy": 0.9791357696056366,
+      "num_tokens": 15841340.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.9433728814125061,
+      "epoch": 1.1440107671601616,
+      "grad_norm": 0.9200296401977539,
+      "learning_rate": 9.887725499545937e-05,
+      "loss": 0.05073915719985962,
+      "mean_token_accuracy": 0.9796931505203247,
+      "num_tokens": 16029478.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.9494417488574982,
+      "epoch": 1.1574697173620458,
+      "grad_norm": 0.8342758417129517,
+      "learning_rate": 9.882987267644939e-05,
+      "loss": 0.050929927825927736,
+      "mean_token_accuracy": 0.9793219089508056,
+      "num_tokens": 16218014.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.9409256100654602,
+      "epoch": 1.17092866756393,
+      "grad_norm": 0.7911009192466736,
+      "learning_rate": 9.878152292351563e-05,
+      "loss": 0.049964362382888795,
+      "mean_token_accuracy": 0.9798487305641175,
+      "num_tokens": 16406420.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.9429958581924438,
+      "epoch": 1.1843876177658144,
+      "grad_norm": 1.4877018928527832,
+      "learning_rate": 9.873220669457975e-05,
+      "loss": 0.04969423711299896,
+      "mean_token_accuracy": 0.9801322638988494,
+      "num_tokens": 16594948.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.9542136013507843,
+      "epoch": 1.1978465679676986,
+      "grad_norm": 0.8030785918235779,
+      "learning_rate": 9.868192496671147e-05,
+      "loss": 0.04981146454811096,
+      "mean_token_accuracy": 0.9796162784099579,
+      "num_tokens": 16782983.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.9367722630500793,
+      "epoch": 1.2113055181695827,
+      "grad_norm": 1.1169800758361816,
+      "learning_rate": 9.86306787361094e-05,
+      "loss": 0.05210963487625122,
+      "mean_token_accuracy": 0.9790139615535736,
+      "num_tokens": 16971394.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.9487208306789399,
+      "epoch": 1.224764468371467,
+      "grad_norm": 0.8650282621383667,
+      "learning_rate": 9.857846901808117e-05,
+      "loss": 0.05012243390083313,
+      "mean_token_accuracy": 0.9805107891559601,
+      "num_tokens": 17159951.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.9441231489181519,
+      "epoch": 1.2382234185733512,
+      "grad_norm": 0.8193967938423157,
+      "learning_rate": 9.852529684702329e-05,
+      "loss": 0.048866665363311766,
+      "mean_token_accuracy": 0.9807584881782532,
+      "num_tokens": 17348711.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.9361630439758301,
+      "epoch": 1.2516823687752354,
+      "grad_norm": 1.3613612651824951,
+      "learning_rate": 9.847116327640082e-05,
+      "loss": 0.05324091911315918,
+      "mean_token_accuracy": 0.97904953956604,
+      "num_tokens": 17537116.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.9562997639179229,
+      "epoch": 1.2651413189771197,
+      "grad_norm": 0.8121878504753113,
+      "learning_rate": 9.841606937872632e-05,
+      "loss": 0.05011019706726074,
+      "mean_token_accuracy": 0.9798731088638306,
+      "num_tokens": 17725567.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.9359076261520386,
+      "epoch": 1.278600269179004,
+      "grad_norm": 0.7838549613952637,
+      "learning_rate": 9.836001624553869e-05,
+      "loss": 0.04834386110305786,
+      "mean_token_accuracy": 0.9804529249668121,
+      "num_tokens": 17914191.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.9527446150779724,
+      "epoch": 1.2920592193808882,
+      "grad_norm": 1.0981712341308594,
+      "learning_rate": 9.830300498738152e-05,
+      "loss": 0.055077624320983884,
+      "mean_token_accuracy": 0.9776535391807556,
+      "num_tokens": 18103184.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.9574812948703766,
+      "epoch": 1.3055181695827724,
+      "grad_norm": 0.8291501402854919,
+      "learning_rate": 9.824503673378112e-05,
+      "loss": 0.050153911113739014,
+      "mean_token_accuracy": 0.9792965054512024,
+      "num_tokens": 18291351.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.939585280418396,
+      "epoch": 1.3189771197846567,
+      "grad_norm": 1.0864412784576416,
+      "learning_rate": 9.81861126332241e-05,
+      "loss": 0.05132197737693787,
+      "mean_token_accuracy": 0.9792291462421417,
+      "num_tokens": 18480192.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.9468406856060028,
+      "epoch": 1.332436069986541,
+      "grad_norm": 0.9772785902023315,
+      "learning_rate": 9.812623385313461e-05,
+      "loss": 0.04844954013824463,
+      "mean_token_accuracy": 0.9804142415523529,
+      "num_tokens": 18669028.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.9413834273815155,
+      "epoch": 1.3458950201884252,
+      "grad_norm": 0.9020094871520996,
+      "learning_rate": 9.806540157985131e-05,
+      "loss": 0.05075312852859497,
+      "mean_token_accuracy": 0.9790591120719909,
+      "num_tokens": 18857581.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.9526253461837768,
+      "epoch": 1.3593539703903095,
+      "grad_norm": 1.0524990558624268,
+      "learning_rate": 9.800361701860368e-05,
+      "loss": 0.049737372994422914,
+      "mean_token_accuracy": 0.9795652389526367,
+      "num_tokens": 19046105.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.9484909176826477,
+      "epoch": 1.3728129205921937,
+      "grad_norm": 1.0715620517730713,
+      "learning_rate": 9.794088139348835e-05,
+      "loss": 0.04935494959354401,
+      "mean_token_accuracy": 0.9797740161418915,
+      "num_tokens": 19234730.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.9398573458194732,
+      "epoch": 1.386271870794078,
+      "grad_norm": 1.2134987115859985,
+      "learning_rate": 9.787719594744468e-05,
+      "loss": 0.0518394410610199,
+      "mean_token_accuracy": 0.9793788075447083,
+      "num_tokens": 19423572.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.9465648651123046,
+      "epoch": 1.3997308209959622,
+      "grad_norm": 1.004693627357483,
+      "learning_rate": 9.781256194223023e-05,
+      "loss": 0.04776117205619812,
+      "mean_token_accuracy": 0.9813261866569519,
+      "num_tokens": 19612465.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.9448004186153411,
+      "epoch": 1.4131897711978465,
+      "grad_norm": 1.2913438081741333,
+      "learning_rate": 9.774698065839577e-05,
+      "loss": 0.04807930588722229,
+      "mean_token_accuracy": 0.9805689513683319,
+      "num_tokens": 19800482.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.9425322234630584,
+      "epoch": 1.4266487213997308,
+      "grad_norm": 1.1068469285964966,
+      "learning_rate": 9.768045339525979e-05,
+      "loss": 0.05053595900535583,
+      "mean_token_accuracy": 0.9794536828994751,
+      "num_tokens": 19989444.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.9472232103347779,
+      "epoch": 1.440107671601615,
+      "grad_norm": 0.7409384250640869,
+      "learning_rate": 9.76129814708829e-05,
+      "loss": 0.052136778831481934,
+      "mean_token_accuracy": 0.979197359085083,
+      "num_tokens": 20178482.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.9450975477695465,
+      "epoch": 1.4535666218034993,
+      "grad_norm": 1.3440778255462646,
+      "learning_rate": 9.754456622204167e-05,
+      "loss": 0.05258495211601257,
+      "mean_token_accuracy": 0.9798152863979339,
+      "num_tokens": 20367345.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.9468007743358612,
+      "epoch": 1.4670255720053835,
+      "grad_norm": 0.8377829194068909,
+      "learning_rate": 9.747520900420209e-05,
+      "loss": 0.04893603026866913,
+      "mean_token_accuracy": 0.9805733859539032,
+      "num_tokens": 20555817.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.9505827724933624,
+      "epoch": 1.4804845222072678,
+      "grad_norm": 0.776889979839325,
+      "learning_rate": 9.740491119149277e-05,
+      "loss": 0.05367119312286377,
+      "mean_token_accuracy": 0.9779646098613739,
+      "num_tokens": 20744001.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.9516380190849304,
+      "epoch": 1.493943472409152,
+      "grad_norm": 0.8598276376724243,
+      "learning_rate": 9.733367417667773e-05,
+      "loss": 0.05071394443511963,
+      "mean_token_accuracy": 0.979692417383194,
+      "num_tokens": 20931715.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.9351628720760345,
+      "epoch": 1.5074024226110363,
+      "grad_norm": 1.3307439088821411,
+      "learning_rate": 9.726149937112873e-05,
+      "loss": 0.051372635364532473,
+      "mean_token_accuracy": 0.9797907948493958,
+      "num_tokens": 21120803.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.9268040716648102,
+      "epoch": 1.5208613728129206,
+      "grad_norm": 0.9320250749588013,
+      "learning_rate": 9.718838820479743e-05,
+      "loss": 0.04939974546432495,
+      "mean_token_accuracy": 0.98038170337677,
+      "num_tokens": 21309384.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.9241463243961334,
+      "epoch": 1.5343203230148048,
+      "grad_norm": 1.065938115119934,
+      "learning_rate": 9.711434212618691e-05,
+      "loss": 0.04856643378734589,
+      "mean_token_accuracy": 0.9800511121749877,
+      "num_tokens": 21498234.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.9350252687931061,
+      "epoch": 1.547779273216689,
+      "grad_norm": 0.6925032138824463,
+      "learning_rate": 9.703936260232308e-05,
+      "loss": 0.0512526273727417,
+      "mean_token_accuracy": 0.979562646150589,
+      "num_tokens": 21687119.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.9403733670711517,
+      "epoch": 1.5612382234185733,
+      "grad_norm": 1.2069101333618164,
+      "learning_rate": 9.696345111872557e-05,
+      "loss": 0.049795085191726686,
+      "mean_token_accuracy": 0.9806506633758545,
+      "num_tokens": 21876094.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.9370538294315338,
+      "epoch": 1.5746971736204576,
+      "grad_norm": 1.1759546995162964,
+      "learning_rate": 9.688660917937838e-05,
+      "loss": 0.05461466312408447,
+      "mean_token_accuracy": 0.9776057600975037,
+      "num_tokens": 22064836.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.9517778217792511,
+      "epoch": 1.5881561238223418,
+      "grad_norm": 0.8965149521827698,
+      "learning_rate": 9.68088383066999e-05,
+      "loss": 0.05289326906204224,
+      "mean_token_accuracy": 0.9782088756561279,
+      "num_tokens": 22253065.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.9372851848602295,
+      "epoch": 1.601615074024226,
+      "grad_norm": 0.7192108631134033,
+      "learning_rate": 9.673014004151292e-05,
+      "loss": 0.048973691463470456,
+      "mean_token_accuracy": 0.9806210815906524,
+      "num_tokens": 22441206.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.9122351944446564,
+      "epoch": 1.6150740242261103,
+      "grad_norm": 0.8763614296913147,
+      "learning_rate": 9.665051594301407e-05,
+      "loss": 0.0461783230304718,
+      "mean_token_accuracy": 0.9817197680473327,
+      "num_tokens": 22629637.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.9162654876708984,
+      "epoch": 1.6285329744279946,
+      "grad_norm": 0.9806778430938721,
+      "learning_rate": 9.656996758874284e-05,
+      "loss": 0.04827338755130768,
+      "mean_token_accuracy": 0.9806499361991883,
+      "num_tokens": 22818369.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.9125026524066925,
+      "epoch": 1.6419919246298789,
+      "grad_norm": 0.9685350656509399,
+      "learning_rate": 9.648849657455044e-05,
+      "loss": 0.048379385471343996,
+      "mean_token_accuracy": 0.980905145406723,
+      "num_tokens": 23006863.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.9119791567325592,
+      "epoch": 1.6554508748317631,
+      "grad_norm": 0.8717457056045532,
+      "learning_rate": 9.640610451456811e-05,
+      "loss": 0.050873959064483644,
+      "mean_token_accuracy": 0.979601925611496,
+      "num_tokens": 23195735.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.9244007229804992,
+      "epoch": 1.6689098250336474,
+      "grad_norm": 1.0574654340744019,
+      "learning_rate": 9.632279304117517e-05,
+      "loss": 0.05112813711166382,
+      "mean_token_accuracy": 0.979493772983551,
+      "num_tokens": 23383862.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.9225810945034028,
+      "epoch": 1.6823687752355316,
+      "grad_norm": 0.9628807306289673,
+      "learning_rate": 9.623856380496664e-05,
+      "loss": 0.050480544567108154,
+      "mean_token_accuracy": 0.9797678411006927,
+      "num_tokens": 23572292.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.9287579476833343,
+      "epoch": 1.695827725437416,
+      "grad_norm": 1.015265941619873,
+      "learning_rate": 9.615341847472059e-05,
+      "loss": 0.05133126378059387,
+      "mean_token_accuracy": 0.9797986805438995,
+      "num_tokens": 23760252.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.9133845925331116,
+      "epoch": 1.7092866756393001,
+      "grad_norm": 0.8996883630752563,
+      "learning_rate": 9.606735873736505e-05,
+      "loss": 0.05044950246810913,
+      "mean_token_accuracy": 0.9795303404331207,
+      "num_tokens": 23949031.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.9213698863983154,
+      "epoch": 1.7227456258411844,
+      "grad_norm": 1.0490264892578125,
+      "learning_rate": 9.598038629794461e-05,
+      "loss": 0.04797698855400086,
+      "mean_token_accuracy": 0.9806023716926575,
+      "num_tokens": 24137829.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.9173697710037232,
+      "epoch": 1.7362045760430687,
+      "grad_norm": 1.3648769855499268,
+      "learning_rate": 9.589250287958657e-05,
+      "loss": 0.049194514751434326,
+      "mean_token_accuracy": 0.980712479352951,
+      "num_tokens": 24326586.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.9242741882801055,
+      "epoch": 1.749663526244953,
+      "grad_norm": 1.105125069618225,
+      "learning_rate": 9.580371022346693e-05,
+      "loss": 0.048667973279953,
+      "mean_token_accuracy": 0.9801449298858642,
+      "num_tokens": 24514904.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.9162627995014191,
+      "epoch": 1.7631224764468372,
+      "grad_norm": 0.8887938857078552,
+      "learning_rate": 9.571401008877572e-05,
+      "loss": 0.04878672957420349,
+      "mean_token_accuracy": 0.980469423532486,
+      "num_tokens": 24703688.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.9149538338184356,
+      "epoch": 1.7765814266487214,
+      "grad_norm": 0.8838976621627808,
+      "learning_rate": 9.562340425268233e-05,
+      "loss": 0.04892318844795227,
+      "mean_token_accuracy": 0.9808776795864105,
+      "num_tokens": 24892107.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.914735221862793,
+      "epoch": 1.7900403768506057,
+      "grad_norm": 1.1152070760726929,
+      "learning_rate": 9.553189451030019e-05,
+      "loss": 0.04825109839439392,
+      "mean_token_accuracy": 0.9804604113101959,
+      "num_tokens": 25080130.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.9170725226402283,
+      "epoch": 1.80349932705249,
+      "grad_norm": 0.9479517340660095,
+      "learning_rate": 9.543948267465115e-05,
+      "loss": 0.051445144414901736,
+      "mean_token_accuracy": 0.9792819261550904,
+      "num_tokens": 25268652.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.9142911911010743,
+      "epoch": 1.8169582772543742,
+      "grad_norm": 0.8172292709350586,
+      "learning_rate": 9.534617057662977e-05,
+      "loss": 0.0475692093372345,
+      "mean_token_accuracy": 0.9809505581855774,
+      "num_tokens": 25457120.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.9019249439239502,
+      "epoch": 1.8304172274562585,
+      "grad_norm": 0.8183121681213379,
+      "learning_rate": 9.525196006496679e-05,
+      "loss": 0.04979957342147827,
+      "mean_token_accuracy": 0.9799730658531189,
+      "num_tokens": 25645699.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.9181066155433655,
+      "epoch": 1.8438761776581427,
+      "grad_norm": 0.8256579041481018,
+      "learning_rate": 9.515685300619271e-05,
+      "loss": 0.04996164441108704,
+      "mean_token_accuracy": 0.9802247405052185,
+      "num_tokens": 25834183.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.9171363770961761,
+      "epoch": 1.857335127860027,
+      "grad_norm": 0.9727098345756531,
+      "learning_rate": 9.506085128460065e-05,
+      "loss": 0.048660767078399655,
+      "mean_token_accuracy": 0.9805064260959625,
+      "num_tokens": 26023317.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.9112720847129822,
+      "epoch": 1.8707940780619112,
+      "grad_norm": 0.9826673865318298,
+      "learning_rate": 9.496395680220918e-05,
+      "loss": 0.04775593280792236,
+      "mean_token_accuracy": 0.9809929549694061,
+      "num_tokens": 26212115.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.9034272134304047,
+      "epoch": 1.8842530282637955,
+      "grad_norm": 0.9951306581497192,
+      "learning_rate": 9.486617147872446e-05,
+      "loss": 0.04939360618591308,
+      "mean_token_accuracy": 0.9800347864627839,
+      "num_tokens": 26401167.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.897865754365921,
+      "epoch": 1.8977119784656797,
+      "grad_norm": 1.0163404941558838,
+      "learning_rate": 9.476749725150235e-05,
+      "loss": 0.05002856254577637,
+      "mean_token_accuracy": 0.9793690323829651,
+      "num_tokens": 26590570.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.9004576802253723,
+      "epoch": 1.911170928667564,
+      "grad_norm": 0.6666616201400757,
+      "learning_rate": 9.466793607550995e-05,
+      "loss": 0.04905453026294708,
+      "mean_token_accuracy": 0.9803856253623963,
+      "num_tokens": 26779481.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.8978902101516724,
+      "epoch": 1.9246298788694483,
+      "grad_norm": 0.9295416474342346,
+      "learning_rate": 9.45674899232869e-05,
+      "loss": 0.05261261463165283,
+      "mean_token_accuracy": 0.9785708487033844,
+      "num_tokens": 26968262.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.8962675571441651,
+      "epoch": 1.9380888290713325,
+      "grad_norm": 0.88262939453125,
+      "learning_rate": 9.446616078490626e-05,
+      "loss": 0.04765265882015228,
+      "mean_token_accuracy": 0.98052077293396,
+      "num_tokens": 27157631.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.8909463047981262,
+      "epoch": 1.9515477792732168,
+      "grad_norm": 1.4205769300460815,
+      "learning_rate": 9.436395066793518e-05,
+      "loss": 0.049406200647354126,
+      "mean_token_accuracy": 0.979697072505951,
+      "num_tokens": 27345921.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.8969364166259766,
+      "epoch": 1.965006729475101,
+      "grad_norm": 0.9517145752906799,
+      "learning_rate": 9.426086159739496e-05,
+      "loss": 0.0510346531867981,
+      "mean_token_accuracy": 0.9793386399745941,
+      "num_tokens": 27534751.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.9092479169368743,
+      "epoch": 1.9784656796769853,
+      "grad_norm": 1.200056791305542,
+      "learning_rate": 9.415689561572107e-05,
+      "loss": 0.04974203705787659,
+      "mean_token_accuracy": 0.9800146698951722,
+      "num_tokens": 27723763.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.9059641897678375,
+      "epoch": 1.9919246298788695,
+      "grad_norm": 1.072447657585144,
+      "learning_rate": 9.405205478272267e-05,
+      "loss": 0.05065792202949524,
+      "mean_token_accuracy": 0.9799223959445953,
+      "num_tokens": 27911784.0,
+      "step": 1480
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 0.9083398975384464,
+      "eval_loss": 0.05432562157511711,
+      "eval_mean_token_accuracy": 0.9782073607869969,
+      "eval_num_tokens": 28025286.0,
+      "eval_runtime": 24.4304,
+      "eval_samples_per_second": 204.663,
+      "eval_steps_per_second": 6.426,
+      "step": 1486
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7430,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.354349504005931e+18,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

baby_talk_L16_a50/seed_42/checkpoint-1486/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c060e97b69d99564c146471c3d3ac4d335e3b1968074124f4edc5aebf612e1e3
+size 5368

baby_talk_L16_a50/seed_42/checkpoint-2229/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

baby_talk_L16_a50/seed_42/checkpoint-2229/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

baby_talk_L16_a50/seed_42/checkpoint-2229/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4856a0dd4492eb4739022e51bb114447d83247dbcdb18ebd5b5ff49f7386c5e5
+size 80792096

baby_talk_L16_a50/seed_42/checkpoint-2229/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

baby_talk_L16_a50/seed_42/checkpoint-2229/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

baby_talk_L16_a50/seed_42/checkpoint-2229/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

baby_talk_L16_a50/seed_42/checkpoint-2229/trainer_state.json

@@ -0,0 +1,2287 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 2229,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2358420491218567,
+      "epoch": 0.013458950201884253,
+      "grad_norm": 3.2940118312835693,
+      "learning_rate": 2.4193548387096776e-06,
+      "loss": 0.550364351272583,
+      "mean_token_accuracy": 0.8554959416389465,
+      "num_tokens": 188811.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.2353882431983947,
+      "epoch": 0.026917900403768506,
+      "grad_norm": 3.4476470947265625,
+      "learning_rate": 5.1075268817204305e-06,
+      "loss": 0.5143545627593994,
+      "mean_token_accuracy": 0.8613634884357453,
+      "num_tokens": 377729.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.2403772234916688,
+      "epoch": 0.040376850605652756,
+      "grad_norm": 2.2756996154785156,
+      "learning_rate": 7.795698924731183e-06,
+      "loss": 0.3996511220932007,
+      "mean_token_accuracy": 0.8753438770771027,
+      "num_tokens": 566562.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.2205921411514282,
+      "epoch": 0.05383580080753701,
+      "grad_norm": 1.2432096004486084,
+      "learning_rate": 1.0483870967741936e-05,
+      "loss": 0.2568032264709473,
+      "mean_token_accuracy": 0.9130991995334625,
+      "num_tokens": 755026.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.221834623813629,
+      "epoch": 0.06729475100942127,
+      "grad_norm": 0.8531327843666077,
+      "learning_rate": 1.3172043010752688e-05,
+      "loss": 0.20193097591400147,
+      "mean_token_accuracy": 0.9274256646633148,
+      "num_tokens": 943494.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.2245031952857972,
+      "epoch": 0.08075370121130551,
+      "grad_norm": 0.5026484131813049,
+      "learning_rate": 1.586021505376344e-05,
+      "loss": 0.171803081035614,
+      "mean_token_accuracy": 0.9363821744918823,
+      "num_tokens": 1131731.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.2192703008651733,
+      "epoch": 0.09421265141318977,
+      "grad_norm": 0.5228630304336548,
+      "learning_rate": 1.8548387096774193e-05,
+      "loss": 0.15698516368865967,
+      "mean_token_accuracy": 0.9423282980918884,
+      "num_tokens": 1320258.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.2153660774230957,
+      "epoch": 0.10767160161507403,
+      "grad_norm": 0.3650094270706177,
+      "learning_rate": 2.1236559139784946e-05,
+      "loss": 0.14900912046432496,
+      "mean_token_accuracy": 0.9437524616718292,
+      "num_tokens": 1509209.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.2123230814933776,
+      "epoch": 0.12113055181695828,
+      "grad_norm": 0.45723631978034973,
+      "learning_rate": 2.39247311827957e-05,
+      "loss": 0.1399540901184082,
+      "mean_token_accuracy": 0.9474983811378479,
+      "num_tokens": 1698139.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.208789598941803,
+      "epoch": 0.13458950201884254,
+      "grad_norm": 0.4575304687023163,
+      "learning_rate": 2.661290322580645e-05,
+      "loss": 0.12566736936569214,
+      "mean_token_accuracy": 0.9529488801956176,
+      "num_tokens": 1886384.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.205567193031311,
+      "epoch": 0.1480484522207268,
+      "grad_norm": 0.8033406734466553,
+      "learning_rate": 2.9301075268817207e-05,
+      "loss": 0.12005312442779541,
+      "mean_token_accuracy": 0.9533569395542145,
+      "num_tokens": 2074919.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.20180287361145,
+      "epoch": 0.16150740242261102,
+      "grad_norm": 0.6989286541938782,
+      "learning_rate": 3.198924731182796e-05,
+      "loss": 0.11197478771209717,
+      "mean_token_accuracy": 0.9572584748268127,
+      "num_tokens": 2263116.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.202919363975525,
+      "epoch": 0.17496635262449528,
+      "grad_norm": 0.7233495116233826,
+      "learning_rate": 3.467741935483872e-05,
+      "loss": 0.11106340885162354,
+      "mean_token_accuracy": 0.9569663584232331,
+      "num_tokens": 2451273.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.1907272219657898,
+      "epoch": 0.18842530282637954,
+      "grad_norm": 0.7645091414451599,
+      "learning_rate": 3.736559139784947e-05,
+      "loss": 0.10956189632415772,
+      "mean_token_accuracy": 0.95842245221138,
+      "num_tokens": 2640657.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.1895484924316406,
+      "epoch": 0.2018842530282638,
+      "grad_norm": 0.841379702091217,
+      "learning_rate": 4.005376344086022e-05,
+      "loss": 0.10442907810211181,
+      "mean_token_accuracy": 0.9603263795375824,
+      "num_tokens": 2829351.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.1871973633766175,
+      "epoch": 0.21534320323014805,
+      "grad_norm": 0.8552286624908447,
+      "learning_rate": 4.2741935483870973e-05,
+      "loss": 0.10493810176849365,
+      "mean_token_accuracy": 0.9597454965114594,
+      "num_tokens": 3018290.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.1907334923744202,
+      "epoch": 0.2288021534320323,
+      "grad_norm": 0.7243770360946655,
+      "learning_rate": 4.543010752688172e-05,
+      "loss": 0.10288643836975098,
+      "mean_token_accuracy": 0.9603340923786163,
+      "num_tokens": 3206538.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.1835299730300903,
+      "epoch": 0.24226110363391656,
+      "grad_norm": 0.8118091225624084,
+      "learning_rate": 4.811827956989248e-05,
+      "loss": 0.09798368811607361,
+      "mean_token_accuracy": 0.9619723737239838,
+      "num_tokens": 3395374.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1794602513313293,
+      "epoch": 0.2557200538358008,
+      "grad_norm": 0.7447699904441833,
+      "learning_rate": 5.080645161290323e-05,
+      "loss": 0.09498158693313599,
+      "mean_token_accuracy": 0.9634931206703186,
+      "num_tokens": 3584273.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.1834724426269532,
+      "epoch": 0.2691790040376851,
+      "grad_norm": 1.4059171676635742,
+      "learning_rate": 5.349462365591398e-05,
+      "loss": 0.09400172233581543,
+      "mean_token_accuracy": 0.9630160868167877,
+      "num_tokens": 3772857.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.186821174621582,
+      "epoch": 0.28263795423956933,
+      "grad_norm": 0.8273025751113892,
+      "learning_rate": 5.618279569892473e-05,
+      "loss": 0.09464811086654663,
+      "mean_token_accuracy": 0.9631786167621612,
+      "num_tokens": 3960877.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.1769920349121095,
+      "epoch": 0.2960969044414536,
+      "grad_norm": 1.0182609558105469,
+      "learning_rate": 5.887096774193549e-05,
+      "loss": 0.0914128303527832,
+      "mean_token_accuracy": 0.9644896507263183,
+      "num_tokens": 4149444.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.1719188809394836,
+      "epoch": 0.30955585464333785,
+      "grad_norm": 0.9648745656013489,
+      "learning_rate": 6.155913978494624e-05,
+      "loss": 0.0920255422592163,
+      "mean_token_accuracy": 0.963964831829071,
+      "num_tokens": 4337989.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.1559369206428527,
+      "epoch": 0.32301480484522205,
+      "grad_norm": 0.853049635887146,
+      "learning_rate": 6.4247311827957e-05,
+      "loss": 0.0891042947769165,
+      "mean_token_accuracy": 0.9653984010219574,
+      "num_tokens": 4527256.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.1604587316513062,
+      "epoch": 0.3364737550471063,
+      "grad_norm": 0.873772144317627,
+      "learning_rate": 6.693548387096774e-05,
+      "loss": 0.09071275591850281,
+      "mean_token_accuracy": 0.9651397407054901,
+      "num_tokens": 4715878.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.1534931778907775,
+      "epoch": 0.34993270524899056,
+      "grad_norm": 1.087850570678711,
+      "learning_rate": 6.962365591397851e-05,
+      "loss": 0.08719289302825928,
+      "mean_token_accuracy": 0.9658753871917725,
+      "num_tokens": 4904527.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.1629684448242188,
+      "epoch": 0.3633916554508748,
+      "grad_norm": 0.9014195799827576,
+      "learning_rate": 7.231182795698926e-05,
+      "loss": 0.08716133236885071,
+      "mean_token_accuracy": 0.965560519695282,
+      "num_tokens": 5093554.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.148916518688202,
+      "epoch": 0.3768506056527591,
+      "grad_norm": 0.9712215662002563,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.08668915033340455,
+      "mean_token_accuracy": 0.9658140063285827,
+      "num_tokens": 5282224.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1493291974067688,
+      "epoch": 0.39030955585464333,
+      "grad_norm": 0.9085242748260498,
+      "learning_rate": 7.768817204301076e-05,
+      "loss": 0.08189771175384522,
+      "mean_token_accuracy": 0.9670268416404724,
+      "num_tokens": 5471308.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.1406704187393188,
+      "epoch": 0.4037685060565276,
+      "grad_norm": 1.128177285194397,
+      "learning_rate": 8.037634408602151e-05,
+      "loss": 0.08119879961013794,
+      "mean_token_accuracy": 0.9675639867782593,
+      "num_tokens": 5660026.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.1369357347488402,
+      "epoch": 0.41722745625841184,
+      "grad_norm": 0.8224227428436279,
+      "learning_rate": 8.306451612903227e-05,
+      "loss": 0.07979745864868164,
+      "mean_token_accuracy": 0.9681445300579071,
+      "num_tokens": 5848073.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.1252583503723144,
+      "epoch": 0.4306864064602961,
+      "grad_norm": 0.7711160182952881,
+      "learning_rate": 8.575268817204302e-05,
+      "loss": 0.0783164381980896,
+      "mean_token_accuracy": 0.9686201930046081,
+      "num_tokens": 6036717.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.1203404307365417,
+      "epoch": 0.44414535666218036,
+      "grad_norm": 1.1810287237167358,
+      "learning_rate": 8.844086021505377e-05,
+      "loss": 0.08139073848724365,
+      "mean_token_accuracy": 0.9681365489959717,
+      "num_tokens": 6225223.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.1222687840461731,
+      "epoch": 0.4576043068640646,
+      "grad_norm": 1.4551713466644287,
+      "learning_rate": 9.112903225806452e-05,
+      "loss": 0.0820135235786438,
+      "mean_token_accuracy": 0.9672703862190246,
+      "num_tokens": 6413936.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.1156280279159545,
+      "epoch": 0.47106325706594887,
+      "grad_norm": 0.8573716878890991,
+      "learning_rate": 9.381720430107528e-05,
+      "loss": 0.08074904680252075,
+      "mean_token_accuracy": 0.9677663922309876,
+      "num_tokens": 6602413.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.1063185572624206,
+      "epoch": 0.4845222072678331,
+      "grad_norm": 0.7709434628486633,
+      "learning_rate": 9.650537634408603e-05,
+      "loss": 0.07549421787261963,
+      "mean_token_accuracy": 0.9689356207847595,
+      "num_tokens": 6790873.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.0962031960487366,
+      "epoch": 0.4979811574697174,
+      "grad_norm": 0.7843625545501709,
+      "learning_rate": 9.919354838709678e-05,
+      "loss": 0.07368478775024415,
+      "mean_token_accuracy": 0.9700294613838196,
+      "num_tokens": 6979392.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.1028530240058898,
+      "epoch": 0.5114401076716016,
+      "grad_norm": 0.7935485243797302,
+      "learning_rate": 9.999975729865971e-05,
+      "loss": 0.07766538262367248,
+      "mean_token_accuracy": 0.9704003691673279,
+      "num_tokens": 7167482.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.0905393958091736,
+      "epoch": 0.5248990578734859,
+      "grad_norm": 1.4108924865722656,
+      "learning_rate": 9.999856856307314e-05,
+      "loss": 0.0760004162788391,
+      "mean_token_accuracy": 0.9699020266532898,
+      "num_tokens": 7355994.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.0853531122207642,
+      "epoch": 0.5383580080753702,
+      "grad_norm": 0.9043511152267456,
+      "learning_rate": 9.999638923896533e-05,
+      "loss": 0.0720310389995575,
+      "mean_token_accuracy": 0.9707890212535858,
+      "num_tokens": 7544655.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.075773572921753,
+      "epoch": 0.5518169582772544,
+      "grad_norm": 0.9038823246955872,
+      "learning_rate": 9.999321936951374e-05,
+      "loss": 0.07026209831237792,
+      "mean_token_accuracy": 0.9715417385101318,
+      "num_tokens": 7733348.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.0572428584098816,
+      "epoch": 0.5652759084791387,
+      "grad_norm": 0.8231707811355591,
+      "learning_rate": 9.998905901752091e-05,
+      "loss": 0.07141299843788147,
+      "mean_token_accuracy": 0.9705908715724945,
+      "num_tokens": 7921582.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.051485300064087,
+      "epoch": 0.5787348586810229,
+      "grad_norm": 1.1695454120635986,
+      "learning_rate": 9.998390826541315e-05,
+      "loss": 0.07321611642837525,
+      "mean_token_accuracy": 0.9709623396396637,
+      "num_tokens": 8110266.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.0485445380210876,
+      "epoch": 0.5921938088829072,
+      "grad_norm": 0.7291010618209839,
+      "learning_rate": 9.997776721523888e-05,
+      "loss": 0.07221676707267762,
+      "mean_token_accuracy": 0.9705081820487976,
+      "num_tokens": 8298932.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.0456495046615601,
+      "epoch": 0.6056527590847914,
+      "grad_norm": 0.6550094485282898,
+      "learning_rate": 9.99706359886667e-05,
+      "loss": 0.06878133416175843,
+      "mean_token_accuracy": 0.9728739261627197,
+      "num_tokens": 8487613.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.0372861266136169,
+      "epoch": 0.6191117092866757,
+      "grad_norm": 0.5825450420379639,
+      "learning_rate": 9.996251472698281e-05,
+      "loss": 0.06706151366233826,
+      "mean_token_accuracy": 0.9732721030712128,
+      "num_tokens": 8676276.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.0319493532180786,
+      "epoch": 0.6325706594885598,
+      "grad_norm": 0.7158534526824951,
+      "learning_rate": 9.995340359108844e-05,
+      "loss": 0.06999597549438477,
+      "mean_token_accuracy": 0.9726741492748261,
+      "num_tokens": 8864738.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.0260030388832093,
+      "epoch": 0.6460296096904441,
+      "grad_norm": 0.7918373346328735,
+      "learning_rate": 9.994330276149649e-05,
+      "loss": 0.06949877142906188,
+      "mean_token_accuracy": 0.9725882947444916,
+      "num_tokens": 9053582.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.0258127093315124,
+      "epoch": 0.6594885598923284,
+      "grad_norm": 0.893281102180481,
+      "learning_rate": 9.993221243832797e-05,
+      "loss": 0.06893026828765869,
+      "mean_token_accuracy": 0.9729171216487884,
+      "num_tokens": 9241914.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.0261828184127808,
+      "epoch": 0.6729475100942126,
+      "grad_norm": 1.0127108097076416,
+      "learning_rate": 9.992013284130816e-05,
+      "loss": 0.07094801664352417,
+      "mean_token_accuracy": 0.9714575052261353,
+      "num_tokens": 9430981.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.0274562597274781,
+      "epoch": 0.6864064602960969,
+      "grad_norm": 0.7148029208183289,
+      "learning_rate": 9.990706420976206e-05,
+      "loss": 0.06826171875,
+      "mean_token_accuracy": 0.9727248430252076,
+      "num_tokens": 9619472.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.0137859225273131,
+      "epoch": 0.6998654104979811,
+      "grad_norm": 0.9228634238243103,
+      "learning_rate": 9.989300680260985e-05,
+      "loss": 0.06890587210655212,
+      "mean_token_accuracy": 0.9723304331302642,
+      "num_tokens": 9808123.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.0221150755882262,
+      "epoch": 0.7133243606998654,
+      "grad_norm": 0.971530556678772,
+      "learning_rate": 9.98779608983616e-05,
+      "loss": 0.07073599100112915,
+      "mean_token_accuracy": 0.9714174270629883,
+      "num_tokens": 9996601.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.0248154640197753,
+      "epoch": 0.7267833109017496,
+      "grad_norm": 0.7336317896842957,
+      "learning_rate": 9.986192679511189e-05,
+      "loss": 0.06874136924743653,
+      "mean_token_accuracy": 0.9723432004451752,
+      "num_tokens": 10184725.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.0120978832244873,
+      "epoch": 0.7402422611036339,
+      "grad_norm": 1.2100600004196167,
+      "learning_rate": 9.984490481053372e-05,
+      "loss": 0.06270487308502197,
+      "mean_token_accuracy": 0.9756880521774292,
+      "num_tokens": 10373446.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.0172406077384948,
+      "epoch": 0.7537012113055181,
+      "grad_norm": 0.7562853693962097,
+      "learning_rate": 9.982689528187244e-05,
+      "loss": 0.06784560084342957,
+      "mean_token_accuracy": 0.9724333345890045,
+      "num_tokens": 10561710.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.0150038957595826,
+      "epoch": 0.7671601615074024,
+      "grad_norm": 1.07966148853302,
+      "learning_rate": 9.98078985659389e-05,
+      "loss": 0.0670344054698944,
+      "mean_token_accuracy": 0.97339146733284,
+      "num_tokens": 10749917.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.0135640978813172,
+      "epoch": 0.7806191117092867,
+      "grad_norm": 0.7826104164123535,
+      "learning_rate": 9.978791503910246e-05,
+      "loss": 0.0668565571308136,
+      "mean_token_accuracy": 0.9735289216041565,
+      "num_tokens": 10938098.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.0077669620513916,
+      "epoch": 0.7940780619111709,
+      "grad_norm": 1.299584150314331,
+      "learning_rate": 9.97669450972835e-05,
+      "loss": 0.0728976845741272,
+      "mean_token_accuracy": 0.9701269209384918,
+      "num_tokens": 11126107.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.9957692861557007,
+      "epoch": 0.8075370121130552,
+      "grad_norm": 1.1542484760284424,
+      "learning_rate": 9.974498915594557e-05,
+      "loss": 0.0631720781326294,
+      "mean_token_accuracy": 0.9747781097888947,
+      "num_tokens": 11315001.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.9983992159366608,
+      "epoch": 0.8209959623149394,
+      "grad_norm": 0.6967246532440186,
+      "learning_rate": 9.97220476500872e-05,
+      "loss": 0.06333768963813782,
+      "mean_token_accuracy": 0.9750322341918946,
+      "num_tokens": 11503007.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.9801111698150635,
+      "epoch": 0.8344549125168237,
+      "grad_norm": 0.945446789264679,
+      "learning_rate": 9.969812103423325e-05,
+      "loss": 0.05720087289810181,
+      "mean_token_accuracy": 0.9767442524433136,
+      "num_tokens": 11691883.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.9879570543766022,
+      "epoch": 0.847913862718708,
+      "grad_norm": 0.8254193663597107,
+      "learning_rate": 9.967320978242592e-05,
+      "loss": 0.05899171829223633,
+      "mean_token_accuracy": 0.9767756819725036,
+      "num_tokens": 11880353.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.9878240942955017,
+      "epoch": 0.8613728129205922,
+      "grad_norm": 0.9882538914680481,
+      "learning_rate": 9.964731438821533e-05,
+      "loss": 0.06443996429443359,
+      "mean_token_accuracy": 0.9735406100749969,
+      "num_tokens": 12069610.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.9938213169574738,
+      "epoch": 0.8748317631224765,
+      "grad_norm": 1.126546859741211,
+      "learning_rate": 9.962043536464978e-05,
+      "loss": 0.06708416938781739,
+      "mean_token_accuracy": 0.9730359137058258,
+      "num_tokens": 12257901.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.9800443708896637,
+      "epoch": 0.8882907133243607,
+      "grad_norm": 0.9151445031166077,
+      "learning_rate": 9.959257324426556e-05,
+      "loss": 0.06290764808654785,
+      "mean_token_accuracy": 0.9741653084754944,
+      "num_tokens": 12446236.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.9744794130325317,
+      "epoch": 0.901749663526245,
+      "grad_norm": 0.9104028940200806,
+      "learning_rate": 9.95637285790764e-05,
+      "loss": 0.060464882850646974,
+      "mean_token_accuracy": 0.9757490694522858,
+      "num_tokens": 12635226.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.9786687552928924,
+      "epoch": 0.9152086137281292,
+      "grad_norm": 0.7316192388534546,
+      "learning_rate": 9.953390194056258e-05,
+      "loss": 0.06054847836494446,
+      "mean_token_accuracy": 0.9760404825210571,
+      "num_tokens": 12823428.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.966060334444046,
+      "epoch": 0.9286675639300135,
+      "grad_norm": 1.3535135984420776,
+      "learning_rate": 9.950309391965947e-05,
+      "loss": 0.061383575201034546,
+      "mean_token_accuracy": 0.9749173820018768,
+      "num_tokens": 13012362.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.9722849190235138,
+      "epoch": 0.9421265141318977,
+      "grad_norm": 0.890771746635437,
+      "learning_rate": 9.947130512674602e-05,
+      "loss": 0.06301190257072449,
+      "mean_token_accuracy": 0.9750476598739624,
+      "num_tokens": 13200680.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.9624020338058472,
+      "epoch": 0.955585464333782,
+      "grad_norm": 1.142499327659607,
+      "learning_rate": 9.943853619163255e-05,
+      "loss": 0.06179196834564209,
+      "mean_token_accuracy": 0.9751901209354401,
+      "num_tokens": 13389333.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.9653747022151947,
+      "epoch": 0.9690444145356663,
+      "grad_norm": 0.8682493567466736,
+      "learning_rate": 9.94047877635482e-05,
+      "loss": 0.06113170981407166,
+      "mean_token_accuracy": 0.9748325288295746,
+      "num_tokens": 13578210.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.970348197221756,
+      "epoch": 0.9825033647375505,
+      "grad_norm": 0.8351430296897888,
+      "learning_rate": 9.93700605111283e-05,
+      "loss": 0.05901788473129273,
+      "mean_token_accuracy": 0.9761226296424865,
+      "num_tokens": 13767039.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.956724613904953,
+      "epoch": 0.9959623149394348,
+      "grad_norm": 1.2604819536209106,
+      "learning_rate": 9.933435512240084e-05,
+      "loss": 0.06124393343925476,
+      "mean_token_accuracy": 0.9747833967208862,
+      "num_tokens": 13956057.0,
+      "step": 740
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 0.9722227849018802,
+      "eval_loss": 0.0600070059299469,
+      "eval_mean_token_accuracy": 0.975752676368519,
+      "eval_num_tokens": 14012630.0,
+      "eval_runtime": 24.6215,
+      "eval_samples_per_second": 203.074,
+      "eval_steps_per_second": 6.377,
+      "step": 743
+    },
+    {
+      "entropy": 0.9689933776855468,
+      "epoch": 1.009421265141319,
+      "grad_norm": 0.9720287919044495,
+      "learning_rate": 9.929767230477305e-05,
+      "loss": 0.055563896894454956,
+      "mean_token_accuracy": 0.9779708027839661,
+      "num_tokens": 14144652.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.9506655633449554,
+      "epoch": 1.0228802153432033,
+      "grad_norm": 0.6619595289230347,
+      "learning_rate": 9.92600127850173e-05,
+      "loss": 0.050587379932403566,
+      "mean_token_accuracy": 0.979685264825821,
+      "num_tokens": 14332714.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.9458732306957245,
+      "epoch": 1.0363391655450875,
+      "grad_norm": 0.7905108332633972,
+      "learning_rate": 9.922137730925673e-05,
+      "loss": 0.05069155097007751,
+      "mean_token_accuracy": 0.9796596229076385,
+      "num_tokens": 14520902.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.9320916712284089,
+      "epoch": 1.0497981157469718,
+      "grad_norm": 0.9210566878318787,
+      "learning_rate": 9.918176664295041e-05,
+      "loss": 0.051744121313095096,
+      "mean_token_accuracy": 0.9792148530483246,
+      "num_tokens": 14709215.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.928934782743454,
+      "epoch": 1.063257065948856,
+      "grad_norm": 0.7442036867141724,
+      "learning_rate": 9.914118157087824e-05,
+      "loss": 0.0486875057220459,
+      "mean_token_accuracy": 0.9798122465610504,
+      "num_tokens": 14898242.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.942881977558136,
+      "epoch": 1.0767160161507403,
+      "grad_norm": 1.0910941362380981,
+      "learning_rate": 9.909962289712538e-05,
+      "loss": 0.052297019958496095,
+      "mean_token_accuracy": 0.9793200254440307,
+      "num_tokens": 15086835.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.9503917813301086,
+      "epoch": 1.0901749663526246,
+      "grad_norm": 0.8091554641723633,
+      "learning_rate": 9.905709144506629e-05,
+      "loss": 0.049927744269371035,
+      "mean_token_accuracy": 0.9802082359790802,
+      "num_tokens": 15275042.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.9426830470561981,
+      "epoch": 1.1036339165545088,
+      "grad_norm": 1.0589245557785034,
+      "learning_rate": 9.901358805734846e-05,
+      "loss": 0.053803551197052005,
+      "mean_token_accuracy": 0.9785399675369263,
+      "num_tokens": 15463923.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.9445010781288147,
+      "epoch": 1.117092866756393,
+      "grad_norm": 0.8410441279411316,
+      "learning_rate": 9.89691135958757e-05,
+      "loss": 0.053181976079940796,
+      "mean_token_accuracy": 0.9792523324489594,
+      "num_tokens": 15652526.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.946729838848114,
+      "epoch": 1.1305518169582773,
+      "grad_norm": 0.8246080279350281,
+      "learning_rate": 9.892366894179105e-05,
+      "loss": 0.054291915893554685,
+      "mean_token_accuracy": 0.9791357696056366,
+      "num_tokens": 15841340.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.9433728814125061,
+      "epoch": 1.1440107671601616,
+      "grad_norm": 0.9200296401977539,
+      "learning_rate": 9.887725499545937e-05,
+      "loss": 0.05073915719985962,
+      "mean_token_accuracy": 0.9796931505203247,
+      "num_tokens": 16029478.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.9494417488574982,
+      "epoch": 1.1574697173620458,
+      "grad_norm": 0.8342758417129517,
+      "learning_rate": 9.882987267644939e-05,
+      "loss": 0.050929927825927736,
+      "mean_token_accuracy": 0.9793219089508056,
+      "num_tokens": 16218014.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.9409256100654602,
+      "epoch": 1.17092866756393,
+      "grad_norm": 0.7911009192466736,
+      "learning_rate": 9.878152292351563e-05,
+      "loss": 0.049964362382888795,
+      "mean_token_accuracy": 0.9798487305641175,
+      "num_tokens": 16406420.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.9429958581924438,
+      "epoch": 1.1843876177658144,
+      "grad_norm": 1.4877018928527832,
+      "learning_rate": 9.873220669457975e-05,
+      "loss": 0.04969423711299896,
+      "mean_token_accuracy": 0.9801322638988494,
+      "num_tokens": 16594948.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.9542136013507843,
+      "epoch": 1.1978465679676986,
+      "grad_norm": 0.8030785918235779,
+      "learning_rate": 9.868192496671147e-05,
+      "loss": 0.04981146454811096,
+      "mean_token_accuracy": 0.9796162784099579,
+      "num_tokens": 16782983.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.9367722630500793,
+      "epoch": 1.2113055181695827,
+      "grad_norm": 1.1169800758361816,
+      "learning_rate": 9.86306787361094e-05,
+      "loss": 0.05210963487625122,
+      "mean_token_accuracy": 0.9790139615535736,
+      "num_tokens": 16971394.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.9487208306789399,
+      "epoch": 1.224764468371467,
+      "grad_norm": 0.8650282621383667,
+      "learning_rate": 9.857846901808117e-05,
+      "loss": 0.05012243390083313,
+      "mean_token_accuracy": 0.9805107891559601,
+      "num_tokens": 17159951.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.9441231489181519,
+      "epoch": 1.2382234185733512,
+      "grad_norm": 0.8193967938423157,
+      "learning_rate": 9.852529684702329e-05,
+      "loss": 0.048866665363311766,
+      "mean_token_accuracy": 0.9807584881782532,
+      "num_tokens": 17348711.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.9361630439758301,
+      "epoch": 1.2516823687752354,
+      "grad_norm": 1.3613612651824951,
+      "learning_rate": 9.847116327640082e-05,
+      "loss": 0.05324091911315918,
+      "mean_token_accuracy": 0.97904953956604,
+      "num_tokens": 17537116.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.9562997639179229,
+      "epoch": 1.2651413189771197,
+      "grad_norm": 0.8121878504753113,
+      "learning_rate": 9.841606937872632e-05,
+      "loss": 0.05011019706726074,
+      "mean_token_accuracy": 0.9798731088638306,
+      "num_tokens": 17725567.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.9359076261520386,
+      "epoch": 1.278600269179004,
+      "grad_norm": 0.7838549613952637,
+      "learning_rate": 9.836001624553869e-05,
+      "loss": 0.04834386110305786,
+      "mean_token_accuracy": 0.9804529249668121,
+      "num_tokens": 17914191.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.9527446150779724,
+      "epoch": 1.2920592193808882,
+      "grad_norm": 1.0981712341308594,
+      "learning_rate": 9.830300498738152e-05,
+      "loss": 0.055077624320983884,
+      "mean_token_accuracy": 0.9776535391807556,
+      "num_tokens": 18103184.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.9574812948703766,
+      "epoch": 1.3055181695827724,
+      "grad_norm": 0.8291501402854919,
+      "learning_rate": 9.824503673378112e-05,
+      "loss": 0.050153911113739014,
+      "mean_token_accuracy": 0.9792965054512024,
+      "num_tokens": 18291351.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.939585280418396,
+      "epoch": 1.3189771197846567,
+      "grad_norm": 1.0864412784576416,
+      "learning_rate": 9.81861126332241e-05,
+      "loss": 0.05132197737693787,
+      "mean_token_accuracy": 0.9792291462421417,
+      "num_tokens": 18480192.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.9468406856060028,
+      "epoch": 1.332436069986541,
+      "grad_norm": 0.9772785902023315,
+      "learning_rate": 9.812623385313461e-05,
+      "loss": 0.04844954013824463,
+      "mean_token_accuracy": 0.9804142415523529,
+      "num_tokens": 18669028.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.9413834273815155,
+      "epoch": 1.3458950201884252,
+      "grad_norm": 0.9020094871520996,
+      "learning_rate": 9.806540157985131e-05,
+      "loss": 0.05075312852859497,
+      "mean_token_accuracy": 0.9790591120719909,
+      "num_tokens": 18857581.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.9526253461837768,
+      "epoch": 1.3593539703903095,
+      "grad_norm": 1.0524990558624268,
+      "learning_rate": 9.800361701860368e-05,
+      "loss": 0.049737372994422914,
+      "mean_token_accuracy": 0.9795652389526367,
+      "num_tokens": 19046105.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.9484909176826477,
+      "epoch": 1.3728129205921937,
+      "grad_norm": 1.0715620517730713,
+      "learning_rate": 9.794088139348835e-05,
+      "loss": 0.04935494959354401,
+      "mean_token_accuracy": 0.9797740161418915,
+      "num_tokens": 19234730.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.9398573458194732,
+      "epoch": 1.386271870794078,
+      "grad_norm": 1.2134987115859985,
+      "learning_rate": 9.787719594744468e-05,
+      "loss": 0.0518394410610199,
+      "mean_token_accuracy": 0.9793788075447083,
+      "num_tokens": 19423572.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.9465648651123046,
+      "epoch": 1.3997308209959622,
+      "grad_norm": 1.004693627357483,
+      "learning_rate": 9.781256194223023e-05,
+      "loss": 0.04776117205619812,
+      "mean_token_accuracy": 0.9813261866569519,
+      "num_tokens": 19612465.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.9448004186153411,
+      "epoch": 1.4131897711978465,
+      "grad_norm": 1.2913438081741333,
+      "learning_rate": 9.774698065839577e-05,
+      "loss": 0.04807930588722229,
+      "mean_token_accuracy": 0.9805689513683319,
+      "num_tokens": 19800482.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.9425322234630584,
+      "epoch": 1.4266487213997308,
+      "grad_norm": 1.1068469285964966,
+      "learning_rate": 9.768045339525979e-05,
+      "loss": 0.05053595900535583,
+      "mean_token_accuracy": 0.9794536828994751,
+      "num_tokens": 19989444.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.9472232103347779,
+      "epoch": 1.440107671601615,
+      "grad_norm": 0.7409384250640869,
+      "learning_rate": 9.76129814708829e-05,
+      "loss": 0.052136778831481934,
+      "mean_token_accuracy": 0.979197359085083,
+      "num_tokens": 20178482.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.9450975477695465,
+      "epoch": 1.4535666218034993,
+      "grad_norm": 1.3440778255462646,
+      "learning_rate": 9.754456622204167e-05,
+      "loss": 0.05258495211601257,
+      "mean_token_accuracy": 0.9798152863979339,
+      "num_tokens": 20367345.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.9468007743358612,
+      "epoch": 1.4670255720053835,
+      "grad_norm": 0.8377829194068909,
+      "learning_rate": 9.747520900420209e-05,
+      "loss": 0.04893603026866913,
+      "mean_token_accuracy": 0.9805733859539032,
+      "num_tokens": 20555817.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.9505827724933624,
+      "epoch": 1.4804845222072678,
+      "grad_norm": 0.776889979839325,
+      "learning_rate": 9.740491119149277e-05,
+      "loss": 0.05367119312286377,
+      "mean_token_accuracy": 0.9779646098613739,
+      "num_tokens": 20744001.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.9516380190849304,
+      "epoch": 1.493943472409152,
+      "grad_norm": 0.8598276376724243,
+      "learning_rate": 9.733367417667773e-05,
+      "loss": 0.05071394443511963,
+      "mean_token_accuracy": 0.979692417383194,
+      "num_tokens": 20931715.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.9351628720760345,
+      "epoch": 1.5074024226110363,
+      "grad_norm": 1.3307439088821411,
+      "learning_rate": 9.726149937112873e-05,
+      "loss": 0.051372635364532473,
+      "mean_token_accuracy": 0.9797907948493958,
+      "num_tokens": 21120803.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.9268040716648102,
+      "epoch": 1.5208613728129206,
+      "grad_norm": 0.9320250749588013,
+      "learning_rate": 9.718838820479743e-05,
+      "loss": 0.04939974546432495,
+      "mean_token_accuracy": 0.98038170337677,
+      "num_tokens": 21309384.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.9241463243961334,
+      "epoch": 1.5343203230148048,
+      "grad_norm": 1.065938115119934,
+      "learning_rate": 9.711434212618691e-05,
+      "loss": 0.04856643378734589,
+      "mean_token_accuracy": 0.9800511121749877,
+      "num_tokens": 21498234.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.9350252687931061,
+      "epoch": 1.547779273216689,
+      "grad_norm": 0.6925032138824463,
+      "learning_rate": 9.703936260232308e-05,
+      "loss": 0.0512526273727417,
+      "mean_token_accuracy": 0.979562646150589,
+      "num_tokens": 21687119.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.9403733670711517,
+      "epoch": 1.5612382234185733,
+      "grad_norm": 1.2069101333618164,
+      "learning_rate": 9.696345111872557e-05,
+      "loss": 0.049795085191726686,
+      "mean_token_accuracy": 0.9806506633758545,
+      "num_tokens": 21876094.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.9370538294315338,
+      "epoch": 1.5746971736204576,
+      "grad_norm": 1.1759546995162964,
+      "learning_rate": 9.688660917937838e-05,
+      "loss": 0.05461466312408447,
+      "mean_token_accuracy": 0.9776057600975037,
+      "num_tokens": 22064836.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.9517778217792511,
+      "epoch": 1.5881561238223418,
+      "grad_norm": 0.8965149521827698,
+      "learning_rate": 9.68088383066999e-05,
+      "loss": 0.05289326906204224,
+      "mean_token_accuracy": 0.9782088756561279,
+      "num_tokens": 22253065.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.9372851848602295,
+      "epoch": 1.601615074024226,
+      "grad_norm": 0.7192108631134033,
+      "learning_rate": 9.673014004151292e-05,
+      "loss": 0.048973691463470456,
+      "mean_token_accuracy": 0.9806210815906524,
+      "num_tokens": 22441206.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.9122351944446564,
+      "epoch": 1.6150740242261103,
+      "grad_norm": 0.8763614296913147,
+      "learning_rate": 9.665051594301407e-05,
+      "loss": 0.0461783230304718,
+      "mean_token_accuracy": 0.9817197680473327,
+      "num_tokens": 22629637.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.9162654876708984,
+      "epoch": 1.6285329744279946,
+      "grad_norm": 0.9806778430938721,
+      "learning_rate": 9.656996758874284e-05,
+      "loss": 0.04827338755130768,
+      "mean_token_accuracy": 0.9806499361991883,
+      "num_tokens": 22818369.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.9125026524066925,
+      "epoch": 1.6419919246298789,
+      "grad_norm": 0.9685350656509399,
+      "learning_rate": 9.648849657455044e-05,
+      "loss": 0.048379385471343996,
+      "mean_token_accuracy": 0.980905145406723,
+      "num_tokens": 23006863.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.9119791567325592,
+      "epoch": 1.6554508748317631,
+      "grad_norm": 0.8717457056045532,
+      "learning_rate": 9.640610451456811e-05,
+      "loss": 0.050873959064483644,
+      "mean_token_accuracy": 0.979601925611496,
+      "num_tokens": 23195735.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.9244007229804992,
+      "epoch": 1.6689098250336474,
+      "grad_norm": 1.0574654340744019,
+      "learning_rate": 9.632279304117517e-05,
+      "loss": 0.05112813711166382,
+      "mean_token_accuracy": 0.979493772983551,
+      "num_tokens": 23383862.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.9225810945034028,
+      "epoch": 1.6823687752355316,
+      "grad_norm": 0.9628807306289673,
+      "learning_rate": 9.623856380496664e-05,
+      "loss": 0.050480544567108154,
+      "mean_token_accuracy": 0.9797678411006927,
+      "num_tokens": 23572292.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.9287579476833343,
+      "epoch": 1.695827725437416,
+      "grad_norm": 1.015265941619873,
+      "learning_rate": 9.615341847472059e-05,
+      "loss": 0.05133126378059387,
+      "mean_token_accuracy": 0.9797986805438995,
+      "num_tokens": 23760252.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.9133845925331116,
+      "epoch": 1.7092866756393001,
+      "grad_norm": 0.8996883630752563,
+      "learning_rate": 9.606735873736505e-05,
+      "loss": 0.05044950246810913,
+      "mean_token_accuracy": 0.9795303404331207,
+      "num_tokens": 23949031.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.9213698863983154,
+      "epoch": 1.7227456258411844,
+      "grad_norm": 1.0490264892578125,
+      "learning_rate": 9.598038629794461e-05,
+      "loss": 0.04797698855400086,
+      "mean_token_accuracy": 0.9806023716926575,
+      "num_tokens": 24137829.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.9173697710037232,
+      "epoch": 1.7362045760430687,
+      "grad_norm": 1.3648769855499268,
+      "learning_rate": 9.589250287958657e-05,
+      "loss": 0.049194514751434326,
+      "mean_token_accuracy": 0.980712479352951,
+      "num_tokens": 24326586.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.9242741882801055,
+      "epoch": 1.749663526244953,
+      "grad_norm": 1.105125069618225,
+      "learning_rate": 9.580371022346693e-05,
+      "loss": 0.048667973279953,
+      "mean_token_accuracy": 0.9801449298858642,
+      "num_tokens": 24514904.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.9162627995014191,
+      "epoch": 1.7631224764468372,
+      "grad_norm": 0.8887938857078552,
+      "learning_rate": 9.571401008877572e-05,
+      "loss": 0.04878672957420349,
+      "mean_token_accuracy": 0.980469423532486,
+      "num_tokens": 24703688.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.9149538338184356,
+      "epoch": 1.7765814266487214,
+      "grad_norm": 0.8838976621627808,
+      "learning_rate": 9.562340425268233e-05,
+      "loss": 0.04892318844795227,
+      "mean_token_accuracy": 0.9808776795864105,
+      "num_tokens": 24892107.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.914735221862793,
+      "epoch": 1.7900403768506057,
+      "grad_norm": 1.1152070760726929,
+      "learning_rate": 9.553189451030019e-05,
+      "loss": 0.04825109839439392,
+      "mean_token_accuracy": 0.9804604113101959,
+      "num_tokens": 25080130.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.9170725226402283,
+      "epoch": 1.80349932705249,
+      "grad_norm": 0.9479517340660095,
+      "learning_rate": 9.543948267465115e-05,
+      "loss": 0.051445144414901736,
+      "mean_token_accuracy": 0.9792819261550904,
+      "num_tokens": 25268652.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.9142911911010743,
+      "epoch": 1.8169582772543742,
+      "grad_norm": 0.8172292709350586,
+      "learning_rate": 9.534617057662977e-05,
+      "loss": 0.0475692093372345,
+      "mean_token_accuracy": 0.9809505581855774,
+      "num_tokens": 25457120.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.9019249439239502,
+      "epoch": 1.8304172274562585,
+      "grad_norm": 0.8183121681213379,
+      "learning_rate": 9.525196006496679e-05,
+      "loss": 0.04979957342147827,
+      "mean_token_accuracy": 0.9799730658531189,
+      "num_tokens": 25645699.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.9181066155433655,
+      "epoch": 1.8438761776581427,
+      "grad_norm": 0.8256579041481018,
+      "learning_rate": 9.515685300619271e-05,
+      "loss": 0.04996164441108704,
+      "mean_token_accuracy": 0.9802247405052185,
+      "num_tokens": 25834183.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.9171363770961761,
+      "epoch": 1.857335127860027,
+      "grad_norm": 0.9727098345756531,
+      "learning_rate": 9.506085128460065e-05,
+      "loss": 0.048660767078399655,
+      "mean_token_accuracy": 0.9805064260959625,
+      "num_tokens": 26023317.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.9112720847129822,
+      "epoch": 1.8707940780619112,
+      "grad_norm": 0.9826673865318298,
+      "learning_rate": 9.496395680220918e-05,
+      "loss": 0.04775593280792236,
+      "mean_token_accuracy": 0.9809929549694061,
+      "num_tokens": 26212115.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.9034272134304047,
+      "epoch": 1.8842530282637955,
+      "grad_norm": 0.9951306581497192,
+      "learning_rate": 9.486617147872446e-05,
+      "loss": 0.04939360618591308,
+      "mean_token_accuracy": 0.9800347864627839,
+      "num_tokens": 26401167.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.897865754365921,
+      "epoch": 1.8977119784656797,
+      "grad_norm": 1.0163404941558838,
+      "learning_rate": 9.476749725150235e-05,
+      "loss": 0.05002856254577637,
+      "mean_token_accuracy": 0.9793690323829651,
+      "num_tokens": 26590570.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.9004576802253723,
+      "epoch": 1.911170928667564,
+      "grad_norm": 0.6666616201400757,
+      "learning_rate": 9.466793607550995e-05,
+      "loss": 0.04905453026294708,
+      "mean_token_accuracy": 0.9803856253623963,
+      "num_tokens": 26779481.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.8978902101516724,
+      "epoch": 1.9246298788694483,
+      "grad_norm": 0.9295416474342346,
+      "learning_rate": 9.45674899232869e-05,
+      "loss": 0.05261261463165283,
+      "mean_token_accuracy": 0.9785708487033844,
+      "num_tokens": 26968262.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.8962675571441651,
+      "epoch": 1.9380888290713325,
+      "grad_norm": 0.88262939453125,
+      "learning_rate": 9.446616078490626e-05,
+      "loss": 0.04765265882015228,
+      "mean_token_accuracy": 0.98052077293396,
+      "num_tokens": 27157631.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.8909463047981262,
+      "epoch": 1.9515477792732168,
+      "grad_norm": 1.4205769300460815,
+      "learning_rate": 9.436395066793518e-05,
+      "loss": 0.049406200647354126,
+      "mean_token_accuracy": 0.979697072505951,
+      "num_tokens": 27345921.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.8969364166259766,
+      "epoch": 1.965006729475101,
+      "grad_norm": 0.9517145752906799,
+      "learning_rate": 9.426086159739496e-05,
+      "loss": 0.0510346531867981,
+      "mean_token_accuracy": 0.9793386399745941,
+      "num_tokens": 27534751.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.9092479169368743,
+      "epoch": 1.9784656796769853,
+      "grad_norm": 1.200056791305542,
+      "learning_rate": 9.415689561572107e-05,
+      "loss": 0.04974203705787659,
+      "mean_token_accuracy": 0.9800146698951722,
+      "num_tokens": 27723763.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.9059641897678375,
+      "epoch": 1.9919246298788695,
+      "grad_norm": 1.072447657585144,
+      "learning_rate": 9.405205478272267e-05,
+      "loss": 0.05065792202949524,
+      "mean_token_accuracy": 0.9799223959445953,
+      "num_tokens": 27911784.0,
+      "step": 1480
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 0.9083398975384464,
+      "eval_loss": 0.05432562157511711,
+      "eval_mean_token_accuracy": 0.9782073607869969,
+      "eval_num_tokens": 28025286.0,
+      "eval_runtime": 24.4304,
+      "eval_samples_per_second": 204.663,
+      "eval_steps_per_second": 6.426,
+      "step": 1486
+    },
+    {
+      "entropy": 0.9113237977027893,
+      "epoch": 2.005383580080754,
+      "grad_norm": 1.0167063474655151,
+      "learning_rate": 9.394634117554173e-05,
+      "loss": 0.044942992925643924,
+      "mean_token_accuracy": 0.9818171083927154,
+      "num_tokens": 28100906.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.8780008792877197,
+      "epoch": 2.018842530282638,
+      "grad_norm": 1.9552241563796997,
+      "learning_rate": 9.38397568886119e-05,
+      "loss": 0.036248764395713805,
+      "mean_token_accuracy": 0.9852245450019836,
+      "num_tokens": 28289445.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.8721176147460937,
+      "epoch": 2.0323014804845223,
+      "grad_norm": 0.8951388001441956,
+      "learning_rate": 9.373230403361712e-05,
+      "loss": 0.037987279891967776,
+      "mean_token_accuracy": 0.9850614190101623,
+      "num_tokens": 28477972.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.8922657191753387,
+      "epoch": 2.0457604306864066,
+      "grad_norm": 0.9028105735778809,
+      "learning_rate": 9.362398473944958e-05,
+      "loss": 0.03588914275169373,
+      "mean_token_accuracy": 0.9863102614879609,
+      "num_tokens": 28666272.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.877799642086029,
+      "epoch": 2.059219380888291,
+      "grad_norm": 1.153886318206787,
+      "learning_rate": 9.35148011521677e-05,
+      "loss": 0.03577309250831604,
+      "mean_token_accuracy": 0.9863148391246795,
+      "num_tokens": 28854780.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.8708971381187439,
+      "epoch": 2.072678331090175,
+      "grad_norm": 1.042955994606018,
+      "learning_rate": 9.340475543495364e-05,
+      "loss": 0.038266432285308835,
+      "mean_token_accuracy": 0.9848419308662415,
+      "num_tokens": 29043741.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.8800721943378449,
+      "epoch": 2.0861372812920593,
+      "grad_norm": 0.6789060235023499,
+      "learning_rate": 9.329384976807023e-05,
+      "loss": 0.032948991656303404,
+      "mean_token_accuracy": 0.9871293902397156,
+      "num_tokens": 29232719.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.8743580460548401,
+      "epoch": 2.0995962314939436,
+      "grad_norm": 1.1780879497528076,
+      "learning_rate": 9.318208634881802e-05,
+      "loss": 0.036874374747276305,
+      "mean_token_accuracy": 0.9859182178974152,
+      "num_tokens": 29421817.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.8851303637027741,
+      "epoch": 2.113055181695828,
+      "grad_norm": 0.747734785079956,
+      "learning_rate": 9.306946739149161e-05,
+      "loss": 0.0364631175994873,
+      "mean_token_accuracy": 0.9862085223197937,
+      "num_tokens": 29610344.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.8798732101917267,
+      "epoch": 2.126514131897712,
+      "grad_norm": 1.5001860857009888,
+      "learning_rate": 9.29559951273358e-05,
+      "loss": 0.03813003897666931,
+      "mean_token_accuracy": 0.9852604746818543,
+      "num_tokens": 29798997.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.8797551989555359,
+      "epoch": 2.1399730820995964,
+      "grad_norm": 0.9593478441238403,
+      "learning_rate": 9.284167180450141e-05,
+      "loss": 0.0394927829504013,
+      "mean_token_accuracy": 0.984604275226593,
+      "num_tokens": 29987809.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.8860546290874481,
+      "epoch": 2.1534320323014806,
+      "grad_norm": 0.8347703218460083,
+      "learning_rate": 9.272649968800069e-05,
+      "loss": 0.036699697375297546,
+      "mean_token_accuracy": 0.985380882024765,
+      "num_tokens": 30176234.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.8783667802810669,
+      "epoch": 2.166890982503365,
+      "grad_norm": 1.1154481172561646,
+      "learning_rate": 9.26104810596625e-05,
+      "loss": 0.03756999969482422,
+      "mean_token_accuracy": 0.9853451430797577,
+      "num_tokens": 30364657.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.8684478521347045,
+      "epoch": 2.180349932705249,
+      "grad_norm": 0.7515475153923035,
+      "learning_rate": 9.249361821808708e-05,
+      "loss": 0.0382376104593277,
+      "mean_token_accuracy": 0.9854029655456543,
+      "num_tokens": 30552904.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.8636964917182922,
+      "epoch": 2.1938088829071334,
+      "grad_norm": 0.7711939215660095,
+      "learning_rate": 9.237591347860052e-05,
+      "loss": 0.036220991611480714,
+      "mean_token_accuracy": 0.9860713243484497,
+      "num_tokens": 30741259.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.8592016279697419,
+      "epoch": 2.2072678331090176,
+      "grad_norm": 1.1143887042999268,
+      "learning_rate": 9.225736917320886e-05,
+      "loss": 0.036316031217575075,
+      "mean_token_accuracy": 0.985757052898407,
+      "num_tokens": 30930144.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.8671079576015472,
+      "epoch": 2.220726783310902,
+      "grad_norm": 0.8980015516281128,
+      "learning_rate": 9.213798765055187e-05,
+      "loss": 0.03822658061981201,
+      "mean_token_accuracy": 0.9847764372825623,
+      "num_tokens": 31118624.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.8785548269748688,
+      "epoch": 2.234185733512786,
+      "grad_norm": 1.094323992729187,
+      "learning_rate": 9.20177712758566e-05,
+      "loss": 0.03736622333526611,
+      "mean_token_accuracy": 0.9850081980228425,
+      "num_tokens": 31306833.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.866977310180664,
+      "epoch": 2.2476446837146704,
+      "grad_norm": 0.8372092843055725,
+      "learning_rate": 9.189672243089046e-05,
+      "loss": 0.0401554524898529,
+      "mean_token_accuracy": 0.9840337932109833,
+      "num_tokens": 31495503.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.8787918210029602,
+      "epoch": 2.2611036339165547,
+      "grad_norm": 1.5408164262771606,
+      "learning_rate": 9.177484351391402e-05,
+      "loss": 0.0368030846118927,
+      "mean_token_accuracy": 0.9847234487533569,
+      "num_tokens": 31683865.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.872721153497696,
+      "epoch": 2.274562584118439,
+      "grad_norm": 1.1115421056747437,
+      "learning_rate": 9.165213693963355e-05,
+      "loss": 0.037859299778938295,
+      "mean_token_accuracy": 0.9851646661758423,
+      "num_tokens": 31871903.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.8774094223976135,
+      "epoch": 2.288021534320323,
+      "grad_norm": 1.0301331281661987,
+      "learning_rate": 9.152860513915314e-05,
+      "loss": 0.038671016693115234,
+      "mean_token_accuracy": 0.9840058028697968,
+      "num_tokens": 32060500.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.889606237411499,
+      "epoch": 2.3014804845222074,
+      "grad_norm": 0.8601903915405273,
+      "learning_rate": 9.140425055992648e-05,
+      "loss": 0.039603835344314574,
+      "mean_token_accuracy": 0.9840235590934754,
+      "num_tokens": 32248744.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.886734277009964,
+      "epoch": 2.3149394347240917,
+      "grad_norm": 1.1029839515686035,
+      "learning_rate": 9.127907566570853e-05,
+      "loss": 0.039513933658599856,
+      "mean_token_accuracy": 0.9844573020935059,
+      "num_tokens": 32437640.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.8843017637729644,
+      "epoch": 2.328398384925976,
+      "grad_norm": 1.2545154094696045,
+      "learning_rate": 9.115308293650653e-05,
+      "loss": 0.036970189213752745,
+      "mean_token_accuracy": 0.985239815711975,
+      "num_tokens": 32625986.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.8913422048091888,
+      "epoch": 2.34185733512786,
+      "grad_norm": 0.9613803625106812,
+      "learning_rate": 9.102627486853099e-05,
+      "loss": 0.03811657428741455,
+      "mean_token_accuracy": 0.9852804243564606,
+      "num_tokens": 32814531.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.8907467782497406,
+      "epoch": 2.3553162853297445,
+      "grad_norm": 1.1811398267745972,
+      "learning_rate": 9.089865397414614e-05,
+      "loss": 0.03903660774230957,
+      "mean_token_accuracy": 0.9842524945735931,
+      "num_tokens": 33002960.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.8833664715290069,
+      "epoch": 2.3687752355316287,
+      "grad_norm": 0.8338477611541748,
+      "learning_rate": 9.077022278182024e-05,
+      "loss": 0.03982087969779968,
+      "mean_token_accuracy": 0.9841201484203339,
+      "num_tokens": 33191183.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.8783826470375061,
+      "epoch": 2.382234185733513,
+      "grad_norm": 0.6904510259628296,
+      "learning_rate": 9.064098383607545e-05,
+      "loss": 0.03699290752410889,
+      "mean_token_accuracy": 0.9854120731353759,
+      "num_tokens": 33379798.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.8871048510074615,
+      "epoch": 2.3956931359353972,
+      "grad_norm": 1.009539246559143,
+      "learning_rate": 9.051093969743738e-05,
+      "loss": 0.03926805555820465,
+      "mean_token_accuracy": 0.9843246698379516,
+      "num_tokens": 33568531.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.8973462700843811,
+      "epoch": 2.409152086137281,
+      "grad_norm": 0.8916401267051697,
+      "learning_rate": 9.03800929423844e-05,
+      "loss": 0.039596831798553465,
+      "mean_token_accuracy": 0.9839228212833404,
+      "num_tokens": 33756333.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.8954619467258453,
+      "epoch": 2.4226110363391653,
+      "grad_norm": 1.056504487991333,
+      "learning_rate": 9.024844616329662e-05,
+      "loss": 0.0396859347820282,
+      "mean_token_accuracy": 0.9837161123752594,
+      "num_tokens": 33944904.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.8952643811702728,
+      "epoch": 2.4360699865410496,
+      "grad_norm": 1.0442084074020386,
+      "learning_rate": 9.011600196840447e-05,
+      "loss": 0.037324142456054685,
+      "mean_token_accuracy": 0.9852897703647614,
+      "num_tokens": 34133530.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.8874911010265351,
+      "epoch": 2.449528936742934,
+      "grad_norm": 0.9926224946975708,
+      "learning_rate": 8.998276298173707e-05,
+      "loss": 0.03637495338916778,
+      "mean_token_accuracy": 0.9853426337242126,
+      "num_tokens": 34322042.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.8900584518909455,
+      "epoch": 2.462987886944818,
+      "grad_norm": 1.001734733581543,
+      "learning_rate": 8.984873184307017e-05,
+      "loss": 0.04029585719108582,
+      "mean_token_accuracy": 0.9842372059822082,
+      "num_tokens": 34510259.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.8986345052719116,
+      "epoch": 2.4764468371467023,
+      "grad_norm": 0.772941529750824,
+      "learning_rate": 8.971391120787397e-05,
+      "loss": 0.0401084691286087,
+      "mean_token_accuracy": 0.9841427087783814,
+      "num_tokens": 34698669.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.8911370873451233,
+      "epoch": 2.4899057873485866,
+      "grad_norm": 0.895031213760376,
+      "learning_rate": 8.957830374726042e-05,
+      "loss": 0.03941032290458679,
+      "mean_token_accuracy": 0.9841966688632965,
+      "num_tokens": 34887160.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.8901963472366333,
+      "epoch": 2.503364737550471,
+      "grad_norm": 1.0677918195724487,
+      "learning_rate": 8.944191214793028e-05,
+      "loss": 0.03503885865211487,
+      "mean_token_accuracy": 0.9856241464614868,
+      "num_tokens": 35075891.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.8817630231380462,
+      "epoch": 2.516823687752355,
+      "grad_norm": 1.1200164556503296,
+      "learning_rate": 8.930473911212e-05,
+      "loss": 0.03878425657749176,
+      "mean_token_accuracy": 0.9846492648124695,
+      "num_tokens": 35264455.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.8897477686405182,
+      "epoch": 2.5302826379542394,
+      "grad_norm": 0.8058190941810608,
+      "learning_rate": 8.916678735754809e-05,
+      "loss": 0.041558006405830385,
+      "mean_token_accuracy": 0.9830883860588073,
+      "num_tokens": 35453251.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.8873181045055389,
+      "epoch": 2.5437415881561236,
+      "grad_norm": 0.7765992283821106,
+      "learning_rate": 8.902805961736123e-05,
+      "loss": 0.03555050492286682,
+      "mean_token_accuracy": 0.9858225345611572,
+      "num_tokens": 35641959.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.8782606959342957,
+      "epoch": 2.557200538358008,
+      "grad_norm": 0.8488866686820984,
+      "learning_rate": 8.88885586400803e-05,
+      "loss": 0.03606923818588257,
+      "mean_token_accuracy": 0.9858632445335388,
+      "num_tokens": 35829636.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.8869808673858642,
+      "epoch": 2.570659488559892,
+      "grad_norm": 1.1734739542007446,
+      "learning_rate": 8.874828718954576e-05,
+      "loss": 0.040956351161003116,
+      "mean_token_accuracy": 0.9835374176502227,
+      "num_tokens": 36017818.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.8850924432277679,
+      "epoch": 2.5841184387617764,
+      "grad_norm": 1.3584387302398682,
+      "learning_rate": 8.86072480448629e-05,
+      "loss": 0.042437011003494264,
+      "mean_token_accuracy": 0.9826479077339172,
+      "num_tokens": 36206505.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.8801687896251679,
+      "epoch": 2.5975773889636606,
+      "grad_norm": 1.2862240076065063,
+      "learning_rate": 8.84654440003469e-05,
+      "loss": 0.03880060315132141,
+      "mean_token_accuracy": 0.9843404710292816,
+      "num_tokens": 36395802.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.8752781748771667,
+      "epoch": 2.611036339165545,
+      "grad_norm": 0.7341249585151672,
+      "learning_rate": 8.83228778654674e-05,
+      "loss": 0.03776344656944275,
+      "mean_token_accuracy": 0.9852877616882324,
+      "num_tokens": 36584370.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.8759494423866272,
+      "epoch": 2.624495289367429,
+      "grad_norm": 0.9807276129722595,
+      "learning_rate": 8.817955246479276e-05,
+      "loss": 0.03780297338962555,
+      "mean_token_accuracy": 0.9853227376937866,
+      "num_tokens": 36772865.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.8753129065036773,
+      "epoch": 2.6379542395693134,
+      "grad_norm": 1.1391628980636597,
+      "learning_rate": 8.803547063793422e-05,
+      "loss": 0.03953765034675598,
+      "mean_token_accuracy": 0.9839197635650635,
+      "num_tokens": 36961462.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.8771899223327637,
+      "epoch": 2.6514131897711977,
+      "grad_norm": 1.0865391492843628,
+      "learning_rate": 8.789063523948958e-05,
+      "loss": 0.03856399655342102,
+      "mean_token_accuracy": 0.9848688066005706,
+      "num_tokens": 37150486.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.8774881184101104,
+      "epoch": 2.664872139973082,
+      "grad_norm": 0.8561595678329468,
+      "learning_rate": 8.774504913898663e-05,
+      "loss": 0.03795175850391388,
+      "mean_token_accuracy": 0.9849309325218201,
+      "num_tokens": 37338890.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.8841517448425293,
+      "epoch": 2.678331090174966,
+      "grad_norm": 0.9458399415016174,
+      "learning_rate": 8.75987152208264e-05,
+      "loss": 0.03860213160514832,
+      "mean_token_accuracy": 0.9842230796813964,
+      "num_tokens": 37526886.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.8809043228626251,
+      "epoch": 2.6917900403768504,
+      "grad_norm": 0.9360871315002441,
+      "learning_rate": 8.745163638422583e-05,
+      "loss": 0.03709094822406769,
+      "mean_token_accuracy": 0.9855118036270142,
+      "num_tokens": 37715408.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.8805519282817841,
+      "epoch": 2.7052489905787347,
+      "grad_norm": 0.8852124214172363,
+      "learning_rate": 8.730381554316051e-05,
+      "loss": 0.03869341611862183,
+      "mean_token_accuracy": 0.9846724212169647,
+      "num_tokens": 37904328.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.8796228408813477,
+      "epoch": 2.718707940780619,
+      "grad_norm": 1.189297080039978,
+      "learning_rate": 8.715525562630687e-05,
+      "loss": 0.03675769567489624,
+      "mean_token_accuracy": 0.9854080140590668,
+      "num_tokens": 38092776.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.8722020089626312,
+      "epoch": 2.732166890982503,
+      "grad_norm": 0.7027947306632996,
+      "learning_rate": 8.700595957698411e-05,
+      "loss": 0.03889244794845581,
+      "mean_token_accuracy": 0.984558242559433,
+      "num_tokens": 38282221.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.8746632814407349,
+      "epoch": 2.7456258411843875,
+      "grad_norm": 0.9799376726150513,
+      "learning_rate": 8.685593035309598e-05,
+      "loss": 0.03815680146217346,
+      "mean_token_accuracy": 0.9845012128353119,
+      "num_tokens": 38470633.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.8766567528247833,
+      "epoch": 2.7590847913862717,
+      "grad_norm": 0.850459098815918,
+      "learning_rate": 8.670517092707213e-05,
+      "loss": 0.039029371738433835,
+      "mean_token_accuracy": 0.9845906794071198,
+      "num_tokens": 38659048.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.8778688013553619,
+      "epoch": 2.772543741588156,
+      "grad_norm": 0.8540134429931641,
+      "learning_rate": 8.655368428580919e-05,
+      "loss": 0.039645448327064514,
+      "mean_token_accuracy": 0.9840566098690033,
+      "num_tokens": 38847370.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.8758600771427154,
+      "epoch": 2.7860026917900402,
+      "grad_norm": 0.8036021590232849,
+      "learning_rate": 8.640147343061165e-05,
+      "loss": 0.038226932287216187,
+      "mean_token_accuracy": 0.9845647215843201,
+      "num_tokens": 39036040.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.8652268946170807,
+      "epoch": 2.7994616419919245,
+      "grad_norm": 0.8526738286018372,
+      "learning_rate": 8.624854137713234e-05,
+      "loss": 0.03798363208770752,
+      "mean_token_accuracy": 0.9845202267169952,
+      "num_tokens": 39224695.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.8707200348377228,
+      "epoch": 2.8129205921938087,
+      "grad_norm": 0.9852389097213745,
+      "learning_rate": 8.609489115531278e-05,
+      "loss": 0.037227436900138855,
+      "mean_token_accuracy": 0.9852050960063934,
+      "num_tokens": 39413360.0,
+      "step": 2090
+    },
+    {
+      "entropy": 0.8711447060108185,
+      "epoch": 2.826379542395693,
+      "grad_norm": 1.228277325630188,
+      "learning_rate": 8.594052580932301e-05,
+      "loss": 0.03698050379753113,
+      "mean_token_accuracy": 0.9852443158626556,
+      "num_tokens": 39602544.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.8742912471294403,
+      "epoch": 2.8398384925975773,
+      "grad_norm": 0.8440544605255127,
+      "learning_rate": 8.578544839750141e-05,
+      "loss": 0.03757670521736145,
+      "mean_token_accuracy": 0.9848618268966675,
+      "num_tokens": 39791340.0,
+      "step": 2110
+    },
+    {
+      "entropy": 0.8801726162433624,
+      "epoch": 2.8532974427994615,
+      "grad_norm": 1.0955252647399902,
+      "learning_rate": 8.562966199229399e-05,
+      "loss": 0.040004149079322815,
+      "mean_token_accuracy": 0.9841345012187958,
+      "num_tokens": 39980212.0,
+      "step": 2120
+    },
+    {
+      "entropy": 0.8931492984294891,
+      "epoch": 2.8667563930013458,
+      "grad_norm": 0.5722652673721313,
+      "learning_rate": 8.547316968019363e-05,
+      "loss": 0.038096648454666135,
+      "mean_token_accuracy": 0.9849136590957641,
+      "num_tokens": 40169122.0,
+      "step": 2130
+    },
+    {
+      "entropy": 0.8767493844032288,
+      "epoch": 2.88021534320323,
+      "grad_norm": 0.7663730382919312,
+      "learning_rate": 8.531597456167885e-05,
+      "loss": 0.03729096055030823,
+      "mean_token_accuracy": 0.985169780254364,
+      "num_tokens": 40357678.0,
+      "step": 2140
+    },
+    {
+      "entropy": 0.8770561516284943,
+      "epoch": 2.8936742934051143,
+      "grad_norm": 1.1505672931671143,
+      "learning_rate": 8.515807975115239e-05,
+      "loss": 0.03984796404838562,
+      "mean_token_accuracy": 0.9838845670223236,
+      "num_tokens": 40546892.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.8789129912853241,
+      "epoch": 2.9071332436069985,
+      "grad_norm": 0.8355761766433716,
+      "learning_rate": 8.499948837687959e-05,
+      "loss": 0.03769223690032959,
+      "mean_token_accuracy": 0.9847559213638306,
+      "num_tokens": 40736109.0,
+      "step": 2160
+    },
+    {
+      "entropy": 0.8781779289245606,
+      "epoch": 2.920592193808883,
+      "grad_norm": 0.9103918075561523,
+      "learning_rate": 8.484020358092625e-05,
+      "loss": 0.038262826204299924,
+      "mean_token_accuracy": 0.984741848707199,
+      "num_tokens": 40924341.0,
+      "step": 2170
+    },
+    {
+      "entropy": 0.8552807629108429,
+      "epoch": 2.934051144010767,
+      "grad_norm": 0.8338510990142822,
+      "learning_rate": 8.468022851909657e-05,
+      "loss": 0.0355743408203125,
+      "mean_token_accuracy": 0.9855036079883576,
+      "num_tokens": 41113849.0,
+      "step": 2180
+    },
+    {
+      "entropy": 0.8705591142177582,
+      "epoch": 2.9475100942126513,
+      "grad_norm": 0.8334057927131653,
+      "learning_rate": 8.451956636087046e-05,
+      "loss": 0.037476515769958495,
+      "mean_token_accuracy": 0.9854551255702972,
+      "num_tokens": 41303130.0,
+      "step": 2190
+    },
+    {
+      "entropy": 0.8809274792671203,
+      "epoch": 2.9609690444145356,
+      "grad_norm": 0.9424103498458862,
+      "learning_rate": 8.435822028934087e-05,
+      "loss": 0.03624279499053955,
+      "mean_token_accuracy": 0.9856135487556458,
+      "num_tokens": 41491750.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.8818133771419525,
+      "epoch": 2.97442799461642,
+      "grad_norm": 1.04243004322052,
+      "learning_rate": 8.41961935011506e-05,
+      "loss": 0.039132320880889894,
+      "mean_token_accuracy": 0.984746390581131,
+      "num_tokens": 41680493.0,
+      "step": 2210
+    },
+    {
+      "entropy": 0.8838395297527313,
+      "epoch": 2.987886944818304,
+      "grad_norm": 0.8650784492492676,
+      "learning_rate": 8.403348920642911e-05,
+      "loss": 0.03822052478790283,
+      "mean_token_accuracy": 0.9852108955383301,
+      "num_tokens": 41868378.0,
+      "step": 2220
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 0.8800126549544608,
+      "eval_loss": 0.05468416586518288,
+      "eval_mean_token_accuracy": 0.9781015617832257,
+      "eval_num_tokens": 42038037.0,
+      "eval_runtime": 24.4825,
+      "eval_samples_per_second": 204.228,
+      "eval_steps_per_second": 6.413,
+      "step": 2229
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7430,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.0311131063058432e+18,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

baby_talk_L16_a50/seed_42/checkpoint-2229/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c060e97b69d99564c146471c3d3ac4d335e3b1968074124f4edc5aebf612e1e3
+size 5368

baby_talk_L16_a50/seed_42/checkpoint-2972/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

baby_talk_L16_a50/seed_42/checkpoint-2972/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

baby_talk_L16_a50/seed_42/checkpoint-2972/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2515d21f30afa1abce215a6e8019acc54800da53f5bc3c4775850dda77992ce
+size 80792096

baby_talk_L16_a50/seed_42/checkpoint-2972/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

baby_talk_L16_a50/seed_42/checkpoint-2972/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

baby_talk_L16_a50/seed_42/checkpoint-2972/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

baby_talk_L16_a50/seed_42/checkpoint-2972/trainer_state.json

@@ -0,0 +1,3048 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 2972,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.2358420491218567,
+      "epoch": 0.013458950201884253,
+      "grad_norm": 3.2940118312835693,
+      "learning_rate": 2.4193548387096776e-06,
+      "loss": 0.550364351272583,
+      "mean_token_accuracy": 0.8554959416389465,
+      "num_tokens": 188811.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.2353882431983947,
+      "epoch": 0.026917900403768506,
+      "grad_norm": 3.4476470947265625,
+      "learning_rate": 5.1075268817204305e-06,
+      "loss": 0.5143545627593994,
+      "mean_token_accuracy": 0.8613634884357453,
+      "num_tokens": 377729.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.2403772234916688,
+      "epoch": 0.040376850605652756,
+      "grad_norm": 2.2756996154785156,
+      "learning_rate": 7.795698924731183e-06,
+      "loss": 0.3996511220932007,
+      "mean_token_accuracy": 0.8753438770771027,
+      "num_tokens": 566562.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.2205921411514282,
+      "epoch": 0.05383580080753701,
+      "grad_norm": 1.2432096004486084,
+      "learning_rate": 1.0483870967741936e-05,
+      "loss": 0.2568032264709473,
+      "mean_token_accuracy": 0.9130991995334625,
+      "num_tokens": 755026.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.221834623813629,
+      "epoch": 0.06729475100942127,
+      "grad_norm": 0.8531327843666077,
+      "learning_rate": 1.3172043010752688e-05,
+      "loss": 0.20193097591400147,
+      "mean_token_accuracy": 0.9274256646633148,
+      "num_tokens": 943494.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.2245031952857972,
+      "epoch": 0.08075370121130551,
+      "grad_norm": 0.5026484131813049,
+      "learning_rate": 1.586021505376344e-05,
+      "loss": 0.171803081035614,
+      "mean_token_accuracy": 0.9363821744918823,
+      "num_tokens": 1131731.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.2192703008651733,
+      "epoch": 0.09421265141318977,
+      "grad_norm": 0.5228630304336548,
+      "learning_rate": 1.8548387096774193e-05,
+      "loss": 0.15698516368865967,
+      "mean_token_accuracy": 0.9423282980918884,
+      "num_tokens": 1320258.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.2153660774230957,
+      "epoch": 0.10767160161507403,
+      "grad_norm": 0.3650094270706177,
+      "learning_rate": 2.1236559139784946e-05,
+      "loss": 0.14900912046432496,
+      "mean_token_accuracy": 0.9437524616718292,
+      "num_tokens": 1509209.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.2123230814933776,
+      "epoch": 0.12113055181695828,
+      "grad_norm": 0.45723631978034973,
+      "learning_rate": 2.39247311827957e-05,
+      "loss": 0.1399540901184082,
+      "mean_token_accuracy": 0.9474983811378479,
+      "num_tokens": 1698139.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.208789598941803,
+      "epoch": 0.13458950201884254,
+      "grad_norm": 0.4575304687023163,
+      "learning_rate": 2.661290322580645e-05,
+      "loss": 0.12566736936569214,
+      "mean_token_accuracy": 0.9529488801956176,
+      "num_tokens": 1886384.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.205567193031311,
+      "epoch": 0.1480484522207268,
+      "grad_norm": 0.8033406734466553,
+      "learning_rate": 2.9301075268817207e-05,
+      "loss": 0.12005312442779541,
+      "mean_token_accuracy": 0.9533569395542145,
+      "num_tokens": 2074919.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.20180287361145,
+      "epoch": 0.16150740242261102,
+      "grad_norm": 0.6989286541938782,
+      "learning_rate": 3.198924731182796e-05,
+      "loss": 0.11197478771209717,
+      "mean_token_accuracy": 0.9572584748268127,
+      "num_tokens": 2263116.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.202919363975525,
+      "epoch": 0.17496635262449528,
+      "grad_norm": 0.7233495116233826,
+      "learning_rate": 3.467741935483872e-05,
+      "loss": 0.11106340885162354,
+      "mean_token_accuracy": 0.9569663584232331,
+      "num_tokens": 2451273.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.1907272219657898,
+      "epoch": 0.18842530282637954,
+      "grad_norm": 0.7645091414451599,
+      "learning_rate": 3.736559139784947e-05,
+      "loss": 0.10956189632415772,
+      "mean_token_accuracy": 0.95842245221138,
+      "num_tokens": 2640657.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.1895484924316406,
+      "epoch": 0.2018842530282638,
+      "grad_norm": 0.841379702091217,
+      "learning_rate": 4.005376344086022e-05,
+      "loss": 0.10442907810211181,
+      "mean_token_accuracy": 0.9603263795375824,
+      "num_tokens": 2829351.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.1871973633766175,
+      "epoch": 0.21534320323014805,
+      "grad_norm": 0.8552286624908447,
+      "learning_rate": 4.2741935483870973e-05,
+      "loss": 0.10493810176849365,
+      "mean_token_accuracy": 0.9597454965114594,
+      "num_tokens": 3018290.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.1907334923744202,
+      "epoch": 0.2288021534320323,
+      "grad_norm": 0.7243770360946655,
+      "learning_rate": 4.543010752688172e-05,
+      "loss": 0.10288643836975098,
+      "mean_token_accuracy": 0.9603340923786163,
+      "num_tokens": 3206538.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.1835299730300903,
+      "epoch": 0.24226110363391656,
+      "grad_norm": 0.8118091225624084,
+      "learning_rate": 4.811827956989248e-05,
+      "loss": 0.09798368811607361,
+      "mean_token_accuracy": 0.9619723737239838,
+      "num_tokens": 3395374.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1794602513313293,
+      "epoch": 0.2557200538358008,
+      "grad_norm": 0.7447699904441833,
+      "learning_rate": 5.080645161290323e-05,
+      "loss": 0.09498158693313599,
+      "mean_token_accuracy": 0.9634931206703186,
+      "num_tokens": 3584273.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.1834724426269532,
+      "epoch": 0.2691790040376851,
+      "grad_norm": 1.4059171676635742,
+      "learning_rate": 5.349462365591398e-05,
+      "loss": 0.09400172233581543,
+      "mean_token_accuracy": 0.9630160868167877,
+      "num_tokens": 3772857.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.186821174621582,
+      "epoch": 0.28263795423956933,
+      "grad_norm": 0.8273025751113892,
+      "learning_rate": 5.618279569892473e-05,
+      "loss": 0.09464811086654663,
+      "mean_token_accuracy": 0.9631786167621612,
+      "num_tokens": 3960877.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.1769920349121095,
+      "epoch": 0.2960969044414536,
+      "grad_norm": 1.0182609558105469,
+      "learning_rate": 5.887096774193549e-05,
+      "loss": 0.0914128303527832,
+      "mean_token_accuracy": 0.9644896507263183,
+      "num_tokens": 4149444.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.1719188809394836,
+      "epoch": 0.30955585464333785,
+      "grad_norm": 0.9648745656013489,
+      "learning_rate": 6.155913978494624e-05,
+      "loss": 0.0920255422592163,
+      "mean_token_accuracy": 0.963964831829071,
+      "num_tokens": 4337989.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.1559369206428527,
+      "epoch": 0.32301480484522205,
+      "grad_norm": 0.853049635887146,
+      "learning_rate": 6.4247311827957e-05,
+      "loss": 0.0891042947769165,
+      "mean_token_accuracy": 0.9653984010219574,
+      "num_tokens": 4527256.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.1604587316513062,
+      "epoch": 0.3364737550471063,
+      "grad_norm": 0.873772144317627,
+      "learning_rate": 6.693548387096774e-05,
+      "loss": 0.09071275591850281,
+      "mean_token_accuracy": 0.9651397407054901,
+      "num_tokens": 4715878.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.1534931778907775,
+      "epoch": 0.34993270524899056,
+      "grad_norm": 1.087850570678711,
+      "learning_rate": 6.962365591397851e-05,
+      "loss": 0.08719289302825928,
+      "mean_token_accuracy": 0.9658753871917725,
+      "num_tokens": 4904527.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.1629684448242188,
+      "epoch": 0.3633916554508748,
+      "grad_norm": 0.9014195799827576,
+      "learning_rate": 7.231182795698926e-05,
+      "loss": 0.08716133236885071,
+      "mean_token_accuracy": 0.965560519695282,
+      "num_tokens": 5093554.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.148916518688202,
+      "epoch": 0.3768506056527591,
+      "grad_norm": 0.9712215662002563,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.08668915033340455,
+      "mean_token_accuracy": 0.9658140063285827,
+      "num_tokens": 5282224.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1493291974067688,
+      "epoch": 0.39030955585464333,
+      "grad_norm": 0.9085242748260498,
+      "learning_rate": 7.768817204301076e-05,
+      "loss": 0.08189771175384522,
+      "mean_token_accuracy": 0.9670268416404724,
+      "num_tokens": 5471308.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.1406704187393188,
+      "epoch": 0.4037685060565276,
+      "grad_norm": 1.128177285194397,
+      "learning_rate": 8.037634408602151e-05,
+      "loss": 0.08119879961013794,
+      "mean_token_accuracy": 0.9675639867782593,
+      "num_tokens": 5660026.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.1369357347488402,
+      "epoch": 0.41722745625841184,
+      "grad_norm": 0.8224227428436279,
+      "learning_rate": 8.306451612903227e-05,
+      "loss": 0.07979745864868164,
+      "mean_token_accuracy": 0.9681445300579071,
+      "num_tokens": 5848073.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.1252583503723144,
+      "epoch": 0.4306864064602961,
+      "grad_norm": 0.7711160182952881,
+      "learning_rate": 8.575268817204302e-05,
+      "loss": 0.0783164381980896,
+      "mean_token_accuracy": 0.9686201930046081,
+      "num_tokens": 6036717.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.1203404307365417,
+      "epoch": 0.44414535666218036,
+      "grad_norm": 1.1810287237167358,
+      "learning_rate": 8.844086021505377e-05,
+      "loss": 0.08139073848724365,
+      "mean_token_accuracy": 0.9681365489959717,
+      "num_tokens": 6225223.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.1222687840461731,
+      "epoch": 0.4576043068640646,
+      "grad_norm": 1.4551713466644287,
+      "learning_rate": 9.112903225806452e-05,
+      "loss": 0.0820135235786438,
+      "mean_token_accuracy": 0.9672703862190246,
+      "num_tokens": 6413936.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.1156280279159545,
+      "epoch": 0.47106325706594887,
+      "grad_norm": 0.8573716878890991,
+      "learning_rate": 9.381720430107528e-05,
+      "loss": 0.08074904680252075,
+      "mean_token_accuracy": 0.9677663922309876,
+      "num_tokens": 6602413.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.1063185572624206,
+      "epoch": 0.4845222072678331,
+      "grad_norm": 0.7709434628486633,
+      "learning_rate": 9.650537634408603e-05,
+      "loss": 0.07549421787261963,
+      "mean_token_accuracy": 0.9689356207847595,
+      "num_tokens": 6790873.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.0962031960487366,
+      "epoch": 0.4979811574697174,
+      "grad_norm": 0.7843625545501709,
+      "learning_rate": 9.919354838709678e-05,
+      "loss": 0.07368478775024415,
+      "mean_token_accuracy": 0.9700294613838196,
+      "num_tokens": 6979392.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.1028530240058898,
+      "epoch": 0.5114401076716016,
+      "grad_norm": 0.7935485243797302,
+      "learning_rate": 9.999975729865971e-05,
+      "loss": 0.07766538262367248,
+      "mean_token_accuracy": 0.9704003691673279,
+      "num_tokens": 7167482.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.0905393958091736,
+      "epoch": 0.5248990578734859,
+      "grad_norm": 1.4108924865722656,
+      "learning_rate": 9.999856856307314e-05,
+      "loss": 0.0760004162788391,
+      "mean_token_accuracy": 0.9699020266532898,
+      "num_tokens": 7355994.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.0853531122207642,
+      "epoch": 0.5383580080753702,
+      "grad_norm": 0.9043511152267456,
+      "learning_rate": 9.999638923896533e-05,
+      "loss": 0.0720310389995575,
+      "mean_token_accuracy": 0.9707890212535858,
+      "num_tokens": 7544655.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.075773572921753,
+      "epoch": 0.5518169582772544,
+      "grad_norm": 0.9038823246955872,
+      "learning_rate": 9.999321936951374e-05,
+      "loss": 0.07026209831237792,
+      "mean_token_accuracy": 0.9715417385101318,
+      "num_tokens": 7733348.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.0572428584098816,
+      "epoch": 0.5652759084791387,
+      "grad_norm": 0.8231707811355591,
+      "learning_rate": 9.998905901752091e-05,
+      "loss": 0.07141299843788147,
+      "mean_token_accuracy": 0.9705908715724945,
+      "num_tokens": 7921582.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.051485300064087,
+      "epoch": 0.5787348586810229,
+      "grad_norm": 1.1695454120635986,
+      "learning_rate": 9.998390826541315e-05,
+      "loss": 0.07321611642837525,
+      "mean_token_accuracy": 0.9709623396396637,
+      "num_tokens": 8110266.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.0485445380210876,
+      "epoch": 0.5921938088829072,
+      "grad_norm": 0.7291010618209839,
+      "learning_rate": 9.997776721523888e-05,
+      "loss": 0.07221676707267762,
+      "mean_token_accuracy": 0.9705081820487976,
+      "num_tokens": 8298932.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.0456495046615601,
+      "epoch": 0.6056527590847914,
+      "grad_norm": 0.6550094485282898,
+      "learning_rate": 9.99706359886667e-05,
+      "loss": 0.06878133416175843,
+      "mean_token_accuracy": 0.9728739261627197,
+      "num_tokens": 8487613.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.0372861266136169,
+      "epoch": 0.6191117092866757,
+      "grad_norm": 0.5825450420379639,
+      "learning_rate": 9.996251472698281e-05,
+      "loss": 0.06706151366233826,
+      "mean_token_accuracy": 0.9732721030712128,
+      "num_tokens": 8676276.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.0319493532180786,
+      "epoch": 0.6325706594885598,
+      "grad_norm": 0.7158534526824951,
+      "learning_rate": 9.995340359108844e-05,
+      "loss": 0.06999597549438477,
+      "mean_token_accuracy": 0.9726741492748261,
+      "num_tokens": 8864738.0,
+      "step": 470
+    },
+    {
+      "entropy": 1.0260030388832093,
+      "epoch": 0.6460296096904441,
+      "grad_norm": 0.7918373346328735,
+      "learning_rate": 9.994330276149649e-05,
+      "loss": 0.06949877142906188,
+      "mean_token_accuracy": 0.9725882947444916,
+      "num_tokens": 9053582.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.0258127093315124,
+      "epoch": 0.6594885598923284,
+      "grad_norm": 0.893281102180481,
+      "learning_rate": 9.993221243832797e-05,
+      "loss": 0.06893026828765869,
+      "mean_token_accuracy": 0.9729171216487884,
+      "num_tokens": 9241914.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.0261828184127808,
+      "epoch": 0.6729475100942126,
+      "grad_norm": 1.0127108097076416,
+      "learning_rate": 9.992013284130816e-05,
+      "loss": 0.07094801664352417,
+      "mean_token_accuracy": 0.9714575052261353,
+      "num_tokens": 9430981.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.0274562597274781,
+      "epoch": 0.6864064602960969,
+      "grad_norm": 0.7148029208183289,
+      "learning_rate": 9.990706420976206e-05,
+      "loss": 0.06826171875,
+      "mean_token_accuracy": 0.9727248430252076,
+      "num_tokens": 9619472.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.0137859225273131,
+      "epoch": 0.6998654104979811,
+      "grad_norm": 0.9228634238243103,
+      "learning_rate": 9.989300680260985e-05,
+      "loss": 0.06890587210655212,
+      "mean_token_accuracy": 0.9723304331302642,
+      "num_tokens": 9808123.0,
+      "step": 520
+    },
+    {
+      "entropy": 1.0221150755882262,
+      "epoch": 0.7133243606998654,
+      "grad_norm": 0.971530556678772,
+      "learning_rate": 9.98779608983616e-05,
+      "loss": 0.07073599100112915,
+      "mean_token_accuracy": 0.9714174270629883,
+      "num_tokens": 9996601.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.0248154640197753,
+      "epoch": 0.7267833109017496,
+      "grad_norm": 0.7336317896842957,
+      "learning_rate": 9.986192679511189e-05,
+      "loss": 0.06874136924743653,
+      "mean_token_accuracy": 0.9723432004451752,
+      "num_tokens": 10184725.0,
+      "step": 540
+    },
+    {
+      "entropy": 1.0120978832244873,
+      "epoch": 0.7402422611036339,
+      "grad_norm": 1.2100600004196167,
+      "learning_rate": 9.984490481053372e-05,
+      "loss": 0.06270487308502197,
+      "mean_token_accuracy": 0.9756880521774292,
+      "num_tokens": 10373446.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.0172406077384948,
+      "epoch": 0.7537012113055181,
+      "grad_norm": 0.7562853693962097,
+      "learning_rate": 9.982689528187244e-05,
+      "loss": 0.06784560084342957,
+      "mean_token_accuracy": 0.9724333345890045,
+      "num_tokens": 10561710.0,
+      "step": 560
+    },
+    {
+      "entropy": 1.0150038957595826,
+      "epoch": 0.7671601615074024,
+      "grad_norm": 1.07966148853302,
+      "learning_rate": 9.98078985659389e-05,
+      "loss": 0.0670344054698944,
+      "mean_token_accuracy": 0.97339146733284,
+      "num_tokens": 10749917.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.0135640978813172,
+      "epoch": 0.7806191117092867,
+      "grad_norm": 0.7826104164123535,
+      "learning_rate": 9.978791503910246e-05,
+      "loss": 0.0668565571308136,
+      "mean_token_accuracy": 0.9735289216041565,
+      "num_tokens": 10938098.0,
+      "step": 580
+    },
+    {
+      "entropy": 1.0077669620513916,
+      "epoch": 0.7940780619111709,
+      "grad_norm": 1.299584150314331,
+      "learning_rate": 9.97669450972835e-05,
+      "loss": 0.0728976845741272,
+      "mean_token_accuracy": 0.9701269209384918,
+      "num_tokens": 11126107.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.9957692861557007,
+      "epoch": 0.8075370121130552,
+      "grad_norm": 1.1542484760284424,
+      "learning_rate": 9.974498915594557e-05,
+      "loss": 0.0631720781326294,
+      "mean_token_accuracy": 0.9747781097888947,
+      "num_tokens": 11315001.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.9983992159366608,
+      "epoch": 0.8209959623149394,
+      "grad_norm": 0.6967246532440186,
+      "learning_rate": 9.97220476500872e-05,
+      "loss": 0.06333768963813782,
+      "mean_token_accuracy": 0.9750322341918946,
+      "num_tokens": 11503007.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.9801111698150635,
+      "epoch": 0.8344549125168237,
+      "grad_norm": 0.945446789264679,
+      "learning_rate": 9.969812103423325e-05,
+      "loss": 0.05720087289810181,
+      "mean_token_accuracy": 0.9767442524433136,
+      "num_tokens": 11691883.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.9879570543766022,
+      "epoch": 0.847913862718708,
+      "grad_norm": 0.8254193663597107,
+      "learning_rate": 9.967320978242592e-05,
+      "loss": 0.05899171829223633,
+      "mean_token_accuracy": 0.9767756819725036,
+      "num_tokens": 11880353.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.9878240942955017,
+      "epoch": 0.8613728129205922,
+      "grad_norm": 0.9882538914680481,
+      "learning_rate": 9.964731438821533e-05,
+      "loss": 0.06443996429443359,
+      "mean_token_accuracy": 0.9735406100749969,
+      "num_tokens": 12069610.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.9938213169574738,
+      "epoch": 0.8748317631224765,
+      "grad_norm": 1.126546859741211,
+      "learning_rate": 9.962043536464978e-05,
+      "loss": 0.06708416938781739,
+      "mean_token_accuracy": 0.9730359137058258,
+      "num_tokens": 12257901.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.9800443708896637,
+      "epoch": 0.8882907133243607,
+      "grad_norm": 0.9151445031166077,
+      "learning_rate": 9.959257324426556e-05,
+      "loss": 0.06290764808654785,
+      "mean_token_accuracy": 0.9741653084754944,
+      "num_tokens": 12446236.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.9744794130325317,
+      "epoch": 0.901749663526245,
+      "grad_norm": 0.9104028940200806,
+      "learning_rate": 9.95637285790764e-05,
+      "loss": 0.060464882850646974,
+      "mean_token_accuracy": 0.9757490694522858,
+      "num_tokens": 12635226.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.9786687552928924,
+      "epoch": 0.9152086137281292,
+      "grad_norm": 0.7316192388534546,
+      "learning_rate": 9.953390194056258e-05,
+      "loss": 0.06054847836494446,
+      "mean_token_accuracy": 0.9760404825210571,
+      "num_tokens": 12823428.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.966060334444046,
+      "epoch": 0.9286675639300135,
+      "grad_norm": 1.3535135984420776,
+      "learning_rate": 9.950309391965947e-05,
+      "loss": 0.061383575201034546,
+      "mean_token_accuracy": 0.9749173820018768,
+      "num_tokens": 13012362.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.9722849190235138,
+      "epoch": 0.9421265141318977,
+      "grad_norm": 0.890771746635437,
+      "learning_rate": 9.947130512674602e-05,
+      "loss": 0.06301190257072449,
+      "mean_token_accuracy": 0.9750476598739624,
+      "num_tokens": 13200680.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.9624020338058472,
+      "epoch": 0.955585464333782,
+      "grad_norm": 1.142499327659607,
+      "learning_rate": 9.943853619163255e-05,
+      "loss": 0.06179196834564209,
+      "mean_token_accuracy": 0.9751901209354401,
+      "num_tokens": 13389333.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.9653747022151947,
+      "epoch": 0.9690444145356663,
+      "grad_norm": 0.8682493567466736,
+      "learning_rate": 9.94047877635482e-05,
+      "loss": 0.06113170981407166,
+      "mean_token_accuracy": 0.9748325288295746,
+      "num_tokens": 13578210.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.970348197221756,
+      "epoch": 0.9825033647375505,
+      "grad_norm": 0.8351430296897888,
+      "learning_rate": 9.93700605111283e-05,
+      "loss": 0.05901788473129273,
+      "mean_token_accuracy": 0.9761226296424865,
+      "num_tokens": 13767039.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.956724613904953,
+      "epoch": 0.9959623149394348,
+      "grad_norm": 1.2604819536209106,
+      "learning_rate": 9.933435512240084e-05,
+      "loss": 0.06124393343925476,
+      "mean_token_accuracy": 0.9747833967208862,
+      "num_tokens": 13956057.0,
+      "step": 740
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 0.9722227849018802,
+      "eval_loss": 0.0600070059299469,
+      "eval_mean_token_accuracy": 0.975752676368519,
+      "eval_num_tokens": 14012630.0,
+      "eval_runtime": 24.6215,
+      "eval_samples_per_second": 203.074,
+      "eval_steps_per_second": 6.377,
+      "step": 743
+    },
+    {
+      "entropy": 0.9689933776855468,
+      "epoch": 1.009421265141319,
+      "grad_norm": 0.9720287919044495,
+      "learning_rate": 9.929767230477305e-05,
+      "loss": 0.055563896894454956,
+      "mean_token_accuracy": 0.9779708027839661,
+      "num_tokens": 14144652.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.9506655633449554,
+      "epoch": 1.0228802153432033,
+      "grad_norm": 0.6619595289230347,
+      "learning_rate": 9.92600127850173e-05,
+      "loss": 0.050587379932403566,
+      "mean_token_accuracy": 0.979685264825821,
+      "num_tokens": 14332714.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.9458732306957245,
+      "epoch": 1.0363391655450875,
+      "grad_norm": 0.7905108332633972,
+      "learning_rate": 9.922137730925673e-05,
+      "loss": 0.05069155097007751,
+      "mean_token_accuracy": 0.9796596229076385,
+      "num_tokens": 14520902.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.9320916712284089,
+      "epoch": 1.0497981157469718,
+      "grad_norm": 0.9210566878318787,
+      "learning_rate": 9.918176664295041e-05,
+      "loss": 0.051744121313095096,
+      "mean_token_accuracy": 0.9792148530483246,
+      "num_tokens": 14709215.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.928934782743454,
+      "epoch": 1.063257065948856,
+      "grad_norm": 0.7442036867141724,
+      "learning_rate": 9.914118157087824e-05,
+      "loss": 0.0486875057220459,
+      "mean_token_accuracy": 0.9798122465610504,
+      "num_tokens": 14898242.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.942881977558136,
+      "epoch": 1.0767160161507403,
+      "grad_norm": 1.0910941362380981,
+      "learning_rate": 9.909962289712538e-05,
+      "loss": 0.052297019958496095,
+      "mean_token_accuracy": 0.9793200254440307,
+      "num_tokens": 15086835.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.9503917813301086,
+      "epoch": 1.0901749663526246,
+      "grad_norm": 0.8091554641723633,
+      "learning_rate": 9.905709144506629e-05,
+      "loss": 0.049927744269371035,
+      "mean_token_accuracy": 0.9802082359790802,
+      "num_tokens": 15275042.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.9426830470561981,
+      "epoch": 1.1036339165545088,
+      "grad_norm": 1.0589245557785034,
+      "learning_rate": 9.901358805734846e-05,
+      "loss": 0.053803551197052005,
+      "mean_token_accuracy": 0.9785399675369263,
+      "num_tokens": 15463923.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.9445010781288147,
+      "epoch": 1.117092866756393,
+      "grad_norm": 0.8410441279411316,
+      "learning_rate": 9.89691135958757e-05,
+      "loss": 0.053181976079940796,
+      "mean_token_accuracy": 0.9792523324489594,
+      "num_tokens": 15652526.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.946729838848114,
+      "epoch": 1.1305518169582773,
+      "grad_norm": 0.8246080279350281,
+      "learning_rate": 9.892366894179105e-05,
+      "loss": 0.054291915893554685,
+      "mean_token_accuracy": 0.9791357696056366,
+      "num_tokens": 15841340.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.9433728814125061,
+      "epoch": 1.1440107671601616,
+      "grad_norm": 0.9200296401977539,
+      "learning_rate": 9.887725499545937e-05,
+      "loss": 0.05073915719985962,
+      "mean_token_accuracy": 0.9796931505203247,
+      "num_tokens": 16029478.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.9494417488574982,
+      "epoch": 1.1574697173620458,
+      "grad_norm": 0.8342758417129517,
+      "learning_rate": 9.882987267644939e-05,
+      "loss": 0.050929927825927736,
+      "mean_token_accuracy": 0.9793219089508056,
+      "num_tokens": 16218014.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.9409256100654602,
+      "epoch": 1.17092866756393,
+      "grad_norm": 0.7911009192466736,
+      "learning_rate": 9.878152292351563e-05,
+      "loss": 0.049964362382888795,
+      "mean_token_accuracy": 0.9798487305641175,
+      "num_tokens": 16406420.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.9429958581924438,
+      "epoch": 1.1843876177658144,
+      "grad_norm": 1.4877018928527832,
+      "learning_rate": 9.873220669457975e-05,
+      "loss": 0.04969423711299896,
+      "mean_token_accuracy": 0.9801322638988494,
+      "num_tokens": 16594948.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.9542136013507843,
+      "epoch": 1.1978465679676986,
+      "grad_norm": 0.8030785918235779,
+      "learning_rate": 9.868192496671147e-05,
+      "loss": 0.04981146454811096,
+      "mean_token_accuracy": 0.9796162784099579,
+      "num_tokens": 16782983.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.9367722630500793,
+      "epoch": 1.2113055181695827,
+      "grad_norm": 1.1169800758361816,
+      "learning_rate": 9.86306787361094e-05,
+      "loss": 0.05210963487625122,
+      "mean_token_accuracy": 0.9790139615535736,
+      "num_tokens": 16971394.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.9487208306789399,
+      "epoch": 1.224764468371467,
+      "grad_norm": 0.8650282621383667,
+      "learning_rate": 9.857846901808117e-05,
+      "loss": 0.05012243390083313,
+      "mean_token_accuracy": 0.9805107891559601,
+      "num_tokens": 17159951.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.9441231489181519,
+      "epoch": 1.2382234185733512,
+      "grad_norm": 0.8193967938423157,
+      "learning_rate": 9.852529684702329e-05,
+      "loss": 0.048866665363311766,
+      "mean_token_accuracy": 0.9807584881782532,
+      "num_tokens": 17348711.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.9361630439758301,
+      "epoch": 1.2516823687752354,
+      "grad_norm": 1.3613612651824951,
+      "learning_rate": 9.847116327640082e-05,
+      "loss": 0.05324091911315918,
+      "mean_token_accuracy": 0.97904953956604,
+      "num_tokens": 17537116.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.9562997639179229,
+      "epoch": 1.2651413189771197,
+      "grad_norm": 0.8121878504753113,
+      "learning_rate": 9.841606937872632e-05,
+      "loss": 0.05011019706726074,
+      "mean_token_accuracy": 0.9798731088638306,
+      "num_tokens": 17725567.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.9359076261520386,
+      "epoch": 1.278600269179004,
+      "grad_norm": 0.7838549613952637,
+      "learning_rate": 9.836001624553869e-05,
+      "loss": 0.04834386110305786,
+      "mean_token_accuracy": 0.9804529249668121,
+      "num_tokens": 17914191.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.9527446150779724,
+      "epoch": 1.2920592193808882,
+      "grad_norm": 1.0981712341308594,
+      "learning_rate": 9.830300498738152e-05,
+      "loss": 0.055077624320983884,
+      "mean_token_accuracy": 0.9776535391807556,
+      "num_tokens": 18103184.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.9574812948703766,
+      "epoch": 1.3055181695827724,
+      "grad_norm": 0.8291501402854919,
+      "learning_rate": 9.824503673378112e-05,
+      "loss": 0.050153911113739014,
+      "mean_token_accuracy": 0.9792965054512024,
+      "num_tokens": 18291351.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.939585280418396,
+      "epoch": 1.3189771197846567,
+      "grad_norm": 1.0864412784576416,
+      "learning_rate": 9.81861126332241e-05,
+      "loss": 0.05132197737693787,
+      "mean_token_accuracy": 0.9792291462421417,
+      "num_tokens": 18480192.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.9468406856060028,
+      "epoch": 1.332436069986541,
+      "grad_norm": 0.9772785902023315,
+      "learning_rate": 9.812623385313461e-05,
+      "loss": 0.04844954013824463,
+      "mean_token_accuracy": 0.9804142415523529,
+      "num_tokens": 18669028.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.9413834273815155,
+      "epoch": 1.3458950201884252,
+      "grad_norm": 0.9020094871520996,
+      "learning_rate": 9.806540157985131e-05,
+      "loss": 0.05075312852859497,
+      "mean_token_accuracy": 0.9790591120719909,
+      "num_tokens": 18857581.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.9526253461837768,
+      "epoch": 1.3593539703903095,
+      "grad_norm": 1.0524990558624268,
+      "learning_rate": 9.800361701860368e-05,
+      "loss": 0.049737372994422914,
+      "mean_token_accuracy": 0.9795652389526367,
+      "num_tokens": 19046105.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.9484909176826477,
+      "epoch": 1.3728129205921937,
+      "grad_norm": 1.0715620517730713,
+      "learning_rate": 9.794088139348835e-05,
+      "loss": 0.04935494959354401,
+      "mean_token_accuracy": 0.9797740161418915,
+      "num_tokens": 19234730.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.9398573458194732,
+      "epoch": 1.386271870794078,
+      "grad_norm": 1.2134987115859985,
+      "learning_rate": 9.787719594744468e-05,
+      "loss": 0.0518394410610199,
+      "mean_token_accuracy": 0.9793788075447083,
+      "num_tokens": 19423572.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.9465648651123046,
+      "epoch": 1.3997308209959622,
+      "grad_norm": 1.004693627357483,
+      "learning_rate": 9.781256194223023e-05,
+      "loss": 0.04776117205619812,
+      "mean_token_accuracy": 0.9813261866569519,
+      "num_tokens": 19612465.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.9448004186153411,
+      "epoch": 1.4131897711978465,
+      "grad_norm": 1.2913438081741333,
+      "learning_rate": 9.774698065839577e-05,
+      "loss": 0.04807930588722229,
+      "mean_token_accuracy": 0.9805689513683319,
+      "num_tokens": 19800482.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.9425322234630584,
+      "epoch": 1.4266487213997308,
+      "grad_norm": 1.1068469285964966,
+      "learning_rate": 9.768045339525979e-05,
+      "loss": 0.05053595900535583,
+      "mean_token_accuracy": 0.9794536828994751,
+      "num_tokens": 19989444.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.9472232103347779,
+      "epoch": 1.440107671601615,
+      "grad_norm": 0.7409384250640869,
+      "learning_rate": 9.76129814708829e-05,
+      "loss": 0.052136778831481934,
+      "mean_token_accuracy": 0.979197359085083,
+      "num_tokens": 20178482.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.9450975477695465,
+      "epoch": 1.4535666218034993,
+      "grad_norm": 1.3440778255462646,
+      "learning_rate": 9.754456622204167e-05,
+      "loss": 0.05258495211601257,
+      "mean_token_accuracy": 0.9798152863979339,
+      "num_tokens": 20367345.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.9468007743358612,
+      "epoch": 1.4670255720053835,
+      "grad_norm": 0.8377829194068909,
+      "learning_rate": 9.747520900420209e-05,
+      "loss": 0.04893603026866913,
+      "mean_token_accuracy": 0.9805733859539032,
+      "num_tokens": 20555817.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.9505827724933624,
+      "epoch": 1.4804845222072678,
+      "grad_norm": 0.776889979839325,
+      "learning_rate": 9.740491119149277e-05,
+      "loss": 0.05367119312286377,
+      "mean_token_accuracy": 0.9779646098613739,
+      "num_tokens": 20744001.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.9516380190849304,
+      "epoch": 1.493943472409152,
+      "grad_norm": 0.8598276376724243,
+      "learning_rate": 9.733367417667773e-05,
+      "loss": 0.05071394443511963,
+      "mean_token_accuracy": 0.979692417383194,
+      "num_tokens": 20931715.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.9351628720760345,
+      "epoch": 1.5074024226110363,
+      "grad_norm": 1.3307439088821411,
+      "learning_rate": 9.726149937112873e-05,
+      "loss": 0.051372635364532473,
+      "mean_token_accuracy": 0.9797907948493958,
+      "num_tokens": 21120803.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.9268040716648102,
+      "epoch": 1.5208613728129206,
+      "grad_norm": 0.9320250749588013,
+      "learning_rate": 9.718838820479743e-05,
+      "loss": 0.04939974546432495,
+      "mean_token_accuracy": 0.98038170337677,
+      "num_tokens": 21309384.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.9241463243961334,
+      "epoch": 1.5343203230148048,
+      "grad_norm": 1.065938115119934,
+      "learning_rate": 9.711434212618691e-05,
+      "loss": 0.04856643378734589,
+      "mean_token_accuracy": 0.9800511121749877,
+      "num_tokens": 21498234.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.9350252687931061,
+      "epoch": 1.547779273216689,
+      "grad_norm": 0.6925032138824463,
+      "learning_rate": 9.703936260232308e-05,
+      "loss": 0.0512526273727417,
+      "mean_token_accuracy": 0.979562646150589,
+      "num_tokens": 21687119.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.9403733670711517,
+      "epoch": 1.5612382234185733,
+      "grad_norm": 1.2069101333618164,
+      "learning_rate": 9.696345111872557e-05,
+      "loss": 0.049795085191726686,
+      "mean_token_accuracy": 0.9806506633758545,
+      "num_tokens": 21876094.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.9370538294315338,
+      "epoch": 1.5746971736204576,
+      "grad_norm": 1.1759546995162964,
+      "learning_rate": 9.688660917937838e-05,
+      "loss": 0.05461466312408447,
+      "mean_token_accuracy": 0.9776057600975037,
+      "num_tokens": 22064836.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.9517778217792511,
+      "epoch": 1.5881561238223418,
+      "grad_norm": 0.8965149521827698,
+      "learning_rate": 9.68088383066999e-05,
+      "loss": 0.05289326906204224,
+      "mean_token_accuracy": 0.9782088756561279,
+      "num_tokens": 22253065.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.9372851848602295,
+      "epoch": 1.601615074024226,
+      "grad_norm": 0.7192108631134033,
+      "learning_rate": 9.673014004151292e-05,
+      "loss": 0.048973691463470456,
+      "mean_token_accuracy": 0.9806210815906524,
+      "num_tokens": 22441206.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.9122351944446564,
+      "epoch": 1.6150740242261103,
+      "grad_norm": 0.8763614296913147,
+      "learning_rate": 9.665051594301407e-05,
+      "loss": 0.0461783230304718,
+      "mean_token_accuracy": 0.9817197680473327,
+      "num_tokens": 22629637.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.9162654876708984,
+      "epoch": 1.6285329744279946,
+      "grad_norm": 0.9806778430938721,
+      "learning_rate": 9.656996758874284e-05,
+      "loss": 0.04827338755130768,
+      "mean_token_accuracy": 0.9806499361991883,
+      "num_tokens": 22818369.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.9125026524066925,
+      "epoch": 1.6419919246298789,
+      "grad_norm": 0.9685350656509399,
+      "learning_rate": 9.648849657455044e-05,
+      "loss": 0.048379385471343996,
+      "mean_token_accuracy": 0.980905145406723,
+      "num_tokens": 23006863.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.9119791567325592,
+      "epoch": 1.6554508748317631,
+      "grad_norm": 0.8717457056045532,
+      "learning_rate": 9.640610451456811e-05,
+      "loss": 0.050873959064483644,
+      "mean_token_accuracy": 0.979601925611496,
+      "num_tokens": 23195735.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.9244007229804992,
+      "epoch": 1.6689098250336474,
+      "grad_norm": 1.0574654340744019,
+      "learning_rate": 9.632279304117517e-05,
+      "loss": 0.05112813711166382,
+      "mean_token_accuracy": 0.979493772983551,
+      "num_tokens": 23383862.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.9225810945034028,
+      "epoch": 1.6823687752355316,
+      "grad_norm": 0.9628807306289673,
+      "learning_rate": 9.623856380496664e-05,
+      "loss": 0.050480544567108154,
+      "mean_token_accuracy": 0.9797678411006927,
+      "num_tokens": 23572292.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.9287579476833343,
+      "epoch": 1.695827725437416,
+      "grad_norm": 1.015265941619873,
+      "learning_rate": 9.615341847472059e-05,
+      "loss": 0.05133126378059387,
+      "mean_token_accuracy": 0.9797986805438995,
+      "num_tokens": 23760252.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.9133845925331116,
+      "epoch": 1.7092866756393001,
+      "grad_norm": 0.8996883630752563,
+      "learning_rate": 9.606735873736505e-05,
+      "loss": 0.05044950246810913,
+      "mean_token_accuracy": 0.9795303404331207,
+      "num_tokens": 23949031.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.9213698863983154,
+      "epoch": 1.7227456258411844,
+      "grad_norm": 1.0490264892578125,
+      "learning_rate": 9.598038629794461e-05,
+      "loss": 0.04797698855400086,
+      "mean_token_accuracy": 0.9806023716926575,
+      "num_tokens": 24137829.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.9173697710037232,
+      "epoch": 1.7362045760430687,
+      "grad_norm": 1.3648769855499268,
+      "learning_rate": 9.589250287958657e-05,
+      "loss": 0.049194514751434326,
+      "mean_token_accuracy": 0.980712479352951,
+      "num_tokens": 24326586.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.9242741882801055,
+      "epoch": 1.749663526244953,
+      "grad_norm": 1.105125069618225,
+      "learning_rate": 9.580371022346693e-05,
+      "loss": 0.048667973279953,
+      "mean_token_accuracy": 0.9801449298858642,
+      "num_tokens": 24514904.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.9162627995014191,
+      "epoch": 1.7631224764468372,
+      "grad_norm": 0.8887938857078552,
+      "learning_rate": 9.571401008877572e-05,
+      "loss": 0.04878672957420349,
+      "mean_token_accuracy": 0.980469423532486,
+      "num_tokens": 24703688.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.9149538338184356,
+      "epoch": 1.7765814266487214,
+      "grad_norm": 0.8838976621627808,
+      "learning_rate": 9.562340425268233e-05,
+      "loss": 0.04892318844795227,
+      "mean_token_accuracy": 0.9808776795864105,
+      "num_tokens": 24892107.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.914735221862793,
+      "epoch": 1.7900403768506057,
+      "grad_norm": 1.1152070760726929,
+      "learning_rate": 9.553189451030019e-05,
+      "loss": 0.04825109839439392,
+      "mean_token_accuracy": 0.9804604113101959,
+      "num_tokens": 25080130.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.9170725226402283,
+      "epoch": 1.80349932705249,
+      "grad_norm": 0.9479517340660095,
+      "learning_rate": 9.543948267465115e-05,
+      "loss": 0.051445144414901736,
+      "mean_token_accuracy": 0.9792819261550904,
+      "num_tokens": 25268652.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.9142911911010743,
+      "epoch": 1.8169582772543742,
+      "grad_norm": 0.8172292709350586,
+      "learning_rate": 9.534617057662977e-05,
+      "loss": 0.0475692093372345,
+      "mean_token_accuracy": 0.9809505581855774,
+      "num_tokens": 25457120.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.9019249439239502,
+      "epoch": 1.8304172274562585,
+      "grad_norm": 0.8183121681213379,
+      "learning_rate": 9.525196006496679e-05,
+      "loss": 0.04979957342147827,
+      "mean_token_accuracy": 0.9799730658531189,
+      "num_tokens": 25645699.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.9181066155433655,
+      "epoch": 1.8438761776581427,
+      "grad_norm": 0.8256579041481018,
+      "learning_rate": 9.515685300619271e-05,
+      "loss": 0.04996164441108704,
+      "mean_token_accuracy": 0.9802247405052185,
+      "num_tokens": 25834183.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.9171363770961761,
+      "epoch": 1.857335127860027,
+      "grad_norm": 0.9727098345756531,
+      "learning_rate": 9.506085128460065e-05,
+      "loss": 0.048660767078399655,
+      "mean_token_accuracy": 0.9805064260959625,
+      "num_tokens": 26023317.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.9112720847129822,
+      "epoch": 1.8707940780619112,
+      "grad_norm": 0.9826673865318298,
+      "learning_rate": 9.496395680220918e-05,
+      "loss": 0.04775593280792236,
+      "mean_token_accuracy": 0.9809929549694061,
+      "num_tokens": 26212115.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.9034272134304047,
+      "epoch": 1.8842530282637955,
+      "grad_norm": 0.9951306581497192,
+      "learning_rate": 9.486617147872446e-05,
+      "loss": 0.04939360618591308,
+      "mean_token_accuracy": 0.9800347864627839,
+      "num_tokens": 26401167.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.897865754365921,
+      "epoch": 1.8977119784656797,
+      "grad_norm": 1.0163404941558838,
+      "learning_rate": 9.476749725150235e-05,
+      "loss": 0.05002856254577637,
+      "mean_token_accuracy": 0.9793690323829651,
+      "num_tokens": 26590570.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.9004576802253723,
+      "epoch": 1.911170928667564,
+      "grad_norm": 0.6666616201400757,
+      "learning_rate": 9.466793607550995e-05,
+      "loss": 0.04905453026294708,
+      "mean_token_accuracy": 0.9803856253623963,
+      "num_tokens": 26779481.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.8978902101516724,
+      "epoch": 1.9246298788694483,
+      "grad_norm": 0.9295416474342346,
+      "learning_rate": 9.45674899232869e-05,
+      "loss": 0.05261261463165283,
+      "mean_token_accuracy": 0.9785708487033844,
+      "num_tokens": 26968262.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.8962675571441651,
+      "epoch": 1.9380888290713325,
+      "grad_norm": 0.88262939453125,
+      "learning_rate": 9.446616078490626e-05,
+      "loss": 0.04765265882015228,
+      "mean_token_accuracy": 0.98052077293396,
+      "num_tokens": 27157631.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.8909463047981262,
+      "epoch": 1.9515477792732168,
+      "grad_norm": 1.4205769300460815,
+      "learning_rate": 9.436395066793518e-05,
+      "loss": 0.049406200647354126,
+      "mean_token_accuracy": 0.979697072505951,
+      "num_tokens": 27345921.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.8969364166259766,
+      "epoch": 1.965006729475101,
+      "grad_norm": 0.9517145752906799,
+      "learning_rate": 9.426086159739496e-05,
+      "loss": 0.0510346531867981,
+      "mean_token_accuracy": 0.9793386399745941,
+      "num_tokens": 27534751.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.9092479169368743,
+      "epoch": 1.9784656796769853,
+      "grad_norm": 1.200056791305542,
+      "learning_rate": 9.415689561572107e-05,
+      "loss": 0.04974203705787659,
+      "mean_token_accuracy": 0.9800146698951722,
+      "num_tokens": 27723763.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.9059641897678375,
+      "epoch": 1.9919246298788695,
+      "grad_norm": 1.072447657585144,
+      "learning_rate": 9.405205478272267e-05,
+      "loss": 0.05065792202949524,
+      "mean_token_accuracy": 0.9799223959445953,
+      "num_tokens": 27911784.0,
+      "step": 1480
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 0.9083398975384464,
+      "eval_loss": 0.05432562157511711,
+      "eval_mean_token_accuracy": 0.9782073607869969,
+      "eval_num_tokens": 28025286.0,
+      "eval_runtime": 24.4304,
+      "eval_samples_per_second": 204.663,
+      "eval_steps_per_second": 6.426,
+      "step": 1486
+    },
+    {
+      "entropy": 0.9113237977027893,
+      "epoch": 2.005383580080754,
+      "grad_norm": 1.0167063474655151,
+      "learning_rate": 9.394634117554173e-05,
+      "loss": 0.044942992925643924,
+      "mean_token_accuracy": 0.9818171083927154,
+      "num_tokens": 28100906.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.8780008792877197,
+      "epoch": 2.018842530282638,
+      "grad_norm": 1.9552241563796997,
+      "learning_rate": 9.38397568886119e-05,
+      "loss": 0.036248764395713805,
+      "mean_token_accuracy": 0.9852245450019836,
+      "num_tokens": 28289445.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.8721176147460937,
+      "epoch": 2.0323014804845223,
+      "grad_norm": 0.8951388001441956,
+      "learning_rate": 9.373230403361712e-05,
+      "loss": 0.037987279891967776,
+      "mean_token_accuracy": 0.9850614190101623,
+      "num_tokens": 28477972.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.8922657191753387,
+      "epoch": 2.0457604306864066,
+      "grad_norm": 0.9028105735778809,
+      "learning_rate": 9.362398473944958e-05,
+      "loss": 0.03588914275169373,
+      "mean_token_accuracy": 0.9863102614879609,
+      "num_tokens": 28666272.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.877799642086029,
+      "epoch": 2.059219380888291,
+      "grad_norm": 1.153886318206787,
+      "learning_rate": 9.35148011521677e-05,
+      "loss": 0.03577309250831604,
+      "mean_token_accuracy": 0.9863148391246795,
+      "num_tokens": 28854780.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.8708971381187439,
+      "epoch": 2.072678331090175,
+      "grad_norm": 1.042955994606018,
+      "learning_rate": 9.340475543495364e-05,
+      "loss": 0.038266432285308835,
+      "mean_token_accuracy": 0.9848419308662415,
+      "num_tokens": 29043741.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.8800721943378449,
+      "epoch": 2.0861372812920593,
+      "grad_norm": 0.6789060235023499,
+      "learning_rate": 9.329384976807023e-05,
+      "loss": 0.032948991656303404,
+      "mean_token_accuracy": 0.9871293902397156,
+      "num_tokens": 29232719.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.8743580460548401,
+      "epoch": 2.0995962314939436,
+      "grad_norm": 1.1780879497528076,
+      "learning_rate": 9.318208634881802e-05,
+      "loss": 0.036874374747276305,
+      "mean_token_accuracy": 0.9859182178974152,
+      "num_tokens": 29421817.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.8851303637027741,
+      "epoch": 2.113055181695828,
+      "grad_norm": 0.747734785079956,
+      "learning_rate": 9.306946739149161e-05,
+      "loss": 0.0364631175994873,
+      "mean_token_accuracy": 0.9862085223197937,
+      "num_tokens": 29610344.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.8798732101917267,
+      "epoch": 2.126514131897712,
+      "grad_norm": 1.5001860857009888,
+      "learning_rate": 9.29559951273358e-05,
+      "loss": 0.03813003897666931,
+      "mean_token_accuracy": 0.9852604746818543,
+      "num_tokens": 29798997.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.8797551989555359,
+      "epoch": 2.1399730820995964,
+      "grad_norm": 0.9593478441238403,
+      "learning_rate": 9.284167180450141e-05,
+      "loss": 0.0394927829504013,
+      "mean_token_accuracy": 0.984604275226593,
+      "num_tokens": 29987809.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.8860546290874481,
+      "epoch": 2.1534320323014806,
+      "grad_norm": 0.8347703218460083,
+      "learning_rate": 9.272649968800069e-05,
+      "loss": 0.036699697375297546,
+      "mean_token_accuracy": 0.985380882024765,
+      "num_tokens": 30176234.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.8783667802810669,
+      "epoch": 2.166890982503365,
+      "grad_norm": 1.1154481172561646,
+      "learning_rate": 9.26104810596625e-05,
+      "loss": 0.03756999969482422,
+      "mean_token_accuracy": 0.9853451430797577,
+      "num_tokens": 30364657.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.8684478521347045,
+      "epoch": 2.180349932705249,
+      "grad_norm": 0.7515475153923035,
+      "learning_rate": 9.249361821808708e-05,
+      "loss": 0.0382376104593277,
+      "mean_token_accuracy": 0.9854029655456543,
+      "num_tokens": 30552904.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.8636964917182922,
+      "epoch": 2.1938088829071334,
+      "grad_norm": 0.7711939215660095,
+      "learning_rate": 9.237591347860052e-05,
+      "loss": 0.036220991611480714,
+      "mean_token_accuracy": 0.9860713243484497,
+      "num_tokens": 30741259.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.8592016279697419,
+      "epoch": 2.2072678331090176,
+      "grad_norm": 1.1143887042999268,
+      "learning_rate": 9.225736917320886e-05,
+      "loss": 0.036316031217575075,
+      "mean_token_accuracy": 0.985757052898407,
+      "num_tokens": 30930144.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.8671079576015472,
+      "epoch": 2.220726783310902,
+      "grad_norm": 0.8980015516281128,
+      "learning_rate": 9.213798765055187e-05,
+      "loss": 0.03822658061981201,
+      "mean_token_accuracy": 0.9847764372825623,
+      "num_tokens": 31118624.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.8785548269748688,
+      "epoch": 2.234185733512786,
+      "grad_norm": 1.094323992729187,
+      "learning_rate": 9.20177712758566e-05,
+      "loss": 0.03736622333526611,
+      "mean_token_accuracy": 0.9850081980228425,
+      "num_tokens": 31306833.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.866977310180664,
+      "epoch": 2.2476446837146704,
+      "grad_norm": 0.8372092843055725,
+      "learning_rate": 9.189672243089046e-05,
+      "loss": 0.0401554524898529,
+      "mean_token_accuracy": 0.9840337932109833,
+      "num_tokens": 31495503.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.8787918210029602,
+      "epoch": 2.2611036339165547,
+      "grad_norm": 1.5408164262771606,
+      "learning_rate": 9.177484351391402e-05,
+      "loss": 0.0368030846118927,
+      "mean_token_accuracy": 0.9847234487533569,
+      "num_tokens": 31683865.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.872721153497696,
+      "epoch": 2.274562584118439,
+      "grad_norm": 1.1115421056747437,
+      "learning_rate": 9.165213693963355e-05,
+      "loss": 0.037859299778938295,
+      "mean_token_accuracy": 0.9851646661758423,
+      "num_tokens": 31871903.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.8774094223976135,
+      "epoch": 2.288021534320323,
+      "grad_norm": 1.0301331281661987,
+      "learning_rate": 9.152860513915314e-05,
+      "loss": 0.038671016693115234,
+      "mean_token_accuracy": 0.9840058028697968,
+      "num_tokens": 32060500.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.889606237411499,
+      "epoch": 2.3014804845222074,
+      "grad_norm": 0.8601903915405273,
+      "learning_rate": 9.140425055992648e-05,
+      "loss": 0.039603835344314574,
+      "mean_token_accuracy": 0.9840235590934754,
+      "num_tokens": 32248744.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.886734277009964,
+      "epoch": 2.3149394347240917,
+      "grad_norm": 1.1029839515686035,
+      "learning_rate": 9.127907566570853e-05,
+      "loss": 0.039513933658599856,
+      "mean_token_accuracy": 0.9844573020935059,
+      "num_tokens": 32437640.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.8843017637729644,
+      "epoch": 2.328398384925976,
+      "grad_norm": 1.2545154094696045,
+      "learning_rate": 9.115308293650653e-05,
+      "loss": 0.036970189213752745,
+      "mean_token_accuracy": 0.985239815711975,
+      "num_tokens": 32625986.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.8913422048091888,
+      "epoch": 2.34185733512786,
+      "grad_norm": 0.9613803625106812,
+      "learning_rate": 9.102627486853099e-05,
+      "loss": 0.03811657428741455,
+      "mean_token_accuracy": 0.9852804243564606,
+      "num_tokens": 32814531.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.8907467782497406,
+      "epoch": 2.3553162853297445,
+      "grad_norm": 1.1811398267745972,
+      "learning_rate": 9.089865397414614e-05,
+      "loss": 0.03903660774230957,
+      "mean_token_accuracy": 0.9842524945735931,
+      "num_tokens": 33002960.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.8833664715290069,
+      "epoch": 2.3687752355316287,
+      "grad_norm": 0.8338477611541748,
+      "learning_rate": 9.077022278182024e-05,
+      "loss": 0.03982087969779968,
+      "mean_token_accuracy": 0.9841201484203339,
+      "num_tokens": 33191183.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.8783826470375061,
+      "epoch": 2.382234185733513,
+      "grad_norm": 0.6904510259628296,
+      "learning_rate": 9.064098383607545e-05,
+      "loss": 0.03699290752410889,
+      "mean_token_accuracy": 0.9854120731353759,
+      "num_tokens": 33379798.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.8871048510074615,
+      "epoch": 2.3956931359353972,
+      "grad_norm": 1.009539246559143,
+      "learning_rate": 9.051093969743738e-05,
+      "loss": 0.03926805555820465,
+      "mean_token_accuracy": 0.9843246698379516,
+      "num_tokens": 33568531.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.8973462700843811,
+      "epoch": 2.409152086137281,
+      "grad_norm": 0.8916401267051697,
+      "learning_rate": 9.03800929423844e-05,
+      "loss": 0.039596831798553465,
+      "mean_token_accuracy": 0.9839228212833404,
+      "num_tokens": 33756333.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.8954619467258453,
+      "epoch": 2.4226110363391653,
+      "grad_norm": 1.056504487991333,
+      "learning_rate": 9.024844616329662e-05,
+      "loss": 0.0396859347820282,
+      "mean_token_accuracy": 0.9837161123752594,
+      "num_tokens": 33944904.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.8952643811702728,
+      "epoch": 2.4360699865410496,
+      "grad_norm": 1.0442084074020386,
+      "learning_rate": 9.011600196840447e-05,
+      "loss": 0.037324142456054685,
+      "mean_token_accuracy": 0.9852897703647614,
+      "num_tokens": 34133530.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.8874911010265351,
+      "epoch": 2.449528936742934,
+      "grad_norm": 0.9926224946975708,
+      "learning_rate": 8.998276298173707e-05,
+      "loss": 0.03637495338916778,
+      "mean_token_accuracy": 0.9853426337242126,
+      "num_tokens": 34322042.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.8900584518909455,
+      "epoch": 2.462987886944818,
+      "grad_norm": 1.001734733581543,
+      "learning_rate": 8.984873184307017e-05,
+      "loss": 0.04029585719108582,
+      "mean_token_accuracy": 0.9842372059822082,
+      "num_tokens": 34510259.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.8986345052719116,
+      "epoch": 2.4764468371467023,
+      "grad_norm": 0.772941529750824,
+      "learning_rate": 8.971391120787397e-05,
+      "loss": 0.0401084691286087,
+      "mean_token_accuracy": 0.9841427087783814,
+      "num_tokens": 34698669.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.8911370873451233,
+      "epoch": 2.4899057873485866,
+      "grad_norm": 0.895031213760376,
+      "learning_rate": 8.957830374726042e-05,
+      "loss": 0.03941032290458679,
+      "mean_token_accuracy": 0.9841966688632965,
+      "num_tokens": 34887160.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.8901963472366333,
+      "epoch": 2.503364737550471,
+      "grad_norm": 1.0677918195724487,
+      "learning_rate": 8.944191214793028e-05,
+      "loss": 0.03503885865211487,
+      "mean_token_accuracy": 0.9856241464614868,
+      "num_tokens": 35075891.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.8817630231380462,
+      "epoch": 2.516823687752355,
+      "grad_norm": 1.1200164556503296,
+      "learning_rate": 8.930473911212e-05,
+      "loss": 0.03878425657749176,
+      "mean_token_accuracy": 0.9846492648124695,
+      "num_tokens": 35264455.0,
+      "step": 1870
+    },
+    {
+      "entropy": 0.8897477686405182,
+      "epoch": 2.5302826379542394,
+      "grad_norm": 0.8058190941810608,
+      "learning_rate": 8.916678735754809e-05,
+      "loss": 0.041558006405830385,
+      "mean_token_accuracy": 0.9830883860588073,
+      "num_tokens": 35453251.0,
+      "step": 1880
+    },
+    {
+      "entropy": 0.8873181045055389,
+      "epoch": 2.5437415881561236,
+      "grad_norm": 0.7765992283821106,
+      "learning_rate": 8.902805961736123e-05,
+      "loss": 0.03555050492286682,
+      "mean_token_accuracy": 0.9858225345611572,
+      "num_tokens": 35641959.0,
+      "step": 1890
+    },
+    {
+      "entropy": 0.8782606959342957,
+      "epoch": 2.557200538358008,
+      "grad_norm": 0.8488866686820984,
+      "learning_rate": 8.88885586400803e-05,
+      "loss": 0.03606923818588257,
+      "mean_token_accuracy": 0.9858632445335388,
+      "num_tokens": 35829636.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.8869808673858642,
+      "epoch": 2.570659488559892,
+      "grad_norm": 1.1734739542007446,
+      "learning_rate": 8.874828718954576e-05,
+      "loss": 0.040956351161003116,
+      "mean_token_accuracy": 0.9835374176502227,
+      "num_tokens": 36017818.0,
+      "step": 1910
+    },
+    {
+      "entropy": 0.8850924432277679,
+      "epoch": 2.5841184387617764,
+      "grad_norm": 1.3584387302398682,
+      "learning_rate": 8.86072480448629e-05,
+      "loss": 0.042437011003494264,
+      "mean_token_accuracy": 0.9826479077339172,
+      "num_tokens": 36206505.0,
+      "step": 1920
+    },
+    {
+      "entropy": 0.8801687896251679,
+      "epoch": 2.5975773889636606,
+      "grad_norm": 1.2862240076065063,
+      "learning_rate": 8.84654440003469e-05,
+      "loss": 0.03880060315132141,
+      "mean_token_accuracy": 0.9843404710292816,
+      "num_tokens": 36395802.0,
+      "step": 1930
+    },
+    {
+      "entropy": 0.8752781748771667,
+      "epoch": 2.611036339165545,
+      "grad_norm": 0.7341249585151672,
+      "learning_rate": 8.83228778654674e-05,
+      "loss": 0.03776344656944275,
+      "mean_token_accuracy": 0.9852877616882324,
+      "num_tokens": 36584370.0,
+      "step": 1940
+    },
+    {
+      "entropy": 0.8759494423866272,
+      "epoch": 2.624495289367429,
+      "grad_norm": 0.9807276129722595,
+      "learning_rate": 8.817955246479276e-05,
+      "loss": 0.03780297338962555,
+      "mean_token_accuracy": 0.9853227376937866,
+      "num_tokens": 36772865.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.8753129065036773,
+      "epoch": 2.6379542395693134,
+      "grad_norm": 1.1391628980636597,
+      "learning_rate": 8.803547063793422e-05,
+      "loss": 0.03953765034675598,
+      "mean_token_accuracy": 0.9839197635650635,
+      "num_tokens": 36961462.0,
+      "step": 1960
+    },
+    {
+      "entropy": 0.8771899223327637,
+      "epoch": 2.6514131897711977,
+      "grad_norm": 1.0865391492843628,
+      "learning_rate": 8.789063523948958e-05,
+      "loss": 0.03856399655342102,
+      "mean_token_accuracy": 0.9848688066005706,
+      "num_tokens": 37150486.0,
+      "step": 1970
+    },
+    {
+      "entropy": 0.8774881184101104,
+      "epoch": 2.664872139973082,
+      "grad_norm": 0.8561595678329468,
+      "learning_rate": 8.774504913898663e-05,
+      "loss": 0.03795175850391388,
+      "mean_token_accuracy": 0.9849309325218201,
+      "num_tokens": 37338890.0,
+      "step": 1980
+    },
+    {
+      "entropy": 0.8841517448425293,
+      "epoch": 2.678331090174966,
+      "grad_norm": 0.9458399415016174,
+      "learning_rate": 8.75987152208264e-05,
+      "loss": 0.03860213160514832,
+      "mean_token_accuracy": 0.9842230796813964,
+      "num_tokens": 37526886.0,
+      "step": 1990
+    },
+    {
+      "entropy": 0.8809043228626251,
+      "epoch": 2.6917900403768504,
+      "grad_norm": 0.9360871315002441,
+      "learning_rate": 8.745163638422583e-05,
+      "loss": 0.03709094822406769,
+      "mean_token_accuracy": 0.9855118036270142,
+      "num_tokens": 37715408.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.8805519282817841,
+      "epoch": 2.7052489905787347,
+      "grad_norm": 0.8852124214172363,
+      "learning_rate": 8.730381554316051e-05,
+      "loss": 0.03869341611862183,
+      "mean_token_accuracy": 0.9846724212169647,
+      "num_tokens": 37904328.0,
+      "step": 2010
+    },
+    {
+      "entropy": 0.8796228408813477,
+      "epoch": 2.718707940780619,
+      "grad_norm": 1.189297080039978,
+      "learning_rate": 8.715525562630687e-05,
+      "loss": 0.03675769567489624,
+      "mean_token_accuracy": 0.9854080140590668,
+      "num_tokens": 38092776.0,
+      "step": 2020
+    },
+    {
+      "entropy": 0.8722020089626312,
+      "epoch": 2.732166890982503,
+      "grad_norm": 0.7027947306632996,
+      "learning_rate": 8.700595957698411e-05,
+      "loss": 0.03889244794845581,
+      "mean_token_accuracy": 0.984558242559433,
+      "num_tokens": 38282221.0,
+      "step": 2030
+    },
+    {
+      "entropy": 0.8746632814407349,
+      "epoch": 2.7456258411843875,
+      "grad_norm": 0.9799376726150513,
+      "learning_rate": 8.685593035309598e-05,
+      "loss": 0.03815680146217346,
+      "mean_token_accuracy": 0.9845012128353119,
+      "num_tokens": 38470633.0,
+      "step": 2040
+    },
+    {
+      "entropy": 0.8766567528247833,
+      "epoch": 2.7590847913862717,
+      "grad_norm": 0.850459098815918,
+      "learning_rate": 8.670517092707213e-05,
+      "loss": 0.039029371738433835,
+      "mean_token_accuracy": 0.9845906794071198,
+      "num_tokens": 38659048.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.8778688013553619,
+      "epoch": 2.772543741588156,
+      "grad_norm": 0.8540134429931641,
+      "learning_rate": 8.655368428580919e-05,
+      "loss": 0.039645448327064514,
+      "mean_token_accuracy": 0.9840566098690033,
+      "num_tokens": 38847370.0,
+      "step": 2060
+    },
+    {
+      "entropy": 0.8758600771427154,
+      "epoch": 2.7860026917900402,
+      "grad_norm": 0.8036021590232849,
+      "learning_rate": 8.640147343061165e-05,
+      "loss": 0.038226932287216187,
+      "mean_token_accuracy": 0.9845647215843201,
+      "num_tokens": 39036040.0,
+      "step": 2070
+    },
+    {
+      "entropy": 0.8652268946170807,
+      "epoch": 2.7994616419919245,
+      "grad_norm": 0.8526738286018372,
+      "learning_rate": 8.624854137713234e-05,
+      "loss": 0.03798363208770752,
+      "mean_token_accuracy": 0.9845202267169952,
+      "num_tokens": 39224695.0,
+      "step": 2080
+    },
+    {
+      "entropy": 0.8707200348377228,
+      "epoch": 2.8129205921938087,
+      "grad_norm": 0.9852389097213745,
+      "learning_rate": 8.609489115531278e-05,
+      "loss": 0.037227436900138855,
+      "mean_token_accuracy": 0.9852050960063934,
+      "num_tokens": 39413360.0,
+      "step": 2090
+    },
+    {
+      "entropy": 0.8711447060108185,
+      "epoch": 2.826379542395693,
+      "grad_norm": 1.228277325630188,
+      "learning_rate": 8.594052580932301e-05,
+      "loss": 0.03698050379753113,
+      "mean_token_accuracy": 0.9852443158626556,
+      "num_tokens": 39602544.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.8742912471294403,
+      "epoch": 2.8398384925975773,
+      "grad_norm": 0.8440544605255127,
+      "learning_rate": 8.578544839750141e-05,
+      "loss": 0.03757670521736145,
+      "mean_token_accuracy": 0.9848618268966675,
+      "num_tokens": 39791340.0,
+      "step": 2110
+    },
+    {
+      "entropy": 0.8801726162433624,
+      "epoch": 2.8532974427994615,
+      "grad_norm": 1.0955252647399902,
+      "learning_rate": 8.562966199229399e-05,
+      "loss": 0.040004149079322815,
+      "mean_token_accuracy": 0.9841345012187958,
+      "num_tokens": 39980212.0,
+      "step": 2120
+    },
+    {
+      "entropy": 0.8931492984294891,
+      "epoch": 2.8667563930013458,
+      "grad_norm": 0.5722652673721313,
+      "learning_rate": 8.547316968019363e-05,
+      "loss": 0.038096648454666135,
+      "mean_token_accuracy": 0.9849136590957641,
+      "num_tokens": 40169122.0,
+      "step": 2130
+    },
+    {
+      "entropy": 0.8767493844032288,
+      "epoch": 2.88021534320323,
+      "grad_norm": 0.7663730382919312,
+      "learning_rate": 8.531597456167885e-05,
+      "loss": 0.03729096055030823,
+      "mean_token_accuracy": 0.985169780254364,
+      "num_tokens": 40357678.0,
+      "step": 2140
+    },
+    {
+      "entropy": 0.8770561516284943,
+      "epoch": 2.8936742934051143,
+      "grad_norm": 1.1505672931671143,
+      "learning_rate": 8.515807975115239e-05,
+      "loss": 0.03984796404838562,
+      "mean_token_accuracy": 0.9838845670223236,
+      "num_tokens": 40546892.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.8789129912853241,
+      "epoch": 2.9071332436069985,
+      "grad_norm": 0.8355761766433716,
+      "learning_rate": 8.499948837687959e-05,
+      "loss": 0.03769223690032959,
+      "mean_token_accuracy": 0.9847559213638306,
+      "num_tokens": 40736109.0,
+      "step": 2160
+    },
+    {
+      "entropy": 0.8781779289245606,
+      "epoch": 2.920592193808883,
+      "grad_norm": 0.9103918075561523,
+      "learning_rate": 8.484020358092625e-05,
+      "loss": 0.038262826204299924,
+      "mean_token_accuracy": 0.984741848707199,
+      "num_tokens": 40924341.0,
+      "step": 2170
+    },
+    {
+      "entropy": 0.8552807629108429,
+      "epoch": 2.934051144010767,
+      "grad_norm": 0.8338510990142822,
+      "learning_rate": 8.468022851909657e-05,
+      "loss": 0.0355743408203125,
+      "mean_token_accuracy": 0.9855036079883576,
+      "num_tokens": 41113849.0,
+      "step": 2180
+    },
+    {
+      "entropy": 0.8705591142177582,
+      "epoch": 2.9475100942126513,
+      "grad_norm": 0.8334057927131653,
+      "learning_rate": 8.451956636087046e-05,
+      "loss": 0.037476515769958495,
+      "mean_token_accuracy": 0.9854551255702972,
+      "num_tokens": 41303130.0,
+      "step": 2190
+    },
+    {
+      "entropy": 0.8809274792671203,
+      "epoch": 2.9609690444145356,
+      "grad_norm": 0.9424103498458862,
+      "learning_rate": 8.435822028934087e-05,
+      "loss": 0.03624279499053955,
+      "mean_token_accuracy": 0.9856135487556458,
+      "num_tokens": 41491750.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.8818133771419525,
+      "epoch": 2.97442799461642,
+      "grad_norm": 1.04243004322052,
+      "learning_rate": 8.41961935011506e-05,
+      "loss": 0.039132320880889894,
+      "mean_token_accuracy": 0.984746390581131,
+      "num_tokens": 41680493.0,
+      "step": 2210
+    },
+    {
+      "entropy": 0.8838395297527313,
+      "epoch": 2.987886944818304,
+      "grad_norm": 0.8650784492492676,
+      "learning_rate": 8.403348920642911e-05,
+      "loss": 0.03822052478790283,
+      "mean_token_accuracy": 0.9852108955383301,
+      "num_tokens": 41868378.0,
+      "step": 2220
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 0.8800126549544608,
+      "eval_loss": 0.05468416586518288,
+      "eval_mean_token_accuracy": 0.9781015617832257,
+      "eval_num_tokens": 42038037.0,
+      "eval_runtime": 24.4825,
+      "eval_samples_per_second": 204.228,
+      "eval_steps_per_second": 6.413,
+      "step": 2229
+    },
+    {
+      "entropy": 0.8773060619831086,
+      "epoch": 3.0013458950201883,
+      "grad_norm": 0.7845072150230408,
+      "learning_rate": 8.387011062872883e-05,
+      "loss": 0.0365300714969635,
+      "mean_token_accuracy": 0.985853761434555,
+      "num_tokens": 42056717.0,
+      "step": 2230
+    },
+    {
+      "entropy": 0.8720550239086151,
+      "epoch": 3.0148048452220726,
+      "grad_norm": 0.9917275905609131,
+      "learning_rate": 8.370606100496128e-05,
+      "loss": 0.030071893334388734,
+      "mean_token_accuracy": 0.9890478909015655,
+      "num_tokens": 42245058.0,
+      "step": 2240
+    },
+    {
+      "entropy": 0.8489451467990875,
+      "epoch": 3.028263795423957,
+      "grad_norm": 0.9414294362068176,
+      "learning_rate": 8.354134358533301e-05,
+      "loss": 0.027054613828659056,
+      "mean_token_accuracy": 0.9894806027412415,
+      "num_tokens": 42433468.0,
+      "step": 2250
+    },
+    {
+      "entropy": 0.848755830526352,
+      "epoch": 3.041722745625841,
+      "grad_norm": 0.7276235818862915,
+      "learning_rate": 8.337596163328114e-05,
+      "loss": 0.026897600293159483,
+      "mean_token_accuracy": 0.9893308103084564,
+      "num_tokens": 42621657.0,
+      "step": 2260
+    },
+    {
+      "entropy": 0.8402929604053497,
+      "epoch": 3.0551816958277254,
+      "grad_norm": 0.9455838799476624,
+      "learning_rate": 8.320991842540875e-05,
+      "loss": 0.027918827533721925,
+      "mean_token_accuracy": 0.9888385117053986,
+      "num_tokens": 42810533.0,
+      "step": 2270
+    },
+    {
+      "entropy": 0.8434606611728668,
+      "epoch": 3.0686406460296096,
+      "grad_norm": 0.7503429651260376,
+      "learning_rate": 8.304321725141995e-05,
+      "loss": 0.025769710540771484,
+      "mean_token_accuracy": 0.989881956577301,
+      "num_tokens": 42999085.0,
+      "step": 2280
+    },
+    {
+      "entropy": 0.8459890782833099,
+      "epoch": 3.082099596231494,
+      "grad_norm": 0.736440896987915,
+      "learning_rate": 8.287586141405464e-05,
+      "loss": 0.028781753778457642,
+      "mean_token_accuracy": 0.9882656812667847,
+      "num_tokens": 43188222.0,
+      "step": 2290
+    },
+    {
+      "entropy": 0.8493306159973144,
+      "epoch": 3.095558546433378,
+      "grad_norm": 0.9577816128730774,
+      "learning_rate": 8.27078542290232e-05,
+      "loss": 0.02689719796180725,
+      "mean_token_accuracy": 0.9889744818210602,
+      "num_tokens": 43376648.0,
+      "step": 2300
+    },
+    {
+      "entropy": 0.8512665331363678,
+      "epoch": 3.1090174966352624,
+      "grad_norm": 0.9631588459014893,
+      "learning_rate": 8.253919902494071e-05,
+      "loss": 0.031439167261123654,
+      "mean_token_accuracy": 0.9874942421913147,
+      "num_tokens": 43564769.0,
+      "step": 2310
+    },
+    {
+      "entropy": 0.8519513130187988,
+      "epoch": 3.1224764468371466,
+      "grad_norm": 1.1011922359466553,
+      "learning_rate": 8.236989914326101e-05,
+      "loss": 0.02696958780288696,
+      "mean_token_accuracy": 0.9900951981544495,
+      "num_tokens": 43753362.0,
+      "step": 2320
+    },
+    {
+      "entropy": 0.8507618844509125,
+      "epoch": 3.135935397039031,
+      "grad_norm": 0.9351623058319092,
+      "learning_rate": 8.21999579382105e-05,
+      "loss": 0.026315575838088988,
+      "mean_token_accuracy": 0.989511913061142,
+      "num_tokens": 43942063.0,
+      "step": 2330
+    },
+    {
+      "entropy": 0.8506478369235992,
+      "epoch": 3.149394347240915,
+      "grad_norm": 1.096564769744873,
+      "learning_rate": 8.202937877672175e-05,
+      "loss": 0.029049152135849,
+      "mean_token_accuracy": 0.9886228442192078,
+      "num_tokens": 44130381.0,
+      "step": 2340
+    },
+    {
+      "entropy": 0.8624941289424897,
+      "epoch": 3.1628532974427994,
+      "grad_norm": 0.7859402298927307,
+      "learning_rate": 8.185816503836665e-05,
+      "loss": 0.027744990587234498,
+      "mean_token_accuracy": 0.9890527307987214,
+      "num_tokens": 44319243.0,
+      "step": 2350
+    },
+    {
+      "entropy": 0.8627866744995117,
+      "epoch": 3.1763122476446837,
+      "grad_norm": 0.7764605283737183,
+      "learning_rate": 8.168632011528961e-05,
+      "loss": 0.02571650445461273,
+      "mean_token_accuracy": 0.9903442323207855,
+      "num_tokens": 44507698.0,
+      "step": 2360
+    },
+    {
+      "entropy": 0.8566473722457886,
+      "epoch": 3.189771197846568,
+      "grad_norm": 0.8182396292686462,
+      "learning_rate": 8.15138474121403e-05,
+      "loss": 0.025625643134117127,
+      "mean_token_accuracy": 0.9897150516510009,
+      "num_tokens": 44695954.0,
+      "step": 2370
+    },
+    {
+      "entropy": 0.8555088877677918,
+      "epoch": 3.203230148048452,
+      "grad_norm": 1.0277098417282104,
+      "learning_rate": 8.134075034600609e-05,
+      "loss": 0.028235083818435668,
+      "mean_token_accuracy": 0.9894341349601745,
+      "num_tokens": 44884558.0,
+      "step": 2380
+    },
+    {
+      "entropy": 0.8562702238559723,
+      "epoch": 3.2166890982503364,
+      "grad_norm": 1.0676493644714355,
+      "learning_rate": 8.116703234634453e-05,
+      "loss": 0.027713948488235475,
+      "mean_token_accuracy": 0.9891053259372711,
+      "num_tokens": 45073381.0,
+      "step": 2390
+    },
+    {
+      "entropy": 0.8520576059818268,
+      "epoch": 3.2301480484522207,
+      "grad_norm": 0.7926117181777954,
+      "learning_rate": 8.099269685491528e-05,
+      "loss": 0.028936928510665892,
+      "mean_token_accuracy": 0.988807988166809,
+      "num_tokens": 45262350.0,
+      "step": 2400
+    },
+    {
+      "entropy": 0.860496312379837,
+      "epoch": 3.243606998654105,
+      "grad_norm": 1.1108510494232178,
+      "learning_rate": 8.081774732571196e-05,
+      "loss": 0.030533093214035033,
+      "mean_token_accuracy": 0.9882748067378998,
+      "num_tokens": 45450782.0,
+      "step": 2410
+    },
+    {
+      "entropy": 0.872324675321579,
+      "epoch": 3.257065948855989,
+      "grad_norm": 1.3057194948196411,
+      "learning_rate": 8.06421872248937e-05,
+      "loss": 0.032421305775642395,
+      "mean_token_accuracy": 0.9879753351211548,
+      "num_tokens": 45639122.0,
+      "step": 2420
+    },
+    {
+      "entropy": 0.8778823137283325,
+      "epoch": 3.2705248990578735,
+      "grad_norm": 1.289680004119873,
+      "learning_rate": 8.046602003071648e-05,
+      "loss": 0.03096393346786499,
+      "mean_token_accuracy": 0.9883370697498322,
+      "num_tokens": 45827730.0,
+      "step": 2430
+    },
+    {
+      "entropy": 0.8627406179904937,
+      "epoch": 3.2839838492597577,
+      "grad_norm": 0.8361398577690125,
+      "learning_rate": 8.028924923346426e-05,
+      "loss": 0.02957139313220978,
+      "mean_token_accuracy": 0.9884674072265625,
+      "num_tokens": 46016274.0,
+      "step": 2440
+    },
+    {
+      "entropy": 0.8584554970264435,
+      "epoch": 3.297442799461642,
+      "grad_norm": 0.9330723881721497,
+      "learning_rate": 8.011187833537972e-05,
+      "loss": 0.029568272829055785,
+      "mean_token_accuracy": 0.9882686555385589,
+      "num_tokens": 46204679.0,
+      "step": 2450
+    },
+    {
+      "entropy": 0.8515178024768829,
+      "epoch": 3.3109017496635262,
+      "grad_norm": 1.0296216011047363,
+      "learning_rate": 7.993391085059502e-05,
+      "loss": 0.027788180112838744,
+      "mean_token_accuracy": 0.9888734877109527,
+      "num_tokens": 46393989.0,
+      "step": 2460
+    },
+    {
+      "entropy": 0.8488850116729736,
+      "epoch": 3.3243606998654105,
+      "grad_norm": 1.1329699754714966,
+      "learning_rate": 7.975535030506203e-05,
+      "loss": 0.028057461977005003,
+      "mean_token_accuracy": 0.9895590543746948,
+      "num_tokens": 46582790.0,
+      "step": 2470
+    },
+    {
+      "entropy": 0.849625188112259,
+      "epoch": 3.3378196500672948,
+      "grad_norm": 0.9267219305038452,
+      "learning_rate": 7.957620023648256e-05,
+      "loss": 0.02802973985671997,
+      "mean_token_accuracy": 0.988851535320282,
+      "num_tokens": 46771138.0,
+      "step": 2480
+    },
+    {
+      "entropy": 0.854446280002594,
+      "epoch": 3.351278600269179,
+      "grad_norm": 0.8713932037353516,
+      "learning_rate": 7.939646419423826e-05,
+      "loss": 0.030220019817352294,
+      "mean_token_accuracy": 0.9882843673229218,
+      "num_tokens": 46960054.0,
+      "step": 2490
+    },
+    {
+      "entropy": 0.8599806547164917,
+      "epoch": 3.3647375504710633,
+      "grad_norm": 0.675019383430481,
+      "learning_rate": 7.92161457393203e-05,
+      "loss": 0.02958865463733673,
+      "mean_token_accuracy": 0.9880296885967255,
+      "num_tokens": 47149567.0,
+      "step": 2500
+    },
+    {
+      "entropy": 0.8549504458904267,
+      "epoch": 3.3781965006729475,
+      "grad_norm": 1.2176353931427002,
+      "learning_rate": 7.903524844425878e-05,
+      "loss": 0.027722162008285523,
+      "mean_token_accuracy": 0.9890090465545655,
+      "num_tokens": 47338060.0,
+      "step": 2510
+    },
+    {
+      "entropy": 0.8552233815193176,
+      "epoch": 3.391655450874832,
+      "grad_norm": 0.9571717977523804,
+      "learning_rate": 7.885377589305197e-05,
+      "loss": 0.028924965858459474,
+      "mean_token_accuracy": 0.9884298503398895,
+      "num_tokens": 47525895.0,
+      "step": 2520
+    },
+    {
+      "entropy": 0.8521630108356476,
+      "epoch": 3.405114401076716,
+      "grad_norm": 0.965713620185852,
+      "learning_rate": 7.867173168109534e-05,
+      "loss": 0.030511701107025148,
+      "mean_token_accuracy": 0.9881398677825928,
+      "num_tokens": 47714524.0,
+      "step": 2530
+    },
+    {
+      "entropy": 0.8537400126457214,
+      "epoch": 3.4185733512786003,
+      "grad_norm": 0.9005307555198669,
+      "learning_rate": 7.84891194151103e-05,
+      "loss": 0.030432754755020143,
+      "mean_token_accuracy": 0.9876708447933197,
+      "num_tokens": 47903188.0,
+      "step": 2540
+    },
+    {
+      "entropy": 0.8596058666706086,
+      "epoch": 3.4320323014804845,
+      "grad_norm": 0.8159075379371643,
+      "learning_rate": 7.830594271307267e-05,
+      "loss": 0.027927806973457335,
+      "mean_token_accuracy": 0.9889210939407349,
+      "num_tokens": 48091334.0,
+      "step": 2550
+    },
+    {
+      "entropy": 0.8482735753059387,
+      "epoch": 3.445491251682369,
+      "grad_norm": 1.0411661863327026,
+      "learning_rate": 7.812220520414115e-05,
+      "loss": 0.03003154993057251,
+      "mean_token_accuracy": 0.9882730603218078,
+      "num_tokens": 48279622.0,
+      "step": 2560
+    },
+    {
+      "entropy": 0.8466426789760589,
+      "epoch": 3.458950201884253,
+      "grad_norm": 1.189595341682434,
+      "learning_rate": 7.793791052858528e-05,
+      "loss": 0.03237656056880951,
+      "mean_token_accuracy": 0.9869072616100312,
+      "num_tokens": 48467527.0,
+      "step": 2570
+    },
+    {
+      "entropy": 0.8545588493347168,
+      "epoch": 3.4724091520861373,
+      "grad_norm": 1.0700056552886963,
+      "learning_rate": 7.775306233771343e-05,
+      "loss": 0.0306060791015625,
+      "mean_token_accuracy": 0.9880664825439454,
+      "num_tokens": 48656151.0,
+      "step": 2580
+    },
+    {
+      "entropy": 0.8503433585166931,
+      "epoch": 3.4858681022880216,
+      "grad_norm": 0.7099478244781494,
+      "learning_rate": 7.756766429380033e-05,
+      "loss": 0.02984013557434082,
+      "mean_token_accuracy": 0.9879442811012268,
+      "num_tokens": 48844796.0,
+      "step": 2590
+    },
+    {
+      "entropy": 0.8464376091957092,
+      "epoch": 3.499327052489906,
+      "grad_norm": 0.9877128005027771,
+      "learning_rate": 7.738172007001465e-05,
+      "loss": 0.0306446373462677,
+      "mean_token_accuracy": 0.9883591532707214,
+      "num_tokens": 49033216.0,
+      "step": 2600
+    },
+    {
+      "entropy": 0.8477609276771545,
+      "epoch": 3.51278600269179,
+      "grad_norm": 1.3324531316757202,
+      "learning_rate": 7.719523335034612e-05,
+      "loss": 0.030148410797119142,
+      "mean_token_accuracy": 0.9879989802837372,
+      "num_tokens": 49221425.0,
+      "step": 2610
+    },
+    {
+      "entropy": 0.8461863219738006,
+      "epoch": 3.5262449528936743,
+      "grad_norm": 0.8938570022583008,
+      "learning_rate": 7.70082078295326e-05,
+      "loss": 0.02682304084300995,
+      "mean_token_accuracy": 0.9897881150245667,
+      "num_tokens": 49409785.0,
+      "step": 2620
+    },
+    {
+      "entropy": 0.8362689018249512,
+      "epoch": 3.5397039030955586,
+      "grad_norm": 0.7423873543739319,
+      "learning_rate": 7.682064721298683e-05,
+      "loss": 0.02689332664012909,
+      "mean_token_accuracy": 0.9899903059005737,
+      "num_tokens": 49598524.0,
+      "step": 2630
+    },
+    {
+      "entropy": 0.8395139217376709,
+      "epoch": 3.553162853297443,
+      "grad_norm": 0.921755850315094,
+      "learning_rate": 7.663255521672308e-05,
+      "loss": 0.030062124133110046,
+      "mean_token_accuracy": 0.9880835592746735,
+      "num_tokens": 49787241.0,
+      "step": 2640
+    },
+    {
+      "entropy": 0.8467226803302765,
+      "epoch": 3.566621803499327,
+      "grad_norm": 1.0296459197998047,
+      "learning_rate": 7.64439355672835e-05,
+      "loss": 0.030666223168373107,
+      "mean_token_accuracy": 0.9877098739147187,
+      "num_tokens": 49975844.0,
+      "step": 2650
+    },
+    {
+      "entropy": 0.833392071723938,
+      "epoch": 3.5800807537012114,
+      "grad_norm": 1.2861948013305664,
+      "learning_rate": 7.625479200166425e-05,
+      "loss": 0.028174835443496703,
+      "mean_token_accuracy": 0.989056545495987,
+      "num_tokens": 50164857.0,
+      "step": 2660
+    },
+    {
+      "entropy": 0.8340730309486389,
+      "epoch": 3.5935397039030956,
+      "grad_norm": 0.8116776347160339,
+      "learning_rate": 7.606512826724155e-05,
+      "loss": 0.028390336036682128,
+      "mean_token_accuracy": 0.9886897742748261,
+      "num_tokens": 50353452.0,
+      "step": 2670
+    },
+    {
+      "entropy": 0.8357116639614105,
+      "epoch": 3.60699865410498,
+      "grad_norm": 0.8362084031105042,
+      "learning_rate": 7.587494812169728e-05,
+      "loss": 0.029806244373321533,
+      "mean_token_accuracy": 0.988309508562088,
+      "num_tokens": 50542461.0,
+      "step": 2680
+    },
+    {
+      "entropy": 0.8363270878791809,
+      "epoch": 3.620457604306864,
+      "grad_norm": 0.8438499569892883,
+      "learning_rate": 7.568425533294476e-05,
+      "loss": 0.028577303886413573,
+      "mean_token_accuracy": 0.9886945366859436,
+      "num_tokens": 50730872.0,
+      "step": 2690
+    },
+    {
+      "entropy": 0.8310261785984039,
+      "epoch": 3.6339165545087484,
+      "grad_norm": 1.1920002698898315,
+      "learning_rate": 7.549305367905385e-05,
+      "loss": 0.030321845412254335,
+      "mean_token_accuracy": 0.9878708899021149,
+      "num_tokens": 50919889.0,
+      "step": 2700
+    },
+    {
+      "entropy": 0.8380252480506897,
+      "epoch": 3.6473755047106327,
+      "grad_norm": 0.9409399032592773,
+      "learning_rate": 7.53013469481763e-05,
+      "loss": 0.030501589179039,
+      "mean_token_accuracy": 0.9872083842754364,
+      "num_tokens": 51108079.0,
+      "step": 2710
+    },
+    {
+      "entropy": 0.8422612130641938,
+      "epoch": 3.660834454912517,
+      "grad_norm": 0.8370440006256104,
+      "learning_rate": 7.510913893847058e-05,
+      "loss": 0.02974823713302612,
+      "mean_token_accuracy": 0.9885554671287536,
+      "num_tokens": 51296913.0,
+      "step": 2720
+    },
+    {
+      "entropy": 0.8387595832347869,
+      "epoch": 3.674293405114401,
+      "grad_norm": 1.1360642910003662,
+      "learning_rate": 7.491643345802667e-05,
+      "loss": 0.026970452070236205,
+      "mean_token_accuracy": 0.9888331353664398,
+      "num_tokens": 51485678.0,
+      "step": 2730
+    },
+    {
+      "entropy": 0.8403031289577484,
+      "epoch": 3.6877523553162854,
+      "grad_norm": 1.1195677518844604,
+      "learning_rate": 7.472323432479062e-05,
+      "loss": 0.029451662302017213,
+      "mean_token_accuracy": 0.988847154378891,
+      "num_tokens": 51674765.0,
+      "step": 2740
+    },
+    {
+      "entropy": 0.8499106168746948,
+      "epoch": 3.7012113055181697,
+      "grad_norm": 0.9573526978492737,
+      "learning_rate": 7.452954536648888e-05,
+      "loss": 0.030826866626739502,
+      "mean_token_accuracy": 0.9875578820705414,
+      "num_tokens": 51863924.0,
+      "step": 2750
+    },
+    {
+      "entropy": 0.8576709866523743,
+      "epoch": 3.714670255720054,
+      "grad_norm": 0.9386286735534668,
+      "learning_rate": 7.433537042055248e-05,
+      "loss": 0.02946900427341461,
+      "mean_token_accuracy": 0.9885939955711365,
+      "num_tokens": 52052011.0,
+      "step": 2760
+    },
+    {
+      "entropy": 0.8404816091060638,
+      "epoch": 3.728129205921938,
+      "grad_norm": 1.095536708831787,
+      "learning_rate": 7.414071333404104e-05,
+      "loss": 0.030070826411247253,
+      "mean_token_accuracy": 0.9881400763988495,
+      "num_tokens": 52240955.0,
+      "step": 2770
+    },
+    {
+      "entropy": 0.8346689164638519,
+      "epoch": 3.7415881561238225,
+      "grad_norm": 0.9964426159858704,
+      "learning_rate": 7.394557796356644e-05,
+      "loss": 0.03005717396736145,
+      "mean_token_accuracy": 0.9883057296276092,
+      "num_tokens": 52429544.0,
+      "step": 2780
+    },
+    {
+      "entropy": 0.8377436757087707,
+      "epoch": 3.7550471063257067,
+      "grad_norm": 0.8499778509140015,
+      "learning_rate": 7.374996817521653e-05,
+      "loss": 0.029283815622329713,
+      "mean_token_accuracy": 0.9882387280464172,
+      "num_tokens": 52618405.0,
+      "step": 2790
+    },
+    {
+      "entropy": 0.8328788638114929,
+      "epoch": 3.768506056527591,
+      "grad_norm": 0.9594390988349915,
+      "learning_rate": 7.35538878444785e-05,
+      "loss": 0.02666555345058441,
+      "mean_token_accuracy": 0.9889323055744171,
+      "num_tokens": 52806689.0,
+      "step": 2800
+    },
+    {
+      "entropy": 0.829415249824524,
+      "epoch": 3.781965006729475,
+      "grad_norm": 1.3587099313735962,
+      "learning_rate": 7.335734085616206e-05,
+      "loss": 0.028349629044532774,
+      "mean_token_accuracy": 0.9890907645225525,
+      "num_tokens": 52995248.0,
+      "step": 2810
+    },
+    {
+      "entropy": 0.8335795283317566,
+      "epoch": 3.7954239569313595,
+      "grad_norm": 1.077789306640625,
+      "learning_rate": 7.316033110432249e-05,
+      "loss": 0.03001731038093567,
+      "mean_token_accuracy": 0.9878358781337738,
+      "num_tokens": 53183607.0,
+      "step": 2820
+    },
+    {
+      "entropy": 0.8316770553588867,
+      "epoch": 3.8088829071332437,
+      "grad_norm": 0.7265951037406921,
+      "learning_rate": 7.296286249218352e-05,
+      "loss": 0.030136841535568237,
+      "mean_token_accuracy": 0.9880174934864044,
+      "num_tokens": 53372735.0,
+      "step": 2830
+    },
+    {
+      "entropy": 0.8310399711132049,
+      "epoch": 3.822341857335128,
+      "grad_norm": 0.6605978012084961,
+      "learning_rate": 7.276493893205995e-05,
+      "loss": 0.026665833592414857,
+      "mean_token_accuracy": 0.9892704486846924,
+      "num_tokens": 53561776.0,
+      "step": 2840
+    },
+    {
+      "entropy": 0.8258654296398162,
+      "epoch": 3.8358008075370122,
+      "grad_norm": 0.6560797691345215,
+      "learning_rate": 7.256656434528018e-05,
+      "loss": 0.026718440651893615,
+      "mean_token_accuracy": 0.989847868680954,
+      "num_tokens": 53750019.0,
+      "step": 2850
+    },
+    {
+      "entropy": 0.8221413195133209,
+      "epoch": 3.8492597577388965,
+      "grad_norm": 0.7495356798171997,
+      "learning_rate": 7.236774266210852e-05,
+      "loss": 0.027431756258010864,
+      "mean_token_accuracy": 0.9892430484294892,
+      "num_tokens": 53938914.0,
+      "step": 2860
+    },
+    {
+      "entropy": 0.8341928541660308,
+      "epoch": 3.8627187079407808,
+      "grad_norm": 1.0676971673965454,
+      "learning_rate": 7.216847782166727e-05,
+      "loss": 0.028432759642601012,
+      "mean_token_accuracy": 0.9889024317264556,
+      "num_tokens": 54127763.0,
+      "step": 2870
+    },
+    {
+      "entropy": 0.8359149813652038,
+      "epoch": 3.876177658142665,
+      "grad_norm": 1.380476713180542,
+      "learning_rate": 7.196877377185872e-05,
+      "loss": 0.028149163722991942,
+      "mean_token_accuracy": 0.9884953200817108,
+      "num_tokens": 54315930.0,
+      "step": 2880
+    },
+    {
+      "entropy": 0.8294597804546356,
+      "epoch": 3.8896366083445493,
+      "grad_norm": 1.0728843212127686,
+      "learning_rate": 7.176863446928694e-05,
+      "loss": 0.028733691573143004,
+      "mean_token_accuracy": 0.9884523034095765,
+      "num_tokens": 54504632.0,
+      "step": 2890
+    },
+    {
+      "entropy": 0.8245523810386658,
+      "epoch": 3.9030955585464335,
+      "grad_norm": 0.9723307490348816,
+      "learning_rate": 7.156806387917937e-05,
+      "loss": 0.02793261408805847,
+      "mean_token_accuracy": 0.9891038477420807,
+      "num_tokens": 54693752.0,
+      "step": 2900
+    },
+    {
+      "entropy": 0.8279620766639709,
+      "epoch": 3.916554508748318,
+      "grad_norm": 1.003624439239502,
+      "learning_rate": 7.136706597530825e-05,
+      "loss": 0.028367668390274048,
+      "mean_token_accuracy": 0.9893603324890137,
+      "num_tokens": 54882030.0,
+      "step": 2910
+    },
+    {
+      "entropy": 0.8306584179401397,
+      "epoch": 3.930013458950202,
+      "grad_norm": 0.8481361865997314,
+      "learning_rate": 7.116564473991192e-05,
+      "loss": 0.029952478408813477,
+      "mean_token_accuracy": 0.9881332099437714,
+      "num_tokens": 55070549.0,
+      "step": 2920
+    },
+    {
+      "entropy": 0.8316469848155975,
+      "epoch": 3.9434724091520863,
+      "grad_norm": 0.7631049156188965,
+      "learning_rate": 7.096380416361588e-05,
+      "loss": 0.029351180791854857,
+      "mean_token_accuracy": 0.9888184070587158,
+      "num_tokens": 55258999.0,
+      "step": 2930
+    },
+    {
+      "entropy": 0.8221287310123444,
+      "epoch": 3.9569313593539706,
+      "grad_norm": 0.7234840393066406,
+      "learning_rate": 7.076154824535381e-05,
+      "loss": 0.027709537744522096,
+      "mean_token_accuracy": 0.9890308439731598,
+      "num_tokens": 55448047.0,
+      "step": 2940
+    },
+    {
+      "entropy": 0.8239192545413971,
+      "epoch": 3.970390309555855,
+      "grad_norm": 0.8497020602226257,
+      "learning_rate": 7.055888099228825e-05,
+      "loss": 0.029604345560073853,
+      "mean_token_accuracy": 0.9881375730037689,
+      "num_tokens": 55636566.0,
+      "step": 2950
+    },
+    {
+      "entropy": 0.8319927275180816,
+      "epoch": 3.983849259757739,
+      "grad_norm": 1.1599552631378174,
+      "learning_rate": 7.035580641973119e-05,
+      "loss": 0.02763860821723938,
+      "mean_token_accuracy": 0.9891406774520874,
+      "num_tokens": 55825049.0,
+      "step": 2960
+    },
+    {
+      "entropy": 0.8404937744140625,
+      "epoch": 3.9973082099596233,
+      "grad_norm": 0.906755805015564,
+      "learning_rate": 7.015232855106468e-05,
+      "loss": 0.029578033089637756,
+      "mean_token_accuracy": 0.9878996431827545,
+      "num_tokens": 56012965.0,
+      "step": 2970
+    },
+    {
+      "epoch": 4.0,
+      "eval_entropy": 0.8419649528849656,
+      "eval_loss": 0.0570383295416832,
+      "eval_mean_token_accuracy": 0.9787763862093543,
+      "eval_num_tokens": 56050518.0,
+      "eval_runtime": 24.4889,
+      "eval_samples_per_second": 204.174,
+      "eval_steps_per_second": 6.411,
+      "step": 2972
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7430,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.7083082838982e+18,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

baby_talk_L16_a50/seed_42/checkpoint-2972/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c060e97b69d99564c146471c3d3ac4d335e3b1968074124f4edc5aebf612e1e3
+size 5368

baby_talk_L16_a50/seed_42/checkpoint-3715/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

baby_talk_L16_a50/seed_42/checkpoint-3715/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

baby_talk_L16_a50/seed_42/checkpoint-3715/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76034427b07f0dba7df52bb96914473f77c4a53c8e7d021b0affc3de7384c88d
+size 80792096

baby_talk_L16_a50/seed_42/checkpoint-3715/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

baby_talk_L16_a50/seed_42/checkpoint-3715/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

baby_talk_L16_a50/seed_42/checkpoint-3715/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

baby_talk_L16_a50/seed_42/checkpoint-3715/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c060e97b69d99564c146471c3d3ac4d335e3b1968074124f4edc5aebf612e1e3
+size 5368

baby_talk_L16_a50/seed_42/checkpoint-4458/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

baby_talk_L16_a50/seed_42/checkpoint-4458/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

baby_talk_L16_a50/seed_42/checkpoint-4458/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a8b6fd518dab5e24f6b28262e81c749857962f0080cb5998e7b9aa57c5eb01a
+size 80792096

baby_talk_L16_a50/seed_42/checkpoint-4458/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

baby_talk_L16_a50/seed_42/checkpoint-4458/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

baby_talk_L16_a50/seed_42/checkpoint-4458/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

baby_talk_L16_a50/seed_42/checkpoint-4458/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c060e97b69d99564c146471c3d3ac4d335e3b1968074124f4edc5aebf612e1e3
+size 5368

baby_talk_L16_a50/seed_42/checkpoint-5201/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen2.5-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B-Instruct
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

baby_talk_L16_a50/seed_42/checkpoint-5201/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

baby_talk_L16_a50/seed_42/checkpoint-5201/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88d555221f6e593a18115c5e50a575aea4bc5817d56745baecbe8d07df0e6cea
+size 80792096

baby_talk_L16_a50/seed_42/checkpoint-5201/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}