Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-1250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-1875/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-625/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,62 @@
+---
+base_model: google/gemma-4-E2B
+library_name: peft
+model_name: gemma4-code-assistant
+tags:
+- base_model:adapter:google/gemma-4-E2B
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for gemma4-code-assistant
+
+This model is a fine-tuned version of [google/gemma-4-E2B](https://huggingface.co/google/gemma-4-E2B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 1.0.0
+- Transformers: 5.6.0.dev0
+- Pytorch: 2.10.0+cu128
+- Datasets: 4.8.4
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

adapter_config.json

@@ -0,0 +1,296 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-4-E2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "lm_head",
+    "embed_tokens"
+  ],
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "26.self_attn.q_proj",
+    "language_model.layers.11.mlp.gate_proj",
+    "34.mlp.gate_proj",
+    "23.mlp.gate_proj",
+    "language_model.layers.5.mlp.gate_proj",
+    "input_proj",
+    "language_model.layers.12.self_attn.q_proj",
+    "32.mlp.down_proj",
+    "20.mlp.up_proj",
+    "language_model.layers.4.self_attn.v_proj",
+    "20.self_attn.k_proj",
+    "input_proj_linear",
+    "24.mlp.gate_proj",
+    "29.self_attn.k_proj",
+    "21.self_attn.v_proj",
+    "21.mlp.down_proj",
+    "16.self_attn.v_proj",
+    "language_model.layers.13.mlp.up_proj",
+    "24.mlp.down_proj",
+    "22.mlp.up_proj",
+    "language_model.layers.9.self_attn.k_proj",
+    "26.mlp.up_proj",
+    "language_model.layers.7.mlp.down_proj",
+    "language_model.layers.2.mlp.gate_proj",
+    "language_model.layers.13.self_attn.k_proj",
+    "25.self_attn.q_proj",
+    "per_layer_model_projection",
+    "18.mlp.up_proj",
+    "language_model.layers.13.self_attn.v_proj",
+    "34.mlp.down_proj",
+    "23.mlp.up_proj",
+    "23.self_attn.o_proj",
+    "24.self_attn.v_proj",
+    "language_model.layers.6.self_attn.q_proj",
+    "29.self_attn.q_proj",
+    "19.self_attn.k_proj",
+    "24.self_attn.k_proj",
+    "language_model.layers.5.self_attn.q_proj",
+    "32.mlp.gate_proj",
+    "language_model.layers.4.mlp.gate_proj",
+    "33.mlp.gate_proj",
+    "19.self_attn.o_proj",
+    "34.self_attn.v_proj",
+    "25.mlp.down_proj",
+    "language_model.layers.8.mlp.up_proj",
+    "language_model.layers.12.mlp.down_proj",
+    "language_model.layers.3.mlp.down_proj",
+    "language_model.layers.7.self_attn.k_proj",
+    "language_model.layers.2.mlp.up_proj",
+    "language_model.layers.13.mlp.down_proj",
+    "28.mlp.gate_proj",
+    "language_model.layers.15.self_attn.v_proj",
+    "20.self_attn.q_proj",
+    "per_layer_projection",
+    "27.mlp.gate_proj",
+    "language_model.layers.11.self_attn.o_proj",
+    "language_model.layers.15.mlp.gate_proj",
+    "30.self_attn.o_proj",
+    "32.self_attn.o_proj",
+    "27.mlp.down_proj",
+    "language_model.layers.1.mlp.gate_proj",
+    "21.self_attn.o_proj",
+    "32.self_attn.q_proj",
+    "language_model.layers.7.self_attn.v_proj",
+    "21.mlp.gate_proj",
+    "16.self_attn.k_proj",
+    "20.mlp.down_proj",
+    "16.mlp.gate_proj",
+    "20.mlp.gate_proj",
+    "language_model.layers.7.mlp.gate_proj",
+    "34.self_attn.q_proj",
+    "31.mlp.down_proj",
+    "language_model.layers.8.mlp.gate_proj",
+    "language_model.layers.9.mlp.up_proj",
+    "language_model.layers.15.self_attn.o_proj",
+    "28.self_attn.o_proj",
+    "32.self_attn.v_proj",
+    "32.self_attn.k_proj",
+    "language_model.layers.4.self_attn.q_proj",
+    "28.self_attn.v_proj",
+    "28.self_attn.k_proj",
+    "26.self_attn.k_proj",
+    "34.self_attn.k_proj",
+    "16.mlp.up_proj",
+    "21.self_attn.q_proj",
+    "17.self_attn.q_proj",
+    "language_model.layers.11.self_attn.k_proj",
+    "24.mlp.up_proj",
+    "23.self_attn.q_proj",
+    "language_model.layers.3.mlp.up_proj",
+    "language_model.layers.12.self_attn.k_proj",
+    "language_model.layers.1.mlp.down_proj",
+    "20.self_attn.v_proj",
+    "language_model.layers.11.mlp.up_proj",
+    "language_model.layers.0.mlp.gate_proj",
+    "language_model.layers.5.self_attn.v_proj",
+    "language_model.layers.6.self_attn.o_proj",
+    "language_model.layers.11.mlp.down_proj",
+    "31.mlp.gate_proj",
+    "31.self_attn.q_proj",
+    "22.self_attn.v_proj",
+    "31.self_attn.v_proj",
+    "25.mlp.up_proj",
+    "language_model.layers.9.self_attn.q_proj",
+    "27.self_attn.o_proj",
+    "17.mlp.up_proj",
+    "25.self_attn.k_proj",
+    "language_model.layers.1.self_attn.o_proj",
+    "19.mlp.down_proj",
+    "language_model.layers.14.self_attn.q_proj",
+    "31.self_attn.k_proj",
+    "language_model.layers.14.mlp.down_proj",
+    "language_model.layers.2.self_attn.v_proj",
+    "language_model.layers.9.mlp.down_proj",
+    "26.mlp.down_proj",
+    "30.self_attn.q_proj",
+    "31.self_attn.o_proj",
+    "18.self_attn.v_proj",
+    "33.self_attn.q_proj",
+    "language_model.layers.2.self_attn.q_proj",
+    "30.mlp.up_proj",
+    "language_model.layers.4.mlp.down_proj",
+    "29.mlp.down_proj",
+    "19.self_attn.q_proj",
+    "language_model.layers.1.self_attn.v_proj",
+    "language_model.layers.10.self_attn.o_proj",
+    "27.self_attn.k_proj",
+    "embedding_projection",
+    "24.self_attn.q_proj",
+    "33.self_attn.v_proj",
+    "29.mlp.up_proj",
+    "25.self_attn.v_proj",
+    "language_model.layers.11.self_attn.q_proj",
+    "33.self_attn.k_proj",
+    "language_model.layers.5.self_attn.o_proj",
+    "language_model.layers.4.self_attn.k_proj",
+    "language_model.layers.3.self_attn.o_proj",
+    "21.self_attn.k_proj",
+    "language_model.layers.8.mlp.down_proj",
+    "language_model.layers.2.self_attn.o_proj",
+    "20.self_attn.o_proj",
+    "language_model.layers.0.self_attn.k_proj",
+    "linear",
+    "language_model.layers.10.self_attn.v_proj",
+    "17.mlp.down_proj",
+    "21.mlp.up_proj",
+    "language_model.layers.2.mlp.down_proj",
+    "22.mlp.gate_proj",
+    "language_model.layers.9.self_attn.o_proj",
+    "language_model.layers.0.self_attn.q_proj",
+    "language_model.layers.14.self_attn.k_proj",
+    "18.mlp.gate_proj",
+    "language_model.layers.3.self_attn.q_proj",
+    "language_model.layers.7.self_attn.q_proj",
+    "language_model.layers.15.mlp.up_proj",
+    "34.self_attn.o_proj",
+    "language_model.layers.8.self_attn.q_proj",
+    "language_model.layers.10.mlp.up_proj",
+    "language_model.layers.14.mlp.up_proj",
+    "29.mlp.gate_proj",
+    "language_model.layers.1.self_attn.k_proj",
+    "language_model.layers.13.self_attn.o_proj",
+    "25.self_attn.o_proj",
+    "17.self_attn.v_proj",
+    "30.mlp.gate_proj",
+    "26.self_attn.v_proj",
+    "language_model.layers.3.self_attn.v_proj",
+    "19.self_attn.v_proj",
+    "language_model.layers.0.mlp.down_proj",
+    "language_model.layers.3.self_attn.k_proj",
+    "18.self_attn.o_proj",
+    "29.self_attn.o_proj",
+    "25.mlp.gate_proj",
+    "language_model.layers.9.mlp.gate_proj",
+    "16.self_attn.o_proj",
+    "language_model.layers.12.self_attn.o_proj",
+    "33.self_attn.o_proj",
+    "output_proj",
+    "22.self_attn.o_proj",
+    "17.self_attn.o_proj",
+    "23.self_attn.v_proj",
+    "language_model.layers.12.mlp.gate_proj",
+    "22.self_attn.q_proj",
+    "relative_k_proj",
+    "30.mlp.down_proj",
+    "22.mlp.down_proj",
+    "31.mlp.up_proj",
+    "18.self_attn.k_proj",
+    "language_model.layers.1.mlp.up_proj",
+    "28.mlp.down_proj",
+    "language_model.layers.15.self_attn.q_proj",
+    "language_model.layers.10.mlp.gate_proj",
+    "language_model.layers.0.self_attn.v_proj",
+    "23.mlp.down_proj",
+    "29.self_attn.v_proj",
+    "language_model.layers.8.self_attn.v_proj",
+    "22.self_attn.k_proj",
+    "17.mlp.gate_proj",
+    "24.self_attn.o_proj",
+    "16.mlp.down_proj",
+    "33.mlp.down_proj",
+    "language_model.layers.2.self_attn.k_proj",
+    "23.self_attn.k_proj",
+    "30.self_attn.v_proj",
+    "language_model.layers.4.self_attn.o_proj",
+    "language_model.layers.1.self_attn.q_proj",
+    "language_model.layers.7.mlp.up_proj",
+    "language_model.layers.5.self_attn.k_proj",
+    "19.mlp.up_proj",
+    "per_layer_input_gate",
+    "language_model.layers.0.mlp.up_proj",
+    "30.self_attn.k_proj",
+    "language_model.layers.8.self_attn.k_proj",
+    "18.self_attn.q_proj",
+    "language_model.layers.4.mlp.up_proj",
+    "19.mlp.gate_proj",
+    "language_model.layers.0.self_attn.o_proj",
+    "language_model.layers.12.self_attn.v_proj",
+    "34.mlp.up_proj",
+    "language_model.layers.13.self_attn.q_proj",
+    "27.mlp.up_proj",
+    "26.mlp.gate_proj",
+    "language_model.layers.6.self_attn.k_proj",
+    "26.self_attn.o_proj",
+    "language_model.layers.10.self_attn.q_proj",
+    "language_model.layers.10.self_attn.k_proj",
+    "language_model.layers.10.mlp.down_proj",
+    "33.mlp.up_proj",
+    "language_model.layers.6.self_attn.v_proj",
+    "16.self_attn.q_proj",
+    "27.self_attn.v_proj",
+    "28.mlp.up_proj",
+    "language_model.layers.15.mlp.down_proj",
+    "language_model.layers.14.self_attn.o_proj",
+    "language_model.layers.5.mlp.up_proj",
+    "language_model.layers.13.mlp.gate_proj",
+    "language_model.layers.14.mlp.gate_proj",
+    "language_model.layers.11.self_attn.v_proj",
+    "language_model.layers.14.self_attn.v_proj",
+    "18.mlp.down_proj",
+    "17.self_attn.k_proj",
+    "language_model.layers.7.self_attn.o_proj",
+    "32.mlp.up_proj",
+    "language_model.layers.6.mlp.up_proj",
+    "language_model.layers.9.self_attn.v_proj",
+    "language_model.layers.3.mlp.gate_proj",
+    "language_model.layers.6.mlp.down_proj",
+    "language_model.layers.12.mlp.up_proj",
+    "language_model.layers.6.mlp.gate_proj",
+    "28.self_attn.q_proj",
+    "language_model.layers.15.self_attn.k_proj",
+    "language_model.layers.8.self_attn.o_proj",
+    "language_model.layers.5.mlp.down_proj",
+    "27.self_attn.q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:189b5596e8b491dd4a55648ab528a50d09009808e7d6767eef9a99df5d2f42e4
+size 1688992024

chat_template.jinja

@@ -0,0 +1,263 @@
+{%- macro format_parameters(properties, required) -%}
+    {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
+    {%- set ns = namespace(found_first=false) -%}
+    {%- for key, value in properties | dictsort -%}
+        {%- set add_comma = false -%}
+        {%- if key not in standard_keys -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {{ key }}:{
+            {%- if value['description'] -%}
+                description:<|"|>{{ value['description'] }}<|"|>
+                {%- set add_comma = true -%}
+            {%- endif -%}
+            {%- if value['nullable'] %}
+                {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                nullable:true
+            {%- endif -%}
+            {%- if value['type'] | upper == 'STRING' -%}
+                {%- if value['enum'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    enum:{{ format_argument(value['enum']) }}
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'OBJECT' -%}
+                ,properties:{
+                {%- if value['properties'] is defined and value['properties'] is mapping -%}
+                    {{- format_parameters(value['properties'], value['required'] | default([])) -}}
+                {%- elif value is mapping -%}
+                    {{- format_parameters(value, value['required'] | default([])) -}}
+                {%- endif -%}
+                }
+                {%- if value['required'] -%}
+                    ,required:[
+                    {%- for item in value['required'] | default([]) -%}
+                        <|"|>{{- item -}}<|"|>
+                        {%- if not loop.last %},{% endif -%}
+                    {%- endfor -%}
+                    ]
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'ARRAY' -%}
+                {%- if value['items'] is mapping and value['items'] -%}
+                    ,items:{
+                    {%- set ns_items = namespace(found_first=false) -%}
+                    {%- for item_key, item_value in value['items'] | dictsort -%}
+                        {%- if item_value is not none -%}
+                            {%- if ns_items.found_first %},{% endif -%}
+                            {%- set ns_items.found_first = true -%}
+                            {%- if item_key == 'properties' -%}
+                                properties:{
+                                {%- if item_value is mapping -%}
+                                    {{- format_parameters(item_value, value['items']['required'] | default([])) -}}
+                                {%- endif -%}
+                                }
+                            {%- elif item_key == 'required' -%}
+                                required:[
+                                {%- for req_item in item_value -%}
+                                    <|"|>{{- req_item -}}<|"|>
+                                    {%- if not loop.last %},{% endif -%}
+                                {%- endfor -%}
+                                ]
+                            {%- elif item_key == 'type' -%}
+                                {%- if item_value is string -%}
+                                    type:{{ format_argument(item_value | upper) }}
+                                {%- else -%}
+                                    type:{{ format_argument(item_value | map('upper') | list) }}
+                                {%- endif -%}
+                            {%- else -%}
+                                {{ item_key }}:{{ format_argument(item_value) }}
+                            {%- endif -%}
+                        {%- endif -%}
+                    {%- endfor -%}
+                    }
+                {%- endif -%}
+            {%- endif -%}
+            {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+            type:<|"|>{{ value['type'] | upper }}<|"|>}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endmacro -%}
+{%- macro format_function_declaration(tool_data) -%}
+    declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
+    {%- set params = tool_data['function']['parameters'] -%}
+    {%- if params -%}
+        ,parameters:{
+        {%- if params['properties'] -%}
+            properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
+        {%- endif -%}
+        {%- if params['required'] -%}
+            required:[
+            {%- for item in params['required'] -%}
+                <|"|>{{- item -}}<|"|>
+                {{- ',' if not loop.last -}}
+            {%- endfor -%}
+            ],
+        {%- endif -%}
+        {%- if params['type'] -%}
+            type:<|"|>{{- params['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    {%- if 'response' in tool_data['function'] -%}
+        {%- set response_declaration = tool_data['function']['response'] -%}
+        ,response:{
+        {%- if response_declaration['description'] -%}
+            description:<|"|>{{- response_declaration['description'] -}}<|"|>,
+        {%- endif -%}
+        {%- if response_declaration['type'] | upper == 'OBJECT' -%}
+            type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    }
+{%- endmacro -%}
+{%- macro format_argument(argument, escape_keys=True) -%}
+    {%- if argument is string -%}
+        {{- '<|"|>' + argument + '<|"|>' -}}
+    {%- elif argument is boolean -%}
+        {{- 'true' if argument else 'false' -}}
+    {%- elif argument is mapping -%}
+        {{- '{' -}}
+        {%- set ns = namespace(found_first=false) -%}
+        {%- for key, value in argument | dictsort -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {%- if escape_keys -%}
+                {{- '<|"|>' + key + '<|"|>' -}}
+            {%- else -%}
+                {{- key -}}
+            {%- endif -%}
+            :{{- format_argument(value, escape_keys=escape_keys) -}}
+        {%- endfor -%}
+        {{- '}' -}}
+    {%- elif argument is sequence -%}
+        {{- '[' -}}
+        {%- for item in argument -%}
+            {{- format_argument(item, escape_keys=escape_keys) -}}
+            {%- if not loop.last %},{% endif -%}
+        {%- endfor -%}
+        {{- ']' -}}
+    {%- else -%}
+        {{- argument -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- macro strip_thinking(text) -%}
+    {%- set ns = namespace(result='') -%}
+    {%- for part in text.split('<channel|>') -%}
+        {%- if '<|channel>' in part -%}
+            {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
+        {%- else -%}
+            {%- set ns.result = ns.result + part -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- ns.result | trim -}}
+{%- endmacro -%}
+
+{%- set ns = namespace(prev_message_type=None) -%}
+{%- set loop_messages = messages -%}
+{{ bos_token }}
+{#- Handle System/Tool Definitions Block -#}
+{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}
+    {{- '<|turn>system\n' -}}
+
+    {#- Inject Thinking token at the very top of the FIRST system turn -#}
+    {%- if enable_thinking is defined and enable_thinking -%}
+        {{- '<|think|>' -}}
+        {%- set ns.prev_message_type = 'think' -%}
+    {%- endif -%}
+
+    {%- if messages[0]['role'] in ['system', 'developer'] -%}
+        {{- messages[0]['content'] | trim -}}
+        {%- set loop_messages = messages[1:] -%}
+    {%- endif -%}
+
+    {%- if tools -%}
+        {%- for tool in tools %}
+            {{- '<|tool>' -}}
+            {{- format_function_declaration(tool) | trim -}}
+            {{- '<tool|>' -}}
+        {%- endfor %}
+        {%- set ns.prev_message_type = 'tool' -%}
+    {%- endif -%}
+
+    {{- '<turn|>\n' -}}
+{%- endif %}
+
+{#- Loop through messages -#}
+{%- for message in loop_messages -%}
+    {%- set ns.prev_message_type = None -%}
+    {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
+        {{- '<|turn>' + role + '\n' }}
+
+            {%- if message['tool_calls'] -%}
+                {%- for tool_call in message['tool_calls'] -%}
+                    {%- set function = tool_call['function'] -%}
+                    {{- '<|tool_call>call:' + function['name'] + '{' -}}
+                    {%- if function['arguments'] is mapping -%}
+                        {%- set ns_args = namespace(found_first=false) -%}
+                        {%- for key, value in function['arguments'] | dictsort -%}
+                            {%- if ns_args.found_first %},{% endif -%}
+                            {%- set ns_args.found_first = true -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                        {%- endfor -%}
+                    {%- elif function['arguments'] is string -%}
+                        {{- function['arguments'] -}}
+                    {%- endif -%}
+                    {{- '}<tool_call|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_call' -%}
+            {%- endif -%}
+
+            {%- if message['tool_responses'] -%}
+                {#- Tool Response handling -#}
+                {%- for tool_response in message['tool_responses'] -%}
+                    {{- '<|tool_response>' -}}
+                    {%- if tool_response['response'] is mapping -%}
+                        {{- 'response:' + tool_response['name'] | default('unknown') + '{' -}}
+                        {%- for key, value in tool_response['response'] | dictsort -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                            {%- if not loop.last %},{% endif -%}
+                        {%- endfor -%}
+                        {{- '}' -}}
+                    {%- else -%}
+                        {{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}}
+                    {%- endif -%}
+                    {{- '<tool_response|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_response' -%}
+            {%- endif -%}
+
+            {%- if message['content'] is string -%}
+                {%- if role == 'model' -%}
+                    {{- strip_thinking(message['content']) -}}
+                {%- else -%}
+                    {{- message['content'] | trim -}}
+                {%- endif -%}
+            {%- elif message['content'] is sequence -%}
+                {%- for item in message['content'] -%}
+                    {%- if item['type'] == 'text' -%}
+                        {%- if role == 'model' -%}
+                            {{- strip_thinking(item['text']) -}}
+                        {%- else -%}
+                            {{- item['text'] | trim -}}
+                        {%- endif -%}
+                    {%- elif item['type'] == 'image' -%}
+                        {{- '\n\n<|image|>\n\n' -}}
+                        {%- set ns.prev_message_type = 'image' -%}
+                    {%- elif item['type'] == 'audio' -%}
+                        {{- '<|audio|>' -}}
+                        {%- set ns.prev_message_type = 'audio' -%}
+                    {%- elif item['type'] == 'video' -%}
+                        {{- '\n\n<|video|>\n\n' -}}
+                        {%- set ns.prev_message_type = 'video' -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+
+        {%- if not (message['tool_responses'] and not message['content']) -%}
+            {{- '<turn|>\n' -}}
+        {%- endif -%}
+{%- endfor -%}
+
+{%- if add_generation_prompt -%}
+    {%- if ns.prev_message_type != 'tool_response' -%}
+        {{- '<|turn>model\n' -}}
+    {%- endif -%}
+{%- endif -%}

checkpoint-1250/README.md

@@ -0,0 +1,209 @@
+---
+base_model: google/gemma-4-E2B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:google/gemma-4-E2B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-1250/adapter_config.json

@@ -0,0 +1,296 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-4-E2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "lm_head",
+    "embed_tokens"
+  ],
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "26.self_attn.q_proj",
+    "language_model.layers.11.mlp.gate_proj",
+    "34.mlp.gate_proj",
+    "23.mlp.gate_proj",
+    "language_model.layers.5.mlp.gate_proj",
+    "input_proj",
+    "language_model.layers.12.self_attn.q_proj",
+    "32.mlp.down_proj",
+    "20.mlp.up_proj",
+    "language_model.layers.4.self_attn.v_proj",
+    "20.self_attn.k_proj",
+    "input_proj_linear",
+    "24.mlp.gate_proj",
+    "29.self_attn.k_proj",
+    "21.self_attn.v_proj",
+    "21.mlp.down_proj",
+    "16.self_attn.v_proj",
+    "language_model.layers.13.mlp.up_proj",
+    "24.mlp.down_proj",
+    "22.mlp.up_proj",
+    "language_model.layers.9.self_attn.k_proj",
+    "26.mlp.up_proj",
+    "language_model.layers.7.mlp.down_proj",
+    "language_model.layers.2.mlp.gate_proj",
+    "language_model.layers.13.self_attn.k_proj",
+    "25.self_attn.q_proj",
+    "per_layer_model_projection",
+    "18.mlp.up_proj",
+    "language_model.layers.13.self_attn.v_proj",
+    "34.mlp.down_proj",
+    "23.mlp.up_proj",
+    "23.self_attn.o_proj",
+    "24.self_attn.v_proj",
+    "language_model.layers.6.self_attn.q_proj",
+    "29.self_attn.q_proj",
+    "19.self_attn.k_proj",
+    "24.self_attn.k_proj",
+    "language_model.layers.5.self_attn.q_proj",
+    "32.mlp.gate_proj",
+    "language_model.layers.4.mlp.gate_proj",
+    "33.mlp.gate_proj",
+    "19.self_attn.o_proj",
+    "34.self_attn.v_proj",
+    "25.mlp.down_proj",
+    "language_model.layers.8.mlp.up_proj",
+    "language_model.layers.12.mlp.down_proj",
+    "language_model.layers.3.mlp.down_proj",
+    "language_model.layers.7.self_attn.k_proj",
+    "language_model.layers.2.mlp.up_proj",
+    "language_model.layers.13.mlp.down_proj",
+    "28.mlp.gate_proj",
+    "language_model.layers.15.self_attn.v_proj",
+    "20.self_attn.q_proj",
+    "per_layer_projection",
+    "27.mlp.gate_proj",
+    "language_model.layers.11.self_attn.o_proj",
+    "language_model.layers.15.mlp.gate_proj",
+    "30.self_attn.o_proj",
+    "32.self_attn.o_proj",
+    "27.mlp.down_proj",
+    "language_model.layers.1.mlp.gate_proj",
+    "21.self_attn.o_proj",
+    "32.self_attn.q_proj",
+    "language_model.layers.7.self_attn.v_proj",
+    "21.mlp.gate_proj",
+    "16.self_attn.k_proj",
+    "20.mlp.down_proj",
+    "16.mlp.gate_proj",
+    "20.mlp.gate_proj",
+    "language_model.layers.7.mlp.gate_proj",
+    "34.self_attn.q_proj",
+    "31.mlp.down_proj",
+    "language_model.layers.8.mlp.gate_proj",
+    "language_model.layers.9.mlp.up_proj",
+    "language_model.layers.15.self_attn.o_proj",
+    "28.self_attn.o_proj",
+    "32.self_attn.v_proj",
+    "32.self_attn.k_proj",
+    "language_model.layers.4.self_attn.q_proj",
+    "28.self_attn.v_proj",
+    "28.self_attn.k_proj",
+    "26.self_attn.k_proj",
+    "34.self_attn.k_proj",
+    "16.mlp.up_proj",
+    "21.self_attn.q_proj",
+    "17.self_attn.q_proj",
+    "language_model.layers.11.self_attn.k_proj",
+    "24.mlp.up_proj",
+    "23.self_attn.q_proj",
+    "language_model.layers.3.mlp.up_proj",
+    "language_model.layers.12.self_attn.k_proj",
+    "language_model.layers.1.mlp.down_proj",
+    "20.self_attn.v_proj",
+    "language_model.layers.11.mlp.up_proj",
+    "language_model.layers.0.mlp.gate_proj",
+    "language_model.layers.5.self_attn.v_proj",
+    "language_model.layers.6.self_attn.o_proj",
+    "language_model.layers.11.mlp.down_proj",
+    "31.mlp.gate_proj",
+    "31.self_attn.q_proj",
+    "22.self_attn.v_proj",
+    "31.self_attn.v_proj",
+    "25.mlp.up_proj",
+    "language_model.layers.9.self_attn.q_proj",
+    "27.self_attn.o_proj",
+    "17.mlp.up_proj",
+    "25.self_attn.k_proj",
+    "language_model.layers.1.self_attn.o_proj",
+    "19.mlp.down_proj",
+    "language_model.layers.14.self_attn.q_proj",
+    "31.self_attn.k_proj",
+    "language_model.layers.14.mlp.down_proj",
+    "language_model.layers.2.self_attn.v_proj",
+    "language_model.layers.9.mlp.down_proj",
+    "26.mlp.down_proj",
+    "30.self_attn.q_proj",
+    "31.self_attn.o_proj",
+    "18.self_attn.v_proj",
+    "33.self_attn.q_proj",
+    "language_model.layers.2.self_attn.q_proj",
+    "30.mlp.up_proj",
+    "language_model.layers.4.mlp.down_proj",
+    "29.mlp.down_proj",
+    "19.self_attn.q_proj",
+    "language_model.layers.1.self_attn.v_proj",
+    "language_model.layers.10.self_attn.o_proj",
+    "27.self_attn.k_proj",
+    "embedding_projection",
+    "24.self_attn.q_proj",
+    "33.self_attn.v_proj",
+    "29.mlp.up_proj",
+    "25.self_attn.v_proj",
+    "language_model.layers.11.self_attn.q_proj",
+    "33.self_attn.k_proj",
+    "language_model.layers.5.self_attn.o_proj",
+    "language_model.layers.4.self_attn.k_proj",
+    "language_model.layers.3.self_attn.o_proj",
+    "21.self_attn.k_proj",
+    "language_model.layers.8.mlp.down_proj",
+    "language_model.layers.2.self_attn.o_proj",
+    "20.self_attn.o_proj",
+    "language_model.layers.0.self_attn.k_proj",
+    "linear",
+    "language_model.layers.10.self_attn.v_proj",
+    "17.mlp.down_proj",
+    "21.mlp.up_proj",
+    "language_model.layers.2.mlp.down_proj",
+    "22.mlp.gate_proj",
+    "language_model.layers.9.self_attn.o_proj",
+    "language_model.layers.0.self_attn.q_proj",
+    "language_model.layers.14.self_attn.k_proj",
+    "18.mlp.gate_proj",
+    "language_model.layers.3.self_attn.q_proj",
+    "language_model.layers.7.self_attn.q_proj",
+    "language_model.layers.15.mlp.up_proj",
+    "34.self_attn.o_proj",
+    "language_model.layers.8.self_attn.q_proj",
+    "language_model.layers.10.mlp.up_proj",
+    "language_model.layers.14.mlp.up_proj",
+    "29.mlp.gate_proj",
+    "language_model.layers.1.self_attn.k_proj",
+    "language_model.layers.13.self_attn.o_proj",
+    "25.self_attn.o_proj",
+    "17.self_attn.v_proj",
+    "30.mlp.gate_proj",
+    "26.self_attn.v_proj",
+    "language_model.layers.3.self_attn.v_proj",
+    "19.self_attn.v_proj",
+    "language_model.layers.0.mlp.down_proj",
+    "language_model.layers.3.self_attn.k_proj",
+    "18.self_attn.o_proj",
+    "29.self_attn.o_proj",
+    "25.mlp.gate_proj",
+    "language_model.layers.9.mlp.gate_proj",
+    "16.self_attn.o_proj",
+    "language_model.layers.12.self_attn.o_proj",
+    "33.self_attn.o_proj",
+    "output_proj",
+    "22.self_attn.o_proj",
+    "17.self_attn.o_proj",
+    "23.self_attn.v_proj",
+    "language_model.layers.12.mlp.gate_proj",
+    "22.self_attn.q_proj",
+    "relative_k_proj",
+    "30.mlp.down_proj",
+    "22.mlp.down_proj",
+    "31.mlp.up_proj",
+    "18.self_attn.k_proj",
+    "language_model.layers.1.mlp.up_proj",
+    "28.mlp.down_proj",
+    "language_model.layers.15.self_attn.q_proj",
+    "language_model.layers.10.mlp.gate_proj",
+    "language_model.layers.0.self_attn.v_proj",
+    "23.mlp.down_proj",
+    "29.self_attn.v_proj",
+    "language_model.layers.8.self_attn.v_proj",
+    "22.self_attn.k_proj",
+    "17.mlp.gate_proj",
+    "24.self_attn.o_proj",
+    "16.mlp.down_proj",
+    "33.mlp.down_proj",
+    "language_model.layers.2.self_attn.k_proj",
+    "23.self_attn.k_proj",
+    "30.self_attn.v_proj",
+    "language_model.layers.4.self_attn.o_proj",
+    "language_model.layers.1.self_attn.q_proj",
+    "language_model.layers.7.mlp.up_proj",
+    "language_model.layers.5.self_attn.k_proj",
+    "19.mlp.up_proj",
+    "per_layer_input_gate",
+    "language_model.layers.0.mlp.up_proj",
+    "30.self_attn.k_proj",
+    "language_model.layers.8.self_attn.k_proj",
+    "18.self_attn.q_proj",
+    "language_model.layers.4.mlp.up_proj",
+    "19.mlp.gate_proj",
+    "language_model.layers.0.self_attn.o_proj",
+    "language_model.layers.12.self_attn.v_proj",
+    "34.mlp.up_proj",
+    "language_model.layers.13.self_attn.q_proj",
+    "27.mlp.up_proj",
+    "26.mlp.gate_proj",
+    "language_model.layers.6.self_attn.k_proj",
+    "26.self_attn.o_proj",
+    "language_model.layers.10.self_attn.q_proj",
+    "language_model.layers.10.self_attn.k_proj",
+    "language_model.layers.10.mlp.down_proj",
+    "33.mlp.up_proj",
+    "language_model.layers.6.self_attn.v_proj",
+    "16.self_attn.q_proj",
+    "27.self_attn.v_proj",
+    "28.mlp.up_proj",
+    "language_model.layers.15.mlp.down_proj",
+    "language_model.layers.14.self_attn.o_proj",
+    "language_model.layers.5.mlp.up_proj",
+    "language_model.layers.13.mlp.gate_proj",
+    "language_model.layers.14.mlp.gate_proj",
+    "language_model.layers.11.self_attn.v_proj",
+    "language_model.layers.14.self_attn.v_proj",
+    "18.mlp.down_proj",
+    "17.self_attn.k_proj",
+    "language_model.layers.7.self_attn.o_proj",
+    "32.mlp.up_proj",
+    "language_model.layers.6.mlp.up_proj",
+    "language_model.layers.9.self_attn.v_proj",
+    "language_model.layers.3.mlp.gate_proj",
+    "language_model.layers.6.mlp.down_proj",
+    "language_model.layers.12.mlp.up_proj",
+    "language_model.layers.6.mlp.gate_proj",
+    "28.self_attn.q_proj",
+    "language_model.layers.15.self_attn.k_proj",
+    "language_model.layers.8.self_attn.o_proj",
+    "language_model.layers.5.mlp.down_proj",
+    "27.self_attn.q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-1250/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91fd92896c8449aae983e31d6265ec4bb7d2d6bc4d0a4fb8b1534c43e57e34aa
+size 1688992024

checkpoint-1250/chat_template.jinja

@@ -0,0 +1,263 @@
+{%- macro format_parameters(properties, required) -%}
+    {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
+    {%- set ns = namespace(found_first=false) -%}
+    {%- for key, value in properties | dictsort -%}
+        {%- set add_comma = false -%}
+        {%- if key not in standard_keys -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {{ key }}:{
+            {%- if value['description'] -%}
+                description:<|"|>{{ value['description'] }}<|"|>
+                {%- set add_comma = true -%}
+            {%- endif -%}
+            {%- if value['nullable'] %}
+                {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                nullable:true
+            {%- endif -%}
+            {%- if value['type'] | upper == 'STRING' -%}
+                {%- if value['enum'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    enum:{{ format_argument(value['enum']) }}
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'OBJECT' -%}
+                ,properties:{
+                {%- if value['properties'] is defined and value['properties'] is mapping -%}
+                    {{- format_parameters(value['properties'], value['required'] | default([])) -}}
+                {%- elif value is mapping -%}
+                    {{- format_parameters(value, value['required'] | default([])) -}}
+                {%- endif -%}
+                }
+                {%- if value['required'] -%}
+                    ,required:[
+                    {%- for item in value['required'] | default([]) -%}
+                        <|"|>{{- item -}}<|"|>
+                        {%- if not loop.last %},{% endif -%}
+                    {%- endfor -%}
+                    ]
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'ARRAY' -%}
+                {%- if value['items'] is mapping and value['items'] -%}
+                    ,items:{
+                    {%- set ns_items = namespace(found_first=false) -%}
+                    {%- for item_key, item_value in value['items'] | dictsort -%}
+                        {%- if item_value is not none -%}
+                            {%- if ns_items.found_first %},{% endif -%}
+                            {%- set ns_items.found_first = true -%}
+                            {%- if item_key == 'properties' -%}
+                                properties:{
+                                {%- if item_value is mapping -%}
+                                    {{- format_parameters(item_value, value['items']['required'] | default([])) -}}
+                                {%- endif -%}
+                                }
+                            {%- elif item_key == 'required' -%}
+                                required:[
+                                {%- for req_item in item_value -%}
+                                    <|"|>{{- req_item -}}<|"|>
+                                    {%- if not loop.last %},{% endif -%}
+                                {%- endfor -%}
+                                ]
+                            {%- elif item_key == 'type' -%}
+                                {%- if item_value is string -%}
+                                    type:{{ format_argument(item_value | upper) }}
+                                {%- else -%}
+                                    type:{{ format_argument(item_value | map('upper') | list) }}
+                                {%- endif -%}
+                            {%- else -%}
+                                {{ item_key }}:{{ format_argument(item_value) }}
+                            {%- endif -%}
+                        {%- endif -%}
+                    {%- endfor -%}
+                    }
+                {%- endif -%}
+            {%- endif -%}
+            {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+            type:<|"|>{{ value['type'] | upper }}<|"|>}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endmacro -%}
+{%- macro format_function_declaration(tool_data) -%}
+    declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
+    {%- set params = tool_data['function']['parameters'] -%}
+    {%- if params -%}
+        ,parameters:{
+        {%- if params['properties'] -%}
+            properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
+        {%- endif -%}
+        {%- if params['required'] -%}
+            required:[
+            {%- for item in params['required'] -%}
+                <|"|>{{- item -}}<|"|>
+                {{- ',' if not loop.last -}}
+            {%- endfor -%}
+            ],
+        {%- endif -%}
+        {%- if params['type'] -%}
+            type:<|"|>{{- params['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    {%- if 'response' in tool_data['function'] -%}
+        {%- set response_declaration = tool_data['function']['response'] -%}
+        ,response:{
+        {%- if response_declaration['description'] -%}
+            description:<|"|>{{- response_declaration['description'] -}}<|"|>,
+        {%- endif -%}
+        {%- if response_declaration['type'] | upper == 'OBJECT' -%}
+            type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    }
+{%- endmacro -%}
+{%- macro format_argument(argument, escape_keys=True) -%}
+    {%- if argument is string -%}
+        {{- '<|"|>' + argument + '<|"|>' -}}
+    {%- elif argument is boolean -%}
+        {{- 'true' if argument else 'false' -}}
+    {%- elif argument is mapping -%}
+        {{- '{' -}}
+        {%- set ns = namespace(found_first=false) -%}
+        {%- for key, value in argument | dictsort -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {%- if escape_keys -%}
+                {{- '<|"|>' + key + '<|"|>' -}}
+            {%- else -%}
+                {{- key -}}
+            {%- endif -%}
+            :{{- format_argument(value, escape_keys=escape_keys) -}}
+        {%- endfor -%}
+        {{- '}' -}}
+    {%- elif argument is sequence -%}
+        {{- '[' -}}
+        {%- for item in argument -%}
+            {{- format_argument(item, escape_keys=escape_keys) -}}
+            {%- if not loop.last %},{% endif -%}
+        {%- endfor -%}
+        {{- ']' -}}
+    {%- else -%}
+        {{- argument -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- macro strip_thinking(text) -%}
+    {%- set ns = namespace(result='') -%}
+    {%- for part in text.split('<channel|>') -%}
+        {%- if '<|channel>' in part -%}
+            {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
+        {%- else -%}
+            {%- set ns.result = ns.result + part -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- ns.result | trim -}}
+{%- endmacro -%}
+
+{%- set ns = namespace(prev_message_type=None) -%}
+{%- set loop_messages = messages -%}
+{{ bos_token }}
+{#- Handle System/Tool Definitions Block -#}
+{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}
+    {{- '<|turn>system\n' -}}
+
+    {#- Inject Thinking token at the very top of the FIRST system turn -#}
+    {%- if enable_thinking is defined and enable_thinking -%}
+        {{- '<|think|>' -}}
+        {%- set ns.prev_message_type = 'think' -%}
+    {%- endif -%}
+
+    {%- if messages[0]['role'] in ['system', 'developer'] -%}
+        {{- messages[0]['content'] | trim -}}
+        {%- set loop_messages = messages[1:] -%}
+    {%- endif -%}
+
+    {%- if tools -%}
+        {%- for tool in tools %}
+            {{- '<|tool>' -}}
+            {{- format_function_declaration(tool) | trim -}}
+            {{- '<tool|>' -}}
+        {%- endfor %}
+        {%- set ns.prev_message_type = 'tool' -%}
+    {%- endif -%}
+
+    {{- '<turn|>\n' -}}
+{%- endif %}
+
+{#- Loop through messages -#}
+{%- for message in loop_messages -%}
+    {%- set ns.prev_message_type = None -%}
+    {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
+        {{- '<|turn>' + role + '\n' }}
+
+            {%- if message['tool_calls'] -%}
+                {%- for tool_call in message['tool_calls'] -%}
+                    {%- set function = tool_call['function'] -%}
+                    {{- '<|tool_call>call:' + function['name'] + '{' -}}
+                    {%- if function['arguments'] is mapping -%}
+                        {%- set ns_args = namespace(found_first=false) -%}
+                        {%- for key, value in function['arguments'] | dictsort -%}
+                            {%- if ns_args.found_first %},{% endif -%}
+                            {%- set ns_args.found_first = true -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                        {%- endfor -%}
+                    {%- elif function['arguments'] is string -%}
+                        {{- function['arguments'] -}}
+                    {%- endif -%}
+                    {{- '}<tool_call|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_call' -%}
+            {%- endif -%}
+
+            {%- if message['tool_responses'] -%}
+                {#- Tool Response handling -#}
+                {%- for tool_response in message['tool_responses'] -%}
+                    {{- '<|tool_response>' -}}
+                    {%- if tool_response['response'] is mapping -%}
+                        {{- 'response:' + tool_response['name'] | default('unknown') + '{' -}}
+                        {%- for key, value in tool_response['response'] | dictsort -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                            {%- if not loop.last %},{% endif -%}
+                        {%- endfor -%}
+                        {{- '}' -}}
+                    {%- else -%}
+                        {{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}}
+                    {%- endif -%}
+                    {{- '<tool_response|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_response' -%}
+            {%- endif -%}
+
+            {%- if message['content'] is string -%}
+                {%- if role == 'model' -%}
+                    {{- strip_thinking(message['content']) -}}
+                {%- else -%}
+                    {{- message['content'] | trim -}}
+                {%- endif -%}
+            {%- elif message['content'] is sequence -%}
+                {%- for item in message['content'] -%}
+                    {%- if item['type'] == 'text' -%}
+                        {%- if role == 'model' -%}
+                            {{- strip_thinking(item['text']) -}}
+                        {%- else -%}
+                            {{- item['text'] | trim -}}
+                        {%- endif -%}
+                    {%- elif item['type'] == 'image' -%}
+                        {{- '\n\n<|image|>\n\n' -}}
+                        {%- set ns.prev_message_type = 'image' -%}
+                    {%- elif item['type'] == 'audio' -%}
+                        {{- '<|audio|>' -}}
+                        {%- set ns.prev_message_type = 'audio' -%}
+                    {%- elif item['type'] == 'video' -%}
+                        {{- '\n\n<|video|>\n\n' -}}
+                        {%- set ns.prev_message_type = 'video' -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+
+        {%- if not (message['tool_responses'] and not message['content']) -%}
+            {{- '<turn|>\n' -}}
+        {%- endif -%}
+{%- endfor -%}
+
+{%- if add_generation_prompt -%}
+    {%- if ns.prev_message_type != 'tool_response' -%}
+        {{- '<|turn>model\n' -}}
+    {%- endif -%}
+{%- endif -%}

checkpoint-1250/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0ae86183738eeb3bbfa0e338f90d4ea1c6750e7a529a9c843e8582dd77e186d
+size 3331821841

checkpoint-1250/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2192e89a6d157174133d8c93c12c8eb1388f9e6ed213d12126868467ae12e9d
+size 14709

checkpoint-1250/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00f276848356c16e5e904d2f5b309482d721c1fea6c2b8972b70cdcaa02157b7
+size 1465

checkpoint-1250/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f
+size 32169626

checkpoint-1250/tokenizer_config.json

@@ -0,0 +1,95 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [
+    "<|video|>"
+  ],
+  "image_token": "<|image|>",
+  "is_local": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "audio_token": "<|audio|>",
+    "boa_token": "<|audio>",
+    "boi_token": "<|image>",
+    "eoa_token": "<audio|>",
+    "eoc_token": "<channel|>",
+    "eoi_token": "<image|>",
+    "eot_token": "<turn|>",
+    "escape_token": "<|\"|>",
+    "etc_token": "<tool_call|>",
+    "etd_token": "<tool|>",
+    "etr_token": "<tool_response|>",
+    "image_token": "<|image|>",
+    "soc_token": "<|channel>",
+    "sot_token": "<|turn>",
+    "stc_token": "<|tool_call>",
+    "std_token": "<|tool>",
+    "str_token": "<|tool_response>",
+    "think_token": "<|think|>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma4Processor",
+  "response_schema": {
+    "properties": {
+      "content": {
+        "type": "string"
+      },
+      "role": {
+        "const": "assistant"
+      },
+      "thinking": {
+        "type": "string"
+      },
+      "tool_calls": {
+        "items": {
+          "properties": {
+            "function": {
+              "properties": {
+                "arguments": {
+                  "additionalProperties": {},
+                  "type": "object",
+                  "x-parser": "gemma4-tool-call"
+                },
+                "name": {
+                  "type": "string"
+                }
+              },
+              "type": "object",
+              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"
+            },
+            "type": {
+              "const": "function"
+            }
+          },
+          "type": "object"
+        },
+        "type": "array",
+        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"
+      }
+    },
+    "type": "object",
+    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<content>(?:(?!\\<\\|tool_call\\>)(?!\\<turn\\|\\>).)+)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?:\\<turn\\|\\>)?"
+  },
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-1250/trainer_state.json

@@ -0,0 +1,1316 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 1250,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.5365671901141895,
+      "epoch": 0.0016,
+      "grad_norm": 4.21875,
+      "learning_rate": 0.0,
+      "loss": 5.494313716888428,
+      "mean_token_accuracy": 0.6573571685482474,
+      "num_tokens": 348701.0,
+      "step": 1
+    },
+    {
+      "entropy": 1.348903712299135,
+      "epoch": 0.016,
+      "grad_norm": 2.59375,
+      "learning_rate": 3.157894736842105e-05,
+      "loss": 5.544558631049262,
+      "mean_token_accuracy": 0.6651431231035126,
+      "num_tokens": 371496.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.360134594142437,
+      "epoch": 0.032,
+      "grad_norm": 3.296875,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 5.742576599121094,
+      "mean_token_accuracy": 0.6662841200828552,
+      "num_tokens": 400839.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.375771728157997,
+      "epoch": 0.048,
+      "grad_norm": 3.140625,
+      "learning_rate": 0.0001017543859649123,
+      "loss": 5.714028167724609,
+      "mean_token_accuracy": 0.6566085889935493,
+      "num_tokens": 431025.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.2748979568481444,
+      "epoch": 0.064,
+      "grad_norm": 2.828125,
+      "learning_rate": 0.0001368421052631579,
+      "loss": 5.11612548828125,
+      "mean_token_accuracy": 0.6816287323832512,
+      "num_tokens": 459089.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.3155348658561707,
+      "epoch": 0.08,
+      "grad_norm": 3.15625,
+      "learning_rate": 0.00017192982456140353,
+      "loss": 5.35040512084961,
+      "mean_token_accuracy": 0.6721501812338829,
+      "num_tokens": 484648.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1791150525212288,
+      "epoch": 0.096,
+      "grad_norm": 2.375,
+      "learning_rate": 0.00019999940277008808,
+      "loss": 4.642951583862304,
+      "mean_token_accuracy": 0.7028968930244446,
+      "num_tokens": 511463.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1177749201655387,
+      "epoch": 0.112,
+      "grad_norm": 2.421875,
+      "learning_rate": 0.0001999785004721968,
+      "loss": 4.523546981811523,
+      "mean_token_accuracy": 0.7057401552796364,
+      "num_tokens": 538438.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.1784485399723053,
+      "epoch": 0.128,
+      "grad_norm": 2.25,
+      "learning_rate": 0.00019992774381199778,
+      "loss": 4.712202072143555,
+      "mean_token_accuracy": 0.7004493281245232,
+      "num_tokens": 569421.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.0666756927967072,
+      "epoch": 0.144,
+      "grad_norm": 4.25,
+      "learning_rate": 0.00019984714794582683,
+      "loss": 4.181539154052734,
+      "mean_token_accuracy": 0.7218294009566307,
+      "num_tokens": 595806.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1410984337329864,
+      "epoch": 0.16,
+      "grad_norm": 2.59375,
+      "learning_rate": 0.00019973673694024,
+      "loss": 4.578577041625977,
+      "mean_token_accuracy": 0.70914496332407,
+      "num_tokens": 626238.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.058976437151432,
+      "epoch": 0.176,
+      "grad_norm": 4.34375,
+      "learning_rate": 0.0001995965437648273,
+      "loss": 4.2924964904785154,
+      "mean_token_accuracy": 0.7180401891469955,
+      "num_tokens": 653441.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.2249037250876427,
+      "epoch": 0.192,
+      "grad_norm": 3.96875,
+      "learning_rate": 0.00019942661028236745,
+      "loss": 5.084912490844727,
+      "mean_token_accuracy": 0.6847136527299881,
+      "num_tokens": 682582.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.1726351112127305,
+      "epoch": 0.208,
+      "grad_norm": 3.296875,
+      "learning_rate": 0.00019922698723632767,
+      "loss": 5.018433380126953,
+      "mean_token_accuracy": 0.7055773109197616,
+      "num_tokens": 710468.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.3469643473625184,
+      "epoch": 0.224,
+      "grad_norm": 2.875,
+      "learning_rate": 0.000198997734235711,
+      "loss": 5.7476848602294925,
+      "mean_token_accuracy": 0.6790369123220443,
+      "num_tokens": 740979.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.187085197865963,
+      "epoch": 0.24,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.0001987389197372567,
+      "loss": 5.085608673095703,
+      "mean_token_accuracy": 0.7057169727981091,
+      "num_tokens": 768427.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.2938067495822907,
+      "epoch": 0.256,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.0001984506210249986,
+      "loss": 5.570721054077149,
+      "mean_token_accuracy": 0.6840770006179809,
+      "num_tokens": 796744.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.1161220327019692,
+      "epoch": 0.272,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.00019813292418718732,
+      "loss": 4.821302795410157,
+      "mean_token_accuracy": 0.7168306574225426,
+      "num_tokens": 822225.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.2217196941375732,
+      "epoch": 0.288,
+      "grad_norm": 2.484375,
+      "learning_rate": 0.00019778592409058378,
+      "loss": 5.171672821044922,
+      "mean_token_accuracy": 0.6998881995677948,
+      "num_tokens": 848049.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1091715931892394,
+      "epoch": 0.304,
+      "grad_norm": 3.109375,
+      "learning_rate": 0.00019740972435213115,
+      "loss": 4.743664169311524,
+      "mean_token_accuracy": 0.7203769966959953,
+      "num_tokens": 873172.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.2154748886823654,
+      "epoch": 0.32,
+      "grad_norm": 2.21875,
+      "learning_rate": 0.00019700443730801413,
+      "loss": 5.212123107910156,
+      "mean_token_accuracy": 0.7076091229915619,
+      "num_tokens": 901716.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.161145742237568,
+      "epoch": 0.336,
+      "grad_norm": 2.078125,
+      "learning_rate": 0.00019657018398011434,
+      "loss": 4.889891052246094,
+      "mean_token_accuracy": 0.7145821407437325,
+      "num_tokens": 928506.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.1209837168455123,
+      "epoch": 0.352,
+      "grad_norm": 2.09375,
+      "learning_rate": 0.00019610709403987246,
+      "loss": 4.862021636962891,
+      "mean_token_accuracy": 0.7152845054864884,
+      "num_tokens": 958723.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.1322738587856294,
+      "epoch": 0.368,
+      "grad_norm": 2.203125,
+      "learning_rate": 0.00019561530576956703,
+      "loss": 4.911846923828125,
+      "mean_token_accuracy": 0.7134342208504677,
+      "num_tokens": 987751.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.1780440196394921,
+      "epoch": 0.384,
+      "grad_norm": 2.421875,
+      "learning_rate": 0.00019509496602102252,
+      "loss": 5.0671440124511715,
+      "mean_token_accuracy": 0.7104542285203934,
+      "num_tokens": 1018315.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.1776200592517854,
+      "epoch": 0.4,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.00019454623017175812,
+      "loss": 5.145317459106446,
+      "mean_token_accuracy": 0.7086253471672534,
+      "num_tokens": 1046627.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.1513659209012985,
+      "epoch": 0.416,
+      "grad_norm": 2.140625,
+      "learning_rate": 0.00019396926207859084,
+      "loss": 4.8750255584716795,
+      "mean_token_accuracy": 0.7203947469592095,
+      "num_tokens": 1075027.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.1364729449152946,
+      "epoch": 0.432,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.00019336423402870653,
+      "loss": 4.866281890869141,
+      "mean_token_accuracy": 0.7150357633829116,
+      "num_tokens": 1102334.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.108440762758255,
+      "epoch": 0.448,
+      "grad_norm": 1.9453125,
+      "learning_rate": 0.00019273132668821364,
+      "loss": 4.697020721435547,
+      "mean_token_accuracy": 0.7265092357993126,
+      "num_tokens": 1131415.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1001321360468865,
+      "epoch": 0.464,
+      "grad_norm": 2.125,
+      "learning_rate": 0.00019207072904819486,
+      "loss": 4.747102355957031,
+      "mean_token_accuracy": 0.7231316924095154,
+      "num_tokens": 1158280.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.1048941344022751,
+      "epoch": 0.48,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.00019138263836827288,
+      "loss": 4.644550323486328,
+      "mean_token_accuracy": 0.7270571634173393,
+      "num_tokens": 1188651.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.0945911705493927,
+      "epoch": 0.496,
+      "grad_norm": 1.984375,
+      "learning_rate": 0.00019066726011770726,
+      "loss": 4.7493640899658205,
+      "mean_token_accuracy": 0.7258608743548394,
+      "num_tokens": 1216504.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.0356923416256905,
+      "epoch": 0.512,
+      "grad_norm": 2.1875,
+      "learning_rate": 0.00018992480791403958,
+      "loss": 4.3843944549560545,
+      "mean_token_accuracy": 0.7372462660074234,
+      "num_tokens": 1243732.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.0773125097155571,
+      "epoch": 0.528,
+      "grad_norm": 2.328125,
+      "learning_rate": 0.0001891555034593055,
+      "loss": 4.6180767059326175,
+      "mean_token_accuracy": 0.7283408164978027,
+      "num_tokens": 1269272.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.050987295806408,
+      "epoch": 0.544,
+      "grad_norm": 1.640625,
+      "learning_rate": 0.00018835957647383303,
+      "loss": 4.4863533020019535,
+      "mean_token_accuracy": 0.7359548002481461,
+      "num_tokens": 1296001.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.0461898490786552,
+      "epoch": 0.56,
+      "grad_norm": 2.453125,
+      "learning_rate": 0.000187537264627646,
+      "loss": 4.434906005859375,
+      "mean_token_accuracy": 0.7349108412861824,
+      "num_tokens": 1322180.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.0046504065394402,
+      "epoch": 0.576,
+      "grad_norm": 2.125,
+      "learning_rate": 0.00018668881346949417,
+      "loss": 4.309226989746094,
+      "mean_token_accuracy": 0.73991359770298,
+      "num_tokens": 1347532.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.0861794739961623,
+      "epoch": 0.592,
+      "grad_norm": 1.890625,
+      "learning_rate": 0.0001858144763535302,
+      "loss": 4.599692535400391,
+      "mean_token_accuracy": 0.7285229310393333,
+      "num_tokens": 1373584.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.0600647926330566,
+      "epoch": 0.608,
+      "grad_norm": 2.203125,
+      "learning_rate": 0.00018491451436365627,
+      "loss": 4.5491493225097654,
+      "mean_token_accuracy": 0.7357145607471466,
+      "num_tokens": 1399594.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.098224511742592,
+      "epoch": 0.624,
+      "grad_norm": 1.9296875,
+      "learning_rate": 0.00018398919623556238,
+      "loss": 4.8250572204589846,
+      "mean_token_accuracy": 0.7251888766884804,
+      "num_tokens": 1428792.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.1232757449150086,
+      "epoch": 0.64,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.00018303879827647975,
+      "loss": 4.712010192871094,
+      "mean_token_accuracy": 0.7287421196699142,
+      "num_tokens": 1457889.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.0411028936505318,
+      "epoch": 0.656,
+      "grad_norm": 1.859375,
+      "learning_rate": 0.00018206360428267332,
+      "loss": 4.405958938598633,
+      "mean_token_accuracy": 0.7350577011704444,
+      "num_tokens": 1484479.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.1146624743938447,
+      "epoch": 0.672,
+      "grad_norm": 1.734375,
+      "learning_rate": 0.00018106390545469795,
+      "loss": 4.742184448242187,
+      "mean_token_accuracy": 0.7282734125852585,
+      "num_tokens": 1514268.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.0480840012431145,
+      "epoch": 0.688,
+      "grad_norm": 1.78125,
+      "learning_rate": 0.0001800400003104436,
+      "loss": 4.4207916259765625,
+      "mean_token_accuracy": 0.7365467861294747,
+      "num_tokens": 1540474.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.0067586719989776,
+      "epoch": 0.704,
+      "grad_norm": 1.921875,
+      "learning_rate": 0.0001789921945959958,
+      "loss": 4.31513671875,
+      "mean_token_accuracy": 0.7437789484858512,
+      "num_tokens": 1568557.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.0166094586253167,
+      "epoch": 0.72,
+      "grad_norm": 1.6171875,
+      "learning_rate": 0.0001779208011943371,
+      "loss": 4.292739486694336,
+      "mean_token_accuracy": 0.7471857726573944,
+      "num_tokens": 1594403.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.0385540708899499,
+      "epoch": 0.736,
+      "grad_norm": 2.75,
+      "learning_rate": 0.00017682614003191807,
+      "loss": 4.415458679199219,
+      "mean_token_accuracy": 0.7400641202926636,
+      "num_tokens": 1624111.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.0706985503435136,
+      "epoch": 0.752,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.0001757085379831246,
+      "loss": 4.554729080200195,
+      "mean_token_accuracy": 0.7342943042516709,
+      "num_tokens": 1652609.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.9689434483647347,
+      "epoch": 0.768,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.00017456832877267084,
+      "loss": 4.2303211212158205,
+      "mean_token_accuracy": 0.7474748462438583,
+      "num_tokens": 1678589.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.1837652444839477,
+      "epoch": 0.784,
+      "grad_norm": 2.078125,
+      "learning_rate": 0.00017340585287594604,
+      "loss": 4.968061447143555,
+      "mean_token_accuracy": 0.7166377156972885,
+      "num_tokens": 1708513.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.042286379635334,
+      "epoch": 0.8,
+      "grad_norm": 2.515625,
+      "learning_rate": 0.00017222145741734626,
+      "loss": 4.422880172729492,
+      "mean_token_accuracy": 0.7405225187540054,
+      "num_tokens": 1736283.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.9832519263029098,
+      "epoch": 0.816,
+      "grad_norm": 1.6171875,
+      "learning_rate": 0.00017101549606662024,
+      "loss": 4.119276428222657,
+      "mean_token_accuracy": 0.7495438739657402,
+      "num_tokens": 1765017.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.9872105062007904,
+      "epoch": 0.832,
+      "grad_norm": 1.71875,
+      "learning_rate": 0.00016978832893326074,
+      "loss": 4.234106826782226,
+      "mean_token_accuracy": 0.7473902180790901,
+      "num_tokens": 1790682.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.9845283895730972,
+      "epoch": 0.848,
+      "grad_norm": 1.4921875,
+      "learning_rate": 0.00016854032245897308,
+      "loss": 4.146430969238281,
+      "mean_token_accuracy": 0.7520387843251228,
+      "num_tokens": 1820050.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.0613365799188614,
+      "epoch": 0.864,
+      "grad_norm": 2.046875,
+      "learning_rate": 0.00016727184930825288,
+      "loss": 4.48931655883789,
+      "mean_token_accuracy": 0.7396015107631684,
+      "num_tokens": 1847079.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.9684726029634476,
+      "epoch": 0.88,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.00016598328825710533,
+      "loss": 4.12475357055664,
+      "mean_token_accuracy": 0.7526269048452378,
+      "num_tokens": 1872383.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.9990086019039154,
+      "epoch": 0.896,
+      "grad_norm": 1.8984375,
+      "learning_rate": 0.00016467502407993992,
+      "loss": 4.360863494873047,
+      "mean_token_accuracy": 0.7387492001056671,
+      "num_tokens": 1903630.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.9708257809281349,
+      "epoch": 0.912,
+      "grad_norm": 1.6328125,
+      "learning_rate": 0.00016334744743467364,
+      "loss": 4.054442596435547,
+      "mean_token_accuracy": 0.7545965671539306,
+      "num_tokens": 1930592.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.0035204842686654,
+      "epoch": 0.928,
+      "grad_norm": 1.8984375,
+      "learning_rate": 0.00016200095474607753,
+      "loss": 4.25194206237793,
+      "mean_token_accuracy": 0.7464351058006287,
+      "num_tokens": 1960212.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.9635734960436821,
+      "epoch": 0.944,
+      "grad_norm": 1.53125,
+      "learning_rate": 0.00016063594808740113,
+      "loss": 4.0587310791015625,
+      "mean_token_accuracy": 0.7547481089830399,
+      "num_tokens": 1986049.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.9986386775970459,
+      "epoch": 0.96,
+      "grad_norm": 1.703125,
+      "learning_rate": 0.0001592528350603103,
+      "loss": 4.331460952758789,
+      "mean_token_accuracy": 0.7503352165222168,
+      "num_tokens": 2013266.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.006168755888939,
+      "epoch": 0.976,
+      "grad_norm": 1.734375,
+      "learning_rate": 0.00015785202867317407,
+      "loss": 4.238505554199219,
+      "mean_token_accuracy": 0.7441465124487877,
+      "num_tokens": 2041824.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.0482723653316497,
+      "epoch": 0.992,
+      "grad_norm": 1.9296875,
+      "learning_rate": 0.0001564339472177373,
+      "loss": 4.394336700439453,
+      "mean_token_accuracy": 0.7416493371129036,
+      "num_tokens": 2071693.0,
+      "step": 620
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.0188530464172363,
+      "eval_loss": 1.0276753902435303,
+      "eval_mean_token_accuracy": 0.7517141411304474,
+      "eval_num_tokens": 2084150.0,
+      "eval_runtime": 72.9791,
+      "eval_samples_per_second": 13.703,
+      "eval_steps_per_second": 3.426,
+      "step": 625
+    },
+    {
+      "entropy": 0.9583894208073616,
+      "epoch": 1.008,
+      "grad_norm": 1.40625,
+      "learning_rate": 0.0001549990141442153,
+      "loss": 3.791181945800781,
+      "mean_token_accuracy": 0.760094690322876,
+      "num_tokens": 2099483.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.7975272431969642,
+      "epoch": 1.024,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.00015354765793484834,
+      "loss": 3.2353271484375,
+      "mean_token_accuracy": 0.7800843045115471,
+      "num_tokens": 2127344.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.8557936266064644,
+      "epoch": 1.04,
+      "grad_norm": 1.921875,
+      "learning_rate": 0.00015208031197595356,
+      "loss": 3.591104507446289,
+      "mean_token_accuracy": 0.769377326965332,
+      "num_tokens": 2157286.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.8815567880868912,
+      "epoch": 1.056,
+      "grad_norm": 1.4765625,
+      "learning_rate": 0.0001505974144285124,
+      "loss": 3.509926986694336,
+      "mean_token_accuracy": 0.7711383983492851,
+      "num_tokens": 2187199.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.7711537912487983,
+      "epoch": 1.072,
+      "grad_norm": 1.3828125,
+      "learning_rate": 0.00014909940809733222,
+      "loss": 3.108415412902832,
+      "mean_token_accuracy": 0.7885575816035271,
+      "num_tokens": 2213742.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.8327512949705124,
+      "epoch": 1.088,
+      "grad_norm": 6.5625,
+      "learning_rate": 0.00014758674029882152,
+      "loss": 3.380535125732422,
+      "mean_token_accuracy": 0.7737385779619217,
+      "num_tokens": 2242448.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.8324707329273224,
+      "epoch": 1.104,
+      "grad_norm": 1.3125,
+      "learning_rate": 0.00014605986272741748,
+      "loss": 3.3809837341308593,
+      "mean_token_accuracy": 0.7749153047800064,
+      "num_tokens": 2273149.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.8442294985055924,
+      "epoch": 1.12,
+      "grad_norm": 1.578125,
+      "learning_rate": 0.0001445192313207067,
+      "loss": 3.3175960540771485,
+      "mean_token_accuracy": 0.7804568380117416,
+      "num_tokens": 2302497.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.7547720111906528,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 1.625,
+      "learning_rate": 0.00014296530612327863,
+      "loss": 3.042662239074707,
+      "mean_token_accuracy": 0.7904633671045304,
+      "num_tokens": 2328935.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.7825665086507797,
+      "epoch": 1.152,
+      "grad_norm": 1.625,
+      "learning_rate": 0.00014139855114935252,
+      "loss": 3.1688852310180664,
+      "mean_token_accuracy": 0.7849456086754799,
+      "num_tokens": 2355711.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.7932837694883347,
+      "epoch": 1.168,
+      "grad_norm": 1.453125,
+      "learning_rate": 0.00013981943424421932,
+      "loss": 3.217195510864258,
+      "mean_token_accuracy": 0.7835724055767059,
+      "num_tokens": 2383138.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.7902459263801574,
+      "epoch": 1.184,
+      "grad_norm": 1.4296875,
+      "learning_rate": 0.00013822842694453924,
+      "loss": 3.1072725296020507,
+      "mean_token_accuracy": 0.7865744397044182,
+      "num_tokens": 2411055.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.7994440570473671,
+      "epoch": 1.2,
+      "grad_norm": 3.6875,
+      "learning_rate": 0.00013662600433753745,
+      "loss": 3.352284622192383,
+      "mean_token_accuracy": 0.7770974606275558,
+      "num_tokens": 2437837.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.7930429011583329,
+      "epoch": 1.216,
+      "grad_norm": 1.765625,
+      "learning_rate": 0.00013501264491913906,
+      "loss": 3.1913455963134765,
+      "mean_token_accuracy": 0.7834275960922241,
+      "num_tokens": 2463749.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.844582536816597,
+      "epoch": 1.232,
+      "grad_norm": 1.7734375,
+      "learning_rate": 0.00013338883045108674,
+      "loss": 3.4333824157714843,
+      "mean_token_accuracy": 0.777095565199852,
+      "num_tokens": 2492017.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.7995493680238723,
+      "epoch": 1.248,
+      "grad_norm": 1.3046875,
+      "learning_rate": 0.0001317550458170826,
+      "loss": 3.26405029296875,
+      "mean_token_accuracy": 0.7810850724577904,
+      "num_tokens": 2518076.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.8197085373103619,
+      "epoch": 1.264,
+      "grad_norm": 1.359375,
+      "learning_rate": 0.00013011177887799845,
+      "loss": 3.2417884826660157,
+      "mean_token_accuracy": 0.7789853543043137,
+      "num_tokens": 2547590.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.8223014809191227,
+      "epoch": 1.28,
+      "grad_norm": 1.3828125,
+      "learning_rate": 0.0001284595203261965,
+      "loss": 3.366427993774414,
+      "mean_token_accuracy": 0.7806236863136291,
+      "num_tokens": 2576314.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.8295105174183846,
+      "epoch": 1.296,
+      "grad_norm": 1.5078125,
+      "learning_rate": 0.00012679876353900482,
+      "loss": 3.4096916198730467,
+      "mean_token_accuracy": 0.7710141837596893,
+      "num_tokens": 2604971.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.8094971597194671,
+      "epoch": 1.312,
+      "grad_norm": 1.609375,
+      "learning_rate": 0.00012513000443139112,
+      "loss": 3.2491512298583984,
+      "mean_token_accuracy": 0.7876769602298737,
+      "num_tokens": 2631986.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.798582112789154,
+      "epoch": 1.328,
+      "grad_norm": 1.984375,
+      "learning_rate": 0.00012345374130787854,
+      "loss": 3.19933967590332,
+      "mean_token_accuracy": 0.7822520866990089,
+      "num_tokens": 2656910.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.8112326949834824,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 1.515625,
+      "learning_rate": 0.00012177047471374807,
+      "loss": 3.2226589202880858,
+      "mean_token_accuracy": 0.7859175249934196,
+      "num_tokens": 2686061.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.8292336650192738,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 1.8515625,
+      "learning_rate": 0.00012008070728557186,
+      "loss": 3.453116226196289,
+      "mean_token_accuracy": 0.7712548315525055,
+      "num_tokens": 2714961.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.8000457942485809,
+      "epoch": 1.376,
+      "grad_norm": 1.4375,
+      "learning_rate": 0.00011838494360112185,
+      "loss": 3.1879623413085936,
+      "mean_token_accuracy": 0.7834105402231216,
+      "num_tokens": 2741521.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.7751885265111923,
+      "epoch": 1.392,
+      "grad_norm": 1.2578125,
+      "learning_rate": 0.00011668369002869912,
+      "loss": 3.129037094116211,
+      "mean_token_accuracy": 0.7885062962770462,
+      "num_tokens": 2768309.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.8135885834693909,
+      "epoch": 1.408,
+      "grad_norm": 1.765625,
+      "learning_rate": 0.00011497745457592816,
+      "loss": 3.372562789916992,
+      "mean_token_accuracy": 0.7782492652535439,
+      "num_tokens": 2796984.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.7925616443157196,
+      "epoch": 1.424,
+      "grad_norm": 1.625,
+      "learning_rate": 0.00011326674673806195,
+      "loss": 3.062746047973633,
+      "mean_token_accuracy": 0.7937909960746765,
+      "num_tokens": 2827932.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.7889664500951767,
+      "epoch": 1.44,
+      "grad_norm": 1.375,
+      "learning_rate": 0.00011155207734584263,
+      "loss": 3.2588306427001954,
+      "mean_token_accuracy": 0.7854808464646339,
+      "num_tokens": 2860126.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.7500537633895874,
+      "epoch": 1.456,
+      "grad_norm": 1.453125,
+      "learning_rate": 0.00010983395841296348,
+      "loss": 2.9640825271606444,
+      "mean_token_accuracy": 0.7949178665876389,
+      "num_tokens": 2884946.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.7727585166692734,
+      "epoch": 1.472,
+      "grad_norm": 1.46875,
+      "learning_rate": 0.00010811290298317755,
+      "loss": 3.129817581176758,
+      "mean_token_accuracy": 0.7860683888196945,
+      "num_tokens": 2908387.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.7516519904136658,
+      "epoch": 1.488,
+      "grad_norm": 1.2890625,
+      "learning_rate": 0.0001063894249770989,
+      "loss": 3.1172218322753906,
+      "mean_token_accuracy": 0.7909497246146202,
+      "num_tokens": 2935394.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.8111906915903091,
+      "epoch": 1.504,
+      "grad_norm": 1.2890625,
+      "learning_rate": 0.00010466403903874176,
+      "loss": 3.222005844116211,
+      "mean_token_accuracy": 0.7841308936476707,
+      "num_tokens": 2966607.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.7766234025359153,
+      "epoch": 1.52,
+      "grad_norm": 1.421875,
+      "learning_rate": 0.00010293726038184393,
+      "loss": 3.127474784851074,
+      "mean_token_accuracy": 0.7897476837038994,
+      "num_tokens": 2993998.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.7666798770427704,
+      "epoch": 1.536,
+      "grad_norm": 2.015625,
+      "learning_rate": 0.00010120960463601976,
+      "loss": 3.038217544555664,
+      "mean_token_accuracy": 0.7863978728652,
+      "num_tokens": 3022959.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.7193653047084808,
+      "epoch": 1.552,
+      "grad_norm": 1.453125,
+      "learning_rate": 9.948158769278939e-05,
+      "loss": 2.9356218338012696,
+      "mean_token_accuracy": 0.7945682421326637,
+      "num_tokens": 3049100.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.7440014734864235,
+      "epoch": 1.568,
+      "grad_norm": 1.3046875,
+      "learning_rate": 9.775372555152912e-05,
+      "loss": 2.9548721313476562,
+      "mean_token_accuracy": 0.7947487100958824,
+      "num_tokens": 3073764.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.7881634041666985,
+      "epoch": 1.584,
+      "grad_norm": 1.3828125,
+      "learning_rate": 9.602653416539031e-05,
+      "loss": 3.2630630493164063,
+      "mean_token_accuracy": 0.7804958373308182,
+      "num_tokens": 3101016.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.7688482075929641,
+      "epoch": 1.6,
+      "grad_norm": 1.1171875,
+      "learning_rate": 9.430052928723153e-05,
+      "loss": 3.0473140716552733,
+      "mean_token_accuracy": 0.7904627084732055,
+      "num_tokens": 3131323.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.8112111315131187,
+      "epoch": 1.616,
+      "grad_norm": 1.3046875,
+      "learning_rate": 9.257622631561085e-05,
+      "loss": 3.270753860473633,
+      "mean_token_accuracy": 0.7852039277553559,
+      "num_tokens": 3161781.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.7603359699249268,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 1.078125,
+      "learning_rate": 9.085414014088369e-05,
+      "loss": 3.0997121810913084,
+      "mean_token_accuracy": 0.7898457184433937,
+      "num_tokens": 3189668.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.789544765651226,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 1.3203125,
+      "learning_rate": 8.913478499145254e-05,
+      "loss": 3.1730154037475584,
+      "mean_token_accuracy": 0.7893433123826981,
+      "num_tokens": 3217248.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.7715817041695118,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 1.125,
+      "learning_rate": 8.741867428021446e-05,
+      "loss": 3.1480581283569338,
+      "mean_token_accuracy": 0.7871334388852119,
+      "num_tokens": 3244290.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.7311166271567344,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 1.734375,
+      "learning_rate": 8.570632045125185e-05,
+      "loss": 2.9483835220336916,
+      "mean_token_accuracy": 0.7995148777961731,
+      "num_tokens": 3270106.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.8038631252944469,
+      "epoch": 1.696,
+      "grad_norm": 1.296875,
+      "learning_rate": 8.399823482681262e-05,
+      "loss": 3.2476577758789062,
+      "mean_token_accuracy": 0.788225382566452,
+      "num_tokens": 3302183.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.7686532281339169,
+      "epoch": 1.712,
+      "grad_norm": 1.484375,
+      "learning_rate": 8.22949274546255e-05,
+      "loss": 3.1519790649414063,
+      "mean_token_accuracy": 0.7915654450654983,
+      "num_tokens": 3329432.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.7671505004167557,
+      "epoch": 1.728,
+      "grad_norm": 1.375,
+      "learning_rate": 8.059690695559568e-05,
+      "loss": 3.085156059265137,
+      "mean_token_accuracy": 0.7906328395009041,
+      "num_tokens": 3353693.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.7490876868367196,
+      "epoch": 1.744,
+      "grad_norm": 1.8203125,
+      "learning_rate": 7.89046803719267e-05,
+      "loss": 3.077616310119629,
+      "mean_token_accuracy": 0.7907149896025658,
+      "num_tokens": 3379146.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.821656309068203,
+      "epoch": 1.76,
+      "grad_norm": 1.1328125,
+      "learning_rate": 7.721875301571359e-05,
+      "loss": 3.2485275268554688,
+      "mean_token_accuracy": 0.7844672784209251,
+      "num_tokens": 3407799.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.7870353177189827,
+      "epoch": 1.776,
+      "grad_norm": 1.390625,
+      "learning_rate": 7.55396283180529e-05,
+      "loss": 3.2128215789794923,
+      "mean_token_accuracy": 0.7861049249768257,
+      "num_tokens": 3436295.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.7809636801481247,
+      "epoch": 1.792,
+      "grad_norm": 2.59375,
+      "learning_rate": 7.386780767871397e-05,
+      "loss": 3.1558361053466797,
+      "mean_token_accuracy": 0.7923071622848511,
+      "num_tokens": 3463552.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.7963082253932953,
+      "epoch": 1.808,
+      "grad_norm": 1.3515625,
+      "learning_rate": 7.22037903164173e-05,
+      "loss": 3.2051555633544924,
+      "mean_token_accuracy": 0.7853836208581925,
+      "num_tokens": 3490070.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.8024965927004815,
+      "epoch": 1.8239999999999998,
+      "grad_norm": 1.4296875,
+      "learning_rate": 7.054807311976379e-05,
+      "loss": 3.256487274169922,
+      "mean_token_accuracy": 0.7800001233816147,
+      "num_tokens": 3519607.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.7608409076929092,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 1.6640625,
+      "learning_rate": 6.890115049885994e-05,
+      "loss": 3.1335351943969725,
+      "mean_token_accuracy": 0.7930847942829132,
+      "num_tokens": 3549954.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.7995478600263596,
+      "epoch": 1.8559999999999999,
+      "grad_norm": 1.0859375,
+      "learning_rate": 6.726351423768322e-05,
+      "loss": 3.192752647399902,
+      "mean_token_accuracy": 0.7869470104575157,
+      "num_tokens": 3579252.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.7790917068719864,
+      "epoch": 1.8719999999999999,
+      "grad_norm": 1.15625,
+      "learning_rate": 6.563565334723134e-05,
+      "loss": 3.1096052169799804,
+      "mean_token_accuracy": 0.7890855461359024,
+      "num_tokens": 3607698.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.6804191634058953,
+      "epoch": 1.888,
+      "grad_norm": 1.4921875,
+      "learning_rate": 6.40180539194999e-05,
+      "loss": 2.7701562881469726,
+      "mean_token_accuracy": 0.8071593567728996,
+      "num_tokens": 3635417.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.7258283078670502,
+      "epoch": 1.904,
+      "grad_norm": 1.28125,
+      "learning_rate": 6.241119898233144e-05,
+      "loss": 2.969502830505371,
+      "mean_token_accuracy": 0.7979315519332886,
+      "num_tokens": 3661222.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.7636997759342193,
+      "epoch": 1.92,
+      "grad_norm": 1.6484375,
+      "learning_rate": 6.0815568355179556e-05,
+      "loss": 3.1156858444213866,
+      "mean_token_accuracy": 0.7898655593395233,
+      "num_tokens": 3692905.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.80526192933321,
+      "epoch": 1.936,
+      "grad_norm": 1.1796875,
+      "learning_rate": 5.923163850583113e-05,
+      "loss": 3.170664596557617,
+      "mean_token_accuracy": 0.7876248195767402,
+      "num_tokens": 3720018.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.7665807068347931,
+      "epoch": 1.952,
+      "grad_norm": 1.28125,
+      "learning_rate": 5.765988240812921e-05,
+      "loss": 3.141133499145508,
+      "mean_token_accuracy": 0.790970367193222,
+      "num_tokens": 3746413.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.7018338434398175,
+      "epoch": 1.968,
+      "grad_norm": 1.375,
+      "learning_rate": 5.6100769400739383e-05,
+      "loss": 2.782573699951172,
+      "mean_token_accuracy": 0.8073293790221214,
+      "num_tokens": 3770477.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.7392673164606094,
+      "epoch": 1.984,
+      "grad_norm": 1.46875,
+      "learning_rate": 5.4554765047001613e-05,
+      "loss": 2.957522964477539,
+      "mean_token_accuracy": 0.7974923148751258,
+      "num_tokens": 3796914.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.7508858695626259,
+      "epoch": 2.0,
+      "grad_norm": 1.2890625,
+      "learning_rate": 5.302233099590928e-05,
+      "loss": 3.049625205993652,
+      "mean_token_accuracy": 0.7928661614656448,
+      "num_tokens": 3822313.0,
+      "step": 1250
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 0.7935182250738144,
+      "eval_loss": 0.9520308971405029,
+      "eval_mean_token_accuracy": 0.7710564243793487,
+      "eval_num_tokens": 3822313.0,
+      "eval_runtime": 72.7945,
+      "eval_samples_per_second": 13.737,
+      "eval_steps_per_second": 3.434,
+      "step": 1250
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1875,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.0800895385989786e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1250/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed89ca2be67eb35da01705488407af1aa1ea83180f94fa1d6688fc37dcb50663
+size 5713

checkpoint-1875/README.md

@@ -0,0 +1,209 @@
+---
+base_model: google/gemma-4-E2B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:google/gemma-4-E2B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-1875/adapter_config.json

@@ -0,0 +1,296 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-4-E2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "lm_head",
+    "embed_tokens"
+  ],
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "26.self_attn.q_proj",
+    "language_model.layers.11.mlp.gate_proj",
+    "34.mlp.gate_proj",
+    "23.mlp.gate_proj",
+    "language_model.layers.5.mlp.gate_proj",
+    "input_proj",
+    "language_model.layers.12.self_attn.q_proj",
+    "32.mlp.down_proj",
+    "20.mlp.up_proj",
+    "language_model.layers.4.self_attn.v_proj",
+    "20.self_attn.k_proj",
+    "input_proj_linear",
+    "24.mlp.gate_proj",
+    "29.self_attn.k_proj",
+    "21.self_attn.v_proj",
+    "21.mlp.down_proj",
+    "16.self_attn.v_proj",
+    "language_model.layers.13.mlp.up_proj",
+    "24.mlp.down_proj",
+    "22.mlp.up_proj",
+    "language_model.layers.9.self_attn.k_proj",
+    "26.mlp.up_proj",
+    "language_model.layers.7.mlp.down_proj",
+    "language_model.layers.2.mlp.gate_proj",
+    "language_model.layers.13.self_attn.k_proj",
+    "25.self_attn.q_proj",
+    "per_layer_model_projection",
+    "18.mlp.up_proj",
+    "language_model.layers.13.self_attn.v_proj",
+    "34.mlp.down_proj",
+    "23.mlp.up_proj",
+    "23.self_attn.o_proj",
+    "24.self_attn.v_proj",
+    "language_model.layers.6.self_attn.q_proj",
+    "29.self_attn.q_proj",
+    "19.self_attn.k_proj",
+    "24.self_attn.k_proj",
+    "language_model.layers.5.self_attn.q_proj",
+    "32.mlp.gate_proj",
+    "language_model.layers.4.mlp.gate_proj",
+    "33.mlp.gate_proj",
+    "19.self_attn.o_proj",
+    "34.self_attn.v_proj",
+    "25.mlp.down_proj",
+    "language_model.layers.8.mlp.up_proj",
+    "language_model.layers.12.mlp.down_proj",
+    "language_model.layers.3.mlp.down_proj",
+    "language_model.layers.7.self_attn.k_proj",
+    "language_model.layers.2.mlp.up_proj",
+    "language_model.layers.13.mlp.down_proj",
+    "28.mlp.gate_proj",
+    "language_model.layers.15.self_attn.v_proj",
+    "20.self_attn.q_proj",
+    "per_layer_projection",
+    "27.mlp.gate_proj",
+    "language_model.layers.11.self_attn.o_proj",
+    "language_model.layers.15.mlp.gate_proj",
+    "30.self_attn.o_proj",
+    "32.self_attn.o_proj",
+    "27.mlp.down_proj",
+    "language_model.layers.1.mlp.gate_proj",
+    "21.self_attn.o_proj",
+    "32.self_attn.q_proj",
+    "language_model.layers.7.self_attn.v_proj",
+    "21.mlp.gate_proj",
+    "16.self_attn.k_proj",
+    "20.mlp.down_proj",
+    "16.mlp.gate_proj",
+    "20.mlp.gate_proj",
+    "language_model.layers.7.mlp.gate_proj",
+    "34.self_attn.q_proj",
+    "31.mlp.down_proj",
+    "language_model.layers.8.mlp.gate_proj",
+    "language_model.layers.9.mlp.up_proj",
+    "language_model.layers.15.self_attn.o_proj",
+    "28.self_attn.o_proj",
+    "32.self_attn.v_proj",
+    "32.self_attn.k_proj",
+    "language_model.layers.4.self_attn.q_proj",
+    "28.self_attn.v_proj",
+    "28.self_attn.k_proj",
+    "26.self_attn.k_proj",
+    "34.self_attn.k_proj",
+    "16.mlp.up_proj",
+    "21.self_attn.q_proj",
+    "17.self_attn.q_proj",
+    "language_model.layers.11.self_attn.k_proj",
+    "24.mlp.up_proj",
+    "23.self_attn.q_proj",
+    "language_model.layers.3.mlp.up_proj",
+    "language_model.layers.12.self_attn.k_proj",
+    "language_model.layers.1.mlp.down_proj",
+    "20.self_attn.v_proj",
+    "language_model.layers.11.mlp.up_proj",
+    "language_model.layers.0.mlp.gate_proj",
+    "language_model.layers.5.self_attn.v_proj",
+    "language_model.layers.6.self_attn.o_proj",
+    "language_model.layers.11.mlp.down_proj",
+    "31.mlp.gate_proj",
+    "31.self_attn.q_proj",
+    "22.self_attn.v_proj",
+    "31.self_attn.v_proj",
+    "25.mlp.up_proj",
+    "language_model.layers.9.self_attn.q_proj",
+    "27.self_attn.o_proj",
+    "17.mlp.up_proj",
+    "25.self_attn.k_proj",
+    "language_model.layers.1.self_attn.o_proj",
+    "19.mlp.down_proj",
+    "language_model.layers.14.self_attn.q_proj",
+    "31.self_attn.k_proj",
+    "language_model.layers.14.mlp.down_proj",
+    "language_model.layers.2.self_attn.v_proj",
+    "language_model.layers.9.mlp.down_proj",
+    "26.mlp.down_proj",
+    "30.self_attn.q_proj",
+    "31.self_attn.o_proj",
+    "18.self_attn.v_proj",
+    "33.self_attn.q_proj",
+    "language_model.layers.2.self_attn.q_proj",
+    "30.mlp.up_proj",
+    "language_model.layers.4.mlp.down_proj",
+    "29.mlp.down_proj",
+    "19.self_attn.q_proj",
+    "language_model.layers.1.self_attn.v_proj",
+    "language_model.layers.10.self_attn.o_proj",
+    "27.self_attn.k_proj",
+    "embedding_projection",
+    "24.self_attn.q_proj",
+    "33.self_attn.v_proj",
+    "29.mlp.up_proj",
+    "25.self_attn.v_proj",
+    "language_model.layers.11.self_attn.q_proj",
+    "33.self_attn.k_proj",
+    "language_model.layers.5.self_attn.o_proj",
+    "language_model.layers.4.self_attn.k_proj",
+    "language_model.layers.3.self_attn.o_proj",
+    "21.self_attn.k_proj",
+    "language_model.layers.8.mlp.down_proj",
+    "language_model.layers.2.self_attn.o_proj",
+    "20.self_attn.o_proj",
+    "language_model.layers.0.self_attn.k_proj",
+    "linear",
+    "language_model.layers.10.self_attn.v_proj",
+    "17.mlp.down_proj",
+    "21.mlp.up_proj",
+    "language_model.layers.2.mlp.down_proj",
+    "22.mlp.gate_proj",
+    "language_model.layers.9.self_attn.o_proj",
+    "language_model.layers.0.self_attn.q_proj",
+    "language_model.layers.14.self_attn.k_proj",
+    "18.mlp.gate_proj",
+    "language_model.layers.3.self_attn.q_proj",
+    "language_model.layers.7.self_attn.q_proj",
+    "language_model.layers.15.mlp.up_proj",
+    "34.self_attn.o_proj",
+    "language_model.layers.8.self_attn.q_proj",
+    "language_model.layers.10.mlp.up_proj",
+    "language_model.layers.14.mlp.up_proj",
+    "29.mlp.gate_proj",
+    "language_model.layers.1.self_attn.k_proj",
+    "language_model.layers.13.self_attn.o_proj",
+    "25.self_attn.o_proj",
+    "17.self_attn.v_proj",
+    "30.mlp.gate_proj",
+    "26.self_attn.v_proj",
+    "language_model.layers.3.self_attn.v_proj",
+    "19.self_attn.v_proj",
+    "language_model.layers.0.mlp.down_proj",
+    "language_model.layers.3.self_attn.k_proj",
+    "18.self_attn.o_proj",
+    "29.self_attn.o_proj",
+    "25.mlp.gate_proj",
+    "language_model.layers.9.mlp.gate_proj",
+    "16.self_attn.o_proj",
+    "language_model.layers.12.self_attn.o_proj",
+    "33.self_attn.o_proj",
+    "output_proj",
+    "22.self_attn.o_proj",
+    "17.self_attn.o_proj",
+    "23.self_attn.v_proj",
+    "language_model.layers.12.mlp.gate_proj",
+    "22.self_attn.q_proj",
+    "relative_k_proj",
+    "30.mlp.down_proj",
+    "22.mlp.down_proj",
+    "31.mlp.up_proj",
+    "18.self_attn.k_proj",
+    "language_model.layers.1.mlp.up_proj",
+    "28.mlp.down_proj",
+    "language_model.layers.15.self_attn.q_proj",
+    "language_model.layers.10.mlp.gate_proj",
+    "language_model.layers.0.self_attn.v_proj",
+    "23.mlp.down_proj",
+    "29.self_attn.v_proj",
+    "language_model.layers.8.self_attn.v_proj",
+    "22.self_attn.k_proj",
+    "17.mlp.gate_proj",
+    "24.self_attn.o_proj",
+    "16.mlp.down_proj",
+    "33.mlp.down_proj",
+    "language_model.layers.2.self_attn.k_proj",
+    "23.self_attn.k_proj",
+    "30.self_attn.v_proj",
+    "language_model.layers.4.self_attn.o_proj",
+    "language_model.layers.1.self_attn.q_proj",
+    "language_model.layers.7.mlp.up_proj",
+    "language_model.layers.5.self_attn.k_proj",
+    "19.mlp.up_proj",
+    "per_layer_input_gate",
+    "language_model.layers.0.mlp.up_proj",
+    "30.self_attn.k_proj",
+    "language_model.layers.8.self_attn.k_proj",
+    "18.self_attn.q_proj",
+    "language_model.layers.4.mlp.up_proj",
+    "19.mlp.gate_proj",
+    "language_model.layers.0.self_attn.o_proj",
+    "language_model.layers.12.self_attn.v_proj",
+    "34.mlp.up_proj",
+    "language_model.layers.13.self_attn.q_proj",
+    "27.mlp.up_proj",
+    "26.mlp.gate_proj",
+    "language_model.layers.6.self_attn.k_proj",
+    "26.self_attn.o_proj",
+    "language_model.layers.10.self_attn.q_proj",
+    "language_model.layers.10.self_attn.k_proj",
+    "language_model.layers.10.mlp.down_proj",
+    "33.mlp.up_proj",
+    "language_model.layers.6.self_attn.v_proj",
+    "16.self_attn.q_proj",
+    "27.self_attn.v_proj",
+    "28.mlp.up_proj",
+    "language_model.layers.15.mlp.down_proj",
+    "language_model.layers.14.self_attn.o_proj",
+    "language_model.layers.5.mlp.up_proj",
+    "language_model.layers.13.mlp.gate_proj",
+    "language_model.layers.14.mlp.gate_proj",
+    "language_model.layers.11.self_attn.v_proj",
+    "language_model.layers.14.self_attn.v_proj",
+    "18.mlp.down_proj",
+    "17.self_attn.k_proj",
+    "language_model.layers.7.self_attn.o_proj",
+    "32.mlp.up_proj",
+    "language_model.layers.6.mlp.up_proj",
+    "language_model.layers.9.self_attn.v_proj",
+    "language_model.layers.3.mlp.gate_proj",
+    "language_model.layers.6.mlp.down_proj",
+    "language_model.layers.12.mlp.up_proj",
+    "language_model.layers.6.mlp.gate_proj",
+    "28.self_attn.q_proj",
+    "language_model.layers.15.self_attn.k_proj",
+    "language_model.layers.8.self_attn.o_proj",
+    "language_model.layers.5.mlp.down_proj",
+    "27.self_attn.q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-1875/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:189b5596e8b491dd4a55648ab528a50d09009808e7d6767eef9a99df5d2f42e4
+size 1688992024

checkpoint-1875/chat_template.jinja

@@ -0,0 +1,263 @@
+{%- macro format_parameters(properties, required) -%}
+    {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
+    {%- set ns = namespace(found_first=false) -%}
+    {%- for key, value in properties | dictsort -%}
+        {%- set add_comma = false -%}
+        {%- if key not in standard_keys -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {{ key }}:{
+            {%- if value['description'] -%}
+                description:<|"|>{{ value['description'] }}<|"|>
+                {%- set add_comma = true -%}
+            {%- endif -%}
+            {%- if value['nullable'] %}
+                {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                nullable:true
+            {%- endif -%}
+            {%- if value['type'] | upper == 'STRING' -%}
+                {%- if value['enum'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    enum:{{ format_argument(value['enum']) }}
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'OBJECT' -%}
+                ,properties:{
+                {%- if value['properties'] is defined and value['properties'] is mapping -%}
+                    {{- format_parameters(value['properties'], value['required'] | default([])) -}}
+                {%- elif value is mapping -%}
+                    {{- format_parameters(value, value['required'] | default([])) -}}
+                {%- endif -%}
+                }
+                {%- if value['required'] -%}
+                    ,required:[
+                    {%- for item in value['required'] | default([]) -%}
+                        <|"|>{{- item -}}<|"|>
+                        {%- if not loop.last %},{% endif -%}
+                    {%- endfor -%}
+                    ]
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'ARRAY' -%}
+                {%- if value['items'] is mapping and value['items'] -%}
+                    ,items:{
+                    {%- set ns_items = namespace(found_first=false) -%}
+                    {%- for item_key, item_value in value['items'] | dictsort -%}
+                        {%- if item_value is not none -%}
+                            {%- if ns_items.found_first %},{% endif -%}
+                            {%- set ns_items.found_first = true -%}
+                            {%- if item_key == 'properties' -%}
+                                properties:{
+                                {%- if item_value is mapping -%}
+                                    {{- format_parameters(item_value, value['items']['required'] | default([])) -}}
+                                {%- endif -%}
+                                }
+                            {%- elif item_key == 'required' -%}
+                                required:[
+                                {%- for req_item in item_value -%}
+                                    <|"|>{{- req_item -}}<|"|>
+                                    {%- if not loop.last %},{% endif -%}
+                                {%- endfor -%}
+                                ]
+                            {%- elif item_key == 'type' -%}
+                                {%- if item_value is string -%}
+                                    type:{{ format_argument(item_value | upper) }}
+                                {%- else -%}
+                                    type:{{ format_argument(item_value | map('upper') | list) }}
+                                {%- endif -%}
+                            {%- else -%}
+                                {{ item_key }}:{{ format_argument(item_value) }}
+                            {%- endif -%}
+                        {%- endif -%}
+                    {%- endfor -%}
+                    }
+                {%- endif -%}
+            {%- endif -%}
+            {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+            type:<|"|>{{ value['type'] | upper }}<|"|>}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endmacro -%}
+{%- macro format_function_declaration(tool_data) -%}
+    declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
+    {%- set params = tool_data['function']['parameters'] -%}
+    {%- if params -%}
+        ,parameters:{
+        {%- if params['properties'] -%}
+            properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
+        {%- endif -%}
+        {%- if params['required'] -%}
+            required:[
+            {%- for item in params['required'] -%}
+                <|"|>{{- item -}}<|"|>
+                {{- ',' if not loop.last -}}
+            {%- endfor -%}
+            ],
+        {%- endif -%}
+        {%- if params['type'] -%}
+            type:<|"|>{{- params['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    {%- if 'response' in tool_data['function'] -%}
+        {%- set response_declaration = tool_data['function']['response'] -%}
+        ,response:{
+        {%- if response_declaration['description'] -%}
+            description:<|"|>{{- response_declaration['description'] -}}<|"|>,
+        {%- endif -%}
+        {%- if response_declaration['type'] | upper == 'OBJECT' -%}
+            type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    }
+{%- endmacro -%}
+{%- macro format_argument(argument, escape_keys=True) -%}
+    {%- if argument is string -%}
+        {{- '<|"|>' + argument + '<|"|>' -}}
+    {%- elif argument is boolean -%}
+        {{- 'true' if argument else 'false' -}}
+    {%- elif argument is mapping -%}
+        {{- '{' -}}
+        {%- set ns = namespace(found_first=false) -%}
+        {%- for key, value in argument | dictsort -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {%- if escape_keys -%}
+                {{- '<|"|>' + key + '<|"|>' -}}
+            {%- else -%}
+                {{- key -}}
+            {%- endif -%}
+            :{{- format_argument(value, escape_keys=escape_keys) -}}
+        {%- endfor -%}
+        {{- '}' -}}
+    {%- elif argument is sequence -%}
+        {{- '[' -}}
+        {%- for item in argument -%}
+            {{- format_argument(item, escape_keys=escape_keys) -}}
+            {%- if not loop.last %},{% endif -%}
+        {%- endfor -%}
+        {{- ']' -}}
+    {%- else -%}
+        {{- argument -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- macro strip_thinking(text) -%}
+    {%- set ns = namespace(result='') -%}
+    {%- for part in text.split('<channel|>') -%}
+        {%- if '<|channel>' in part -%}
+            {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
+        {%- else -%}
+            {%- set ns.result = ns.result + part -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- ns.result | trim -}}
+{%- endmacro -%}
+
+{%- set ns = namespace(prev_message_type=None) -%}
+{%- set loop_messages = messages -%}
+{{ bos_token }}
+{#- Handle System/Tool Definitions Block -#}
+{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}
+    {{- '<|turn>system\n' -}}
+
+    {#- Inject Thinking token at the very top of the FIRST system turn -#}
+    {%- if enable_thinking is defined and enable_thinking -%}
+        {{- '<|think|>' -}}
+        {%- set ns.prev_message_type = 'think' -%}
+    {%- endif -%}
+
+    {%- if messages[0]['role'] in ['system', 'developer'] -%}
+        {{- messages[0]['content'] | trim -}}
+        {%- set loop_messages = messages[1:] -%}
+    {%- endif -%}
+
+    {%- if tools -%}
+        {%- for tool in tools %}
+            {{- '<|tool>' -}}
+            {{- format_function_declaration(tool) | trim -}}
+            {{- '<tool|>' -}}
+        {%- endfor %}
+        {%- set ns.prev_message_type = 'tool' -%}
+    {%- endif -%}
+
+    {{- '<turn|>\n' -}}
+{%- endif %}
+
+{#- Loop through messages -#}
+{%- for message in loop_messages -%}
+    {%- set ns.prev_message_type = None -%}
+    {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
+        {{- '<|turn>' + role + '\n' }}
+
+            {%- if message['tool_calls'] -%}
+                {%- for tool_call in message['tool_calls'] -%}
+                    {%- set function = tool_call['function'] -%}
+                    {{- '<|tool_call>call:' + function['name'] + '{' -}}
+                    {%- if function['arguments'] is mapping -%}
+                        {%- set ns_args = namespace(found_first=false) -%}
+                        {%- for key, value in function['arguments'] | dictsort -%}
+                            {%- if ns_args.found_first %},{% endif -%}
+                            {%- set ns_args.found_first = true -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                        {%- endfor -%}
+                    {%- elif function['arguments'] is string -%}
+                        {{- function['arguments'] -}}
+                    {%- endif -%}
+                    {{- '}<tool_call|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_call' -%}
+            {%- endif -%}
+
+            {%- if message['tool_responses'] -%}
+                {#- Tool Response handling -#}
+                {%- for tool_response in message['tool_responses'] -%}
+                    {{- '<|tool_response>' -}}
+                    {%- if tool_response['response'] is mapping -%}
+                        {{- 'response:' + tool_response['name'] | default('unknown') + '{' -}}
+                        {%- for key, value in tool_response['response'] | dictsort -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                            {%- if not loop.last %},{% endif -%}
+                        {%- endfor -%}
+                        {{- '}' -}}
+                    {%- else -%}
+                        {{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}}
+                    {%- endif -%}
+                    {{- '<tool_response|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_response' -%}
+            {%- endif -%}
+
+            {%- if message['content'] is string -%}
+                {%- if role == 'model' -%}
+                    {{- strip_thinking(message['content']) -}}
+                {%- else -%}
+                    {{- message['content'] | trim -}}
+                {%- endif -%}
+            {%- elif message['content'] is sequence -%}
+                {%- for item in message['content'] -%}
+                    {%- if item['type'] == 'text' -%}
+                        {%- if role == 'model' -%}
+                            {{- strip_thinking(item['text']) -}}
+                        {%- else -%}
+                            {{- item['text'] | trim -}}
+                        {%- endif -%}
+                    {%- elif item['type'] == 'image' -%}
+                        {{- '\n\n<|image|>\n\n' -}}
+                        {%- set ns.prev_message_type = 'image' -%}
+                    {%- elif item['type'] == 'audio' -%}
+                        {{- '<|audio|>' -}}
+                        {%- set ns.prev_message_type = 'audio' -%}
+                    {%- elif item['type'] == 'video' -%}
+                        {{- '\n\n<|video|>\n\n' -}}
+                        {%- set ns.prev_message_type = 'video' -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+
+        {%- if not (message['tool_responses'] and not message['content']) -%}
+            {{- '<turn|>\n' -}}
+        {%- endif -%}
+{%- endfor -%}
+
+{%- if add_generation_prompt -%}
+    {%- if ns.prev_message_type != 'tool_response' -%}
+        {{- '<|turn>model\n' -}}
+    {%- endif -%}
+{%- endif -%}

checkpoint-1875/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fc8707a1d8c24906c9eb5e80ee57b512720aacef79a08a837c07fb09d5e5cd3
+size 3331821841

checkpoint-1875/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d5ba88fe62a37ece60c20d432b18ab977bdec84877b620580feadbc1eefe3a8
+size 14709

checkpoint-1875/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:533def6869947a2b92c494f2c895e6d073f06cec414ab54ae1ab3ed805984837
+size 1465

checkpoint-1875/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f
+size 32169626

checkpoint-1875/tokenizer_config.json

@@ -0,0 +1,95 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [
+    "<|video|>"
+  ],
+  "image_token": "<|image|>",
+  "is_local": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "audio_token": "<|audio|>",
+    "boa_token": "<|audio>",
+    "boi_token": "<|image>",
+    "eoa_token": "<audio|>",
+    "eoc_token": "<channel|>",
+    "eoi_token": "<image|>",
+    "eot_token": "<turn|>",
+    "escape_token": "<|\"|>",
+    "etc_token": "<tool_call|>",
+    "etd_token": "<tool|>",
+    "etr_token": "<tool_response|>",
+    "image_token": "<|image|>",
+    "soc_token": "<|channel>",
+    "sot_token": "<|turn>",
+    "stc_token": "<|tool_call>",
+    "std_token": "<|tool>",
+    "str_token": "<|tool_response>",
+    "think_token": "<|think|>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma4Processor",
+  "response_schema": {
+    "properties": {
+      "content": {
+        "type": "string"
+      },
+      "role": {
+        "const": "assistant"
+      },
+      "thinking": {
+        "type": "string"
+      },
+      "tool_calls": {
+        "items": {
+          "properties": {
+            "function": {
+              "properties": {
+                "arguments": {
+                  "additionalProperties": {},
+                  "type": "object",
+                  "x-parser": "gemma4-tool-call"
+                },
+                "name": {
+                  "type": "string"
+                }
+              },
+              "type": "object",
+              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"
+            },
+            "type": {
+              "const": "function"
+            }
+          },
+          "type": "object"
+        },
+        "type": "array",
+        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"
+      }
+    },
+    "type": "object",
+    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<content>(?:(?!\\<\\|tool_call\\>)(?!\\<turn\\|\\>).)+)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?:\\<turn\\|\\>)?"
+  },
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-1875/trainer_state.json

@@ -0,0 +1,1947 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 1875,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.5365671901141895,
+      "epoch": 0.0016,
+      "grad_norm": 4.21875,
+      "learning_rate": 0.0,
+      "loss": 5.494313716888428,
+      "mean_token_accuracy": 0.6573571685482474,
+      "num_tokens": 348701.0,
+      "step": 1
+    },
+    {
+      "entropy": 1.348903712299135,
+      "epoch": 0.016,
+      "grad_norm": 2.59375,
+      "learning_rate": 3.157894736842105e-05,
+      "loss": 5.544558631049262,
+      "mean_token_accuracy": 0.6651431231035126,
+      "num_tokens": 371496.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.360134594142437,
+      "epoch": 0.032,
+      "grad_norm": 3.296875,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 5.742576599121094,
+      "mean_token_accuracy": 0.6662841200828552,
+      "num_tokens": 400839.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.375771728157997,
+      "epoch": 0.048,
+      "grad_norm": 3.140625,
+      "learning_rate": 0.0001017543859649123,
+      "loss": 5.714028167724609,
+      "mean_token_accuracy": 0.6566085889935493,
+      "num_tokens": 431025.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.2748979568481444,
+      "epoch": 0.064,
+      "grad_norm": 2.828125,
+      "learning_rate": 0.0001368421052631579,
+      "loss": 5.11612548828125,
+      "mean_token_accuracy": 0.6816287323832512,
+      "num_tokens": 459089.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.3155348658561707,
+      "epoch": 0.08,
+      "grad_norm": 3.15625,
+      "learning_rate": 0.00017192982456140353,
+      "loss": 5.35040512084961,
+      "mean_token_accuracy": 0.6721501812338829,
+      "num_tokens": 484648.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1791150525212288,
+      "epoch": 0.096,
+      "grad_norm": 2.375,
+      "learning_rate": 0.00019999940277008808,
+      "loss": 4.642951583862304,
+      "mean_token_accuracy": 0.7028968930244446,
+      "num_tokens": 511463.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1177749201655387,
+      "epoch": 0.112,
+      "grad_norm": 2.421875,
+      "learning_rate": 0.0001999785004721968,
+      "loss": 4.523546981811523,
+      "mean_token_accuracy": 0.7057401552796364,
+      "num_tokens": 538438.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.1784485399723053,
+      "epoch": 0.128,
+      "grad_norm": 2.25,
+      "learning_rate": 0.00019992774381199778,
+      "loss": 4.712202072143555,
+      "mean_token_accuracy": 0.7004493281245232,
+      "num_tokens": 569421.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.0666756927967072,
+      "epoch": 0.144,
+      "grad_norm": 4.25,
+      "learning_rate": 0.00019984714794582683,
+      "loss": 4.181539154052734,
+      "mean_token_accuracy": 0.7218294009566307,
+      "num_tokens": 595806.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1410984337329864,
+      "epoch": 0.16,
+      "grad_norm": 2.59375,
+      "learning_rate": 0.00019973673694024,
+      "loss": 4.578577041625977,
+      "mean_token_accuracy": 0.70914496332407,
+      "num_tokens": 626238.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.058976437151432,
+      "epoch": 0.176,
+      "grad_norm": 4.34375,
+      "learning_rate": 0.0001995965437648273,
+      "loss": 4.2924964904785154,
+      "mean_token_accuracy": 0.7180401891469955,
+      "num_tokens": 653441.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.2249037250876427,
+      "epoch": 0.192,
+      "grad_norm": 3.96875,
+      "learning_rate": 0.00019942661028236745,
+      "loss": 5.084912490844727,
+      "mean_token_accuracy": 0.6847136527299881,
+      "num_tokens": 682582.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.1726351112127305,
+      "epoch": 0.208,
+      "grad_norm": 3.296875,
+      "learning_rate": 0.00019922698723632767,
+      "loss": 5.018433380126953,
+      "mean_token_accuracy": 0.7055773109197616,
+      "num_tokens": 710468.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.3469643473625184,
+      "epoch": 0.224,
+      "grad_norm": 2.875,
+      "learning_rate": 0.000198997734235711,
+      "loss": 5.7476848602294925,
+      "mean_token_accuracy": 0.6790369123220443,
+      "num_tokens": 740979.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.187085197865963,
+      "epoch": 0.24,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.0001987389197372567,
+      "loss": 5.085608673095703,
+      "mean_token_accuracy": 0.7057169727981091,
+      "num_tokens": 768427.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.2938067495822907,
+      "epoch": 0.256,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.0001984506210249986,
+      "loss": 5.570721054077149,
+      "mean_token_accuracy": 0.6840770006179809,
+      "num_tokens": 796744.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.1161220327019692,
+      "epoch": 0.272,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.00019813292418718732,
+      "loss": 4.821302795410157,
+      "mean_token_accuracy": 0.7168306574225426,
+      "num_tokens": 822225.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.2217196941375732,
+      "epoch": 0.288,
+      "grad_norm": 2.484375,
+      "learning_rate": 0.00019778592409058378,
+      "loss": 5.171672821044922,
+      "mean_token_accuracy": 0.6998881995677948,
+      "num_tokens": 848049.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1091715931892394,
+      "epoch": 0.304,
+      "grad_norm": 3.109375,
+      "learning_rate": 0.00019740972435213115,
+      "loss": 4.743664169311524,
+      "mean_token_accuracy": 0.7203769966959953,
+      "num_tokens": 873172.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.2154748886823654,
+      "epoch": 0.32,
+      "grad_norm": 2.21875,
+      "learning_rate": 0.00019700443730801413,
+      "loss": 5.212123107910156,
+      "mean_token_accuracy": 0.7076091229915619,
+      "num_tokens": 901716.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.161145742237568,
+      "epoch": 0.336,
+      "grad_norm": 2.078125,
+      "learning_rate": 0.00019657018398011434,
+      "loss": 4.889891052246094,
+      "mean_token_accuracy": 0.7145821407437325,
+      "num_tokens": 928506.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.1209837168455123,
+      "epoch": 0.352,
+      "grad_norm": 2.09375,
+      "learning_rate": 0.00019610709403987246,
+      "loss": 4.862021636962891,
+      "mean_token_accuracy": 0.7152845054864884,
+      "num_tokens": 958723.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.1322738587856294,
+      "epoch": 0.368,
+      "grad_norm": 2.203125,
+      "learning_rate": 0.00019561530576956703,
+      "loss": 4.911846923828125,
+      "mean_token_accuracy": 0.7134342208504677,
+      "num_tokens": 987751.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.1780440196394921,
+      "epoch": 0.384,
+      "grad_norm": 2.421875,
+      "learning_rate": 0.00019509496602102252,
+      "loss": 5.0671440124511715,
+      "mean_token_accuracy": 0.7104542285203934,
+      "num_tokens": 1018315.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.1776200592517854,
+      "epoch": 0.4,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.00019454623017175812,
+      "loss": 5.145317459106446,
+      "mean_token_accuracy": 0.7086253471672534,
+      "num_tokens": 1046627.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.1513659209012985,
+      "epoch": 0.416,
+      "grad_norm": 2.140625,
+      "learning_rate": 0.00019396926207859084,
+      "loss": 4.8750255584716795,
+      "mean_token_accuracy": 0.7203947469592095,
+      "num_tokens": 1075027.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.1364729449152946,
+      "epoch": 0.432,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.00019336423402870653,
+      "loss": 4.866281890869141,
+      "mean_token_accuracy": 0.7150357633829116,
+      "num_tokens": 1102334.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.108440762758255,
+      "epoch": 0.448,
+      "grad_norm": 1.9453125,
+      "learning_rate": 0.00019273132668821364,
+      "loss": 4.697020721435547,
+      "mean_token_accuracy": 0.7265092357993126,
+      "num_tokens": 1131415.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1001321360468865,
+      "epoch": 0.464,
+      "grad_norm": 2.125,
+      "learning_rate": 0.00019207072904819486,
+      "loss": 4.747102355957031,
+      "mean_token_accuracy": 0.7231316924095154,
+      "num_tokens": 1158280.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.1048941344022751,
+      "epoch": 0.48,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.00019138263836827288,
+      "loss": 4.644550323486328,
+      "mean_token_accuracy": 0.7270571634173393,
+      "num_tokens": 1188651.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.0945911705493927,
+      "epoch": 0.496,
+      "grad_norm": 1.984375,
+      "learning_rate": 0.00019066726011770726,
+      "loss": 4.7493640899658205,
+      "mean_token_accuracy": 0.7258608743548394,
+      "num_tokens": 1216504.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.0356923416256905,
+      "epoch": 0.512,
+      "grad_norm": 2.1875,
+      "learning_rate": 0.00018992480791403958,
+      "loss": 4.3843944549560545,
+      "mean_token_accuracy": 0.7372462660074234,
+      "num_tokens": 1243732.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.0773125097155571,
+      "epoch": 0.528,
+      "grad_norm": 2.328125,
+      "learning_rate": 0.0001891555034593055,
+      "loss": 4.6180767059326175,
+      "mean_token_accuracy": 0.7283408164978027,
+      "num_tokens": 1269272.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.050987295806408,
+      "epoch": 0.544,
+      "grad_norm": 1.640625,
+      "learning_rate": 0.00018835957647383303,
+      "loss": 4.4863533020019535,
+      "mean_token_accuracy": 0.7359548002481461,
+      "num_tokens": 1296001.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.0461898490786552,
+      "epoch": 0.56,
+      "grad_norm": 2.453125,
+      "learning_rate": 0.000187537264627646,
+      "loss": 4.434906005859375,
+      "mean_token_accuracy": 0.7349108412861824,
+      "num_tokens": 1322180.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.0046504065394402,
+      "epoch": 0.576,
+      "grad_norm": 2.125,
+      "learning_rate": 0.00018668881346949417,
+      "loss": 4.309226989746094,
+      "mean_token_accuracy": 0.73991359770298,
+      "num_tokens": 1347532.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.0861794739961623,
+      "epoch": 0.592,
+      "grad_norm": 1.890625,
+      "learning_rate": 0.0001858144763535302,
+      "loss": 4.599692535400391,
+      "mean_token_accuracy": 0.7285229310393333,
+      "num_tokens": 1373584.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.0600647926330566,
+      "epoch": 0.608,
+      "grad_norm": 2.203125,
+      "learning_rate": 0.00018491451436365627,
+      "loss": 4.5491493225097654,
+      "mean_token_accuracy": 0.7357145607471466,
+      "num_tokens": 1399594.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.098224511742592,
+      "epoch": 0.624,
+      "grad_norm": 1.9296875,
+      "learning_rate": 0.00018398919623556238,
+      "loss": 4.8250572204589846,
+      "mean_token_accuracy": 0.7251888766884804,
+      "num_tokens": 1428792.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.1232757449150086,
+      "epoch": 0.64,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.00018303879827647975,
+      "loss": 4.712010192871094,
+      "mean_token_accuracy": 0.7287421196699142,
+      "num_tokens": 1457889.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.0411028936505318,
+      "epoch": 0.656,
+      "grad_norm": 1.859375,
+      "learning_rate": 0.00018206360428267332,
+      "loss": 4.405958938598633,
+      "mean_token_accuracy": 0.7350577011704444,
+      "num_tokens": 1484479.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.1146624743938447,
+      "epoch": 0.672,
+      "grad_norm": 1.734375,
+      "learning_rate": 0.00018106390545469795,
+      "loss": 4.742184448242187,
+      "mean_token_accuracy": 0.7282734125852585,
+      "num_tokens": 1514268.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.0480840012431145,
+      "epoch": 0.688,
+      "grad_norm": 1.78125,
+      "learning_rate": 0.0001800400003104436,
+      "loss": 4.4207916259765625,
+      "mean_token_accuracy": 0.7365467861294747,
+      "num_tokens": 1540474.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.0067586719989776,
+      "epoch": 0.704,
+      "grad_norm": 1.921875,
+      "learning_rate": 0.0001789921945959958,
+      "loss": 4.31513671875,
+      "mean_token_accuracy": 0.7437789484858512,
+      "num_tokens": 1568557.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.0166094586253167,
+      "epoch": 0.72,
+      "grad_norm": 1.6171875,
+      "learning_rate": 0.0001779208011943371,
+      "loss": 4.292739486694336,
+      "mean_token_accuracy": 0.7471857726573944,
+      "num_tokens": 1594403.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.0385540708899499,
+      "epoch": 0.736,
+      "grad_norm": 2.75,
+      "learning_rate": 0.00017682614003191807,
+      "loss": 4.415458679199219,
+      "mean_token_accuracy": 0.7400641202926636,
+      "num_tokens": 1624111.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.0706985503435136,
+      "epoch": 0.752,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.0001757085379831246,
+      "loss": 4.554729080200195,
+      "mean_token_accuracy": 0.7342943042516709,
+      "num_tokens": 1652609.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.9689434483647347,
+      "epoch": 0.768,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.00017456832877267084,
+      "loss": 4.2303211212158205,
+      "mean_token_accuracy": 0.7474748462438583,
+      "num_tokens": 1678589.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.1837652444839477,
+      "epoch": 0.784,
+      "grad_norm": 2.078125,
+      "learning_rate": 0.00017340585287594604,
+      "loss": 4.968061447143555,
+      "mean_token_accuracy": 0.7166377156972885,
+      "num_tokens": 1708513.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.042286379635334,
+      "epoch": 0.8,
+      "grad_norm": 2.515625,
+      "learning_rate": 0.00017222145741734626,
+      "loss": 4.422880172729492,
+      "mean_token_accuracy": 0.7405225187540054,
+      "num_tokens": 1736283.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.9832519263029098,
+      "epoch": 0.816,
+      "grad_norm": 1.6171875,
+      "learning_rate": 0.00017101549606662024,
+      "loss": 4.119276428222657,
+      "mean_token_accuracy": 0.7495438739657402,
+      "num_tokens": 1765017.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.9872105062007904,
+      "epoch": 0.832,
+      "grad_norm": 1.71875,
+      "learning_rate": 0.00016978832893326074,
+      "loss": 4.234106826782226,
+      "mean_token_accuracy": 0.7473902180790901,
+      "num_tokens": 1790682.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.9845283895730972,
+      "epoch": 0.848,
+      "grad_norm": 1.4921875,
+      "learning_rate": 0.00016854032245897308,
+      "loss": 4.146430969238281,
+      "mean_token_accuracy": 0.7520387843251228,
+      "num_tokens": 1820050.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.0613365799188614,
+      "epoch": 0.864,
+      "grad_norm": 2.046875,
+      "learning_rate": 0.00016727184930825288,
+      "loss": 4.48931655883789,
+      "mean_token_accuracy": 0.7396015107631684,
+      "num_tokens": 1847079.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.9684726029634476,
+      "epoch": 0.88,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.00016598328825710533,
+      "loss": 4.12475357055664,
+      "mean_token_accuracy": 0.7526269048452378,
+      "num_tokens": 1872383.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.9990086019039154,
+      "epoch": 0.896,
+      "grad_norm": 1.8984375,
+      "learning_rate": 0.00016467502407993992,
+      "loss": 4.360863494873047,
+      "mean_token_accuracy": 0.7387492001056671,
+      "num_tokens": 1903630.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.9708257809281349,
+      "epoch": 0.912,
+      "grad_norm": 1.6328125,
+      "learning_rate": 0.00016334744743467364,
+      "loss": 4.054442596435547,
+      "mean_token_accuracy": 0.7545965671539306,
+      "num_tokens": 1930592.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.0035204842686654,
+      "epoch": 0.928,
+      "grad_norm": 1.8984375,
+      "learning_rate": 0.00016200095474607753,
+      "loss": 4.25194206237793,
+      "mean_token_accuracy": 0.7464351058006287,
+      "num_tokens": 1960212.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.9635734960436821,
+      "epoch": 0.944,
+      "grad_norm": 1.53125,
+      "learning_rate": 0.00016063594808740113,
+      "loss": 4.0587310791015625,
+      "mean_token_accuracy": 0.7547481089830399,
+      "num_tokens": 1986049.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.9986386775970459,
+      "epoch": 0.96,
+      "grad_norm": 1.703125,
+      "learning_rate": 0.0001592528350603103,
+      "loss": 4.331460952758789,
+      "mean_token_accuracy": 0.7503352165222168,
+      "num_tokens": 2013266.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.006168755888939,
+      "epoch": 0.976,
+      "grad_norm": 1.734375,
+      "learning_rate": 0.00015785202867317407,
+      "loss": 4.238505554199219,
+      "mean_token_accuracy": 0.7441465124487877,
+      "num_tokens": 2041824.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.0482723653316497,
+      "epoch": 0.992,
+      "grad_norm": 1.9296875,
+      "learning_rate": 0.0001564339472177373,
+      "loss": 4.394336700439453,
+      "mean_token_accuracy": 0.7416493371129036,
+      "num_tokens": 2071693.0,
+      "step": 620
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.0188530464172363,
+      "eval_loss": 1.0276753902435303,
+      "eval_mean_token_accuracy": 0.7517141411304474,
+      "eval_num_tokens": 2084150.0,
+      "eval_runtime": 72.9791,
+      "eval_samples_per_second": 13.703,
+      "eval_steps_per_second": 3.426,
+      "step": 625
+    },
+    {
+      "entropy": 0.9583894208073616,
+      "epoch": 1.008,
+      "grad_norm": 1.40625,
+      "learning_rate": 0.0001549990141442153,
+      "loss": 3.791181945800781,
+      "mean_token_accuracy": 0.760094690322876,
+      "num_tokens": 2099483.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.7975272431969642,
+      "epoch": 1.024,
+      "grad_norm": 1.390625,
+      "learning_rate": 0.00015354765793484834,
+      "loss": 3.2353271484375,
+      "mean_token_accuracy": 0.7800843045115471,
+      "num_tokens": 2127344.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.8557936266064644,
+      "epoch": 1.04,
+      "grad_norm": 1.921875,
+      "learning_rate": 0.00015208031197595356,
+      "loss": 3.591104507446289,
+      "mean_token_accuracy": 0.769377326965332,
+      "num_tokens": 2157286.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.8815567880868912,
+      "epoch": 1.056,
+      "grad_norm": 1.4765625,
+      "learning_rate": 0.0001505974144285124,
+      "loss": 3.509926986694336,
+      "mean_token_accuracy": 0.7711383983492851,
+      "num_tokens": 2187199.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.7711537912487983,
+      "epoch": 1.072,
+      "grad_norm": 1.3828125,
+      "learning_rate": 0.00014909940809733222,
+      "loss": 3.108415412902832,
+      "mean_token_accuracy": 0.7885575816035271,
+      "num_tokens": 2213742.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.8327512949705124,
+      "epoch": 1.088,
+      "grad_norm": 6.5625,
+      "learning_rate": 0.00014758674029882152,
+      "loss": 3.380535125732422,
+      "mean_token_accuracy": 0.7737385779619217,
+      "num_tokens": 2242448.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.8324707329273224,
+      "epoch": 1.104,
+      "grad_norm": 1.3125,
+      "learning_rate": 0.00014605986272741748,
+      "loss": 3.3809837341308593,
+      "mean_token_accuracy": 0.7749153047800064,
+      "num_tokens": 2273149.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.8442294985055924,
+      "epoch": 1.12,
+      "grad_norm": 1.578125,
+      "learning_rate": 0.0001445192313207067,
+      "loss": 3.3175960540771485,
+      "mean_token_accuracy": 0.7804568380117416,
+      "num_tokens": 2302497.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.7547720111906528,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 1.625,
+      "learning_rate": 0.00014296530612327863,
+      "loss": 3.042662239074707,
+      "mean_token_accuracy": 0.7904633671045304,
+      "num_tokens": 2328935.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.7825665086507797,
+      "epoch": 1.152,
+      "grad_norm": 1.625,
+      "learning_rate": 0.00014139855114935252,
+      "loss": 3.1688852310180664,
+      "mean_token_accuracy": 0.7849456086754799,
+      "num_tokens": 2355711.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.7932837694883347,
+      "epoch": 1.168,
+      "grad_norm": 1.453125,
+      "learning_rate": 0.00013981943424421932,
+      "loss": 3.217195510864258,
+      "mean_token_accuracy": 0.7835724055767059,
+      "num_tokens": 2383138.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.7902459263801574,
+      "epoch": 1.184,
+      "grad_norm": 1.4296875,
+      "learning_rate": 0.00013822842694453924,
+      "loss": 3.1072725296020507,
+      "mean_token_accuracy": 0.7865744397044182,
+      "num_tokens": 2411055.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.7994440570473671,
+      "epoch": 1.2,
+      "grad_norm": 3.6875,
+      "learning_rate": 0.00013662600433753745,
+      "loss": 3.352284622192383,
+      "mean_token_accuracy": 0.7770974606275558,
+      "num_tokens": 2437837.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.7930429011583329,
+      "epoch": 1.216,
+      "grad_norm": 1.765625,
+      "learning_rate": 0.00013501264491913906,
+      "loss": 3.1913455963134765,
+      "mean_token_accuracy": 0.7834275960922241,
+      "num_tokens": 2463749.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.844582536816597,
+      "epoch": 1.232,
+      "grad_norm": 1.7734375,
+      "learning_rate": 0.00013338883045108674,
+      "loss": 3.4333824157714843,
+      "mean_token_accuracy": 0.777095565199852,
+      "num_tokens": 2492017.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.7995493680238723,
+      "epoch": 1.248,
+      "grad_norm": 1.3046875,
+      "learning_rate": 0.0001317550458170826,
+      "loss": 3.26405029296875,
+      "mean_token_accuracy": 0.7810850724577904,
+      "num_tokens": 2518076.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.8197085373103619,
+      "epoch": 1.264,
+      "grad_norm": 1.359375,
+      "learning_rate": 0.00013011177887799845,
+      "loss": 3.2417884826660157,
+      "mean_token_accuracy": 0.7789853543043137,
+      "num_tokens": 2547590.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.8223014809191227,
+      "epoch": 1.28,
+      "grad_norm": 1.3828125,
+      "learning_rate": 0.0001284595203261965,
+      "loss": 3.366427993774414,
+      "mean_token_accuracy": 0.7806236863136291,
+      "num_tokens": 2576314.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.8295105174183846,
+      "epoch": 1.296,
+      "grad_norm": 1.5078125,
+      "learning_rate": 0.00012679876353900482,
+      "loss": 3.4096916198730467,
+      "mean_token_accuracy": 0.7710141837596893,
+      "num_tokens": 2604971.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.8094971597194671,
+      "epoch": 1.312,
+      "grad_norm": 1.609375,
+      "learning_rate": 0.00012513000443139112,
+      "loss": 3.2491512298583984,
+      "mean_token_accuracy": 0.7876769602298737,
+      "num_tokens": 2631986.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.798582112789154,
+      "epoch": 1.328,
+      "grad_norm": 1.984375,
+      "learning_rate": 0.00012345374130787854,
+      "loss": 3.19933967590332,
+      "mean_token_accuracy": 0.7822520866990089,
+      "num_tokens": 2656910.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.8112326949834824,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 1.515625,
+      "learning_rate": 0.00012177047471374807,
+      "loss": 3.2226589202880858,
+      "mean_token_accuracy": 0.7859175249934196,
+      "num_tokens": 2686061.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.8292336650192738,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 1.8515625,
+      "learning_rate": 0.00012008070728557186,
+      "loss": 3.453116226196289,
+      "mean_token_accuracy": 0.7712548315525055,
+      "num_tokens": 2714961.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.8000457942485809,
+      "epoch": 1.376,
+      "grad_norm": 1.4375,
+      "learning_rate": 0.00011838494360112185,
+      "loss": 3.1879623413085936,
+      "mean_token_accuracy": 0.7834105402231216,
+      "num_tokens": 2741521.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.7751885265111923,
+      "epoch": 1.392,
+      "grad_norm": 1.2578125,
+      "learning_rate": 0.00011668369002869912,
+      "loss": 3.129037094116211,
+      "mean_token_accuracy": 0.7885062962770462,
+      "num_tokens": 2768309.0,
+      "step": 870
+    },
+    {
+      "entropy": 0.8135885834693909,
+      "epoch": 1.408,
+      "grad_norm": 1.765625,
+      "learning_rate": 0.00011497745457592816,
+      "loss": 3.372562789916992,
+      "mean_token_accuracy": 0.7782492652535439,
+      "num_tokens": 2796984.0,
+      "step": 880
+    },
+    {
+      "entropy": 0.7925616443157196,
+      "epoch": 1.424,
+      "grad_norm": 1.625,
+      "learning_rate": 0.00011326674673806195,
+      "loss": 3.062746047973633,
+      "mean_token_accuracy": 0.7937909960746765,
+      "num_tokens": 2827932.0,
+      "step": 890
+    },
+    {
+      "entropy": 0.7889664500951767,
+      "epoch": 1.44,
+      "grad_norm": 1.375,
+      "learning_rate": 0.00011155207734584263,
+      "loss": 3.2588306427001954,
+      "mean_token_accuracy": 0.7854808464646339,
+      "num_tokens": 2860126.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.7500537633895874,
+      "epoch": 1.456,
+      "grad_norm": 1.453125,
+      "learning_rate": 0.00010983395841296348,
+      "loss": 2.9640825271606444,
+      "mean_token_accuracy": 0.7949178665876389,
+      "num_tokens": 2884946.0,
+      "step": 910
+    },
+    {
+      "entropy": 0.7727585166692734,
+      "epoch": 1.472,
+      "grad_norm": 1.46875,
+      "learning_rate": 0.00010811290298317755,
+      "loss": 3.129817581176758,
+      "mean_token_accuracy": 0.7860683888196945,
+      "num_tokens": 2908387.0,
+      "step": 920
+    },
+    {
+      "entropy": 0.7516519904136658,
+      "epoch": 1.488,
+      "grad_norm": 1.2890625,
+      "learning_rate": 0.0001063894249770989,
+      "loss": 3.1172218322753906,
+      "mean_token_accuracy": 0.7909497246146202,
+      "num_tokens": 2935394.0,
+      "step": 930
+    },
+    {
+      "entropy": 0.8111906915903091,
+      "epoch": 1.504,
+      "grad_norm": 1.2890625,
+      "learning_rate": 0.00010466403903874176,
+      "loss": 3.222005844116211,
+      "mean_token_accuracy": 0.7841308936476707,
+      "num_tokens": 2966607.0,
+      "step": 940
+    },
+    {
+      "entropy": 0.7766234025359153,
+      "epoch": 1.52,
+      "grad_norm": 1.421875,
+      "learning_rate": 0.00010293726038184393,
+      "loss": 3.127474784851074,
+      "mean_token_accuracy": 0.7897476837038994,
+      "num_tokens": 2993998.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.7666798770427704,
+      "epoch": 1.536,
+      "grad_norm": 2.015625,
+      "learning_rate": 0.00010120960463601976,
+      "loss": 3.038217544555664,
+      "mean_token_accuracy": 0.7863978728652,
+      "num_tokens": 3022959.0,
+      "step": 960
+    },
+    {
+      "entropy": 0.7193653047084808,
+      "epoch": 1.552,
+      "grad_norm": 1.453125,
+      "learning_rate": 9.948158769278939e-05,
+      "loss": 2.9356218338012696,
+      "mean_token_accuracy": 0.7945682421326637,
+      "num_tokens": 3049100.0,
+      "step": 970
+    },
+    {
+      "entropy": 0.7440014734864235,
+      "epoch": 1.568,
+      "grad_norm": 1.3046875,
+      "learning_rate": 9.775372555152912e-05,
+      "loss": 2.9548721313476562,
+      "mean_token_accuracy": 0.7947487100958824,
+      "num_tokens": 3073764.0,
+      "step": 980
+    },
+    {
+      "entropy": 0.7881634041666985,
+      "epoch": 1.584,
+      "grad_norm": 1.3828125,
+      "learning_rate": 9.602653416539031e-05,
+      "loss": 3.2630630493164063,
+      "mean_token_accuracy": 0.7804958373308182,
+      "num_tokens": 3101016.0,
+      "step": 990
+    },
+    {
+      "entropy": 0.7688482075929641,
+      "epoch": 1.6,
+      "grad_norm": 1.1171875,
+      "learning_rate": 9.430052928723153e-05,
+      "loss": 3.0473140716552733,
+      "mean_token_accuracy": 0.7904627084732055,
+      "num_tokens": 3131323.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.8112111315131187,
+      "epoch": 1.616,
+      "grad_norm": 1.3046875,
+      "learning_rate": 9.257622631561085e-05,
+      "loss": 3.270753860473633,
+      "mean_token_accuracy": 0.7852039277553559,
+      "num_tokens": 3161781.0,
+      "step": 1010
+    },
+    {
+      "entropy": 0.7603359699249268,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 1.078125,
+      "learning_rate": 9.085414014088369e-05,
+      "loss": 3.0997121810913084,
+      "mean_token_accuracy": 0.7898457184433937,
+      "num_tokens": 3189668.0,
+      "step": 1020
+    },
+    {
+      "entropy": 0.789544765651226,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 1.3203125,
+      "learning_rate": 8.913478499145254e-05,
+      "loss": 3.1730154037475584,
+      "mean_token_accuracy": 0.7893433123826981,
+      "num_tokens": 3217248.0,
+      "step": 1030
+    },
+    {
+      "entropy": 0.7715817041695118,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 1.125,
+      "learning_rate": 8.741867428021446e-05,
+      "loss": 3.1480581283569338,
+      "mean_token_accuracy": 0.7871334388852119,
+      "num_tokens": 3244290.0,
+      "step": 1040
+    },
+    {
+      "entropy": 0.7311166271567344,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 1.734375,
+      "learning_rate": 8.570632045125185e-05,
+      "loss": 2.9483835220336916,
+      "mean_token_accuracy": 0.7995148777961731,
+      "num_tokens": 3270106.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.8038631252944469,
+      "epoch": 1.696,
+      "grad_norm": 1.296875,
+      "learning_rate": 8.399823482681262e-05,
+      "loss": 3.2476577758789062,
+      "mean_token_accuracy": 0.788225382566452,
+      "num_tokens": 3302183.0,
+      "step": 1060
+    },
+    {
+      "entropy": 0.7686532281339169,
+      "epoch": 1.712,
+      "grad_norm": 1.484375,
+      "learning_rate": 8.22949274546255e-05,
+      "loss": 3.1519790649414063,
+      "mean_token_accuracy": 0.7915654450654983,
+      "num_tokens": 3329432.0,
+      "step": 1070
+    },
+    {
+      "entropy": 0.7671505004167557,
+      "epoch": 1.728,
+      "grad_norm": 1.375,
+      "learning_rate": 8.059690695559568e-05,
+      "loss": 3.085156059265137,
+      "mean_token_accuracy": 0.7906328395009041,
+      "num_tokens": 3353693.0,
+      "step": 1080
+    },
+    {
+      "entropy": 0.7490876868367196,
+      "epoch": 1.744,
+      "grad_norm": 1.8203125,
+      "learning_rate": 7.89046803719267e-05,
+      "loss": 3.077616310119629,
+      "mean_token_accuracy": 0.7907149896025658,
+      "num_tokens": 3379146.0,
+      "step": 1090
+    },
+    {
+      "entropy": 0.821656309068203,
+      "epoch": 1.76,
+      "grad_norm": 1.1328125,
+      "learning_rate": 7.721875301571359e-05,
+      "loss": 3.2485275268554688,
+      "mean_token_accuracy": 0.7844672784209251,
+      "num_tokens": 3407799.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.7870353177189827,
+      "epoch": 1.776,
+      "grad_norm": 1.390625,
+      "learning_rate": 7.55396283180529e-05,
+      "loss": 3.2128215789794923,
+      "mean_token_accuracy": 0.7861049249768257,
+      "num_tokens": 3436295.0,
+      "step": 1110
+    },
+    {
+      "entropy": 0.7809636801481247,
+      "epoch": 1.792,
+      "grad_norm": 2.59375,
+      "learning_rate": 7.386780767871397e-05,
+      "loss": 3.1558361053466797,
+      "mean_token_accuracy": 0.7923071622848511,
+      "num_tokens": 3463552.0,
+      "step": 1120
+    },
+    {
+      "entropy": 0.7963082253932953,
+      "epoch": 1.808,
+      "grad_norm": 1.3515625,
+      "learning_rate": 7.22037903164173e-05,
+      "loss": 3.2051555633544924,
+      "mean_token_accuracy": 0.7853836208581925,
+      "num_tokens": 3490070.0,
+      "step": 1130
+    },
+    {
+      "entropy": 0.8024965927004815,
+      "epoch": 1.8239999999999998,
+      "grad_norm": 1.4296875,
+      "learning_rate": 7.054807311976379e-05,
+      "loss": 3.256487274169922,
+      "mean_token_accuracy": 0.7800001233816147,
+      "num_tokens": 3519607.0,
+      "step": 1140
+    },
+    {
+      "entropy": 0.7608409076929092,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 1.6640625,
+      "learning_rate": 6.890115049885994e-05,
+      "loss": 3.1335351943969725,
+      "mean_token_accuracy": 0.7930847942829132,
+      "num_tokens": 3549954.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.7995478600263596,
+      "epoch": 1.8559999999999999,
+      "grad_norm": 1.0859375,
+      "learning_rate": 6.726351423768322e-05,
+      "loss": 3.192752647399902,
+      "mean_token_accuracy": 0.7869470104575157,
+      "num_tokens": 3579252.0,
+      "step": 1160
+    },
+    {
+      "entropy": 0.7790917068719864,
+      "epoch": 1.8719999999999999,
+      "grad_norm": 1.15625,
+      "learning_rate": 6.563565334723134e-05,
+      "loss": 3.1096052169799804,
+      "mean_token_accuracy": 0.7890855461359024,
+      "num_tokens": 3607698.0,
+      "step": 1170
+    },
+    {
+      "entropy": 0.6804191634058953,
+      "epoch": 1.888,
+      "grad_norm": 1.4921875,
+      "learning_rate": 6.40180539194999e-05,
+      "loss": 2.7701562881469726,
+      "mean_token_accuracy": 0.8071593567728996,
+      "num_tokens": 3635417.0,
+      "step": 1180
+    },
+    {
+      "entropy": 0.7258283078670502,
+      "epoch": 1.904,
+      "grad_norm": 1.28125,
+      "learning_rate": 6.241119898233144e-05,
+      "loss": 2.969502830505371,
+      "mean_token_accuracy": 0.7979315519332886,
+      "num_tokens": 3661222.0,
+      "step": 1190
+    },
+    {
+      "entropy": 0.7636997759342193,
+      "epoch": 1.92,
+      "grad_norm": 1.6484375,
+      "learning_rate": 6.0815568355179556e-05,
+      "loss": 3.1156858444213866,
+      "mean_token_accuracy": 0.7898655593395233,
+      "num_tokens": 3692905.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.80526192933321,
+      "epoch": 1.936,
+      "grad_norm": 1.1796875,
+      "learning_rate": 5.923163850583113e-05,
+      "loss": 3.170664596557617,
+      "mean_token_accuracy": 0.7876248195767402,
+      "num_tokens": 3720018.0,
+      "step": 1210
+    },
+    {
+      "entropy": 0.7665807068347931,
+      "epoch": 1.952,
+      "grad_norm": 1.28125,
+      "learning_rate": 5.765988240812921e-05,
+      "loss": 3.141133499145508,
+      "mean_token_accuracy": 0.790970367193222,
+      "num_tokens": 3746413.0,
+      "step": 1220
+    },
+    {
+      "entropy": 0.7018338434398175,
+      "epoch": 1.968,
+      "grad_norm": 1.375,
+      "learning_rate": 5.6100769400739383e-05,
+      "loss": 2.782573699951172,
+      "mean_token_accuracy": 0.8073293790221214,
+      "num_tokens": 3770477.0,
+      "step": 1230
+    },
+    {
+      "entropy": 0.7392673164606094,
+      "epoch": 1.984,
+      "grad_norm": 1.46875,
+      "learning_rate": 5.4554765047001613e-05,
+      "loss": 2.957522964477539,
+      "mean_token_accuracy": 0.7974923148751258,
+      "num_tokens": 3796914.0,
+      "step": 1240
+    },
+    {
+      "entropy": 0.7508858695626259,
+      "epoch": 2.0,
+      "grad_norm": 1.2890625,
+      "learning_rate": 5.302233099590928e-05,
+      "loss": 3.049625205993652,
+      "mean_token_accuracy": 0.7928661614656448,
+      "num_tokens": 3822313.0,
+      "step": 1250
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 0.7935182250738144,
+      "eval_loss": 0.9520308971405029,
+      "eval_mean_token_accuracy": 0.7710564243793487,
+      "eval_num_tokens": 3822313.0,
+      "eval_runtime": 72.7945,
+      "eval_samples_per_second": 13.737,
+      "eval_steps_per_second": 3.434,
+      "step": 1250
+    },
+    {
+      "entropy": 0.7053400471806526,
+      "epoch": 2.016,
+      "grad_norm": 1.3125,
+      "learning_rate": 5.150392484425728e-05,
+      "loss": 2.6305599212646484,
+      "mean_token_accuracy": 0.8136913523077964,
+      "num_tokens": 3849460.0,
+      "step": 1260
+    },
+    {
+      "entropy": 0.6759374007582665,
+      "epoch": 2.032,
+      "grad_norm": 1.1796875,
+      "learning_rate": 5.000000000000002e-05,
+      "loss": 2.740474891662598,
+      "mean_token_accuracy": 0.810079038143158,
+      "num_tokens": 3881699.0,
+      "step": 1270
+    },
+    {
+      "entropy": 0.6673750668764115,
+      "epoch": 2.048,
+      "grad_norm": 1.234375,
+      "learning_rate": 4.851100554686021e-05,
+      "loss": 2.5454484939575197,
+      "mean_token_accuracy": 0.8159348770976067,
+      "num_tokens": 3909562.0,
+      "step": 1280
+    },
+    {
+      "entropy": 0.5898188591003418,
+      "epoch": 2.064,
+      "grad_norm": 1.2578125,
+      "learning_rate": 4.7037386110228985e-05,
+      "loss": 2.3230581283569336,
+      "mean_token_accuracy": 0.8328281179070472,
+      "num_tokens": 3936217.0,
+      "step": 1290
+    },
+    {
+      "entropy": 0.6469230554997921,
+      "epoch": 2.08,
+      "grad_norm": 1.59375,
+      "learning_rate": 4.5579581724397255e-05,
+      "loss": 2.592478942871094,
+      "mean_token_accuracy": 0.8147690609097481,
+      "num_tokens": 3966142.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.6723301410675049,
+      "epoch": 2.096,
+      "grad_norm": 1.125,
+      "learning_rate": 4.413802770115816e-05,
+      "loss": 2.6196125030517576,
+      "mean_token_accuracy": 0.8145923808217048,
+      "num_tokens": 3995663.0,
+      "step": 1310
+    },
+    {
+      "entropy": 0.6221730917692184,
+      "epoch": 2.112,
+      "grad_norm": 1.2578125,
+      "learning_rate": 4.271315449981934e-05,
+      "loss": 2.4203453063964844,
+      "mean_token_accuracy": 0.8260695144534111,
+      "num_tokens": 4025537.0,
+      "step": 1320
+    },
+    {
+      "entropy": 0.6446548275649547,
+      "epoch": 2.128,
+      "grad_norm": 1.1015625,
+      "learning_rate": 4.130538759866457e-05,
+      "loss": 2.562292289733887,
+      "mean_token_accuracy": 0.8144417598843574,
+      "num_tokens": 4053555.0,
+      "step": 1330
+    },
+    {
+      "entropy": 0.6655827894806862,
+      "epoch": 2.144,
+      "grad_norm": 1.2421875,
+      "learning_rate": 3.991514736790258e-05,
+      "loss": 2.6614761352539062,
+      "mean_token_accuracy": 0.8111246332526207,
+      "num_tokens": 4080361.0,
+      "step": 1340
+    },
+    {
+      "entropy": 0.6002881981432437,
+      "epoch": 2.16,
+      "grad_norm": 1.078125,
+      "learning_rate": 3.854284894414122e-05,
+      "loss": 2.3379238128662108,
+      "mean_token_accuracy": 0.8291780337691307,
+      "num_tokens": 4108641.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.6322725959122181,
+      "epoch": 2.176,
+      "grad_norm": 1.34375,
+      "learning_rate": 3.7188902106424416e-05,
+      "loss": 2.5648292541503905,
+      "mean_token_accuracy": 0.819225026667118,
+      "num_tokens": 4137407.0,
+      "step": 1360
+    },
+    {
+      "entropy": 0.6220526427030564,
+      "epoch": 2.192,
+      "grad_norm": 1.265625,
+      "learning_rate": 3.5853711153868965e-05,
+      "loss": 2.521045112609863,
+      "mean_token_accuracy": 0.8181931138038635,
+      "num_tokens": 4165637.0,
+      "step": 1370
+    },
+    {
+      "entropy": 0.59385786652565,
+      "epoch": 2.208,
+      "grad_norm": 1.1484375,
+      "learning_rate": 3.4537674784937614e-05,
+      "loss": 2.256278419494629,
+      "mean_token_accuracy": 0.8328502863645554,
+      "num_tokens": 4190019.0,
+      "step": 1380
+    },
+    {
+      "entropy": 0.6182742714881897,
+      "epoch": 2.224,
+      "grad_norm": 1.390625,
+      "learning_rate": 3.324118597838464e-05,
+      "loss": 2.4784978866577148,
+      "mean_token_accuracy": 0.8253998696804047,
+      "num_tokens": 4218065.0,
+      "step": 1390
+    },
+    {
+      "entropy": 0.625076462328434,
+      "epoch": 2.24,
+      "grad_norm": 1.375,
+      "learning_rate": 3.196463187590929e-05,
+      "loss": 2.5287513732910156,
+      "mean_token_accuracy": 0.8231741398572922,
+      "num_tokens": 4245504.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.6474951952695847,
+      "epoch": 2.2560000000000002,
+      "grad_norm": 1.65625,
+      "learning_rate": 3.070839366655215e-05,
+      "loss": 2.574743461608887,
+      "mean_token_accuracy": 0.8156981885433197,
+      "num_tokens": 4275439.0,
+      "step": 1410
+    },
+    {
+      "entropy": 0.6507719412446022,
+      "epoch": 2.2720000000000002,
+      "grad_norm": 1.3984375,
+      "learning_rate": 2.9472846472869298e-05,
+      "loss": 2.5274782180786133,
+      "mean_token_accuracy": 0.8186039045453072,
+      "num_tokens": 4300521.0,
+      "step": 1420
+    },
+    {
+      "entropy": 0.6078799493610859,
+      "epoch": 2.288,
+      "grad_norm": 1.203125,
+      "learning_rate": 2.8258359238917665e-05,
+      "loss": 2.3472341537475585,
+      "mean_token_accuracy": 0.8310479253530503,
+      "num_tokens": 4325611.0,
+      "step": 1430
+    },
+    {
+      "entropy": 0.6164450705051422,
+      "epoch": 2.304,
+      "grad_norm": 1.2109375,
+      "learning_rate": 2.7065294620085424e-05,
+      "loss": 2.4567951202392577,
+      "mean_token_accuracy": 0.8222395464777946,
+      "num_tokens": 4351381.0,
+      "step": 1440
+    },
+    {
+      "entropy": 0.6051406018435955,
+      "epoch": 2.32,
+      "grad_norm": 1.5078125,
+      "learning_rate": 2.5894008874800325e-05,
+      "loss": 2.4069543838500977,
+      "mean_token_accuracy": 0.8269070222973823,
+      "num_tokens": 4380875.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.6406742617487907,
+      "epoch": 2.336,
+      "grad_norm": 1.5,
+      "learning_rate": 2.4744851758148156e-05,
+      "loss": 2.575749969482422,
+      "mean_token_accuracy": 0.8187732085585594,
+      "num_tokens": 4410021.0,
+      "step": 1460
+    },
+    {
+      "entropy": 0.6262726046144962,
+      "epoch": 2.352,
+      "grad_norm": 11.6875,
+      "learning_rate": 2.361816641743303e-05,
+      "loss": 2.5055971145629883,
+      "mean_token_accuracy": 0.8225668385624886,
+      "num_tokens": 4437438.0,
+      "step": 1470
+    },
+    {
+      "entropy": 0.6012745849788189,
+      "epoch": 2.368,
+      "grad_norm": 1.265625,
+      "learning_rate": 2.251428928971102e-05,
+      "loss": 2.387685775756836,
+      "mean_token_accuracy": 0.8289849147200584,
+      "num_tokens": 4463678.0,
+      "step": 1480
+    },
+    {
+      "entropy": 0.6096144415438175,
+      "epoch": 2.384,
+      "grad_norm": 0.98828125,
+      "learning_rate": 2.1433550001327373e-05,
+      "loss": 2.4208652496337892,
+      "mean_token_accuracy": 0.825342382490635,
+      "num_tokens": 4488019.0,
+      "step": 1490
+    },
+    {
+      "entropy": 0.6083463601768017,
+      "epoch": 2.4,
+      "grad_norm": 1.2265625,
+      "learning_rate": 2.0376271269487514e-05,
+      "loss": 2.388985824584961,
+      "mean_token_accuracy": 0.824395664036274,
+      "num_tokens": 4517306.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.6329524978995323,
+      "epoch": 2.416,
+      "grad_norm": 1.671875,
+      "learning_rate": 1.9342768805891178e-05,
+      "loss": 2.51919002532959,
+      "mean_token_accuracy": 0.8179876819252968,
+      "num_tokens": 4545893.0,
+      "step": 1510
+    },
+    {
+      "entropy": 0.5982788041234016,
+      "epoch": 2.432,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.8333351222458407e-05,
+      "loss": 2.3680145263671877,
+      "mean_token_accuracy": 0.8285414636135101,
+      "num_tokens": 4573653.0,
+      "step": 1520
+    },
+    {
+      "entropy": 0.680705762654543,
+      "epoch": 2.448,
+      "grad_norm": 1.140625,
+      "learning_rate": 1.7348319939175637e-05,
+      "loss": 2.6735370635986326,
+      "mean_token_accuracy": 0.8125541433691978,
+      "num_tokens": 4601669.0,
+      "step": 1530
+    },
+    {
+      "entropy": 0.563636127859354,
+      "epoch": 2.464,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.6387969094089316e-05,
+      "loss": 2.165860176086426,
+      "mean_token_accuracy": 0.838480469584465,
+      "num_tokens": 4627149.0,
+      "step": 1540
+    },
+    {
+      "entropy": 0.5980882495641708,
+      "epoch": 2.48,
+      "grad_norm": 1.046875,
+      "learning_rate": 1.5452585455473977e-05,
+      "loss": 2.3885900497436525,
+      "mean_token_accuracy": 0.8232199549674988,
+      "num_tokens": 4653051.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.5893117256462574,
+      "epoch": 2.496,
+      "grad_norm": 1.375,
+      "learning_rate": 1.454244833620102e-05,
+      "loss": 2.361526107788086,
+      "mean_token_accuracy": 0.8287293270230294,
+      "num_tokens": 4679592.0,
+      "step": 1560
+    },
+    {
+      "entropy": 0.6897482767701149,
+      "epoch": 2.512,
+      "grad_norm": 1.1328125,
+      "learning_rate": 1.3657829510333654e-05,
+      "loss": 2.8343505859375,
+      "mean_token_accuracy": 0.8045855790376664,
+      "num_tokens": 4712580.0,
+      "step": 1570
+    },
+    {
+      "entropy": 0.5980210550129413,
+      "epoch": 2.528,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.2798993131973091e-05,
+      "loss": 2.393186569213867,
+      "mean_token_accuracy": 0.8249497979879379,
+      "num_tokens": 4740546.0,
+      "step": 1580
+    },
+    {
+      "entropy": 0.6140701808035374,
+      "epoch": 2.544,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.1966195656380031e-05,
+      "loss": 2.440609169006348,
+      "mean_token_accuracy": 0.8237545773386955,
+      "num_tokens": 4767938.0,
+      "step": 1590
+    },
+    {
+      "entropy": 0.6063477545976639,
+      "epoch": 2.56,
+      "grad_norm": 1.1875,
+      "learning_rate": 1.1159685763395111e-05,
+      "loss": 2.3470132827758787,
+      "mean_token_accuracy": 0.8265559047460556,
+      "num_tokens": 4794286.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.6648287683725357,
+      "epoch": 2.576,
+      "grad_norm": 1.1328125,
+      "learning_rate": 1.0379704283181179e-05,
+      "loss": 2.7020286560058593,
+      "mean_token_accuracy": 0.808851246535778,
+      "num_tokens": 4823293.0,
+      "step": 1610
+    },
+    {
+      "entropy": 0.6474792867898941,
+      "epoch": 2.592,
+      "grad_norm": 1.1953125,
+      "learning_rate": 9.62648412430951e-06,
+      "loss": 2.606777000427246,
+      "mean_token_accuracy": 0.8219101890921593,
+      "num_tokens": 4852606.0,
+      "step": 1620
+    },
+    {
+      "entropy": 0.6176077917218208,
+      "epoch": 2.608,
+      "grad_norm": 1.5625,
+      "learning_rate": 8.900250204211514e-06,
+      "loss": 2.4370376586914064,
+      "mean_token_accuracy": 0.8261922746896744,
+      "num_tokens": 4880168.0,
+      "step": 1630
+    },
+    {
+      "entropy": 0.6533839575946331,
+      "epoch": 2.624,
+      "grad_norm": 1.15625,
+      "learning_rate": 8.201219382016556e-06,
+      "loss": 2.6114227294921877,
+      "mean_token_accuracy": 0.815986056625843,
+      "num_tokens": 4909334.0,
+      "step": 1640
+    },
+    {
+      "entropy": 0.617949029058218,
+      "epoch": 2.64,
+      "grad_norm": 1.2265625,
+      "learning_rate": 7.529600393796232e-06,
+      "loss": 2.4090667724609376,
+      "mean_token_accuracy": 0.8269602835178376,
+      "num_tokens": 4936371.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.6316794902086258,
+      "epoch": 2.656,
+      "grad_norm": 1.1640625,
+      "learning_rate": 6.8855937902340576e-06,
+      "loss": 2.516379165649414,
+      "mean_token_accuracy": 0.819202584028244,
+      "num_tokens": 4966014.0,
+      "step": 1660
+    },
+    {
+      "entropy": 0.6159006580710411,
+      "epoch": 2.672,
+      "grad_norm": 1.34375,
+      "learning_rate": 6.269391876739495e-06,
+      "loss": 2.430217170715332,
+      "mean_token_accuracy": 0.8262986287474632,
+      "num_tokens": 4991945.0,
+      "step": 1670
+    },
+    {
+      "entropy": 0.6038173116743565,
+      "epoch": 2.6879999999999997,
+      "grad_norm": 0.9765625,
+      "learning_rate": 5.681178656024055e-06,
+      "loss": 2.3973724365234377,
+      "mean_token_accuracy": 0.8248073413968087,
+      "num_tokens": 5019869.0,
+      "step": 1680
+    },
+    {
+      "entropy": 0.6135182313621044,
+      "epoch": 2.7039999999999997,
+      "grad_norm": 1.2578125,
+      "learning_rate": 5.121129773156663e-06,
+      "loss": 2.4443332672119142,
+      "mean_token_accuracy": 0.824867345392704,
+      "num_tokens": 5048093.0,
+      "step": 1690
+    },
+    {
+      "entropy": 0.6408688805997371,
+      "epoch": 2.7199999999999998,
+      "grad_norm": 1.1640625,
+      "learning_rate": 4.58941246311464e-06,
+      "loss": 2.571330261230469,
+      "mean_token_accuracy": 0.8133804589509964,
+      "num_tokens": 5078113.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.6208270855247975,
+      "epoch": 2.7359999999999998,
+      "grad_norm": 1.1953125,
+      "learning_rate": 4.0861855008460405e-06,
+      "loss": 2.4142126083374023,
+      "mean_token_accuracy": 0.8225093990564346,
+      "num_tokens": 5106452.0,
+      "step": 1710
+    },
+    {
+      "entropy": 0.6045813173055649,
+      "epoch": 2.752,
+      "grad_norm": 1.1875,
+      "learning_rate": 3.611599153858214e-06,
+      "loss": 2.3853626251220703,
+      "mean_token_accuracy": 0.8237989127635956,
+      "num_tokens": 5133206.0,
+      "step": 1720
+    },
+    {
+      "entropy": 0.5914519041776657,
+      "epoch": 2.768,
+      "grad_norm": 1.328125,
+      "learning_rate": 3.1657951373467497e-06,
+      "loss": 2.298448181152344,
+      "mean_token_accuracy": 0.8312044337391853,
+      "num_tokens": 5156203.0,
+      "step": 1730
+    },
+    {
+      "entropy": 0.6053797014057636,
+      "epoch": 2.784,
+      "grad_norm": 1.2734375,
+      "learning_rate": 2.748906571878207e-06,
+      "loss": 2.4279769897460937,
+      "mean_token_accuracy": 0.8242192730307579,
+      "num_tokens": 5183714.0,
+      "step": 1740
+    },
+    {
+      "entropy": 0.6214097492396832,
+      "epoch": 2.8,
+      "grad_norm": 1.0078125,
+      "learning_rate": 2.3610579436393e-06,
+      "loss": 2.4572864532470704,
+      "mean_token_accuracy": 0.8246369242668152,
+      "num_tokens": 5212940.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.6008337244391442,
+      "epoch": 2.816,
+      "grad_norm": 1.1875,
+      "learning_rate": 2.002365067264289e-06,
+      "loss": 2.364995002746582,
+      "mean_token_accuracy": 0.8302600085735321,
+      "num_tokens": 5236919.0,
+      "step": 1760
+    },
+    {
+      "entropy": 0.5900142095983029,
+      "epoch": 2.832,
+      "grad_norm": 1.1875,
+      "learning_rate": 1.6729350512519005e-06,
+      "loss": 2.3344017028808595,
+      "mean_token_accuracy": 0.8315937608480454,
+      "num_tokens": 5264983.0,
+      "step": 1770
+    },
+    {
+      "entropy": 0.6705081954598426,
+      "epoch": 2.848,
+      "grad_norm": 1.2734375,
+      "learning_rate": 1.3728662659818204e-06,
+      "loss": 2.7036811828613283,
+      "mean_token_accuracy": 0.8103000715374946,
+      "num_tokens": 5294775.0,
+      "step": 1780
+    },
+    {
+      "entropy": 0.6234683454036712,
+      "epoch": 2.864,
+      "grad_norm": 1.3046875,
+      "learning_rate": 1.1022483143405705e-06,
+      "loss": 2.4741338729858398,
+      "mean_token_accuracy": 0.8195253252983093,
+      "num_tokens": 5322219.0,
+      "step": 1790
+    },
+    {
+      "entropy": 0.6028523907065392,
+      "epoch": 2.88,
+      "grad_norm": 1.875,
+      "learning_rate": 8.611620049653879e-07,
+      "loss": 2.3432369232177734,
+      "mean_token_accuracy": 0.8303986221551896,
+      "num_tokens": 5348043.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.6217740766704083,
+      "epoch": 2.896,
+      "grad_norm": 1.3515625,
+      "learning_rate": 6.496793281141056e-07,
+      "loss": 2.472747230529785,
+      "mean_token_accuracy": 0.8228062316775322,
+      "num_tokens": 5377321.0,
+      "step": 1810
+    },
+    {
+      "entropy": 0.651834636926651,
+      "epoch": 2.912,
+      "grad_norm": 1.390625,
+      "learning_rate": 4.678634341683252e-07,
+      "loss": 2.596288871765137,
+      "mean_token_accuracy": 0.8179462268948555,
+      "num_tokens": 5407751.0,
+      "step": 1820
+    },
+    {
+      "entropy": 0.602143744379282,
+      "epoch": 2.928,
+      "grad_norm": 1.4453125,
+      "learning_rate": 3.1576861477621287e-07,
+      "loss": 2.424236297607422,
+      "mean_token_accuracy": 0.8246050551533699,
+      "num_tokens": 5435212.0,
+      "step": 1830
+    },
+    {
+      "entropy": 0.6053410112857819,
+      "epoch": 2.944,
+      "grad_norm": 1.1796875,
+      "learning_rate": 1.9344028664056713e-07,
+      "loss": 2.388152313232422,
+      "mean_token_accuracy": 0.8283460915088654,
+      "num_tokens": 5462690.0,
+      "step": 1840
+    },
+    {
+      "entropy": 0.6017510555684566,
+      "epoch": 2.96,
+      "grad_norm": 1.4765625,
+      "learning_rate": 1.0091497795706728e-07,
+      "loss": 2.410584831237793,
+      "mean_token_accuracy": 0.8255392298102379,
+      "num_tokens": 5489364.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.6333115980029106,
+      "epoch": 2.976,
+      "grad_norm": 1.2734375,
+      "learning_rate": 3.8220317506654226e-08,
+      "loss": 2.480808067321777,
+      "mean_token_accuracy": 0.8206309258937836,
+      "num_tokens": 5518093.0,
+      "step": 1860
+    },
+    {
+      "entropy": 0.6766974046826363,
+      "epoch": 2.992,
+      "grad_norm": 1.2421875,
+      "learning_rate": 5.375026405352035e-09,
+      "loss": 2.673609161376953,
+      "mean_token_accuracy": 0.8086716338992119,
+      "num_tokens": 5547228.0,
+      "step": 1870
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 0.6697398825883866,
+      "eval_loss": 0.986240029335022,
+      "eval_mean_token_accuracy": 0.7718619239330292,
+      "eval_num_tokens": 5560476.0,
+      "eval_runtime": 72.585,
+      "eval_samples_per_second": 13.777,
+      "eval_steps_per_second": 3.444,
+      "step": 1875
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1875,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.618119414518085e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1875/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed89ca2be67eb35da01705488407af1aa1ea83180f94fa1d6688fc37dcb50663
+size 5713

checkpoint-625/README.md

@@ -0,0 +1,209 @@
+---
+base_model: google/gemma-4-E2B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:google/gemma-4-E2B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-625/adapter_config.json

@@ -0,0 +1,296 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-4-E2B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "lm_head",
+    "embed_tokens"
+  ],
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "26.self_attn.q_proj",
+    "language_model.layers.11.mlp.gate_proj",
+    "34.mlp.gate_proj",
+    "23.mlp.gate_proj",
+    "language_model.layers.5.mlp.gate_proj",
+    "input_proj",
+    "language_model.layers.12.self_attn.q_proj",
+    "32.mlp.down_proj",
+    "20.mlp.up_proj",
+    "language_model.layers.4.self_attn.v_proj",
+    "20.self_attn.k_proj",
+    "input_proj_linear",
+    "24.mlp.gate_proj",
+    "29.self_attn.k_proj",
+    "21.self_attn.v_proj",
+    "21.mlp.down_proj",
+    "16.self_attn.v_proj",
+    "language_model.layers.13.mlp.up_proj",
+    "24.mlp.down_proj",
+    "22.mlp.up_proj",
+    "language_model.layers.9.self_attn.k_proj",
+    "26.mlp.up_proj",
+    "language_model.layers.7.mlp.down_proj",
+    "language_model.layers.2.mlp.gate_proj",
+    "language_model.layers.13.self_attn.k_proj",
+    "25.self_attn.q_proj",
+    "per_layer_model_projection",
+    "18.mlp.up_proj",
+    "language_model.layers.13.self_attn.v_proj",
+    "34.mlp.down_proj",
+    "23.mlp.up_proj",
+    "23.self_attn.o_proj",
+    "24.self_attn.v_proj",
+    "language_model.layers.6.self_attn.q_proj",
+    "29.self_attn.q_proj",
+    "19.self_attn.k_proj",
+    "24.self_attn.k_proj",
+    "language_model.layers.5.self_attn.q_proj",
+    "32.mlp.gate_proj",
+    "language_model.layers.4.mlp.gate_proj",
+    "33.mlp.gate_proj",
+    "19.self_attn.o_proj",
+    "34.self_attn.v_proj",
+    "25.mlp.down_proj",
+    "language_model.layers.8.mlp.up_proj",
+    "language_model.layers.12.mlp.down_proj",
+    "language_model.layers.3.mlp.down_proj",
+    "language_model.layers.7.self_attn.k_proj",
+    "language_model.layers.2.mlp.up_proj",
+    "language_model.layers.13.mlp.down_proj",
+    "28.mlp.gate_proj",
+    "language_model.layers.15.self_attn.v_proj",
+    "20.self_attn.q_proj",
+    "per_layer_projection",
+    "27.mlp.gate_proj",
+    "language_model.layers.11.self_attn.o_proj",
+    "language_model.layers.15.mlp.gate_proj",
+    "30.self_attn.o_proj",
+    "32.self_attn.o_proj",
+    "27.mlp.down_proj",
+    "language_model.layers.1.mlp.gate_proj",
+    "21.self_attn.o_proj",
+    "32.self_attn.q_proj",
+    "language_model.layers.7.self_attn.v_proj",
+    "21.mlp.gate_proj",
+    "16.self_attn.k_proj",
+    "20.mlp.down_proj",
+    "16.mlp.gate_proj",
+    "20.mlp.gate_proj",
+    "language_model.layers.7.mlp.gate_proj",
+    "34.self_attn.q_proj",
+    "31.mlp.down_proj",
+    "language_model.layers.8.mlp.gate_proj",
+    "language_model.layers.9.mlp.up_proj",
+    "language_model.layers.15.self_attn.o_proj",
+    "28.self_attn.o_proj",
+    "32.self_attn.v_proj",
+    "32.self_attn.k_proj",
+    "language_model.layers.4.self_attn.q_proj",
+    "28.self_attn.v_proj",
+    "28.self_attn.k_proj",
+    "26.self_attn.k_proj",
+    "34.self_attn.k_proj",
+    "16.mlp.up_proj",
+    "21.self_attn.q_proj",
+    "17.self_attn.q_proj",
+    "language_model.layers.11.self_attn.k_proj",
+    "24.mlp.up_proj",
+    "23.self_attn.q_proj",
+    "language_model.layers.3.mlp.up_proj",
+    "language_model.layers.12.self_attn.k_proj",
+    "language_model.layers.1.mlp.down_proj",
+    "20.self_attn.v_proj",
+    "language_model.layers.11.mlp.up_proj",
+    "language_model.layers.0.mlp.gate_proj",
+    "language_model.layers.5.self_attn.v_proj",
+    "language_model.layers.6.self_attn.o_proj",
+    "language_model.layers.11.mlp.down_proj",
+    "31.mlp.gate_proj",
+    "31.self_attn.q_proj",
+    "22.self_attn.v_proj",
+    "31.self_attn.v_proj",
+    "25.mlp.up_proj",
+    "language_model.layers.9.self_attn.q_proj",
+    "27.self_attn.o_proj",
+    "17.mlp.up_proj",
+    "25.self_attn.k_proj",
+    "language_model.layers.1.self_attn.o_proj",
+    "19.mlp.down_proj",
+    "language_model.layers.14.self_attn.q_proj",
+    "31.self_attn.k_proj",
+    "language_model.layers.14.mlp.down_proj",
+    "language_model.layers.2.self_attn.v_proj",
+    "language_model.layers.9.mlp.down_proj",
+    "26.mlp.down_proj",
+    "30.self_attn.q_proj",
+    "31.self_attn.o_proj",
+    "18.self_attn.v_proj",
+    "33.self_attn.q_proj",
+    "language_model.layers.2.self_attn.q_proj",
+    "30.mlp.up_proj",
+    "language_model.layers.4.mlp.down_proj",
+    "29.mlp.down_proj",
+    "19.self_attn.q_proj",
+    "language_model.layers.1.self_attn.v_proj",
+    "language_model.layers.10.self_attn.o_proj",
+    "27.self_attn.k_proj",
+    "embedding_projection",
+    "24.self_attn.q_proj",
+    "33.self_attn.v_proj",
+    "29.mlp.up_proj",
+    "25.self_attn.v_proj",
+    "language_model.layers.11.self_attn.q_proj",
+    "33.self_attn.k_proj",
+    "language_model.layers.5.self_attn.o_proj",
+    "language_model.layers.4.self_attn.k_proj",
+    "language_model.layers.3.self_attn.o_proj",
+    "21.self_attn.k_proj",
+    "language_model.layers.8.mlp.down_proj",
+    "language_model.layers.2.self_attn.o_proj",
+    "20.self_attn.o_proj",
+    "language_model.layers.0.self_attn.k_proj",
+    "linear",
+    "language_model.layers.10.self_attn.v_proj",
+    "17.mlp.down_proj",
+    "21.mlp.up_proj",
+    "language_model.layers.2.mlp.down_proj",
+    "22.mlp.gate_proj",
+    "language_model.layers.9.self_attn.o_proj",
+    "language_model.layers.0.self_attn.q_proj",
+    "language_model.layers.14.self_attn.k_proj",
+    "18.mlp.gate_proj",
+    "language_model.layers.3.self_attn.q_proj",
+    "language_model.layers.7.self_attn.q_proj",
+    "language_model.layers.15.mlp.up_proj",
+    "34.self_attn.o_proj",
+    "language_model.layers.8.self_attn.q_proj",
+    "language_model.layers.10.mlp.up_proj",
+    "language_model.layers.14.mlp.up_proj",
+    "29.mlp.gate_proj",
+    "language_model.layers.1.self_attn.k_proj",
+    "language_model.layers.13.self_attn.o_proj",
+    "25.self_attn.o_proj",
+    "17.self_attn.v_proj",
+    "30.mlp.gate_proj",
+    "26.self_attn.v_proj",
+    "language_model.layers.3.self_attn.v_proj",
+    "19.self_attn.v_proj",
+    "language_model.layers.0.mlp.down_proj",
+    "language_model.layers.3.self_attn.k_proj",
+    "18.self_attn.o_proj",
+    "29.self_attn.o_proj",
+    "25.mlp.gate_proj",
+    "language_model.layers.9.mlp.gate_proj",
+    "16.self_attn.o_proj",
+    "language_model.layers.12.self_attn.o_proj",
+    "33.self_attn.o_proj",
+    "output_proj",
+    "22.self_attn.o_proj",
+    "17.self_attn.o_proj",
+    "23.self_attn.v_proj",
+    "language_model.layers.12.mlp.gate_proj",
+    "22.self_attn.q_proj",
+    "relative_k_proj",
+    "30.mlp.down_proj",
+    "22.mlp.down_proj",
+    "31.mlp.up_proj",
+    "18.self_attn.k_proj",
+    "language_model.layers.1.mlp.up_proj",
+    "28.mlp.down_proj",
+    "language_model.layers.15.self_attn.q_proj",
+    "language_model.layers.10.mlp.gate_proj",
+    "language_model.layers.0.self_attn.v_proj",
+    "23.mlp.down_proj",
+    "29.self_attn.v_proj",
+    "language_model.layers.8.self_attn.v_proj",
+    "22.self_attn.k_proj",
+    "17.mlp.gate_proj",
+    "24.self_attn.o_proj",
+    "16.mlp.down_proj",
+    "33.mlp.down_proj",
+    "language_model.layers.2.self_attn.k_proj",
+    "23.self_attn.k_proj",
+    "30.self_attn.v_proj",
+    "language_model.layers.4.self_attn.o_proj",
+    "language_model.layers.1.self_attn.q_proj",
+    "language_model.layers.7.mlp.up_proj",
+    "language_model.layers.5.self_attn.k_proj",
+    "19.mlp.up_proj",
+    "per_layer_input_gate",
+    "language_model.layers.0.mlp.up_proj",
+    "30.self_attn.k_proj",
+    "language_model.layers.8.self_attn.k_proj",
+    "18.self_attn.q_proj",
+    "language_model.layers.4.mlp.up_proj",
+    "19.mlp.gate_proj",
+    "language_model.layers.0.self_attn.o_proj",
+    "language_model.layers.12.self_attn.v_proj",
+    "34.mlp.up_proj",
+    "language_model.layers.13.self_attn.q_proj",
+    "27.mlp.up_proj",
+    "26.mlp.gate_proj",
+    "language_model.layers.6.self_attn.k_proj",
+    "26.self_attn.o_proj",
+    "language_model.layers.10.self_attn.q_proj",
+    "language_model.layers.10.self_attn.k_proj",
+    "language_model.layers.10.mlp.down_proj",
+    "33.mlp.up_proj",
+    "language_model.layers.6.self_attn.v_proj",
+    "16.self_attn.q_proj",
+    "27.self_attn.v_proj",
+    "28.mlp.up_proj",
+    "language_model.layers.15.mlp.down_proj",
+    "language_model.layers.14.self_attn.o_proj",
+    "language_model.layers.5.mlp.up_proj",
+    "language_model.layers.13.mlp.gate_proj",
+    "language_model.layers.14.mlp.gate_proj",
+    "language_model.layers.11.self_attn.v_proj",
+    "language_model.layers.14.self_attn.v_proj",
+    "18.mlp.down_proj",
+    "17.self_attn.k_proj",
+    "language_model.layers.7.self_attn.o_proj",
+    "32.mlp.up_proj",
+    "language_model.layers.6.mlp.up_proj",
+    "language_model.layers.9.self_attn.v_proj",
+    "language_model.layers.3.mlp.gate_proj",
+    "language_model.layers.6.mlp.down_proj",
+    "language_model.layers.12.mlp.up_proj",
+    "language_model.layers.6.mlp.gate_proj",
+    "28.self_attn.q_proj",
+    "language_model.layers.15.self_attn.k_proj",
+    "language_model.layers.8.self_attn.o_proj",
+    "language_model.layers.5.mlp.down_proj",
+    "27.self_attn.q_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-625/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbe41a8d88c8071ea2abd0648b2ebdbcd2fc74aea4f0e890372aca66e56a0ddc
+size 1688992024

checkpoint-625/chat_template.jinja

@@ -0,0 +1,263 @@
+{%- macro format_parameters(properties, required) -%}
+    {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
+    {%- set ns = namespace(found_first=false) -%}
+    {%- for key, value in properties | dictsort -%}
+        {%- set add_comma = false -%}
+        {%- if key not in standard_keys -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {{ key }}:{
+            {%- if value['description'] -%}
+                description:<|"|>{{ value['description'] }}<|"|>
+                {%- set add_comma = true -%}
+            {%- endif -%}
+            {%- if value['nullable'] %}
+                {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                nullable:true
+            {%- endif -%}
+            {%- if value['type'] | upper == 'STRING' -%}
+                {%- if value['enum'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    enum:{{ format_argument(value['enum']) }}
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'OBJECT' -%}
+                ,properties:{
+                {%- if value['properties'] is defined and value['properties'] is mapping -%}
+                    {{- format_parameters(value['properties'], value['required'] | default([])) -}}
+                {%- elif value is mapping -%}
+                    {{- format_parameters(value, value['required'] | default([])) -}}
+                {%- endif -%}
+                }
+                {%- if value['required'] -%}
+                    ,required:[
+                    {%- for item in value['required'] | default([]) -%}
+                        <|"|>{{- item -}}<|"|>
+                        {%- if not loop.last %},{% endif -%}
+                    {%- endfor -%}
+                    ]
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'ARRAY' -%}
+                {%- if value['items'] is mapping and value['items'] -%}
+                    ,items:{
+                    {%- set ns_items = namespace(found_first=false) -%}
+                    {%- for item_key, item_value in value['items'] | dictsort -%}
+                        {%- if item_value is not none -%}
+                            {%- if ns_items.found_first %},{% endif -%}
+                            {%- set ns_items.found_first = true -%}
+                            {%- if item_key == 'properties' -%}
+                                properties:{
+                                {%- if item_value is mapping -%}
+                                    {{- format_parameters(item_value, value['items']['required'] | default([])) -}}
+                                {%- endif -%}
+                                }
+                            {%- elif item_key == 'required' -%}
+                                required:[
+                                {%- for req_item in item_value -%}
+                                    <|"|>{{- req_item -}}<|"|>
+                                    {%- if not loop.last %},{% endif -%}
+                                {%- endfor -%}
+                                ]
+                            {%- elif item_key == 'type' -%}
+                                {%- if item_value is string -%}
+                                    type:{{ format_argument(item_value | upper) }}
+                                {%- else -%}
+                                    type:{{ format_argument(item_value | map('upper') | list) }}
+                                {%- endif -%}
+                            {%- else -%}
+                                {{ item_key }}:{{ format_argument(item_value) }}
+                            {%- endif -%}
+                        {%- endif -%}
+                    {%- endfor -%}
+                    }
+                {%- endif -%}
+            {%- endif -%}
+            {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+            type:<|"|>{{ value['type'] | upper }}<|"|>}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endmacro -%}
+{%- macro format_function_declaration(tool_data) -%}
+    declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
+    {%- set params = tool_data['function']['parameters'] -%}
+    {%- if params -%}
+        ,parameters:{
+        {%- if params['properties'] -%}
+            properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
+        {%- endif -%}
+        {%- if params['required'] -%}
+            required:[
+            {%- for item in params['required'] -%}
+                <|"|>{{- item -}}<|"|>
+                {{- ',' if not loop.last -}}
+            {%- endfor -%}
+            ],
+        {%- endif -%}
+        {%- if params['type'] -%}
+            type:<|"|>{{- params['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    {%- if 'response' in tool_data['function'] -%}
+        {%- set response_declaration = tool_data['function']['response'] -%}
+        ,response:{
+        {%- if response_declaration['description'] -%}
+            description:<|"|>{{- response_declaration['description'] -}}<|"|>,
+        {%- endif -%}
+        {%- if response_declaration['type'] | upper == 'OBJECT' -%}
+            type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    }
+{%- endmacro -%}
+{%- macro format_argument(argument, escape_keys=True) -%}
+    {%- if argument is string -%}
+        {{- '<|"|>' + argument + '<|"|>' -}}
+    {%- elif argument is boolean -%}
+        {{- 'true' if argument else 'false' -}}
+    {%- elif argument is mapping -%}
+        {{- '{' -}}
+        {%- set ns = namespace(found_first=false) -%}
+        {%- for key, value in argument | dictsort -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {%- if escape_keys -%}
+                {{- '<|"|>' + key + '<|"|>' -}}
+            {%- else -%}
+                {{- key -}}
+            {%- endif -%}
+            :{{- format_argument(value, escape_keys=escape_keys) -}}
+        {%- endfor -%}
+        {{- '}' -}}
+    {%- elif argument is sequence -%}
+        {{- '[' -}}
+        {%- for item in argument -%}
+            {{- format_argument(item, escape_keys=escape_keys) -}}
+            {%- if not loop.last %},{% endif -%}
+        {%- endfor -%}
+        {{- ']' -}}
+    {%- else -%}
+        {{- argument -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- macro strip_thinking(text) -%}
+    {%- set ns = namespace(result='') -%}
+    {%- for part in text.split('<channel|>') -%}
+        {%- if '<|channel>' in part -%}
+            {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
+        {%- else -%}
+            {%- set ns.result = ns.result + part -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- ns.result | trim -}}
+{%- endmacro -%}
+
+{%- set ns = namespace(prev_message_type=None) -%}
+{%- set loop_messages = messages -%}
+{{ bos_token }}
+{#- Handle System/Tool Definitions Block -#}
+{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}
+    {{- '<|turn>system\n' -}}
+
+    {#- Inject Thinking token at the very top of the FIRST system turn -#}
+    {%- if enable_thinking is defined and enable_thinking -%}
+        {{- '<|think|>' -}}
+        {%- set ns.prev_message_type = 'think' -%}
+    {%- endif -%}
+
+    {%- if messages[0]['role'] in ['system', 'developer'] -%}
+        {{- messages[0]['content'] | trim -}}
+        {%- set loop_messages = messages[1:] -%}
+    {%- endif -%}
+
+    {%- if tools -%}
+        {%- for tool in tools %}
+            {{- '<|tool>' -}}
+            {{- format_function_declaration(tool) | trim -}}
+            {{- '<tool|>' -}}
+        {%- endfor %}
+        {%- set ns.prev_message_type = 'tool' -%}
+    {%- endif -%}
+
+    {{- '<turn|>\n' -}}
+{%- endif %}
+
+{#- Loop through messages -#}
+{%- for message in loop_messages -%}
+    {%- set ns.prev_message_type = None -%}
+    {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
+        {{- '<|turn>' + role + '\n' }}
+
+            {%- if message['tool_calls'] -%}
+                {%- for tool_call in message['tool_calls'] -%}
+                    {%- set function = tool_call['function'] -%}
+                    {{- '<|tool_call>call:' + function['name'] + '{' -}}
+                    {%- if function['arguments'] is mapping -%}
+                        {%- set ns_args = namespace(found_first=false) -%}
+                        {%- for key, value in function['arguments'] | dictsort -%}
+                            {%- if ns_args.found_first %},{% endif -%}
+                            {%- set ns_args.found_first = true -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                        {%- endfor -%}
+                    {%- elif function['arguments'] is string -%}
+                        {{- function['arguments'] -}}
+                    {%- endif -%}
+                    {{- '}<tool_call|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_call' -%}
+            {%- endif -%}
+
+            {%- if message['tool_responses'] -%}
+                {#- Tool Response handling -#}
+                {%- for tool_response in message['tool_responses'] -%}
+                    {{- '<|tool_response>' -}}
+                    {%- if tool_response['response'] is mapping -%}
+                        {{- 'response:' + tool_response['name'] | default('unknown') + '{' -}}
+                        {%- for key, value in tool_response['response'] | dictsort -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                            {%- if not loop.last %},{% endif -%}
+                        {%- endfor -%}
+                        {{- '}' -}}
+                    {%- else -%}
+                        {{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}}
+                    {%- endif -%}
+                    {{- '<tool_response|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_response' -%}
+            {%- endif -%}
+
+            {%- if message['content'] is string -%}
+                {%- if role == 'model' -%}
+                    {{- strip_thinking(message['content']) -}}
+                {%- else -%}
+                    {{- message['content'] | trim -}}
+                {%- endif -%}
+            {%- elif message['content'] is sequence -%}
+                {%- for item in message['content'] -%}
+                    {%- if item['type'] == 'text' -%}
+                        {%- if role == 'model' -%}
+                            {{- strip_thinking(item['text']) -}}
+                        {%- else -%}
+                            {{- item['text'] | trim -}}
+                        {%- endif -%}
+                    {%- elif item['type'] == 'image' -%}
+                        {{- '\n\n<|image|>\n\n' -}}
+                        {%- set ns.prev_message_type = 'image' -%}
+                    {%- elif item['type'] == 'audio' -%}
+                        {{- '<|audio|>' -}}
+                        {%- set ns.prev_message_type = 'audio' -%}
+                    {%- elif item['type'] == 'video' -%}
+                        {{- '\n\n<|video|>\n\n' -}}
+                        {%- set ns.prev_message_type = 'video' -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+
+        {%- if not (message['tool_responses'] and not message['content']) -%}
+            {{- '<turn|>\n' -}}
+        {%- endif -%}
+{%- endfor -%}
+
+{%- if add_generation_prompt -%}
+    {%- if ns.prev_message_type != 'tool_response' -%}
+        {{- '<|turn>model\n' -}}
+    {%- endif -%}
+{%- endif -%}

checkpoint-625/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43ae2016363ef546eaf19ce0daa11db0b0cd615b166a3224b952e7b8f5b426bf
+size 3331821841

checkpoint-625/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5377bc98986921ac2e2e82cb69b0a97c56332dcabe8c71af82c971b6c76800e7
+size 14709

checkpoint-625/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27b837f13019ce2d83e8a5c43aaf9374d2b3b575cafb43a578c35223db9760e2
+size 1465

checkpoint-625/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f
+size 32169626

checkpoint-625/tokenizer_config.json

@@ -0,0 +1,95 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [
+    "<|video|>"
+  ],
+  "image_token": "<|image|>",
+  "is_local": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "audio_token": "<|audio|>",
+    "boa_token": "<|audio>",
+    "boi_token": "<|image>",
+    "eoa_token": "<audio|>",
+    "eoc_token": "<channel|>",
+    "eoi_token": "<image|>",
+    "eot_token": "<turn|>",
+    "escape_token": "<|\"|>",
+    "etc_token": "<tool_call|>",
+    "etd_token": "<tool|>",
+    "etr_token": "<tool_response|>",
+    "image_token": "<|image|>",
+    "soc_token": "<|channel>",
+    "sot_token": "<|turn>",
+    "stc_token": "<|tool_call>",
+    "std_token": "<|tool>",
+    "str_token": "<|tool_response>",
+    "think_token": "<|think|>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma4Processor",
+  "response_schema": {
+    "properties": {
+      "content": {
+        "type": "string"
+      },
+      "role": {
+        "const": "assistant"
+      },
+      "thinking": {
+        "type": "string"
+      },
+      "tool_calls": {
+        "items": {
+          "properties": {
+            "function": {
+              "properties": {
+                "arguments": {
+                  "additionalProperties": {},
+                  "type": "object",
+                  "x-parser": "gemma4-tool-call"
+                },
+                "name": {
+                  "type": "string"
+                }
+              },
+              "type": "object",
+              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"
+            },
+            "type": {
+              "const": "function"
+            }
+          },
+          "type": "object"
+        },
+        "type": "array",
+        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"
+      }
+    },
+    "type": "object",
+    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<content>(?:(?!\\<\\|tool_call\\>)(?!\\<turn\\|\\>).)+)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?:\\<turn\\|\\>)?"
+  },
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-625/trainer_state.json

@@ -0,0 +1,675 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 625,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.5365671901141895,
+      "epoch": 0.0016,
+      "grad_norm": 4.21875,
+      "learning_rate": 0.0,
+      "loss": 5.494313716888428,
+      "mean_token_accuracy": 0.6573571685482474,
+      "num_tokens": 348701.0,
+      "step": 1
+    },
+    {
+      "entropy": 1.348903712299135,
+      "epoch": 0.016,
+      "grad_norm": 2.59375,
+      "learning_rate": 3.157894736842105e-05,
+      "loss": 5.544558631049262,
+      "mean_token_accuracy": 0.6651431231035126,
+      "num_tokens": 371496.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.360134594142437,
+      "epoch": 0.032,
+      "grad_norm": 3.296875,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 5.742576599121094,
+      "mean_token_accuracy": 0.6662841200828552,
+      "num_tokens": 400839.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.375771728157997,
+      "epoch": 0.048,
+      "grad_norm": 3.140625,
+      "learning_rate": 0.0001017543859649123,
+      "loss": 5.714028167724609,
+      "mean_token_accuracy": 0.6566085889935493,
+      "num_tokens": 431025.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.2748979568481444,
+      "epoch": 0.064,
+      "grad_norm": 2.828125,
+      "learning_rate": 0.0001368421052631579,
+      "loss": 5.11612548828125,
+      "mean_token_accuracy": 0.6816287323832512,
+      "num_tokens": 459089.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.3155348658561707,
+      "epoch": 0.08,
+      "grad_norm": 3.15625,
+      "learning_rate": 0.00017192982456140353,
+      "loss": 5.35040512084961,
+      "mean_token_accuracy": 0.6721501812338829,
+      "num_tokens": 484648.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1791150525212288,
+      "epoch": 0.096,
+      "grad_norm": 2.375,
+      "learning_rate": 0.00019999940277008808,
+      "loss": 4.642951583862304,
+      "mean_token_accuracy": 0.7028968930244446,
+      "num_tokens": 511463.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.1177749201655387,
+      "epoch": 0.112,
+      "grad_norm": 2.421875,
+      "learning_rate": 0.0001999785004721968,
+      "loss": 4.523546981811523,
+      "mean_token_accuracy": 0.7057401552796364,
+      "num_tokens": 538438.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.1784485399723053,
+      "epoch": 0.128,
+      "grad_norm": 2.25,
+      "learning_rate": 0.00019992774381199778,
+      "loss": 4.712202072143555,
+      "mean_token_accuracy": 0.7004493281245232,
+      "num_tokens": 569421.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.0666756927967072,
+      "epoch": 0.144,
+      "grad_norm": 4.25,
+      "learning_rate": 0.00019984714794582683,
+      "loss": 4.181539154052734,
+      "mean_token_accuracy": 0.7218294009566307,
+      "num_tokens": 595806.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.1410984337329864,
+      "epoch": 0.16,
+      "grad_norm": 2.59375,
+      "learning_rate": 0.00019973673694024,
+      "loss": 4.578577041625977,
+      "mean_token_accuracy": 0.70914496332407,
+      "num_tokens": 626238.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.058976437151432,
+      "epoch": 0.176,
+      "grad_norm": 4.34375,
+      "learning_rate": 0.0001995965437648273,
+      "loss": 4.2924964904785154,
+      "mean_token_accuracy": 0.7180401891469955,
+      "num_tokens": 653441.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.2249037250876427,
+      "epoch": 0.192,
+      "grad_norm": 3.96875,
+      "learning_rate": 0.00019942661028236745,
+      "loss": 5.084912490844727,
+      "mean_token_accuracy": 0.6847136527299881,
+      "num_tokens": 682582.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.1726351112127305,
+      "epoch": 0.208,
+      "grad_norm": 3.296875,
+      "learning_rate": 0.00019922698723632767,
+      "loss": 5.018433380126953,
+      "mean_token_accuracy": 0.7055773109197616,
+      "num_tokens": 710468.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.3469643473625184,
+      "epoch": 0.224,
+      "grad_norm": 2.875,
+      "learning_rate": 0.000198997734235711,
+      "loss": 5.7476848602294925,
+      "mean_token_accuracy": 0.6790369123220443,
+      "num_tokens": 740979.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.187085197865963,
+      "epoch": 0.24,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.0001987389197372567,
+      "loss": 5.085608673095703,
+      "mean_token_accuracy": 0.7057169727981091,
+      "num_tokens": 768427.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.2938067495822907,
+      "epoch": 0.256,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.0001984506210249986,
+      "loss": 5.570721054077149,
+      "mean_token_accuracy": 0.6840770006179809,
+      "num_tokens": 796744.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.1161220327019692,
+      "epoch": 0.272,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.00019813292418718732,
+      "loss": 4.821302795410157,
+      "mean_token_accuracy": 0.7168306574225426,
+      "num_tokens": 822225.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.2217196941375732,
+      "epoch": 0.288,
+      "grad_norm": 2.484375,
+      "learning_rate": 0.00019778592409058378,
+      "loss": 5.171672821044922,
+      "mean_token_accuracy": 0.6998881995677948,
+      "num_tokens": 848049.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.1091715931892394,
+      "epoch": 0.304,
+      "grad_norm": 3.109375,
+      "learning_rate": 0.00019740972435213115,
+      "loss": 4.743664169311524,
+      "mean_token_accuracy": 0.7203769966959953,
+      "num_tokens": 873172.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.2154748886823654,
+      "epoch": 0.32,
+      "grad_norm": 2.21875,
+      "learning_rate": 0.00019700443730801413,
+      "loss": 5.212123107910156,
+      "mean_token_accuracy": 0.7076091229915619,
+      "num_tokens": 901716.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.161145742237568,
+      "epoch": 0.336,
+      "grad_norm": 2.078125,
+      "learning_rate": 0.00019657018398011434,
+      "loss": 4.889891052246094,
+      "mean_token_accuracy": 0.7145821407437325,
+      "num_tokens": 928506.0,
+      "step": 210
+    },
+    {
+      "entropy": 1.1209837168455123,
+      "epoch": 0.352,
+      "grad_norm": 2.09375,
+      "learning_rate": 0.00019610709403987246,
+      "loss": 4.862021636962891,
+      "mean_token_accuracy": 0.7152845054864884,
+      "num_tokens": 958723.0,
+      "step": 220
+    },
+    {
+      "entropy": 1.1322738587856294,
+      "epoch": 0.368,
+      "grad_norm": 2.203125,
+      "learning_rate": 0.00019561530576956703,
+      "loss": 4.911846923828125,
+      "mean_token_accuracy": 0.7134342208504677,
+      "num_tokens": 987751.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.1780440196394921,
+      "epoch": 0.384,
+      "grad_norm": 2.421875,
+      "learning_rate": 0.00019509496602102252,
+      "loss": 5.0671440124511715,
+      "mean_token_accuracy": 0.7104542285203934,
+      "num_tokens": 1018315.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.1776200592517854,
+      "epoch": 0.4,
+      "grad_norm": 2.359375,
+      "learning_rate": 0.00019454623017175812,
+      "loss": 5.145317459106446,
+      "mean_token_accuracy": 0.7086253471672534,
+      "num_tokens": 1046627.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.1513659209012985,
+      "epoch": 0.416,
+      "grad_norm": 2.140625,
+      "learning_rate": 0.00019396926207859084,
+      "loss": 4.8750255584716795,
+      "mean_token_accuracy": 0.7203947469592095,
+      "num_tokens": 1075027.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.1364729449152946,
+      "epoch": 0.432,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.00019336423402870653,
+      "loss": 4.866281890869141,
+      "mean_token_accuracy": 0.7150357633829116,
+      "num_tokens": 1102334.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.108440762758255,
+      "epoch": 0.448,
+      "grad_norm": 1.9453125,
+      "learning_rate": 0.00019273132668821364,
+      "loss": 4.697020721435547,
+      "mean_token_accuracy": 0.7265092357993126,
+      "num_tokens": 1131415.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.1001321360468865,
+      "epoch": 0.464,
+      "grad_norm": 2.125,
+      "learning_rate": 0.00019207072904819486,
+      "loss": 4.747102355957031,
+      "mean_token_accuracy": 0.7231316924095154,
+      "num_tokens": 1158280.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.1048941344022751,
+      "epoch": 0.48,
+      "grad_norm": 2.234375,
+      "learning_rate": 0.00019138263836827288,
+      "loss": 4.644550323486328,
+      "mean_token_accuracy": 0.7270571634173393,
+      "num_tokens": 1188651.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.0945911705493927,
+      "epoch": 0.496,
+      "grad_norm": 1.984375,
+      "learning_rate": 0.00019066726011770726,
+      "loss": 4.7493640899658205,
+      "mean_token_accuracy": 0.7258608743548394,
+      "num_tokens": 1216504.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.0356923416256905,
+      "epoch": 0.512,
+      "grad_norm": 2.1875,
+      "learning_rate": 0.00018992480791403958,
+      "loss": 4.3843944549560545,
+      "mean_token_accuracy": 0.7372462660074234,
+      "num_tokens": 1243732.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.0773125097155571,
+      "epoch": 0.528,
+      "grad_norm": 2.328125,
+      "learning_rate": 0.0001891555034593055,
+      "loss": 4.6180767059326175,
+      "mean_token_accuracy": 0.7283408164978027,
+      "num_tokens": 1269272.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.050987295806408,
+      "epoch": 0.544,
+      "grad_norm": 1.640625,
+      "learning_rate": 0.00018835957647383303,
+      "loss": 4.4863533020019535,
+      "mean_token_accuracy": 0.7359548002481461,
+      "num_tokens": 1296001.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.0461898490786552,
+      "epoch": 0.56,
+      "grad_norm": 2.453125,
+      "learning_rate": 0.000187537264627646,
+      "loss": 4.434906005859375,
+      "mean_token_accuracy": 0.7349108412861824,
+      "num_tokens": 1322180.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.0046504065394402,
+      "epoch": 0.576,
+      "grad_norm": 2.125,
+      "learning_rate": 0.00018668881346949417,
+      "loss": 4.309226989746094,
+      "mean_token_accuracy": 0.73991359770298,
+      "num_tokens": 1347532.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.0861794739961623,
+      "epoch": 0.592,
+      "grad_norm": 1.890625,
+      "learning_rate": 0.0001858144763535302,
+      "loss": 4.599692535400391,
+      "mean_token_accuracy": 0.7285229310393333,
+      "num_tokens": 1373584.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.0600647926330566,
+      "epoch": 0.608,
+      "grad_norm": 2.203125,
+      "learning_rate": 0.00018491451436365627,
+      "loss": 4.5491493225097654,
+      "mean_token_accuracy": 0.7357145607471466,
+      "num_tokens": 1399594.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.098224511742592,
+      "epoch": 0.624,
+      "grad_norm": 1.9296875,
+      "learning_rate": 0.00018398919623556238,
+      "loss": 4.8250572204589846,
+      "mean_token_accuracy": 0.7251888766884804,
+      "num_tokens": 1428792.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.1232757449150086,
+      "epoch": 0.64,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.00018303879827647975,
+      "loss": 4.712010192871094,
+      "mean_token_accuracy": 0.7287421196699142,
+      "num_tokens": 1457889.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.0411028936505318,
+      "epoch": 0.656,
+      "grad_norm": 1.859375,
+      "learning_rate": 0.00018206360428267332,
+      "loss": 4.405958938598633,
+      "mean_token_accuracy": 0.7350577011704444,
+      "num_tokens": 1484479.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.1146624743938447,
+      "epoch": 0.672,
+      "grad_norm": 1.734375,
+      "learning_rate": 0.00018106390545469795,
+      "loss": 4.742184448242187,
+      "mean_token_accuracy": 0.7282734125852585,
+      "num_tokens": 1514268.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.0480840012431145,
+      "epoch": 0.688,
+      "grad_norm": 1.78125,
+      "learning_rate": 0.0001800400003104436,
+      "loss": 4.4207916259765625,
+      "mean_token_accuracy": 0.7365467861294747,
+      "num_tokens": 1540474.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.0067586719989776,
+      "epoch": 0.704,
+      "grad_norm": 1.921875,
+      "learning_rate": 0.0001789921945959958,
+      "loss": 4.31513671875,
+      "mean_token_accuracy": 0.7437789484858512,
+      "num_tokens": 1568557.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.0166094586253167,
+      "epoch": 0.72,
+      "grad_norm": 1.6171875,
+      "learning_rate": 0.0001779208011943371,
+      "loss": 4.292739486694336,
+      "mean_token_accuracy": 0.7471857726573944,
+      "num_tokens": 1594403.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.0385540708899499,
+      "epoch": 0.736,
+      "grad_norm": 2.75,
+      "learning_rate": 0.00017682614003191807,
+      "loss": 4.415458679199219,
+      "mean_token_accuracy": 0.7400641202926636,
+      "num_tokens": 1624111.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.0706985503435136,
+      "epoch": 0.752,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.0001757085379831246,
+      "loss": 4.554729080200195,
+      "mean_token_accuracy": 0.7342943042516709,
+      "num_tokens": 1652609.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.9689434483647347,
+      "epoch": 0.768,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.00017456832877267084,
+      "loss": 4.2303211212158205,
+      "mean_token_accuracy": 0.7474748462438583,
+      "num_tokens": 1678589.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.1837652444839477,
+      "epoch": 0.784,
+      "grad_norm": 2.078125,
+      "learning_rate": 0.00017340585287594604,
+      "loss": 4.968061447143555,
+      "mean_token_accuracy": 0.7166377156972885,
+      "num_tokens": 1708513.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.042286379635334,
+      "epoch": 0.8,
+      "grad_norm": 2.515625,
+      "learning_rate": 0.00017222145741734626,
+      "loss": 4.422880172729492,
+      "mean_token_accuracy": 0.7405225187540054,
+      "num_tokens": 1736283.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.9832519263029098,
+      "epoch": 0.816,
+      "grad_norm": 1.6171875,
+      "learning_rate": 0.00017101549606662024,
+      "loss": 4.119276428222657,
+      "mean_token_accuracy": 0.7495438739657402,
+      "num_tokens": 1765017.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.9872105062007904,
+      "epoch": 0.832,
+      "grad_norm": 1.71875,
+      "learning_rate": 0.00016978832893326074,
+      "loss": 4.234106826782226,
+      "mean_token_accuracy": 0.7473902180790901,
+      "num_tokens": 1790682.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.9845283895730972,
+      "epoch": 0.848,
+      "grad_norm": 1.4921875,
+      "learning_rate": 0.00016854032245897308,
+      "loss": 4.146430969238281,
+      "mean_token_accuracy": 0.7520387843251228,
+      "num_tokens": 1820050.0,
+      "step": 530
+    },
+    {
+      "entropy": 1.0613365799188614,
+      "epoch": 0.864,
+      "grad_norm": 2.046875,
+      "learning_rate": 0.00016727184930825288,
+      "loss": 4.48931655883789,
+      "mean_token_accuracy": 0.7396015107631684,
+      "num_tokens": 1847079.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.9684726029634476,
+      "epoch": 0.88,
+      "grad_norm": 2.03125,
+      "learning_rate": 0.00016598328825710533,
+      "loss": 4.12475357055664,
+      "mean_token_accuracy": 0.7526269048452378,
+      "num_tokens": 1872383.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.9990086019039154,
+      "epoch": 0.896,
+      "grad_norm": 1.8984375,
+      "learning_rate": 0.00016467502407993992,
+      "loss": 4.360863494873047,
+      "mean_token_accuracy": 0.7387492001056671,
+      "num_tokens": 1903630.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.9708257809281349,
+      "epoch": 0.912,
+      "grad_norm": 1.6328125,
+      "learning_rate": 0.00016334744743467364,
+      "loss": 4.054442596435547,
+      "mean_token_accuracy": 0.7545965671539306,
+      "num_tokens": 1930592.0,
+      "step": 570
+    },
+    {
+      "entropy": 1.0035204842686654,
+      "epoch": 0.928,
+      "grad_norm": 1.8984375,
+      "learning_rate": 0.00016200095474607753,
+      "loss": 4.25194206237793,
+      "mean_token_accuracy": 0.7464351058006287,
+      "num_tokens": 1960212.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.9635734960436821,
+      "epoch": 0.944,
+      "grad_norm": 1.53125,
+      "learning_rate": 0.00016063594808740113,
+      "loss": 4.0587310791015625,
+      "mean_token_accuracy": 0.7547481089830399,
+      "num_tokens": 1986049.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.9986386775970459,
+      "epoch": 0.96,
+      "grad_norm": 1.703125,
+      "learning_rate": 0.0001592528350603103,
+      "loss": 4.331460952758789,
+      "mean_token_accuracy": 0.7503352165222168,
+      "num_tokens": 2013266.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.006168755888939,
+      "epoch": 0.976,
+      "grad_norm": 1.734375,
+      "learning_rate": 0.00015785202867317407,
+      "loss": 4.238505554199219,
+      "mean_token_accuracy": 0.7441465124487877,
+      "num_tokens": 2041824.0,
+      "step": 610
+    },
+    {
+      "entropy": 1.0482723653316497,
+      "epoch": 0.992,
+      "grad_norm": 1.9296875,
+      "learning_rate": 0.0001564339472177373,
+      "loss": 4.394336700439453,
+      "mean_token_accuracy": 0.7416493371129036,
+      "num_tokens": 2071693.0,
+      "step": 620
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.0188530464172363,
+      "eval_loss": 1.0276753902435303,
+      "eval_mean_token_accuracy": 0.7517141411304474,
+      "eval_num_tokens": 2084150.0,
+      "eval_runtime": 72.9791,
+      "eval_samples_per_second": 13.703,
+      "eval_steps_per_second": 3.426,
+      "step": 625
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1875,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.400684024323021e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-625/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed89ca2be67eb35da01705488407af1aa1ea83180f94fa1d6688fc37dcb50663
+size 5713

runs/Apr04_02-44-14_c6dd9530df0f/events.out.tfevents.1775270654.c6dd9530df0f.4098.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77baebe588f7bf758899505b80a216428b593a936818c6b2b9cf0e4fbd1ea323
+size 97599

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f
+size 32169626

tokenizer_config.json

@@ -0,0 +1,95 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [
+    "<|video|>"
+  ],
+  "image_token": "<|image|>",
+  "is_local": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "audio_token": "<|audio|>",
+    "boa_token": "<|audio>",
+    "boi_token": "<|image>",
+    "eoa_token": "<audio|>",
+    "eoc_token": "<channel|>",
+    "eoi_token": "<image|>",
+    "eot_token": "<turn|>",
+    "escape_token": "<|\"|>",
+    "etc_token": "<tool_call|>",
+    "etd_token": "<tool|>",
+    "etr_token": "<tool_response|>",
+    "image_token": "<|image|>",
+    "soc_token": "<|channel>",
+    "sot_token": "<|turn>",
+    "stc_token": "<|tool_call>",
+    "std_token": "<|tool>",
+    "str_token": "<|tool_response>",
+    "think_token": "<|think|>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma4Processor",
+  "response_schema": {
+    "properties": {
+      "content": {
+        "type": "string"
+      },
+      "role": {
+        "const": "assistant"
+      },
+      "thinking": {
+        "type": "string"
+      },
+      "tool_calls": {
+        "items": {
+          "properties": {
+            "function": {
+              "properties": {
+                "arguments": {
+                  "additionalProperties": {},
+                  "type": "object",
+                  "x-parser": "gemma4-tool-call"
+                },
+                "name": {
+                  "type": "string"
+                }
+              },
+              "type": "object",
+              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"
+            },
+            "type": {
+              "const": "function"
+            }
+          },
+          "type": "object"
+        },
+        "type": "array",
+        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"
+      }
+    },
+    "type": "object",
+    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<content>(?:(?!\\<\\|tool_call\\>)(?!\\<turn\\|\\>).)+)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?:\\<turn\\|\\>)?"
+  },
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed89ca2be67eb35da01705488407af1aa1ea83180f94fa1d6688fc37dcb50663
+size 5713