Duplicate from prxy5607/d8bab1ad-d4c5-4a11-9355-c7bdbdca2625

Co-authored-by: PRXY 5607 <prxy5607@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,165 @@
+---
+library_name: peft
+license: other
+base_model: Qwen/Qwen1.5-0.5B
+tags:
+- axolotl
+- generated_from_trainer
+model-index:
+- name: d8bab1ad-d4c5-4a11-9355-c7bdbdca2625
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.4.1`
+```yaml
+adapter: lora
+base_model: Qwen/Qwen1.5-0.5B
+bf16: true
+chat_template: llama3
+data_processes: 16
+dataset_prepared_path: null
+datasets:
+- data_files:
+  - 860b8d4c90d8e589_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/860b8d4c90d8e589_train_data.json
+  type:
+    field_instruction: instruction
+    field_output: output
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+debug: null
+deepspeed: null
+device_map: auto
+do_eval: true
+early_stopping_patience: 5
+eval_batch_size: 4
+eval_max_new_tokens: 128
+eval_steps: 50
+eval_table_size: null
+evals_per_epoch: null
+flash_attention: true
+fp16: false
+fsdp: null
+fsdp_config: null
+gradient_accumulation_steps: 4
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: prxy5607/d8bab1ad-d4c5-4a11-9355-c7bdbdca2625
+hub_repo: null
+hub_strategy: checkpoint
+hub_token: null
+learning_rate: 0.0001
+load_in_4bit: false
+load_in_8bit: false
+local_rank: null
+logging_steps: 1
+lora_alpha: 128
+lora_dropout: 0.05
+lora_fan_in_fan_out: null
+lora_model_dir: null
+lora_r: 64
+lora_target_linear: true
+lr_scheduler: cosine
+max_grad_norm: 1.0
+max_memory:
+  0: 75GB
+max_steps: 200
+micro_batch_size: 8
+mlflow_experiment_name: /tmp/860b8d4c90d8e589_train_data.json
+model_type: AutoModelForCausalLM
+num_epochs: 3
+optim_args:
+  adam_beta1: 0.9
+  adam_beta2: 0.95
+  adam_epsilon: 1e-5
+optimizer: adamw_bnb_8bit
+output_dir: miner_id_24
+pad_to_sequence_len: true
+resume_from_checkpoint: null
+s2_attention: null
+sample_packing: false
+save_steps: 50
+saves_per_epoch: null
+sequence_len: 1024
+strict: false
+tf32: true
+tokenizer_type: AutoTokenizer
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.05
+wandb_entity: null
+wandb_mode: online
+wandb_name: c3048255-83e5-4714-a7da-aaac6d54116a
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: c3048255-83e5-4714-a7da-aaac6d54116a
+warmup_steps: 10
+weight_decay: 0.0
+xformers_attention: null
+
+```
+
+</details><br>
+
+# d8bab1ad-d4c5-4a11-9355-c7bdbdca2625
+
+This model is a fine-tuned version of [Qwen/Qwen1.5-0.5B](https://huggingface.co/Qwen/Qwen1.5-0.5B) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.4917
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 8
+- eval_batch_size: 4
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=adam_beta1=0.9,adam_beta2=0.95,adam_epsilon=1e-5
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 10
+- training_steps: 200
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 0.4368        | 0.0013 | 1    | 0.5608          |
+| 0.7765        | 0.0638 | 50   | 0.5404          |
+| 0.7028        | 0.1276 | 100  | 0.5071          |
+| 0.718         | 0.1914 | 150  | 0.4941          |
+| 0.9528        | 0.2553 | 200  | 0.4917          |
+
+
+### Framework versions
+
+- PEFT 0.13.2
+- Transformers 4.46.0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.1
+- Tokenizers 0.20.1

adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen1.5-0.5B",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "gate_proj",
+    "down_proj",
+    "up_proj",
+    "v_proj",
+    "o_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e58925830ef6ff7b848c03757a804a02559a07561d30b5abb62b52bdbbac104
+size 121232010

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e69c01b0baa97f76913d3bae86869baf01d5778081769bf6557083c3ad8cd35
+size 121155320

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "Qwen/Qwen1.5-0.5B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 2816,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 16,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

last-checkpoint/README.md

@@ -0,0 +1,202 @@
+---
+base_model: Qwen/Qwen1.5-0.5B
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.13.2

last-checkpoint/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen1.5-0.5B",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "gate_proj",
+    "down_proj",
+    "up_proj",
+    "v_proj",
+    "o_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

last-checkpoint/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e69c01b0baa97f76913d3bae86869baf01d5778081769bf6557083c3ad8cd35
+size 121155320

last-checkpoint/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:065b766bf8fdab244b6ec3fea66bc9a2d1e024f92ce7bb6f9c9823b5a251ad81
+size 61896852

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75acfddc06afa3ab2513f063faf0174c897a7fd26463ca4378ef071a29e56ecd
+size 14244

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2d754412c61116546142914503e7369d0cc35d3c380a07e5218f595d76b6d96
+size 1064

last-checkpoint/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

last-checkpoint/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcfe42da0a4497e8b2b172c1f9f4ec423a46dc12907f4349c55025f670422ba9
+size 11418266

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,1482 @@
+{
+  "best_metric": 0.4917088747024536,
+  "best_model_checkpoint": "miner_id_24/checkpoint-200",
+  "epoch": 0.2552648372686662,
+  "eval_steps": 50,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0012763241863433313,
+      "grad_norm": 0.37641966342926025,
+      "learning_rate": 1e-05,
+      "loss": 0.4368,
+      "step": 1
+    },
+    {
+      "epoch": 0.0012763241863433313,
+      "eval_loss": 0.5608253479003906,
+      "eval_runtime": 23.1399,
+      "eval_samples_per_second": 57.044,
+      "eval_steps_per_second": 14.261,
+      "step": 1
+    },
+    {
+      "epoch": 0.0025526483726866626,
+      "grad_norm": 0.4534890353679657,
+      "learning_rate": 2e-05,
+      "loss": 0.5335,
+      "step": 2
+    },
+    {
+      "epoch": 0.0038289725590299937,
+      "grad_norm": 0.4631837010383606,
+      "learning_rate": 3e-05,
+      "loss": 0.5789,
+      "step": 3
+    },
+    {
+      "epoch": 0.005105296745373325,
+      "grad_norm": 0.4761736989021301,
+      "learning_rate": 4e-05,
+      "loss": 0.4519,
+      "step": 4
+    },
+    {
+      "epoch": 0.006381620931716656,
+      "grad_norm": 0.514417827129364,
+      "learning_rate": 5e-05,
+      "loss": 0.552,
+      "step": 5
+    },
+    {
+      "epoch": 0.007657945118059987,
+      "grad_norm": 0.5431033372879028,
+      "learning_rate": 6e-05,
+      "loss": 0.5444,
+      "step": 6
+    },
+    {
+      "epoch": 0.008934269304403318,
+      "grad_norm": 0.6033271551132202,
+      "learning_rate": 7e-05,
+      "loss": 0.5097,
+      "step": 7
+    },
+    {
+      "epoch": 0.01021059349074665,
+      "grad_norm": 0.6013683080673218,
+      "learning_rate": 8e-05,
+      "loss": 0.4829,
+      "step": 8
+    },
+    {
+      "epoch": 0.01148691767708998,
+      "grad_norm": 0.5697230100631714,
+      "learning_rate": 9e-05,
+      "loss": 0.5655,
+      "step": 9
+    },
+    {
+      "epoch": 0.012763241863433313,
+      "grad_norm": 0.6451380252838135,
+      "learning_rate": 0.0001,
+      "loss": 0.4481,
+      "step": 10
+    },
+    {
+      "epoch": 0.014039566049776643,
+      "grad_norm": 0.6178256273269653,
+      "learning_rate": 9.999316524962345e-05,
+      "loss": 0.5175,
+      "step": 11
+    },
+    {
+      "epoch": 0.015315890236119975,
+      "grad_norm": 0.5703033208847046,
+      "learning_rate": 9.997266286704631e-05,
+      "loss": 0.4731,
+      "step": 12
+    },
+    {
+      "epoch": 0.016592214422463305,
+      "grad_norm": 0.6212876439094543,
+      "learning_rate": 9.993849845741524e-05,
+      "loss": 0.4671,
+      "step": 13
+    },
+    {
+      "epoch": 0.017868538608806637,
+      "grad_norm": 0.6187449097633362,
+      "learning_rate": 9.989068136093873e-05,
+      "loss": 0.4454,
+      "step": 14
+    },
+    {
+      "epoch": 0.01914486279514997,
+      "grad_norm": 0.6634529829025269,
+      "learning_rate": 9.98292246503335e-05,
+      "loss": 0.4789,
+      "step": 15
+    },
+    {
+      "epoch": 0.0204211869814933,
+      "grad_norm": 0.6257879734039307,
+      "learning_rate": 9.975414512725057e-05,
+      "loss": 0.4996,
+      "step": 16
+    },
+    {
+      "epoch": 0.02169751116783663,
+      "grad_norm": 0.631335973739624,
+      "learning_rate": 9.966546331768191e-05,
+      "loss": 0.503,
+      "step": 17
+    },
+    {
+      "epoch": 0.02297383535417996,
+      "grad_norm": 0.6125084161758423,
+      "learning_rate": 9.956320346634876e-05,
+      "loss": 0.5093,
+      "step": 18
+    },
+    {
+      "epoch": 0.024250159540523293,
+      "grad_norm": 0.5686100125312805,
+      "learning_rate": 9.944739353007344e-05,
+      "loss": 0.4747,
+      "step": 19
+    },
+    {
+      "epoch": 0.025526483726866625,
+      "grad_norm": 0.5883579254150391,
+      "learning_rate": 9.931806517013612e-05,
+      "loss": 0.4663,
+      "step": 20
+    },
+    {
+      "epoch": 0.026802807913209957,
+      "grad_norm": 0.5903049111366272,
+      "learning_rate": 9.917525374361912e-05,
+      "loss": 0.5262,
+      "step": 21
+    },
+    {
+      "epoch": 0.028079132099553285,
+      "grad_norm": 0.5621379613876343,
+      "learning_rate": 9.901899829374047e-05,
+      "loss": 0.4749,
+      "step": 22
+    },
+    {
+      "epoch": 0.029355456285896617,
+      "grad_norm": 0.5512058734893799,
+      "learning_rate": 9.884934153917997e-05,
+      "loss": 0.4451,
+      "step": 23
+    },
+    {
+      "epoch": 0.03063178047223995,
+      "grad_norm": 0.5691269636154175,
+      "learning_rate": 9.86663298624003e-05,
+      "loss": 0.4989,
+      "step": 24
+    },
+    {
+      "epoch": 0.03190810465858328,
+      "grad_norm": 0.6421544551849365,
+      "learning_rate": 9.847001329696653e-05,
+      "loss": 0.5056,
+      "step": 25
+    },
+    {
+      "epoch": 0.03318442884492661,
+      "grad_norm": 0.6291079521179199,
+      "learning_rate": 9.826044551386744e-05,
+      "loss": 0.4711,
+      "step": 26
+    },
+    {
+      "epoch": 0.03446075303126994,
+      "grad_norm": 0.5773482918739319,
+      "learning_rate": 9.803768380684242e-05,
+      "loss": 0.4504,
+      "step": 27
+    },
+    {
+      "epoch": 0.035737077217613274,
+      "grad_norm": 0.5747997164726257,
+      "learning_rate": 9.780178907671789e-05,
+      "loss": 0.4807,
+      "step": 28
+    },
+    {
+      "epoch": 0.037013401403956606,
+      "grad_norm": 0.5618578195571899,
+      "learning_rate": 9.755282581475769e-05,
+      "loss": 0.4023,
+      "step": 29
+    },
+    {
+      "epoch": 0.03828972559029994,
+      "grad_norm": 0.5824413299560547,
+      "learning_rate": 9.729086208503174e-05,
+      "loss": 0.4787,
+      "step": 30
+    },
+    {
+      "epoch": 0.03956604977664327,
+      "grad_norm": 0.5838544964790344,
+      "learning_rate": 9.701596950580806e-05,
+      "loss": 0.492,
+      "step": 31
+    },
+    {
+      "epoch": 0.0408423739629866,
+      "grad_norm": 0.6019923686981201,
+      "learning_rate": 9.672822322997305e-05,
+      "loss": 0.4381,
+      "step": 32
+    },
+    {
+      "epoch": 0.042118698149329926,
+      "grad_norm": 0.5955802202224731,
+      "learning_rate": 9.642770192448536e-05,
+      "loss": 0.4111,
+      "step": 33
+    },
+    {
+      "epoch": 0.04339502233567326,
+      "grad_norm": 0.5979661345481873,
+      "learning_rate": 9.611448774886924e-05,
+      "loss": 0.4346,
+      "step": 34
+    },
+    {
+      "epoch": 0.04467134652201659,
+      "grad_norm": 0.5971465110778809,
+      "learning_rate": 9.578866633275288e-05,
+      "loss": 0.4648,
+      "step": 35
+    },
+    {
+      "epoch": 0.04594767070835992,
+      "grad_norm": 0.5893446803092957,
+      "learning_rate": 9.545032675245813e-05,
+      "loss": 0.4582,
+      "step": 36
+    },
+    {
+      "epoch": 0.047223994894703254,
+      "grad_norm": 0.6208324432373047,
+      "learning_rate": 9.509956150664796e-05,
+      "loss": 0.3868,
+      "step": 37
+    },
+    {
+      "epoch": 0.048500319081046586,
+      "grad_norm": 0.6666813492774963,
+      "learning_rate": 9.473646649103818e-05,
+      "loss": 0.4443,
+      "step": 38
+    },
+    {
+      "epoch": 0.04977664326738992,
+      "grad_norm": 0.6952598094940186,
+      "learning_rate": 9.43611409721806e-05,
+      "loss": 0.4523,
+      "step": 39
+    },
+    {
+      "epoch": 0.05105296745373325,
+      "grad_norm": 0.7287755012512207,
+      "learning_rate": 9.397368756032445e-05,
+      "loss": 0.4863,
+      "step": 40
+    },
+    {
+      "epoch": 0.05232929164007658,
+      "grad_norm": 0.7538745403289795,
+      "learning_rate": 9.357421218136386e-05,
+      "loss": 0.4104,
+      "step": 41
+    },
+    {
+      "epoch": 0.053605615826419914,
+      "grad_norm": 0.7233603000640869,
+      "learning_rate": 9.316282404787871e-05,
+      "loss": 0.3859,
+      "step": 42
+    },
+    {
+      "epoch": 0.05488194001276324,
+      "grad_norm": 0.7828330397605896,
+      "learning_rate": 9.273963562927695e-05,
+      "loss": 0.4195,
+      "step": 43
+    },
+    {
+      "epoch": 0.05615826419910657,
+      "grad_norm": 0.9844853281974792,
+      "learning_rate": 9.230476262104677e-05,
+      "loss": 0.5328,
+      "step": 44
+    },
+    {
+      "epoch": 0.0574345883854499,
+      "grad_norm": 1.2196927070617676,
+      "learning_rate": 9.185832391312644e-05,
+      "loss": 0.6285,
+      "step": 45
+    },
+    {
+      "epoch": 0.058710912571793235,
+      "grad_norm": 1.5032840967178345,
+      "learning_rate": 9.140044155740101e-05,
+      "loss": 0.6508,
+      "step": 46
+    },
+    {
+      "epoch": 0.05998723675813657,
+      "grad_norm": 1.3031989336013794,
+      "learning_rate": 9.093124073433463e-05,
+      "loss": 0.6553,
+      "step": 47
+    },
+    {
+      "epoch": 0.0612635609444799,
+      "grad_norm": 1.5585204362869263,
+      "learning_rate": 9.045084971874738e-05,
+      "loss": 0.7208,
+      "step": 48
+    },
+    {
+      "epoch": 0.06253988513082323,
+      "grad_norm": 2.1215412616729736,
+      "learning_rate": 8.995939984474624e-05,
+      "loss": 0.6276,
+      "step": 49
+    },
+    {
+      "epoch": 0.06381620931716656,
+      "grad_norm": 2.615065097808838,
+      "learning_rate": 8.945702546981969e-05,
+      "loss": 0.7765,
+      "step": 50
+    },
+    {
+      "epoch": 0.06381620931716656,
+      "eval_loss": 0.5404265522956848,
+      "eval_runtime": 22.9302,
+      "eval_samples_per_second": 57.566,
+      "eval_steps_per_second": 14.391,
+      "step": 50
+    },
+    {
+      "epoch": 0.0650925335035099,
+      "grad_norm": 0.430701345205307,
+      "learning_rate": 8.894386393810563e-05,
+      "loss": 0.4316,
+      "step": 51
+    },
+    {
+      "epoch": 0.06636885768985322,
+      "grad_norm": 0.47454386949539185,
+      "learning_rate": 8.842005554284296e-05,
+      "loss": 0.4918,
+      "step": 52
+    },
+    {
+      "epoch": 0.06764518187619656,
+      "grad_norm": 0.4706365466117859,
+      "learning_rate": 8.788574348801675e-05,
+      "loss": 0.5707,
+      "step": 53
+    },
+    {
+      "epoch": 0.06892150606253988,
+      "grad_norm": 0.4887033998966217,
+      "learning_rate": 8.73410738492077e-05,
+      "loss": 0.6148,
+      "step": 54
+    },
+    {
+      "epoch": 0.07019783024888322,
+      "grad_norm": 0.4659351110458374,
+      "learning_rate": 8.678619553365659e-05,
+      "loss": 0.5816,
+      "step": 55
+    },
+    {
+      "epoch": 0.07147415443522655,
+      "grad_norm": 0.4376816153526306,
+      "learning_rate": 8.622126023955446e-05,
+      "loss": 0.5145,
+      "step": 56
+    },
+    {
+      "epoch": 0.07275047862156987,
+      "grad_norm": 0.43677350878715515,
+      "learning_rate": 8.564642241456986e-05,
+      "loss": 0.5502,
+      "step": 57
+    },
+    {
+      "epoch": 0.07402680280791321,
+      "grad_norm": 0.4679816961288452,
+      "learning_rate": 8.506183921362443e-05,
+      "loss": 0.5514,
+      "step": 58
+    },
+    {
+      "epoch": 0.07530312699425654,
+      "grad_norm": 0.4348083734512329,
+      "learning_rate": 8.44676704559283e-05,
+      "loss": 0.5216,
+      "step": 59
+    },
+    {
+      "epoch": 0.07657945118059988,
+      "grad_norm": 0.4561150074005127,
+      "learning_rate": 8.386407858128706e-05,
+      "loss": 0.5618,
+      "step": 60
+    },
+    {
+      "epoch": 0.0778557753669432,
+      "grad_norm": 0.468262642621994,
+      "learning_rate": 8.32512286056924e-05,
+      "loss": 0.5123,
+      "step": 61
+    },
+    {
+      "epoch": 0.07913209955328654,
+      "grad_norm": 0.46827855706214905,
+      "learning_rate": 8.262928807620843e-05,
+      "loss": 0.5597,
+      "step": 62
+    },
+    {
+      "epoch": 0.08040842373962986,
+      "grad_norm": 0.4949502646923065,
+      "learning_rate": 8.199842702516583e-05,
+      "loss": 0.519,
+      "step": 63
+    },
+    {
+      "epoch": 0.0816847479259732,
+      "grad_norm": 0.44436025619506836,
+      "learning_rate": 8.135881792367686e-05,
+      "loss": 0.4945,
+      "step": 64
+    },
+    {
+      "epoch": 0.08296107211231653,
+      "grad_norm": 0.44837185740470886,
+      "learning_rate": 8.07106356344834e-05,
+      "loss": 0.5026,
+      "step": 65
+    },
+    {
+      "epoch": 0.08423739629865985,
+      "grad_norm": 0.47120752930641174,
+      "learning_rate": 8.005405736415126e-05,
+      "loss": 0.5032,
+      "step": 66
+    },
+    {
+      "epoch": 0.08551372048500319,
+      "grad_norm": 0.4542293846607208,
+      "learning_rate": 7.938926261462366e-05,
+      "loss": 0.4768,
+      "step": 67
+    },
+    {
+      "epoch": 0.08679004467134652,
+      "grad_norm": 0.45541873574256897,
+      "learning_rate": 7.871643313414718e-05,
+      "loss": 0.4579,
+      "step": 68
+    },
+    {
+      "epoch": 0.08806636885768986,
+      "grad_norm": 0.4422976076602936,
+      "learning_rate": 7.803575286758364e-05,
+      "loss": 0.465,
+      "step": 69
+    },
+    {
+      "epoch": 0.08934269304403318,
+      "grad_norm": 0.4453737437725067,
+      "learning_rate": 7.734740790612136e-05,
+      "loss": 0.4724,
+      "step": 70
+    },
+    {
+      "epoch": 0.09061901723037652,
+      "grad_norm": 0.4480217695236206,
+      "learning_rate": 7.66515864363997e-05,
+      "loss": 0.5033,
+      "step": 71
+    },
+    {
+      "epoch": 0.09189534141671984,
+      "grad_norm": 0.44391363859176636,
+      "learning_rate": 7.594847868906076e-05,
+      "loss": 0.4314,
+      "step": 72
+    },
+    {
+      "epoch": 0.09317166560306318,
+      "grad_norm": 0.4313923418521881,
+      "learning_rate": 7.52382768867422e-05,
+      "loss": 0.3964,
+      "step": 73
+    },
+    {
+      "epoch": 0.09444798978940651,
+      "grad_norm": 0.4617796838283539,
+      "learning_rate": 7.452117519152542e-05,
+      "loss": 0.4351,
+      "step": 74
+    },
+    {
+      "epoch": 0.09572431397574985,
+      "grad_norm": 0.47440049052238464,
+      "learning_rate": 7.379736965185368e-05,
+      "loss": 0.4476,
+      "step": 75
+    },
+    {
+      "epoch": 0.09700063816209317,
+      "grad_norm": 0.47849926352500916,
+      "learning_rate": 7.30670581489344e-05,
+      "loss": 0.5245,
+      "step": 76
+    },
+    {
+      "epoch": 0.0982769623484365,
+      "grad_norm": 0.4690335988998413,
+      "learning_rate": 7.233044034264034e-05,
+      "loss": 0.4633,
+      "step": 77
+    },
+    {
+      "epoch": 0.09955328653477984,
+      "grad_norm": 0.4430462718009949,
+      "learning_rate": 7.158771761692464e-05,
+      "loss": 0.4268,
+      "step": 78
+    },
+    {
+      "epoch": 0.10082961072112316,
+      "grad_norm": 0.5094535946846008,
+      "learning_rate": 7.083909302476453e-05,
+      "loss": 0.4849,
+      "step": 79
+    },
+    {
+      "epoch": 0.1021059349074665,
+      "grad_norm": 0.4931361973285675,
+      "learning_rate": 7.008477123264848e-05,
+      "loss": 0.4241,
+      "step": 80
+    },
+    {
+      "epoch": 0.10338225909380983,
+      "grad_norm": 0.4746374487876892,
+      "learning_rate": 6.932495846462261e-05,
+      "loss": 0.4575,
+      "step": 81
+    },
+    {
+      "epoch": 0.10465858328015316,
+      "grad_norm": 0.4942508637905121,
+      "learning_rate": 6.855986244591104e-05,
+      "loss": 0.4278,
+      "step": 82
+    },
+    {
+      "epoch": 0.10593490746649649,
+      "grad_norm": 0.5018263459205627,
+      "learning_rate": 6.778969234612584e-05,
+      "loss": 0.4473,
+      "step": 83
+    },
+    {
+      "epoch": 0.10721123165283983,
+      "grad_norm": 0.5011287927627563,
+      "learning_rate": 6.701465872208216e-05,
+      "loss": 0.3925,
+      "step": 84
+    },
+    {
+      "epoch": 0.10848755583918315,
+      "grad_norm": 0.49989867210388184,
+      "learning_rate": 6.623497346023418e-05,
+      "loss": 0.3619,
+      "step": 85
+    },
+    {
+      "epoch": 0.10976388002552648,
+      "grad_norm": 0.5609748363494873,
+      "learning_rate": 6.545084971874738e-05,
+      "loss": 0.4623,
+      "step": 86
+    },
+    {
+      "epoch": 0.11104020421186982,
+      "grad_norm": 0.5378284454345703,
+      "learning_rate": 6.466250186922325e-05,
+      "loss": 0.4022,
+      "step": 87
+    },
+    {
+      "epoch": 0.11231652839821314,
+      "grad_norm": 0.5313222408294678,
+      "learning_rate": 6.387014543809223e-05,
+      "loss": 0.4028,
+      "step": 88
+    },
+    {
+      "epoch": 0.11359285258455648,
+      "grad_norm": 0.6048693656921387,
+      "learning_rate": 6.307399704769099e-05,
+      "loss": 0.4676,
+      "step": 89
+    },
+    {
+      "epoch": 0.1148691767708998,
+      "grad_norm": 0.6088370680809021,
+      "learning_rate": 6.227427435703997e-05,
+      "loss": 0.4231,
+      "step": 90
+    },
+    {
+      "epoch": 0.11614550095724314,
+      "grad_norm": 0.6291335225105286,
+      "learning_rate": 6.147119600233758e-05,
+      "loss": 0.3802,
+      "step": 91
+    },
+    {
+      "epoch": 0.11742182514358647,
+      "grad_norm": 0.7526514530181885,
+      "learning_rate": 6.066498153718735e-05,
+      "loss": 0.4127,
+      "step": 92
+    },
+    {
+      "epoch": 0.11869814932992981,
+      "grad_norm": 0.6645894646644592,
+      "learning_rate": 5.985585137257401e-05,
+      "loss": 0.3678,
+      "step": 93
+    },
+    {
+      "epoch": 0.11997447351627313,
+      "grad_norm": 0.7924028635025024,
+      "learning_rate": 5.90440267166055e-05,
+      "loss": 0.5067,
+      "step": 94
+    },
+    {
+      "epoch": 0.12125079770261646,
+      "grad_norm": 0.94552081823349,
+      "learning_rate": 5.8229729514036705e-05,
+      "loss": 0.5221,
+      "step": 95
+    },
+    {
+      "epoch": 0.1225271218889598,
+      "grad_norm": 1.0852935314178467,
+      "learning_rate": 5.74131823855921e-05,
+      "loss": 0.7297,
+      "step": 96
+    },
+    {
+      "epoch": 0.12380344607530312,
+      "grad_norm": 1.1597877740859985,
+      "learning_rate": 5.6594608567103456e-05,
+      "loss": 0.7244,
+      "step": 97
+    },
+    {
+      "epoch": 0.12507977026164646,
+      "grad_norm": 1.403279423713684,
+      "learning_rate": 5.577423184847932e-05,
+      "loss": 0.7322,
+      "step": 98
+    },
+    {
+      "epoch": 0.1263560944479898,
+      "grad_norm": 1.5290478467941284,
+      "learning_rate": 5.495227651252315e-05,
+      "loss": 0.5653,
+      "step": 99
+    },
+    {
+      "epoch": 0.1276324186343331,
+      "grad_norm": 2.1053667068481445,
+      "learning_rate": 5.4128967273616625e-05,
+      "loss": 0.7028,
+      "step": 100
+    },
+    {
+      "epoch": 0.1276324186343331,
+      "eval_loss": 0.5070851445198059,
+      "eval_runtime": 23.1224,
+      "eval_samples_per_second": 57.087,
+      "eval_steps_per_second": 14.272,
+      "step": 100
+    },
+    {
+      "epoch": 0.12890874282067646,
+      "grad_norm": 0.31127122044563293,
+      "learning_rate": 5.330452921628497e-05,
+      "loss": 0.4352,
+      "step": 101
+    },
+    {
+      "epoch": 0.1301850670070198,
+      "grad_norm": 0.35046932101249695,
+      "learning_rate": 5.247918773366112e-05,
+      "loss": 0.5297,
+      "step": 102
+    },
+    {
+      "epoch": 0.13146139119336311,
+      "grad_norm": 0.3474259376525879,
+      "learning_rate": 5.165316846586541e-05,
+      "loss": 0.5379,
+      "step": 103
+    },
+    {
+      "epoch": 0.13273771537970644,
+      "grad_norm": 0.36887326836586,
+      "learning_rate": 5.0826697238317935e-05,
+      "loss": 0.548,
+      "step": 104
+    },
+    {
+      "epoch": 0.13401403956604976,
+      "grad_norm": 0.38822340965270996,
+      "learning_rate": 5e-05,
+      "loss": 0.5126,
+      "step": 105
+    },
+    {
+      "epoch": 0.13529036375239312,
+      "grad_norm": 0.3616776764392853,
+      "learning_rate": 4.917330276168208e-05,
+      "loss": 0.5528,
+      "step": 106
+    },
+    {
+      "epoch": 0.13656668793873644,
+      "grad_norm": 0.3938593566417694,
+      "learning_rate": 4.834683153413459e-05,
+      "loss": 0.4775,
+      "step": 107
+    },
+    {
+      "epoch": 0.13784301212507977,
+      "grad_norm": 0.3966670632362366,
+      "learning_rate": 4.7520812266338885e-05,
+      "loss": 0.5506,
+      "step": 108
+    },
+    {
+      "epoch": 0.1391193363114231,
+      "grad_norm": 0.4023497998714447,
+      "learning_rate": 4.669547078371504e-05,
+      "loss": 0.502,
+      "step": 109
+    },
+    {
+      "epoch": 0.14039566049776644,
+      "grad_norm": 0.39788365364074707,
+      "learning_rate": 4.5871032726383386e-05,
+      "loss": 0.4577,
+      "step": 110
+    },
+    {
+      "epoch": 0.14167198468410977,
+      "grad_norm": 0.3886997699737549,
+      "learning_rate": 4.504772348747687e-05,
+      "loss": 0.4873,
+      "step": 111
+    },
+    {
+      "epoch": 0.1429483088704531,
+      "grad_norm": 0.39564603567123413,
+      "learning_rate": 4.4225768151520694e-05,
+      "loss": 0.4624,
+      "step": 112
+    },
+    {
+      "epoch": 0.14422463305679642,
+      "grad_norm": 0.38617604970932007,
+      "learning_rate": 4.3405391432896555e-05,
+      "loss": 0.4683,
+      "step": 113
+    },
+    {
+      "epoch": 0.14550095724313974,
+      "grad_norm": 0.4043361246585846,
+      "learning_rate": 4.2586817614407895e-05,
+      "loss": 0.5375,
+      "step": 114
+    },
+    {
+      "epoch": 0.1467772814294831,
+      "grad_norm": 0.4169708788394928,
+      "learning_rate": 4.17702704859633e-05,
+      "loss": 0.5361,
+      "step": 115
+    },
+    {
+      "epoch": 0.14805360561582642,
+      "grad_norm": 0.43478041887283325,
+      "learning_rate": 4.095597328339452e-05,
+      "loss": 0.5032,
+      "step": 116
+    },
+    {
+      "epoch": 0.14932992980216975,
+      "grad_norm": 0.41572943329811096,
+      "learning_rate": 4.0144148627425993e-05,
+      "loss": 0.495,
+      "step": 117
+    },
+    {
+      "epoch": 0.15060625398851307,
+      "grad_norm": 0.409906268119812,
+      "learning_rate": 3.933501846281267e-05,
+      "loss": 0.4699,
+      "step": 118
+    },
+    {
+      "epoch": 0.15188257817485643,
+      "grad_norm": 0.43988335132598877,
+      "learning_rate": 3.852880399766243e-05,
+      "loss": 0.5225,
+      "step": 119
+    },
+    {
+      "epoch": 0.15315890236119975,
+      "grad_norm": 0.4484773874282837,
+      "learning_rate": 3.772572564296005e-05,
+      "loss": 0.4719,
+      "step": 120
+    },
+    {
+      "epoch": 0.15443522654754308,
+      "grad_norm": 0.4349138140678406,
+      "learning_rate": 3.6926002952309016e-05,
+      "loss": 0.4777,
+      "step": 121
+    },
+    {
+      "epoch": 0.1557115507338864,
+      "grad_norm": 0.4240596294403076,
+      "learning_rate": 3.612985456190778e-05,
+      "loss": 0.4537,
+      "step": 122
+    },
+    {
+      "epoch": 0.15698787492022973,
+      "grad_norm": 0.42663252353668213,
+      "learning_rate": 3.533749813077677e-05,
+      "loss": 0.5012,
+      "step": 123
+    },
+    {
+      "epoch": 0.15826419910657308,
+      "grad_norm": 0.43581223487854004,
+      "learning_rate": 3.4549150281252636e-05,
+      "loss": 0.4523,
+      "step": 124
+    },
+    {
+      "epoch": 0.1595405232929164,
+      "grad_norm": 0.4443126618862152,
+      "learning_rate": 3.3765026539765834e-05,
+      "loss": 0.5059,
+      "step": 125
+    },
+    {
+      "epoch": 0.16081684747925973,
+      "grad_norm": 0.4689054787158966,
+      "learning_rate": 3.298534127791785e-05,
+      "loss": 0.5195,
+      "step": 126
+    },
+    {
+      "epoch": 0.16209317166560305,
+      "grad_norm": 0.4573354125022888,
+      "learning_rate": 3.221030765387417e-05,
+      "loss": 0.4797,
+      "step": 127
+    },
+    {
+      "epoch": 0.1633694958519464,
+      "grad_norm": 0.4319009482860565,
+      "learning_rate": 3.144013755408895e-05,
+      "loss": 0.4067,
+      "step": 128
+    },
+    {
+      "epoch": 0.16464582003828973,
+      "grad_norm": 0.48260587453842163,
+      "learning_rate": 3.0675041535377405e-05,
+      "loss": 0.436,
+      "step": 129
+    },
+    {
+      "epoch": 0.16592214422463306,
+      "grad_norm": 0.4620423913002014,
+      "learning_rate": 2.991522876735154e-05,
+      "loss": 0.4464,
+      "step": 130
+    },
+    {
+      "epoch": 0.16719846841097638,
+      "grad_norm": 0.4515093266963959,
+      "learning_rate": 2.916090697523549e-05,
+      "loss": 0.4042,
+      "step": 131
+    },
+    {
+      "epoch": 0.1684747925973197,
+      "grad_norm": 0.4574383497238159,
+      "learning_rate": 2.8412282383075363e-05,
+      "loss": 0.4391,
+      "step": 132
+    },
+    {
+      "epoch": 0.16975111678366306,
+      "grad_norm": 0.5026492476463318,
+      "learning_rate": 2.766955965735968e-05,
+      "loss": 0.4646,
+      "step": 133
+    },
+    {
+      "epoch": 0.17102744097000638,
+      "grad_norm": 0.5044828057289124,
+      "learning_rate": 2.693294185106562e-05,
+      "loss": 0.4666,
+      "step": 134
+    },
+    {
+      "epoch": 0.1723037651563497,
+      "grad_norm": 0.4921793043613434,
+      "learning_rate": 2.6202630348146324e-05,
+      "loss": 0.4727,
+      "step": 135
+    },
+    {
+      "epoch": 0.17358008934269303,
+      "grad_norm": 0.49983012676239014,
+      "learning_rate": 2.547882480847461e-05,
+      "loss": 0.4222,
+      "step": 136
+    },
+    {
+      "epoch": 0.1748564135290364,
+      "grad_norm": 0.5211167931556702,
+      "learning_rate": 2.476172311325783e-05,
+      "loss": 0.4483,
+      "step": 137
+    },
+    {
+      "epoch": 0.1761327377153797,
+      "grad_norm": 0.5635063648223877,
+      "learning_rate": 2.405152131093926e-05,
+      "loss": 0.4456,
+      "step": 138
+    },
+    {
+      "epoch": 0.17740906190172304,
+      "grad_norm": 0.5445062518119812,
+      "learning_rate": 2.3348413563600325e-05,
+      "loss": 0.4423,
+      "step": 139
+    },
+    {
+      "epoch": 0.17868538608806636,
+      "grad_norm": 0.5959829688072205,
+      "learning_rate": 2.2652592093878666e-05,
+      "loss": 0.4147,
+      "step": 140
+    },
+    {
+      "epoch": 0.1799617102744097,
+      "grad_norm": 0.567952036857605,
+      "learning_rate": 2.196424713241637e-05,
+      "loss": 0.3891,
+      "step": 141
+    },
+    {
+      "epoch": 0.18123803446075304,
+      "grad_norm": 0.5882492661476135,
+      "learning_rate": 2.128356686585282e-05,
+      "loss": 0.4155,
+      "step": 142
+    },
+    {
+      "epoch": 0.18251435864709636,
+      "grad_norm": 0.6992138624191284,
+      "learning_rate": 2.061073738537635e-05,
+      "loss": 0.4634,
+      "step": 143
+    },
+    {
+      "epoch": 0.1837906828334397,
+      "grad_norm": 0.7090771198272705,
+      "learning_rate": 1.9945942635848748e-05,
+      "loss": 0.4381,
+      "step": 144
+    },
+    {
+      "epoch": 0.18506700701978301,
+      "grad_norm": 0.7691441178321838,
+      "learning_rate": 1.928936436551661e-05,
+      "loss": 0.4575,
+      "step": 145
+    },
+    {
+      "epoch": 0.18634333120612637,
+      "grad_norm": 0.9078056812286377,
+      "learning_rate": 1.8641182076323148e-05,
+      "loss": 0.6049,
+      "step": 146
+    },
+    {
+      "epoch": 0.1876196553924697,
+      "grad_norm": 1.0223463773727417,
+      "learning_rate": 1.800157297483417e-05,
+      "loss": 0.6327,
+      "step": 147
+    },
+    {
+      "epoch": 0.18889597957881302,
+      "grad_norm": 1.2614026069641113,
+      "learning_rate": 1.7370711923791567e-05,
+      "loss": 0.6936,
+      "step": 148
+    },
+    {
+      "epoch": 0.19017230376515634,
+      "grad_norm": 1.3089138269424438,
+      "learning_rate": 1.6748771394307585e-05,
+      "loss": 0.7366,
+      "step": 149
+    },
+    {
+      "epoch": 0.1914486279514997,
+      "grad_norm": 2.0128531455993652,
+      "learning_rate": 1.6135921418712956e-05,
+      "loss": 0.718,
+      "step": 150
+    },
+    {
+      "epoch": 0.1914486279514997,
+      "eval_loss": 0.4940585196018219,
+      "eval_runtime": 22.9839,
+      "eval_samples_per_second": 57.431,
+      "eval_steps_per_second": 14.358,
+      "step": 150
+    },
+    {
+      "epoch": 0.19272495213784302,
+      "grad_norm": 0.3091920018196106,
+      "learning_rate": 1.553232954407171e-05,
+      "loss": 0.4487,
+      "step": 151
+    },
+    {
+      "epoch": 0.19400127632418634,
+      "grad_norm": 0.3527967929840088,
+      "learning_rate": 1.4938160786375572e-05,
+      "loss": 0.5823,
+      "step": 152
+    },
+    {
+      "epoch": 0.19527760051052967,
+      "grad_norm": 0.3343672752380371,
+      "learning_rate": 1.435357758543015e-05,
+      "loss": 0.4653,
+      "step": 153
+    },
+    {
+      "epoch": 0.196553924696873,
+      "grad_norm": 0.335879921913147,
+      "learning_rate": 1.3778739760445552e-05,
+      "loss": 0.5055,
+      "step": 154
+    },
+    {
+      "epoch": 0.19783024888321635,
+      "grad_norm": 0.3466244637966156,
+      "learning_rate": 1.3213804466343421e-05,
+      "loss": 0.4781,
+      "step": 155
+    },
+    {
+      "epoch": 0.19910657306955967,
+      "grad_norm": 0.36654630303382874,
+      "learning_rate": 1.2658926150792322e-05,
+      "loss": 0.4866,
+      "step": 156
+    },
+    {
+      "epoch": 0.200382897255903,
+      "grad_norm": 0.3677193224430084,
+      "learning_rate": 1.2114256511983274e-05,
+      "loss": 0.5397,
+      "step": 157
+    },
+    {
+      "epoch": 0.20165922144224632,
+      "grad_norm": 0.3839276134967804,
+      "learning_rate": 1.157994445715706e-05,
+      "loss": 0.5531,
+      "step": 158
+    },
+    {
+      "epoch": 0.20293554562858968,
+      "grad_norm": 0.3790455460548401,
+      "learning_rate": 1.1056136061894384e-05,
+      "loss": 0.446,
+      "step": 159
+    },
+    {
+      "epoch": 0.204211869814933,
+      "grad_norm": 0.3889751732349396,
+      "learning_rate": 1.0542974530180327e-05,
+      "loss": 0.4816,
+      "step": 160
+    },
+    {
+      "epoch": 0.20548819400127633,
+      "grad_norm": 0.38132384419441223,
+      "learning_rate": 1.0040600155253765e-05,
+      "loss": 0.4639,
+      "step": 161
+    },
+    {
+      "epoch": 0.20676451818761965,
+      "grad_norm": 0.4053707420825958,
+      "learning_rate": 9.549150281252633e-06,
+      "loss": 0.502,
+      "step": 162
+    },
+    {
+      "epoch": 0.20804084237396298,
+      "grad_norm": 0.393968790769577,
+      "learning_rate": 9.068759265665384e-06,
+      "loss": 0.4814,
+      "step": 163
+    },
+    {
+      "epoch": 0.20931716656030633,
+      "grad_norm": 0.3983076512813568,
+      "learning_rate": 8.599558442598998e-06,
+      "loss": 0.4585,
+      "step": 164
+    },
+    {
+      "epoch": 0.21059349074664965,
+      "grad_norm": 0.3859368562698364,
+      "learning_rate": 8.141676086873572e-06,
+      "loss": 0.4193,
+      "step": 165
+    },
+    {
+      "epoch": 0.21186981493299298,
+      "grad_norm": 0.400826096534729,
+      "learning_rate": 7.695237378953223e-06,
+      "loss": 0.4521,
+      "step": 166
+    },
+    {
+      "epoch": 0.2131461391193363,
+      "grad_norm": 0.4179336726665497,
+      "learning_rate": 7.260364370723044e-06,
+      "loss": 0.4817,
+      "step": 167
+    },
+    {
+      "epoch": 0.21442246330567966,
+      "grad_norm": 0.3994908332824707,
+      "learning_rate": 6.837175952121306e-06,
+      "loss": 0.4296,
+      "step": 168
+    },
+    {
+      "epoch": 0.21569878749202298,
+      "grad_norm": 0.41591042280197144,
+      "learning_rate": 6.425787818636131e-06,
+      "loss": 0.5005,
+      "step": 169
+    },
+    {
+      "epoch": 0.2169751116783663,
+      "grad_norm": 0.40266382694244385,
+      "learning_rate": 6.026312439675552e-06,
+      "loss": 0.4779,
+      "step": 170
+    },
+    {
+      "epoch": 0.21825143586470963,
+      "grad_norm": 0.42550334334373474,
+      "learning_rate": 5.6388590278194096e-06,
+      "loss": 0.4363,
+      "step": 171
+    },
+    {
+      "epoch": 0.21952776005105296,
+      "grad_norm": 0.41087865829467773,
+      "learning_rate": 5.263533508961827e-06,
+      "loss": 0.4108,
+      "step": 172
+    },
+    {
+      "epoch": 0.2208040842373963,
+      "grad_norm": 0.4212132394313812,
+      "learning_rate": 4.900438493352055e-06,
+      "loss": 0.4745,
+      "step": 173
+    },
+    {
+      "epoch": 0.22208040842373963,
+      "grad_norm": 0.44137516617774963,
+      "learning_rate": 4.549673247541875e-06,
+      "loss": 0.433,
+      "step": 174
+    },
+    {
+      "epoch": 0.22335673261008296,
+      "grad_norm": 0.4842296242713928,
+      "learning_rate": 4.2113336672471245e-06,
+      "loss": 0.552,
+      "step": 175
+    },
+    {
+      "epoch": 0.22463305679642628,
+      "grad_norm": 0.4331275522708893,
+      "learning_rate": 3.885512251130763e-06,
+      "loss": 0.4176,
+      "step": 176
+    },
+    {
+      "epoch": 0.22590938098276964,
+      "grad_norm": 0.4455384910106659,
+      "learning_rate": 3.5722980755146517e-06,
+      "loss": 0.4351,
+      "step": 177
+    },
+    {
+      "epoch": 0.22718570516911296,
+      "grad_norm": 0.444436639547348,
+      "learning_rate": 3.271776770026963e-06,
+      "loss": 0.4778,
+      "step": 178
+    },
+    {
+      "epoch": 0.2284620293554563,
+      "grad_norm": 0.46339622139930725,
+      "learning_rate": 2.9840304941919415e-06,
+      "loss": 0.4955,
+      "step": 179
+    },
+    {
+      "epoch": 0.2297383535417996,
+      "grad_norm": 0.4468136429786682,
+      "learning_rate": 2.7091379149682685e-06,
+      "loss": 0.4556,
+      "step": 180
+    },
+    {
+      "epoch": 0.23101467772814294,
+      "grad_norm": 0.43544065952301025,
+      "learning_rate": 2.4471741852423237e-06,
+      "loss": 0.4106,
+      "step": 181
+    },
+    {
+      "epoch": 0.2322910019144863,
+      "grad_norm": 0.4690147638320923,
+      "learning_rate": 2.1982109232821178e-06,
+      "loss": 0.4371,
+      "step": 182
+    },
+    {
+      "epoch": 0.23356732610082961,
+      "grad_norm": 0.4728126525878906,
+      "learning_rate": 1.962316193157593e-06,
+      "loss": 0.4733,
+      "step": 183
+    },
+    {
+      "epoch": 0.23484365028717294,
+      "grad_norm": 0.4679579734802246,
+      "learning_rate": 1.7395544861325718e-06,
+      "loss": 0.4264,
+      "step": 184
+    },
+    {
+      "epoch": 0.23611997447351626,
+      "grad_norm": 0.4710768163204193,
+      "learning_rate": 1.5299867030334814e-06,
+      "loss": 0.4202,
+      "step": 185
+    },
+    {
+      "epoch": 0.23739629865985962,
+      "grad_norm": 0.48057445883750916,
+      "learning_rate": 1.333670137599713e-06,
+      "loss": 0.4006,
+      "step": 186
+    },
+    {
+      "epoch": 0.23867262284620294,
+      "grad_norm": 0.49291718006134033,
+      "learning_rate": 1.1506584608200367e-06,
+      "loss": 0.4297,
+      "step": 187
+    },
+    {
+      "epoch": 0.23994894703254627,
+      "grad_norm": 0.5248841643333435,
+      "learning_rate": 9.810017062595322e-07,
+      "loss": 0.5074,
+      "step": 188
+    },
+    {
+      "epoch": 0.2412252712188896,
+      "grad_norm": 0.5448639988899231,
+      "learning_rate": 8.247462563808817e-07,
+      "loss": 0.4247,
+      "step": 189
+    },
+    {
+      "epoch": 0.24250159540523292,
+      "grad_norm": 0.528141975402832,
+      "learning_rate": 6.819348298638839e-07,
+      "loss": 0.431,
+      "step": 190
+    },
+    {
+      "epoch": 0.24377791959157627,
+      "grad_norm": 0.5867920517921448,
+      "learning_rate": 5.526064699265753e-07,
+      "loss": 0.4193,
+      "step": 191
+    },
+    {
+      "epoch": 0.2450542437779196,
+      "grad_norm": 0.5877797603607178,
+      "learning_rate": 4.367965336512403e-07,
+      "loss": 0.3454,
+      "step": 192
+    },
+    {
+      "epoch": 0.24633056796426292,
+      "grad_norm": 0.6367771029472351,
+      "learning_rate": 3.3453668231809286e-07,
+      "loss": 0.4904,
+      "step": 193
+    },
+    {
+      "epoch": 0.24760689215060624,
+      "grad_norm": 0.796617865562439,
+      "learning_rate": 2.458548727494292e-07,
+      "loss": 0.4901,
+      "step": 194
+    },
+    {
+      "epoch": 0.2488832163369496,
+      "grad_norm": 0.9110091328620911,
+      "learning_rate": 1.7077534966650766e-07,
+      "loss": 0.6218,
+      "step": 195
+    },
+    {
+      "epoch": 0.2501595405232929,
+      "grad_norm": 1.0279269218444824,
+      "learning_rate": 1.0931863906127327e-07,
+      "loss": 0.5675,
+      "step": 196
+    },
+    {
+      "epoch": 0.25143586470963625,
+      "grad_norm": 1.1644656658172607,
+      "learning_rate": 6.150154258476315e-08,
+      "loss": 0.695,
+      "step": 197
+    },
+    {
+      "epoch": 0.2527121888959796,
+      "grad_norm": 1.1470328569412231,
+      "learning_rate": 2.7337132953697554e-08,
+      "loss": 0.5349,
+      "step": 198
+    },
+    {
+      "epoch": 0.2539885130823229,
+      "grad_norm": 1.6212947368621826,
+      "learning_rate": 6.834750376549792e-09,
+      "loss": 0.7102,
+      "step": 199
+    },
+    {
+      "epoch": 0.2552648372686662,
+      "grad_norm": 2.6510043144226074,
+      "learning_rate": 0.0,
+      "loss": 0.9528,
+      "step": 200
+    },
+    {
+      "epoch": 0.2552648372686662,
+      "eval_loss": 0.4917088747024536,
+      "eval_runtime": 22.9973,
+      "eval_samples_per_second": 57.398,
+      "eval_steps_per_second": 14.35,
+      "step": 200
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 200,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 5,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.391684260724736e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2b306aef7ff0da3ba7d6800372cc91d809e0c278dab6aab206d8bf3cadcaceb
+size 6840

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcfe42da0a4497e8b2b172c1f9f4ec423a46dc12907f4349c55025f670422ba9
+size 11418266

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2b306aef7ff0da3ba7d6800372cc91d809e0c278dab6aab206d8bf3cadcaceb
+size 6840