Model save

README.md

@@ -0,0 +1,76 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-dpo-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-dpo-full
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5023
+- Rewards/chosen: -1.0875
+- Rewards/rejected: -2.0687
+- Rewards/accuracies: 0.7695
+- Rewards/margins: 0.9812
+- Logps/rejected: -469.5586
+- Logps/chosen: -371.3432
+- Logits/rejected: -0.3666
+- Logits/chosen: -1.1044
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.5767        | 0.21  | 100  | 0.5762          | -0.5418        | -1.0929          | 0.7148             | 0.5512          | -371.9847      | -316.7739    | -1.7636         | -1.7624       |
+| 0.5185        | 0.42  | 200  | 0.5312          | -1.0541        | -1.7947          | 0.7656             | 0.7407          | -442.1638      | -368.0010    | -0.6928         | -1.0192       |
+| 0.5166        | 0.63  | 300  | 0.5114          | -1.1529        | -2.0654          | 0.7539             | 0.9125          | -469.2292      | -377.8804    | -0.6568         | -1.2977       |
+| 0.5003        | 0.84  | 400  | 0.5023          | -1.0875        | -2.0687          | 0.7695             | 0.9812          | -469.5586      | -371.3432    | -0.3666         | -1.1044       |
+
+
+### Framework versions
+
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.2
+- Datasets 2.18.1.dev0
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5374924991918907,
+    "train_runtime": 7413.7576,
+    "train_samples": 61134,
+    "train_samples_per_second": 8.246,
+    "train_steps_per_second": 0.064
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4daa4e80bef7d5469132e4f1aa1f40f3977a1d74be33e7646c039f79530a8f51
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee6c858bdc80db43b9f432ea41c97766ff1dcdf01daaef5fc87a92186e4135c2
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5434ad99ac49d9ccc2d6b02956ab83b649e6c16514cbd59079ab14ac6b1007e0
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5374924991918907,
+    "train_runtime": 7413.7576,
+    "train_samples": 61134,
+    "train_samples_per_second": 8.246,
+    "train_steps_per_second": 0.064
+}

trainer_state.json

@@ -0,0 +1,814 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 478,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 8.988967747415137,
+      "learning_rate": 1.0416666666666666e-08,
+      "logits/chosen": -2.7205495834350586,
+      "logits/rejected": -2.556058883666992,
+      "logps/chosen": -245.96978759765625,
+      "logps/rejected": -198.16636657714844,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 8.972775503730468,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.706871271133423,
+      "logits/rejected": -2.728226900100708,
+      "logps/chosen": -299.9327087402344,
+      "logps/rejected": -282.5617370605469,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": -0.0002042697014985606,
+      "rewards/margins": -0.00041398685425519943,
+      "rewards/rejected": 0.0002097171382047236,
+      "step": 10
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 8.863587069504122,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.7274839878082275,
+      "logits/rejected": -2.6729283332824707,
+      "logps/chosen": -302.313232421875,
+      "logps/rejected": -296.7969665527344,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0005617194110527635,
+      "rewards/margins": 0.001121552544645965,
+      "rewards/rejected": -0.0005598333082161844,
+      "step": 20
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 8.692790204491699,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.668670415878296,
+      "logits/rejected": -2.6144020557403564,
+      "logps/chosen": -250.7909393310547,
+      "logps/rejected": -229.73324584960938,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.00824995618313551,
+      "rewards/margins": 0.010436030104756355,
+      "rewards/rejected": -0.0021860746201127768,
+      "step": 30
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 8.064484646395186,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.6780455112457275,
+      "logits/rejected": -2.65267276763916,
+      "logps/chosen": -300.8070983886719,
+      "logps/rejected": -262.91119384765625,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.040893860161304474,
+      "rewards/margins": 0.03940137103199959,
+      "rewards/rejected": 0.0014924891293048859,
+      "step": 40
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 10.103335337067259,
+      "learning_rate": 4.999733114418725e-07,
+      "logits/chosen": -2.5915541648864746,
+      "logits/rejected": -2.560572624206543,
+      "logps/chosen": -279.72900390625,
+      "logps/rejected": -273.87506103515625,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.039791837334632874,
+      "rewards/margins": 0.09251143783330917,
+      "rewards/rejected": -0.052719611674547195,
+      "step": 50
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 11.50466311253595,
+      "learning_rate": 4.990398100856366e-07,
+      "logits/chosen": -2.5843663215637207,
+      "logits/rejected": -2.552401304244995,
+      "logps/chosen": -267.9306335449219,
+      "logps/rejected": -254.7063751220703,
+      "loss": 0.6306,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.029593652114272118,
+      "rewards/margins": 0.16877658665180206,
+      "rewards/rejected": -0.19837023317813873,
+      "step": 60
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 11.669269626157586,
+      "learning_rate": 4.967775735898179e-07,
+      "logits/chosen": -2.5480875968933105,
+      "logits/rejected": -2.4989640712738037,
+      "logps/chosen": -279.6777038574219,
+      "logps/rejected": -264.2624206542969,
+      "loss": 0.6215,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.05579886958003044,
+      "rewards/margins": 0.20252302289009094,
+      "rewards/rejected": -0.2583218514919281,
+      "step": 70
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 14.944916591668084,
+      "learning_rate": 4.931986719649298e-07,
+      "logits/chosen": -2.443418025970459,
+      "logits/rejected": -2.3847784996032715,
+      "logps/chosen": -280.1128845214844,
+      "logps/rejected": -303.17889404296875,
+      "loss": 0.5961,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.059926122426986694,
+      "rewards/margins": 0.2583328187465668,
+      "rewards/rejected": -0.31825894117355347,
+      "step": 80
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 14.497805589416837,
+      "learning_rate": 4.883222001996351e-07,
+      "logits/chosen": -2.4620890617370605,
+      "logits/rejected": -2.39471173286438,
+      "logps/chosen": -307.2780456542969,
+      "logps/rejected": -313.0209045410156,
+      "loss": 0.5964,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.45970097184181213,
+      "rewards/margins": 0.2758905589580536,
+      "rewards/rejected": -0.7355915307998657,
+      "step": 90
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 24.280462666043302,
+      "learning_rate": 4.821741763807186e-07,
+      "logits/chosen": -2.2708218097686768,
+      "logits/rejected": -2.1873369216918945,
+      "logps/chosen": -369.2454833984375,
+      "logps/rejected": -385.3036804199219,
+      "loss": 0.5767,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.7413482666015625,
+      "rewards/margins": 0.38414236903190613,
+      "rewards/rejected": -1.125490665435791,
+      "step": 100
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": -1.7623814344406128,
+      "eval_logits/rejected": -1.7635838985443115,
+      "eval_logps/chosen": -316.7738952636719,
+      "eval_logps/rejected": -371.9847412109375,
+      "eval_loss": 0.5761595964431763,
+      "eval_rewards/accuracies": 0.71484375,
+      "eval_rewards/chosen": -0.5417935848236084,
+      "eval_rewards/margins": 0.5511539578437805,
+      "eval_rewards/rejected": -1.0929476022720337,
+      "eval_runtime": 91.9636,
+      "eval_samples_per_second": 21.748,
+      "eval_steps_per_second": 0.348,
+      "step": 100
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 21.276395436310406,
+      "learning_rate": 4.747874028753375e-07,
+      "logits/chosen": -1.6442521810531616,
+      "logits/rejected": -1.3894662857055664,
+      "logps/chosen": -327.5511169433594,
+      "logps/rejected": -338.51129150390625,
+      "loss": 0.5675,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.5326968431472778,
+      "rewards/margins": 0.6152414083480835,
+      "rewards/rejected": -1.1479383707046509,
+      "step": 110
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 31.303120852952276,
+      "learning_rate": 4.662012913161997e-07,
+      "logits/chosen": -1.3023754358291626,
+      "logits/rejected": -1.3187789916992188,
+      "logps/chosen": -339.4796142578125,
+      "logps/rejected": -399.40771484375,
+      "loss": 0.5346,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.6499887704849243,
+      "rewards/margins": 0.667434811592102,
+      "rewards/rejected": -1.3174235820770264,
+      "step": 120
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 27.028821109643975,
+      "learning_rate": 4.5646165232345103e-07,
+      "logits/chosen": -0.633685290813446,
+      "logits/rejected": -0.5499342679977417,
+      "logps/chosen": -389.3062438964844,
+      "logps/rejected": -421.11199951171875,
+      "loss": 0.5396,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.011283278465271,
+      "rewards/margins": 0.5636219382286072,
+      "rewards/rejected": -1.5749051570892334,
+      "step": 130
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 18.250619925903354,
+      "learning_rate": 4.456204510851956e-07,
+      "logits/chosen": -1.371741771697998,
+      "logits/rejected": -0.8369172215461731,
+      "logps/chosen": -359.0638122558594,
+      "logps/rejected": -422.20184326171875,
+      "loss": 0.5313,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7278968095779419,
+      "rewards/margins": 0.7612239718437195,
+      "rewards/rejected": -1.4891207218170166,
+      "step": 140
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 28.832530099775454,
+      "learning_rate": 4.337355301007335e-07,
+      "logits/chosen": -1.2246290445327759,
+      "logits/rejected": -0.8883364796638489,
+      "logps/chosen": -313.1046447753906,
+      "logps/rejected": -389.2489013671875,
+      "loss": 0.525,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.7995457053184509,
+      "rewards/margins": 0.6876853704452515,
+      "rewards/rejected": -1.4872313737869263,
+      "step": 150
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 26.24455420439153,
+      "learning_rate": 4.2087030056579986e-07,
+      "logits/chosen": -0.6042385101318359,
+      "logits/rejected": -0.29473963379859924,
+      "logps/chosen": -385.8511047363281,
+      "logps/rejected": -433.1654357910156,
+      "loss": 0.5306,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.2439239025115967,
+      "rewards/margins": 0.5850938558578491,
+      "rewards/rejected": -1.8290178775787354,
+      "step": 160
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 21.466673908999535,
+      "learning_rate": 4.070934040463998e-07,
+      "logits/chosen": -1.0178248882293701,
+      "logits/rejected": -0.41918307542800903,
+      "logps/chosen": -386.2198181152344,
+      "logps/rejected": -461.2767028808594,
+      "loss": 0.5222,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.9928082227706909,
+      "rewards/margins": 0.9411848783493042,
+      "rewards/rejected": -1.9339931011199951,
+      "step": 170
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 24.51504275177038,
+      "learning_rate": 3.9247834624635404e-07,
+      "logits/chosen": -1.2531163692474365,
+      "logits/rejected": -0.820971667766571,
+      "logps/chosen": -364.41583251953125,
+      "logps/rejected": -421.32159423828125,
+      "loss": 0.5303,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8646942973136902,
+      "rewards/margins": 0.7480314373970032,
+      "rewards/rejected": -1.6127256155014038,
+      "step": 180
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 28.270571178832476,
+      "learning_rate": 3.7710310482256523e-07,
+      "logits/chosen": -0.3600873351097107,
+      "logits/rejected": 0.8002864122390747,
+      "logps/chosen": -387.2593994140625,
+      "logps/rejected": -469.677978515625,
+      "loss": 0.5083,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.9201375842094421,
+      "rewards/margins": 1.1224908828735352,
+      "rewards/rejected": -2.042628526687622,
+      "step": 190
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 19.44371035878493,
+      "learning_rate": 3.610497133404795e-07,
+      "logits/chosen": -0.532671332359314,
+      "logits/rejected": -0.07579895108938217,
+      "logps/chosen": -405.0849304199219,
+      "logps/rejected": -445.6201171875,
+      "loss": 0.5185,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0358976125717163,
+      "rewards/margins": 0.725575864315033,
+      "rewards/rejected": -1.7614734172821045,
+      "step": 200
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": -1.0192277431488037,
+      "eval_logits/rejected": -0.6928036212921143,
+      "eval_logps/chosen": -368.00103759765625,
+      "eval_logps/rejected": -442.16375732421875,
+      "eval_loss": 0.5312252044677734,
+      "eval_rewards/accuracies": 0.765625,
+      "eval_rewards/chosen": -1.0540651082992554,
+      "eval_rewards/margins": 0.7406728863716125,
+      "eval_rewards/rejected": -1.7947381734848022,
+      "eval_runtime": 92.0234,
+      "eval_samples_per_second": 21.734,
+      "eval_steps_per_second": 0.348,
+      "step": 200
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 24.268514536481177,
+      "learning_rate": 3.4440382358952115e-07,
+      "logits/chosen": -0.8844076991081238,
+      "logits/rejected": -0.5308712720870972,
+      "logps/chosen": -373.499755859375,
+      "logps/rejected": -436.2884216308594,
+      "loss": 0.5033,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.951114296913147,
+      "rewards/margins": 0.8899469375610352,
+      "rewards/rejected": -1.8410612344741821,
+      "step": 210
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 22.946027324747533,
+      "learning_rate": 3.272542485937368e-07,
+      "logits/chosen": -0.5064371824264526,
+      "logits/rejected": 0.23283937573432922,
+      "logps/chosen": -383.8258972167969,
+      "logps/rejected": -451.389892578125,
+      "loss": 0.5141,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0778086185455322,
+      "rewards/margins": 0.8345571756362915,
+      "rewards/rejected": -1.9123656749725342,
+      "step": 220
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 26.66216276685979,
+      "learning_rate": 3.096924887558854e-07,
+      "logits/chosen": -0.37716662883758545,
+      "logits/rejected": 0.36690014600753784,
+      "logps/chosen": -421.6444396972656,
+      "logps/rejected": -493.63543701171875,
+      "loss": 0.5256,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.408128261566162,
+      "rewards/margins": 0.7821764945983887,
+      "rewards/rejected": -2.19030499458313,
+      "step": 230
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 20.689211336064318,
+      "learning_rate": 2.9181224366319943e-07,
+      "logits/chosen": -0.7800679206848145,
+      "logits/rejected": 0.013299018144607544,
+      "logps/chosen": -401.562744140625,
+      "logps/rejected": -449.37384033203125,
+      "loss": 0.4774,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1366697549819946,
+      "rewards/margins": 0.8018972277641296,
+      "rewards/rejected": -1.93856680393219,
+      "step": 240
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 30.629232248630764,
+      "learning_rate": 2.7370891215954565e-07,
+      "logits/chosen": -0.8394690752029419,
+      "logits/rejected": -0.07095638662576675,
+      "logps/chosen": -385.07147216796875,
+      "logps/rejected": -440.8536682128906,
+      "loss": 0.5339,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0770814418792725,
+      "rewards/margins": 0.7553308606147766,
+      "rewards/rejected": -1.8324124813079834,
+      "step": 250
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 22.359924477710283,
+      "learning_rate": 2.55479083351317e-07,
+      "logits/chosen": -0.6873140931129456,
+      "logits/rejected": -0.23746924102306366,
+      "logps/chosen": -373.2782287597656,
+      "logps/rejected": -458.29498291015625,
+      "loss": 0.4991,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0945470333099365,
+      "rewards/margins": 0.8836139440536499,
+      "rewards/rejected": -1.9781608581542969,
+      "step": 260
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 24.352377084503114,
+      "learning_rate": 2.3722002126275822e-07,
+      "logits/chosen": -1.4399245977401733,
+      "logits/rejected": -0.610245943069458,
+      "logps/chosen": -363.4355773925781,
+      "logps/rejected": -440.0086364746094,
+      "loss": 0.5162,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.7005236148834229,
+      "rewards/margins": 1.0190067291259766,
+      "rewards/rejected": -1.719530701637268,
+      "step": 270
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 27.016744918953567,
+      "learning_rate": 2.19029145890313e-07,
+      "logits/chosen": -1.5887335538864136,
+      "logits/rejected": -0.9962981343269348,
+      "logps/chosen": -386.28643798828125,
+      "logps/rejected": -435.70574951171875,
+      "loss": 0.5132,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.7019662857055664,
+      "rewards/margins": 0.9548131227493286,
+      "rewards/rejected": -1.6567795276641846,
+      "step": 280
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 26.13224420416667,
+      "learning_rate": 2.0100351342479216e-07,
+      "logits/chosen": -1.5274226665496826,
+      "logits/rejected": -0.6684475541114807,
+      "logps/chosen": -367.9737243652344,
+      "logps/rejected": -433.135986328125,
+      "loss": 0.518,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.8575841188430786,
+      "rewards/margins": 0.9145325422286987,
+      "rewards/rejected": -1.7721166610717773,
+      "step": 290
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 25.118365240619237,
+      "learning_rate": 1.8323929841460178e-07,
+      "logits/chosen": -1.3744531869888306,
+      "logits/rejected": -0.6543569564819336,
+      "logps/chosen": -374.47381591796875,
+      "logps/rejected": -437.312255859375,
+      "loss": 0.5166,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1720250844955444,
+      "rewards/margins": 0.730822741985321,
+      "rewards/rejected": -1.9028480052947998,
+      "step": 300
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -1.2976927757263184,
+      "eval_logits/rejected": -0.6567528247833252,
+      "eval_logps/chosen": -377.8804016113281,
+      "eval_logps/rejected": -469.2291564941406,
+      "eval_loss": 0.5114231705665588,
+      "eval_rewards/accuracies": 0.75390625,
+      "eval_rewards/chosen": -1.1528586149215698,
+      "eval_rewards/margins": 0.9125331044197083,
+      "eval_rewards/rejected": -2.065391778945923,
+      "eval_runtime": 91.9206,
+      "eval_samples_per_second": 21.758,
+      "eval_steps_per_second": 0.348,
+      "step": 300
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 28.40886458453979,
+      "learning_rate": 1.6583128063291573e-07,
+      "logits/chosen": -1.1402945518493652,
+      "logits/rejected": -0.41577059030532837,
+      "logps/chosen": -433.451416015625,
+      "logps/rejected": -484.820068359375,
+      "loss": 0.5033,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.3210188150405884,
+      "rewards/margins": 0.8473700284957886,
+      "rewards/rejected": -2.168388843536377,
+      "step": 310
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 28.30583390412626,
+      "learning_rate": 1.488723393865766e-07,
+      "logits/chosen": -0.8680181503295898,
+      "logits/rejected": -0.25706297159194946,
+      "logps/chosen": -432.978759765625,
+      "logps/rejected": -507.5543518066406,
+      "loss": 0.4662,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.384337067604065,
+      "rewards/margins": 0.9784086346626282,
+      "rewards/rejected": -2.362745761871338,
+      "step": 320
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 23.94955155668742,
+      "learning_rate": 1.3245295796480788e-07,
+      "logits/chosen": -1.0475196838378906,
+      "logits/rejected": -0.02384273335337639,
+      "logps/chosen": -420.4109802246094,
+      "logps/rejected": -500.77447509765625,
+      "loss": 0.5191,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.4072248935699463,
+      "rewards/margins": 0.9297706484794617,
+      "rewards/rejected": -2.3369956016540527,
+      "step": 330
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 26.88145170346802,
+      "learning_rate": 1.1666074087171627e-07,
+      "logits/chosen": -1.1260825395584106,
+      "logits/rejected": -0.4045921862125397,
+      "logps/chosen": -386.1277160644531,
+      "logps/rejected": -480.0000915527344,
+      "loss": 0.4851,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.2617071866989136,
+      "rewards/margins": 0.9609416723251343,
+      "rewards/rejected": -2.222649097442627,
+      "step": 340
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 32.127198200608866,
+      "learning_rate": 1.0157994641835734e-07,
+      "logits/chosen": -1.157327651977539,
+      "logits/rejected": -0.39939287304878235,
+      "logps/chosen": -410.75958251953125,
+      "logps/rejected": -485.366455078125,
+      "loss": 0.5184,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.3121212720870972,
+      "rewards/margins": 0.8859411478042603,
+      "rewards/rejected": -2.1980621814727783,
+      "step": 350
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 23.47201224640711,
+      "learning_rate": 8.729103716819111e-08,
+      "logits/chosen": -1.2730977535247803,
+      "logits/rejected": -0.21225647628307343,
+      "logps/chosen": -370.95758056640625,
+      "logps/rejected": -467.11773681640625,
+      "loss": 0.4815,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.087627649307251,
+      "rewards/margins": 1.1211528778076172,
+      "rewards/rejected": -2.2087807655334473,
+      "step": 360
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 25.572187464491854,
+      "learning_rate": 7.387025063449081e-08,
+      "logits/chosen": -1.204538345336914,
+      "logits/rejected": -0.6417055726051331,
+      "logps/chosen": -380.7297668457031,
+      "logps/rejected": -450.6810607910156,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9736254811286926,
+      "rewards/margins": 0.8355528712272644,
+      "rewards/rejected": -1.8091785907745361,
+      "step": 370
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 24.442713207132563,
+      "learning_rate": 6.138919252022435e-08,
+      "logits/chosen": -0.7307707667350769,
+      "logits/rejected": 0.02534063719213009,
+      "logps/chosen": -357.3538818359375,
+      "logps/rejected": -461.0918884277344,
+      "loss": 0.4757,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.3051798343658447,
+      "rewards/margins": 0.8298481702804565,
+      "rewards/rejected": -2.135028123855591,
+      "step": 380
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 31.55979798759149,
+      "learning_rate": 4.991445467064689e-08,
+      "logits/chosen": -0.8228236436843872,
+      "logits/rejected": 0.12945809960365295,
+      "logps/chosen": -374.20416259765625,
+      "logps/rejected": -449.0730895996094,
+      "loss": 0.5154,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.171090841293335,
+      "rewards/margins": 0.9261363744735718,
+      "rewards/rejected": -2.0972273349761963,
+      "step": 390
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 25.25735911338149,
+      "learning_rate": 3.9507259776993954e-08,
+      "logits/chosen": -0.7915847301483154,
+      "logits/rejected": 0.3253408670425415,
+      "logps/chosen": -416.76593017578125,
+      "logps/rejected": -465.4586486816406,
+      "loss": 0.5003,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2516233921051025,
+      "rewards/margins": 0.8954402208328247,
+      "rewards/rejected": -2.147063732147217,
+      "step": 400
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": -1.104366660118103,
+      "eval_logits/rejected": -0.3666176497936249,
+      "eval_logps/chosen": -371.3431701660156,
+      "eval_logps/rejected": -469.5586242675781,
+      "eval_loss": 0.5023010969161987,
+      "eval_rewards/accuracies": 0.76953125,
+      "eval_rewards/chosen": -1.0874860286712646,
+      "eval_rewards/margins": 0.9812002778053284,
+      "eval_rewards/rejected": -2.0686862468719482,
+      "eval_runtime": 91.8817,
+      "eval_samples_per_second": 21.767,
+      "eval_steps_per_second": 0.348,
+      "step": 400
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 32.88722785510891,
+      "learning_rate": 3.022313472693447e-08,
+      "logits/chosen": -1.1924324035644531,
+      "logits/rejected": -0.2936061918735504,
+      "logps/chosen": -390.41717529296875,
+      "logps/rejected": -483.9921875,
+      "loss": 0.505,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.2092020511627197,
+      "rewards/margins": 0.7791404128074646,
+      "rewards/rejected": -1.98834228515625,
+      "step": 410
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 25.99174775937039,
+      "learning_rate": 2.2111614344599684e-08,
+      "logits/chosen": -1.1985347270965576,
+      "logits/rejected": -0.4221798777580261,
+      "logps/chosen": -412.852294921875,
+      "logps/rejected": -503.3661193847656,
+      "loss": 0.4894,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.1841309070587158,
+      "rewards/margins": 0.8794403076171875,
+      "rewards/rejected": -2.0635714530944824,
+      "step": 420
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 27.355228620514268,
+      "learning_rate": 1.521597710086439e-08,
+      "logits/chosen": -0.8364534378051758,
+      "logits/rejected": -0.08403267711400986,
+      "logps/chosen": -412.7071228027344,
+      "logps/rejected": -494.9200744628906,
+      "loss": 0.5068,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.187975525856018,
+      "rewards/margins": 1.0082122087478638,
+      "rewards/rejected": -2.1961874961853027,
+      "step": 430
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 27.4807676908952,
+      "learning_rate": 9.57301420397924e-09,
+      "logits/chosen": -1.0082981586456299,
+      "logits/rejected": -0.2474936544895172,
+      "logps/chosen": -454.13555908203125,
+      "logps/rejected": -466.0726623535156,
+      "loss": 0.4931,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.326205849647522,
+      "rewards/margins": 0.6482640504837036,
+      "rewards/rejected": -1.9744701385498047,
+      "step": 440
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 28.832598416935205,
+      "learning_rate": 5.212833302556258e-09,
+      "logits/chosen": -1.285772681236267,
+      "logits/rejected": -0.31841930747032166,
+      "logps/chosen": -393.72039794921875,
+      "logps/rejected": -450.94482421875,
+      "loss": 0.4967,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1269813776016235,
+      "rewards/margins": 0.8901702761650085,
+      "rewards/rejected": -2.0171515941619873,
+      "step": 450
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 24.265990822993565,
+      "learning_rate": 2.158697848236607e-09,
+      "logits/chosen": -1.114770770072937,
+      "logits/rejected": -0.18640299141407013,
+      "logps/chosen": -425.88958740234375,
+      "logps/rejected": -475.578125,
+      "loss": 0.5203,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2242448329925537,
+      "rewards/margins": 0.7737809419631958,
+      "rewards/rejected": -1.99802565574646,
+      "step": 460
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 24.93617729504318,
+      "learning_rate": 4.269029751107489e-10,
+      "logits/chosen": -1.35390305519104,
+      "logits/rejected": -0.3422853946685791,
+      "logps/chosen": -402.8608703613281,
+      "logps/rejected": -424.72259521484375,
+      "loss": 0.4941,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0088200569152832,
+      "rewards/margins": 0.8776105046272278,
+      "rewards/rejected": -1.8864305019378662,
+      "step": 470
+    },
+    {
+      "epoch": 1.0,
+      "step": 478,
+      "total_flos": 0.0,
+      "train_loss": 0.5374924991918907,
+      "train_runtime": 7413.7576,
+      "train_samples_per_second": 8.246,
+      "train_steps_per_second": 0.064
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 478,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}