Model save

README.md

@@ -0,0 +1,74 @@
+---
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-dpo-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-dpo-full
+
+This model was trained from scratch on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5284
+- Rewards/chosen: -0.6175
+- Rewards/rejected: -1.4117
+- Rewards/accuracies: 0.7539
+- Rewards/margins: 0.7941
+- Logps/rejected: -405.0050
+- Logps/chosen: -325.5808
+- Logits/rejected: 0.5323
+- Logits/chosen: 0.2839
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.5992        | 0.2092 | 100  | 0.5956          | -0.2932        | -0.6564          | 0.7148             | 0.3632          | -329.4821      | -293.1491    | -2.2402         | -2.2843       |
+| 0.57          | 0.4184 | 200  | 0.5591          | -0.3908        | -0.9608          | 0.7422             | 0.5700          | -359.9165      | -302.9073    | -1.6390         | -1.7197       |
+| 0.5222        | 0.6276 | 300  | 0.5473          | -0.4814        | -1.1717          | 0.7461             | 0.6902          | -381.0072      | -311.9707    | -1.3133         | -1.4138       |
+| 0.5332        | 0.8368 | 400  | 0.5284          | -0.6175        | -1.4117          | 0.7539             | 0.7941          | -405.0050      | -325.5808    | 0.5323          | 0.2839        |
+
+
+### Framework versions
+
+- Transformers 4.40.2
+- Pytorch 2.1.0a0+32f93b1
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.5645424491690791,
+    "train_runtime": 10520.2198,
+    "train_samples": 61134,
+    "train_samples_per_second": 5.811,
+    "train_steps_per_second": 0.045
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.40.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b076187b90081ad1c6a4f915a911c338cc541a48d5fa9cb7f3687fbbc7737f2
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b7a9fe43e8871b043a4c16072317cda846eb79ae0f11558ea84fbafd67a12c0
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:053d35552ab20765b56f60cc3a6d666c06ae39b537eac7523d5120e8d4a9a494
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.5645424491690791,
+    "train_runtime": 10520.2198,
+    "train_samples": 61134,
+    "train_samples_per_second": 5.811,
+    "train_steps_per_second": 0.045
+}

trainer_state.json

@@ -0,0 +1,814 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 478,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0020920502092050207,
+      "grad_norm": 9.060302588570952,
+      "learning_rate": 1.0416666666666666e-08,
+      "logits/chosen": -2.564940929412842,
+      "logits/rejected": -2.551347255706787,
+      "logps/chosen": -272.16363525390625,
+      "logps/rejected": -360.77850341796875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02092050209205021,
+      "grad_norm": 8.648628131955448,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.382624864578247,
+      "logits/rejected": -2.3470795154571533,
+      "logps/chosen": -268.0838623046875,
+      "logps/rejected": -252.37567138671875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": -2.9887629352742806e-05,
+      "rewards/margins": -0.00016839371528476477,
+      "rewards/rejected": 0.00013850608957000077,
+      "step": 10
+    },
+    {
+      "epoch": 0.04184100418410042,
+      "grad_norm": 7.617974823658716,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.4419467449188232,
+      "logits/rejected": -2.3880372047424316,
+      "logps/chosen": -281.94671630859375,
+      "logps/rejected": -296.29632568359375,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0021942686289548874,
+      "rewards/margins": 0.0010914185550063848,
+      "rewards/rejected": 0.0011028503067791462,
+      "step": 20
+    },
+    {
+      "epoch": 0.06276150627615062,
+      "grad_norm": 7.730667240255409,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.473952054977417,
+      "logits/rejected": -2.393177032470703,
+      "logps/chosen": -299.96917724609375,
+      "logps/rejected": -263.9479064941406,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.011055229231715202,
+      "rewards/margins": 0.004415138624608517,
+      "rewards/rejected": 0.00664009153842926,
+      "step": 30
+    },
+    {
+      "epoch": 0.08368200836820083,
+      "grad_norm": 7.607331913148092,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.365400791168213,
+      "logits/rejected": -2.3343396186828613,
+      "logps/chosen": -264.3897399902344,
+      "logps/rejected": -244.12796020507812,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.034082163125276566,
+      "rewards/margins": 0.02488231658935547,
+      "rewards/rejected": 0.009199848398566246,
+      "step": 40
+    },
+    {
+      "epoch": 0.10460251046025104,
+      "grad_norm": 7.967409342630165,
+      "learning_rate": 4.999733114418725e-07,
+      "logits/chosen": -2.3081209659576416,
+      "logits/rejected": -2.2759177684783936,
+      "logps/chosen": -279.1407165527344,
+      "logps/rejected": -292.4937744140625,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.026544462889432907,
+      "rewards/margins": 0.03468267619609833,
+      "rewards/rejected": -0.008138209581375122,
+      "step": 50
+    },
+    {
+      "epoch": 0.12552301255230125,
+      "grad_norm": 9.27127742261999,
+      "learning_rate": 4.990398100856366e-07,
+      "logits/chosen": -2.399071216583252,
+      "logits/rejected": -2.350670337677002,
+      "logps/chosen": -272.9728698730469,
+      "logps/rejected": -307.12005615234375,
+      "loss": 0.6532,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0034440518356859684,
+      "rewards/margins": 0.08366598933935165,
+      "rewards/rejected": -0.08022192865610123,
+      "step": 60
+    },
+    {
+      "epoch": 0.14644351464435146,
+      "grad_norm": 10.443914565176241,
+      "learning_rate": 4.967775735898179e-07,
+      "logits/chosen": -2.260702133178711,
+      "logits/rejected": -2.2748961448669434,
+      "logps/chosen": -274.5917663574219,
+      "logps/rejected": -279.2971496582031,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.07422757148742676,
+      "rewards/margins": 0.13401368260383606,
+      "rewards/rejected": -0.208241268992424,
+      "step": 70
+    },
+    {
+      "epoch": 0.16736401673640167,
+      "grad_norm": 15.459859654874244,
+      "learning_rate": 4.931986719649298e-07,
+      "logits/chosen": -2.4126198291778564,
+      "logits/rejected": -2.3573036193847656,
+      "logps/chosen": -338.80572509765625,
+      "logps/rejected": -296.1191711425781,
+      "loss": 0.6162,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.12377747148275375,
+      "rewards/margins": 0.18862910568714142,
+      "rewards/rejected": -0.31240659952163696,
+      "step": 80
+    },
+    {
+      "epoch": 0.18828451882845187,
+      "grad_norm": 11.90521728524108,
+      "learning_rate": 4.883222001996351e-07,
+      "logits/chosen": -2.2599902153015137,
+      "logits/rejected": -2.2168564796447754,
+      "logps/chosen": -292.635009765625,
+      "logps/rejected": -317.6386413574219,
+      "loss": 0.5958,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3297635614871979,
+      "rewards/margins": 0.3308764398097992,
+      "rewards/rejected": -0.6606400012969971,
+      "step": 90
+    },
+    {
+      "epoch": 0.20920502092050208,
+      "grad_norm": 13.96062398758623,
+      "learning_rate": 4.821741763807186e-07,
+      "logits/chosen": -2.213327169418335,
+      "logits/rejected": -2.182467222213745,
+      "logps/chosen": -301.69793701171875,
+      "logps/rejected": -330.6862487792969,
+      "loss": 0.5992,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3573867082595825,
+      "rewards/margins": 0.3630400002002716,
+      "rewards/rejected": -0.7204266786575317,
+      "step": 100
+    },
+    {
+      "epoch": 0.20920502092050208,
+      "eval_logits/chosen": -2.2843189239501953,
+      "eval_logits/rejected": -2.2402381896972656,
+      "eval_logps/chosen": -293.1490783691406,
+      "eval_logps/rejected": -329.4821472167969,
+      "eval_loss": 0.5955678820610046,
+      "eval_rewards/accuracies": 0.71484375,
+      "eval_rewards/chosen": -0.2932242155075073,
+      "eval_rewards/margins": 0.3632100224494934,
+      "eval_rewards/rejected": -0.6564342379570007,
+      "eval_runtime": 138.6513,
+      "eval_samples_per_second": 14.425,
+      "eval_steps_per_second": 0.231,
+      "step": 100
+    },
+    {
+      "epoch": 0.2301255230125523,
+      "grad_norm": 14.60592607361745,
+      "learning_rate": 4.747874028753375e-07,
+      "logits/chosen": -2.263821840286255,
+      "logits/rejected": -2.1686301231384277,
+      "logps/chosen": -341.34893798828125,
+      "logps/rejected": -324.3285217285156,
+      "loss": 0.5993,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.3308233618736267,
+      "rewards/margins": 0.32568514347076416,
+      "rewards/rejected": -0.6565085649490356,
+      "step": 110
+    },
+    {
+      "epoch": 0.2510460251046025,
+      "grad_norm": 18.287627836325075,
+      "learning_rate": 4.662012913161997e-07,
+      "logits/chosen": -2.0950424671173096,
+      "logits/rejected": -2.068326950073242,
+      "logps/chosen": -325.9095458984375,
+      "logps/rejected": -342.905517578125,
+      "loss": 0.5752,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.41224637627601624,
+      "rewards/margins": 0.3889158368110657,
+      "rewards/rejected": -0.8011622428894043,
+      "step": 120
+    },
+    {
+      "epoch": 0.2719665271966527,
+      "grad_norm": 12.97679659726723,
+      "learning_rate": 4.5646165232345103e-07,
+      "logits/chosen": -2.099592447280884,
+      "logits/rejected": -2.0621869564056396,
+      "logps/chosen": -327.469482421875,
+      "logps/rejected": -356.67864990234375,
+      "loss": 0.5625,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.4445687234401703,
+      "rewards/margins": 0.41623586416244507,
+      "rewards/rejected": -0.860804557800293,
+      "step": 130
+    },
+    {
+      "epoch": 0.2928870292887029,
+      "grad_norm": 15.103124909375557,
+      "learning_rate": 4.456204510851956e-07,
+      "logits/chosen": -2.050687313079834,
+      "logits/rejected": -2.0436277389526367,
+      "logps/chosen": -359.5152587890625,
+      "logps/rejected": -391.96661376953125,
+      "loss": 0.5646,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.40217551589012146,
+      "rewards/margins": 0.4963633418083191,
+      "rewards/rejected": -0.8985388875007629,
+      "step": 140
+    },
+    {
+      "epoch": 0.3138075313807531,
+      "grad_norm": 20.299745999142388,
+      "learning_rate": 4.337355301007335e-07,
+      "logits/chosen": -1.984320044517517,
+      "logits/rejected": -1.9091389179229736,
+      "logps/chosen": -336.2827453613281,
+      "logps/rejected": -366.0244140625,
+      "loss": 0.5728,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.5187880396842957,
+      "rewards/margins": 0.3677828311920166,
+      "rewards/rejected": -0.8865708112716675,
+      "step": 150
+    },
+    {
+      "epoch": 0.33472803347280333,
+      "grad_norm": 15.479459276886582,
+      "learning_rate": 4.2087030056579986e-07,
+      "logits/chosen": -1.9620025157928467,
+      "logits/rejected": -1.8357343673706055,
+      "logps/chosen": -324.5109558105469,
+      "logps/rejected": -350.720703125,
+      "loss": 0.5729,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5085537433624268,
+      "rewards/margins": 0.5033678412437439,
+      "rewards/rejected": -1.0119216442108154,
+      "step": 160
+    },
+    {
+      "epoch": 0.35564853556485354,
+      "grad_norm": 17.79663918416807,
+      "learning_rate": 4.070934040463998e-07,
+      "logits/chosen": -1.863904595375061,
+      "logits/rejected": -1.797764539718628,
+      "logps/chosen": -308.15771484375,
+      "logps/rejected": -330.265380859375,
+      "loss": 0.568,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5722733736038208,
+      "rewards/margins": 0.4044536054134369,
+      "rewards/rejected": -0.9767271280288696,
+      "step": 170
+    },
+    {
+      "epoch": 0.37656903765690375,
+      "grad_norm": 16.83875212046203,
+      "learning_rate": 3.9247834624635404e-07,
+      "logits/chosen": -1.6904703378677368,
+      "logits/rejected": -1.617736577987671,
+      "logps/chosen": -314.245849609375,
+      "logps/rejected": -330.62835693359375,
+      "loss": 0.5374,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5731824636459351,
+      "rewards/margins": 0.46102070808410645,
+      "rewards/rejected": -1.034203290939331,
+      "step": 180
+    },
+    {
+      "epoch": 0.39748953974895396,
+      "grad_norm": 19.18049452997649,
+      "learning_rate": 3.7710310482256523e-07,
+      "logits/chosen": -1.7148773670196533,
+      "logits/rejected": -1.6501872539520264,
+      "logps/chosen": -320.56182861328125,
+      "logps/rejected": -354.6867980957031,
+      "loss": 0.5561,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4821714460849762,
+      "rewards/margins": 0.4589623808860779,
+      "rewards/rejected": -0.9411337971687317,
+      "step": 190
+    },
+    {
+      "epoch": 0.41841004184100417,
+      "grad_norm": 35.07989580391586,
+      "learning_rate": 3.610497133404795e-07,
+      "logits/chosen": -1.5612151622772217,
+      "logits/rejected": -1.5434982776641846,
+      "logps/chosen": -309.4225158691406,
+      "logps/rejected": -352.44219970703125,
+      "loss": 0.57,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.4960923194885254,
+      "rewards/margins": 0.5161901712417603,
+      "rewards/rejected": -1.0122824907302856,
+      "step": 200
+    },
+    {
+      "epoch": 0.41841004184100417,
+      "eval_logits/chosen": -1.7196840047836304,
+      "eval_logits/rejected": -1.639003038406372,
+      "eval_logps/chosen": -302.9073181152344,
+      "eval_logps/rejected": -359.9164733886719,
+      "eval_loss": 0.559145987033844,
+      "eval_rewards/accuracies": 0.7421875,
+      "eval_rewards/chosen": -0.3908064365386963,
+      "eval_rewards/margins": 0.569970965385437,
+      "eval_rewards/rejected": -0.9607773423194885,
+      "eval_runtime": 138.5868,
+      "eval_samples_per_second": 14.431,
+      "eval_steps_per_second": 0.231,
+      "step": 200
+    },
+    {
+      "epoch": 0.4393305439330544,
+      "grad_norm": 16.173387035217118,
+      "learning_rate": 3.4440382358952115e-07,
+      "logits/chosen": -1.554513692855835,
+      "logits/rejected": -1.4561774730682373,
+      "logps/chosen": -341.60223388671875,
+      "logps/rejected": -353.4090576171875,
+      "loss": 0.5688,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.4863763749599457,
+      "rewards/margins": 0.5323517322540283,
+      "rewards/rejected": -1.0187281370162964,
+      "step": 210
+    },
+    {
+      "epoch": 0.4602510460251046,
+      "grad_norm": 21.968084457428354,
+      "learning_rate": 3.272542485937368e-07,
+      "logits/chosen": -1.538390040397644,
+      "logits/rejected": -1.434029459953308,
+      "logps/chosen": -316.9021911621094,
+      "logps/rejected": -343.46759033203125,
+      "loss": 0.5586,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.532656729221344,
+      "rewards/margins": 0.5030606389045715,
+      "rewards/rejected": -1.0357173681259155,
+      "step": 220
+    },
+    {
+      "epoch": 0.4811715481171548,
+      "grad_norm": 17.796130138386662,
+      "learning_rate": 3.096924887558854e-07,
+      "logits/chosen": -1.4872758388519287,
+      "logits/rejected": -1.4086310863494873,
+      "logps/chosen": -305.5406494140625,
+      "logps/rejected": -364.971923828125,
+      "loss": 0.5506,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.5540087819099426,
+      "rewards/margins": 0.6236208081245422,
+      "rewards/rejected": -1.1776294708251953,
+      "step": 230
+    },
+    {
+      "epoch": 0.502092050209205,
+      "grad_norm": 22.063726081874908,
+      "learning_rate": 2.9181224366319943e-07,
+      "logits/chosen": -1.5888181924819946,
+      "logits/rejected": -1.5095860958099365,
+      "logps/chosen": -316.6210632324219,
+      "logps/rejected": -359.104248046875,
+      "loss": 0.5263,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.5185328722000122,
+      "rewards/margins": 0.6040252447128296,
+      "rewards/rejected": -1.1225581169128418,
+      "step": 240
+    },
+    {
+      "epoch": 0.5230125523012552,
+      "grad_norm": 19.61380627881043,
+      "learning_rate": 2.7370891215954565e-07,
+      "logits/chosen": -1.4691661596298218,
+      "logits/rejected": -1.3506040573120117,
+      "logps/chosen": -353.1708984375,
+      "logps/rejected": -376.46856689453125,
+      "loss": 0.5438,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.5244424939155579,
+      "rewards/margins": 0.6430000066757202,
+      "rewards/rejected": -1.1674424409866333,
+      "step": 250
+    },
+    {
+      "epoch": 0.5439330543933054,
+      "grad_norm": 23.636957052718806,
+      "learning_rate": 2.55479083351317e-07,
+      "logits/chosen": -1.4749513864517212,
+      "logits/rejected": -1.3565380573272705,
+      "logps/chosen": -350.9261779785156,
+      "logps/rejected": -368.041748046875,
+      "loss": 0.547,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5501475930213928,
+      "rewards/margins": 0.5780745148658752,
+      "rewards/rejected": -1.128222107887268,
+      "step": 260
+    },
+    {
+      "epoch": 0.5648535564853556,
+      "grad_norm": 17.97300559505276,
+      "learning_rate": 2.3722002126275822e-07,
+      "logits/chosen": -1.517932653427124,
+      "logits/rejected": -1.4447662830352783,
+      "logps/chosen": -327.8120422363281,
+      "logps/rejected": -353.8706970214844,
+      "loss": 0.5444,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4883030951023102,
+      "rewards/margins": 0.4679597318172455,
+      "rewards/rejected": -0.9562627673149109,
+      "step": 270
+    },
+    {
+      "epoch": 0.5857740585774058,
+      "grad_norm": 22.292312183806402,
+      "learning_rate": 2.19029145890313e-07,
+      "logits/chosen": -1.3661726713180542,
+      "logits/rejected": -1.218951940536499,
+      "logps/chosen": -325.18756103515625,
+      "logps/rejected": -358.7313232421875,
+      "loss": 0.5516,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6136071085929871,
+      "rewards/margins": 0.5374716520309448,
+      "rewards/rejected": -1.1510788202285767,
+      "step": 280
+    },
+    {
+      "epoch": 0.606694560669456,
+      "grad_norm": 18.485623156759097,
+      "learning_rate": 2.0100351342479216e-07,
+      "logits/chosen": -1.4040883779525757,
+      "logits/rejected": -1.3219355344772339,
+      "logps/chosen": -316.49322509765625,
+      "logps/rejected": -359.63519287109375,
+      "loss": 0.5484,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.6615027189254761,
+      "rewards/margins": 0.5264458060264587,
+      "rewards/rejected": -1.1879485845565796,
+      "step": 290
+    },
+    {
+      "epoch": 0.6276150627615062,
+      "grad_norm": 16.713586027443107,
+      "learning_rate": 1.8323929841460178e-07,
+      "logits/chosen": -1.4240423440933228,
+      "logits/rejected": -1.248325228691101,
+      "logps/chosen": -362.2311706542969,
+      "logps/rejected": -374.8503112792969,
+      "loss": 0.5222,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5985096096992493,
+      "rewards/margins": 0.5667251348495483,
+      "rewards/rejected": -1.1652346849441528,
+      "step": 300
+    },
+    {
+      "epoch": 0.6276150627615062,
+      "eval_logits/chosen": -1.4138315916061401,
+      "eval_logits/rejected": -1.313331961631775,
+      "eval_logps/chosen": -311.970703125,
+      "eval_logps/rejected": -381.0071716308594,
+      "eval_loss": 0.5472748279571533,
+      "eval_rewards/accuracies": 0.74609375,
+      "eval_rewards/chosen": -0.4814407229423523,
+      "eval_rewards/margins": 0.6902433633804321,
+      "eval_rewards/rejected": -1.1716840267181396,
+      "eval_runtime": 138.4988,
+      "eval_samples_per_second": 14.441,
+      "eval_steps_per_second": 0.231,
+      "step": 300
+    },
+    {
+      "epoch": 0.6485355648535565,
+      "grad_norm": 19.79799392754639,
+      "learning_rate": 1.6583128063291573e-07,
+      "logits/chosen": -1.2533605098724365,
+      "logits/rejected": -1.190824270248413,
+      "logps/chosen": -354.2838439941406,
+      "logps/rejected": -371.8002014160156,
+      "loss": 0.5314,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5662702322006226,
+      "rewards/margins": 0.5699962973594666,
+      "rewards/rejected": -1.1362664699554443,
+      "step": 310
+    },
+    {
+      "epoch": 0.6694560669456067,
+      "grad_norm": 30.909931043164597,
+      "learning_rate": 1.488723393865766e-07,
+      "logits/chosen": -1.2598419189453125,
+      "logits/rejected": -1.156743049621582,
+      "logps/chosen": -353.2414245605469,
+      "logps/rejected": -360.50372314453125,
+      "loss": 0.5207,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.5773128271102905,
+      "rewards/margins": 0.5609738230705261,
+      "rewards/rejected": -1.1382867097854614,
+      "step": 320
+    },
+    {
+      "epoch": 0.6903765690376569,
+      "grad_norm": 23.359866281207143,
+      "learning_rate": 1.3245295796480788e-07,
+      "logits/chosen": -1.1778898239135742,
+      "logits/rejected": -1.0287997722625732,
+      "logps/chosen": -322.0570983886719,
+      "logps/rejected": -371.15679931640625,
+      "loss": 0.5342,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5950717329978943,
+      "rewards/margins": 0.5082412958145142,
+      "rewards/rejected": -1.1033132076263428,
+      "step": 330
+    },
+    {
+      "epoch": 0.7112970711297071,
+      "grad_norm": 26.5996382586134,
+      "learning_rate": 1.1666074087171627e-07,
+      "logits/chosen": -0.9922504425048828,
+      "logits/rejected": -0.8580983877182007,
+      "logps/chosen": -352.2337341308594,
+      "logps/rejected": -416.58740234375,
+      "loss": 0.5263,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.5986633896827698,
+      "rewards/margins": 0.736039400100708,
+      "rewards/rejected": -1.3347028493881226,
+      "step": 340
+    },
+    {
+      "epoch": 0.7322175732217573,
+      "grad_norm": 21.5234507080226,
+      "learning_rate": 1.0157994641835734e-07,
+      "logits/chosen": -0.6279062628746033,
+      "logits/rejected": -0.44238734245300293,
+      "logps/chosen": -329.3270263671875,
+      "logps/rejected": -378.29486083984375,
+      "loss": 0.5064,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.6930121183395386,
+      "rewards/margins": 0.6956107020378113,
+      "rewards/rejected": -1.388622760772705,
+      "step": 350
+    },
+    {
+      "epoch": 0.7531380753138075,
+      "grad_norm": 21.10296184322806,
+      "learning_rate": 8.729103716819111e-08,
+      "logits/chosen": -0.42689481377601624,
+      "logits/rejected": -0.05037093162536621,
+      "logps/chosen": -380.1378479003906,
+      "logps/rejected": -405.749755859375,
+      "loss": 0.5475,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.7636948823928833,
+      "rewards/margins": 0.6744855642318726,
+      "rewards/rejected": -1.4381804466247559,
+      "step": 360
+    },
+    {
+      "epoch": 0.7740585774058577,
+      "grad_norm": 20.405513047162312,
+      "learning_rate": 7.387025063449081e-08,
+      "logits/chosen": 0.00676759472116828,
+      "logits/rejected": 0.27322202920913696,
+      "logps/chosen": -355.5211181640625,
+      "logps/rejected": -372.49932861328125,
+      "loss": 0.5303,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.7726585268974304,
+      "rewards/margins": 0.5654970407485962,
+      "rewards/rejected": -1.3381555080413818,
+      "step": 370
+    },
+    {
+      "epoch": 0.7949790794979079,
+      "grad_norm": 20.074958448970854,
+      "learning_rate": 6.138919252022435e-08,
+      "logits/chosen": 0.2522502541542053,
+      "logits/rejected": 0.30653125047683716,
+      "logps/chosen": -323.6443786621094,
+      "logps/rejected": -412.6951599121094,
+      "loss": 0.5236,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.8133866190910339,
+      "rewards/margins": 0.7043942213058472,
+      "rewards/rejected": -1.5177809000015259,
+      "step": 380
+    },
+    {
+      "epoch": 0.8158995815899581,
+      "grad_norm": 29.718090261041645,
+      "learning_rate": 4.991445467064689e-08,
+      "logits/chosen": 0.1301044225692749,
+      "logits/rejected": 0.2582958936691284,
+      "logps/chosen": -368.65142822265625,
+      "logps/rejected": -415.99212646484375,
+      "loss": 0.52,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.6479382514953613,
+      "rewards/margins": 0.6408324241638184,
+      "rewards/rejected": -1.2887706756591797,
+      "step": 390
+    },
+    {
+      "epoch": 0.8368200836820083,
+      "grad_norm": 23.237016960868214,
+      "learning_rate": 3.9507259776993954e-08,
+      "logits/chosen": 0.4682826101779938,
+      "logits/rejected": 0.722841203212738,
+      "logps/chosen": -349.4666442871094,
+      "logps/rejected": -410.83270263671875,
+      "loss": 0.5332,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.7660902738571167,
+      "rewards/margins": 0.668927013874054,
+      "rewards/rejected": -1.4350173473358154,
+      "step": 400
+    },
+    {
+      "epoch": 0.8368200836820083,
+      "eval_logits/chosen": 0.2838735580444336,
+      "eval_logits/rejected": 0.5322580933570862,
+      "eval_logps/chosen": -325.5808410644531,
+      "eval_logps/rejected": -405.0050048828125,
+      "eval_loss": 0.5284038186073303,
+      "eval_rewards/accuracies": 0.75390625,
+      "eval_rewards/chosen": -0.6175417900085449,
+      "eval_rewards/margins": 0.7941210269927979,
+      "eval_rewards/rejected": -1.4116626977920532,
+      "eval_runtime": 138.4597,
+      "eval_samples_per_second": 14.445,
+      "eval_steps_per_second": 0.231,
+      "step": 400
+    },
+    {
+      "epoch": 0.8577405857740585,
+      "grad_norm": 21.410899974880437,
+      "learning_rate": 3.022313472693447e-08,
+      "logits/chosen": 0.4737936854362488,
+      "logits/rejected": 0.8527080416679382,
+      "logps/chosen": -365.0519104003906,
+      "logps/rejected": -407.4276428222656,
+      "loss": 0.5338,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.6539891362190247,
+      "rewards/margins": 0.7270708680152893,
+      "rewards/rejected": -1.3810601234436035,
+      "step": 410
+    },
+    {
+      "epoch": 0.8786610878661087,
+      "grad_norm": 23.815834641200247,
+      "learning_rate": 2.2111614344599684e-08,
+      "logits/chosen": 0.43105998635292053,
+      "logits/rejected": 0.7020677328109741,
+      "logps/chosen": -367.0744323730469,
+      "logps/rejected": -411.14324951171875,
+      "loss": 0.5204,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.7023344039916992,
+      "rewards/margins": 0.6693269610404968,
+      "rewards/rejected": -1.3716614246368408,
+      "step": 420
+    },
+    {
+      "epoch": 0.899581589958159,
+      "grad_norm": 22.437685513304036,
+      "learning_rate": 1.521597710086439e-08,
+      "logits/chosen": 0.8259914517402649,
+      "logits/rejected": 1.0822646617889404,
+      "logps/chosen": -352.48974609375,
+      "logps/rejected": -403.360595703125,
+      "loss": 0.5046,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7168700695037842,
+      "rewards/margins": 0.7234013676643372,
+      "rewards/rejected": -1.4402713775634766,
+      "step": 430
+    },
+    {
+      "epoch": 0.9205020920502092,
+      "grad_norm": 23.732152862050572,
+      "learning_rate": 9.57301420397924e-09,
+      "logits/chosen": 0.6262695789337158,
+      "logits/rejected": 0.9615359306335449,
+      "logps/chosen": -348.96661376953125,
+      "logps/rejected": -406.67974853515625,
+      "loss": 0.5137,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.6009017825126648,
+      "rewards/margins": 0.7451079487800598,
+      "rewards/rejected": -1.346009612083435,
+      "step": 440
+    },
+    {
+      "epoch": 0.9414225941422594,
+      "grad_norm": 21.68475406897744,
+      "learning_rate": 5.212833302556258e-09,
+      "logits/chosen": 0.8079266548156738,
+      "logits/rejected": 0.969477653503418,
+      "logps/chosen": -369.1518249511719,
+      "logps/rejected": -449.89385986328125,
+      "loss": 0.5229,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.7800692319869995,
+      "rewards/margins": 0.6285091638565063,
+      "rewards/rejected": -1.4085785150527954,
+      "step": 450
+    },
+    {
+      "epoch": 0.9623430962343096,
+      "grad_norm": 23.158227656060077,
+      "learning_rate": 2.158697848236607e-09,
+      "logits/chosen": 0.8919423818588257,
+      "logits/rejected": 1.1862996816635132,
+      "logps/chosen": -344.595703125,
+      "logps/rejected": -376.88714599609375,
+      "loss": 0.5221,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.70527184009552,
+      "rewards/margins": 0.6562052965164185,
+      "rewards/rejected": -1.3614771366119385,
+      "step": 460
+    },
+    {
+      "epoch": 0.9832635983263598,
+      "grad_norm": 19.603511735209654,
+      "learning_rate": 4.269029751107489e-10,
+      "logits/chosen": 0.8390657305717468,
+      "logits/rejected": 1.1047471761703491,
+      "logps/chosen": -347.3504943847656,
+      "logps/rejected": -415.75677490234375,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.7157813906669617,
+      "rewards/margins": 0.6578011512756348,
+      "rewards/rejected": -1.3735826015472412,
+      "step": 470
+    },
+    {
+      "epoch": 1.0,
+      "step": 478,
+      "total_flos": 0.0,
+      "train_loss": 0.5645424491690791,
+      "train_runtime": 10520.2198,
+      "train_samples_per_second": 5.811,
+      "train_steps_per_second": 0.045
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 478,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}