Model save

README.md

@@ -0,0 +1,59 @@
+---
+license: mit
+base_model: HuggingFaceH4/mistral-7b-sft-beta
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: DPO-Zephyr-7B
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# DPO-Zephyr-7B
+
+This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 3
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 3
+- total_train_batch_size: 9
+- total_eval_batch_size: 12
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.41.2
+- Pytorch 2.2.2+cu118
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.3274427829874902,
+    "train_runtime": 3901.2433,
+    "train_samples": 15283,
+    "train_samples_per_second": 3.917,
+    "train_steps_per_second": 0.436
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.41.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e31fb4d82e99207bc7003cbecaca462412a4118bf0620396c9dbf7d6415e0986
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d8a374c1588b189db65a81ef788f6209c9af93191d8af95c774e8a54a40a0d2
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd1c846ebb9bb5d0a800fcb67891e7854237198c53e5a70101de5ed5fa17296f
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.3274427829874902,
+    "train_runtime": 3901.2433,
+    "train_samples": 15283,
+    "train_samples_per_second": 3.917,
+    "train_steps_per_second": 0.436
+}

trainer_state.json

@@ -0,0 +1,2592 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1699,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0005885815185403178,
+      "grad_norm": 32.780718251913974,
+      "learning_rate": 2.942907592701589e-10,
+      "logits/chosen": -2.603684186935425,
+      "logits/rejected": -2.7838211059570312,
+      "logps/chosen": -294.7889099121094,
+      "logps/rejected": -69.72528076171875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.005885815185403178,
+      "grad_norm": 30.421018457904477,
+      "learning_rate": 2.9429075927015888e-09,
+      "logits/chosen": -2.8964638710021973,
+      "logits/rejected": -2.901930809020996,
+      "logps/chosen": -331.2449645996094,
+      "logps/rejected": -301.0341796875,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.2222222238779068,
+      "rewards/chosen": -0.0001181578409159556,
+      "rewards/margins": -0.0006874248501844704,
+      "rewards/rejected": 0.0005692669074051082,
+      "step": 10
+    },
+    {
+      "epoch": 0.011771630370806356,
+      "grad_norm": 39.77348485571547,
+      "learning_rate": 5.8858151854031776e-09,
+      "logits/chosen": -2.890000581741333,
+      "logits/rejected": -2.8872923851013184,
+      "logps/chosen": -299.2838439941406,
+      "logps/rejected": -242.45925903320312,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": 0.00012406618043314666,
+      "rewards/margins": 0.0003421924775466323,
+      "rewards/rejected": -0.0002181262243539095,
+      "step": 20
+    },
+    {
+      "epoch": 0.017657445556209534,
+      "grad_norm": 37.15226364119724,
+      "learning_rate": 8.828722778104766e-09,
+      "logits/chosen": -2.871821403503418,
+      "logits/rejected": -2.821002721786499,
+      "logps/chosen": -275.4599609375,
+      "logps/rejected": -248.410888671875,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.3999999761581421,
+      "rewards/chosen": 7.882448699092492e-05,
+      "rewards/margins": -0.0011783873196691275,
+      "rewards/rejected": 0.0012572116684168577,
+      "step": 30
+    },
+    {
+      "epoch": 0.023543260741612712,
+      "grad_norm": 28.986298484577823,
+      "learning_rate": 1.1771630370806355e-08,
+      "logits/chosen": -2.8257861137390137,
+      "logits/rejected": -2.782503366470337,
+      "logps/chosen": -278.97601318359375,
+      "logps/rejected": -273.42205810546875,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.0018755996134132147,
+      "rewards/margins": -0.0013799888547509909,
+      "rewards/rejected": -0.0004956105258315802,
+      "step": 40
+    },
+    {
+      "epoch": 0.02942907592701589,
+      "grad_norm": 28.82026354510354,
+      "learning_rate": 1.4714537963507944e-08,
+      "logits/chosen": -2.855421543121338,
+      "logits/rejected": -2.8377156257629395,
+      "logps/chosen": -253.5948944091797,
+      "logps/rejected": -274.3114318847656,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0006353737553581595,
+      "rewards/margins": -0.0002903261920437217,
+      "rewards/rejected": -0.00034504756331443787,
+      "step": 50
+    },
+    {
+      "epoch": 0.03531489111241907,
+      "grad_norm": 30.822584821190027,
+      "learning_rate": 1.7657445556209532e-08,
+      "logits/chosen": -2.824986696243286,
+      "logits/rejected": -2.8068385124206543,
+      "logps/chosen": -330.98846435546875,
+      "logps/rejected": -329.9060974121094,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.4666667580604553,
+      "rewards/chosen": 0.00020488860900513828,
+      "rewards/margins": -0.0003379290283191949,
+      "rewards/rejected": 0.000542817753739655,
+      "step": 60
+    },
+    {
+      "epoch": 0.04120070629782225,
+      "grad_norm": 32.409000763227326,
+      "learning_rate": 2.0600353148911124e-08,
+      "logits/chosen": -2.6729750633239746,
+      "logits/rejected": -2.7752492427825928,
+      "logps/chosen": -288.5548400878906,
+      "logps/rejected": -286.0675964355469,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.0015738470247015357,
+      "rewards/margins": -0.001438200124539435,
+      "rewards/rejected": -0.0001356468565063551,
+      "step": 70
+    },
+    {
+      "epoch": 0.047086521483225424,
+      "grad_norm": 28.66094375035891,
+      "learning_rate": 2.354326074161271e-08,
+      "logits/chosen": -2.8610806465148926,
+      "logits/rejected": -2.809129476547241,
+      "logps/chosen": -202.50711059570312,
+      "logps/rejected": -256.8191833496094,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.0010831685503944755,
+      "rewards/margins": -0.0023852435406297445,
+      "rewards/rejected": 0.001302075106650591,
+      "step": 80
+    },
+    {
+      "epoch": 0.052972336668628606,
+      "grad_norm": 31.002618375475574,
+      "learning_rate": 2.6486168334314303e-08,
+      "logits/chosen": -2.9170703887939453,
+      "logits/rejected": -2.8157236576080322,
+      "logps/chosen": -280.6930236816406,
+      "logps/rejected": -256.5555725097656,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0004723949823528528,
+      "rewards/margins": 0.0002984645252581686,
+      "rewards/rejected": 0.00017393044254276901,
+      "step": 90
+    },
+    {
+      "epoch": 0.05885815185403178,
+      "grad_norm": 35.75851601438966,
+      "learning_rate": 2.942907592701589e-08,
+      "logits/chosen": -2.882338285446167,
+      "logits/rejected": -2.8756601810455322,
+      "logps/chosen": -284.1132507324219,
+      "logps/rejected": -261.82562255859375,
+      "loss": 0.693,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0010607184376567602,
+      "rewards/margins": 0.0015401893761008978,
+      "rewards/rejected": -0.0004794707929249853,
+      "step": 100
+    },
+    {
+      "epoch": 0.06474396703943497,
+      "grad_norm": 26.830290079245987,
+      "learning_rate": 3.2371983519717484e-08,
+      "logits/chosen": -2.8224923610687256,
+      "logits/rejected": -2.786034107208252,
+      "logps/chosen": -213.18856811523438,
+      "logps/rejected": -209.486572265625,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5000000596046448,
+      "rewards/chosen": 0.0004384171334095299,
+      "rewards/margins": 0.000683752354234457,
+      "rewards/rejected": -0.00024533524992875755,
+      "step": 110
+    },
+    {
+      "epoch": 0.07062978222483814,
+      "grad_norm": 33.59257452712036,
+      "learning_rate": 3.5314891112419064e-08,
+      "logits/chosen": -2.7929792404174805,
+      "logits/rejected": -2.787545919418335,
+      "logps/chosen": -296.0240478515625,
+      "logps/rejected": -260.3634033203125,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.4333333373069763,
+      "rewards/chosen": 8.877464279066771e-05,
+      "rewards/margins": -0.0005403327522799373,
+      "rewards/rejected": 0.000629107526037842,
+      "step": 120
+    },
+    {
+      "epoch": 0.07651559741024132,
+      "grad_norm": 31.167055741749806,
+      "learning_rate": 3.8257798705120656e-08,
+      "logits/chosen": -2.929863452911377,
+      "logits/rejected": -2.8793587684631348,
+      "logps/chosen": -192.31356811523438,
+      "logps/rejected": -197.66647338867188,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5000000596046448,
+      "rewards/chosen": 0.001261854893527925,
+      "rewards/margins": 0.00017420703079551458,
+      "rewards/rejected": 0.0010876477463170886,
+      "step": 130
+    },
+    {
+      "epoch": 0.0824014125956445,
+      "grad_norm": 25.597516159244545,
+      "learning_rate": 4.120070629782225e-08,
+      "logits/chosen": -2.818716526031494,
+      "logits/rejected": -2.8491830825805664,
+      "logps/chosen": -267.6719970703125,
+      "logps/rejected": -269.23138427734375,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5666667222976685,
+      "rewards/chosen": 0.002183220349252224,
+      "rewards/margins": 0.0013612338807433844,
+      "rewards/rejected": 0.0008219865849241614,
+      "step": 140
+    },
+    {
+      "epoch": 0.08828722778104768,
+      "grad_norm": 30.754492049125638,
+      "learning_rate": 4.414361389052384e-08,
+      "logits/chosen": -2.807731866836548,
+      "logits/rejected": -2.82275652885437,
+      "logps/chosen": -256.2826843261719,
+      "logps/rejected": -216.8761749267578,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": 0.0029171775095164776,
+      "rewards/margins": 0.0019290171330794692,
+      "rewards/rejected": 0.0009881604928523302,
+      "step": 150
+    },
+    {
+      "epoch": 0.09417304296645085,
+      "grad_norm": 33.216197696590626,
+      "learning_rate": 4.708652148322542e-08,
+      "logits/chosen": -2.7577600479125977,
+      "logits/rejected": -2.686497449874878,
+      "logps/chosen": -271.95977783203125,
+      "logps/rejected": -216.9535675048828,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0015044666361063719,
+      "rewards/margins": -0.0006265610572881997,
+      "rewards/rejected": 0.0021310276351869106,
+      "step": 160
+    },
+    {
+      "epoch": 0.10005885815185403,
+      "grad_norm": 31.85341360598595,
+      "learning_rate": 5.002942907592701e-08,
+      "logits/chosen": -2.860220432281494,
+      "logits/rejected": -2.824690580368042,
+      "logps/chosen": -229.2205047607422,
+      "logps/rejected": -216.49691772460938,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": 0.005175250582396984,
+      "rewards/margins": 0.004574229009449482,
+      "rewards/rejected": 0.0006010220386087894,
+      "step": 170
+    },
+    {
+      "epoch": 0.10594467333725721,
+      "grad_norm": 33.35020421307099,
+      "learning_rate": 5.2972336668628605e-08,
+      "logits/chosen": -2.8476788997650146,
+      "logits/rejected": -2.8345282077789307,
+      "logps/chosen": -276.9328918457031,
+      "logps/rejected": -297.92138671875,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": 0.002554533304646611,
+      "rewards/margins": 0.002964629326015711,
+      "rewards/rejected": -0.0004100958467461169,
+      "step": 180
+    },
+    {
+      "epoch": 0.11183048852266039,
+      "grad_norm": 27.907028115997573,
+      "learning_rate": 5.591524426133019e-08,
+      "logits/chosen": -2.8730475902557373,
+      "logits/rejected": -2.839439868927002,
+      "logps/chosen": -272.5675048828125,
+      "logps/rejected": -264.9402770996094,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": 0.002410594839602709,
+      "rewards/margins": 0.000803423929028213,
+      "rewards/rejected": 0.0016071706777438521,
+      "step": 190
+    },
+    {
+      "epoch": 0.11771630370806356,
+      "grad_norm": 29.481219977820295,
+      "learning_rate": 5.885815185403178e-08,
+      "logits/chosen": -2.9408724308013916,
+      "logits/rejected": -2.815699338912964,
+      "logps/chosen": -310.2392883300781,
+      "logps/rejected": -236.49404907226562,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": 0.006228293292224407,
+      "rewards/margins": 0.005451700650155544,
+      "rewards/rejected": 0.0007765936315990984,
+      "step": 200
+    },
+    {
+      "epoch": 0.12360211889346674,
+      "grad_norm": 35.046475552370886,
+      "learning_rate": 6.180105944673337e-08,
+      "logits/chosen": -2.6355643272399902,
+      "logits/rejected": -2.668175458908081,
+      "logps/chosen": -314.09619140625,
+      "logps/rejected": -340.42645263671875,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": 0.010181600227952003,
+      "rewards/margins": 0.004080074839293957,
+      "rewards/rejected": 0.006101525854319334,
+      "step": 210
+    },
+    {
+      "epoch": 0.12948793407886994,
+      "grad_norm": 30.183227607657507,
+      "learning_rate": 6.474396703943497e-08,
+      "logits/chosen": -2.922337293624878,
+      "logits/rejected": -2.9165425300598145,
+      "logps/chosen": -311.33917236328125,
+      "logps/rejected": -271.88800048828125,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.008037504740059376,
+      "rewards/margins": 0.006833515129983425,
+      "rewards/rejected": 0.0012039883295074105,
+      "step": 220
+    },
+    {
+      "epoch": 0.1353737492642731,
+      "grad_norm": 28.651320388398528,
+      "learning_rate": 6.768687463213654e-08,
+      "logits/chosen": -2.764054536819458,
+      "logits/rejected": -2.791746139526367,
+      "logps/chosen": -342.35638427734375,
+      "logps/rejected": -269.53033447265625,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": 0.011332266964018345,
+      "rewards/margins": 0.010512511245906353,
+      "rewards/rejected": 0.0008197550778277218,
+      "step": 230
+    },
+    {
+      "epoch": 0.14125956444967627,
+      "grad_norm": 30.194472525654817,
+      "learning_rate": 7.062978222483813e-08,
+      "logits/chosen": -2.9200429916381836,
+      "logits/rejected": -2.8475396633148193,
+      "logps/chosen": -330.425537109375,
+      "logps/rejected": -282.93597412109375,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": 0.015913909301161766,
+      "rewards/margins": 0.013330700807273388,
+      "rewards/rejected": 0.0025832061655819416,
+      "step": 240
+    },
+    {
+      "epoch": 0.14714537963507945,
+      "grad_norm": 49.29351755297091,
+      "learning_rate": 7.357268981753973e-08,
+      "logits/chosen": -2.809460401535034,
+      "logits/rejected": -2.847788095474243,
+      "logps/chosen": -228.82962036132812,
+      "logps/rejected": -260.50067138671875,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.010601197369396687,
+      "rewards/margins": 0.0034695893991738558,
+      "rewards/rejected": 0.007131607737392187,
+      "step": 250
+    },
+    {
+      "epoch": 0.15303119482048264,
+      "grad_norm": 35.22606986399358,
+      "learning_rate": 7.651559741024131e-08,
+      "logits/chosen": -2.829927921295166,
+      "logits/rejected": -2.7731313705444336,
+      "logps/chosen": -368.4117736816406,
+      "logps/rejected": -278.7731018066406,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": 0.014880454167723656,
+      "rewards/margins": 0.008975975215435028,
+      "rewards/rejected": 0.00590447848662734,
+      "step": 260
+    },
+    {
+      "epoch": 0.15891701000588582,
+      "grad_norm": 31.079626356652042,
+      "learning_rate": 7.945850500294291e-08,
+      "logits/chosen": -2.8230886459350586,
+      "logits/rejected": -2.8253014087677,
+      "logps/chosen": -246.8427276611328,
+      "logps/rejected": -257.07440185546875,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": 0.011582859791815281,
+      "rewards/margins": 0.00953072588890791,
+      "rewards/rejected": 0.002052133437246084,
+      "step": 270
+    },
+    {
+      "epoch": 0.164802825191289,
+      "grad_norm": 34.16689629310999,
+      "learning_rate": 8.24014125956445e-08,
+      "logits/chosen": -2.790585994720459,
+      "logits/rejected": -2.741240978240967,
+      "logps/chosen": -194.43751525878906,
+      "logps/rejected": -246.21109008789062,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.011640877462923527,
+      "rewards/margins": 0.013520603068172932,
+      "rewards/rejected": -0.0018797263037413359,
+      "step": 280
+    },
+    {
+      "epoch": 0.17068864037669218,
+      "grad_norm": 26.650476040593702,
+      "learning_rate": 8.534432018834608e-08,
+      "logits/chosen": -2.877924680709839,
+      "logits/rejected": -2.9167160987854004,
+      "logps/chosen": -200.64999389648438,
+      "logps/rejected": -239.98623657226562,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.015060337260365486,
+      "rewards/margins": 0.004761075135320425,
+      "rewards/rejected": 0.0102992607280612,
+      "step": 290
+    },
+    {
+      "epoch": 0.17657445556209536,
+      "grad_norm": 37.84975273963207,
+      "learning_rate": 8.828722778104768e-08,
+      "logits/chosen": -2.7102274894714355,
+      "logits/rejected": -2.723665475845337,
+      "logps/chosen": -241.3960418701172,
+      "logps/rejected": -201.9961395263672,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": 0.023427072912454605,
+      "rewards/margins": 0.020558174699544907,
+      "rewards/rejected": 0.002868895884603262,
+      "step": 300
+    },
+    {
+      "epoch": 0.18246027074749852,
+      "grad_norm": 32.582230477501014,
+      "learning_rate": 9.123013537374925e-08,
+      "logits/chosen": -2.771923303604126,
+      "logits/rejected": -2.755645275115967,
+      "logps/chosen": -236.4650421142578,
+      "logps/rejected": -176.6122589111328,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.8333331942558289,
+      "rewards/chosen": 0.03548317030072212,
+      "rewards/margins": 0.026407241821289062,
+      "rewards/rejected": 0.00907592847943306,
+      "step": 310
+    },
+    {
+      "epoch": 0.1883460859329017,
+      "grad_norm": 35.39155087276743,
+      "learning_rate": 9.417304296645084e-08,
+      "logits/chosen": -2.809967517852783,
+      "logits/rejected": -2.7024905681610107,
+      "logps/chosen": -267.1817932128906,
+      "logps/rejected": -216.0163116455078,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.027452057227492332,
+      "rewards/margins": 0.02637614868581295,
+      "rewards/rejected": 0.0010759088909253478,
+      "step": 320
+    },
+    {
+      "epoch": 0.19423190111830488,
+      "grad_norm": 25.00506965748009,
+      "learning_rate": 9.711595055915244e-08,
+      "logits/chosen": -2.8656439781188965,
+      "logits/rejected": -2.797421932220459,
+      "logps/chosen": -223.9487762451172,
+      "logps/rejected": -238.2415313720703,
+      "loss": 0.687,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.018507182598114014,
+      "rewards/margins": 0.011660409159958363,
+      "rewards/rejected": 0.006846773438155651,
+      "step": 330
+    },
+    {
+      "epoch": 0.20011771630370806,
+      "grad_norm": 31.48968715365344,
+      "learning_rate": 1.0005885815185403e-07,
+      "logits/chosen": -2.8663744926452637,
+      "logits/rejected": -2.829288959503174,
+      "logps/chosen": -227.17904663085938,
+      "logps/rejected": -254.73300170898438,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03473003953695297,
+      "rewards/margins": -0.002185368211939931,
+      "rewards/rejected": 0.036915406584739685,
+      "step": 340
+    },
+    {
+      "epoch": 0.20600353148911124,
+      "grad_norm": 33.13341184823187,
+      "learning_rate": 1.0300176574455561e-07,
+      "logits/chosen": -2.8728482723236084,
+      "logits/rejected": -2.849656581878662,
+      "logps/chosen": -232.64306640625,
+      "logps/rejected": -189.30641174316406,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.5999999642372131,
+      "rewards/chosen": 0.025649067014455795,
+      "rewards/margins": 0.01318146288394928,
+      "rewards/rejected": 0.01246760506182909,
+      "step": 350
+    },
+    {
+      "epoch": 0.21188934667451442,
+      "grad_norm": 33.57559926375768,
+      "learning_rate": 1.0594467333725721e-07,
+      "logits/chosen": -2.94728422164917,
+      "logits/rejected": -2.8343241214752197,
+      "logps/chosen": -330.82861328125,
+      "logps/rejected": -255.1486053466797,
+      "loss": 0.6704,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": 0.060690779238939285,
+      "rewards/margins": 0.051527559757232666,
+      "rewards/rejected": 0.009163224138319492,
+      "step": 360
+    },
+    {
+      "epoch": 0.2177751618599176,
+      "grad_norm": 30.406704666801055,
+      "learning_rate": 1.088875809299588e-07,
+      "logits/chosen": -2.847501039505005,
+      "logits/rejected": -2.859219551086426,
+      "logps/chosen": -226.4930877685547,
+      "logps/rejected": -411.73291015625,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": 0.029315758496522903,
+      "rewards/margins": 0.03848721459507942,
+      "rewards/rejected": -0.009171457961201668,
+      "step": 370
+    },
+    {
+      "epoch": 0.22366097704532079,
+      "grad_norm": 32.000980950184385,
+      "learning_rate": 1.1183048852266038e-07,
+      "logits/chosen": -2.750246047973633,
+      "logits/rejected": -2.683522939682007,
+      "logps/chosen": -312.06695556640625,
+      "logps/rejected": -251.6304931640625,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": 0.029916947707533836,
+      "rewards/margins": 0.033031001687049866,
+      "rewards/rejected": -0.003114058170467615,
+      "step": 380
+    },
+    {
+      "epoch": 0.22954679223072397,
+      "grad_norm": 34.96067667192858,
+      "learning_rate": 1.1477339611536198e-07,
+      "logits/chosen": -2.855989694595337,
+      "logits/rejected": -2.815114974975586,
+      "logps/chosen": -262.18450927734375,
+      "logps/rejected": -189.57838439941406,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.025654960423707962,
+      "rewards/margins": 0.0397619903087616,
+      "rewards/rejected": -0.01410702895373106,
+      "step": 390
+    },
+    {
+      "epoch": 0.23543260741612712,
+      "grad_norm": 30.784886423492903,
+      "learning_rate": 1.1771630370806355e-07,
+      "logits/chosen": -2.8241119384765625,
+      "logits/rejected": -2.836333751678467,
+      "logps/chosen": -233.1301727294922,
+      "logps/rejected": -277.805419921875,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": 0.03232261911034584,
+      "rewards/margins": 0.03198341652750969,
+      "rewards/rejected": 0.00033920182613655925,
+      "step": 400
+    },
+    {
+      "epoch": 0.2413184226015303,
+      "grad_norm": 27.065262812921755,
+      "learning_rate": 1.2065921130076514e-07,
+      "logits/chosen": -2.756892442703247,
+      "logits/rejected": -2.6843674182891846,
+      "logps/chosen": -287.8220520019531,
+      "logps/rejected": -222.5053253173828,
+      "loss": 0.67,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.03421301394701004,
+      "rewards/margins": 0.054109979420900345,
+      "rewards/rejected": -0.019896967336535454,
+      "step": 410
+    },
+    {
+      "epoch": 0.24720423778693348,
+      "grad_norm": 33.86568434240492,
+      "learning_rate": 1.2360211889346674e-07,
+      "logits/chosen": -2.8926842212677,
+      "logits/rejected": -2.817612648010254,
+      "logps/chosen": -268.4040222167969,
+      "logps/rejected": -255.06655883789062,
+      "loss": 0.6698,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": 0.014465409331023693,
+      "rewards/margins": 0.02410224825143814,
+      "rewards/rejected": -0.009636839851737022,
+      "step": 420
+    },
+    {
+      "epoch": 0.25309005297233667,
+      "grad_norm": 30.66902266053471,
+      "learning_rate": 1.2654502648616834e-07,
+      "logits/chosen": -2.7713985443115234,
+      "logits/rejected": -2.76814341545105,
+      "logps/chosen": -209.8004913330078,
+      "logps/rejected": -219.0537567138672,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": 0.0189804807305336,
+      "rewards/margins": 0.026658236980438232,
+      "rewards/rejected": -0.007677753455936909,
+      "step": 430
+    },
+    {
+      "epoch": 0.2589758681577399,
+      "grad_norm": 37.33663950349735,
+      "learning_rate": 1.2948793407886994e-07,
+      "logits/chosen": -2.762784481048584,
+      "logits/rejected": -2.707094669342041,
+      "logps/chosen": -250.97177124023438,
+      "logps/rejected": -232.8260040283203,
+      "loss": 0.6651,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": 0.03512561693787575,
+      "rewards/margins": 0.09343008697032928,
+      "rewards/rejected": -0.05830447003245354,
+      "step": 440
+    },
+    {
+      "epoch": 0.26486168334314303,
+      "grad_norm": 38.55541735391845,
+      "learning_rate": 1.324308416715715e-07,
+      "logits/chosen": -2.7505548000335693,
+      "logits/rejected": -2.790361166000366,
+      "logps/chosen": -261.1943664550781,
+      "logps/rejected": -230.3916473388672,
+      "loss": 0.6643,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": 0.03301350399851799,
+      "rewards/margins": 0.08106649667024612,
+      "rewards/rejected": -0.04805298522114754,
+      "step": 450
+    },
+    {
+      "epoch": 0.2707474985285462,
+      "grad_norm": 30.70708391983573,
+      "learning_rate": 1.3537374926427308e-07,
+      "logits/chosen": -2.782029390335083,
+      "logits/rejected": -2.710545539855957,
+      "logps/chosen": -238.57211303710938,
+      "logps/rejected": -251.1454315185547,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -0.005198891274631023,
+      "rewards/margins": 0.015401232056319714,
+      "rewards/rejected": -0.020600125193595886,
+      "step": 460
+    },
+    {
+      "epoch": 0.2766333137139494,
+      "grad_norm": 33.06918545223034,
+      "learning_rate": 1.3831665685697468e-07,
+      "logits/chosen": -2.7734148502349854,
+      "logits/rejected": -2.722306728363037,
+      "logps/chosen": -236.19290161132812,
+      "logps/rejected": -218.6464080810547,
+      "loss": 0.6545,
+      "rewards/accuracies": 0.5666666030883789,
+      "rewards/chosen": 0.004786822013556957,
+      "rewards/margins": 0.04201732203364372,
+      "rewards/rejected": -0.03723050281405449,
+      "step": 470
+    },
+    {
+      "epoch": 0.28251912889935255,
+      "grad_norm": 34.67042804510441,
+      "learning_rate": 1.4125956444967625e-07,
+      "logits/chosen": -2.847014904022217,
+      "logits/rejected": -2.759166717529297,
+      "logps/chosen": -291.9137878417969,
+      "logps/rejected": -247.39566040039062,
+      "loss": 0.6612,
+      "rewards/accuracies": 0.46666669845581055,
+      "rewards/chosen": -0.01318571250885725,
+      "rewards/margins": 0.02275732532143593,
+      "rewards/rejected": -0.03594304248690605,
+      "step": 480
+    },
+    {
+      "epoch": 0.28840494408475575,
+      "grad_norm": 41.62165688207727,
+      "learning_rate": 1.4420247204237788e-07,
+      "logits/chosen": -2.8126304149627686,
+      "logits/rejected": -2.8055615425109863,
+      "logps/chosen": -289.50091552734375,
+      "logps/rejected": -277.8962707519531,
+      "loss": 0.6614,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": 0.020298918709158897,
+      "rewards/margins": 0.0497608408331871,
+      "rewards/rejected": -0.02946191094815731,
+      "step": 490
+    },
+    {
+      "epoch": 0.2942907592701589,
+      "grad_norm": 36.97324400383161,
+      "learning_rate": 1.4714537963507945e-07,
+      "logits/chosen": -2.885690212249756,
+      "logits/rejected": -2.8348922729492188,
+      "logps/chosen": -307.3095703125,
+      "logps/rejected": -290.29180908203125,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": 0.024543384090065956,
+      "rewards/margins": 0.09910109639167786,
+      "rewards/rejected": -0.07455770671367645,
+      "step": 500
+    },
+    {
+      "epoch": 0.3001765744555621,
+      "grad_norm": 36.070076135435244,
+      "learning_rate": 1.5008828722778105e-07,
+      "logits/chosen": -2.842216968536377,
+      "logits/rejected": -2.820446491241455,
+      "logps/chosen": -256.1488342285156,
+      "logps/rejected": -247.92672729492188,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.0209733247756958,
+      "rewards/margins": 0.04726904630661011,
+      "rewards/rejected": -0.0682423785328865,
+      "step": 510
+    },
+    {
+      "epoch": 0.30606238964096527,
+      "grad_norm": 42.70438629438365,
+      "learning_rate": 1.5303119482048262e-07,
+      "logits/chosen": -2.6641478538513184,
+      "logits/rejected": -2.66444993019104,
+      "logps/chosen": -259.26348876953125,
+      "logps/rejected": -269.11572265625,
+      "loss": 0.6363,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.010639377869665623,
+      "rewards/margins": 0.09347447007894516,
+      "rewards/rejected": -0.10411383956670761,
+      "step": 520
+    },
+    {
+      "epoch": 0.3119482048263684,
+      "grad_norm": 35.960710299811815,
+      "learning_rate": 1.559741024131842e-07,
+      "logits/chosen": -2.805729866027832,
+      "logits/rejected": -2.7076404094696045,
+      "logps/chosen": -255.88052368164062,
+      "logps/rejected": -234.35415649414062,
+      "loss": 0.6335,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.007552222814410925,
+      "rewards/margins": 0.11313612759113312,
+      "rewards/rejected": -0.12068834155797958,
+      "step": 530
+    },
+    {
+      "epoch": 0.31783402001177163,
+      "grad_norm": 44.82672193345242,
+      "learning_rate": 1.5891701000588582e-07,
+      "logits/chosen": -2.8400981426239014,
+      "logits/rejected": -2.8063666820526123,
+      "logps/chosen": -294.94451904296875,
+      "logps/rejected": -278.1987609863281,
+      "loss": 0.6372,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.08216958492994308,
+      "rewards/margins": 0.09824585914611816,
+      "rewards/rejected": -0.18041543662548065,
+      "step": 540
+    },
+    {
+      "epoch": 0.3237198351971748,
+      "grad_norm": 35.13933484391953,
+      "learning_rate": 1.618599175985874e-07,
+      "logits/chosen": -2.7300307750701904,
+      "logits/rejected": -2.7713351249694824,
+      "logps/chosen": -230.0304718017578,
+      "logps/rejected": -262.08721923828125,
+      "loss": 0.6309,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.08895792812108994,
+      "rewards/margins": 0.13027846813201904,
+      "rewards/rejected": -0.2192363739013672,
+      "step": 550
+    },
+    {
+      "epoch": 0.329605650382578,
+      "grad_norm": 52.54257576269853,
+      "learning_rate": 1.64802825191289e-07,
+      "logits/chosen": -2.753162145614624,
+      "logits/rejected": -2.768341541290283,
+      "logps/chosen": -252.78878784179688,
+      "logps/rejected": -274.1339111328125,
+      "loss": 0.6457,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.1532047688961029,
+      "rewards/margins": 0.1562652289867401,
+      "rewards/rejected": -0.309469997882843,
+      "step": 560
+    },
+    {
+      "epoch": 0.33549146556798115,
+      "grad_norm": 38.27091133745243,
+      "learning_rate": 1.6774573278399057e-07,
+      "logits/chosen": -2.804091215133667,
+      "logits/rejected": -2.776109457015991,
+      "logps/chosen": -287.103271484375,
+      "logps/rejected": -222.2604217529297,
+      "loss": 0.6418,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04693541303277016,
+      "rewards/margins": 0.12523198127746582,
+      "rewards/rejected": -0.17216739058494568,
+      "step": 570
+    },
+    {
+      "epoch": 0.34137728075338436,
+      "grad_norm": 46.61649976864576,
+      "learning_rate": 1.7068864037669217e-07,
+      "logits/chosen": -2.802682399749756,
+      "logits/rejected": -2.7890288829803467,
+      "logps/chosen": -285.7334899902344,
+      "logps/rejected": -294.05181884765625,
+      "loss": 0.6359,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.16137059032917023,
+      "rewards/margins": 0.1200031265616417,
+      "rewards/rejected": -0.2813737392425537,
+      "step": 580
+    },
+    {
+      "epoch": 0.3472630959387875,
+      "grad_norm": 38.73511849094784,
+      "learning_rate": 1.7363154796939374e-07,
+      "logits/chosen": -2.7510268688201904,
+      "logits/rejected": -2.6245083808898926,
+      "logps/chosen": -265.32757568359375,
+      "logps/rejected": -240.0128631591797,
+      "loss": 0.6388,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.2014482468366623,
+      "rewards/margins": 0.0063117193058133125,
+      "rewards/rejected": -0.20775997638702393,
+      "step": 590
+    },
+    {
+      "epoch": 0.3531489111241907,
+      "grad_norm": 33.48738056386425,
+      "learning_rate": 1.7657445556209536e-07,
+      "logits/chosen": -2.8242006301879883,
+      "logits/rejected": -2.828608274459839,
+      "logps/chosen": -249.8431396484375,
+      "logps/rejected": -261.53948974609375,
+      "loss": 0.6199,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.0753629207611084,
+      "rewards/margins": 0.17823639512062073,
+      "rewards/rejected": -0.25359928607940674,
+      "step": 600
+    },
+    {
+      "epoch": 0.3590347263095939,
+      "grad_norm": 44.06827645574314,
+      "learning_rate": 1.7951736315479694e-07,
+      "logits/chosen": -2.652758836746216,
+      "logits/rejected": -2.7120182514190674,
+      "logps/chosen": -283.7317810058594,
+      "logps/rejected": -316.36004638671875,
+      "loss": 0.6189,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2105875015258789,
+      "rewards/margins": 0.14809870719909668,
+      "rewards/rejected": -0.35868626832962036,
+      "step": 610
+    },
+    {
+      "epoch": 0.36492054149499703,
+      "grad_norm": 77.88854058346173,
+      "learning_rate": 1.824602707474985e-07,
+      "logits/chosen": -2.792132616043091,
+      "logits/rejected": -2.735396146774292,
+      "logps/chosen": -308.66094970703125,
+      "logps/rejected": -297.17120361328125,
+      "loss": 0.615,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.38594353199005127,
+      "rewards/margins": 0.13582220673561096,
+      "rewards/rejected": -0.5217657685279846,
+      "step": 620
+    },
+    {
+      "epoch": 0.37080635668040024,
+      "grad_norm": 51.14005626306302,
+      "learning_rate": 1.854031783402001e-07,
+      "logits/chosen": -2.6652121543884277,
+      "logits/rejected": -2.6964430809020996,
+      "logps/chosen": -246.6635284423828,
+      "logps/rejected": -252.4509735107422,
+      "loss": 0.6434,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.09110568463802338,
+      "rewards/margins": 0.1668696105480194,
+      "rewards/rejected": -0.2579752802848816,
+      "step": 630
+    },
+    {
+      "epoch": 0.3766921718658034,
+      "grad_norm": 51.12058774641213,
+      "learning_rate": 1.8834608593290168e-07,
+      "logits/chosen": -2.837921380996704,
+      "logits/rejected": -2.8364148139953613,
+      "logps/chosen": -307.8941955566406,
+      "logps/rejected": -311.6710205078125,
+      "loss": 0.6426,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": -0.19721655547618866,
+      "rewards/margins": 0.042520731687545776,
+      "rewards/rejected": -0.23973730206489563,
+      "step": 640
+    },
+    {
+      "epoch": 0.3825779870512066,
+      "grad_norm": 55.28695481091037,
+      "learning_rate": 1.9128899352560328e-07,
+      "logits/chosen": -2.7750084400177,
+      "logits/rejected": -2.816450595855713,
+      "logps/chosen": -338.82080078125,
+      "logps/rejected": -358.87921142578125,
+      "loss": 0.6245,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.21482959389686584,
+      "rewards/margins": 0.2418464869260788,
+      "rewards/rejected": -0.45667609572410583,
+      "step": 650
+    },
+    {
+      "epoch": 0.38846380223660976,
+      "grad_norm": 60.1584611934493,
+      "learning_rate": 1.9423190111830488e-07,
+      "logits/chosen": -2.808283805847168,
+      "logits/rejected": -2.733508586883545,
+      "logps/chosen": -268.6055908203125,
+      "logps/rejected": -279.4296569824219,
+      "loss": 0.6244,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -0.24825648963451385,
+      "rewards/margins": 0.07473016530275345,
+      "rewards/rejected": -0.3229866623878479,
+      "step": 660
+    },
+    {
+      "epoch": 0.39434961742201297,
+      "grad_norm": 38.16884117276723,
+      "learning_rate": 1.9717480871100648e-07,
+      "logits/chosen": -2.763615131378174,
+      "logits/rejected": -2.742448329925537,
+      "logps/chosen": -214.5966796875,
+      "logps/rejected": -293.02313232421875,
+      "loss": 0.6017,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1877942830324173,
+      "rewards/margins": 0.23820248246192932,
+      "rewards/rejected": -0.42599672079086304,
+      "step": 670
+    },
+    {
+      "epoch": 0.4002354326074161,
+      "grad_norm": 40.282667153165065,
+      "learning_rate": 2.0011771630370805e-07,
+      "logits/chosen": -2.7557873725891113,
+      "logits/rejected": -2.777015209197998,
+      "logps/chosen": -384.47186279296875,
+      "logps/rejected": -303.30877685546875,
+      "loss": 0.6045,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.15859545767307281,
+      "rewards/margins": 0.22654417157173157,
+      "rewards/rejected": -0.3851396441459656,
+      "step": 680
+    },
+    {
+      "epoch": 0.40612124779281933,
+      "grad_norm": 50.42522303402139,
+      "learning_rate": 2.0306062389640965e-07,
+      "logits/chosen": -2.7922425270080566,
+      "logits/rejected": -2.707688093185425,
+      "logps/chosen": -314.8849792480469,
+      "logps/rejected": -246.93618774414062,
+      "loss": 0.6023,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.20941126346588135,
+      "rewards/margins": 0.2189422845840454,
+      "rewards/rejected": -0.42835354804992676,
+      "step": 690
+    },
+    {
+      "epoch": 0.4120070629782225,
+      "grad_norm": 86.59616940333294,
+      "learning_rate": 2.0600353148911122e-07,
+      "logits/chosen": -2.7496159076690674,
+      "logits/rejected": -2.7615320682525635,
+      "logps/chosen": -316.3775939941406,
+      "logps/rejected": -316.27752685546875,
+      "loss": 0.6312,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.4660351276397705,
+      "rewards/margins": 0.19507920742034912,
+      "rewards/rejected": -0.6611142754554749,
+      "step": 700
+    },
+    {
+      "epoch": 0.41789287816362564,
+      "grad_norm": 113.88657488515956,
+      "learning_rate": 3.61272271440356e-07,
+      "logits/chosen": -2.713832139968872,
+      "logits/rejected": -2.639629364013672,
+      "logps/chosen": -292.7522277832031,
+      "logps/rejected": -276.9837951660156,
+      "loss": 0.6111,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.42747750878334045,
+      "rewards/margins": 0.30797505378723145,
+      "rewards/rejected": -0.7354525923728943,
+      "step": 710
+    },
+    {
+      "epoch": 0.42377869334902885,
+      "grad_norm": 58.276303041068935,
+      "learning_rate": 3.5664928281860625e-07,
+      "logits/chosen": -2.7681069374084473,
+      "logits/rejected": -2.6781115531921387,
+      "logps/chosen": -293.0718078613281,
+      "logps/rejected": -281.61212158203125,
+      "loss": 0.5922,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3552438020706177,
+      "rewards/margins": 0.20946629345417023,
+      "rewards/rejected": -0.5647100806236267,
+      "step": 720
+    },
+    {
+      "epoch": 0.429664508534432,
+      "grad_norm": 34.26366533936239,
+      "learning_rate": 3.5198127192199e-07,
+      "logits/chosen": -2.7726988792419434,
+      "logits/rejected": -2.7980246543884277,
+      "logps/chosen": -358.5955810546875,
+      "logps/rejected": -417.20281982421875,
+      "loss": 0.5593,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -0.27696719765663147,
+      "rewards/margins": 0.5634738802909851,
+      "rewards/rejected": -0.8404410481452942,
+      "step": 730
+    },
+    {
+      "epoch": 0.4355503237198352,
+      "grad_norm": 48.29459945401339,
+      "learning_rate": 3.472702093635682e-07,
+      "logits/chosen": -2.822880268096924,
+      "logits/rejected": -2.754981279373169,
+      "logps/chosen": -232.97921752929688,
+      "logps/rejected": -225.82296752929688,
+      "loss": 0.6444,
+      "rewards/accuracies": 0.5333333015441895,
+      "rewards/chosen": -0.568128228187561,
+      "rewards/margins": -0.030178595334291458,
+      "rewards/rejected": -0.5379496216773987,
+      "step": 740
+    },
+    {
+      "epoch": 0.44143613890523836,
+      "grad_norm": 88.73737377845674,
+      "learning_rate": 3.42518083930774e-07,
+      "logits/chosen": -2.6345629692077637,
+      "logits/rejected": -2.6091387271881104,
+      "logps/chosen": -272.2128601074219,
+      "logps/rejected": -344.5741882324219,
+      "loss": 0.6375,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.46700435876846313,
+      "rewards/margins": 0.42694124579429626,
+      "rewards/rejected": -0.8939456939697266,
+      "step": 750
+    },
+    {
+      "epoch": 0.44732195409064157,
+      "grad_norm": 74.5070307164555,
+      "learning_rate": 3.377269017458409e-07,
+      "logits/chosen": -2.767118453979492,
+      "logits/rejected": -2.7667033672332764,
+      "logps/chosen": -392.59454345703125,
+      "logps/rejected": -410.0496520996094,
+      "loss": 0.5401,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.3810432553291321,
+      "rewards/margins": 0.6425568461418152,
+      "rewards/rejected": -1.0236002206802368,
+      "step": 760
+    },
+    {
+      "epoch": 0.4532077692760447,
+      "grad_norm": 102.79976551674226,
+      "learning_rate": 3.3289868541891255e-07,
+      "logits/chosen": -2.604182720184326,
+      "logits/rejected": -2.527705430984497,
+      "logps/chosen": -296.3623046875,
+      "logps/rejected": -354.95819091796875,
+      "loss": 0.6152,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -0.5251516103744507,
+      "rewards/margins": 0.46194180846214294,
+      "rewards/rejected": -0.9870933294296265,
+      "step": 770
+    },
+    {
+      "epoch": 0.45909358446144793,
+      "grad_norm": 67.73780235887719,
+      "learning_rate": 3.280354731941932e-07,
+      "logits/chosen": -2.76226544380188,
+      "logits/rejected": -2.6936070919036865,
+      "logps/chosen": -315.97216796875,
+      "logps/rejected": -342.60546875,
+      "loss": 0.5639,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.4423181116580963,
+      "rewards/margins": 0.47899603843688965,
+      "rewards/rejected": -0.9213140606880188,
+      "step": 780
+    },
+    {
+      "epoch": 0.4649793996468511,
+      "grad_norm": 38.93430396007463,
+      "learning_rate": 3.231393180894973e-07,
+      "logits/chosen": -2.795037031173706,
+      "logits/rejected": -2.7878684997558594,
+      "logps/chosen": -255.4384765625,
+      "logps/rejected": -313.3453369140625,
+      "loss": 0.5453,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.3891940712928772,
+      "rewards/margins": 0.5739094614982605,
+      "rewards/rejected": -0.9631035923957825,
+      "step": 790
+    },
+    {
+      "epoch": 0.47086521483225424,
+      "grad_norm": 78.21647434951635,
+      "learning_rate": 3.1821228702956253e-07,
+      "logits/chosen": -2.5857443809509277,
+      "logits/rejected": -2.5272181034088135,
+      "logps/chosen": -269.7416076660156,
+      "logps/rejected": -359.6760559082031,
+      "loss": 0.6535,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.6391388773918152,
+      "rewards/margins": 0.2941351532936096,
+      "rewards/rejected": -0.9332740902900696,
+      "step": 800
+    },
+    {
+      "epoch": 0.47675103001765745,
+      "grad_norm": 59.73596890978745,
+      "learning_rate": 3.132564599734921e-07,
+      "logits/chosen": -2.7386457920074463,
+      "logits/rejected": -2.754356861114502,
+      "logps/chosen": -292.2206726074219,
+      "logps/rejected": -324.42816162109375,
+      "loss": 0.6368,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.3809540867805481,
+      "rewards/margins": 0.2144467830657959,
+      "rewards/rejected": -0.5954009294509888,
+      "step": 810
+    },
+    {
+      "epoch": 0.4826368452030606,
+      "grad_norm": 58.04453100186318,
+      "learning_rate": 3.082739290366946e-07,
+      "logits/chosen": -2.731048107147217,
+      "logits/rejected": -2.7188940048217773,
+      "logps/chosen": -367.1738586425781,
+      "logps/rejected": -365.79803466796875,
+      "loss": 0.5855,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.40680187940597534,
+      "rewards/margins": 0.5839999318122864,
+      "rewards/rejected": -0.9908016920089722,
+      "step": 820
+    },
+    {
+      "epoch": 0.4885226603884638,
+      "grad_norm": 60.703085038442914,
+      "learning_rate": 3.0326679760769226e-07,
+      "logits/chosen": -2.633038282394409,
+      "logits/rejected": -2.6379826068878174,
+      "logps/chosen": -270.77972412109375,
+      "logps/rejected": -361.93170166015625,
+      "loss": 0.6076,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -0.29258260130882263,
+      "rewards/margins": 0.578423798084259,
+      "rewards/rejected": -0.8710063695907593,
+      "step": 830
+    },
+    {
+      "epoch": 0.49440847557386697,
+      "grad_norm": 68.79606387966484,
+      "learning_rate": 2.982371794601694e-07,
+      "logits/chosen": -2.7220444679260254,
+      "logits/rejected": -2.665432929992676,
+      "logps/chosen": -456.65557861328125,
+      "logps/rejected": -338.92926025390625,
+      "loss": 0.6101,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.38200992345809937,
+      "rewards/margins": 0.4571179747581482,
+      "rewards/rejected": -0.8391278386116028,
+      "step": 840
+    },
+    {
+      "epoch": 0.5002942907592701,
+      "grad_norm": 39.349812488010386,
+      "learning_rate": 2.931871978606379e-07,
+      "logits/chosen": -2.7449920177459717,
+      "logits/rejected": -2.785231590270996,
+      "logps/chosen": -353.2920837402344,
+      "logps/rejected": -434.22979736328125,
+      "loss": 0.5017,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.3858256936073303,
+      "rewards/margins": 0.6933655738830566,
+      "rewards/rejected": -1.0791910886764526,
+      "step": 850
+    },
+    {
+      "epoch": 0.5061801059446733,
+      "grad_norm": 65.24381800745265,
+      "learning_rate": 2.881189846720941e-07,
+      "logits/chosen": -2.7466366291046143,
+      "logits/rejected": -2.7046473026275635,
+      "logps/chosen": -334.4920959472656,
+      "logps/rejected": -363.40838623046875,
+      "loss": 0.5525,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.41190671920776367,
+      "rewards/margins": 0.559307873249054,
+      "rewards/rejected": -0.9712145924568176,
+      "step": 860
+    },
+    {
+      "epoch": 0.5120659211300765,
+      "grad_norm": 51.09800858702452,
+      "learning_rate": 2.830346794540479e-07,
+      "logits/chosen": -2.6742959022521973,
+      "logits/rejected": -2.6640892028808594,
+      "logps/chosen": -266.5992736816406,
+      "logps/rejected": -313.98828125,
+      "loss": 0.5778,
+      "rewards/accuracies": 0.4999999403953552,
+      "rewards/chosen": -0.6418452262878418,
+      "rewards/margins": 0.16775313019752502,
+      "rewards/rejected": -0.8095983266830444,
+      "step": 870
+    },
+    {
+      "epoch": 0.5179517363154797,
+      "grad_norm": 54.983291652847775,
+      "learning_rate": 2.7793642855930133e-07,
+      "logits/chosen": -2.7972800731658936,
+      "logits/rejected": -2.657902240753174,
+      "logps/chosen": -379.5608825683594,
+      "logps/rejected": -415.2431640625,
+      "loss": 0.6336,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -0.5884305834770203,
+      "rewards/margins": 0.4619186818599701,
+      "rewards/rejected": -1.0503493547439575,
+      "step": 880
+    },
+    {
+      "epoch": 0.5238375515008828,
+      "grad_norm": 52.1793743861904,
+      "learning_rate": 2.728263842278608e-07,
+      "logits/chosen": -2.7541468143463135,
+      "logits/rejected": -2.7553248405456543,
+      "logps/chosen": -340.1725158691406,
+      "logps/rejected": -431.4141540527344,
+      "loss": 0.5083,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -0.5956707000732422,
+      "rewards/margins": 0.6665983200073242,
+      "rewards/rejected": -1.2622690200805664,
+      "step": 890
+    },
+    {
+      "epoch": 0.5297233666862861,
+      "grad_norm": 62.43287153719138,
+      "learning_rate": 2.6770670367836276e-07,
+      "logits/chosen": -2.731095790863037,
+      "logits/rejected": -2.734687328338623,
+      "logps/chosen": -333.5868835449219,
+      "logps/rejected": -405.0290222167969,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -0.7237066030502319,
+      "rewards/margins": 0.5302045941352844,
+      "rewards/rejected": -1.2539112567901611,
+      "step": 900
+    },
+    {
+      "epoch": 0.5356091818716893,
+      "grad_norm": 82.01285085767707,
+      "learning_rate": 2.625795481973985e-07,
+      "logits/chosen": -2.6702892780303955,
+      "logits/rejected": -2.5133378505706787,
+      "logps/chosen": -338.619873046875,
+      "logps/rejected": -311.3938903808594,
+      "loss": 0.5897,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.5718550086021423,
+      "rewards/margins": 0.4375828206539154,
+      "rewards/rejected": -1.0094377994537354,
+      "step": 910
+    },
+    {
+      "epoch": 0.5414949970570924,
+      "grad_norm": 95.54109435400643,
+      "learning_rate": 2.5744708222712137e-07,
+      "logits/chosen": -2.7311172485351562,
+      "logits/rejected": -2.739924907684326,
+      "logps/chosen": -272.01995849609375,
+      "logps/rejected": -311.47833251953125,
+      "loss": 0.5762,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5505859851837158,
+      "rewards/margins": 0.6296683549880981,
+      "rewards/rejected": -1.180254340171814,
+      "step": 920
+    },
+    {
+      "epoch": 0.5473808122424956,
+      "grad_norm": 36.48587725082776,
+      "learning_rate": 2.5231147245152156e-07,
+      "logits/chosen": -2.4503333568573,
+      "logits/rejected": -2.468564033508301,
+      "logps/chosen": -298.019775390625,
+      "logps/rejected": -372.83026123046875,
+      "loss": 0.5587,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.6832865476608276,
+      "rewards/margins": 0.8449433445930481,
+      "rewards/rejected": -1.5282299518585205,
+      "step": 930
+    },
+    {
+      "epoch": 0.5532666274278988,
+      "grad_norm": 71.24789616161829,
+      "learning_rate": 2.471748868817551e-07,
+      "logits/chosen": -2.7247791290283203,
+      "logits/rejected": -2.7448885440826416,
+      "logps/chosen": -316.38726806640625,
+      "logps/rejected": -418.75830078125,
+      "loss": 0.5429,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.43857964873313904,
+      "rewards/margins": 0.8119268417358398,
+      "rewards/rejected": -1.2505066394805908,
+      "step": 940
+    },
+    {
+      "epoch": 0.559152442613302,
+      "grad_norm": 57.37226759319327,
+      "learning_rate": 2.4203949394091186e-07,
+      "logits/chosen": -2.7133288383483887,
+      "logits/rejected": -2.6669368743896484,
+      "logps/chosen": -384.8581848144531,
+      "logps/rejected": -477.08538818359375,
+      "loss": 0.5152,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -0.9175896644592285,
+      "rewards/margins": 1.0064847469329834,
+      "rewards/rejected": -1.9240741729736328,
+      "step": 950
+    },
+    {
+      "epoch": 0.5650382577987051,
+      "grad_norm": 102.83287628895852,
+      "learning_rate": 2.3690746154861048e-07,
+      "logits/chosen": -2.5697152614593506,
+      "logits/rejected": -2.5663580894470215,
+      "logps/chosen": -361.67926025390625,
+      "logps/rejected": -425.91571044921875,
+      "loss": 0.5645,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.9139325022697449,
+      "rewards/margins": 0.5984528064727783,
+      "rewards/rejected": -1.5123852491378784,
+      "step": 960
+    },
+    {
+      "epoch": 0.5709240729841083,
+      "grad_norm": 103.97188710606014,
+      "learning_rate": 2.3178095620580524e-07,
+      "logits/chosen": -2.642576217651367,
+      "logits/rejected": -2.576850175857544,
+      "logps/chosen": -394.79290771484375,
+      "logps/rejected": -357.9416809082031,
+      "loss": 0.628,
+      "rewards/accuracies": 0.5999999642372131,
+      "rewards/chosen": -1.2030242681503296,
+      "rewards/margins": 0.2695932388305664,
+      "rewards/rejected": -1.4726173877716064,
+      "step": 970
+    },
+    {
+      "epoch": 0.5768098881695115,
+      "grad_norm": 122.23429828784404,
+      "learning_rate": 2.266621420801918e-07,
+      "logits/chosen": -2.3418798446655273,
+      "logits/rejected": -2.3062217235565186,
+      "logps/chosen": -334.18182373046875,
+      "logps/rejected": -375.04083251953125,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.9751673936843872,
+      "rewards/margins": 0.36949509382247925,
+      "rewards/rejected": -1.3446625471115112,
+      "step": 980
+    },
+    {
+      "epoch": 0.5826957033549146,
+      "grad_norm": 53.90742131111234,
+      "learning_rate": 2.2155318009259821e-07,
+      "logits/chosen": -2.517277479171753,
+      "logits/rejected": -2.552461862564087,
+      "logps/chosen": -351.4830322265625,
+      "logps/rejected": -347.33770751953125,
+      "loss": 0.6017,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.7861822843551636,
+      "rewards/margins": 0.5186395645141602,
+      "rewards/rejected": -1.3048217296600342,
+      "step": 990
+    },
+    {
+      "epoch": 0.5885815185403178,
+      "grad_norm": 79.56155835501555,
+      "learning_rate": 2.1645622700474695e-07,
+      "logits/chosen": -2.53051495552063,
+      "logits/rejected": -2.50981068611145,
+      "logps/chosen": -300.99261474609375,
+      "logps/rejected": -307.985107421875,
+      "loss": 0.6313,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9508684873580933,
+      "rewards/margins": 0.4335688054561615,
+      "rewards/rejected": -1.3844374418258667,
+      "step": 1000
+    },
+    {
+      "epoch": 0.594467333725721,
+      "grad_norm": 178.75847915531864,
+      "learning_rate": 2.113734345087716e-07,
+      "logits/chosen": -2.699451208114624,
+      "logits/rejected": -2.7512571811676025,
+      "logps/chosen": -213.2908172607422,
+      "logps/rejected": -347.00457763671875,
+      "loss": 0.4959,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -0.5839646458625793,
+      "rewards/margins": 0.5955744385719299,
+      "rewards/rejected": -1.1795389652252197,
+      "step": 1010
+    },
+    {
+      "epoch": 0.6003531489111242,
+      "grad_norm": 144.9877689912697,
+      "learning_rate": 2.0630694831887473e-07,
+      "logits/chosen": -2.3605151176452637,
+      "logits/rejected": -2.377347469329834,
+      "logps/chosen": -302.31072998046875,
+      "logps/rejected": -364.3263854980469,
+      "loss": 0.5393,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.8714925646781921,
+      "rewards/margins": 0.615392804145813,
+      "rewards/rejected": -1.4868853092193604,
+      "step": 1020
+    },
+    {
+      "epoch": 0.6062389640965273,
+      "grad_norm": 90.93107910277243,
+      "learning_rate": 2.0125890726550896e-07,
+      "logits/chosen": -2.774040937423706,
+      "logits/rejected": -2.749173641204834,
+      "logps/chosen": -312.368896484375,
+      "logps/rejected": -329.6822814941406,
+      "loss": 0.5044,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -0.3795444369316101,
+      "rewards/margins": 0.5973297357559204,
+      "rewards/rejected": -0.9768741726875305,
+      "step": 1030
+    },
+    {
+      "epoch": 0.6121247792819305,
+      "grad_norm": 53.759522144352374,
+      "learning_rate": 1.9623144239246432e-07,
+      "logits/chosen": -2.555093765258789,
+      "logits/rejected": -2.572145700454712,
+      "logps/chosen": -320.54248046875,
+      "logps/rejected": -383.291748046875,
+      "loss": 0.5761,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7120213508605957,
+      "rewards/margins": 0.4912422299385071,
+      "rewards/rejected": -1.203263521194458,
+      "step": 1040
+    },
+    {
+      "epoch": 0.6180105944673338,
+      "grad_norm": 96.22956917547782,
+      "learning_rate": 1.91226676057242e-07,
+      "logits/chosen": -2.5407838821411133,
+      "logits/rejected": -2.471928596496582,
+      "logps/chosen": -419.9786071777344,
+      "logps/rejected": -386.1612854003906,
+      "loss": 0.5582,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.8196290135383606,
+      "rewards/margins": 0.6197575330734253,
+      "rewards/rejected": -1.4393866062164307,
+      "step": 1050
+    },
+    {
+      "epoch": 0.6238964096527368,
+      "grad_norm": 84.05938732777776,
+      "learning_rate": 1.8624672103509578e-07,
+      "logits/chosen": -2.4780263900756836,
+      "logits/rejected": -2.3386425971984863,
+      "logps/chosen": -417.42724609375,
+      "logps/rejected": -414.38262939453125,
+      "loss": 0.5922,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0109527111053467,
+      "rewards/margins": 0.40876656770706177,
+      "rewards/rejected": -1.4197193384170532,
+      "step": 1060
+    },
+    {
+      "epoch": 0.6297822248381401,
+      "grad_norm": 125.25073964918506,
+      "learning_rate": 1.8129367962711882e-07,
+      "logits/chosen": -2.6754302978515625,
+      "logits/rejected": -2.6922547817230225,
+      "logps/chosen": -416.434326171875,
+      "logps/rejected": -419.4442443847656,
+      "loss": 0.5897,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8944675326347351,
+      "rewards/margins": 0.5295539498329163,
+      "rewards/rejected": -1.4240214824676514,
+      "step": 1070
+    },
+    {
+      "epoch": 0.6356680400235433,
+      "grad_norm": 87.45258473406294,
+      "learning_rate": 1.7636964277275108e-07,
+      "logits/chosen": -2.5234591960906982,
+      "logits/rejected": -2.5611681938171387,
+      "logps/chosen": -326.291015625,
+      "logps/rejected": -389.78839111328125,
+      "loss": 0.4929,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -0.7259222269058228,
+      "rewards/margins": 0.8374055624008179,
+      "rewards/rejected": -1.5633279085159302,
+      "step": 1080
+    },
+    {
+      "epoch": 0.6415538552089465,
+      "grad_norm": 53.279206385221514,
+      "learning_rate": 1.7147668916708398e-07,
+      "logits/chosen": -2.480179786682129,
+      "logits/rejected": -2.4370687007904053,
+      "logps/chosen": -358.24444580078125,
+      "logps/rejected": -384.9347229003906,
+      "loss": 0.5857,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.065148115158081,
+      "rewards/margins": 0.38636088371276855,
+      "rewards/rejected": -1.4515092372894287,
+      "step": 1090
+    },
+    {
+      "epoch": 0.6474396703943496,
+      "grad_norm": 52.091641562005016,
+      "learning_rate": 1.666168843833339e-07,
+      "logits/chosen": -2.3647563457489014,
+      "logits/rejected": -2.3754029273986816,
+      "logps/chosen": -263.02447509765625,
+      "logps/rejected": -354.109619140625,
+      "loss": 0.5574,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -0.5551760792732239,
+      "rewards/margins": 0.844565749168396,
+      "rewards/rejected": -1.399741768836975,
+      "step": 1100
+    },
+    {
+      "epoch": 0.6533254855797528,
+      "grad_norm": 112.76354894611299,
+      "learning_rate": 1.6179228000085532e-07,
+      "logits/chosen": -2.617955446243286,
+      "logits/rejected": -2.5767765045166016,
+      "logps/chosen": -301.1888122558594,
+      "logps/rejected": -372.39239501953125,
+      "loss": 0.6259,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.8070073127746582,
+      "rewards/margins": 0.5521813035011292,
+      "rewards/rejected": -1.3591885566711426,
+      "step": 1110
+    },
+    {
+      "epoch": 0.659211300765156,
+      "grad_norm": 72.36752468174844,
+      "learning_rate": 1.570049127390604e-07,
+      "logits/chosen": -2.607243299484253,
+      "logits/rejected": -2.6284470558166504,
+      "logps/chosen": -382.6817321777344,
+      "logps/rejected": -473.088134765625,
+      "loss": 0.5981,
+      "rewards/accuracies": 0.5999999642372131,
+      "rewards/chosen": -0.9492104649543762,
+      "rewards/margins": 0.6012105941772461,
+      "rewards/rejected": -1.5504209995269775,
+      "step": 1120
+    },
+    {
+      "epoch": 0.6650971159505592,
+      "grad_norm": 99.38680104764003,
+      "learning_rate": 1.5225680359761325e-07,
+      "logits/chosen": -2.555722713470459,
+      "logits/rejected": -2.5732975006103516,
+      "logps/chosen": -235.4317169189453,
+      "logps/rejected": -325.0386962890625,
+      "loss": 0.5918,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.6323750019073486,
+      "rewards/margins": 0.4906527101993561,
+      "rewards/rejected": -1.1230276823043823,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6709829311359623,
+      "grad_norm": 113.18996714616782,
+      "learning_rate": 1.4754995700325917e-07,
+      "logits/chosen": -2.628831624984741,
+      "logits/rejected": -2.61437726020813,
+      "logps/chosen": -304.4407043457031,
+      "logps/rejected": -390.73321533203125,
+      "loss": 0.5161,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -0.7466219067573547,
+      "rewards/margins": 0.5649534463882446,
+      "rewards/rejected": -1.3115752935409546,
+      "step": 1140
+    },
+    {
+      "epoch": 0.6768687463213655,
+      "grad_norm": 213.32924667526723,
+      "learning_rate": 1.428863599636511e-07,
+      "logits/chosen": -2.4004693031311035,
+      "logits/rejected": -2.5256259441375732,
+      "logps/chosen": -326.31280517578125,
+      "logps/rejected": -392.92529296875,
+      "loss": 0.4886,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": -0.8898285627365112,
+      "rewards/margins": 0.6658565998077393,
+      "rewards/rejected": -1.5556851625442505,
+      "step": 1150
+    },
+    {
+      "epoch": 0.6827545615067687,
+      "grad_norm": 169.10265364288662,
+      "learning_rate": 1.3826798122852868e-07,
+      "logits/chosen": -2.6862120628356934,
+      "logits/rejected": -2.5977320671081543,
+      "logps/chosen": -360.7554016113281,
+      "logps/rejected": -339.32513427734375,
+      "loss": 0.5558,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9063922166824341,
+      "rewards/margins": 0.3310279846191406,
+      "rewards/rejected": -1.2374203205108643,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6886403766921718,
+      "grad_norm": 173.65784239229404,
+      "learning_rate": 1.3369677045860544e-07,
+      "logits/chosen": -2.625609874725342,
+      "logits/rejected": -2.5897438526153564,
+      "logps/chosen": -319.73748779296875,
+      "logps/rejected": -373.0987854003906,
+      "loss": 0.4981,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -0.6690064668655396,
+      "rewards/margins": 0.8604344129562378,
+      "rewards/rejected": -1.5294408798217773,
+      "step": 1170
+    },
+    {
+      "epoch": 0.694526191877575,
+      "grad_norm": 124.29493358693395,
+      "learning_rate": 1.291746574025148e-07,
+      "logits/chosen": -2.736769914627075,
+      "logits/rejected": -2.7117438316345215,
+      "logps/chosen": -440.48779296875,
+      "logps/rejected": -379.79132080078125,
+      "loss": 0.5748,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.1783974170684814,
+      "rewards/margins": 0.2958952486515045,
+      "rewards/rejected": -1.4742927551269531,
+      "step": 1180
+    },
+    {
+      "epoch": 0.7004120070629782,
+      "grad_norm": 78.3006038615153,
+      "learning_rate": 1.247035510821611e-07,
+      "logits/chosen": -2.7176737785339355,
+      "logits/rejected": -2.6454663276672363,
+      "logps/chosen": -376.97381591796875,
+      "logps/rejected": -335.2852783203125,
+      "loss": 0.5881,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9591771960258484,
+      "rewards/margins": 0.3564377725124359,
+      "rewards/rejected": -1.315614938735962,
+      "step": 1190
+    },
+    {
+      "epoch": 0.7062978222483814,
+      "grad_norm": 173.0463100342363,
+      "learning_rate": 1.2028533898682123e-07,
+      "logits/chosen": -2.7012906074523926,
+      "logits/rejected": -2.6718831062316895,
+      "logps/chosen": -352.58294677734375,
+      "logps/rejected": -393.17572021484375,
+      "loss": 0.5845,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.8266950845718384,
+      "rewards/margins": 0.6615980863571167,
+      "rewards/rejected": -1.4882930517196655,
+      "step": 1200
+    },
+    {
+      "epoch": 0.7121836374337845,
+      "grad_norm": 80.48548009967331,
+      "learning_rate": 1.1592188627633598e-07,
+      "logits/chosen": -2.583117961883545,
+      "logits/rejected": -2.6228084564208984,
+      "logps/chosen": -317.26708984375,
+      "logps/rejected": -329.43719482421875,
+      "loss": 0.632,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.8232676386833191,
+      "rewards/margins": 0.3029004633426666,
+      "rewards/rejected": -1.1261680126190186,
+      "step": 1210
+    },
+    {
+      "epoch": 0.7180694526191878,
+      "grad_norm": 62.307227511632156,
+      "learning_rate": 1.1161503499372835e-07,
+      "logits/chosen": -2.5381112098693848,
+      "logits/rejected": -2.426206350326538,
+      "logps/chosen": -356.4806823730469,
+      "logps/rejected": -495.2960510253906,
+      "loss": 0.4735,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.9427182078361511,
+      "rewards/margins": 0.6752321124076843,
+      "rewards/rejected": -1.617950439453125,
+      "step": 1220
+    },
+    {
+      "epoch": 0.723955267804591,
+      "grad_norm": 102.88342581327655,
+      "learning_rate": 1.0736660328757954e-07,
+      "logits/chosen": -2.63267183303833,
+      "logits/rejected": -2.4871132373809814,
+      "logps/chosen": -303.36163330078125,
+      "logps/rejected": -332.2799987792969,
+      "loss": 0.5103,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.6926143765449524,
+      "rewards/margins": 0.48562532663345337,
+      "rewards/rejected": -1.1782395839691162,
+      "step": 1230
+    },
+    {
+      "epoch": 0.7298410829899941,
+      "grad_norm": 69.73162807423068,
+      "learning_rate": 1.0317838464449318e-07,
+      "logits/chosen": -2.5889840126037598,
+      "logits/rejected": -2.594730854034424,
+      "logps/chosen": -305.60546875,
+      "logps/rejected": -347.56988525390625,
+      "loss": 0.5767,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6716678142547607,
+      "rewards/margins": 0.4861713945865631,
+      "rewards/rejected": -1.1578391790390015,
+      "step": 1240
+    },
+    {
+      "epoch": 0.7357268981753973,
+      "grad_norm": 99.72345975810403,
+      "learning_rate": 9.905214713197019e-08,
+      "logits/chosen": -2.780867338180542,
+      "logits/rejected": -2.7212603092193604,
+      "logps/chosen": -319.85809326171875,
+      "logps/rejected": -362.9429931640625,
+      "loss": 0.5763,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.7506597638130188,
+      "rewards/margins": 0.7295576333999634,
+      "rewards/rejected": -1.480217456817627,
+      "step": 1250
+    },
+    {
+      "epoch": 0.7416127133608005,
+      "grad_norm": 42.256039163587936,
+      "learning_rate": 9.498963265201454e-08,
+      "logits/chosen": -2.5502285957336426,
+      "logits/rejected": -2.4693973064422607,
+      "logps/chosen": -301.47845458984375,
+      "logps/rejected": -348.4825439453125,
+      "loss": 0.5623,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.7193578481674194,
+      "rewards/margins": 0.40803733468055725,
+      "rewards/rejected": -1.1273951530456543,
+      "step": 1260
+    },
+    {
+      "epoch": 0.7474985285462037,
+      "grad_norm": 51.960392917513985,
+      "learning_rate": 9.09925562057845e-08,
+      "logits/chosen": -2.5758213996887207,
+      "logits/rejected": -2.6676766872406006,
+      "logps/chosen": -319.75018310546875,
+      "logps/rejected": -392.013916015625,
+      "loss": 0.5471,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.8297551870346069,
+      "rewards/margins": 0.583861231803894,
+      "rewards/rejected": -1.4136162996292114,
+      "step": 1270
+    },
+    {
+      "epoch": 0.7533843437316068,
+      "grad_norm": 96.3763472702637,
+      "learning_rate": 8.706260516960043e-08,
+      "logits/chosen": -2.6656923294067383,
+      "logits/rejected": -2.706164836883545,
+      "logps/chosen": -342.38623046875,
+      "logps/rejected": -424.30706787109375,
+      "loss": 0.6335,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.1196720600128174,
+      "rewards/margins": 0.4281339645385742,
+      "rewards/rejected": -1.5478060245513916,
+      "step": 1280
+    },
+    {
+      "epoch": 0.75927015891701,
+      "grad_norm": 82.6639251391733,
+      "learning_rate": 8.320143858261486e-08,
+      "logits/chosen": -2.746375560760498,
+      "logits/rejected": -2.6994574069976807,
+      "logps/chosen": -360.11285400390625,
+      "logps/rejected": -339.80926513671875,
+      "loss": 0.5403,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.801996111869812,
+      "rewards/margins": 0.3962213695049286,
+      "rewards/rejected": -1.1982173919677734,
+      "step": 1290
+    },
+    {
+      "epoch": 0.7651559741024132,
+      "grad_norm": 58.83037988316151,
+      "learning_rate": 7.941068644644405e-08,
+      "logits/chosen": -2.7802703380584717,
+      "logits/rejected": -2.643925428390503,
+      "logps/chosen": -391.9327697753906,
+      "logps/rejected": -413.474609375,
+      "loss": 0.5632,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.1195310354232788,
+      "rewards/margins": 0.615394115447998,
+      "rewards/rejected": -1.7349250316619873,
+      "step": 1300
+    },
+    {
+      "epoch": 0.7710417892878164,
+      "grad_norm": 73.64616872899245,
+      "learning_rate": 7.569194903705913e-08,
+      "logits/chosen": -2.7224087715148926,
+      "logits/rejected": -2.751784324645996,
+      "logps/chosen": -406.634521484375,
+      "logps/rejected": -462.2149963378906,
+      "loss": 0.5761,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.9177020788192749,
+      "rewards/margins": 0.5259424448013306,
+      "rewards/rejected": -1.4436447620391846,
+      "step": 1310
+    },
+    {
+      "epoch": 0.7769276044732195,
+      "grad_norm": 226.35543077516525,
+      "learning_rate": 7.204679622922502e-08,
+      "logits/chosen": -2.6812875270843506,
+      "logits/rejected": -2.663227081298828,
+      "logps/chosen": -432.437744140625,
+      "logps/rejected": -406.0752258300781,
+      "loss": 0.6171,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.8880939483642578,
+      "rewards/margins": 0.6238939166069031,
+      "rewards/rejected": -1.5119879245758057,
+      "step": 1320
+    },
+    {
+      "epoch": 0.7828134196586227,
+      "grad_norm": 52.032206635043295,
+      "learning_rate": 6.847676683377398e-08,
+      "logits/chosen": -2.6559484004974365,
+      "logits/rejected": -2.5057730674743652,
+      "logps/chosen": -473.78338623046875,
+      "logps/rejected": -408.1139221191406,
+      "loss": 0.5067,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.8587116003036499,
+      "rewards/margins": 0.6869573593139648,
+      "rewards/rejected": -1.5456688404083252,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7886992348440259,
+      "grad_norm": 313.43589076362645,
+      "learning_rate": 6.498336794799234e-08,
+      "logits/chosen": -2.5412328243255615,
+      "logits/rejected": -2.4910500049591064,
+      "logps/chosen": -325.08929443359375,
+      "logps/rejected": -471.27276611328125,
+      "loss": 0.5095,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.8783607482910156,
+      "rewards/margins": 0.6811479926109314,
+      "rewards/rejected": -1.5595088005065918,
+      "step": 1340
+    },
+    {
+      "epoch": 0.794585050029429,
+      "grad_norm": 89.6143819430957,
+      "learning_rate": 6.156807431939537e-08,
+      "logits/chosen": -2.522613763809204,
+      "logits/rejected": -2.5427727699279785,
+      "logps/chosen": -385.08453369140625,
+      "logps/rejected": -421.5887756347656,
+      "loss": 0.5045,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -0.6811944246292114,
+      "rewards/margins": 0.6432855725288391,
+      "rewards/rejected": -1.3244801759719849,
+      "step": 1350
+    },
+    {
+      "epoch": 0.8004708652148322,
+      "grad_norm": 67.09755437948724,
+      "learning_rate": 5.8232327723159e-08,
+      "logits/chosen": -2.4834189414978027,
+      "logits/rejected": -2.527981996536255,
+      "logps/chosen": -332.33831787109375,
+      "logps/rejected": -436.96856689453125,
+      "loss": 0.5009,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.9011951684951782,
+      "rewards/margins": 1.019168496131897,
+      "rewards/rejected": -1.920363426208496,
+      "step": 1360
+    },
+    {
+      "epoch": 0.8063566804002354,
+      "grad_norm": 58.01149580333495,
+      "learning_rate": 5.49775363534706e-08,
+      "logits/chosen": -2.6389904022216797,
+      "logits/rejected": -2.5821375846862793,
+      "logps/chosen": -338.70745849609375,
+      "logps/rejected": -419.2255859375,
+      "loss": 0.4777,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.6776565909385681,
+      "rewards/margins": 0.8060138821601868,
+      "rewards/rejected": -1.4836704730987549,
+      "step": 1370
+    },
+    {
+      "epoch": 0.8122424955856387,
+      "grad_norm": 102.8449817133206,
+      "learning_rate": 5.180507422905583e-08,
+      "logits/chosen": -2.7128119468688965,
+      "logits/rejected": -2.673996925354004,
+      "logps/chosen": -322.67779541015625,
+      "logps/rejected": -363.71270751953125,
+      "loss": 0.5261,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -0.8014975786209106,
+      "rewards/margins": 0.7609502673149109,
+      "rewards/rejected": -1.5624477863311768,
+      "step": 1380
+    },
+    {
+      "epoch": 0.8181283107710418,
+      "grad_norm": 65.36829776160975,
+      "learning_rate": 4.871628061313321e-08,
+      "logits/chosen": -2.581416368484497,
+      "logits/rejected": -2.682572364807129,
+      "logps/chosen": -318.94915771484375,
+      "logps/rejected": -439.8599548339844,
+      "loss": 0.5081,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.6443731188774109,
+      "rewards/margins": 1.2208740711212158,
+      "rewards/rejected": -1.8652470111846924,
+      "step": 1390
+    },
+    {
+      "epoch": 0.824014125956445,
+      "grad_norm": 178.13037283488754,
+      "learning_rate": 4.571245944804089e-08,
+      "logits/chosen": -2.489166736602783,
+      "logits/rejected": -2.398707866668701,
+      "logps/chosen": -375.8815612792969,
+      "logps/rejected": -497.11761474609375,
+      "loss": 0.5097,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -0.9333661198616028,
+      "rewards/margins": 1.0528299808502197,
+      "rewards/rejected": -1.9861959218978882,
+      "step": 1400
+    },
+    {
+      "epoch": 0.8298999411418482,
+      "grad_norm": 172.4847999858577,
+      "learning_rate": 4.279487880477342e-08,
+      "logits/chosen": -2.5642523765563965,
+      "logits/rejected": -2.607612133026123,
+      "logps/chosen": -372.1091003417969,
+      "logps/rejected": -393.6594543457031,
+      "loss": 0.5695,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.8643810153007507,
+      "rewards/margins": 0.6560724973678589,
+      "rewards/rejected": -1.5204535722732544,
+      "step": 1410
+    },
+    {
+      "epoch": 0.8357857563272513,
+      "grad_norm": 88.7462733661287,
+      "learning_rate": 3.9964770347662715e-08,
+      "logits/chosen": -2.615901470184326,
+      "logits/rejected": -2.5380148887634277,
+      "logps/chosen": -269.59002685546875,
+      "logps/rejected": -408.44708251953125,
+      "loss": 0.5431,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -0.8203733563423157,
+      "rewards/margins": 1.1023374795913696,
+      "rewards/rejected": -1.9227107763290405,
+      "step": 1420
+    },
+    {
+      "epoch": 0.8416715715126545,
+      "grad_norm": 48.812190967043605,
+      "learning_rate": 3.7223328814427414e-08,
+      "logits/chosen": -2.7706921100616455,
+      "logits/rejected": -2.6520023345947266,
+      "logps/chosen": -346.84747314453125,
+      "logps/rejected": -356.3780212402344,
+      "loss": 0.4967,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -0.7521100640296936,
+      "rewards/margins": 0.6091283559799194,
+      "rewards/rejected": -1.3612383604049683,
+      "step": 1430
+    },
+    {
+      "epoch": 0.8475573866980577,
+      "grad_norm": 81.36099469807638,
+      "learning_rate": 3.457171151181154e-08,
+      "logits/chosen": -2.6654701232910156,
+      "logits/rejected": -2.5508956909179688,
+      "logps/chosen": -465.358154296875,
+      "logps/rejected": -375.4325866699219,
+      "loss": 0.5797,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.9726330041885376,
+      "rewards/margins": 0.5235468149185181,
+      "rewards/rejected": -1.4961798191070557,
+      "step": 1440
+    },
+    {
+      "epoch": 0.8534432018834609,
+      "grad_norm": 58.24549104706358,
+      "learning_rate": 3.2011037827024054e-08,
+      "logits/chosen": -2.6841177940368652,
+      "logits/rejected": -2.628507137298584,
+      "logps/chosen": -382.93841552734375,
+      "logps/rejected": -372.9692687988281,
+      "loss": 0.5755,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.9706902503967285,
+      "rewards/margins": 0.5649546384811401,
+      "rewards/rejected": -1.5356448888778687,
+      "step": 1450
+    },
+    {
+      "epoch": 0.859329017068864,
+      "grad_norm": 52.79125544847847,
+      "learning_rate": 2.9542388755186804e-08,
+      "logits/chosen": -2.5700201988220215,
+      "logits/rejected": -2.6171364784240723,
+      "logps/chosen": -345.32354736328125,
+      "logps/rejected": -433.79632568359375,
+      "loss": 0.5057,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.8421486020088196,
+      "rewards/margins": 0.6202816963195801,
+      "rewards/rejected": -1.4624302387237549,
+      "step": 1460
+    },
+    {
+      "epoch": 0.8652148322542672,
+      "grad_norm": 199.09195971504573,
+      "learning_rate": 2.7166806442989743e-08,
+      "logits/chosen": -2.468412399291992,
+      "logits/rejected": -2.5492916107177734,
+      "logps/chosen": -419.80133056640625,
+      "logps/rejected": -409.2911682128906,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1756455898284912,
+      "rewards/margins": 0.4044803977012634,
+      "rewards/rejected": -1.5801259279251099,
+      "step": 1470
+    },
+    {
+      "epoch": 0.8711006474396704,
+      "grad_norm": 186.66911649670288,
+      "learning_rate": 2.488529374874557e-08,
+      "logits/chosen": -2.59171724319458,
+      "logits/rejected": -2.5773372650146484,
+      "logps/chosen": -398.6797790527344,
+      "logps/rejected": -408.37554931640625,
+      "loss": 0.4744,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.9879580736160278,
+      "rewards/margins": 0.6637744903564453,
+      "rewards/rejected": -1.6517324447631836,
+      "step": 1480
+    },
+    {
+      "epoch": 0.8769864626250736,
+      "grad_norm": 121.13095753685242,
+      "learning_rate": 2.26988138190308e-08,
+      "logits/chosen": -2.511080265045166,
+      "logits/rejected": -2.476285696029663,
+      "logps/chosen": -388.0962219238281,
+      "logps/rejected": -352.10211181640625,
+      "loss": 0.586,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.7947136163711548,
+      "rewards/margins": 0.43633508682250977,
+      "rewards/rejected": -1.2310487031936646,
+      "step": 1490
+    },
+    {
+      "epoch": 0.8828722778104767,
+      "grad_norm": 60.22203345102361,
+      "learning_rate": 2.0608289682090663e-08,
+      "logits/chosen": -2.488114833831787,
+      "logits/rejected": -2.5074970722198486,
+      "logps/chosen": -347.0746765136719,
+      "logps/rejected": -471.3238220214844,
+      "loss": 0.5907,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.9873822927474976,
+      "rewards/margins": 0.9621238708496094,
+      "rewards/rejected": -1.9495060443878174,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8887580929958799,
+      "grad_norm": 83.04601367710045,
+      "learning_rate": 1.86146038581807e-08,
+      "logits/chosen": -2.6847586631774902,
+      "logits/rejected": -2.6667001247406006,
+      "logps/chosen": -379.8851013183594,
+      "logps/rejected": -402.2454528808594,
+      "loss": 0.5593,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.6927786469459534,
+      "rewards/margins": 0.8660175204277039,
+      "rewards/rejected": -1.5587961673736572,
+      "step": 1510
+    },
+    {
+      "epoch": 0.8946439081812831,
+      "grad_norm": 138.65953465833107,
+      "learning_rate": 1.6718597987007977e-08,
+      "logits/chosen": -2.6951823234558105,
+      "logits/rejected": -2.6642472743988037,
+      "logps/chosen": -458.5162658691406,
+      "logps/rejected": -486.06890869140625,
+      "loss": 0.5285,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -0.9807785153388977,
+      "rewards/margins": 0.7777236700057983,
+      "rewards/rejected": -1.7585022449493408,
+      "step": 1520
+    },
+    {
+      "epoch": 0.9005297233666862,
+      "grad_norm": 235.95269304208253,
+      "learning_rate": 1.4921072472431073e-08,
+      "logits/chosen": -2.582733392715454,
+      "logits/rejected": -2.54697847366333,
+      "logps/chosen": -380.550048828125,
+      "logps/rejected": -469.8357849121094,
+      "loss": 0.465,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": -0.8054933547973633,
+      "rewards/margins": 1.0806586742401123,
+      "rewards/rejected": -1.8861520290374756,
+      "step": 1530
+    },
+    {
+      "epoch": 0.9064155385520895,
+      "grad_norm": 108.02460840596761,
+      "learning_rate": 1.3222786144567317e-08,
+      "logits/chosen": -2.5321972370147705,
+      "logits/rejected": -2.446223735809326,
+      "logps/chosen": -361.153564453125,
+      "logps/rejected": -472.0574645996094,
+      "loss": 0.5382,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.0409860610961914,
+      "rewards/margins": 0.5477335453033447,
+      "rewards/rejected": -1.5887196063995361,
+      "step": 1540
+    },
+    {
+      "epoch": 0.9123013537374927,
+      "grad_norm": 109.47512454441822,
+      "learning_rate": 1.1624455939450773e-08,
+      "logits/chosen": -2.5635602474212646,
+      "logits/rejected": -2.5231423377990723,
+      "logps/chosen": -390.92144775390625,
+      "logps/rejected": -435.39874267578125,
+      "loss": 0.5468,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -1.122955083847046,
+      "rewards/margins": 0.48533105850219727,
+      "rewards/rejected": -1.6082861423492432,
+      "step": 1550
+    },
+    {
+      "epoch": 0.9181871689228959,
+      "grad_norm": 124.59873217557653,
+      "learning_rate": 1.0126756596375685e-08,
+      "logits/chosen": -2.5917229652404785,
+      "logits/rejected": -2.533984661102295,
+      "logps/chosen": -311.223388671875,
+      "logps/rejected": -396.94775390625,
+      "loss": 0.5317,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.0927371978759766,
+      "rewards/margins": 0.5735793113708496,
+      "rewards/rejected": -1.6663163900375366,
+      "step": 1560
+    },
+    {
+      "epoch": 0.924072984108299,
+      "grad_norm": 115.76739079691396,
+      "learning_rate": 8.730320373053545e-09,
+      "logits/chosen": -2.56396484375,
+      "logits/rejected": -2.548893451690674,
+      "logps/chosen": -375.27008056640625,
+      "logps/rejected": -411.82183837890625,
+      "loss": 0.5228,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8550286293029785,
+      "rewards/margins": 0.5478664636611938,
+      "rewards/rejected": -1.402895212173462,
+      "step": 1570
+    },
+    {
+      "epoch": 0.9299587992937022,
+      "grad_norm": 92.62228298124931,
+      "learning_rate": 7.4357367787039605e-09,
+      "logits/chosen": -2.5038106441497803,
+      "logits/rejected": -2.4670755863189697,
+      "logps/chosen": -403.1292724609375,
+      "logps/rejected": -438.26580810546875,
+      "loss": 0.5464,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9969646334648132,
+      "rewards/margins": 0.7654547691345215,
+      "rewards/rejected": -1.7624194622039795,
+      "step": 1580
+    },
+    {
+      "epoch": 0.9358446144791054,
+      "grad_norm": 55.929288108030285,
+      "learning_rate": 6.243552325191581e-09,
+      "logits/chosen": -2.4634852409362793,
+      "logits/rejected": -2.364482879638672,
+      "logps/chosen": -391.3017578125,
+      "logps/rejected": -450.46734619140625,
+      "loss": 0.4938,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.8056437373161316,
+      "rewards/margins": 1.2021862268447876,
+      "rewards/rejected": -2.0078299045562744,
+      "step": 1590
+    },
+    {
+      "epoch": 0.9417304296645085,
+      "grad_norm": 61.15914730289007,
+      "learning_rate": 5.154270296314878e-09,
+      "logits/chosen": -2.6435704231262207,
+      "logits/rejected": -2.6495299339294434,
+      "logps/chosen": -374.0923767089844,
+      "logps/rejected": -469.6061096191406,
+      "loss": 0.4876,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.7130493521690369,
+      "rewards/margins": 0.9921830296516418,
+      "rewards/rejected": -1.7052322626113892,
+      "step": 1600
+    },
+    {
+      "epoch": 0.9476162448499117,
+      "grad_norm": 87.06035022289088,
+      "learning_rate": 4.168350535343451e-09,
+      "logits/chosen": -2.5816283226013184,
+      "logits/rejected": -2.508742570877075,
+      "logps/chosen": -431.2406311035156,
+      "logps/rejected": -444.27606201171875,
+      "loss": 0.6038,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.034767508506775,
+      "rewards/margins": 0.6255626082420349,
+      "rewards/rejected": -1.6603301763534546,
+      "step": 1610
+    },
+    {
+      "epoch": 0.9535020600353149,
+      "grad_norm": 71.26805126819406,
+      "learning_rate": 3.286209250894012e-09,
+      "logits/chosen": -2.616842746734619,
+      "logits/rejected": -2.5755038261413574,
+      "logps/chosen": -404.6748352050781,
+      "logps/rejected": -424.21905517578125,
+      "loss": 0.6546,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.873518168926239,
+      "rewards/margins": 0.5968537926673889,
+      "rewards/rejected": -1.4703717231750488,
+      "step": 1620
+    },
+    {
+      "epoch": 0.9593878752207181,
+      "grad_norm": 88.13236089767452,
+      "learning_rate": 2.5082188412267614e-09,
+      "logits/chosen": -2.623947858810425,
+      "logits/rejected": -2.5589542388916016,
+      "logps/chosen": -363.2535095214844,
+      "logps/rejected": -465.86297607421875,
+      "loss": 0.555,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -0.8597814440727234,
+      "rewards/margins": 0.6761683225631714,
+      "rewards/rejected": -1.53594970703125,
+      "step": 1630
+    },
+    {
+      "epoch": 0.9652736904061212,
+      "grad_norm": 104.76645086528532,
+      "learning_rate": 1.8347077370366726e-09,
+      "logits/chosen": -2.449385404586792,
+      "logits/rejected": -2.401689052581787,
+      "logps/chosen": -358.8383483886719,
+      "logps/rejected": -446.76092529296875,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -0.9526700973510742,
+      "rewards/margins": 0.987733006477356,
+      "rewards/rejected": -1.9404027462005615,
+      "step": 1640
+    },
+    {
+      "epoch": 0.9711595055915244,
+      "grad_norm": 119.412074414206,
+      "learning_rate": 1.2659602628053712e-09,
+      "logits/chosen": -2.5362093448638916,
+      "logits/rejected": -2.5611374378204346,
+      "logps/chosen": -341.82537841796875,
+      "logps/rejected": -383.3554382324219,
+      "loss": 0.5503,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.9428917169570923,
+      "rewards/margins": 0.6708999872207642,
+      "rewards/rejected": -1.6137914657592773,
+      "step": 1650
+    },
+    {
+      "epoch": 0.9770453207769276,
+      "grad_norm": 91.85897396152326,
+      "learning_rate": 8.022165167730355e-10,
+      "logits/chosen": -2.5891966819763184,
+      "logits/rejected": -2.598982334136963,
+      "logps/chosen": -303.7531433105469,
+      "logps/rejected": -403.8943176269531,
+      "loss": 0.5337,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -0.7517716884613037,
+      "rewards/margins": 0.8414438962936401,
+      "rewards/rejected": -1.5932157039642334,
+      "step": 1660
+    },
+    {
+      "epoch": 0.9829311359623308,
+      "grad_norm": 49.42793849018654,
+      "learning_rate": 4.4367226958005877e-10,
+      "logits/chosen": -2.6086266040802,
+      "logits/rejected": -2.586604356765747,
+      "logps/chosen": -384.49041748046875,
+      "logps/rejected": -449.49365234375,
+      "loss": 0.5456,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.9067045450210571,
+      "rewards/margins": 0.4888109266757965,
+      "rewards/rejected": -1.3955154418945312,
+      "step": 1670
+    },
+    {
+      "epoch": 0.9888169511477339,
+      "grad_norm": 114.79577809839017,
+      "learning_rate": 1.9047888162218584e-10,
+      "logits/chosen": -2.6199536323547363,
+      "logits/rejected": -2.5287978649139404,
+      "logps/chosen": -333.8753662109375,
+      "logps/rejected": -358.57916259765625,
+      "loss": 0.4755,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.7598250508308411,
+      "rewards/margins": 0.8381843566894531,
+      "rewards/rejected": -1.5980093479156494,
+      "step": 1680
+    },
+    {
+      "epoch": 0.9947027663331371,
+      "grad_norm": 149.79667335956157,
+      "learning_rate": 4.2743239153153474e-11,
+      "logits/chosen": -2.4809188842773438,
+      "logits/rejected": -2.3862576484680176,
+      "logps/chosen": -299.9741516113281,
+      "logps/rejected": -355.7412414550781,
+      "loss": 0.5547,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.1635828018188477,
+      "rewards/margins": 0.4038754403591156,
+      "rewards/rejected": -1.5674583911895752,
+      "step": 1690
+    },
+    {
+      "epoch": 1.0,
+      "step": 1699,
+      "total_flos": 0.0,
+      "train_loss": 0.3274427829874902,
+      "train_runtime": 3901.2433,
+      "train_samples_per_second": 3.917,
+      "train_steps_per_second": 0.436
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1699,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}