Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- kto
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-kto
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/nlee28/huggingface/runs/odmtxd9p)
+# zephyr-7b-kto
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.42.4
+- Pytorch 2.1.2+cu121
+- Datasets 2.16.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.321955375749023,
+    "train_runtime": 31031.977,
+    "train_samples": 60917,
+    "train_samples_per_second": 1.963,
+    "train_steps_per_second": 0.061
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.42.4"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46f8b429e795066757438ee4f5e111f05d1dedd5a204e1768338157d806ac453
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f5488369d7fb93a6995ce580fc61441f62b827c72815e00d96007d231690c51
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6636ec0ebc90538d14353926a784e25a86cbc48be9ada07792068ff299a0c7d6
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.321955375749023,
+    "train_runtime": 31031.977,
+    "train_samples": 60917,
+    "train_samples_per_second": 1.963,
+    "train_steps_per_second": 0.061
+}

trainer_state.json

@@ -0,0 +1,2512 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1904,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005252100840336135,
+      "grad_norm": 70.43905184347379,
+      "kl": 0.21706357598304749,
+      "learning_rate": 4.999659696812289e-07,
+      "logps/chosen": -305.59059320494185,
+      "logps/rejected": -267.9389252533784,
+      "loss": 0.5004,
+      "rewards/chosen": -0.09660225136335505,
+      "rewards/margins": 0.019717185921372296,
+      "rewards/rejected": -0.11631943728472735,
+      "step": 10
+    },
+    {
+      "epoch": 0.01050420168067227,
+      "grad_norm": 81.8070303563997,
+      "kl": 1.2398526668548584,
+      "learning_rate": 4.998638879894165e-07,
+      "logps/chosen": -279.9350071957237,
+      "logps/rejected": -239.82217261904762,
+      "loss": 0.4612,
+      "rewards/chosen": 0.36437626888877467,
+      "rewards/margins": 0.3501945868470615,
+      "rewards/rejected": 0.01418168204171317,
+      "step": 20
+    },
+    {
+      "epoch": 0.015756302521008403,
+      "grad_norm": 73.53961446495941,
+      "kl": 0.0,
+      "learning_rate": 4.996937827155428e-07,
+      "logps/chosen": -327.1117964181287,
+      "logps/rejected": -270.5936713506711,
+      "loss": 0.4185,
+      "rewards/chosen": -0.6132015652126737,
+      "rewards/margins": 1.3433878967603758,
+      "rewards/rejected": -1.9565894619730495,
+      "step": 30
+    },
+    {
+      "epoch": 0.02100840336134454,
+      "grad_norm": 51.541845841038615,
+      "kl": 3.030362129211426,
+      "learning_rate": 4.994557001695013e-07,
+      "logps/chosen": -286.4127286585366,
+      "logps/rejected": -238.16353665865384,
+      "loss": 0.4045,
+      "rewards/chosen": 1.3511776807831555,
+      "rewards/margins": 1.1392201160624147,
+      "rewards/rejected": 0.21195756472074068,
+      "step": 40
+    },
+    {
+      "epoch": 0.026260504201680673,
+      "grad_norm": 83.73385866105204,
+      "kl": 0.0,
+      "learning_rate": 4.991497051674917e-07,
+      "logps/chosen": -295.27088246855345,
+      "logps/rejected": -278.80997670807454,
+      "loss": 0.3944,
+      "rewards/chosen": 0.2137240523812156,
+      "rewards/margins": 1.5743222196584865,
+      "rewards/rejected": -1.360598167277271,
+      "step": 50
+    },
+    {
+      "epoch": 0.031512605042016806,
+      "grad_norm": 57.616560952806196,
+      "kl": 0.0,
+      "learning_rate": 4.987758810143735e-07,
+      "logps/chosen": -325.4705078125,
+      "logps/rejected": -255.959326171875,
+      "loss": 0.4034,
+      "rewards/chosen": 1.1321415901184082,
+      "rewards/margins": 1.2807276964187622,
+      "rewards/rejected": -0.14858610630035402,
+      "step": 60
+    },
+    {
+      "epoch": 0.03676470588235294,
+      "grad_norm": 62.734358016783204,
+      "kl": 0.0,
+      "learning_rate": 4.983343294809874e-07,
+      "logps/chosen": -306.6684683866279,
+      "logps/rejected": -258.4501953125,
+      "loss": 0.3822,
+      "rewards/chosen": 1.1383993459302326,
+      "rewards/margins": 1.6607683698461164,
+      "rewards/rejected": -0.5223690239158837,
+      "step": 70
+    },
+    {
+      "epoch": 0.04201680672268908,
+      "grad_norm": 52.446040187552235,
+      "kl": 0.0,
+      "learning_rate": 4.978251707764491e-07,
+      "logps/chosen": -304.0558176100629,
+      "logps/rejected": -276.5557793090062,
+      "loss": 0.3593,
+      "rewards/chosen": 0.427140049964377,
+      "rewards/margins": 2.316681770716027,
+      "rewards/rejected": -1.8895417207516498,
+      "step": 80
+    },
+    {
+      "epoch": 0.04726890756302521,
+      "grad_norm": 51.1401597083277,
+      "kl": 0.0,
+      "learning_rate": 4.972485435154228e-07,
+      "logps/chosen": -284.92168090062114,
+      "logps/rejected": -289.27459217767296,
+      "loss": 0.3601,
+      "rewards/chosen": 1.139393302964868,
+      "rewards/margins": 2.435457571832792,
+      "rewards/rejected": -1.2960642688679245,
+      "step": 90
+    },
+    {
+      "epoch": 0.052521008403361345,
+      "grad_norm": 44.236393372962524,
+      "kl": 0.0,
+      "learning_rate": 4.966046046803842e-07,
+      "logps/chosen": -312.7745820063694,
+      "logps/rejected": -278.74285851226995,
+      "loss": 0.3593,
+      "rewards/chosen": 0.3532914568664162,
+      "rewards/margins": 2.3767495515510406,
+      "rewards/rejected": -2.0234580946846243,
+      "step": 100
+    },
+    {
+      "epoch": 0.05777310924369748,
+      "grad_norm": 38.95152051376054,
+      "kl": 0.0,
+      "learning_rate": 4.958935295788841e-07,
+      "logps/chosen": -364.43832781456956,
+      "logps/rejected": -306.8252588757396,
+      "loss": 0.3823,
+      "rewards/chosen": -1.0973136851329677,
+      "rewards/margins": 2.0935938101286817,
+      "rewards/rejected": -3.1909074952616496,
+      "step": 110
+    },
+    {
+      "epoch": 0.06302521008403361,
+      "grad_norm": 45.75317629434847,
+      "kl": 0.0,
+      "learning_rate": 4.951155117958216e-07,
+      "logps/chosen": -304.1672453703704,
+      "logps/rejected": -288.4255340189873,
+      "loss": 0.3693,
+      "rewards/chosen": -0.012564458964783469,
+      "rewards/margins": 2.483852622843959,
+      "rewards/rejected": -2.496417081808742,
+      "step": 120
+    },
+    {
+      "epoch": 0.06827731092436974,
+      "grad_norm": 52.69834029914945,
+      "kl": 0.0,
+      "learning_rate": 4.942707631407419e-07,
+      "logps/chosen": -331.2020513523392,
+      "logps/rejected": -275.2519662332215,
+      "loss": 0.3931,
+      "rewards/chosen": 0.4922319378769189,
+      "rewards/margins": 1.7257208016235108,
+      "rewards/rejected": -1.233488863746592,
+      "step": 130
+    },
+    {
+      "epoch": 0.07352941176470588,
+      "grad_norm": 65.97698112794365,
+      "kl": 0.0,
+      "learning_rate": 4.933595135901732e-07,
+      "logps/chosen": -314.8364361702128,
+      "logps/rejected": -269.268156424581,
+      "loss": 0.3674,
+      "rewards/chosen": 0.04331512315898922,
+      "rewards/margins": 1.9166126173759934,
+      "rewards/rejected": -1.8732974942170042,
+      "step": 140
+    },
+    {
+      "epoch": 0.07878151260504201,
+      "grad_norm": 75.72251586801518,
+      "kl": 0.0,
+      "learning_rate": 4.923820112250169e-07,
+      "logps/chosen": -310.11692533557044,
+      "logps/rejected": -272.4901087353801,
+      "loss": 0.3438,
+      "rewards/chosen": -0.31625601749292154,
+      "rewards/margins": 2.9441549382819323,
+      "rewards/rejected": -3.2604109557748537,
+      "step": 150
+    },
+    {
+      "epoch": 0.08403361344537816,
+      "grad_norm": 51.266519524836305,
+      "kl": 0.0,
+      "learning_rate": 4.913385221630096e-07,
+      "logps/chosen": -264.80689858490564,
+      "logps/rejected": -284.40986510093165,
+      "loss": 0.3589,
+      "rewards/chosen": 0.2699741627435264,
+      "rewards/margins": 2.429860872395118,
+      "rewards/rejected": -2.1598867096515915,
+      "step": 160
+    },
+    {
+      "epoch": 0.08928571428571429,
+      "grad_norm": 69.780370721156,
+      "kl": 0.0,
+      "learning_rate": 4.902293304862749e-07,
+      "logps/chosen": -281.5589111328125,
+      "logps/rejected": -274.93525390625,
+      "loss": 0.361,
+      "rewards/chosen": 0.5642091274261475,
+      "rewards/margins": 2.2661412715911866,
+      "rewards/rejected": -1.701932144165039,
+      "step": 170
+    },
+    {
+      "epoch": 0.09453781512605042,
+      "grad_norm": 48.456143880910204,
+      "kl": 0.0,
+      "learning_rate": 4.890547381639833e-07,
+      "logps/chosen": -312.7041968368902,
+      "logps/rejected": -251.9474158653846,
+      "loss": 0.3564,
+      "rewards/chosen": -0.15778241506436977,
+      "rewards/margins": 2.705699155448451,
+      "rewards/rejected": -2.8634815705128207,
+      "step": 180
+    },
+    {
+      "epoch": 0.09978991596638656,
+      "grad_norm": 49.10681910714073,
+      "kl": 0.0,
+      "learning_rate": 4.878150649701439e-07,
+      "logps/chosen": -356.0400260416667,
+      "logps/rejected": -285.11312040441175,
+      "loss": 0.3345,
+      "rewards/chosen": -0.45484156290690103,
+      "rewards/margins": 3.4065278505811505,
+      "rewards/rejected": -3.8613694134880516,
+      "step": 190
+    },
+    {
+      "epoch": 0.10504201680672269,
+      "grad_norm": 42.17787499105965,
+      "kl": 0.0,
+      "learning_rate": 4.865106483965486e-07,
+      "logps/chosen": -297.6003605769231,
+      "logps/rejected": -252.6407440929878,
+      "loss": 0.3495,
+      "rewards/chosen": -0.08823707164862217,
+      "rewards/margins": 2.8701916601301507,
+      "rewards/rejected": -2.9584287317787727,
+      "step": 200
+    },
+    {
+      "epoch": 0.11029411764705882,
+      "grad_norm": 32.7456875390811,
+      "kl": 0.0,
+      "learning_rate": 4.851418435608919e-07,
+      "logps/chosen": -292.8467514124294,
+      "logps/rejected": -265.97642591783216,
+      "loss": 0.3655,
+      "rewards/chosen": 0.9715841692046258,
+      "rewards/margins": 2.2475568689901917,
+      "rewards/rejected": -1.275972699785566,
+      "step": 210
+    },
+    {
+      "epoch": 0.11554621848739496,
+      "grad_norm": 46.476558893396145,
+      "kl": 0.0,
+      "learning_rate": 4.837090231100927e-07,
+      "logps/chosen": -315.36924463757396,
+      "logps/rejected": -251.48538389900662,
+      "loss": 0.3217,
+      "rewards/chosen": 1.1581101276465422,
+      "rewards/margins": 2.9979615575668648,
+      "rewards/rejected": -1.8398514299203228,
+      "step": 220
+    },
+    {
+      "epoch": 0.1207983193277311,
+      "grad_norm": 58.692837949072945,
+      "kl": 0.0,
+      "learning_rate": 4.822125771188448e-07,
+      "logps/chosen": -306.7650669642857,
+      "logps/rejected": -301.322265625,
+      "loss": 0.3701,
+      "rewards/chosen": -0.7039821178882153,
+      "rewards/margins": 2.5890142082365966,
+      "rewards/rejected": -3.2929963261248116,
+      "step": 230
+    },
+    {
+      "epoch": 0.12605042016806722,
+      "grad_norm": 38.77396195360521,
+      "kl": 0.0,
+      "learning_rate": 4.806529129834207e-07,
+      "logps/chosen": -299.9251123715753,
+      "logps/rejected": -295.51858836206895,
+      "loss": 0.3211,
+      "rewards/chosen": -0.4328058843743311,
+      "rewards/margins": 3.400510611674519,
+      "rewards/rejected": -3.8333164960488504,
+      "step": 240
+    },
+    {
+      "epoch": 0.13130252100840337,
+      "grad_norm": 54.771749584890465,
+      "kl": 0.0,
+      "learning_rate": 4.790304553107622e-07,
+      "logps/chosen": -305.0079280695266,
+      "logps/rejected": -256.60885761589407,
+      "loss": 0.3722,
+      "rewards/chosen": 0.8469177787825906,
+      "rewards/margins": 2.0538868820589253,
+      "rewards/rejected": -1.2069691032763348,
+      "step": 250
+    },
+    {
+      "epoch": 0.13655462184873948,
+      "grad_norm": 56.498632681940265,
+      "kl": 0.0,
+      "learning_rate": 4.773456458028837e-07,
+      "logps/chosen": -278.4545183121019,
+      "logps/rejected": -264.5601514570552,
+      "loss": 0.3744,
+      "rewards/chosen": 1.3172516063520103,
+      "rewards/margins": 2.152952080254426,
+      "rewards/rejected": -0.8357004739024156,
+      "step": 260
+    },
+    {
+      "epoch": 0.14180672268907563,
+      "grad_norm": 56.9574995100581,
+      "kl": 0.0,
+      "learning_rate": 4.755989431366221e-07,
+      "logps/chosen": -304.06099759615387,
+      "logps/rejected": -293.3189310213415,
+      "loss": 0.3093,
+      "rewards/chosen": 1.1587672111315606,
+      "rewards/margins": 4.173915839180341,
+      "rewards/rejected": -3.0151486280487805,
+      "step": 270
+    },
+    {
+      "epoch": 0.14705882352941177,
+      "grad_norm": 63.035012985547,
+      "kl": 0.0,
+      "learning_rate": 4.737908228387656e-07,
+      "logps/chosen": -297.4018322172619,
+      "logps/rejected": -267.7578895970395,
+      "loss": 0.3581,
+      "rewards/chosen": 1.0978340875534784,
+      "rewards/margins": 2.6742270142213442,
+      "rewards/rejected": -1.576392926667866,
+      "step": 280
+    },
+    {
+      "epoch": 0.15231092436974789,
+      "grad_norm": 57.87624552889083,
+      "kl": 0.0,
+      "learning_rate": 4.7192177715659516e-07,
+      "logps/chosen": -293.31778630239523,
+      "logps/rejected": -253.0896905637255,
+      "loss": 0.3583,
+      "rewards/chosen": 0.7335890809932869,
+      "rewards/margins": 2.5885067194353826,
+      "rewards/rejected": -1.8549176384420956,
+      "step": 290
+    },
+    {
+      "epoch": 0.15756302521008403,
+      "grad_norm": 63.42262644789701,
+      "kl": 0.0,
+      "learning_rate": 4.699923149238736e-07,
+      "logps/chosen": -300.68055867805754,
+      "logps/rejected": -294.7720778660221,
+      "loss": 0.3416,
+      "rewards/chosen": -0.4132621820024449,
+      "rewards/margins": 3.4406516889618164,
+      "rewards/rejected": -3.853913870964261,
+      "step": 300
+    },
+    {
+      "epoch": 0.16281512605042017,
+      "grad_norm": 58.34203205576481,
+      "kl": 0.0,
+      "learning_rate": 4.680029614223198e-07,
+      "logps/chosen": -284.8943819665605,
+      "logps/rejected": -281.19528853527606,
+      "loss": 0.374,
+      "rewards/chosen": -0.6231775951992934,
+      "rewards/margins": 2.335680965000093,
+      "rewards/rejected": -2.9588585601993866,
+      "step": 310
+    },
+    {
+      "epoch": 0.16806722689075632,
+      "grad_norm": 45.38290295645773,
+      "kl": 0.0,
+      "learning_rate": 4.65954258238604e-07,
+      "logps/chosen": -297.1282980913174,
+      "logps/rejected": -263.37903390522877,
+      "loss": 0.3695,
+      "rewards/chosen": 0.311750309196061,
+      "rewards/margins": 2.5421297005359302,
+      "rewards/rejected": -2.230379391339869,
+      "step": 320
+    },
+    {
+      "epoch": 0.17331932773109243,
+      "grad_norm": 67.2098515597697,
+      "kl": 0.0,
+      "learning_rate": 4.638467631169056e-07,
+      "logps/chosen": -328.67982700892856,
+      "logps/rejected": -309.46533203125,
+      "loss": 0.3203,
+      "rewards/chosen": 0.9687714349655878,
+      "rewards/margins": 3.498681472357652,
+      "rewards/rejected": -2.529910037392064,
+      "step": 330
+    },
+    {
+      "epoch": 0.17857142857142858,
+      "grad_norm": 55.44522631657296,
+      "kl": 0.0,
+      "learning_rate": 4.6168104980707103e-07,
+      "logps/chosen": -285.8288395579268,
+      "logps/rejected": -275.2672526041667,
+      "loss": 0.3315,
+      "rewards/chosen": 0.7993925141125191,
+      "rewards/margins": 3.4423916251902433,
+      "rewards/rejected": -2.6429991110777245,
+      "step": 340
+    },
+    {
+      "epoch": 0.18382352941176472,
+      "grad_norm": 36.50055695974426,
+      "kl": 0.0,
+      "learning_rate": 4.594577079084145e-07,
+      "logps/chosen": -287.8186279296875,
+      "logps/rejected": -295.969091796875,
+      "loss": 0.3146,
+      "rewards/chosen": 1.0129197120666504,
+      "rewards/margins": 3.7097062110900882,
+      "rewards/rejected": -2.6967864990234376,
+      "step": 350
+    },
+    {
+      "epoch": 0.18907563025210083,
+      "grad_norm": 48.72297305247662,
+      "kl": 0.0,
+      "learning_rate": 4.5717734270920466e-07,
+      "logps/chosen": -280.1525594325153,
+      "logps/rejected": -232.07081011146497,
+      "loss": 0.351,
+      "rewards/chosen": 0.9978786538715011,
+      "rewards/margins": 2.654015719394491,
+      "rewards/rejected": -1.6561370655229897,
+      "step": 360
+    },
+    {
+      "epoch": 0.19432773109243698,
+      "grad_norm": 47.17739577906298,
+      "kl": 0.0,
+      "learning_rate": 4.548405750218785e-07,
+      "logps/chosen": -290.4170778508772,
+      "logps/rejected": -278.8102191694631,
+      "loss": 0.3535,
+      "rewards/chosen": 0.4387444390190972,
+      "rewards/margins": 3.595384203792063,
+      "rewards/rejected": -3.1566397647729656,
+      "step": 370
+    },
+    {
+      "epoch": 0.19957983193277312,
+      "grad_norm": 55.83663877353586,
+      "kl": 0.0,
+      "learning_rate": 4.5244804101403025e-07,
+      "logps/chosen": -262.20204133064516,
+      "logps/rejected": -258.4055634469697,
+      "loss": 0.3611,
+      "rewards/chosen": 0.3024998326455393,
+      "rewards/margins": 2.7435985024490432,
+      "rewards/rejected": -2.441098669803504,
+      "step": 380
+    },
+    {
+      "epoch": 0.20483193277310924,
+      "grad_norm": 53.266164401529096,
+      "kl": 0.0,
+      "learning_rate": 4.5000039203521976e-07,
+      "logps/chosen": -300.6577662417763,
+      "logps/rejected": -298.0972609747024,
+      "loss": 0.348,
+      "rewards/chosen": 1.187171132940995,
+      "rewards/margins": 3.403816691616125,
+      "rewards/rejected": -2.2166455586751304,
+      "step": 390
+    },
+    {
+      "epoch": 0.21008403361344538,
+      "grad_norm": 51.236491583825575,
+      "kl": 0.0,
+      "learning_rate": 4.47498294439647e-07,
+      "logps/chosen": -291.7605892319277,
+      "logps/rejected": -267.51894784902595,
+      "loss": 0.3199,
+      "rewards/chosen": 1.1110430797898625,
+      "rewards/margins": 3.199955449158775,
+      "rewards/rejected": -2.0889123693689124,
+      "step": 400
+    },
+    {
+      "epoch": 0.21533613445378152,
+      "grad_norm": 66.69622404112168,
+      "kl": 0.0,
+      "learning_rate": 4.449424294047419e-07,
+      "logps/chosen": -301.6258148006135,
+      "logps/rejected": -284.4103801751592,
+      "loss": 0.3232,
+      "rewards/chosen": 1.0998908668939322,
+      "rewards/margins": 3.629001137594569,
+      "rewards/rejected": -2.5291102707006368,
+      "step": 410
+    },
+    {
+      "epoch": 0.22058823529411764,
+      "grad_norm": 52.10847099745242,
+      "kl": 0.0,
+      "learning_rate": 4.4233349274571974e-07,
+      "logps/chosen": -303.1148280201342,
+      "logps/rejected": -273.5418494152047,
+      "loss": 0.3532,
+      "rewards/chosen": 0.6703597203197095,
+      "rewards/margins": 3.336041436614026,
+      "rewards/rejected": -2.6656817162943165,
+      "step": 420
+    },
+    {
+      "epoch": 0.22584033613445378,
+      "grad_norm": 48.65147057788595,
+      "kl": 0.0,
+      "learning_rate": 4.396721947261496e-07,
+      "logps/chosen": -293.84951524849396,
+      "logps/rejected": -279.7598924512987,
+      "loss": 0.345,
+      "rewards/chosen": 0.35480995637824736,
+      "rewards/margins": 3.4659533905807636,
+      "rewards/rejected": -3.1111434342025164,
+      "step": 430
+    },
+    {
+      "epoch": 0.23109243697478993,
+      "grad_norm": 60.821733992897606,
+      "kl": 0.0,
+      "learning_rate": 4.3695925986459107e-07,
+      "logps/chosen": -284.638457507622,
+      "logps/rejected": -260.53165064102564,
+      "loss": 0.3469,
+      "rewards/chosen": 0.6963288376970989,
+      "rewards/margins": 3.8638249564871634,
+      "rewards/rejected": -3.1674961187900643,
+      "step": 440
+    },
+    {
+      "epoch": 0.23634453781512604,
+      "grad_norm": 40.515343648966464,
+      "kl": 0.0,
+      "learning_rate": 4.341954267373494e-07,
+      "logps/chosen": -288.7482045807453,
+      "logps/rejected": -257.25950766509436,
+      "loss": 0.3392,
+      "rewards/chosen": 1.8534261573175466,
+      "rewards/margins": 2.916300141600418,
+      "rewards/rejected": -1.0628739842828714,
+      "step": 450
+    },
+    {
+      "epoch": 0.2415966386554622,
+      "grad_norm": 59.00202211258808,
+      "kl": 0.0,
+      "learning_rate": 4.313814477774035e-07,
+      "logps/chosen": -450.8061615566038,
+      "logps/rejected": -373.24873835403724,
+      "loss": 0.3372,
+      "rewards/chosen": -13.37514433470912,
+      "rewards/margins": -0.7130592221314807,
+      "rewards/rejected": -12.662085112577639,
+      "step": 460
+    },
+    {
+      "epoch": 0.24684873949579833,
+      "grad_norm": 47.46663221207845,
+      "kl": 0.0,
+      "learning_rate": 4.2851808906956134e-07,
+      "logps/chosen": -293.39712000739644,
+      "logps/rejected": -281.5295685016556,
+      "loss": 0.3252,
+      "rewards/chosen": 1.1083643986628606,
+      "rewards/margins": 3.8087977886928686,
+      "rewards/rejected": -2.700433390030008,
+      "step": 470
+    },
+    {
+      "epoch": 0.25210084033613445,
+      "grad_norm": 51.531711526626296,
+      "kl": 0.0,
+      "learning_rate": 4.256061301418996e-07,
+      "logps/chosen": -276.60667242005815,
+      "logps/rejected": -270.2625897381757,
+      "loss": 0.3145,
+      "rewards/chosen": 1.4284691034361374,
+      "rewards/margins": 3.5734818300910147,
+      "rewards/rejected": -2.1450127266548775,
+      "step": 480
+    },
+    {
+      "epoch": 0.25735294117647056,
+      "grad_norm": 45.63717278781431,
+      "kl": 0.0,
+      "learning_rate": 4.2264636375354283e-07,
+      "logps/chosen": -291.6655943627451,
+      "logps/rejected": -258.8236573727545,
+      "loss": 0.3174,
+      "rewards/chosen": 1.2060089111328125,
+      "rewards/margins": 4.036167989947838,
+      "rewards/rejected": -2.830159078815026,
+      "step": 490
+    },
+    {
+      "epoch": 0.26260504201680673,
+      "grad_norm": 52.57777553035371,
+      "kl": 0.0,
+      "learning_rate": 4.1963959567884045e-07,
+      "logps/chosen": -308.52905933277026,
+      "logps/rejected": -288.6666061046512,
+      "loss": 0.3152,
+      "rewards/chosen": 1.2610940675477724,
+      "rewards/margins": 4.6736672387641605,
+      "rewards/rejected": -3.4125731712163883,
+      "step": 500
+    },
+    {
+      "epoch": 0.26785714285714285,
+      "grad_norm": 40.78741469978887,
+      "kl": 0.0,
+      "learning_rate": 4.1658664448800094e-07,
+      "logps/chosen": -287.10628043831167,
+      "logps/rejected": -265.04310993975906,
+      "loss": 0.334,
+      "rewards/chosen": 0.06761282140558417,
+      "rewards/margins": 3.2680445532156366,
+      "rewards/rejected": -3.2004317318100526,
+      "step": 510
+    },
+    {
+      "epoch": 0.27310924369747897,
+      "grad_norm": 53.20527576804437,
+      "kl": 0.0,
+      "learning_rate": 4.1348834132424204e-07,
+      "logps/chosen": -319.5769211871069,
+      "logps/rejected": -310.2074679736025,
+      "loss": 0.3435,
+      "rewards/chosen": -0.4044905368636989,
+      "rewards/margins": 4.406604517241503,
+      "rewards/rejected": -4.811095054105202,
+      "step": 520
+    },
+    {
+      "epoch": 0.27836134453781514,
+      "grad_norm": 56.13445331912671,
+      "kl": 0.0,
+      "learning_rate": 4.103455296775181e-07,
+      "logps/chosen": -318.1049981174699,
+      "logps/rejected": -280.2241020698052,
+      "loss": 0.3043,
+      "rewards/chosen": 0.8269406973597515,
+      "rewards/margins": 4.47310245796243,
+      "rewards/rejected": -3.6461617606026784,
+      "step": 530
+    },
+    {
+      "epoch": 0.28361344537815125,
+      "grad_norm": 35.13309154843364,
+      "kl": 0.0,
+      "learning_rate": 4.071590651548867e-07,
+      "logps/chosen": -300.40650531045753,
+      "logps/rejected": -319.7241766467066,
+      "loss": 0.3379,
+      "rewards/chosen": -1.0223476434844772,
+      "rewards/margins": 4.300584251514588,
+      "rewards/rejected": -5.322931894999065,
+      "step": 540
+    },
+    {
+      "epoch": 0.28886554621848737,
+      "grad_norm": 52.825979085690484,
+      "kl": 0.0,
+      "learning_rate": 4.039298152475754e-07,
+      "logps/chosen": -309.6330613057325,
+      "logps/rejected": -316.5918328220859,
+      "loss": 0.3449,
+      "rewards/chosen": -0.2137961903954767,
+      "rewards/margins": 3.9556471390236183,
+      "rewards/rejected": -4.169443329419095,
+      "step": 550
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 51.016496924749916,
+      "kl": 0.0,
+      "learning_rate": 4.006586590948141e-07,
+      "logps/chosen": -282.8731328616352,
+      "logps/rejected": -268.71367915372673,
+      "loss": 0.3175,
+      "rewards/chosen": 1.230227104522897,
+      "rewards/margins": 3.262244593180214,
+      "rewards/rejected": -2.0320174886573175,
+      "step": 560
+    },
+    {
+      "epoch": 0.29936974789915966,
+      "grad_norm": 38.1092514993909,
+      "kl": 0.0,
+      "learning_rate": 3.973464872444958e-07,
+      "logps/chosen": -294.08426339285717,
+      "logps/rejected": -278.8594487028302,
+      "loss": 0.3418,
+      "rewards/chosen": 1.2165776602229716,
+      "rewards/margins": 3.751641013947421,
+      "rewards/rejected": -2.53506335372445,
+      "step": 570
+    },
+    {
+      "epoch": 0.30462184873949577,
+      "grad_norm": 58.704387106710655,
+      "kl": 0.0,
+      "learning_rate": 3.939942014107318e-07,
+      "logps/chosen": -271.2850392964072,
+      "logps/rejected": -297.9468954248366,
+      "loss": 0.3425,
+      "rewards/chosen": 0.49638238781226607,
+      "rewards/margins": 3.3393096099706607,
+      "rewards/rejected": -2.8429272221583948,
+      "step": 580
+    },
+    {
+      "epoch": 0.30987394957983194,
+      "grad_norm": 40.68813636391329,
+      "kl": 0.0,
+      "learning_rate": 3.9060271422836624e-07,
+      "logps/chosen": -287.60264185855266,
+      "logps/rejected": -279.2552780877976,
+      "loss": 0.3002,
+      "rewards/chosen": 1.2695540377968235,
+      "rewards/margins": 4.471388216903036,
+      "rewards/rejected": -3.201834179106213,
+      "step": 590
+    },
+    {
+      "epoch": 0.31512605042016806,
+      "grad_norm": 50.475157194053196,
+      "kl": 0.0,
+      "learning_rate": 3.871729490045185e-07,
+      "logps/chosen": -290.81252297794117,
+      "logps/rejected": -269.86484375,
+      "loss": 0.2895,
+      "rewards/chosen": 1.230052095301011,
+      "rewards/margins": 4.965939546472886,
+      "rewards/rejected": -3.735887451171875,
+      "step": 600
+    },
+    {
+      "epoch": 0.32037815126050423,
+      "grad_norm": 52.61703361665091,
+      "kl": 0.0,
+      "learning_rate": 3.837058394672196e-07,
+      "logps/chosen": -272.8394775390625,
+      "logps/rejected": -298.7532958984375,
+      "loss": 0.3103,
+      "rewards/chosen": 0.9608588218688965,
+      "rewards/margins": 4.247009944915772,
+      "rewards/rejected": -3.286151123046875,
+      "step": 610
+    },
+    {
+      "epoch": 0.32563025210084034,
+      "grad_norm": 46.87592251872074,
+      "kl": 0.0,
+      "learning_rate": 3.8020232951121166e-07,
+      "logps/chosen": -291.6056034482759,
+      "logps/rejected": -298.2428348214286,
+      "loss": 0.3197,
+      "rewards/chosen": 0.4831507189520474,
+      "rewards/margins": 3.7980579455145476,
+      "rewards/rejected": -3.3149072265625,
+      "step": 620
+    },
+    {
+      "epoch": 0.33088235294117646,
+      "grad_norm": 50.37994120078213,
+      "kl": 0.0,
+      "learning_rate": 3.7666337294097985e-07,
+      "logps/chosen": -306.0224609375,
+      "logps/rejected": -271.47445401278407,
+      "loss": 0.3324,
+      "rewards/chosen": 0.5180339813232422,
+      "rewards/margins": 4.124568072232333,
+      "rewards/rejected": -3.606534090909091,
+      "step": 630
+    },
+    {
+      "epoch": 0.33613445378151263,
+      "grad_norm": 42.5115670992028,
+      "kl": 0.0,
+      "learning_rate": 3.730899332110855e-07,
+      "logps/chosen": -283.47386259191177,
+      "logps/rejected": -306.39444633152175,
+      "loss": 0.2874,
+      "rewards/chosen": 0.3301387113683364,
+      "rewards/margins": 5.416864360994695,
+      "rewards/rejected": -5.086725649626358,
+      "step": 640
+    },
+    {
+      "epoch": 0.34138655462184875,
+      "grad_norm": 41.86018982710908,
+      "kl": 0.0,
+      "learning_rate": 3.694829831638738e-07,
+      "logps/chosen": -277.6752025462963,
+      "logps/rejected": -316.3676819620253,
+      "loss": 0.3214,
+      "rewards/chosen": -0.18712226255440417,
+      "rewards/margins": 4.334243518819211,
+      "rewards/rejected": -4.521365781373616,
+      "step": 650
+    },
+    {
+      "epoch": 0.34663865546218486,
+      "grad_norm": 48.14500081411287,
+      "kl": 0.0,
+      "learning_rate": 3.658435047646238e-07,
+      "logps/chosen": -287.62355587121215,
+      "logps/rejected": -288.9054939516129,
+      "loss": 0.2986,
+      "rewards/chosen": 0.6268967137192235,
+      "rewards/margins": 5.107723799152692,
+      "rewards/rejected": -4.480827085433468,
+      "step": 660
+    },
+    {
+      "epoch": 0.35189075630252103,
+      "grad_norm": 56.47592313913895,
+      "kl": 0.0,
+      "learning_rate": 3.621724888342161e-07,
+      "logps/chosen": -315.55562279929575,
+      "logps/rejected": -265.77844101123594,
+      "loss": 0.3345,
+      "rewards/chosen": 0.5301809713874065,
+      "rewards/margins": 3.9119891344623716,
+      "rewards/rejected": -3.381808163074965,
+      "step": 670
+    },
+    {
+      "epoch": 0.35714285714285715,
+      "grad_norm": 61.01903580408772,
+      "kl": 0.0,
+      "learning_rate": 3.584709347793895e-07,
+      "logps/chosen": -333.9532463121118,
+      "logps/rejected": -284.3685878537736,
+      "loss": 0.3061,
+      "rewards/chosen": 0.6283516735764023,
+      "rewards/margins": 4.496458923478132,
+      "rewards/rejected": -3.8681072499017297,
+      "step": 680
+    },
+    {
+      "epoch": 0.36239495798319327,
+      "grad_norm": 65.14201524354712,
+      "kl": 0.0,
+      "learning_rate": 3.5473985032065946e-07,
+      "logps/chosen": -337.2389689700704,
+      "logps/rejected": -293.1407566713483,
+      "loss": 0.3049,
+      "rewards/chosen": 1.298159102318992,
+      "rewards/margins": 4.470166802349715,
+      "rewards/rejected": -3.1720077000307234,
+      "step": 690
+    },
+    {
+      "epoch": 0.36764705882352944,
+      "grad_norm": 83.73292739825986,
+      "kl": 0.0,
+      "learning_rate": 3.509802512179737e-07,
+      "logps/chosen": -641.1551411290322,
+      "logps/rejected": -469.2648674242424,
+      "loss": 0.3542,
+      "rewards/chosen": -31.990240675403225,
+      "rewards/margins": -6.357922375213832,
+      "rewards/rejected": -25.632318300189393,
+      "step": 700
+    },
+    {
+      "epoch": 0.37289915966386555,
+      "grad_norm": 59.59503648599958,
+      "kl": 0.0,
+      "learning_rate": 3.4719316099417983e-07,
+      "logps/chosen": -305.71205357142856,
+      "logps/rejected": -302.07416961477986,
+      "loss": 0.3297,
+      "rewards/chosen": 0.6071341378348214,
+      "rewards/margins": 4.526243746227117,
+      "rewards/rejected": -3.9191096083922954,
+      "step": 710
+    },
+    {
+      "epoch": 0.37815126050420167,
+      "grad_norm": 52.12511665719596,
+      "kl": 0.0,
+      "learning_rate": 3.4337961065637786e-07,
+      "logps/chosen": -354.86205286949684,
+      "logps/rejected": -341.6544788431677,
+      "loss": 0.3121,
+      "rewards/chosen": -0.023094465147774173,
+      "rewards/margins": 5.537508137603584,
+      "rewards/rejected": -5.560602602751358,
+      "step": 720
+    },
+    {
+      "epoch": 0.38340336134453784,
+      "grad_norm": 57.5402410664677,
+      "kl": 0.0,
+      "learning_rate": 3.395406384152371e-07,
+      "logps/chosen": -315.799042492378,
+      "logps/rejected": -294.5041316105769,
+      "loss": 0.3199,
+      "rewards/chosen": 0.43840366456566787,
+      "rewards/margins": 4.3100072146207555,
+      "rewards/rejected": -3.871603550055088,
+      "step": 730
+    },
+    {
+      "epoch": 0.38865546218487396,
+      "grad_norm": 49.22922457925009,
+      "kl": 0.0,
+      "learning_rate": 3.356772894023505e-07,
+      "logps/chosen": -268.91911764705884,
+      "logps/rejected": -281.14432565789474,
+      "loss": 0.3627,
+      "rewards/chosen": 1.405719124697109,
+      "rewards/margins": 3.5240630802892143,
+      "rewards/rejected": -2.1183439555921053,
+      "step": 740
+    },
+    {
+      "epoch": 0.3939075630252101,
+      "grad_norm": 46.913670160363154,
+      "kl": 0.0,
+      "learning_rate": 3.317906153857054e-07,
+      "logps/chosen": -283.1244277468153,
+      "logps/rejected": -271.5762078220859,
+      "loss": 0.3514,
+      "rewards/chosen": 1.5512054832118332,
+      "rewards/margins": 3.464116740341338,
+      "rewards/rejected": -1.9129112571295053,
+      "step": 750
+    },
+    {
+      "epoch": 0.39915966386554624,
+      "grad_norm": 39.898837555375735,
+      "kl": 0.0,
+      "learning_rate": 3.2788167448334784e-07,
+      "logps/chosen": -253.08657625786162,
+      "logps/rejected": -273.2616459627329,
+      "loss": 0.3454,
+      "rewards/chosen": 0.4627175001228381,
+      "rewards/margins": 3.103327146558688,
+      "rewards/rejected": -2.64060964643585,
+      "step": 760
+    },
+    {
+      "epoch": 0.40441176470588236,
+      "grad_norm": 78.57819058757303,
+      "kl": 0.0,
+      "learning_rate": 3.2395153087531763e-07,
+      "logps/chosen": -297.46610213926175,
+      "logps/rejected": -265.79100420321635,
+      "loss": 0.3314,
+      "rewards/chosen": 0.0033199643128670304,
+      "rewards/margins": 4.24907213159631,
+      "rewards/rejected": -4.245752167283443,
+      "step": 770
+    },
+    {
+      "epoch": 0.4096638655462185,
+      "grad_norm": 58.66647545765705,
+      "kl": 0.0,
+      "learning_rate": 3.20001254513933e-07,
+      "logps/chosen": -316.0476090604027,
+      "logps/rejected": -338.84203673245617,
+      "loss": 0.3352,
+      "rewards/chosen": -0.9521625698012793,
+      "rewards/margins": 3.994815012092909,
+      "rewards/rejected": -4.946977581894188,
+      "step": 780
+    },
+    {
+      "epoch": 0.41491596638655465,
+      "grad_norm": 61.099435670035334,
+      "kl": 0.0,
+      "learning_rate": 3.160319208325044e-07,
+      "logps/chosen": -328.2163245506536,
+      "logps/rejected": -299.7488070733533,
+      "loss": 0.3143,
+      "rewards/chosen": -0.38961004120072507,
+      "rewards/margins": 4.762490664614208,
+      "rewards/rejected": -5.152100705814933,
+      "step": 790
+    },
+    {
+      "epoch": 0.42016806722689076,
+      "grad_norm": 55.61375515356607,
+      "kl": 0.0,
+      "learning_rate": 3.1204461045255597e-07,
+      "logps/chosen": -310.96216982886904,
+      "logps/rejected": -292.1716951069079,
+      "loss": 0.3393,
+      "rewards/chosen": 0.4771306628272647,
+      "rewards/margins": 4.309103688500579,
+      "rewards/rejected": -3.831973025673314,
+      "step": 800
+    },
+    {
+      "epoch": 0.4254201680672269,
+      "grad_norm": 64.83942546607469,
+      "kl": 0.0,
+      "learning_rate": 3.0804040888963367e-07,
+      "logps/chosen": -306.39021236795776,
+      "logps/rejected": -276.6224543539326,
+      "loss": 0.3007,
+      "rewards/chosen": 0.6496220978213029,
+      "rewards/margins": 5.41400753551881,
+      "rewards/rejected": -4.764385437697507,
+      "step": 810
+    },
+    {
+      "epoch": 0.43067226890756305,
+      "grad_norm": 47.53109374375667,
+      "kl": 0.0,
+      "learning_rate": 3.040204062577824e-07,
+      "logps/chosen": -319.8095262096774,
+      "logps/rejected": -276.47109375,
+      "loss": 0.3453,
+      "rewards/chosen": 0.12265574547552294,
+      "rewards/margins": 3.962314170546546,
+      "rewards/rejected": -3.839658425071023,
+      "step": 820
+    },
+    {
+      "epoch": 0.43592436974789917,
+      "grad_norm": 50.597587143566045,
+      "kl": 0.0,
+      "learning_rate": 2.999856969727704e-07,
+      "logps/chosen": -301.44910453216374,
+      "logps/rejected": -281.371670511745,
+      "loss": 0.3375,
+      "rewards/chosen": 0.8545954297160545,
+      "rewards/margins": 4.381973588797431,
+      "rewards/rejected": -3.5273781590813758,
+      "step": 830
+    },
+    {
+      "epoch": 0.4411764705882353,
+      "grad_norm": 65.2431749840782,
+      "kl": 0.0,
+      "learning_rate": 2.959373794541426e-07,
+      "logps/chosen": -288.58415743670884,
+      "logps/rejected": -275.80381944444446,
+      "loss": 0.3174,
+      "rewards/chosen": 0.5184780072562302,
+      "rewards/margins": 4.098698170860589,
+      "rewards/rejected": -3.5802201636043596,
+      "step": 840
+    },
+    {
+      "epoch": 0.44642857142857145,
+      "grad_norm": 43.10826918884011,
+      "kl": 0.0,
+      "learning_rate": 2.9187655582618407e-07,
+      "logps/chosen": -317.565112154908,
+      "logps/rejected": -297.73979896496814,
+      "loss": 0.3039,
+      "rewards/chosen": 1.0487361978168137,
+      "rewards/margins": 4.458561225185444,
+      "rewards/rejected": -3.4098250273686306,
+      "step": 850
+    },
+    {
+      "epoch": 0.45168067226890757,
+      "grad_norm": 49.36926037963924,
+      "kl": 0.0,
+      "learning_rate": 2.878043316178753e-07,
+      "logps/chosen": -290.1714082154088,
+      "logps/rejected": -275.16697399068323,
+      "loss": 0.339,
+      "rewards/chosen": -0.1980593579370271,
+      "rewards/margins": 4.149451317528811,
+      "rewards/rejected": -4.347510675465839,
+      "step": 860
+    },
+    {
+      "epoch": 0.4569327731092437,
+      "grad_norm": 31.59893236803937,
+      "kl": 0.0,
+      "learning_rate": 2.837218154619193e-07,
+      "logps/chosen": -310.3774646577381,
+      "logps/rejected": -279.89951685855266,
+      "loss": 0.3072,
+      "rewards/chosen": 0.8483944847470238,
+      "rewards/margins": 4.661724482562608,
+      "rewards/rejected": -3.813329997815584,
+      "step": 870
+    },
+    {
+      "epoch": 0.46218487394957986,
+      "grad_norm": 78.02492897783358,
+      "kl": 0.0,
+      "learning_rate": 2.796301187929257e-07,
+      "logps/chosen": -328.6445046768707,
+      "logps/rejected": -285.24498735549133,
+      "loss": 0.3409,
+      "rewards/chosen": -0.49578680959688565,
+      "rewards/margins": 3.4661660613839667,
+      "rewards/rejected": -3.9619528709808525,
+      "step": 880
+    },
+    {
+      "epoch": 0.46743697478991597,
+      "grad_norm": 47.42133886253641,
+      "kl": 0.0,
+      "learning_rate": 2.755303555448301e-07,
+      "logps/chosen": -321.7949578220859,
+      "logps/rejected": -317.8714171974522,
+      "loss": 0.3471,
+      "rewards/chosen": -1.7351532918543904,
+      "rewards/margins": 4.654421281443778,
+      "rewards/rejected": -6.389574573298169,
+      "step": 890
+    },
+    {
+      "epoch": 0.4726890756302521,
+      "grad_norm": 50.39237398761096,
+      "kl": 0.0,
+      "learning_rate": 2.7142364184763424e-07,
+      "logps/chosen": -284.39781663907286,
+      "logps/rejected": -317.32179178994085,
+      "loss": 0.3519,
+      "rewards/chosen": -0.08173387413782789,
+      "rewards/margins": 4.825091084257143,
+      "rewards/rejected": -4.906824958394971,
+      "step": 900
+    },
+    {
+      "epoch": 0.47794117647058826,
+      "grad_norm": 56.7884471471947,
+      "kl": 0.0,
+      "learning_rate": 2.673110957235479e-07,
+      "logps/chosen": -309.5387185534591,
+      "logps/rejected": -299.7813227872671,
+      "loss": 0.3017,
+      "rewards/chosen": 0.5868375766202338,
+      "rewards/margins": 4.1676898245336655,
+      "rewards/rejected": -3.5808522479134317,
+      "step": 910
+    },
+    {
+      "epoch": 0.4831932773109244,
+      "grad_norm": 55.26734943859427,
+      "kl": 0.0,
+      "learning_rate": 2.6319383678261557e-07,
+      "logps/chosen": -322.89912539308176,
+      "logps/rejected": -293.4796680900621,
+      "loss": 0.3131,
+      "rewards/chosen": 0.8568071089450668,
+      "rewards/margins": 4.327623205849182,
+      "rewards/rejected": -3.470816096904115,
+      "step": 920
+    },
+    {
+      "epoch": 0.4884453781512605,
+      "grad_norm": 43.186318715354865,
+      "kl": 0.0,
+      "learning_rate": 2.5907298591791105e-07,
+      "logps/chosen": -297.91787462349396,
+      "logps/rejected": -322.5411931818182,
+      "loss": 0.2897,
+      "rewards/chosen": 0.7378294151949595,
+      "rewards/margins": 5.026299398580715,
+      "rewards/rejected": -4.288469983385755,
+      "step": 930
+    },
+    {
+      "epoch": 0.49369747899159666,
+      "grad_norm": 75.34771457681897,
+      "kl": 0.0,
+      "learning_rate": 2.5494966500038264e-07,
+      "logps/chosen": -304.362060546875,
+      "logps/rejected": -301.68695746527777,
+      "loss": 0.3408,
+      "rewards/chosen": -0.0454999641938643,
+      "rewards/margins": 4.585332284070025,
+      "rewards/rejected": -4.630832248263889,
+      "step": 940
+    },
+    {
+      "epoch": 0.4989495798319328,
+      "grad_norm": 70.74960017005274,
+      "kl": 0.0,
+      "learning_rate": 2.508249965734319e-07,
+      "logps/chosen": -322.87849884969324,
+      "logps/rejected": -298.17384056528664,
+      "loss": 0.2932,
+      "rewards/chosen": 0.08947515780209032,
+      "rewards/margins": 5.383114517002927,
+      "rewards/rejected": -5.293639359200836,
+      "step": 950
+    },
+    {
+      "epoch": 0.5042016806722689,
+      "grad_norm": 71.05303842998583,
+      "kl": 0.0,
+      "learning_rate": 2.467001035473103e-07,
+      "logps/chosen": -305.84786676646706,
+      "logps/rejected": -311.00056168300654,
+      "loss": 0.3135,
+      "rewards/chosen": 0.8616841024981288,
+      "rewards/margins": 4.789382175542451,
+      "rewards/rejected": -3.927698073044322,
+      "step": 960
+    },
+    {
+      "epoch": 0.509453781512605,
+      "grad_norm": 51.90598218741456,
+      "kl": 0.0,
+      "learning_rate": 2.425761088934142e-07,
+      "logps/chosen": -275.5298913043478,
+      "logps/rejected": -261.6807439072327,
+      "loss": 0.3269,
+      "rewards/chosen": 0.6217543797463364,
+      "rewards/margins": 4.42957770484952,
+      "rewards/rejected": -3.807823325103184,
+      "step": 970
+    },
+    {
+      "epoch": 0.5147058823529411,
+      "grad_norm": 49.01333342668485,
+      "kl": 0.0,
+      "learning_rate": 2.3845413533856514e-07,
+      "logps/chosen": -303.1413620283019,
+      "logps/rejected": -294.9575892857143,
+      "loss": 0.2877,
+      "rewards/chosen": 0.7997542567223123,
+      "rewards/margins": 5.2810229084595015,
+      "rewards/rejected": -4.48126865173719,
+      "step": 980
+    },
+    {
+      "epoch": 0.5199579831932774,
+      "grad_norm": 45.4420006215638,
+      "kl": 0.0,
+      "learning_rate": 2.343353050593553e-07,
+      "logps/chosen": -315.00589576863354,
+      "logps/rejected": -299.73405562106916,
+      "loss": 0.3082,
+      "rewards/chosen": 0.48438277155716225,
+      "rewards/margins": 5.411952082927052,
+      "rewards/rejected": -4.92756931136989,
+      "step": 990
+    },
+    {
+      "epoch": 0.5252100840336135,
+      "grad_norm": 44.01774159207231,
+      "kl": 0.0,
+      "learning_rate": 2.3022073937664383e-07,
+      "logps/chosen": -304.87012987012986,
+      "logps/rejected": -277.4141330948795,
+      "loss": 0.3005,
+      "rewards/chosen": 1.3989817631709112,
+      "rewards/margins": 4.661985928268803,
+      "rewards/rejected": -3.2630041650978914,
+      "step": 1000
+    },
+    {
+      "epoch": 0.5304621848739496,
+      "grad_norm": 48.00123102792709,
+      "kl": 0.0,
+      "learning_rate": 2.261115584502849e-07,
+      "logps/chosen": -307.63991253930817,
+      "logps/rejected": -270.22658676242236,
+      "loss": 0.326,
+      "rewards/chosen": 1.3268984428741648,
+      "rewards/margins": 3.5992706708924103,
+      "rewards/rejected": -2.2723722280182455,
+      "step": 1010
+    },
+    {
+      "epoch": 0.5357142857142857,
+      "grad_norm": 101.16998848852474,
+      "kl": 0.0,
+      "learning_rate": 2.2200888097417302e-07,
+      "logps/chosen": -260.3294677734375,
+      "logps/rejected": -272.8283203125,
+      "loss": 0.3344,
+      "rewards/chosen": 1.6289764404296876,
+      "rewards/margins": 3.6363025665283204,
+      "rewards/rejected": -2.007326126098633,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5409663865546218,
+      "grad_norm": 45.84798251584408,
+      "kl": 0.0,
+      "learning_rate": 2.1791382387168684e-07,
+      "logps/chosen": -293.09831912878786,
+      "logps/rejected": -303.9574848790323,
+      "loss": 0.3042,
+      "rewards/chosen": 0.6066324407404119,
+      "rewards/margins": 5.404102435070048,
+      "rewards/rejected": -4.797469994329637,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5462184873949579,
+      "grad_norm": 40.19468881639845,
+      "kl": 0.0,
+      "learning_rate": 2.1382750199161495e-07,
+      "logps/chosen": -319.7741268382353,
+      "logps/rejected": -292.6617708333333,
+      "loss": 0.3023,
+      "rewards/chosen": 1.018133544921875,
+      "rewards/margins": 4.818516031901042,
+      "rewards/rejected": -3.8003824869791667,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5514705882352942,
+      "grad_norm": 33.38152973598087,
+      "kl": 0.0,
+      "learning_rate": 2.0975102780464673e-07,
+      "logps/chosen": -311.2853467987805,
+      "logps/rejected": -304.88221153846155,
+      "loss": 0.3141,
+      "rewards/chosen": 1.0162078113090702,
+      "rewards/margins": 4.974026258324295,
+      "rewards/rejected": -3.9578184470152245,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5567226890756303,
+      "grad_norm": 63.954166267932834,
+      "kl": 0.0,
+      "learning_rate": 2.0568551110051074e-07,
+      "logps/chosen": -264.285549331761,
+      "logps/rejected": -278.77358307453414,
+      "loss": 0.3119,
+      "rewards/chosen": 1.0249821884827044,
+      "rewards/margins": 4.825221324983093,
+      "rewards/rejected": -3.8002391365003882,
+      "step": 1060
+    },
+    {
+      "epoch": 0.5619747899159664,
+      "grad_norm": 41.06781071234312,
+      "kl": 0.0,
+      "learning_rate": 2.016320586858422e-07,
+      "logps/chosen": -276.17345252403845,
+      "logps/rejected": -310.3708555640244,
+      "loss": 0.2879,
+      "rewards/chosen": 1.260654547275641,
+      "rewards/margins": 5.546377977630062,
+      "rewards/rejected": -4.2857234303544205,
+      "step": 1070
+    },
+    {
+      "epoch": 0.5672268907563025,
+      "grad_norm": 48.37440945890147,
+      "kl": 0.0,
+      "learning_rate": 1.9759177408286337e-07,
+      "logps/chosen": -316.57014678030305,
+      "logps/rejected": -284.27633568548384,
+      "loss": 0.3181,
+      "rewards/chosen": 1.2529100822679924,
+      "rewards/margins": 4.161886384876359,
+      "rewards/rejected": -2.908976302608367,
+      "step": 1080
+    },
+    {
+      "epoch": 0.5724789915966386,
+      "grad_norm": 43.39106017396027,
+      "kl": 0.0,
+      "learning_rate": 1.9356575722895808e-07,
+      "logps/chosen": -298.2481151660839,
+      "logps/rejected": -282.445334569209,
+      "loss": 0.3094,
+      "rewards/chosen": 1.7649928806545017,
+      "rewards/margins": 4.975185889680809,
+      "rewards/rejected": -3.2101930090263067,
+      "step": 1090
+    },
+    {
+      "epoch": 0.5777310924369747,
+      "grad_norm": 60.62146730688155,
+      "kl": 0.0,
+      "learning_rate": 1.895551041772216e-07,
+      "logps/chosen": -307.72318892045456,
+      "logps/rejected": -269.39409722222223,
+      "loss": 0.3155,
+      "rewards/chosen": 1.027849023992365,
+      "rewards/margins": 3.9311439051772608,
+      "rewards/rejected": -2.903294881184896,
+      "step": 1100
+    },
+    {
+      "epoch": 0.582983193277311,
+      "grad_norm": 87.64642171640351,
+      "kl": 0.0,
+      "learning_rate": 1.8556090679806847e-07,
+      "logps/chosen": -302.53110881024094,
+      "logps/rejected": -283.19252232142856,
+      "loss": 0.299,
+      "rewards/chosen": 1.1432133869952465,
+      "rewards/margins": 5.261560554963997,
+      "rewards/rejected": -4.11834716796875,
+      "step": 1110
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 61.71028625828252,
+      "kl": 0.0,
+      "learning_rate": 1.8158425248197928e-07,
+      "logps/chosen": -325.427734375,
+      "logps/rejected": -312.36092748397436,
+      "loss": 0.2687,
+      "rewards/chosen": 1.890699060951791,
+      "rewards/margins": 5.755905657130677,
+      "rewards/rejected": -3.8652065961788864,
+      "step": 1120
+    },
+    {
+      "epoch": 0.5934873949579832,
+      "grad_norm": 65.2119562803559,
+      "kl": 0.0,
+      "learning_rate": 1.7762622384346609e-07,
+      "logps/chosen": -291.2824315200617,
+      "logps/rejected": -258.82664161392404,
+      "loss": 0.3013,
+      "rewards/chosen": 1.8866918116440008,
+      "rewards/margins": 4.205873229761089,
+      "rewards/rejected": -2.3191814181170884,
+      "step": 1130
+    },
+    {
+      "epoch": 0.5987394957983193,
+      "grad_norm": 38.01395730830506,
+      "kl": 0.0,
+      "learning_rate": 1.7368789842633907e-07,
+      "logps/chosen": -322.3194526336478,
+      "logps/rejected": -307.19121215062114,
+      "loss": 0.3003,
+      "rewards/chosen": 1.5120626005736537,
+      "rewards/margins": 4.748744047099275,
+      "rewards/rejected": -3.2366814465256213,
+      "step": 1140
+    },
+    {
+      "epoch": 0.6039915966386554,
+      "grad_norm": 57.24723300086865,
+      "kl": 0.0,
+      "learning_rate": 1.697703484103532e-07,
+      "logps/chosen": -297.3913395579268,
+      "logps/rejected": -276.43687399839746,
+      "loss": 0.3062,
+      "rewards/chosen": 1.2956105674185403,
+      "rewards/margins": 5.099581686834606,
+      "rewards/rejected": -3.8039711194160657,
+      "step": 1150
+    },
+    {
+      "epoch": 0.6092436974789915,
+      "grad_norm": 46.71083438381099,
+      "kl": 0.0,
+      "learning_rate": 1.6587464031931526e-07,
+      "logps/chosen": -307.08378031716416,
+      "logps/rejected": -292.77835181451616,
+      "loss": 0.3292,
+      "rewards/chosen": 1.054947810386544,
+      "rewards/margins": 4.291957494631335,
+      "rewards/rejected": -3.2370096842447915,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6144957983193278,
+      "grad_norm": 28.548641210768817,
+      "kl": 0.0,
+      "learning_rate": 1.6200183473073048e-07,
+      "logps/chosen": -301.33444602272726,
+      "logps/rejected": -287.6282510080645,
+      "loss": 0.2939,
+      "rewards/chosen": 0.8960710005326704,
+      "rewards/margins": 4.915845997382469,
+      "rewards/rejected": -4.019774996849798,
+      "step": 1170
+    },
+    {
+      "epoch": 0.6197478991596639,
+      "grad_norm": 75.28423549841818,
+      "kl": 0.0,
+      "learning_rate": 1.5815298598706888e-07,
+      "logps/chosen": -294.9990295031056,
+      "logps/rejected": -297.1986536949685,
+      "loss": 0.3023,
+      "rewards/chosen": 0.4812743382424301,
+      "rewards/margins": 5.439699416244593,
+      "rewards/rejected": -4.958425078002162,
+      "step": 1180
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 65.4384114129384,
+      "kl": 0.0,
+      "learning_rate": 1.5432914190872756e-07,
+      "logps/chosen": -287.07041139240505,
+      "logps/rejected": -303.2984905478395,
+      "loss": 0.3376,
+      "rewards/chosen": 0.6506334135803995,
+      "rewards/margins": 4.541042906732406,
+      "rewards/rejected": -3.8904094931520063,
+      "step": 1190
+    },
+    {
+      "epoch": 0.6302521008403361,
+      "grad_norm": 39.03209593199308,
+      "kl": 0.0,
+      "learning_rate": 1.505313435087698e-07,
+      "logps/chosen": -302.3798961900685,
+      "logps/rejected": -300.5006510416667,
+      "loss": 0.3026,
+      "rewards/chosen": 0.7183382217198202,
+      "rewards/margins": 5.01667280381842,
+      "rewards/rejected": -4.298334582098599,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6355042016806722,
+      "grad_norm": 37.74806766053673,
+      "kl": 0.0,
+      "learning_rate": 1.4676062470951705e-07,
+      "logps/chosen": -320.5517877684049,
+      "logps/rejected": -318.0521248009554,
+      "loss": 0.3071,
+      "rewards/chosen": 0.636018296692269,
+      "rewards/margins": 5.225012997130167,
+      "rewards/rejected": -4.588994700437898,
+      "step": 1210
+    },
+    {
+      "epoch": 0.6407563025210085,
+      "grad_norm": 48.11396951304232,
+      "kl": 0.0,
+      "learning_rate": 1.430180120610711e-07,
+      "logps/chosen": -294.4245869824841,
+      "logps/rejected": -295.2305885736196,
+      "loss": 0.2893,
+      "rewards/chosen": 0.30213877198043143,
+      "rewards/margins": 5.5401721368673185,
+      "rewards/rejected": -5.238033364886887,
+      "step": 1220
+    },
+    {
+      "epoch": 0.6460084033613446,
+      "grad_norm": 54.817845691829724,
+      "kl": 0.0,
+      "learning_rate": 1.3930452446184385e-07,
+      "logps/chosen": -329.90924310064935,
+      "logps/rejected": -325.61871705572287,
+      "loss": 0.2779,
+      "rewards/chosen": 0.11350006252140193,
+      "rewards/margins": 6.256528965241658,
+      "rewards/rejected": -6.143028902720256,
+      "step": 1230
+    },
+    {
+      "epoch": 0.6512605042016807,
+      "grad_norm": 58.30938790650043,
+      "kl": 0.0,
+      "learning_rate": 1.3562117288116923e-07,
+      "logps/chosen": -327.2169507575758,
+      "logps/rejected": -302.0602318548387,
+      "loss": 0.31,
+      "rewards/chosen": 0.08168421658602627,
+      "rewards/margins": 5.523900462175744,
+      "rewards/rejected": -5.442216245589718,
+      "step": 1240
+    },
+    {
+      "epoch": 0.6565126050420168,
+      "grad_norm": 55.044099371688475,
+      "kl": 0.0,
+      "learning_rate": 1.319689600840747e-07,
+      "logps/chosen": -308.8638286226115,
+      "logps/rejected": -301.6549079754601,
+      "loss": 0.323,
+      "rewards/chosen": -0.07818331384355096,
+      "rewards/margins": 4.454612785298512,
+      "rewards/rejected": -4.532796099142063,
+      "step": 1250
+    },
+    {
+      "epoch": 0.6617647058823529,
+      "grad_norm": 102.96993890295445,
+      "kl": 0.0,
+      "learning_rate": 1.2834888035828596e-07,
+      "logps/chosen": -300.1451028963415,
+      "logps/rejected": -284.04051482371796,
+      "loss": 0.2618,
+      "rewards/chosen": 1.2945330550030965,
+      "rewards/margins": 6.069371233588834,
+      "rewards/rejected": -4.774838178585737,
+      "step": 1260
+    },
+    {
+      "epoch": 0.667016806722689,
+      "grad_norm": 62.28101991590411,
+      "kl": 0.0,
+      "learning_rate": 1.2476191924353932e-07,
+      "logps/chosen": -340.08214285714286,
+      "logps/rejected": -286.0248114224138,
+      "loss": 0.2961,
+      "rewards/chosen": 1.140911167689732,
+      "rewards/margins": 5.149687181294258,
+      "rewards/rejected": -4.008776013604526,
+      "step": 1270
+    },
+    {
+      "epoch": 0.6722689075630253,
+      "grad_norm": 56.339091641639506,
+      "kl": 0.0,
+      "learning_rate": 1.2120905326327596e-07,
+      "logps/chosen": -294.5368897928994,
+      "logps/rejected": -291.57833195364236,
+      "loss": 0.3268,
+      "rewards/chosen": 0.9549788977267474,
+      "rewards/margins": 4.6167907768658205,
+      "rewards/rejected": -3.661811879139073,
+      "step": 1280
+    },
+    {
+      "epoch": 0.6775210084033614,
+      "grad_norm": 66.6477563654386,
+      "kl": 0.0,
+      "learning_rate": 1.1769124965879091e-07,
+      "logps/chosen": -307.14776490066225,
+      "logps/rejected": -255.9688424556213,
+      "loss": 0.3042,
+      "rewards/chosen": 1.2051307728748448,
+      "rewards/margins": 4.886082675727101,
+      "rewards/rejected": -3.680951902852256,
+      "step": 1290
+    },
+    {
+      "epoch": 0.6827731092436975,
+      "grad_norm": 51.16719779623548,
+      "kl": 0.0,
+      "learning_rate": 1.1420946612590837e-07,
+      "logps/chosen": -274.20879836309524,
+      "logps/rejected": -287.1597193667763,
+      "loss": 0.3073,
+      "rewards/chosen": 0.7768129621233258,
+      "rewards/margins": 4.585587408309592,
+      "rewards/rejected": -3.8087744461862663,
+      "step": 1300
+    },
+    {
+      "epoch": 0.6880252100840336,
+      "grad_norm": 46.424902003345146,
+      "kl": 0.0,
+      "learning_rate": 1.1076465055425646e-07,
+      "logps/chosen": -295.8868777252907,
+      "logps/rejected": -271.63410578547297,
+      "loss": 0.2858,
+      "rewards/chosen": 1.1238191294115643,
+      "rewards/margins": 5.138592936421699,
+      "rewards/rejected": -4.014773807010135,
+      "step": 1310
+    },
+    {
+      "epoch": 0.6932773109243697,
+      "grad_norm": 68.63654254788709,
+      "kl": 0.0,
+      "learning_rate": 1.0735774076921128e-07,
+      "logps/chosen": -290.4839599609375,
+      "logps/rejected": -246.5270751953125,
+      "loss": 0.3271,
+      "rewards/chosen": 1.0010972976684571,
+      "rewards/margins": 4.268711280822754,
+      "rewards/rejected": -3.267613983154297,
+      "step": 1320
+    },
+    {
+      "epoch": 0.6985294117647058,
+      "grad_norm": 41.396174001243224,
+      "kl": 0.0,
+      "learning_rate": 1.039896642765809e-07,
+      "logps/chosen": -301.9284396701389,
+      "logps/rejected": -323.622314453125,
+      "loss": 0.2906,
+      "rewards/chosen": 0.9389337963528104,
+      "rewards/margins": 5.783896870083279,
+      "rewards/rejected": -4.844963073730469,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7037815126050421,
+      "grad_norm": 42.28257001832088,
+      "kl": 0.0,
+      "learning_rate": 1.0066133801009871e-07,
+      "logps/chosen": -330.92494419642856,
+      "logps/rejected": -265.96412417763156,
+      "loss": 0.2927,
+      "rewards/chosen": 1.0747142973400297,
+      "rewards/margins": 5.422101510796033,
+      "rewards/rejected": -4.347387213456003,
+      "step": 1340
+    },
+    {
+      "epoch": 0.7090336134453782,
+      "grad_norm": 84.07485027336253,
+      "kl": 0.0,
+      "learning_rate": 9.737366808179553e-08,
+      "logps/chosen": -309.5953733766234,
+      "logps/rejected": -269.1078219126506,
+      "loss": 0.2967,
+      "rewards/chosen": 0.7130060567484273,
+      "rewards/margins": 5.262509245283819,
+      "rewards/rejected": -4.549503188535391,
+      "step": 1350
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 46.16115962232735,
+      "kl": 0.0,
+      "learning_rate": 9.412754953531663e-08,
+      "logps/chosen": -268.95225954341316,
+      "logps/rejected": -302.0711039624183,
+      "loss": 0.3274,
+      "rewards/chosen": 0.8310946161875468,
+      "rewards/margins": 4.501142151803355,
+      "rewards/rejected": -3.670047535615809,
+      "step": 1360
+    },
+    {
+      "epoch": 0.7195378151260504,
+      "grad_norm": 125.1880813626159,
+      "kl": 0.0,
+      "learning_rate": 9.092386610225325e-08,
+      "logps/chosen": -269.0572060032895,
+      "logps/rejected": -291.4349655877976,
+      "loss": 0.308,
+      "rewards/chosen": 1.3923439226652448,
+      "rewards/margins": 4.397153347655944,
+      "rewards/rejected": -3.0048094249906994,
+      "step": 1370
+    },
+    {
+      "epoch": 0.7247899159663865,
+      "grad_norm": 72.34232290742432,
+      "kl": 0.0,
+      "learning_rate": 8.776348996155317e-08,
+      "logps/chosen": -295.1348721590909,
+      "logps/rejected": -292.1137348790323,
+      "loss": 0.2973,
+      "rewards/chosen": 0.5286340886896307,
+      "rewards/margins": 5.645270303407373,
+      "rewards/rejected": -5.116636214717742,
+      "step": 1380
+    },
+    {
+      "epoch": 0.7300420168067226,
+      "grad_norm": 101.10706766564086,
+      "kl": 0.0,
+      "learning_rate": 8.464728150207636e-08,
+      "logps/chosen": -298.3083235062893,
+      "logps/rejected": -309.3073563664596,
+      "loss": 0.2942,
+      "rewards/chosen": 1.189275585630405,
+      "rewards/margins": 5.284051052075474,
+      "rewards/rejected": -4.0947754664450695,
+      "step": 1390
+    },
+    {
+      "epoch": 0.7352941176470589,
+      "grad_norm": 32.92022315972127,
+      "kl": 0.0,
+      "learning_rate": 8.15760890883607e-08,
+      "logps/chosen": -282.652443272293,
+      "logps/rejected": -268.00280387269936,
+      "loss": 0.3095,
+      "rewards/chosen": 0.9466076504652667,
+      "rewards/margins": 4.818131267700696,
+      "rewards/rejected": -3.8715236172354293,
+      "step": 1400
+    },
+    {
+      "epoch": 0.740546218487395,
+      "grad_norm": 66.0924893411411,
+      "kl": 0.0,
+      "learning_rate": 7.855074882966103e-08,
+      "logps/chosen": -317.7045433407738,
+      "logps/rejected": -290.1392115542763,
+      "loss": 0.2904,
+      "rewards/chosen": 0.8157288687569755,
+      "rewards/margins": 5.654886776343324,
+      "rewards/rejected": -4.8391579075863485,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7457983193277311,
+      "grad_norm": 29.988518434005318,
+      "kl": 0.0,
+      "learning_rate": 7.557208435232449e-08,
+      "logps/chosen": -281.4327616494083,
+      "logps/rejected": -294.1578797599338,
+      "loss": 0.2786,
+      "rewards/chosen": 1.3708069028233636,
+      "rewards/margins": 5.1220817695969565,
+      "rewards/rejected": -3.751274866773593,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7510504201680672,
+      "grad_norm": 137.787205313578,
+      "kl": 0.0,
+      "learning_rate": 7.264090657556443e-08,
+      "logps/chosen": -301.6008921967456,
+      "logps/rejected": -260.6448158112583,
+      "loss": 0.3048,
+      "rewards/chosen": 0.6026445196930473,
+      "rewards/margins": 5.368352381991268,
+      "rewards/rejected": -4.76570786229822,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7563025210084033,
+      "grad_norm": 72.85612810540347,
+      "kl": 0.0,
+      "learning_rate": 6.975801349069385e-08,
+      "logps/chosen": -284.1950284090909,
+      "logps/rejected": -288.5327872983871,
+      "loss": 0.2889,
+      "rewards/chosen": 0.9271378950639204,
+      "rewards/margins": 5.736759792115331,
+      "rewards/rejected": -4.809621897051411,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7615546218487395,
+      "grad_norm": 102.78241617760347,
+      "kl": 0.0,
+      "learning_rate": 6.692418994387799e-08,
+      "logps/chosen": -288.68474264705884,
+      "logps/rejected": -295.6461452095808,
+      "loss": 0.2864,
+      "rewards/chosen": 1.031971950157016,
+      "rewards/margins": 5.655328021218018,
+      "rewards/rejected": -4.623356071061003,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7668067226890757,
+      "grad_norm": 65.20920984482183,
+      "kl": 0.0,
+      "learning_rate": 6.414020742246593e-08,
+      "logps/chosen": -290.9591749237805,
+      "logps/rejected": -305.4570562900641,
+      "loss": 0.2474,
+      "rewards/chosen": 1.242940390982279,
+      "rewards/margins": 6.340856431647342,
+      "rewards/rejected": -5.097916040665064,
+      "step": 1460
+    },
+    {
+      "epoch": 0.7720588235294118,
+      "grad_norm": 66.73235542937682,
+      "kl": 0.0,
+      "learning_rate": 6.140682384495902e-08,
+      "logps/chosen": -311.04422530594405,
+      "logps/rejected": -286.8678937146893,
+      "loss": 0.2764,
+      "rewards/chosen": 1.3596580878837958,
+      "rewards/margins": 5.264105654201769,
+      "rewards/rejected": -3.904447566317973,
+      "step": 1470
+    },
+    {
+      "epoch": 0.7773109243697479,
+      "grad_norm": 86.01646021421267,
+      "kl": 0.0,
+      "learning_rate": 5.872478335467298e-08,
+      "logps/chosen": -266.1013243140244,
+      "logps/rejected": -272.6937850560897,
+      "loss": 0.3282,
+      "rewards/chosen": 1.3123749523628048,
+      "rewards/margins": 4.397385187489007,
+      "rewards/rejected": -3.085010235126202,
+      "step": 1480
+    },
+    {
+      "epoch": 0.782563025210084,
+      "grad_norm": 40.2582163158674,
+      "kl": 0.0,
+      "learning_rate": 5.60948161171505e-08,
+      "logps/chosen": -293.6717694256757,
+      "logps/rejected": -290.36904978197674,
+      "loss": 0.3256,
+      "rewards/chosen": 1.6847073323017843,
+      "rewards/margins": 4.269535376394717,
+      "rewards/rejected": -2.5848280440929323,
+      "step": 1490
+    },
+    {
+      "epoch": 0.7878151260504201,
+      "grad_norm": 63.304715322652115,
+      "kl": 0.0,
+      "learning_rate": 5.351763812137916e-08,
+      "logps/chosen": -276.54271343954247,
+      "logps/rejected": -297.85116485778445,
+      "loss": 0.307,
+      "rewards/chosen": 1.3718405330882353,
+      "rewards/margins": 4.822991575729516,
+      "rewards/rejected": -3.45115104264128,
+      "step": 1500
+    },
+    {
+      "epoch": 0.7930672268907563,
+      "grad_norm": 52.372015337826504,
+      "kl": 0.0,
+      "learning_rate": 5.0993950984868836e-08,
+      "logps/chosen": -268.07744565217394,
+      "logps/rejected": -287.0620332154088,
+      "loss": 0.331,
+      "rewards/chosen": 1.229935924458948,
+      "rewards/margins": 4.45942081844971,
+      "rewards/rejected": -3.2294848939907626,
+      "step": 1510
+    },
+    {
+      "epoch": 0.7983193277310925,
+      "grad_norm": 32.103803649440515,
+      "kl": 0.0,
+      "learning_rate": 4.8524441762641284e-08,
+      "logps/chosen": -314.27061222484275,
+      "logps/rejected": -264.5042944487578,
+      "loss": 0.273,
+      "rewards/chosen": 1.570641667587952,
+      "rewards/margins": 5.312982324129101,
+      "rewards/rejected": -3.742340656541149,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8035714285714286,
+      "grad_norm": 43.137151503875806,
+      "kl": 0.0,
+      "learning_rate": 4.6109782760184956e-08,
+      "logps/chosen": -277.4168693862275,
+      "logps/rejected": -293.28860294117646,
+      "loss": 0.2971,
+      "rewards/chosen": 1.7373914889946669,
+      "rewards/margins": 4.780555711063703,
+      "rewards/rejected": -3.043164222069036,
+      "step": 1530
+    },
+    {
+      "epoch": 0.8088235294117647,
+      "grad_norm": 50.02695394326994,
+      "kl": 0.0,
+      "learning_rate": 4.375063135042445e-08,
+      "logps/chosen": -295.5123428254438,
+      "logps/rejected": -317.11524730960264,
+      "loss": 0.2655,
+      "rewards/chosen": 1.741723596697023,
+      "rewards/margins": 5.875752757758695,
+      "rewards/rejected": -4.134029161061672,
+      "step": 1540
+    },
+    {
+      "epoch": 0.8140756302521008,
+      "grad_norm": 39.113276722110314,
+      "kl": 0.0,
+      "learning_rate": 4.144762979475575e-08,
+      "logps/chosen": -296.8016826923077,
+      "logps/rejected": -275.8460451977401,
+      "loss": 0.3124,
+      "rewards/chosen": 1.1531088235494973,
+      "rewards/margins": 4.638992460886185,
+      "rewards/rejected": -3.4858836373366877,
+      "step": 1550
+    },
+    {
+      "epoch": 0.819327731092437,
+      "grad_norm": 65.74166961412382,
+      "kl": 0.0,
+      "learning_rate": 3.9201405068195385e-08,
+      "logps/chosen": -300.7455797697368,
+      "logps/rejected": -287.8294735863095,
+      "loss": 0.3239,
+      "rewards/chosen": 1.3694068507144326,
+      "rewards/margins": 4.3731968432739565,
+      "rewards/rejected": -3.0037899925595237,
+      "step": 1560
+    },
+    {
+      "epoch": 0.8245798319327731,
+      "grad_norm": 40.97531017956201,
+      "kl": 0.0,
+      "learning_rate": 3.701256868869124e-08,
+      "logps/chosen": -308.8232851808176,
+      "logps/rejected": -251.1597923136646,
+      "loss": 0.3387,
+      "rewards/chosen": 1.3816734649850138,
+      "rewards/margins": 4.159259081007917,
+      "rewards/rejected": -2.7775856160229035,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8298319327731093,
+      "grad_norm": 67.8764255365854,
+      "kl": 0.0,
+      "learning_rate": 3.488171655064107e-08,
+      "logps/chosen": -283.0639042075163,
+      "logps/rejected": -281.1170705464072,
+      "loss": 0.3203,
+      "rewards/chosen": 1.294641232958027,
+      "rewards/margins": 4.51022142511278,
+      "rewards/rejected": -3.215580192154753,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8350840336134454,
+      "grad_norm": 50.875889688456084,
+      "kl": 0.0,
+      "learning_rate": 3.28094287626651e-08,
+      "logps/chosen": -326.3169806985294,
+      "logps/rejected": -297.6423697916667,
+      "loss": 0.295,
+      "rewards/chosen": 1.5313853544347427,
+      "rewards/margins": 5.293017434512867,
+      "rewards/rejected": -3.761632080078125,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8403361344537815,
+      "grad_norm": 46.374855210212814,
+      "kl": 0.0,
+      "learning_rate": 3.079626948967534e-08,
+      "logps/chosen": -304.4027423469388,
+      "logps/rejected": -287.12953847543355,
+      "loss": 0.3133,
+      "rewards/chosen": 1.1032969156901042,
+      "rewards/margins": 4.994385853438478,
+      "rewards/rejected": -3.8910889377483744,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8455882352941176,
+      "grad_norm": 55.05693685423068,
+      "kl": 0.0,
+      "learning_rate": 2.88427867992862e-08,
+      "logps/chosen": -289.62710160818716,
+      "logps/rejected": -261.55342911073825,
+      "loss": 0.2771,
+      "rewards/chosen": 1.5187704521313048,
+      "rewards/margins": 5.328108912963939,
+      "rewards/rejected": -3.809338460832634,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8508403361344538,
+      "grad_norm": 40.13605931494857,
+      "kl": 0.0,
+      "learning_rate": 2.6949512512606965e-08,
+      "logps/chosen": -292.2077305169753,
+      "logps/rejected": -295.6835195806962,
+      "loss": 0.2883,
+      "rewards/chosen": 1.2319991500289351,
+      "rewards/margins": 5.180050506091039,
+      "rewards/rejected": -3.9480513560621042,
+      "step": 1620
+    },
+    {
+      "epoch": 0.8560924369747899,
+      "grad_norm": 26.234795280971806,
+      "kl": 0.0,
+      "learning_rate": 2.5116962059457653e-08,
+      "logps/chosen": -302.0387451171875,
+      "logps/rejected": -288.72607421875,
+      "loss": 0.2664,
+      "rewards/chosen": 1.2181474685668945,
+      "rewards/margins": 5.965513038635254,
+      "rewards/rejected": -4.747365570068359,
+      "step": 1630
+    },
+    {
+      "epoch": 0.8613445378151261,
+      "grad_norm": 88.66056550298477,
+      "kl": 0.0,
+      "learning_rate": 2.334563433804687e-08,
+      "logps/chosen": -321.8770623059006,
+      "logps/rejected": -290.4252161949685,
+      "loss": 0.2794,
+      "rewards/chosen": 1.0824178375812792,
+      "rewards/margins": 5.4567103671852495,
+      "rewards/rejected": -4.37429252960397,
+      "step": 1640
+    },
+    {
+      "epoch": 0.8665966386554622,
+      "grad_norm": 46.4344164784028,
+      "kl": 0.0,
+      "learning_rate": 2.1636011579150793e-08,
+      "logps/chosen": -295.313525390625,
+      "logps/rejected": -301.4338134765625,
+      "loss": 0.2569,
+      "rewards/chosen": 1.2288617134094237,
+      "rewards/margins": 5.767054080963135,
+      "rewards/rejected": -4.538192367553711,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8718487394957983,
+      "grad_norm": 46.720887932263636,
+      "kl": 0.0,
+      "learning_rate": 1.998855921482906e-08,
+      "logps/chosen": -277.0703369140625,
+      "logps/rejected": -252.481689453125,
+      "loss": 0.321,
+      "rewards/chosen": 0.9479250907897949,
+      "rewards/margins": 4.605030727386475,
+      "rewards/rejected": -3.65710563659668,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8771008403361344,
+      "grad_norm": 77.29491195750472,
+      "kl": 0.0,
+      "learning_rate": 1.8403725751714615e-08,
+      "logps/chosen": -291.567009066358,
+      "logps/rejected": -321.13864715189874,
+      "loss": 0.3081,
+      "rewards/chosen": 0.9058116394796489,
+      "rewards/margins": 5.581022712956706,
+      "rewards/rejected": -4.675211073477057,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8823529411764706,
+      "grad_norm": 35.132417644186994,
+      "kl": 0.0,
+      "learning_rate": 1.6881942648911074e-08,
+      "logps/chosen": -289.9874625748503,
+      "logps/rejected": -282.17639399509807,
+      "loss": 0.3084,
+      "rewards/chosen": 1.0200939064254304,
+      "rewards/margins": 5.139812570258151,
+      "rewards/rejected": -4.119718663832721,
+      "step": 1680
+    },
+    {
+      "epoch": 0.8876050420168067,
+      "grad_norm": 41.03832392131175,
+      "kl": 0.0,
+      "learning_rate": 1.5423624200531115e-08,
+      "logps/chosen": -295.63090376420456,
+      "logps/rejected": -312.71739366319446,
+      "loss": 0.267,
+      "rewards/chosen": 1.9761036959561435,
+      "rewards/margins": 5.501615119702889,
+      "rewards/rejected": -3.5255114237467446,
+      "step": 1690
+    },
+    {
+      "epoch": 0.8928571428571429,
+      "grad_norm": 75.37154437951729,
+      "kl": 0.0,
+      "learning_rate": 1.4029167422908105e-08,
+      "logps/chosen": -276.8064123376623,
+      "logps/rejected": -277.1538968373494,
+      "loss": 0.2886,
+      "rewards/chosen": 1.2620481020444398,
+      "rewards/margins": 5.54502657354764,
+      "rewards/rejected": -4.2829784715032,
+      "step": 1700
+    },
+    {
+      "epoch": 0.898109243697479,
+      "grad_norm": 40.567317410299374,
+      "kl": 0.0,
+      "learning_rate": 1.2698951946511327e-08,
+      "logps/chosen": -308.1201601808176,
+      "logps/rejected": -281.5419497282609,
+      "loss": 0.3105,
+      "rewards/chosen": 0.3164803127072892,
+      "rewards/margins": 4.992804434407599,
+      "rewards/rejected": -4.67632412170031,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9033613445378151,
+      "grad_norm": 54.46098979124272,
+      "kl": 0.0,
+      "learning_rate": 1.1433339912594265e-08,
+      "logps/chosen": -306.5632858727811,
+      "logps/rejected": -280.34178394039736,
+      "loss": 0.2749,
+      "rewards/chosen": 1.2275824010724852,
+      "rewards/margins": 6.236800730697899,
+      "rewards/rejected": -5.009218329625414,
+      "step": 1720
+    },
+    {
+      "epoch": 0.9086134453781513,
+      "grad_norm": 40.407188173526485,
+      "kl": 0.0,
+      "learning_rate": 1.0232675874604608e-08,
+      "logps/chosen": -309.16411713286715,
+      "logps/rejected": -310.2364936440678,
+      "loss": 0.2894,
+      "rewards/chosen": 0.768409675651497,
+      "rewards/margins": 5.153310129725472,
+      "rewards/rejected": -4.384900454073976,
+      "step": 1730
+    },
+    {
+      "epoch": 0.9138655462184874,
+      "grad_norm": 30.438351627967617,
+      "kl": 0.0,
+      "learning_rate": 9.097286704381896e-09,
+      "logps/chosen": -317.2416330645161,
+      "logps/rejected": -278.5894886363636,
+      "loss": 0.3153,
+      "rewards/chosen": 1.0305794992754536,
+      "rewards/margins": 4.64966315812962,
+      "rewards/rejected": -3.6190836588541666,
+      "step": 1740
+    },
+    {
+      "epoch": 0.9191176470588235,
+      "grad_norm": 56.93756466862378,
+      "kl": 0.0,
+      "learning_rate": 8.02748150316937e-09,
+      "logps/chosen": -286.57449070411394,
+      "logps/rejected": -278.80803915895063,
+      "loss": 0.3173,
+      "rewards/chosen": 1.041691840449466,
+      "rewards/margins": 4.70056070076486,
+      "rewards/rejected": -3.6588688603153936,
+      "step": 1750
+    },
+    {
+      "epoch": 0.9243697478991597,
+      "grad_norm": 59.48003945550571,
+      "kl": 0.0,
+      "learning_rate": 7.023551517463089e-09,
+      "logps/chosen": -276.2664721385542,
+      "logps/rejected": -300.2993861607143,
+      "loss": 0.3055,
+      "rewards/chosen": 1.1007473497505647,
+      "rewards/margins": 4.7333291844198015,
+      "rewards/rejected": -3.632581834669237,
+      "step": 1760
+    },
+    {
+      "epoch": 0.9296218487394958,
+      "grad_norm": 75.70187200694772,
+      "kl": 0.0,
+      "learning_rate": 6.085770059722634e-09,
+      "logps/chosen": -276.1528105345912,
+      "logps/rejected": -285.99791343167703,
+      "loss": 0.2734,
+      "rewards/chosen": 1.5460262658461086,
+      "rewards/margins": 5.593088920519051,
+      "rewards/rejected": -4.047062654672943,
+      "step": 1770
+    },
+    {
+      "epoch": 0.9348739495798319,
+      "grad_norm": 62.362510517390575,
+      "kl": 0.0,
+      "learning_rate": 5.214392433963488e-09,
+      "logps/chosen": -291.6478470203488,
+      "logps/rejected": -285.2013038429054,
+      "loss": 0.324,
+      "rewards/chosen": 0.7867310989734738,
+      "rewards/margins": 4.6428056641392255,
+      "rewards/rejected": -3.8560745651657515,
+      "step": 1780
+    },
+    {
+      "epoch": 0.9401260504201681,
+      "grad_norm": 57.685751276785766,
+      "kl": 0.0,
+      "learning_rate": 4.409655866252693e-09,
+      "logps/chosen": -284.08316022398844,
+      "logps/rejected": -311.69903273809524,
+      "loss": 0.2987,
+      "rewards/chosen": 1.1329683513310604,
+      "rewards/margins": 5.054558935807038,
+      "rewards/rejected": -3.921590584475978,
+      "step": 1790
+    },
+    {
+      "epoch": 0.9453781512605042,
+      "grad_norm": 64.21944849726562,
+      "kl": 0.0,
+      "learning_rate": 3.671779440125644e-09,
+      "logps/chosen": -296.63870919585986,
+      "logps/rejected": -292.5702645705521,
+      "loss": 0.282,
+      "rewards/chosen": 1.5888889580015924,
+      "rewards/margins": 5.482801244116431,
+      "rewards/rejected": -3.8939122861148387,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9506302521008403,
+      "grad_norm": 120.91274575574961,
+      "kl": 0.0,
+      "learning_rate": 3.000964036942305e-09,
+      "logps/chosen": -286.46690883757964,
+      "logps/rejected": -307.89421970858893,
+      "loss": 0.3187,
+      "rewards/chosen": 0.656261176820014,
+      "rewards/margins": 5.1038937616551365,
+      "rewards/rejected": -4.447632584835123,
+      "step": 1810
+    },
+    {
+      "epoch": 0.9558823529411765,
+      "grad_norm": 36.13196162142592,
+      "kl": 0.0,
+      "learning_rate": 2.397392281198729e-09,
+      "logps/chosen": -302.49788306451615,
+      "logps/rejected": -277.52104640151515,
+      "loss": 0.2937,
+      "rewards/chosen": 0.9888374574722782,
+      "rewards/margins": 5.233212605436293,
+      "rewards/rejected": -4.244375147964015,
+      "step": 1820
+    },
+    {
+      "epoch": 0.9611344537815126,
+      "grad_norm": 28.996205179253028,
+      "kl": 0.0,
+      "learning_rate": 1.861228490808886e-09,
+      "logps/chosen": -293.10886452414775,
+      "logps/rejected": -297.74403211805554,
+      "loss": 0.2997,
+      "rewards/chosen": 0.9712606776844371,
+      "rewards/margins": 5.202846710128014,
+      "rewards/rejected": -4.231586032443577,
+      "step": 1830
+    },
+    {
+      "epoch": 0.9663865546218487,
+      "grad_norm": 45.07816045720644,
+      "kl": 0.0,
+      "learning_rate": 1.3926186323703903e-09,
+      "logps/chosen": -278.5950362042683,
+      "logps/rejected": -270.61345653044873,
+      "loss": 0.3545,
+      "rewards/chosen": 0.5840071236214986,
+      "rewards/margins": 3.794906672274343,
+      "rewards/rejected": -3.2108995486528444,
+      "step": 1840
+    },
+    {
+      "epoch": 0.9716386554621849,
+      "grad_norm": 37.183516112349096,
+      "kl": 0.0,
+      "learning_rate": 9.916902814261774e-10,
+      "logps/chosen": -289.34506048387095,
+      "logps/rejected": -296.7537168560606,
+      "loss": 0.2669,
+      "rewards/chosen": 1.187039283014113,
+      "rewards/margins": 5.528245041773583,
+      "rewards/rejected": -4.34120575875947,
+      "step": 1850
+    },
+    {
+      "epoch": 0.976890756302521,
+      "grad_norm": 99.37300393766564,
+      "kl": 0.0,
+      "learning_rate": 6.585525877328968e-10,
+      "logps/chosen": -291.4291068412162,
+      "logps/rejected": -333.1322901526163,
+      "loss": 0.3206,
+      "rewards/chosen": 0.7791987238703547,
+      "rewards/margins": 4.7422238391274405,
+      "rewards/rejected": -3.9630251152570857,
+      "step": 1860
+    },
+    {
+      "epoch": 0.9821428571428571,
+      "grad_norm": 91.23926744066455,
+      "kl": 0.0,
+      "learning_rate": 3.9329624554584883e-10,
+      "logps/chosen": -284.56466490963857,
+      "logps/rejected": -287.7052049512987,
+      "loss": 0.3167,
+      "rewards/chosen": 0.6357683848185711,
+      "rewards/margins": 4.381204223394058,
+      "rewards/rejected": -3.745435838575487,
+      "step": 1870
+    },
+    {
+      "epoch": 0.9873949579831933,
+      "grad_norm": 56.11511499932997,
+      "kl": 0.0,
+      "learning_rate": 1.959934689280962e-10,
+      "logps/chosen": -295.48760695684524,
+      "logps/rejected": -301.8758994654605,
+      "loss": 0.2857,
+      "rewards/chosen": 1.2652909415108817,
+      "rewards/margins": 6.0682341209927895,
+      "rewards/rejected": -4.8029431794819075,
+      "step": 1880
+    },
+    {
+      "epoch": 0.9926470588235294,
+      "grad_norm": 71.18992657480682,
+      "kl": 0.0,
+      "learning_rate": 6.669797209069017e-11,
+      "logps/chosen": -269.54678721910113,
+      "logps/rejected": -283.6238996478873,
+      "loss": 0.3009,
+      "rewards/chosen": 0.8121567629696278,
+      "rewards/margins": 5.090732821370816,
+      "rewards/rejected": -4.278576058401188,
+      "step": 1890
+    },
+    {
+      "epoch": 0.9978991596638656,
+      "grad_norm": 25.085484740800712,
+      "kl": 0.0,
+      "learning_rate": 5.444954769395771e-12,
+      "logps/chosen": -256.12128784937886,
+      "logps/rejected": -292.2530709512579,
+      "loss": 0.2815,
+      "rewards/chosen": 1.2973005401421778,
+      "rewards/margins": 5.830910950052752,
+      "rewards/rejected": -4.533610409910574,
+      "step": 1900
+    },
+    {
+      "epoch": 1.0,
+      "step": 1904,
+      "total_flos": 0.0,
+      "train_loss": 0.321955375749023,
+      "train_runtime": 31031.977,
+      "train_samples_per_second": 1.963,
+      "train_steps_per_second": 0.061
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1904,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}