Model save

README.md

@@ -0,0 +1,59 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: kaist-mistral-orpo-OHP-15k-Mathcode
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# kaist-mistral-orpo-OHP-15k-Mathcode
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 3
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.39.3
+- Pytorch 2.1.2.post303
+- Datasets 2.18.0
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.3926089354220279,
+    "train_runtime": 9824.6416,
+    "train_samples": 14963,
+    "train_samples_per_second": 4.569,
+    "train_steps_per_second": 0.071
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.3"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:530727d17679ed1bb28a8868fea4ac21785730dcbed50b4b36ff0bdbace38968
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:481d3979e9e98525e3aa070ae715ec0fea335e00706dab3989e4e93926b622a7
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a049612c9c82fff6b809d90893ce0fd8e7aa48d221dac6db6dfb2c2bcf8c0068
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.3926089354220279,
+    "train_runtime": 9824.6416,
+    "train_samples": 14963,
+    "train_samples_per_second": 4.569,
+    "train_steps_per_second": 0.071
+}

trainer_state.json

@@ -0,0 +1,1290 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 702,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.04,
+      "grad_norm": 3.8125,
+      "learning_rate": 4.997496987311431e-06,
+      "log_odds_chosen": 0.16457681357860565,
+      "log_odds_ratio": -0.6622492671012878,
+      "logits/chosen": -2.9562315940856934,
+      "logits/rejected": -2.943472385406494,
+      "logps/chosen": -0.6016947031021118,
+      "logps/rejected": -0.675441563129425,
+      "loss": 0.9961,
+      "nll_loss": 0.7968690991401672,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12033893167972565,
+      "rewards/margins": 0.014749372377991676,
+      "rewards/rejected": -0.13508832454681396,
+      "step": 10
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 3.5,
+      "learning_rate": 4.989992961303738e-06,
+      "log_odds_chosen": 0.09340940415859222,
+      "log_odds_ratio": -0.6970354914665222,
+      "logits/chosen": -3.0346553325653076,
+      "logits/rejected": -3.047511577606201,
+      "logps/chosen": -0.5685297846794128,
+      "logps/rejected": -0.6139942407608032,
+      "loss": 0.4901,
+      "nll_loss": 0.35993438959121704,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.11370595544576645,
+      "rewards/margins": 0.00909288041293621,
+      "rewards/rejected": -0.1227988451719284,
+      "step": 20
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 3.703125,
+      "learning_rate": 4.977502948114772e-06,
+      "log_odds_chosen": 0.06927172094583511,
+      "log_odds_ratio": -0.7004188895225525,
+      "logits/chosen": -3.142512798309326,
+      "logits/rejected": -3.1354269981384277,
+      "logps/chosen": -0.5674165487289429,
+      "logps/rejected": -0.590361475944519,
+      "loss": 0.4797,
+      "nll_loss": 0.3510151505470276,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.11348330974578857,
+      "rewards/margins": 0.00458899512887001,
+      "rewards/rejected": -0.11807229369878769,
+      "step": 30
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 3.484375,
+      "learning_rate": 4.960051957873726e-06,
+      "log_odds_chosen": 0.049079131335020065,
+      "log_odds_ratio": -0.709546685218811,
+      "logits/chosen": -3.2224674224853516,
+      "logits/rejected": -3.239351749420166,
+      "logps/chosen": -0.564799427986145,
+      "logps/rejected": -0.5826238393783569,
+      "loss": 0.4883,
+      "nll_loss": 0.3356490135192871,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.11295988410711288,
+      "rewards/margins": 0.003564885351806879,
+      "rewards/rejected": -0.11652477085590363,
+      "step": 40
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 3.59375,
+      "learning_rate": 4.937674934620601e-06,
+      "log_odds_chosen": 0.1465633362531662,
+      "log_odds_ratio": -0.6634026169776917,
+      "logits/chosen": -3.2599196434020996,
+      "logits/rejected": -3.257382869720459,
+      "logps/chosen": -0.5410924553871155,
+      "logps/rejected": -0.6060024499893188,
+      "loss": 0.4577,
+      "nll_loss": 0.3106100261211395,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.10821850597858429,
+      "rewards/margins": 0.012981968931853771,
+      "rewards/rejected": -0.12120047956705093,
+      "step": 50
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 3.234375,
+      "learning_rate": 4.9104166863339065e-06,
+      "log_odds_chosen": 0.061258088797330856,
+      "log_odds_ratio": -0.7155165076255798,
+      "logits/chosen": -3.3071017265319824,
+      "logits/rejected": -3.3195369243621826,
+      "logps/chosen": -0.6092182397842407,
+      "logps/rejected": -0.6360430121421814,
+      "loss": 0.494,
+      "nll_loss": 0.3727017939090729,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.12184363603591919,
+      "rewards/margins": 0.005364974495023489,
+      "rewards/rejected": -0.1272086203098297,
+      "step": 60
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 3.28125,
+      "learning_rate": 4.878331795206705e-06,
+      "log_odds_chosen": 0.09167635440826416,
+      "log_odds_ratio": -0.7004056572914124,
+      "logits/chosen": -3.2954325675964355,
+      "logits/rejected": -3.311220169067383,
+      "logps/chosen": -0.5609530210494995,
+      "logps/rejected": -0.6090016961097717,
+      "loss": 0.4762,
+      "nll_loss": 0.31441202759742737,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.1121906191110611,
+      "rewards/margins": 0.009609726257622242,
+      "rewards/rejected": -0.12180032581090927,
+      "step": 70
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 3.40625,
+      "learning_rate": 4.841484508350679e-06,
+      "log_odds_chosen": 0.1308586597442627,
+      "log_odds_ratio": -0.6873519420623779,
+      "logits/chosen": -3.2609565258026123,
+      "logits/rejected": -3.302786350250244,
+      "logps/chosen": -0.5968475937843323,
+      "logps/rejected": -0.6581549644470215,
+      "loss": 0.4895,
+      "nll_loss": 0.36970850825309753,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1193695068359375,
+      "rewards/margins": 0.012261486612260342,
+      "rewards/rejected": -0.13163098692893982,
+      "step": 80
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 3.78125,
+      "learning_rate": 4.799948609147061e-06,
+      "log_odds_chosen": 0.06979366391897202,
+      "log_odds_ratio": -0.7083112001419067,
+      "logits/chosen": -3.289924144744873,
+      "logits/rejected": -3.326308012008667,
+      "logps/chosen": -0.6024026274681091,
+      "logps/rejected": -0.6323495507240295,
+      "loss": 0.4868,
+      "nll_loss": 0.376103937625885,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.12048052251338959,
+      "rewards/margins": 0.005989375524222851,
+      "rewards/rejected": -0.12646988034248352,
+      "step": 90
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 3.4375,
+      "learning_rate": 4.753807269502041e-06,
+      "log_odds_chosen": 0.12073998153209686,
+      "log_odds_ratio": -0.6814596652984619,
+      "logits/chosen": -3.2658774852752686,
+      "logits/rejected": -3.276677370071411,
+      "logps/chosen": -0.5358774662017822,
+      "logps/rejected": -0.5820270776748657,
+      "loss": 0.4686,
+      "nll_loss": 0.3315598964691162,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.10717550665140152,
+      "rewards/margins": 0.009229922667145729,
+      "rewards/rejected": -0.1164054125547409,
+      "step": 100
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 3.140625,
+      "learning_rate": 4.703152883302498e-06,
+      "log_odds_chosen": 0.12317148596048355,
+      "log_odds_ratio": -0.6822798848152161,
+      "logits/chosen": -3.279986619949341,
+      "logits/rejected": -3.3097903728485107,
+      "logps/chosen": -0.5460432171821594,
+      "logps/rejected": -0.6105918884277344,
+      "loss": 0.4745,
+      "nll_loss": 0.3556433618068695,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.10920864343643188,
+      "rewards/margins": 0.01290974486619234,
+      "rewards/rejected": -0.12211839854717255,
+      "step": 110
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 3.03125,
+      "learning_rate": 4.648086881405542e-06,
+      "log_odds_chosen": 0.11489371955394745,
+      "log_odds_ratio": -0.689535915851593,
+      "logits/chosen": -3.255175828933716,
+      "logits/rejected": -3.275517225265503,
+      "logps/chosen": -0.5755141377449036,
+      "logps/rejected": -0.6235055923461914,
+      "loss": 0.475,
+      "nll_loss": 0.32820188999176025,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.11510282754898071,
+      "rewards/margins": 0.00959828682243824,
+      "rewards/rejected": -0.1247011050581932,
+      "step": 120
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 2.890625,
+      "learning_rate": 4.588719528532342e-06,
+      "log_odds_chosen": 0.09126593172550201,
+      "log_odds_ratio": -0.697300910949707,
+      "logits/chosen": -3.2629482746124268,
+      "logits/rejected": -3.2472147941589355,
+      "logps/chosen": -0.5701687335968018,
+      "logps/rejected": -0.618691623210907,
+      "loss": 0.4766,
+      "nll_loss": 0.3278309404850006,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.1140337586402893,
+      "rewards/margins": 0.00970458984375,
+      "rewards/rejected": -0.12373832613229752,
+      "step": 130
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 3.234375,
+      "learning_rate": 4.525169702472917e-06,
+      "log_odds_chosen": 0.055292803794145584,
+      "log_odds_ratio": -0.7131302356719971,
+      "logits/chosen": -3.1946372985839844,
+      "logits/rejected": -3.208055019378662,
+      "logps/chosen": -0.5659917593002319,
+      "logps/rejected": -0.5857821702957153,
+      "loss": 0.4741,
+      "nll_loss": 0.32717862725257874,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.11319835484027863,
+      "rewards/margins": 0.003958091139793396,
+      "rewards/rejected": -0.11715643107891083,
+      "step": 140
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 3.28125,
+      "learning_rate": 4.457564656044056e-06,
+      "log_odds_chosen": 0.10363437980413437,
+      "log_odds_ratio": -0.6924600005149841,
+      "logits/chosen": -3.227095127105713,
+      "logits/rejected": -3.269847869873047,
+      "logps/chosen": -0.540108859539032,
+      "logps/rejected": -0.5899955034255981,
+      "loss": 0.4874,
+      "nll_loss": 0.3099691569805145,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.10802175849676132,
+      "rewards/margins": 0.009977328591048717,
+      "rewards/rejected": -0.11799909919500351,
+      "step": 150
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 3.25,
+      "learning_rate": 4.386039762276976e-06,
+      "log_odds_chosen": 0.12246614694595337,
+      "log_odds_ratio": -0.6819506883621216,
+      "logits/chosen": -3.1787781715393066,
+      "logits/rejected": -3.1843771934509277,
+      "logps/chosen": -0.556765615940094,
+      "logps/rejected": -0.6073147058486938,
+      "loss": 0.4718,
+      "nll_loss": 0.34596166014671326,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.11135313659906387,
+      "rewards/margins": 0.010109812021255493,
+      "rewards/rejected": -0.12146292626857758,
+      "step": 160
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 3.609375,
+      "learning_rate": 4.310738243344996e-06,
+      "log_odds_chosen": 0.09136945009231567,
+      "log_odds_ratio": -0.6929733753204346,
+      "logits/chosen": -3.207054853439331,
+      "logits/rejected": -3.225612163543701,
+      "logps/chosen": -0.5217511653900146,
+      "logps/rejected": -0.5743865966796875,
+      "loss": 0.4775,
+      "nll_loss": 0.34531891345977783,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.10435023158788681,
+      "rewards/margins": 0.010527082718908787,
+      "rewards/rejected": -0.11487732082605362,
+      "step": 170
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 3.21875,
+      "learning_rate": 4.231810883773999e-06,
+      "log_odds_chosen": 0.13569065928459167,
+      "log_odds_ratio": -0.6776682138442993,
+      "logits/chosen": -3.176665782928467,
+      "logits/rejected": -3.197835922241211,
+      "logps/chosen": -0.5234608054161072,
+      "logps/rejected": -0.5825861692428589,
+      "loss": 0.4797,
+      "nll_loss": 0.358028769493103,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.10469217598438263,
+      "rewards/margins": 0.011825057677924633,
+      "rewards/rejected": -0.11651722341775894,
+      "step": 180
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 2.96875,
+      "learning_rate": 4.149415728509971e-06,
+      "log_odds_chosen": 0.11225181818008423,
+      "log_odds_ratio": -0.6859180331230164,
+      "logits/chosen": -3.2352004051208496,
+      "logits/rejected": -3.2568259239196777,
+      "logps/chosen": -0.5570780634880066,
+      "logps/rejected": -0.6075831651687622,
+      "loss": 0.4563,
+      "nll_loss": 0.33723753690719604,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.11141560971736908,
+      "rewards/margins": 0.010101011954247952,
+      "rewards/rejected": -0.1215166226029396,
+      "step": 190
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 3.09375,
+      "learning_rate": 4.063717766448194e-06,
+      "log_odds_chosen": 0.09286753088235855,
+      "log_odds_ratio": -0.6982024908065796,
+      "logits/chosen": -3.177328109741211,
+      "logits/rejected": -3.204103946685791,
+      "logps/chosen": -0.5489121675491333,
+      "logps/rejected": -0.5975584387779236,
+      "loss": 0.4871,
+      "nll_loss": 0.36433055996894836,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.10978243499994278,
+      "rewards/margins": 0.009729253128170967,
+      "rewards/rejected": -0.1195116862654686,
+      "step": 200
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 3.234375,
+      "learning_rate": 3.974888600057808e-06,
+      "log_odds_chosen": 0.06380079686641693,
+      "log_odds_ratio": -0.7078055143356323,
+      "logits/chosen": -3.179089307785034,
+      "logits/rejected": -3.1974921226501465,
+      "logps/chosen": -0.5259910821914673,
+      "logps/rejected": -0.5546124577522278,
+      "loss": 0.4842,
+      "nll_loss": 0.3254118859767914,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.1051982045173645,
+      "rewards/margins": 0.0057242861948907375,
+      "rewards/rejected": -0.11092247813940048,
+      "step": 210
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 3.1875,
+      "learning_rate": 3.883106101763285e-06,
+      "log_odds_chosen": 0.07409624010324478,
+      "log_odds_ratio": -0.7109658122062683,
+      "logits/chosen": -3.2040557861328125,
+      "logits/rejected": -3.232853651046753,
+      "logps/chosen": -0.5239503979682922,
+      "logps/rejected": -0.5682771801948547,
+      "loss": 0.4671,
+      "nll_loss": 0.32592883706092834,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.10479007661342621,
+      "rewards/margins": 0.008865350857377052,
+      "rewards/rejected": -0.1136554479598999,
+      "step": 220
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 3.375,
+      "learning_rate": 3.7885540577708806e-06,
+      "log_odds_chosen": 0.17323985695838928,
+      "log_odds_ratio": -0.6644941568374634,
+      "logits/chosen": -3.242084503173828,
+      "logits/rejected": -3.256577730178833,
+      "logps/chosen": -0.5062168836593628,
+      "logps/rejected": -0.5952838063240051,
+      "loss": 0.4719,
+      "nll_loss": 0.31021052598953247,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.10124337673187256,
+      "rewards/margins": 0.01781339757144451,
+      "rewards/rejected": -0.11905677616596222,
+      "step": 230
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 3.84375,
+      "learning_rate": 3.6914218000532697e-06,
+      "log_odds_chosen": 0.369870126247406,
+      "log_odds_ratio": -0.5746415257453918,
+      "logits/chosen": -3.2143750190734863,
+      "logits/rejected": -3.2405693531036377,
+      "logps/chosen": -0.4556041657924652,
+      "logps/rejected": -0.6035604476928711,
+      "loss": 0.4236,
+      "nll_loss": 0.2927762269973755,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.09112082421779633,
+      "rewards/margins": 0.029591253027319908,
+      "rewards/rejected": -0.12071208655834198,
+      "step": 240
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 3.8125,
+      "learning_rate": 3.5919038272292824e-06,
+      "log_odds_chosen": 0.6750356554985046,
+      "log_odds_ratio": -0.4590001702308655,
+      "logits/chosen": -3.1941792964935303,
+      "logits/rejected": -3.229973554611206,
+      "logps/chosen": -0.41714033484458923,
+      "logps/rejected": -0.676815390586853,
+      "loss": 0.3754,
+      "nll_loss": 0.2869933545589447,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.08342806994915009,
+      "rewards/margins": 0.05193500965833664,
+      "rewards/rejected": -0.13536307215690613,
+      "step": 250
+    },
+    {
+      "epoch": 1.11,
+      "grad_norm": 4.0,
+      "learning_rate": 3.4901994150978926e-06,
+      "log_odds_chosen": 0.7411336302757263,
+      "log_odds_ratio": -0.43500590324401855,
+      "logits/chosen": -3.248504161834717,
+      "logits/rejected": -3.2485687732696533,
+      "logps/chosen": -0.45796918869018555,
+      "logps/rejected": -0.7692248225212097,
+      "loss": 0.3752,
+      "nll_loss": 0.29675063490867615,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.09159384667873383,
+      "rewards/margins": 0.06225113198161125,
+      "rewards/rejected": -0.15384498238563538,
+      "step": 260
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 4.125,
+      "learning_rate": 3.386512217606339e-06,
+      "log_odds_chosen": 0.672085702419281,
+      "log_odds_ratio": -0.4590581953525543,
+      "logits/chosen": -3.210575580596924,
+      "logits/rejected": -3.2388033866882324,
+      "logps/chosen": -0.4334833025932312,
+      "logps/rejected": -0.7109585404396057,
+      "loss": 0.3831,
+      "nll_loss": 0.29120856523513794,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.08669666945934296,
+      "rewards/margins": 0.055495042353868484,
+      "rewards/rejected": -0.14219172298908234,
+      "step": 270
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 4.1875,
+      "learning_rate": 3.281049859051394e-06,
+      "log_odds_chosen": 0.6232529282569885,
+      "log_odds_ratio": -0.4691483974456787,
+      "logits/chosen": -3.1913466453552246,
+      "logits/rejected": -3.213522434234619,
+      "logps/chosen": -0.457376629114151,
+      "logps/rejected": -0.7198972702026367,
+      "loss": 0.3748,
+      "nll_loss": 0.2882610559463501,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.09147532284259796,
+      "rewards/margins": 0.05250414088368416,
+      "rewards/rejected": -0.14397946000099182,
+      "step": 280
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 4.125,
+      "learning_rate": 3.17402351833036e-06,
+      "log_odds_chosen": 0.710340678691864,
+      "log_odds_ratio": -0.44652828574180603,
+      "logits/chosen": -3.149742841720581,
+      "logits/rejected": -3.189366579055786,
+      "logps/chosen": -0.43936777114868164,
+      "logps/rejected": -0.72941654920578,
+      "loss": 0.3629,
+      "nll_loss": 0.29283395409584045,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.08787355571985245,
+      "rewards/margins": 0.05800975114107132,
+      "rewards/rejected": -0.14588329195976257,
+      "step": 290
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 4.09375,
+      "learning_rate": 3.0656475060743065e-06,
+      "log_odds_chosen": 0.7323763966560364,
+      "log_odds_ratio": -0.4345877766609192,
+      "logits/chosen": -3.1514878273010254,
+      "logits/rejected": -3.199571132659912,
+      "logps/chosen": -0.44486841559410095,
+      "logps/rejected": -0.7583059072494507,
+      "loss": 0.3627,
+      "nll_loss": 0.29908788204193115,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.08897367864847183,
+      "rewards/margins": 0.06268750131130219,
+      "rewards/rejected": -0.15166118741035461,
+      "step": 300
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 4.4375,
+      "learning_rate": 2.956138835510282e-06,
+      "log_odds_chosen": 0.7746503353118896,
+      "log_odds_ratio": -0.42378655076026917,
+      "logits/chosen": -3.1741645336151123,
+      "logits/rejected": -3.2151100635528564,
+      "logps/chosen": -0.4183032512664795,
+      "logps/rejected": -0.7279097437858582,
+      "loss": 0.3604,
+      "nll_loss": 0.26949578523635864,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.08366064727306366,
+      "rewards/margins": 0.06192130595445633,
+      "rewards/rejected": -0.14558196067810059,
+      "step": 310
+    },
+    {
+      "epoch": 1.37,
+      "grad_norm": 4.28125,
+      "learning_rate": 2.8457167879118332e-06,
+      "log_odds_chosen": 0.7689862251281738,
+      "log_odds_ratio": -0.43063870072364807,
+      "logits/chosen": -3.159554958343506,
+      "logits/rejected": -3.188426971435547,
+      "logps/chosen": -0.3951800465583801,
+      "logps/rejected": -0.7116678953170776,
+      "loss": 0.3763,
+      "nll_loss": 0.27707797288894653,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.07903601229190826,
+      "rewards/margins": 0.0632975846529007,
+      "rewards/rejected": -0.14233359694480896,
+      "step": 320
+    },
+    {
+      "epoch": 1.41,
+      "grad_norm": 3.921875,
+      "learning_rate": 2.7346024735079483e-06,
+      "log_odds_chosen": 0.7219299674034119,
+      "log_odds_ratio": -0.44966334104537964,
+      "logits/chosen": -3.127485752105713,
+      "logits/rejected": -3.198312997817993,
+      "logps/chosen": -0.41490626335144043,
+      "logps/rejected": -0.7052132487297058,
+      "loss": 0.3787,
+      "nll_loss": 0.2818581163883209,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.08298125118017197,
+      "rewards/margins": 0.058061398565769196,
+      "rewards/rejected": -0.14104264974594116,
+      "step": 330
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 4.34375,
+      "learning_rate": 2.6230183887296955e-06,
+      "log_odds_chosen": 0.7794576287269592,
+      "log_odds_ratio": -0.42089906334877014,
+      "logits/chosen": -3.129941463470459,
+      "logits/rejected": -3.1785807609558105,
+      "logps/chosen": -0.4315931797027588,
+      "logps/rejected": -0.7535022497177124,
+      "loss": 0.3603,
+      "nll_loss": 0.30001071095466614,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.08631863445043564,
+      "rewards/margins": 0.06438182294368744,
+      "rewards/rejected": -0.15070047974586487,
+      "step": 340
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 4.53125,
+      "learning_rate": 2.511187970681109e-06,
+      "log_odds_chosen": 0.6614913940429688,
+      "log_odds_ratio": -0.4707488417625427,
+      "logits/chosen": -3.1603000164031982,
+      "logits/rejected": -3.1471877098083496,
+      "logps/chosen": -0.4360755980014801,
+      "logps/rejected": -0.6958964467048645,
+      "loss": 0.3618,
+      "nll_loss": 0.2804470360279083,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.0872151255607605,
+      "rewards/margins": 0.0519641637802124,
+      "rewards/rejected": -0.1391792744398117,
+      "step": 350
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 3.828125,
+      "learning_rate": 2.399335149726463e-06,
+      "log_odds_chosen": 0.6727465391159058,
+      "log_odds_ratio": -0.4600442945957184,
+      "logits/chosen": -3.1209638118743896,
+      "logits/rejected": -3.1621603965759277,
+      "logps/chosen": -0.4566575884819031,
+      "logps/rejected": -0.7309259176254272,
+      "loss": 0.3719,
+      "nll_loss": 0.3025999665260315,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.09133151918649673,
+      "rewards/margins": 0.0548536591231823,
+      "rewards/rejected": -0.14618518948554993,
+      "step": 360
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 4.46875,
+      "learning_rate": 2.2876839010898377e-06,
+      "log_odds_chosen": 0.7592717409133911,
+      "log_odds_ratio": -0.43831682205200195,
+      "logits/chosen": -3.1502082347869873,
+      "logits/rejected": -3.207881212234497,
+      "logps/chosen": -0.42622488737106323,
+      "logps/rejected": -0.7278318405151367,
+      "loss": 0.3498,
+      "nll_loss": 0.2541278004646301,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.08524497598409653,
+      "rewards/margins": 0.0603213906288147,
+      "rewards/rejected": -0.14556637406349182,
+      "step": 370
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 4.90625,
+      "learning_rate": 2.1764577963648616e-06,
+      "log_odds_chosen": 0.6783360838890076,
+      "log_odds_ratio": -0.46248960494995117,
+      "logits/chosen": -3.146242141723633,
+      "logits/rejected": -3.164093017578125,
+      "logps/chosen": -0.42164331674575806,
+      "logps/rejected": -0.6987554430961609,
+      "loss": 0.3636,
+      "nll_loss": 0.30008167028427124,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.08432865887880325,
+      "rewards/margins": 0.05542243644595146,
+      "rewards/rejected": -0.1397511065006256,
+      "step": 380
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 4.4375,
+      "learning_rate": 2.0658795558326745e-06,
+      "log_odds_chosen": 0.6860665082931519,
+      "log_odds_ratio": -0.44539427757263184,
+      "logits/chosen": -3.1428816318511963,
+      "logits/rejected": -3.202255964279175,
+      "logps/chosen": -0.4127088487148285,
+      "logps/rejected": -0.6815465688705444,
+      "loss": 0.3548,
+      "nll_loss": 0.25606799125671387,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.08254177868366241,
+      "rewards/margins": 0.05376753211021423,
+      "rewards/rejected": -0.13630932569503784,
+      "step": 390
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 4.15625,
+      "learning_rate": 1.956170602484582e-06,
+      "log_odds_chosen": 0.8614572286605835,
+      "log_odds_ratio": -0.3955555558204651,
+      "logits/chosen": -3.1366238594055176,
+      "logits/rejected": -3.1604573726654053,
+      "logps/chosen": -0.3913766145706177,
+      "logps/rejected": -0.7423630952835083,
+      "loss": 0.3487,
+      "nll_loss": 0.24887590110301971,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.07827533036470413,
+      "rewards/margins": 0.07019731402397156,
+      "rewards/rejected": -0.1484726369380951,
+      "step": 400
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 4.625,
+      "learning_rate": 1.8475506186424075e-06,
+      "log_odds_chosen": 0.7284751534461975,
+      "log_odds_ratio": -0.4521869122982025,
+      "logits/chosen": -3.1315054893493652,
+      "logits/rejected": -3.1535370349884033,
+      "logps/chosen": -0.4371741712093353,
+      "logps/rejected": -0.7353378534317017,
+      "loss": 0.3711,
+      "nll_loss": 0.2623272240161896,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.08743483573198318,
+      "rewards/margins": 0.059632737189531326,
+      "rewards/rejected": -0.1470675766468048,
+      "step": 410
+    },
+    {
+      "epoch": 1.79,
+      "grad_norm": 3.796875,
+      "learning_rate": 1.740237106064383e-06,
+      "log_odds_chosen": 0.7518190145492554,
+      "log_odds_ratio": -0.43122825026512146,
+      "logits/chosen": -3.155817747116089,
+      "logits/rejected": -3.1722676753997803,
+      "logps/chosen": -0.44021105766296387,
+      "logps/rejected": -0.752731442451477,
+      "loss": 0.3521,
+      "nll_loss": 0.27316415309906006,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.08804221451282501,
+      "rewards/margins": 0.06250409036874771,
+      "rewards/rejected": -0.15054629743099213,
+      "step": 420
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.6344449504174193e-06,
+      "log_odds_chosen": 0.7577536106109619,
+      "log_odds_ratio": -0.4262828230857849,
+      "logits/chosen": -3.112555980682373,
+      "logits/rejected": -3.175693988800049,
+      "logps/chosen": -0.4249524176120758,
+      "logps/rejected": -0.7365177273750305,
+      "loss": 0.3772,
+      "nll_loss": 0.29327088594436646,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.084990493953228,
+      "rewards/margins": 0.06231304258108139,
+      "rewards/rejected": -0.14730355143547058,
+      "step": 430
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 4.625,
+      "learning_rate": 1.5303859909878632e-06,
+      "log_odds_chosen": 0.6891047954559326,
+      "log_odds_ratio": -0.45858821272850037,
+      "logits/chosen": -3.18835711479187,
+      "logits/rejected": -3.198822021484375,
+      "logps/chosen": -0.43595075607299805,
+      "logps/rejected": -0.707006573677063,
+      "loss": 0.3569,
+      "nll_loss": 0.2663270831108093,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.08719015121459961,
+      "rewards/margins": 0.054211169481277466,
+      "rewards/rejected": -0.14140132069587708,
+      "step": 440
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 4.84375,
+      "learning_rate": 1.4282685964923643e-06,
+      "log_odds_chosen": 0.8139818906784058,
+      "log_odds_ratio": -0.41608935594558716,
+      "logits/chosen": -3.1146390438079834,
+      "logits/rejected": -3.167144775390625,
+      "logps/chosen": -0.41870421171188354,
+      "logps/rejected": -0.7421842813491821,
+      "loss": 0.3699,
+      "nll_loss": 0.2976505160331726,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.08374085277318954,
+      "rewards/margins": 0.06469599902629852,
+      "rewards/rejected": -0.14843687415122986,
+      "step": 450
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 4.53125,
+      "learning_rate": 1.328297247838241e-06,
+      "log_odds_chosen": 0.7360786199569702,
+      "log_odds_ratio": -0.4329991936683655,
+      "logits/chosen": -3.162118434906006,
+      "logits/rejected": -3.2087559700012207,
+      "logps/chosen": -0.4136020541191101,
+      "logps/rejected": -0.7124483585357666,
+      "loss": 0.3642,
+      "nll_loss": 0.2634555995464325,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.08272042125463486,
+      "rewards/margins": 0.05976926162838936,
+      "rewards/rejected": -0.14248968660831451,
+      "step": 460
+    },
+    {
+      "epoch": 2.01,
+      "grad_norm": 5.3125,
+      "learning_rate": 1.2306721286688312e-06,
+      "log_odds_chosen": 0.8309246301651001,
+      "log_odds_ratio": -0.4013638496398926,
+      "logits/chosen": -3.150937795639038,
+      "logits/rejected": -3.205261707305908,
+      "logps/chosen": -0.4131666123867035,
+      "logps/rejected": -0.7659183740615845,
+      "loss": 0.3651,
+      "nll_loss": 0.2736993134021759,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.08263333141803741,
+      "rewards/margins": 0.07055036723613739,
+      "rewards/rejected": -0.1531836986541748,
+      "step": 470
+    },
+    {
+      "epoch": 2.05,
+      "grad_norm": 4.5625,
+      "learning_rate": 1.1355887245137383e-06,
+      "log_odds_chosen": 1.1850926876068115,
+      "log_odds_ratio": -0.30928006768226624,
+      "logits/chosen": -3.197258234024048,
+      "logits/rejected": -3.233253002166748,
+      "logps/chosen": -0.3570306599140167,
+      "logps/rejected": -0.8269041776657104,
+      "loss": 0.306,
+      "nll_loss": 0.21892933547496796,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -0.07140612602233887,
+      "rewards/margins": 0.09397471696138382,
+      "rewards/rejected": -0.16538085043430328,
+      "step": 480
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 4.3125,
+      "learning_rate": 1.043237431346622e-06,
+      "log_odds_chosen": 1.2696115970611572,
+      "log_odds_ratio": -0.29188066720962524,
+      "logits/chosen": -3.060274600982666,
+      "logits/rejected": -3.112060785293579,
+      "logps/chosen": -0.37511518597602844,
+      "logps/rejected": -0.8814083933830261,
+      "loss": 0.3133,
+      "nll_loss": 0.2801322340965271,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.07502304017543793,
+      "rewards/margins": 0.10125863552093506,
+      "rewards/rejected": -0.17628169059753418,
+      "step": 490
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 4.40625,
+      "learning_rate": 9.538031743343628e-07,
+      "log_odds_chosen": 1.2528076171875,
+      "log_odds_ratio": -0.3035999834537506,
+      "logits/chosen": -3.0861003398895264,
+      "logits/rejected": -3.111483097076416,
+      "logps/chosen": -0.39375776052474976,
+      "logps/rejected": -0.9319614171981812,
+      "loss": 0.3176,
+      "nll_loss": 0.2627008259296417,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -0.07875155657529831,
+      "rewards/margins": 0.10764074325561523,
+      "rewards/rejected": -0.18639230728149414,
+      "step": 500
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 4.75,
+      "learning_rate": 8.674650375410379e-07,
+      "log_odds_chosen": 1.0996912717819214,
+      "log_odds_ratio": -0.34020066261291504,
+      "logits/chosen": -3.0855681896209717,
+      "logits/rejected": -3.1118760108947754,
+      "logps/chosen": -0.426668643951416,
+      "logps/rejected": -0.8785163760185242,
+      "loss": 0.3107,
+      "nll_loss": 0.2832065224647522,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.08533372730016708,
+      "rewards/margins": 0.09036955237388611,
+      "rewards/rejected": -0.17570330202579498,
+      "step": 510
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 4.3125,
+      "learning_rate": 7.843959053281663e-07,
+      "log_odds_chosen": 1.3144631385803223,
+      "log_odds_ratio": -0.2888423502445221,
+      "logits/chosen": -3.060899019241333,
+      "logits/rejected": -3.110773801803589,
+      "logps/chosen": -0.3530345559120178,
+      "logps/rejected": -0.8913728594779968,
+      "loss": 0.3117,
+      "nll_loss": 0.2591019570827484,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -0.07060691714286804,
+      "rewards/margins": 0.10766766965389252,
+      "rewards/rejected": -0.17827460169792175,
+      "step": 520
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 5.0,
+      "learning_rate": 7.047621161693152e-07,
+      "log_odds_chosen": 1.327051043510437,
+      "log_odds_ratio": -0.2728634774684906,
+      "logits/chosen": -3.0373597145080566,
+      "logits/rejected": -3.121220350265503,
+      "logps/chosen": -0.35444390773773193,
+      "logps/rejected": -0.9054730534553528,
+      "loss": 0.3201,
+      "nll_loss": 0.2551073729991913,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -0.07088878005743027,
+      "rewards/margins": 0.11020584404468536,
+      "rewards/rejected": -0.18109463155269623,
+      "step": 530
+    },
+    {
+      "epoch": 2.31,
+      "grad_norm": 4.8125,
+      "learning_rate": 6.28723129572247e-07,
+      "log_odds_chosen": 1.3342939615249634,
+      "log_odds_ratio": -0.27417218685150146,
+      "logits/chosen": -3.0797994136810303,
+      "logits/rejected": -3.1183524131774902,
+      "logps/chosen": -0.362510621547699,
+      "logps/rejected": -0.9240224957466125,
+      "loss": 0.314,
+      "nll_loss": 0.2499639093875885,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.07250212132930756,
+      "rewards/margins": 0.11230238527059555,
+      "rewards/rejected": -0.1848045140504837,
+      "step": 540
+    },
+    {
+      "epoch": 2.35,
+      "grad_norm": 4.5,
+      "learning_rate": 5.564312067755856e-07,
+      "log_odds_chosen": 1.2629302740097046,
+      "log_odds_ratio": -0.29583701491355896,
+      "logits/chosen": -3.106447696685791,
+      "logits/rejected": -3.153026580810547,
+      "logps/chosen": -0.36336833238601685,
+      "logps/rejected": -0.8715957403182983,
+      "loss": 0.3098,
+      "nll_loss": 0.2292967140674591,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07267366349697113,
+      "rewards/margins": 0.10164550691843033,
+      "rewards/rejected": -0.17431916296482086,
+      "step": 550
+    },
+    {
+      "epoch": 2.39,
+      "grad_norm": 4.375,
+      "learning_rate": 4.880311058593617e-07,
+      "log_odds_chosen": 1.3260443210601807,
+      "log_odds_ratio": -0.2782818675041199,
+      "logits/chosen": -3.0871644020080566,
+      "logits/rejected": -3.1364452838897705,
+      "logps/chosen": -0.36632782220840454,
+      "logps/rejected": -0.924032986164093,
+      "loss": 0.2976,
+      "nll_loss": 0.22606725990772247,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07326555997133255,
+      "rewards/margins": 0.11154104769229889,
+      "rewards/rejected": -0.18480661511421204,
+      "step": 560
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 4.96875,
+      "learning_rate": 4.2365979187997094e-07,
+      "log_odds_chosen": 1.277055025100708,
+      "log_odds_ratio": -0.29783302545547485,
+      "logits/chosen": -3.0671963691711426,
+      "logits/rejected": -3.1140341758728027,
+      "logps/chosen": -0.381780207157135,
+      "logps/rejected": -0.9046441912651062,
+      "loss": 0.3148,
+      "nll_loss": 0.2656724154949188,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.07635603845119476,
+      "rewards/margins": 0.10457281768321991,
+      "rewards/rejected": -0.18092885613441467,
+      "step": 570
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 4.9375,
+      "learning_rate": 3.634461626099495e-07,
+      "log_odds_chosen": 1.2928454875946045,
+      "log_odds_ratio": -0.28668928146362305,
+      "logits/chosen": -3.0869479179382324,
+      "logits/rejected": -3.1252989768981934,
+      "logps/chosen": -0.345980703830719,
+      "logps/rejected": -0.8593694567680359,
+      "loss": 0.3035,
+      "nll_loss": 0.22678887844085693,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.06919614970684052,
+      "rewards/margins": 0.10267776250839233,
+      "rewards/rejected": -0.17187389731407166,
+      "step": 580
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 5.5,
+      "learning_rate": 3.075107904317667e-07,
+      "log_odds_chosen": 1.2825336456298828,
+      "log_odds_ratio": -0.29012447595596313,
+      "logits/chosen": -3.0542449951171875,
+      "logits/rejected": -3.106863021850586,
+      "logps/chosen": -0.36031287908554077,
+      "logps/rejected": -0.8658483624458313,
+      "loss": 0.3156,
+      "nll_loss": 0.2495698481798172,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.07206258177757263,
+      "rewards/margins": 0.10110710561275482,
+      "rewards/rejected": -0.17316967248916626,
+      "step": 590
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 4.90625,
+      "learning_rate": 2.5596568090246546e-07,
+      "log_odds_chosen": 1.2836467027664185,
+      "log_odds_ratio": -0.28635507822036743,
+      "logits/chosen": -3.0764803886413574,
+      "logits/rejected": -3.1473774909973145,
+      "logps/chosen": -0.37658047676086426,
+      "logps/rejected": -0.9045608639717102,
+      "loss": 0.3141,
+      "nll_loss": 0.24526798725128174,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07531610131263733,
+      "rewards/margins": 0.10559606552124023,
+      "rewards/rejected": -0.18091216683387756,
+      "step": 600
+    },
+    {
+      "epoch": 2.61,
+      "grad_norm": 4.28125,
+      "learning_rate": 2.0891404847259267e-07,
+      "log_odds_chosen": 1.2697139978408813,
+      "log_odds_ratio": -0.29423031210899353,
+      "logits/chosen": -3.090924024581909,
+      "logits/rejected": -3.146167039871216,
+      "logps/chosen": -0.38669395446777344,
+      "logps/rejected": -0.9116611480712891,
+      "loss": 0.313,
+      "nll_loss": 0.25895369052886963,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.0773387923836708,
+      "rewards/margins": 0.10499344766139984,
+      "rewards/rejected": -0.18233224749565125,
+      "step": 610
+    },
+    {
+      "epoch": 2.65,
+      "grad_norm": 4.96875,
+      "learning_rate": 1.6645010980854082e-07,
+      "log_odds_chosen": 1.3362077474594116,
+      "log_odds_ratio": -0.2740064263343811,
+      "logits/chosen": -3.075295925140381,
+      "logits/rejected": -3.110752582550049,
+      "logps/chosen": -0.3653913140296936,
+      "logps/rejected": -0.9164104461669922,
+      "loss": 0.3052,
+      "nll_loss": 0.26205652952194214,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07307826727628708,
+      "rewards/margins": 0.1102038249373436,
+      "rewards/rejected": -0.18328210711479187,
+      "step": 620
+    },
+    {
+      "epoch": 2.69,
+      "grad_norm": 5.21875,
+      "learning_rate": 1.286588951321363e-07,
+      "log_odds_chosen": 1.2346899509429932,
+      "log_odds_ratio": -0.29600512981414795,
+      "logits/chosen": -3.094604015350342,
+      "logits/rejected": -3.125540256500244,
+      "logps/chosen": -0.393343448638916,
+      "logps/rejected": -0.9184632301330566,
+      "loss": 0.3236,
+      "nll_loss": 0.2790059447288513,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.07866868376731873,
+      "rewards/margins": 0.10502395778894424,
+      "rewards/rejected": -0.18369261920452118,
+      "step": 630
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 4.6875,
+      "learning_rate": 9.561607795526007e-08,
+      "log_odds_chosen": 1.2945499420166016,
+      "log_odds_ratio": -0.2860717177391052,
+      "logits/chosen": -3.092331647872925,
+      "logits/rejected": -3.149021863937378,
+      "logps/chosen": -0.3724428415298462,
+      "logps/rejected": -0.9131635427474976,
+      "loss": 0.312,
+      "nll_loss": 0.2520996630191803,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.074488565325737,
+      "rewards/margins": 0.1081441268324852,
+      "rewards/rejected": -0.182632714509964,
+      "step": 640
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 4.375,
+      "learning_rate": 6.738782355044048e-08,
+      "log_odds_chosen": 1.303377628326416,
+      "log_odds_ratio": -0.28758224844932556,
+      "logits/chosen": -3.0791025161743164,
+      "logits/rejected": -3.130547046661377,
+      "logps/chosen": -0.3604482114315033,
+      "logps/rejected": -0.8876269459724426,
+      "loss": 0.3161,
+      "nll_loss": 0.2479972094297409,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07208964973688126,
+      "rewards/margins": 0.10543572902679443,
+      "rewards/rejected": -0.1775253713130951,
+      "step": 650
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 4.8125,
+      "learning_rate": 4.40306564608381e-08,
+      "log_odds_chosen": 1.220916509628296,
+      "log_odds_ratio": -0.3061557412147522,
+      "logits/chosen": -3.0849318504333496,
+      "logits/rejected": -3.0918779373168945,
+      "logps/chosen": -0.38813266158103943,
+      "logps/rejected": -0.8825920224189758,
+      "loss": 0.3064,
+      "nll_loss": 0.26622262597084045,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.0776265487074852,
+      "rewards/margins": 0.09889186918735504,
+      "rewards/rejected": -0.17651841044425964,
+      "step": 660
+    },
+    {
+      "epoch": 2.86,
+      "grad_norm": 4.9375,
+      "learning_rate": 2.5591347314928572e-08,
+      "log_odds_chosen": 1.1736371517181396,
+      "log_odds_ratio": -0.3195890486240387,
+      "logits/chosen": -3.0899367332458496,
+      "logits/rejected": -3.1268470287323,
+      "logps/chosen": -0.3975283205509186,
+      "logps/rejected": -0.897696316242218,
+      "loss": 0.3181,
+      "nll_loss": 0.2676974833011627,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.07950566709041595,
+      "rewards/margins": 0.10003360360860825,
+      "rewards/rejected": -0.1795392781496048,
+      "step": 670
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 3.921875,
+      "learning_rate": 1.2106819172520434e-08,
+      "log_odds_chosen": 1.2462742328643799,
+      "log_odds_ratio": -0.29843610525131226,
+      "logits/chosen": -3.0916683673858643,
+      "logits/rejected": -3.126143217086792,
+      "logps/chosen": -0.37134823203086853,
+      "logps/rejected": -0.8719061613082886,
+      "loss": 0.3068,
+      "nll_loss": 0.2421465665102005,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.07426965236663818,
+      "rewards/margins": 0.10011158138513565,
+      "rewards/rejected": -0.17438122630119324,
+      "step": 680
+    },
+    {
+      "epoch": 2.95,
+      "grad_norm": 4.0625,
+      "learning_rate": 3.6040735896455957e-09,
+      "log_odds_chosen": 1.2757344245910645,
+      "log_odds_ratio": -0.29123741388320923,
+      "logits/chosen": -3.0802972316741943,
+      "logits/rejected": -3.130801200866699,
+      "logps/chosen": -0.37011533975601196,
+      "logps/rejected": -0.8979229927062988,
+      "loss": 0.3024,
+      "nll_loss": 0.23298993706703186,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -0.07402306795120239,
+      "rewards/margins": 0.10556153953075409,
+      "rewards/rejected": -0.17958460748195648,
+      "step": 690
+    },
+    {
+      "epoch": 2.99,
+      "grad_norm": 5.0,
+      "learning_rate": 1.0013655036916758e-10,
+      "log_odds_chosen": 1.3622093200683594,
+      "log_odds_ratio": -0.2736757695674896,
+      "logits/chosen": -3.140601396560669,
+      "logits/rejected": -3.1706244945526123,
+      "logps/chosen": -0.34326601028442383,
+      "logps/rejected": -0.8912754058837891,
+      "loss": 0.3068,
+      "nll_loss": 0.21759584546089172,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.06865320354700089,
+      "rewards/margins": 0.10960187017917633,
+      "rewards/rejected": -0.1782551109790802,
+      "step": 700
+    },
+    {
+      "epoch": 3.0,
+      "step": 702,
+      "total_flos": 0.0,
+      "train_loss": 0.3926089354220279,
+      "train_runtime": 9824.6416,
+      "train_samples_per_second": 4.569,
+      "train_steps_per_second": 0.071
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 702,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}