Model save

README.md

@@ -0,0 +1,74 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-dpo-maximal
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-dpo-maximal
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.3380
+- Rewards/chosen: -0.1339
+- Rewards/rejected: -3.0976
+- Rewards/accuracies: 0.8790
+- Rewards/margins: 2.9637
+- Logps/rejected: -275.9525
+- Logps/chosen: -285.9466
+- Logits/rejected: -2.1375
+- Logits/chosen: -2.2908
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.3619        | 0.26  | 500  | 0.3822          | 0.1843         | -2.0970          | 0.8651             | 2.2812          | -265.9466      | -282.7652    | -2.1994         | -2.3618       |
+| 0.396         | 0.52  | 1000 | 0.3747          | -0.7559        | -3.2293          | 0.8730             | 2.4733          | -277.2696      | -292.1672    | -2.1335         | -2.2927       |
+| 0.3618        | 0.78  | 1500 | 0.3452          | -0.4962        | -3.2836          | 0.875              | 2.7874          | -277.8134      | -289.5698    | -2.1794         | -2.3280       |
+
+
+### Framework versions
+
+- Transformers 4.36.1
+- Pytorch 2.0.1+cu117
+- Datasets 2.16.1
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.290846109390259,
+    "eval_logits/rejected": -2.137458324432373,
+    "eval_logps/chosen": -285.9466247558594,
+    "eval_logps/rejected": -275.9525146484375,
+    "eval_loss": 0.3380393981933594,
+    "eval_rewards/accuracies": 0.8789682388305664,
+    "eval_rewards/chosen": -0.13388866186141968,
+    "eval_rewards/margins": 2.963660955429077,
+    "eval_rewards/rejected": -3.0975501537323,
+    "eval_runtime": 720.1571,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 2.777,
+    "eval_steps_per_second": 0.087,
+    "train_loss": 0.3594729615826684,
+    "train_runtime": 33818.9349,
+    "train_samples": 61135,
+    "train_samples_per_second": 1.808,
+    "train_steps_per_second": 0.057
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.1",
+  "use_cache": false,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.290846109390259,
+    "eval_logits/rejected": -2.137458324432373,
+    "eval_logps/chosen": -285.9466247558594,
+    "eval_logps/rejected": -275.9525146484375,
+    "eval_loss": 0.3380393981933594,
+    "eval_rewards/accuracies": 0.8789682388305664,
+    "eval_rewards/chosen": -0.13388866186141968,
+    "eval_rewards/margins": 2.963660955429077,
+    "eval_rewards/rejected": -3.0975501537323,
+    "eval_runtime": 720.1571,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 2.777,
+    "eval_steps_per_second": 0.087
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.1"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c34b5370f69683957a12be5437dce1f22c5d71fad464a1f92a7ee9b2fdc7d788
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7de3a4da759a44b11af01603249a6ee4332bb7c0f8b0ee907145744b768227c5
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2597d437a1ea8c189b726774d3b4b7df3c07b1808adc29b32b860a8a187cbf3c
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Feb06_22-14-23_node01/events.out.tfevents.1707229034.node01.64050.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32a87d9a15f11435166fcfeba010908cdaee0e32b1964c71ca5d9d7eca541196
+size 128616

runs/Feb06_22-14-23_node01/events.out.tfevents.1707263573.node01.64050.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24a07e0027b92d68fcd39069fb9db2c0e8b3742de8b63767490851f56a490720
+size 828

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 4096,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.3594729615826684,
+    "train_runtime": 33818.9349,
+    "train_samples": 61135,
+    "train_samples_per_second": 1.808,
+    "train_steps_per_second": 0.057
+}

trainer_state.json

@@ -0,0 +1,2766 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1911,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6041666666666664e-09,
+      "logits/chosen": -2.4222943782806396,
+      "logits/rejected": -2.300191879272461,
+      "logps/chosen": -242.6232147216797,
+      "logps/rejected": -173.6947784423828,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.6041666666666667e-08,
+      "logits/chosen": -2.580005645751953,
+      "logits/rejected": -2.4936344623565674,
+      "logps/chosen": -268.85345458984375,
+      "logps/rejected": -210.68524169921875,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.5833333134651184,
+      "rewards/chosen": 0.004795679822564125,
+      "rewards/margins": 0.013473947532474995,
+      "rewards/rejected": -0.008678268641233444,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.208333333333333e-08,
+      "logits/chosen": -2.693087339401245,
+      "logits/rejected": -2.5713565349578857,
+      "logps/chosen": -301.3067321777344,
+      "logps/rejected": -251.20126342773438,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0008392825839109719,
+      "rewards/margins": 0.013224393129348755,
+      "rewards/rejected": -0.012385109439492226,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.812499999999999e-08,
+      "logits/chosen": -2.725404977798462,
+      "logits/rejected": -2.4892578125,
+      "logps/chosen": -297.3845520019531,
+      "logps/rejected": -222.6193389892578,
+      "loss": 0.6594,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.059859294444322586,
+      "rewards/margins": 0.09642773121595383,
+      "rewards/rejected": -0.03656843304634094,
+      "step": 30
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.66707706451416,
+      "logits/rejected": -2.561459541320801,
+      "logps/chosen": -258.4730224609375,
+      "logps/rejected": -230.8173065185547,
+      "loss": 0.6212,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.16431955993175507,
+      "rewards/margins": 0.18100649118423462,
+      "rewards/rejected": -0.016686933115124702,
+      "step": 40
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3020833333333334e-07,
+      "logits/chosen": -2.6738762855529785,
+      "logits/rejected": -2.5567877292633057,
+      "logps/chosen": -321.64422607421875,
+      "logps/rejected": -266.5691833496094,
+      "loss": 0.5692,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.4113697111606598,
+      "rewards/margins": 0.3625262379646301,
+      "rewards/rejected": 0.048843465745449066,
+      "step": 50
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -2.6034302711486816,
+      "logits/rejected": -2.437656879425049,
+      "logps/chosen": -301.7183837890625,
+      "logps/rejected": -295.6407165527344,
+      "loss": 0.5304,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.41556516289711,
+      "rewards/margins": 0.5151094198226929,
+      "rewards/rejected": -0.09954418987035751,
+      "step": 60
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8229166666666666e-07,
+      "logits/chosen": -2.563300132751465,
+      "logits/rejected": -2.472400426864624,
+      "logps/chosen": -265.8623046875,
+      "logps/rejected": -256.015869140625,
+      "loss": 0.5141,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.4251571297645569,
+      "rewards/margins": 0.5559587478637695,
+      "rewards/rejected": -0.13080164790153503,
+      "step": 70
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.6306231021881104,
+      "logits/rejected": -2.4977340698242188,
+      "logps/chosen": -248.59957885742188,
+      "logps/rejected": -230.89956665039062,
+      "loss": 0.4591,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.5105770826339722,
+      "rewards/margins": 0.6713550686836243,
+      "rewards/rejected": -0.1607779711484909,
+      "step": 80
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3437499999999998e-07,
+      "logits/chosen": -2.5985584259033203,
+      "logits/rejected": -2.4967856407165527,
+      "logps/chosen": -270.8584289550781,
+      "logps/rejected": -265.82183837890625,
+      "loss": 0.4679,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.3090444505214691,
+      "rewards/margins": 0.8424568176269531,
+      "rewards/rejected": -0.5334123373031616,
+      "step": 90
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.604166666666667e-07,
+      "logits/chosen": -2.5061540603637695,
+      "logits/rejected": -2.3507046699523926,
+      "logps/chosen": -235.2123565673828,
+      "logps/rejected": -198.79722595214844,
+      "loss": 0.4467,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.6367200613021851,
+      "rewards/margins": 1.0976603031158447,
+      "rewards/rejected": -0.4609402120113373,
+      "step": 100
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.864583333333333e-07,
+      "logits/chosen": -2.569188117980957,
+      "logits/rejected": -2.4755890369415283,
+      "logps/chosen": -298.9624938964844,
+      "logps/rejected": -258.39947509765625,
+      "loss": 0.4311,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.614601731300354,
+      "rewards/margins": 1.216589331626892,
+      "rewards/rejected": -0.6019877195358276,
+      "step": 110
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.5569252967834473,
+      "logits/rejected": -2.398772716522217,
+      "logps/chosen": -280.63787841796875,
+      "logps/rejected": -212.21267700195312,
+      "loss": 0.4719,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.8052973747253418,
+      "rewards/margins": 1.0679086446762085,
+      "rewards/rejected": -0.26261135935783386,
+      "step": 120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3854166666666667e-07,
+      "logits/chosen": -2.6138534545898438,
+      "logits/rejected": -2.3813750743865967,
+      "logps/chosen": -311.6466064453125,
+      "logps/rejected": -238.57833862304688,
+      "loss": 0.4179,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.803765594959259,
+      "rewards/margins": 1.384441614151001,
+      "rewards/rejected": -0.5806761980056763,
+      "step": 130
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.645833333333333e-07,
+      "logits/chosen": -2.620131015777588,
+      "logits/rejected": -2.4834771156311035,
+      "logps/chosen": -280.6990051269531,
+      "logps/rejected": -241.31381225585938,
+      "loss": 0.4193,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.6271090507507324,
+      "rewards/margins": 1.3870524168014526,
+      "rewards/rejected": -0.7599433660507202,
+      "step": 140
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.9062499999999997e-07,
+      "logits/chosen": -2.4387025833129883,
+      "logits/rejected": -2.3931989669799805,
+      "logps/chosen": -277.7287292480469,
+      "logps/rejected": -276.62030029296875,
+      "loss": 0.3709,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.10160081088542938,
+      "rewards/margins": 1.4848428964614868,
+      "rewards/rejected": -1.3832420110702515,
+      "step": 150
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.5423970222473145,
+      "logits/rejected": -2.4555859565734863,
+      "logps/chosen": -271.20391845703125,
+      "logps/rejected": -272.84075927734375,
+      "loss": 0.3887,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.8339279294013977,
+      "rewards/margins": 2.0397675037384033,
+      "rewards/rejected": -1.2058393955230713,
+      "step": 160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.427083333333333e-07,
+      "logits/chosen": -2.5736887454986572,
+      "logits/rejected": -2.423067331314087,
+      "logps/chosen": -296.3034973144531,
+      "logps/rejected": -254.79971313476562,
+      "loss": 0.4365,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1544642299413681,
+      "rewards/margins": 1.3991954326629639,
+      "rewards/rejected": -1.5536595582962036,
+      "step": 170
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -2.571223735809326,
+      "logits/rejected": -2.438267946243286,
+      "logps/chosen": -305.3669128417969,
+      "logps/rejected": -290.1111145019531,
+      "loss": 0.3981,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.318599134683609,
+      "rewards/margins": 1.9614486694335938,
+      "rewards/rejected": -1.642849326133728,
+      "step": 180
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.947916666666667e-07,
+      "logits/chosen": -2.4852206707000732,
+      "logits/rejected": -2.301888942718506,
+      "logps/chosen": -280.9864196777344,
+      "logps/rejected": -234.48117065429688,
+      "loss": 0.3926,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.24488838016986847,
+      "rewards/margins": 2.234631061553955,
+      "rewards/rejected": -1.9897428750991821,
+      "step": 190
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.976730657358929e-07,
+      "logits/chosen": -2.3834030628204346,
+      "logits/rejected": -2.3198955059051514,
+      "logps/chosen": -260.1601257324219,
+      "logps/rejected": -241.60348510742188,
+      "loss": 0.4712,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1966917514801025,
+      "rewards/margins": 1.3776226043701172,
+      "rewards/rejected": -2.5743143558502197,
+      "step": 200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.947643979057592e-07,
+      "logits/chosen": -2.4042606353759766,
+      "logits/rejected": -2.3647255897521973,
+      "logps/chosen": -265.72344970703125,
+      "logps/rejected": -300.393798828125,
+      "loss": 0.405,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.035271644592285156,
+      "rewards/margins": 1.8820867538452148,
+      "rewards/rejected": -1.8468151092529297,
+      "step": 210
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.918557300756254e-07,
+      "logits/chosen": -2.2479195594787598,
+      "logits/rejected": -2.126936435699463,
+      "logps/chosen": -286.0814514160156,
+      "logps/rejected": -230.5002899169922,
+      "loss": 0.4365,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.2404344528913498,
+      "rewards/margins": 1.7336857318878174,
+      "rewards/rejected": -1.9741201400756836,
+      "step": 220
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.889470622454916e-07,
+      "logits/chosen": -2.402921199798584,
+      "logits/rejected": -2.213413715362549,
+      "logps/chosen": -304.3617248535156,
+      "logps/rejected": -279.91497802734375,
+      "loss": 0.4291,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.006418424658477306,
+      "rewards/margins": 2.0425734519958496,
+      "rewards/rejected": -2.0489919185638428,
+      "step": 230
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.860383944153578e-07,
+      "logits/chosen": -2.4241273403167725,
+      "logits/rejected": -2.272402048110962,
+      "logps/chosen": -287.0025939941406,
+      "logps/rejected": -263.2986145019531,
+      "loss": 0.381,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.207146555185318,
+      "rewards/margins": 1.902012586593628,
+      "rewards/rejected": -2.109158992767334,
+      "step": 240
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.83129726585224e-07,
+      "logits/chosen": -2.429861545562744,
+      "logits/rejected": -2.3017280101776123,
+      "logps/chosen": -257.54071044921875,
+      "logps/rejected": -265.039794921875,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.5596320629119873,
+      "rewards/margins": 1.9522535800933838,
+      "rewards/rejected": -2.511885404586792,
+      "step": 250
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.802210587550902e-07,
+      "logits/chosen": -2.3487319946289062,
+      "logits/rejected": -2.2305383682250977,
+      "logps/chosen": -304.7814025878906,
+      "logps/rejected": -278.561279296875,
+      "loss": 0.3971,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.19774925708770752,
+      "rewards/margins": 1.9618850946426392,
+      "rewards/rejected": -2.1596343517303467,
+      "step": 260
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.773123909249563e-07,
+      "logits/chosen": -2.4337475299835205,
+      "logits/rejected": -2.2924251556396484,
+      "logps/chosen": -274.05218505859375,
+      "logps/rejected": -279.85198974609375,
+      "loss": 0.4099,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.34399348497390747,
+      "rewards/margins": 3.3459434509277344,
+      "rewards/rejected": -3.689936399459839,
+      "step": 270
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.7440372309482255e-07,
+      "logits/chosen": -2.390204906463623,
+      "logits/rejected": -2.249260663986206,
+      "logps/chosen": -292.68243408203125,
+      "logps/rejected": -268.0016174316406,
+      "loss": 0.6235,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.024298882111907005,
+      "rewards/margins": 2.0128695964813232,
+      "rewards/rejected": -2.037168502807617,
+      "step": 280
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.7149505526468876e-07,
+      "logits/chosen": -2.356079578399658,
+      "logits/rejected": -2.1153652667999268,
+      "logps/chosen": -292.0235595703125,
+      "logps/rejected": -256.1002502441406,
+      "loss": 0.322,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.1279529184103012,
+      "rewards/margins": 2.2269229888916016,
+      "rewards/rejected": -2.0989699363708496,
+      "step": 290
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.6858638743455497e-07,
+      "logits/chosen": -2.472639322280884,
+      "logits/rejected": -2.3683533668518066,
+      "logps/chosen": -338.8008728027344,
+      "logps/rejected": -282.82843017578125,
+      "loss": 0.4142,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.14725220203399658,
+      "rewards/margins": 1.3299568891525269,
+      "rewards/rejected": -1.4772090911865234,
+      "step": 300
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.656777196044212e-07,
+      "logits/chosen": -2.4232888221740723,
+      "logits/rejected": -2.2172470092773438,
+      "logps/chosen": -296.7711181640625,
+      "logps/rejected": -261.01995849609375,
+      "loss": 0.4658,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.06699763238430023,
+      "rewards/margins": 2.3037867546081543,
+      "rewards/rejected": -2.2367894649505615,
+      "step": 310
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.627690517742874e-07,
+      "logits/chosen": -2.327702283859253,
+      "logits/rejected": -2.200692892074585,
+      "logps/chosen": -241.1982421875,
+      "logps/rejected": -220.4070281982422,
+      "loss": 0.3734,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.3033676743507385,
+      "rewards/margins": 2.032435417175293,
+      "rewards/rejected": -1.7290678024291992,
+      "step": 320
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.5986038394415354e-07,
+      "logits/chosen": -2.321364641189575,
+      "logits/rejected": -2.122469902038574,
+      "logps/chosen": -284.7498779296875,
+      "logps/rejected": -232.0214385986328,
+      "loss": 0.6356,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.3031235337257385,
+      "rewards/margins": 2.075382709503174,
+      "rewards/rejected": -2.3785061836242676,
+      "step": 330
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.569517161140198e-07,
+      "logits/chosen": -2.420304775238037,
+      "logits/rejected": -2.304600954055786,
+      "logps/chosen": -280.8135070800781,
+      "logps/rejected": -293.92681884765625,
+      "loss": 0.5416,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.15361396968364716,
+      "rewards/margins": 1.9998279809951782,
+      "rewards/rejected": -1.846213936805725,
+      "step": 340
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.5404304828388595e-07,
+      "logits/chosen": -2.367790699005127,
+      "logits/rejected": -2.225597381591797,
+      "logps/chosen": -298.0318298339844,
+      "logps/rejected": -295.94256591796875,
+      "loss": 0.3371,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.41088590025901794,
+      "rewards/margins": 2.291506767272949,
+      "rewards/rejected": -2.702393054962158,
+      "step": 350
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.5113438045375216e-07,
+      "logits/chosen": -2.25712251663208,
+      "logits/rejected": -2.0587408542633057,
+      "logps/chosen": -273.820068359375,
+      "logps/rejected": -240.53372192382812,
+      "loss": 0.3219,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.08009258657693863,
+      "rewards/margins": 2.750244617462158,
+      "rewards/rejected": -2.8303377628326416,
+      "step": 360
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.4822571262361837e-07,
+      "logits/chosen": -2.2759785652160645,
+      "logits/rejected": -2.10341477394104,
+      "logps/chosen": -232.6130828857422,
+      "logps/rejected": -221.0072784423828,
+      "loss": 0.4257,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.17208755016326904,
+      "rewards/margins": 2.1418089866638184,
+      "rewards/rejected": -2.313896417617798,
+      "step": 370
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.453170447934846e-07,
+      "logits/chosen": -2.2166106700897217,
+      "logits/rejected": -2.0448429584503174,
+      "logps/chosen": -300.9383239746094,
+      "logps/rejected": -248.05429077148438,
+      "loss": 0.3711,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.026472803205251694,
+      "rewards/margins": 2.597238779067993,
+      "rewards/rejected": -2.623711585998535,
+      "step": 380
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.424083769633508e-07,
+      "logits/chosen": -2.3724656105041504,
+      "logits/rejected": -2.2095799446105957,
+      "logps/chosen": -301.7389221191406,
+      "logps/rejected": -317.2711181640625,
+      "loss": 0.4157,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.7330515384674072,
+      "rewards/margins": 2.163984775543213,
+      "rewards/rejected": -2.897036552429199,
+      "step": 390
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.39499709133217e-07,
+      "logits/chosen": -2.490659713745117,
+      "logits/rejected": -2.345731496810913,
+      "logps/chosen": -308.53558349609375,
+      "logps/rejected": -294.06500244140625,
+      "loss": 0.4574,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.5682181715965271,
+      "rewards/margins": 2.168735980987549,
+      "rewards/rejected": -2.7369542121887207,
+      "step": 400
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.3659104130308314e-07,
+      "logits/chosen": -2.3574600219726562,
+      "logits/rejected": -2.245293617248535,
+      "logps/chosen": -264.3704833984375,
+      "logps/rejected": -270.0927734375,
+      "loss": 0.3066,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.676201581954956,
+      "rewards/margins": 2.4835641384124756,
+      "rewards/rejected": -3.1597657203674316,
+      "step": 410
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.336823734729494e-07,
+      "logits/chosen": -2.542794704437256,
+      "logits/rejected": -2.2256128787994385,
+      "logps/chosen": -327.1799621582031,
+      "logps/rejected": -270.61065673828125,
+      "loss": 0.3053,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.38001373410224915,
+      "rewards/margins": 2.6601920127868652,
+      "rewards/rejected": -3.040205478668213,
+      "step": 420
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.3077370564281556e-07,
+      "logits/chosen": -2.393115997314453,
+      "logits/rejected": -2.141357421875,
+      "logps/chosen": -291.4068908691406,
+      "logps/rejected": -254.27560424804688,
+      "loss": 0.3113,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.41613179445266724,
+      "rewards/margins": 2.6375651359558105,
+      "rewards/rejected": -3.053696870803833,
+      "step": 430
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.278650378126818e-07,
+      "logits/chosen": -2.2429466247558594,
+      "logits/rejected": -2.1097540855407715,
+      "logps/chosen": -283.1539001464844,
+      "logps/rejected": -235.98074340820312,
+      "loss": 0.3876,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.28720611333847046,
+      "rewards/margins": 2.393641233444214,
+      "rewards/rejected": -2.680847406387329,
+      "step": 440
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.24956369982548e-07,
+      "logits/chosen": -2.3428943157196045,
+      "logits/rejected": -2.1891512870788574,
+      "logps/chosen": -305.74981689453125,
+      "logps/rejected": -277.8802185058594,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.138112872838974,
+      "rewards/margins": 2.6869564056396484,
+      "rewards/rejected": -2.8250691890716553,
+      "step": 450
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.220477021524142e-07,
+      "logits/chosen": -2.3777918815612793,
+      "logits/rejected": -2.1941428184509277,
+      "logps/chosen": -264.9512023925781,
+      "logps/rejected": -269.465087890625,
+      "loss": 0.3758,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.27588149905204773,
+      "rewards/margins": 2.346616506576538,
+      "rewards/rejected": -2.622498035430908,
+      "step": 460
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.191390343222804e-07,
+      "logits/chosen": -2.412323474884033,
+      "logits/rejected": -2.234257698059082,
+      "logps/chosen": -298.27630615234375,
+      "logps/rejected": -266.40460205078125,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.25904446840286255,
+      "rewards/margins": 2.111635684967041,
+      "rewards/rejected": -2.370680093765259,
+      "step": 470
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.162303664921466e-07,
+      "logits/chosen": -2.35739803314209,
+      "logits/rejected": -2.168689250946045,
+      "logps/chosen": -281.678955078125,
+      "logps/rejected": -260.5196838378906,
+      "loss": 0.3468,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.45242181420326233,
+      "rewards/margins": 2.4778125286102295,
+      "rewards/rejected": -2.930233955383301,
+      "step": 480
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.1332169866201275e-07,
+      "logits/chosen": -2.286667585372925,
+      "logits/rejected": -2.049354076385498,
+      "logps/chosen": -283.5222473144531,
+      "logps/rejected": -245.1978759765625,
+      "loss": 0.4088,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.19852662086486816,
+      "rewards/margins": 2.8068137168884277,
+      "rewards/rejected": -2.6082868576049805,
+      "step": 490
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.10413030831879e-07,
+      "logits/chosen": -2.423564910888672,
+      "logits/rejected": -2.1486616134643555,
+      "logps/chosen": -315.94171142578125,
+      "logps/rejected": -240.4951629638672,
+      "loss": 0.3619,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.18976499140262604,
+      "rewards/margins": 2.241330146789551,
+      "rewards/rejected": -2.4310948848724365,
+      "step": 500
+    },
+    {
+      "epoch": 0.26,
+      "eval_logits/chosen": -2.3617773056030273,
+      "eval_logits/rejected": -2.199384927749634,
+      "eval_logps/chosen": -282.76519775390625,
+      "eval_logps/rejected": -265.9465637207031,
+      "eval_loss": 0.3821512758731842,
+      "eval_rewards/accuracies": 0.8650793433189392,
+      "eval_rewards/chosen": 0.18425174057483673,
+      "eval_rewards/margins": 2.281205654144287,
+      "eval_rewards/rejected": -2.096953868865967,
+      "eval_runtime": 717.3734,
+      "eval_samples_per_second": 2.788,
+      "eval_steps_per_second": 0.088,
+      "step": 500
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.0750436300174517e-07,
+      "logits/chosen": -2.295250415802002,
+      "logits/rejected": -2.2096400260925293,
+      "logps/chosen": -291.8120422363281,
+      "logps/rejected": -260.5565490722656,
+      "loss": 0.3481,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.032511450350284576,
+      "rewards/margins": 1.999352216720581,
+      "rewards/rejected": -2.0318636894226074,
+      "step": 510
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.0459569517161143e-07,
+      "logits/chosen": -2.3412113189697266,
+      "logits/rejected": -2.168757915496826,
+      "logps/chosen": -289.66015625,
+      "logps/rejected": -284.6536560058594,
+      "loss": 0.4214,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.024832209572196007,
+      "rewards/margins": 2.256009817123413,
+      "rewards/rejected": -2.2808423042297363,
+      "step": 520
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.016870273414776e-07,
+      "logits/chosen": -2.3721296787261963,
+      "logits/rejected": -2.216839551925659,
+      "logps/chosen": -246.75814819335938,
+      "logps/rejected": -261.1737365722656,
+      "loss": 0.3284,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.24080362915992737,
+      "rewards/margins": 2.464489459991455,
+      "rewards/rejected": -2.7052931785583496,
+      "step": 530
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 3.987783595113438e-07,
+      "logits/chosen": -2.3687021732330322,
+      "logits/rejected": -2.1532135009765625,
+      "logps/chosen": -329.71002197265625,
+      "logps/rejected": -260.98931884765625,
+      "loss": 0.3913,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.147232323884964,
+      "rewards/margins": 2.5936901569366455,
+      "rewards/rejected": -2.740922451019287,
+      "step": 540
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.9586969168121e-07,
+      "logits/chosen": -2.3451223373413086,
+      "logits/rejected": -2.1668312549591064,
+      "logps/chosen": -337.485595703125,
+      "logps/rejected": -296.2718200683594,
+      "loss": 0.3295,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.21495577692985535,
+      "rewards/margins": 2.442884922027588,
+      "rewards/rejected": -2.227928876876831,
+      "step": 550
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.929610238510762e-07,
+      "logits/chosen": -2.26320481300354,
+      "logits/rejected": -2.1082823276519775,
+      "logps/chosen": -280.9175720214844,
+      "logps/rejected": -272.5048522949219,
+      "loss": 0.4014,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.1318025290966034,
+      "rewards/margins": 2.3364672660827637,
+      "rewards/rejected": -2.204664707183838,
+      "step": 560
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 3.900523560209424e-07,
+      "logits/chosen": -2.3491129875183105,
+      "logits/rejected": -2.1811981201171875,
+      "logps/chosen": -303.96441650390625,
+      "logps/rejected": -286.36212158203125,
+      "loss": 0.3572,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.1058625802397728,
+      "rewards/margins": 2.76606822013855,
+      "rewards/rejected": -2.660205364227295,
+      "step": 570
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 3.871436881908086e-07,
+      "logits/chosen": -2.396277904510498,
+      "logits/rejected": -2.1476378440856934,
+      "logps/chosen": -274.61334228515625,
+      "logps/rejected": -237.1002960205078,
+      "loss": 0.3261,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.008923185057938099,
+      "rewards/margins": 2.317211151123047,
+      "rewards/rejected": -2.326134443283081,
+      "step": 580
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.842350203606748e-07,
+      "logits/chosen": -2.30973482131958,
+      "logits/rejected": -2.0597481727600098,
+      "logps/chosen": -255.37875366210938,
+      "logps/rejected": -235.5735626220703,
+      "loss": 0.3845,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5673273801803589,
+      "rewards/margins": 2.377692222595215,
+      "rewards/rejected": -2.945019245147705,
+      "step": 590
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.8132635253054103e-07,
+      "logits/chosen": -2.2930376529693604,
+      "logits/rejected": -2.230609893798828,
+      "logps/chosen": -288.5751953125,
+      "logps/rejected": -318.2559509277344,
+      "loss": 0.354,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.0748153105378151,
+      "rewards/margins": 2.7162458896636963,
+      "rewards/rejected": -2.7910614013671875,
+      "step": 600
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.784176847004072e-07,
+      "logits/chosen": -2.3762922286987305,
+      "logits/rejected": -2.3032281398773193,
+      "logps/chosen": -266.0533447265625,
+      "logps/rejected": -305.07257080078125,
+      "loss": 0.3032,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.32922041416168213,
+      "rewards/margins": 2.986255645751953,
+      "rewards/rejected": -3.315476179122925,
+      "step": 610
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.755090168702734e-07,
+      "logits/chosen": -2.313537120819092,
+      "logits/rejected": -2.25026273727417,
+      "logps/chosen": -306.7781982421875,
+      "logps/rejected": -339.4481506347656,
+      "loss": 0.3557,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.6963660717010498,
+      "rewards/margins": 2.4231371879577637,
+      "rewards/rejected": -3.1195030212402344,
+      "step": 620
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.726003490401396e-07,
+      "logits/chosen": -2.2563579082489014,
+      "logits/rejected": -2.1553292274475098,
+      "logps/chosen": -288.6074523925781,
+      "logps/rejected": -262.7868957519531,
+      "loss": 0.3146,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.44406232237815857,
+      "rewards/margins": 3.098081588745117,
+      "rewards/rejected": -3.5421440601348877,
+      "step": 630
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.696916812100058e-07,
+      "logits/chosen": -2.227241277694702,
+      "logits/rejected": -2.0303165912628174,
+      "logps/chosen": -301.3541259765625,
+      "logps/rejected": -224.4669189453125,
+      "loss": 0.2901,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.028151262551546097,
+      "rewards/margins": 3.3005497455596924,
+      "rewards/rejected": -3.2723987102508545,
+      "step": 640
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 3.66783013379872e-07,
+      "logits/chosen": -2.342437744140625,
+      "logits/rejected": -2.1738288402557373,
+      "logps/chosen": -291.0592346191406,
+      "logps/rejected": -245.4440155029297,
+      "loss": 0.3027,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.018262123689055443,
+      "rewards/margins": 3.1664016246795654,
+      "rewards/rejected": -3.184663772583008,
+      "step": 650
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.6387434554973823e-07,
+      "logits/chosen": -2.269165515899658,
+      "logits/rejected": -2.113163709640503,
+      "logps/chosen": -308.91119384765625,
+      "logps/rejected": -277.9391174316406,
+      "loss": 0.3326,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.8833945393562317,
+      "rewards/margins": 2.456275701522827,
+      "rewards/rejected": -3.339670181274414,
+      "step": 660
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.609656777196044e-07,
+      "logits/chosen": -2.3179445266723633,
+      "logits/rejected": -2.203679084777832,
+      "logps/chosen": -320.14263916015625,
+      "logps/rejected": -286.0535888671875,
+      "loss": 0.3371,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.80712890625,
+      "rewards/margins": 2.284970760345459,
+      "rewards/rejected": -3.09209942817688,
+      "step": 670
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.5805700988947064e-07,
+      "logits/chosen": -2.2682688236236572,
+      "logits/rejected": -2.1642825603485107,
+      "logps/chosen": -253.39920043945312,
+      "logps/rejected": -242.94921875,
+      "loss": 0.3251,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.13453222811222076,
+      "rewards/margins": 2.803748369216919,
+      "rewards/rejected": -2.9382805824279785,
+      "step": 680
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.551483420593368e-07,
+      "logits/chosen": -2.312835454940796,
+      "logits/rejected": -2.162968158721924,
+      "logps/chosen": -321.691162109375,
+      "logps/rejected": -285.85479736328125,
+      "loss": 0.3158,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.3543187081813812,
+      "rewards/margins": 2.611062526702881,
+      "rewards/rejected": -2.965381622314453,
+      "step": 690
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.5223967422920306e-07,
+      "logits/chosen": -2.2975497245788574,
+      "logits/rejected": -2.1003618240356445,
+      "logps/chosen": -346.7196960449219,
+      "logps/rejected": -309.878662109375,
+      "loss": 0.3841,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.6169618368148804,
+      "rewards/margins": 2.551251173019409,
+      "rewards/rejected": -3.168212890625,
+      "step": 700
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.493310063990692e-07,
+      "logits/chosen": -2.368072032928467,
+      "logits/rejected": -2.1905136108398438,
+      "logps/chosen": -249.8124542236328,
+      "logps/rejected": -255.54690551757812,
+      "loss": 0.3306,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.10715383291244507,
+      "rewards/margins": 3.3935387134552,
+      "rewards/rejected": -3.500692367553711,
+      "step": 710
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.464223385689354e-07,
+      "logits/chosen": -2.3153772354125977,
+      "logits/rejected": -2.2586145401000977,
+      "logps/chosen": -242.65805053710938,
+      "logps/rejected": -254.4687042236328,
+      "loss": 0.3583,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.1414814442396164,
+      "rewards/margins": 2.854738235473633,
+      "rewards/rejected": -2.713256359100342,
+      "step": 720
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.4351367073880163e-07,
+      "logits/chosen": -2.318925380706787,
+      "logits/rejected": -2.2675676345825195,
+      "logps/chosen": -257.1809997558594,
+      "logps/rejected": -271.678955078125,
+      "loss": 0.2901,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.037356503307819366,
+      "rewards/margins": 2.468323230743408,
+      "rewards/rejected": -2.43096661567688,
+      "step": 730
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.4060500290866783e-07,
+      "logits/chosen": -2.3555076122283936,
+      "logits/rejected": -2.1742148399353027,
+      "logps/chosen": -342.202880859375,
+      "logps/rejected": -292.80706787109375,
+      "loss": 0.271,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.6387542486190796,
+      "rewards/margins": 2.901257038116455,
+      "rewards/rejected": -3.540011167526245,
+      "step": 740
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.37696335078534e-07,
+      "logits/chosen": -2.3542723655700684,
+      "logits/rejected": -2.159390687942505,
+      "logps/chosen": -291.72039794921875,
+      "logps/rejected": -241.2707977294922,
+      "loss": 0.3751,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.3051406145095825,
+      "rewards/margins": 2.620650053024292,
+      "rewards/rejected": -2.925791025161743,
+      "step": 750
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.3478766724840025e-07,
+      "logits/chosen": -2.2971510887145996,
+      "logits/rejected": -2.1574819087982178,
+      "logps/chosen": -272.34698486328125,
+      "logps/rejected": -245.53323364257812,
+      "loss": 0.3298,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.02699587121605873,
+      "rewards/margins": 2.5259757041931152,
+      "rewards/rejected": -2.552971363067627,
+      "step": 760
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.318789994182664e-07,
+      "logits/chosen": -2.3655948638916016,
+      "logits/rejected": -2.1962826251983643,
+      "logps/chosen": -256.5421142578125,
+      "logps/rejected": -283.6791687011719,
+      "loss": 0.3064,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.1341879814863205,
+      "rewards/margins": 2.915113925933838,
+      "rewards/rejected": -2.78092622756958,
+      "step": 770
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.2897033158813266e-07,
+      "logits/chosen": -2.2602274417877197,
+      "logits/rejected": -2.1949830055236816,
+      "logps/chosen": -282.25225830078125,
+      "logps/rejected": -250.8028106689453,
+      "loss": 0.4087,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.5231667757034302,
+      "rewards/margins": 2.3779044151306152,
+      "rewards/rejected": -2.901071548461914,
+      "step": 780
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.260616637579988e-07,
+      "logits/chosen": -2.3795268535614014,
+      "logits/rejected": -2.273205041885376,
+      "logps/chosen": -300.8672180175781,
+      "logps/rejected": -285.58538818359375,
+      "loss": 0.3704,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.10278744995594025,
+      "rewards/margins": 1.8986539840698242,
+      "rewards/rejected": -2.001441478729248,
+      "step": 790
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.2315299592786503e-07,
+      "logits/chosen": -2.3038344383239746,
+      "logits/rejected": -2.1164727210998535,
+      "logps/chosen": -280.07183837890625,
+      "logps/rejected": -238.2751007080078,
+      "loss": 0.3589,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.01835661195218563,
+      "rewards/margins": 3.2359859943389893,
+      "rewards/rejected": -3.2176296710968018,
+      "step": 800
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.2024432809773123e-07,
+      "logits/chosen": -2.2451796531677246,
+      "logits/rejected": -2.0506649017333984,
+      "logps/chosen": -264.8363342285156,
+      "logps/rejected": -231.02664184570312,
+      "loss": 0.2797,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.010631388053297997,
+      "rewards/margins": 3.197535991668701,
+      "rewards/rejected": -3.1869049072265625,
+      "step": 810
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.1733566026759744e-07,
+      "logits/chosen": -2.3159565925598145,
+      "logits/rejected": -2.1475565433502197,
+      "logps/chosen": -281.8079528808594,
+      "logps/rejected": -262.15545654296875,
+      "loss": 0.4088,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.36606311798095703,
+      "rewards/margins": 2.4230146408081055,
+      "rewards/rejected": -2.7890777587890625,
+      "step": 820
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.144269924374636e-07,
+      "logits/chosen": -2.312073230743408,
+      "logits/rejected": -2.1491856575012207,
+      "logps/chosen": -263.62823486328125,
+      "logps/rejected": -274.19891357421875,
+      "loss": 0.374,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.2555127739906311,
+      "rewards/margins": 2.482483386993408,
+      "rewards/rejected": -2.7379961013793945,
+      "step": 830
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.1151832460732986e-07,
+      "logits/chosen": -2.22873592376709,
+      "logits/rejected": -2.0442733764648438,
+      "logps/chosen": -298.53509521484375,
+      "logps/rejected": -266.8551025390625,
+      "loss": 0.3152,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.09185002744197845,
+      "rewards/margins": 3.16621994972229,
+      "rewards/rejected": -3.0743699073791504,
+      "step": 840
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.08609656777196e-07,
+      "logits/chosen": -2.271857738494873,
+      "logits/rejected": -2.1578991413116455,
+      "logps/chosen": -263.89007568359375,
+      "logps/rejected": -253.24179077148438,
+      "loss": 0.3368,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.23096470534801483,
+      "rewards/margins": 2.82230281829834,
+      "rewards/rejected": -3.05326771736145,
+      "step": 850
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.0570098894706227e-07,
+      "logits/chosen": -2.3438827991485596,
+      "logits/rejected": -2.2087767124176025,
+      "logps/chosen": -291.0592346191406,
+      "logps/rejected": -278.04412841796875,
+      "loss": 0.2865,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4321475923061371,
+      "rewards/margins": 2.5872695446014404,
+      "rewards/rejected": -3.0194175243377686,
+      "step": 860
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.0279232111692843e-07,
+      "logits/chosen": -2.15918231010437,
+      "logits/rejected": -2.081557035446167,
+      "logps/chosen": -260.660888671875,
+      "logps/rejected": -263.269287109375,
+      "loss": 0.3036,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.07870881259441376,
+      "rewards/margins": 3.0273680686950684,
+      "rewards/rejected": -3.106077194213867,
+      "step": 870
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 2.9988365328679463e-07,
+      "logits/chosen": -2.30003023147583,
+      "logits/rejected": -2.122143507003784,
+      "logps/chosen": -305.5731201171875,
+      "logps/rejected": -266.93487548828125,
+      "loss": 0.3189,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.4314422011375427,
+      "rewards/margins": 3.166658401489258,
+      "rewards/rejected": -3.5981006622314453,
+      "step": 880
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.9697498545666084e-07,
+      "logits/chosen": -2.2059383392333984,
+      "logits/rejected": -1.9731464385986328,
+      "logps/chosen": -342.48468017578125,
+      "logps/rejected": -278.19580078125,
+      "loss": 0.3215,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.6458533406257629,
+      "rewards/margins": 3.3348402976989746,
+      "rewards/rejected": -3.980693817138672,
+      "step": 890
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.9406631762652705e-07,
+      "logits/chosen": -2.264359951019287,
+      "logits/rejected": -2.0387301445007324,
+      "logps/chosen": -330.88446044921875,
+      "logps/rejected": -297.0821228027344,
+      "loss": 0.2575,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5124462246894836,
+      "rewards/margins": 3.263385772705078,
+      "rewards/rejected": -3.775831937789917,
+      "step": 900
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.9115764979639326e-07,
+      "logits/chosen": -2.270777463912964,
+      "logits/rejected": -2.169574022293091,
+      "logps/chosen": -307.5442810058594,
+      "logps/rejected": -281.299072265625,
+      "loss": 0.2926,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.295340895652771,
+      "rewards/margins": 3.029505491256714,
+      "rewards/rejected": -3.3248462677001953,
+      "step": 910
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.8824898196625947e-07,
+      "logits/chosen": -2.193376302719116,
+      "logits/rejected": -1.9907703399658203,
+      "logps/chosen": -290.02862548828125,
+      "logps/rejected": -275.67919921875,
+      "loss": 0.2917,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.3032880127429962,
+      "rewards/margins": 3.0413851737976074,
+      "rewards/rejected": -3.3446731567382812,
+      "step": 920
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.853403141361256e-07,
+      "logits/chosen": -2.1926026344299316,
+      "logits/rejected": -1.989322304725647,
+      "logps/chosen": -297.0482482910156,
+      "logps/rejected": -234.0426788330078,
+      "loss": 0.3039,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.6774950623512268,
+      "rewards/margins": 2.4298861026763916,
+      "rewards/rejected": -3.1073811054229736,
+      "step": 930
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.824316463059919e-07,
+      "logits/chosen": -2.2069790363311768,
+      "logits/rejected": -2.064375877380371,
+      "logps/chosen": -295.0317077636719,
+      "logps/rejected": -293.92242431640625,
+      "loss": 0.3263,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.0432120561599731,
+      "rewards/margins": 3.1925106048583984,
+      "rewards/rejected": -4.235722541809082,
+      "step": 940
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.7952297847585803e-07,
+      "logits/chosen": -2.217855930328369,
+      "logits/rejected": -2.084010601043701,
+      "logps/chosen": -272.4216613769531,
+      "logps/rejected": -253.54348754882812,
+      "loss": 0.3049,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.8211439847946167,
+      "rewards/margins": 2.7119455337524414,
+      "rewards/rejected": -3.5330893993377686,
+      "step": 950
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.766143106457243e-07,
+      "logits/chosen": -2.1376349925994873,
+      "logits/rejected": -2.015172004699707,
+      "logps/chosen": -259.86810302734375,
+      "logps/rejected": -262.43756103515625,
+      "loss": 0.3498,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.7693565487861633,
+      "rewards/margins": 2.9703726768493652,
+      "rewards/rejected": -3.739729404449463,
+      "step": 960
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.7370564281559045e-07,
+      "logits/chosen": -2.34848952293396,
+      "logits/rejected": -2.0991873741149902,
+      "logps/chosen": -343.0187683105469,
+      "logps/rejected": -266.3363342285156,
+      "loss": 0.3301,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.8317963480949402,
+      "rewards/margins": 2.7026162147521973,
+      "rewards/rejected": -3.534412384033203,
+      "step": 970
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.7079697498545666e-07,
+      "logits/chosen": -2.2526087760925293,
+      "logits/rejected": -2.0652668476104736,
+      "logps/chosen": -316.2718811035156,
+      "logps/rejected": -283.3409729003906,
+      "loss": 0.2974,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.2208749055862427,
+      "rewards/margins": 3.0918426513671875,
+      "rewards/rejected": -4.312718391418457,
+      "step": 980
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.6788830715532287e-07,
+      "logits/chosen": -2.316199541091919,
+      "logits/rejected": -2.1947271823883057,
+      "logps/chosen": -274.7453308105469,
+      "logps/rejected": -276.6575927734375,
+      "loss": 0.4123,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3303449153900146,
+      "rewards/margins": 2.126737117767334,
+      "rewards/rejected": -3.4570820331573486,
+      "step": 990
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.6497963932518907e-07,
+      "logits/chosen": -2.1468493938446045,
+      "logits/rejected": -2.0539629459381104,
+      "logps/chosen": -293.9585266113281,
+      "logps/rejected": -318.6316223144531,
+      "loss": 0.396,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.9897568821907043,
+      "rewards/margins": 2.690471649169922,
+      "rewards/rejected": -3.6802284717559814,
+      "step": 1000
+    },
+    {
+      "epoch": 0.52,
+      "eval_logits/chosen": -2.2926905155181885,
+      "eval_logits/rejected": -2.133467435836792,
+      "eval_logps/chosen": -292.16717529296875,
+      "eval_logps/rejected": -277.26959228515625,
+      "eval_loss": 0.37469375133514404,
+      "eval_rewards/accuracies": 0.8730158805847168,
+      "eval_rewards/chosen": -0.7559432983398438,
+      "eval_rewards/margins": 2.473315715789795,
+      "eval_rewards/rejected": -3.2292590141296387,
+      "eval_runtime": 715.3867,
+      "eval_samples_per_second": 2.796,
+      "eval_steps_per_second": 0.088,
+      "step": 1000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6207097149505523e-07,
+      "logits/chosen": -2.3242342472076416,
+      "logits/rejected": -2.152332067489624,
+      "logps/chosen": -317.60345458984375,
+      "logps/rejected": -260.40057373046875,
+      "loss": 0.3001,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.3242790102958679,
+      "rewards/margins": 2.7641217708587646,
+      "rewards/rejected": -3.088400363922119,
+      "step": 1010
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.591623036649215e-07,
+      "logits/chosen": -2.2785840034484863,
+      "logits/rejected": -2.119656562805176,
+      "logps/chosen": -301.8367614746094,
+      "logps/rejected": -281.8397521972656,
+      "loss": 0.2918,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5784929394721985,
+      "rewards/margins": 2.6694836616516113,
+      "rewards/rejected": -3.247976779937744,
+      "step": 1020
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5625363583478764e-07,
+      "logits/chosen": -2.2209084033966064,
+      "logits/rejected": -2.115307569503784,
+      "logps/chosen": -272.38092041015625,
+      "logps/rejected": -277.6880187988281,
+      "loss": 0.2436,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.6559833288192749,
+      "rewards/margins": 2.454627513885498,
+      "rewards/rejected": -3.1106107234954834,
+      "step": 1030
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.533449680046539e-07,
+      "logits/chosen": -2.2398033142089844,
+      "logits/rejected": -2.1163482666015625,
+      "logps/chosen": -284.6607360839844,
+      "logps/rejected": -289.515625,
+      "loss": 0.3228,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.2610655725002289,
+      "rewards/margins": 3.0918660163879395,
+      "rewards/rejected": -3.352931261062622,
+      "step": 1040
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5043630017452006e-07,
+      "logits/chosen": -2.2711968421936035,
+      "logits/rejected": -2.1295342445373535,
+      "logps/chosen": -284.3731689453125,
+      "logps/rejected": -258.94189453125,
+      "loss": 0.3527,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.4090282917022705,
+      "rewards/margins": 2.8344674110412598,
+      "rewards/rejected": -3.2434959411621094,
+      "step": 1050
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4752763234438627e-07,
+      "logits/chosen": -2.252903699874878,
+      "logits/rejected": -2.149430274963379,
+      "logps/chosen": -246.1130828857422,
+      "logps/rejected": -267.55377197265625,
+      "loss": 0.3028,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.19610390067100525,
+      "rewards/margins": 2.8517351150512695,
+      "rewards/rejected": -3.0478389263153076,
+      "step": 1060
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4461896451425247e-07,
+      "logits/chosen": -2.221062183380127,
+      "logits/rejected": -2.1321158409118652,
+      "logps/chosen": -254.9988250732422,
+      "logps/rejected": -250.8503875732422,
+      "loss": 0.3461,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.6029731035232544,
+      "rewards/margins": 2.7440178394317627,
+      "rewards/rejected": -3.3469905853271484,
+      "step": 1070
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.417102966841187e-07,
+      "logits/chosen": -2.27197527885437,
+      "logits/rejected": -2.067380428314209,
+      "logps/chosen": -265.71539306640625,
+      "logps/rejected": -266.55487060546875,
+      "loss": 0.293,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.19143518805503845,
+      "rewards/margins": 3.0950098037719727,
+      "rewards/rejected": -3.286444902420044,
+      "step": 1080
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3880162885398483e-07,
+      "logits/chosen": -2.286116600036621,
+      "logits/rejected": -2.009376287460327,
+      "logps/chosen": -283.79241943359375,
+      "logps/rejected": -210.90280151367188,
+      "loss": 0.3128,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3403369188308716,
+      "rewards/margins": 3.371168613433838,
+      "rewards/rejected": -3.71150541305542,
+      "step": 1090
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.3589296102385107e-07,
+      "logits/chosen": -2.3620998859405518,
+      "logits/rejected": -2.1931567192077637,
+      "logps/chosen": -285.1781921386719,
+      "logps/rejected": -266.1251220703125,
+      "loss": 0.3443,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.029187727719545364,
+      "rewards/margins": 2.814174175262451,
+      "rewards/rejected": -2.8433618545532227,
+      "step": 1100
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.3298429319371725e-07,
+      "logits/chosen": -2.351217746734619,
+      "logits/rejected": -2.1239571571350098,
+      "logps/chosen": -285.3910217285156,
+      "logps/rejected": -278.2947082519531,
+      "loss": 0.3532,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.21914640069007874,
+      "rewards/margins": 2.5958099365234375,
+      "rewards/rejected": -2.8149561882019043,
+      "step": 1110
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.3007562536358346e-07,
+      "logits/chosen": -2.3129007816314697,
+      "logits/rejected": -2.1005890369415283,
+      "logps/chosen": -281.8247985839844,
+      "logps/rejected": -266.05230712890625,
+      "loss": 0.4081,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.002577108098194003,
+      "rewards/margins": 2.926461935043335,
+      "rewards/rejected": -2.929039239883423,
+      "step": 1120
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.2716695753344967e-07,
+      "logits/chosen": -2.3015172481536865,
+      "logits/rejected": -2.125819444656372,
+      "logps/chosen": -280.98382568359375,
+      "logps/rejected": -234.7398681640625,
+      "loss": 0.3517,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.1859440803527832,
+      "rewards/margins": 2.21616268157959,
+      "rewards/rejected": -2.402106761932373,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.2425828970331587e-07,
+      "logits/chosen": -2.346259355545044,
+      "logits/rejected": -2.2613463401794434,
+      "logps/chosen": -250.6602325439453,
+      "logps/rejected": -252.2750244140625,
+      "loss": 0.3576,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.09936543554067612,
+      "rewards/margins": 2.3562135696411133,
+      "rewards/rejected": -2.4555790424346924,
+      "step": 1140
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.2134962187318208e-07,
+      "logits/chosen": -2.2582526206970215,
+      "logits/rejected": -2.185497283935547,
+      "logps/chosen": -289.13775634765625,
+      "logps/rejected": -283.5538330078125,
+      "loss": 0.3806,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.5999955534934998,
+      "rewards/margins": 2.2260944843292236,
+      "rewards/rejected": -2.82608962059021,
+      "step": 1150
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1844095404304826e-07,
+      "logits/chosen": -2.275233745574951,
+      "logits/rejected": -2.1149070262908936,
+      "logps/chosen": -315.2784423828125,
+      "logps/rejected": -286.3872375488281,
+      "loss": 0.3783,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.8392481803894043,
+      "rewards/margins": 2.358086109161377,
+      "rewards/rejected": -3.197334051132202,
+      "step": 1160
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1553228621291447e-07,
+      "logits/chosen": -2.2347631454467773,
+      "logits/rejected": -2.108426332473755,
+      "logps/chosen": -282.07763671875,
+      "logps/rejected": -275.0281982421875,
+      "loss": 0.4327,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.1532267332077026,
+      "rewards/margins": 2.362797498703003,
+      "rewards/rejected": -3.516024112701416,
+      "step": 1170
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.1262361838278068e-07,
+      "logits/chosen": -2.277366876602173,
+      "logits/rejected": -2.1091060638427734,
+      "logps/chosen": -299.38275146484375,
+      "logps/rejected": -269.4599609375,
+      "loss": 0.2882,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.1344071626663208,
+      "rewards/margins": 2.7073185443878174,
+      "rewards/rejected": -3.8417255878448486,
+      "step": 1180
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.0971495055264688e-07,
+      "logits/chosen": -2.2836697101593018,
+      "logits/rejected": -2.1914501190185547,
+      "logps/chosen": -287.3064880371094,
+      "logps/rejected": -263.22406005859375,
+      "loss": 0.316,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.27397674322128296,
+      "rewards/margins": 2.519117593765259,
+      "rewards/rejected": -2.7930939197540283,
+      "step": 1190
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0680628272251307e-07,
+      "logits/chosen": -2.2928688526153564,
+      "logits/rejected": -2.176222085952759,
+      "logps/chosen": -279.0005187988281,
+      "logps/rejected": -303.7047424316406,
+      "loss": 0.359,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.5041743516921997,
+      "rewards/margins": 2.464326858520508,
+      "rewards/rejected": -2.968501329421997,
+      "step": 1200
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0389761489237927e-07,
+      "logits/chosen": -2.3216004371643066,
+      "logits/rejected": -2.187290906906128,
+      "logps/chosen": -282.7243957519531,
+      "logps/rejected": -283.76904296875,
+      "loss": 0.2816,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.3544381260871887,
+      "rewards/margins": 2.6914665699005127,
+      "rewards/rejected": -3.0459046363830566,
+      "step": 1210
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 2.0098894706224548e-07,
+      "logits/chosen": -2.3324356079101562,
+      "logits/rejected": -2.1998660564422607,
+      "logps/chosen": -239.81320190429688,
+      "logps/rejected": -246.91592407226562,
+      "loss": 0.3403,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.48790687322616577,
+      "rewards/margins": 2.5268945693969727,
+      "rewards/rejected": -3.0148017406463623,
+      "step": 1220
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.980802792321117e-07,
+      "logits/chosen": -2.2460789680480957,
+      "logits/rejected": -2.1282896995544434,
+      "logps/chosen": -331.67144775390625,
+      "logps/rejected": -274.24932861328125,
+      "loss": 0.3193,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.4981555938720703,
+      "rewards/margins": 2.2620322704315186,
+      "rewards/rejected": -2.760187864303589,
+      "step": 1230
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.9517161140197787e-07,
+      "logits/chosen": -2.4429068565368652,
+      "logits/rejected": -2.3785741329193115,
+      "logps/chosen": -296.1825256347656,
+      "logps/rejected": -311.277099609375,
+      "loss": 0.324,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.26397719979286194,
+      "rewards/margins": 2.5085294246673584,
+      "rewards/rejected": -2.7725067138671875,
+      "step": 1240
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.9226294357184408e-07,
+      "logits/chosen": -2.3132803440093994,
+      "logits/rejected": -2.1668436527252197,
+      "logps/chosen": -297.1078186035156,
+      "logps/rejected": -279.4066162109375,
+      "loss": 0.3619,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.23463638126850128,
+      "rewards/margins": 2.710824489593506,
+      "rewards/rejected": -2.9454610347747803,
+      "step": 1250
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.8935427574171028e-07,
+      "logits/chosen": -2.3645033836364746,
+      "logits/rejected": -2.1661763191223145,
+      "logps/chosen": -317.00616455078125,
+      "logps/rejected": -259.47955322265625,
+      "loss": 0.3045,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.00973432045429945,
+      "rewards/margins": 2.873765468597412,
+      "rewards/rejected": -2.8640313148498535,
+      "step": 1260
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.864456079115765e-07,
+      "logits/chosen": -2.3081579208374023,
+      "logits/rejected": -2.2391886711120605,
+      "logps/chosen": -259.70318603515625,
+      "logps/rejected": -275.3518981933594,
+      "loss": 0.3005,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.2516987919807434,
+      "rewards/margins": 2.7073814868927,
+      "rewards/rejected": -2.959080219268799,
+      "step": 1270
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.835369400814427e-07,
+      "logits/chosen": -2.3735592365264893,
+      "logits/rejected": -2.216593027114868,
+      "logps/chosen": -287.75701904296875,
+      "logps/rejected": -258.850341796875,
+      "loss": 0.2848,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.4840475916862488,
+      "rewards/margins": 2.552049160003662,
+      "rewards/rejected": -3.0360965728759766,
+      "step": 1280
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.8062827225130888e-07,
+      "logits/chosen": -2.294996738433838,
+      "logits/rejected": -2.132660150527954,
+      "logps/chosen": -298.65631103515625,
+      "logps/rejected": -278.43450927734375,
+      "loss": 0.3069,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.8069319725036621,
+      "rewards/margins": 2.365262985229492,
+      "rewards/rejected": -3.1721949577331543,
+      "step": 1290
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.777196044211751e-07,
+      "logits/chosen": -2.289973735809326,
+      "logits/rejected": -2.19386625289917,
+      "logps/chosen": -263.17120361328125,
+      "logps/rejected": -262.7763977050781,
+      "loss": 0.3533,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5058833360671997,
+      "rewards/margins": 2.9816837310791016,
+      "rewards/rejected": -3.487567186355591,
+      "step": 1300
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.748109365910413e-07,
+      "logits/chosen": -2.2870774269104004,
+      "logits/rejected": -2.1734931468963623,
+      "logps/chosen": -235.30648803710938,
+      "logps/rejected": -267.2663269042969,
+      "loss": 0.3568,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.08289147913455963,
+      "rewards/margins": 2.5056369304656982,
+      "rewards/rejected": -2.422745704650879,
+      "step": 1310
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.719022687609075e-07,
+      "logits/chosen": -2.383655071258545,
+      "logits/rejected": -2.16994309425354,
+      "logps/chosen": -342.08282470703125,
+      "logps/rejected": -297.896728515625,
+      "loss": 0.4754,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.2975102961063385,
+      "rewards/margins": 2.276179075241089,
+      "rewards/rejected": -2.5736892223358154,
+      "step": 1320
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.6899360093077368e-07,
+      "logits/chosen": -2.3111634254455566,
+      "logits/rejected": -2.197882890701294,
+      "logps/chosen": -274.7708740234375,
+      "logps/rejected": -243.1353759765625,
+      "loss": 0.4027,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.430492103099823,
+      "rewards/margins": 2.385375499725342,
+      "rewards/rejected": -2.8158674240112305,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.660849331006399e-07,
+      "logits/chosen": -2.374952793121338,
+      "logits/rejected": -2.1876838207244873,
+      "logps/chosen": -325.642578125,
+      "logps/rejected": -267.1546630859375,
+      "loss": 0.2825,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.023454055190086365,
+      "rewards/margins": 3.0091123580932617,
+      "rewards/rejected": -2.9856581687927246,
+      "step": 1340
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.631762652705061e-07,
+      "logits/chosen": -2.3101236820220947,
+      "logits/rejected": -2.2971105575561523,
+      "logps/chosen": -227.51797485351562,
+      "logps/rejected": -258.02642822265625,
+      "loss": 0.2928,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.2340981513261795,
+      "rewards/margins": 2.7252297401428223,
+      "rewards/rejected": -2.9593281745910645,
+      "step": 1350
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.602675974403723e-07,
+      "logits/chosen": -2.3820908069610596,
+      "logits/rejected": -2.2365851402282715,
+      "logps/chosen": -270.7075500488281,
+      "logps/rejected": -252.9586944580078,
+      "loss": 0.3034,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.36182206869125366,
+      "rewards/margins": 3.0204334259033203,
+      "rewards/rejected": -3.3822555541992188,
+      "step": 1360
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.573589296102385e-07,
+      "logits/chosen": -2.3668997287750244,
+      "logits/rejected": -2.200810670852661,
+      "logps/chosen": -286.9949645996094,
+      "logps/rejected": -287.580078125,
+      "loss": 0.3634,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.7097588777542114,
+      "rewards/margins": 2.247542142868042,
+      "rewards/rejected": -2.957301139831543,
+      "step": 1370
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.544502617801047e-07,
+      "logits/chosen": -2.3244481086730957,
+      "logits/rejected": -2.2366535663604736,
+      "logps/chosen": -292.60992431640625,
+      "logps/rejected": -292.54779052734375,
+      "loss": 0.3295,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.16679362952709198,
+      "rewards/margins": 2.6909871101379395,
+      "rewards/rejected": -2.857780694961548,
+      "step": 1380
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.515415939499709e-07,
+      "logits/chosen": -2.213742256164551,
+      "logits/rejected": -2.143993616104126,
+      "logps/chosen": -292.03692626953125,
+      "logps/rejected": -270.4903259277344,
+      "loss": 0.3432,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.5152915120124817,
+      "rewards/margins": 2.64485764503479,
+      "rewards/rejected": -3.160149097442627,
+      "step": 1390
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.486329261198371e-07,
+      "logits/chosen": -2.2537856101989746,
+      "logits/rejected": -2.138474702835083,
+      "logps/chosen": -267.24884033203125,
+      "logps/rejected": -268.0120849609375,
+      "loss": 0.274,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.7926814556121826,
+      "rewards/margins": 2.544640064239502,
+      "rewards/rejected": -3.3373215198516846,
+      "step": 1400
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.4572425828970332e-07,
+      "logits/chosen": -2.3915152549743652,
+      "logits/rejected": -2.170942544937134,
+      "logps/chosen": -314.9630126953125,
+      "logps/rejected": -270.1300964355469,
+      "loss": 0.3387,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.9004882574081421,
+      "rewards/margins": 2.366654872894287,
+      "rewards/rejected": -3.2671427726745605,
+      "step": 1410
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.428155904595695e-07,
+      "logits/chosen": -2.2214369773864746,
+      "logits/rejected": -2.0794403553009033,
+      "logps/chosen": -300.19415283203125,
+      "logps/rejected": -284.22613525390625,
+      "loss": 0.3149,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.6136606931686401,
+      "rewards/margins": 3.0374813079833984,
+      "rewards/rejected": -3.65114164352417,
+      "step": 1420
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.399069226294357e-07,
+      "logits/chosen": -2.1593339443206787,
+      "logits/rejected": -1.9660701751708984,
+      "logps/chosen": -250.0261993408203,
+      "logps/rejected": -228.28903198242188,
+      "loss": 0.2967,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.6077468395233154,
+      "rewards/margins": 2.963073253631592,
+      "rewards/rejected": -3.570819854736328,
+      "step": 1430
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.3699825479930191e-07,
+      "logits/chosen": -2.282439708709717,
+      "logits/rejected": -2.080564498901367,
+      "logps/chosen": -331.7917785644531,
+      "logps/rejected": -233.09375,
+      "loss": 0.3336,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.7863529324531555,
+      "rewards/margins": 2.6935837268829346,
+      "rewards/rejected": -3.479936122894287,
+      "step": 1440
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.3408958696916812e-07,
+      "logits/chosen": -2.2574408054351807,
+      "logits/rejected": -2.13787841796875,
+      "logps/chosen": -265.936767578125,
+      "logps/rejected": -284.001953125,
+      "loss": 0.2295,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.5051341652870178,
+      "rewards/margins": 3.3120696544647217,
+      "rewards/rejected": -3.817204236984253,
+      "step": 1450
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.311809191390343e-07,
+      "logits/chosen": -2.2214787006378174,
+      "logits/rejected": -2.0134081840515137,
+      "logps/chosen": -292.0023498535156,
+      "logps/rejected": -292.8784484863281,
+      "loss": 0.406,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.8032021522521973,
+      "rewards/margins": 3.317984104156494,
+      "rewards/rejected": -4.121186256408691,
+      "step": 1460
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.282722513089005e-07,
+      "logits/chosen": -2.3705811500549316,
+      "logits/rejected": -2.2448134422302246,
+      "logps/chosen": -289.64300537109375,
+      "logps/rejected": -262.7651062011719,
+      "loss": 0.3156,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.44253435730934143,
+      "rewards/margins": 2.5564846992492676,
+      "rewards/rejected": -2.999018907546997,
+      "step": 1470
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.2536358347876672e-07,
+      "logits/chosen": -2.2889206409454346,
+      "logits/rejected": -2.227792501449585,
+      "logps/chosen": -267.8201599121094,
+      "logps/rejected": -258.48626708984375,
+      "loss": 0.295,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.5226227045059204,
+      "rewards/margins": 2.639317035675049,
+      "rewards/rejected": -3.1619396209716797,
+      "step": 1480
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 1.2245491564863293e-07,
+      "logits/chosen": -2.357090473175049,
+      "logits/rejected": -2.142202615737915,
+      "logps/chosen": -330.6837158203125,
+      "logps/rejected": -281.90185546875,
+      "loss": 0.2878,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.4217701852321625,
+      "rewards/margins": 2.895638942718506,
+      "rewards/rejected": -3.317409038543701,
+      "step": 1490
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 1.195462478184991e-07,
+      "logits/chosen": -2.3041255474090576,
+      "logits/rejected": -2.1481804847717285,
+      "logps/chosen": -262.95391845703125,
+      "logps/rejected": -265.3773193359375,
+      "loss": 0.3618,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.6990595459938049,
+      "rewards/margins": 2.7094345092773438,
+      "rewards/rejected": -3.408494234085083,
+      "step": 1500
+    },
+    {
+      "epoch": 0.78,
+      "eval_logits/chosen": -2.328040599822998,
+      "eval_logits/rejected": -2.1793718338012695,
+      "eval_logps/chosen": -289.56976318359375,
+      "eval_logps/rejected": -277.8134460449219,
+      "eval_loss": 0.34519457817077637,
+      "eval_rewards/accuracies": 0.875,
+      "eval_rewards/chosen": -0.4962008595466614,
+      "eval_rewards/margins": 2.787445068359375,
+      "eval_rewards/rejected": -3.2836461067199707,
+      "eval_runtime": 718.9118,
+      "eval_samples_per_second": 2.782,
+      "eval_steps_per_second": 0.088,
+      "step": 1500
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.1663757998836531e-07,
+      "logits/chosen": -2.265958786010742,
+      "logits/rejected": -2.0687148571014404,
+      "logps/chosen": -284.8194580078125,
+      "logps/rejected": -248.05648803710938,
+      "loss": 0.2666,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.4019474387168884,
+      "rewards/margins": 3.251687526702881,
+      "rewards/rejected": -3.653635025024414,
+      "step": 1510
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.1372891215823152e-07,
+      "logits/chosen": -2.3123621940612793,
+      "logits/rejected": -2.2089877128601074,
+      "logps/chosen": -271.91351318359375,
+      "logps/rejected": -272.6903076171875,
+      "loss": 0.3357,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.38390928506851196,
+      "rewards/margins": 2.608208417892456,
+      "rewards/rejected": -2.9921176433563232,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.1082024432809772e-07,
+      "logits/chosen": -2.300546646118164,
+      "logits/rejected": -2.1590487957000732,
+      "logps/chosen": -271.7149963378906,
+      "logps/rejected": -244.64279174804688,
+      "loss": 0.3112,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3754982352256775,
+      "rewards/margins": 2.3724427223205566,
+      "rewards/rejected": -2.747941255569458,
+      "step": 1530
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0791157649796392e-07,
+      "logits/chosen": -2.3611299991607666,
+      "logits/rejected": -2.319636106491089,
+      "logps/chosen": -261.41375732421875,
+      "logps/rejected": -291.7661437988281,
+      "loss": 0.3872,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.07669395208358765,
+      "rewards/margins": 2.4173684120178223,
+      "rewards/rejected": -2.34067440032959,
+      "step": 1540
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0500290866783013e-07,
+      "logits/chosen": -2.212877035140991,
+      "logits/rejected": -2.085379123687744,
+      "logps/chosen": -261.783203125,
+      "logps/rejected": -255.25588989257812,
+      "loss": 0.2901,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.059961654245853424,
+      "rewards/margins": 2.9282031059265137,
+      "rewards/rejected": -2.9881649017333984,
+      "step": 1550
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.0209424083769633e-07,
+      "logits/chosen": -2.261627674102783,
+      "logits/rejected": -2.1716694831848145,
+      "logps/chosen": -273.3644714355469,
+      "logps/rejected": -261.2357177734375,
+      "loss": 0.3538,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.17286448180675507,
+      "rewards/margins": 2.786151885986328,
+      "rewards/rejected": -2.9590163230895996,
+      "step": 1560
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.918557300756253e-08,
+      "logits/chosen": -2.303070545196533,
+      "logits/rejected": -2.106238842010498,
+      "logps/chosen": -286.3017883300781,
+      "logps/rejected": -230.15576171875,
+      "loss": 0.3195,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.3066893517971039,
+      "rewards/margins": 3.2129788398742676,
+      "rewards/rejected": -3.5196681022644043,
+      "step": 1570
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.627690517742873e-08,
+      "logits/chosen": -2.271367311477661,
+      "logits/rejected": -2.1737093925476074,
+      "logps/chosen": -298.4072570800781,
+      "logps/rejected": -290.3675842285156,
+      "loss": 0.288,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.25732770562171936,
+      "rewards/margins": 3.0521702766418457,
+      "rewards/rejected": -2.794842481613159,
+      "step": 1580
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.336823734729494e-08,
+      "logits/chosen": -2.271235227584839,
+      "logits/rejected": -2.1804494857788086,
+      "logps/chosen": -297.44476318359375,
+      "logps/rejected": -279.959228515625,
+      "loss": 0.2927,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.32572048902511597,
+      "rewards/margins": 2.5924339294433594,
+      "rewards/rejected": -2.918154239654541,
+      "step": 1590
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 9.045956951716113e-08,
+      "logits/chosen": -2.292823076248169,
+      "logits/rejected": -2.1955997943878174,
+      "logps/chosen": -247.9260711669922,
+      "logps/rejected": -235.42709350585938,
+      "loss": 0.3524,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.17039936780929565,
+      "rewards/margins": 2.6811909675598145,
+      "rewards/rejected": -2.851590156555176,
+      "step": 1600
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.755090168702734e-08,
+      "logits/chosen": -2.2323901653289795,
+      "logits/rejected": -2.106067657470703,
+      "logps/chosen": -279.44683837890625,
+      "logps/rejected": -286.6934814453125,
+      "loss": 0.2906,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.4923517107963562,
+      "rewards/margins": 3.07497239112854,
+      "rewards/rejected": -3.567323684692383,
+      "step": 1610
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.464223385689353e-08,
+      "logits/chosen": -2.305323362350464,
+      "logits/rejected": -2.233548641204834,
+      "logps/chosen": -288.0763244628906,
+      "logps/rejected": -268.65594482421875,
+      "loss": 0.3689,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.27551114559173584,
+      "rewards/margins": 2.574982166290283,
+      "rewards/rejected": -2.8504936695098877,
+      "step": 1620
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.173356602675974e-08,
+      "logits/chosen": -2.312969923019409,
+      "logits/rejected": -2.1368892192840576,
+      "logps/chosen": -264.1705627441406,
+      "logps/rejected": -252.60848999023438,
+      "loss": 0.3067,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.0068013668060303,
+      "rewards/margins": 2.1217563152313232,
+      "rewards/rejected": -3.1285574436187744,
+      "step": 1630
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.882489819662593e-08,
+      "logits/chosen": -2.2855401039123535,
+      "logits/rejected": -2.19065523147583,
+      "logps/chosen": -282.502197265625,
+      "logps/rejected": -347.73345947265625,
+      "loss": 0.2969,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4646262228488922,
+      "rewards/margins": 2.8088560104370117,
+      "rewards/rejected": -3.273482084274292,
+      "step": 1640
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.591623036649214e-08,
+      "logits/chosen": -2.1352343559265137,
+      "logits/rejected": -2.088813066482544,
+      "logps/chosen": -287.72955322265625,
+      "logps/rejected": -267.5281677246094,
+      "loss": 0.4208,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.43632712960243225,
+      "rewards/margins": 2.592315196990967,
+      "rewards/rejected": -3.028642177581787,
+      "step": 1650
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 7.300756253635834e-08,
+      "logits/chosen": -2.300978183746338,
+      "logits/rejected": -2.151585578918457,
+      "logps/chosen": -266.4535827636719,
+      "logps/rejected": -282.6994323730469,
+      "loss": 0.3317,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.1617407649755478,
+      "rewards/margins": 2.6548030376434326,
+      "rewards/rejected": -2.8165440559387207,
+      "step": 1660
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 7.009889470622454e-08,
+      "logits/chosen": -2.3190159797668457,
+      "logits/rejected": -2.1111197471618652,
+      "logps/chosen": -304.418701171875,
+      "logps/rejected": -261.68865966796875,
+      "loss": 0.296,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.013362640514969826,
+      "rewards/margins": 3.4856789112091064,
+      "rewards/rejected": -3.4990413188934326,
+      "step": 1670
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.719022687609075e-08,
+      "logits/chosen": -2.247602939605713,
+      "logits/rejected": -2.0082693099975586,
+      "logps/chosen": -326.18359375,
+      "logps/rejected": -308.87628173828125,
+      "loss": 0.2498,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1832130253314972,
+      "rewards/margins": 3.8249638080596924,
+      "rewards/rejected": -4.008176326751709,
+      "step": 1680
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.428155904595695e-08,
+      "logits/chosen": -2.1355159282684326,
+      "logits/rejected": -2.1326820850372314,
+      "logps/chosen": -290.07147216796875,
+      "logps/rejected": -286.3025207519531,
+      "loss": 0.2702,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.38392263650894165,
+      "rewards/margins": 3.633469343185425,
+      "rewards/rejected": -3.249546766281128,
+      "step": 1690
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.137289121582315e-08,
+      "logits/chosen": -2.3396739959716797,
+      "logits/rejected": -2.0457844734191895,
+      "logps/chosen": -317.99639892578125,
+      "logps/rejected": -273.2770690917969,
+      "loss": 0.3243,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.09367147088050842,
+      "rewards/margins": 3.0268261432647705,
+      "rewards/rejected": -3.120497941970825,
+      "step": 1700
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.846422338568935e-08,
+      "logits/chosen": -2.2484824657440186,
+      "logits/rejected": -2.0729587078094482,
+      "logps/chosen": -283.7063293457031,
+      "logps/rejected": -268.8601379394531,
+      "loss": 0.2804,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1501619815826416,
+      "rewards/margins": 2.7162556648254395,
+      "rewards/rejected": -2.8664181232452393,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.555555555555555e-08,
+      "logits/chosen": -2.256999969482422,
+      "logits/rejected": -2.092594623565674,
+      "logps/chosen": -333.51324462890625,
+      "logps/rejected": -271.24493408203125,
+      "loss": 0.3285,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.4324091374874115,
+      "rewards/margins": 2.6822280883789062,
+      "rewards/rejected": -3.1146373748779297,
+      "step": 1720
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 5.264688772542175e-08,
+      "logits/chosen": -2.190694570541382,
+      "logits/rejected": -2.0392961502075195,
+      "logps/chosen": -283.0891418457031,
+      "logps/rejected": -250.1522979736328,
+      "loss": 0.2861,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.07667428255081177,
+      "rewards/margins": 2.9937844276428223,
+      "rewards/rejected": -3.0704586505889893,
+      "step": 1730
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.973821989528795e-08,
+      "logits/chosen": -2.285017490386963,
+      "logits/rejected": -2.125302314758301,
+      "logps/chosen": -256.6737365722656,
+      "logps/rejected": -242.7133026123047,
+      "loss": 0.3628,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.20822696387767792,
+      "rewards/margins": 3.1273458003997803,
+      "rewards/rejected": -2.919118881225586,
+      "step": 1740
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.682955206515416e-08,
+      "logits/chosen": -2.3101308345794678,
+      "logits/rejected": -2.1478629112243652,
+      "logps/chosen": -289.7849426269531,
+      "logps/rejected": -312.31524658203125,
+      "loss": 0.3225,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.10851460695266724,
+      "rewards/margins": 3.011849880218506,
+      "rewards/rejected": -3.12036395072937,
+      "step": 1750
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.392088423502036e-08,
+      "logits/chosen": -2.308870553970337,
+      "logits/rejected": -2.1753602027893066,
+      "logps/chosen": -251.39785766601562,
+      "logps/rejected": -231.73434448242188,
+      "loss": 0.3189,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.27802008390426636,
+      "rewards/margins": 2.905123233795166,
+      "rewards/rejected": -3.183143377304077,
+      "step": 1760
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 4.101221640488656e-08,
+      "logits/chosen": -2.264986276626587,
+      "logits/rejected": -2.116339921951294,
+      "logps/chosen": -288.02484130859375,
+      "logps/rejected": -278.58892822265625,
+      "loss": 0.2915,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.2735270857810974,
+      "rewards/margins": 2.9817147254943848,
+      "rewards/rejected": -3.255242109298706,
+      "step": 1770
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.810354857475276e-08,
+      "logits/chosen": -2.1339287757873535,
+      "logits/rejected": -1.9782625436782837,
+      "logps/chosen": -311.86077880859375,
+      "logps/rejected": -266.83270263671875,
+      "loss": 0.3428,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.10870571434497833,
+      "rewards/margins": 3.3522801399230957,
+      "rewards/rejected": -3.4609856605529785,
+      "step": 1780
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.519488074461896e-08,
+      "logits/chosen": -2.223109245300293,
+      "logits/rejected": -2.0661864280700684,
+      "logps/chosen": -251.7618865966797,
+      "logps/rejected": -270.4107971191406,
+      "loss": 0.253,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.07158003002405167,
+      "rewards/margins": 3.161891460418701,
+      "rewards/rejected": -3.090311288833618,
+      "step": 1790
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.228621291448516e-08,
+      "logits/chosen": -2.2715201377868652,
+      "logits/rejected": -2.0951197147369385,
+      "logps/chosen": -315.7049255371094,
+      "logps/rejected": -259.9617919921875,
+      "loss": 0.3807,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.14460697770118713,
+      "rewards/margins": 2.736274480819702,
+      "rewards/rejected": -2.8808817863464355,
+      "step": 1800
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.9377545084351366e-08,
+      "logits/chosen": -2.212463855743408,
+      "logits/rejected": -2.1107747554779053,
+      "logps/chosen": -365.3102111816406,
+      "logps/rejected": -310.04986572265625,
+      "loss": 0.3168,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.15414926409721375,
+      "rewards/margins": 3.325876235961914,
+      "rewards/rejected": -3.171726942062378,
+      "step": 1810
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6468877254217567e-08,
+      "logits/chosen": -2.240906000137329,
+      "logits/rejected": -2.081852436065674,
+      "logps/chosen": -307.33056640625,
+      "logps/rejected": -293.54486083984375,
+      "loss": 0.3318,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.0472014881670475,
+      "rewards/margins": 3.150693416595459,
+      "rewards/rejected": -3.103492021560669,
+      "step": 1820
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.3560209424083768e-08,
+      "logits/chosen": -2.2496564388275146,
+      "logits/rejected": -2.1368825435638428,
+      "logps/chosen": -304.8343200683594,
+      "logps/rejected": -285.279052734375,
+      "loss": 0.3409,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.13887426257133484,
+      "rewards/margins": 2.7014544010162354,
+      "rewards/rejected": -2.8403282165527344,
+      "step": 1830
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.065154159394997e-08,
+      "logits/chosen": -2.22584867477417,
+      "logits/rejected": -2.1327052116394043,
+      "logps/chosen": -314.4709777832031,
+      "logps/rejected": -261.6623229980469,
+      "loss": 0.4897,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.23914465308189392,
+      "rewards/margins": 2.23769474029541,
+      "rewards/rejected": -2.476839542388916,
+      "step": 1840
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7742873763816174e-08,
+      "logits/chosen": -2.220412015914917,
+      "logits/rejected": -2.1215548515319824,
+      "logps/chosen": -315.8754577636719,
+      "logps/rejected": -295.4253845214844,
+      "loss": 0.3629,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.12752804160118103,
+      "rewards/margins": 3.037806510925293,
+      "rewards/rejected": -3.165334701538086,
+      "step": 1850
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4834205933682373e-08,
+      "logits/chosen": -2.270265817642212,
+      "logits/rejected": -2.0846986770629883,
+      "logps/chosen": -285.07525634765625,
+      "logps/rejected": -246.28506469726562,
+      "loss": 0.4131,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.09410500526428223,
+      "rewards/margins": 3.04309344291687,
+      "rewards/rejected": -3.1371986865997314,
+      "step": 1860
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1925538103548575e-08,
+      "logits/chosen": -2.172769546508789,
+      "logits/rejected": -1.9685335159301758,
+      "logps/chosen": -270.36553955078125,
+      "logps/rejected": -245.4918670654297,
+      "loss": 0.3484,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.11144036054611206,
+      "rewards/margins": 2.7038679122924805,
+      "rewards/rejected": -2.815308094024658,
+      "step": 1870
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.016870273414776e-09,
+      "logits/chosen": -2.237025737762451,
+      "logits/rejected": -2.0877583026885986,
+      "logps/chosen": -329.714111328125,
+      "logps/rejected": -306.50347900390625,
+      "loss": 0.2283,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.1830584704875946,
+      "rewards/margins": 3.397716999053955,
+      "rewards/rejected": -3.214658737182617,
+      "step": 1880
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.1082024432809765e-09,
+      "logits/chosen": -2.13305401802063,
+      "logits/rejected": -2.096346378326416,
+      "logps/chosen": -262.7850036621094,
+      "logps/rejected": -269.12371826171875,
+      "loss": 0.3068,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.39133384823799133,
+      "rewards/margins": 2.5701076984405518,
+      "rewards/rejected": -2.9614415168762207,
+      "step": 1890
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.1995346131471783e-09,
+      "logits/chosen": -2.2220492362976074,
+      "logits/rejected": -2.0965094566345215,
+      "logps/chosen": -257.00225830078125,
+      "logps/rejected": -267.1251525878906,
+      "loss": 0.318,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.30209580063819885,
+      "rewards/margins": 3.022385358810425,
+      "rewards/rejected": -3.324481248855591,
+      "step": 1900
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.9086678301337986e-10,
+      "logits/chosen": -2.2612786293029785,
+      "logits/rejected": -2.0825133323669434,
+      "logps/chosen": -295.4743957519531,
+      "logps/rejected": -249.2105255126953,
+      "loss": 0.3212,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.1671399027109146,
+      "rewards/margins": 2.760793447494507,
+      "rewards/rejected": -2.9279332160949707,
+      "step": 1910
+    },
+    {
+      "epoch": 1.0,
+      "step": 1911,
+      "total_flos": 0.0,
+      "train_loss": 0.3594729615826684,
+      "train_runtime": 33818.9349,
+      "train_samples_per_second": 1.808,
+      "train_steps_per_second": 0.057
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1911,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88cd95f98b524d76481ebe26aeee0cba759bd9f52c8e58b76719adc45111641d
+size 5371