Model save

README.md

@@ -0,0 +1,76 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-dpo-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-dpo-full
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5026
+- Rewards/chosen: -1.0472
+- Rewards/rejected: -1.9901
+- Rewards/accuracies: 0.7619
+- Rewards/margins: 0.9430
+- Logps/rejected: -460.7913
+- Logps/chosen: -388.8264
+- Logits/rejected: 1.7625
+- Logits/chosen: 1.0392
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 615
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.5736        | 0.21  | 100  | 0.5842          | -0.3837        | -0.8468          | 0.7242             | 0.4632          | -346.4596      | -322.4754    | -2.3510         | -2.4330       |
+| 0.5116        | 0.42  | 200  | 0.5308          | -0.9042        | -1.7331          | 0.7520             | 0.8289          | -435.0859      | -374.5288    | 0.5012          | 0.0459        |
+| 0.5027        | 0.63  | 300  | 0.5084          | -0.8877        | -1.7467          | 0.7639             | 0.8590          | -436.4478      | -372.8834    | 1.8224          | 1.1385        |
+| 0.4823        | 0.84  | 400  | 0.5037          | -1.1953        | -2.1521          | 0.7619             | 0.9568          | -476.9852      | -403.6375    | 1.9978          | 1.3075        |
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": 1.0391736030578613,
+    "eval_logits/rejected": 1.7624521255493164,
+    "eval_logps/chosen": -388.8264465332031,
+    "eval_logps/rejected": -460.791259765625,
+    "eval_loss": 0.5025966167449951,
+    "eval_rewards/accuracies": 0.761904776096344,
+    "eval_rewards/chosen": -1.0471669435501099,
+    "eval_rewards/margins": 0.9429601430892944,
+    "eval_rewards/rejected": -1.9901270866394043,
+    "eval_runtime": 213.579,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 9.364,
+    "eval_steps_per_second": 0.295,
+    "train_loss": 0.540120158305458,
+    "train_runtime": 18392.9814,
+    "train_samples": 61135,
+    "train_samples_per_second": 3.324,
+    "train_steps_per_second": 0.026
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": 1.0391736030578613,
+    "eval_logits/rejected": 1.7624521255493164,
+    "eval_logps/chosen": -388.8264465332031,
+    "eval_logps/rejected": -460.791259765625,
+    "eval_loss": 0.5025966167449951,
+    "eval_rewards/accuracies": 0.761904776096344,
+    "eval_rewards/chosen": -1.0471669435501099,
+    "eval_rewards/margins": 0.9429601430892944,
+    "eval_rewards/rejected": -1.9901270866394043,
+    "eval_runtime": 213.579,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 9.364,
+    "eval_steps_per_second": 0.295
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:894e6998a1d708164c8c052ed82994019c554d2a754775ff0fdc103c8408ad30
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2d77ba29b74e70c07ce5f8f6247ecc1ae20d7a7667614f2f54e9a30c3aefb16
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89d0c0092c7a58fa205b887a9eb67dcdda279e8fa945df623c07c020e8e91950
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Mar05_22-40-24_gpu-q-23/events.out.tfevents.1709678476.gpu-q-23.3419918.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3159d05fa8a91b5ad9283b195c6288c3b7d5c27294f10434e9e439ea43ac0d0e
-size 33360
+oid sha256:b804cfb78595f57a987bf96bb079c10040c20dce8c912625434320b00ed65afd
+size 38152

runs/Mar05_22-40-24_gpu-q-23/events.out.tfevents.1709697083.gpu-q-23.3419918.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b20c564f74d25abd5261ec555e3587a1c4488cf02e9e61f721bb4bdf716ab615
+size 828

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.540120158305458,
+    "train_runtime": 18392.9814,
+    "train_samples": 61135,
+    "train_samples_per_second": 3.324,
+    "train_steps_per_second": 0.026
+}

trainer_state.json

@@ -0,0 +1,766 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9984301412872841,
+  "eval_steps": 100,
+  "global_step": 477,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.0416666666666666e-08,
+      "logits/chosen": -2.6597137451171875,
+      "logits/rejected": -2.5902962684631348,
+      "logps/chosen": -296.01092529296875,
+      "logps/rejected": -290.09039306640625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.624401807785034,
+      "logits/rejected": -2.58382511138916,
+      "logps/chosen": -278.5157775878906,
+      "logps/rejected": -242.1708984375,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4618055522441864,
+      "rewards/chosen": 0.00044631207128986716,
+      "rewards/margins": 0.0005090843187645078,
+      "rewards/rejected": -6.277219654293731e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.6496779918670654,
+      "logits/rejected": -2.5882985591888428,
+      "logps/chosen": -276.2167663574219,
+      "logps/rejected": -246.029052734375,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0011385921388864517,
+      "rewards/margins": 0.001740106614306569,
+      "rewards/rejected": -0.0006015143590047956,
+      "step": 20
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.6365559101104736,
+      "logits/rejected": -2.5856800079345703,
+      "logps/chosen": -291.7817077636719,
+      "logps/rejected": -274.3924255371094,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": 0.0060002789832651615,
+      "rewards/margins": 0.009898515418171883,
+      "rewards/rejected": -0.0038982369005680084,
+      "step": 30
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.5962817668914795,
+      "logits/rejected": -2.533491611480713,
+      "logps/chosen": -304.540283203125,
+      "logps/rejected": -277.9544677734375,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.02992216870188713,
+      "rewards/margins": 0.03784631937742233,
+      "rewards/rejected": -0.007924148812890053,
+      "step": 40
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999731868769026e-07,
+      "logits/chosen": -2.550187110900879,
+      "logits/rejected": -2.5181498527526855,
+      "logps/chosen": -298.8063049316406,
+      "logps/rejected": -287.06878662109375,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": -0.0016534685855731368,
+      "rewards/margins": 0.07625629007816315,
+      "rewards/rejected": -0.07790975272655487,
+      "step": 50
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": -2.5104401111602783,
+      "logits/rejected": -2.409574031829834,
+      "logps/chosen": -286.75982666015625,
+      "logps/rejected": -255.2505340576172,
+      "loss": 0.6393,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": -0.038099173456430435,
+      "rewards/margins": 0.16247370839118958,
+      "rewards/rejected": -0.2005728781223297,
+      "step": 60
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967625656594781e-07,
+      "logits/chosen": -2.5491867065429688,
+      "logits/rejected": -2.4775824546813965,
+      "logps/chosen": -305.6695556640625,
+      "logps/rejected": -294.116455078125,
+      "loss": 0.6222,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": -0.25629425048828125,
+      "rewards/margins": 0.2100234478712082,
+      "rewards/rejected": -0.466317743062973,
+      "step": 70
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.93167072587771e-07,
+      "logits/chosen": -2.512619972229004,
+      "logits/rejected": -2.459233522415161,
+      "logps/chosen": -313.1577453613281,
+      "logps/rejected": -333.73199462890625,
+      "loss": 0.6115,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": -0.26731860637664795,
+      "rewards/margins": 0.19400887191295624,
+      "rewards/rejected": -0.4613274931907654,
+      "step": 80
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": -2.453090190887451,
+      "logits/rejected": -2.3753929138183594,
+      "logps/chosen": -286.51220703125,
+      "logps/rejected": -299.60174560546875,
+      "loss": 0.5834,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": -0.2537994682788849,
+      "rewards/margins": 0.3461161255836487,
+      "rewards/rejected": -0.5999155044555664,
+      "step": 90
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.820919832540181e-07,
+      "logits/chosen": -2.481619358062744,
+      "logits/rejected": -2.3836333751678467,
+      "logps/chosen": -316.19061279296875,
+      "logps/rejected": -341.30596923828125,
+      "loss": 0.5736,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.3343900740146637,
+      "rewards/margins": 0.4206913113594055,
+      "rewards/rejected": -0.755081295967102,
+      "step": 100
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": -2.432966470718384,
+      "eval_logits/rejected": -2.3509771823883057,
+      "eval_logps/chosen": -322.4753723144531,
+      "eval_logps/rejected": -346.4595947265625,
+      "eval_loss": 0.584247350692749,
+      "eval_rewards/accuracies": 0.7242063283920288,
+      "eval_rewards/chosen": -0.3836560547351837,
+      "eval_rewards/margins": 0.46315449476242065,
+      "eval_rewards/rejected": -0.846810519695282,
+      "eval_runtime": 210.4195,
+      "eval_samples_per_second": 9.505,
+      "eval_steps_per_second": 0.299,
+      "step": 100
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7467175306295647e-07,
+      "logits/chosen": -2.392451763153076,
+      "logits/rejected": -2.2609496116638184,
+      "logps/chosen": -329.49127197265625,
+      "logps/rejected": -338.2959289550781,
+      "loss": 0.5785,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": -0.32086434960365295,
+      "rewards/margins": 0.4109874665737152,
+      "rewards/rejected": -0.7318518161773682,
+      "step": 110
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": -1.5522091388702393,
+      "logits/rejected": -1.2740141153335571,
+      "logps/chosen": -365.6708984375,
+      "logps/rejected": -378.64117431640625,
+      "loss": 0.5441,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": -0.8018746376037598,
+      "rewards/margins": 0.6111066341400146,
+      "rewards/rejected": -1.4129812717437744,
+      "step": 120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5626458262912735e-07,
+      "logits/chosen": -1.4644163846969604,
+      "logits/rejected": -1.1872146129608154,
+      "logps/chosen": -324.7268371582031,
+      "logps/rejected": -374.5579833984375,
+      "loss": 0.549,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.4890199601650238,
+      "rewards/margins": 0.5601330995559692,
+      "rewards/rejected": -1.0491530895233154,
+      "step": 130
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.453763107901675e-07,
+      "logits/chosen": -0.9067952036857605,
+      "logits/rejected": -0.45260438323020935,
+      "logps/chosen": -365.0186462402344,
+      "logps/rejected": -414.08526611328125,
+      "loss": 0.5265,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": -0.6849763989448547,
+      "rewards/margins": 0.7878872752189636,
+      "rewards/rejected": -1.4728636741638184,
+      "step": 140
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": -0.6669697761535645,
+      "logits/rejected": -0.12073500454425812,
+      "logps/chosen": -363.00494384765625,
+      "logps/rejected": -401.21533203125,
+      "loss": 0.5248,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.7877362966537476,
+      "rewards/margins": 0.7080036401748657,
+      "rewards/rejected": -1.4957398176193237,
+      "step": 150
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2052190435769554e-07,
+      "logits/chosen": -0.41461697220802307,
+      "logits/rejected": 0.06870967149734497,
+      "logps/chosen": -365.81103515625,
+      "logps/rejected": -428.4544982910156,
+      "loss": 0.5073,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.8387645483016968,
+      "rewards/margins": 0.7532976269721985,
+      "rewards/rejected": -1.59206223487854,
+      "step": 160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.0668899744407567e-07,
+      "logits/chosen": -0.7587008476257324,
+      "logits/rejected": -0.3191295862197876,
+      "logps/chosen": -342.67425537109375,
+      "logps/rejected": -367.0728759765625,
+      "loss": 0.5653,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": -0.8000960350036621,
+      "rewards/margins": 0.5834600329399109,
+      "rewards/rejected": -1.3835561275482178,
+      "step": 170
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": -0.17186331748962402,
+      "logits/rejected": 0.25430601835250854,
+      "logps/chosen": -365.8907165527344,
+      "logps/rejected": -429.11688232421875,
+      "loss": 0.5225,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.884574294090271,
+      "rewards/margins": 0.7979339361190796,
+      "rewards/rejected": -1.6825082302093506,
+      "step": 180
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.765821230985757e-07,
+      "logits/chosen": 0.03502330183982849,
+      "logits/rejected": 0.6186638474464417,
+      "logps/chosen": -348.1022644042969,
+      "logps/rejected": -428.8955993652344,
+      "loss": 0.5105,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.7972752451896667,
+      "rewards/margins": 0.9095140695571899,
+      "rewards/rejected": -1.706789255142212,
+      "step": 190
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.604695382782159e-07,
+      "logits/chosen": -0.39503103494644165,
+      "logits/rejected": 0.1367359161376953,
+      "logps/chosen": -365.77545166015625,
+      "logps/rejected": -400.91998291015625,
+      "loss": 0.5116,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": -0.7570338249206543,
+      "rewards/margins": 0.7489927411079407,
+      "rewards/rejected": -1.5060265064239502,
+      "step": 200
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": 0.04591968283057213,
+      "eval_logits/rejected": 0.5012260675430298,
+      "eval_logps/chosen": -374.5287780761719,
+      "eval_logps/rejected": -435.0859375,
+      "eval_loss": 0.5307875871658325,
+      "eval_rewards/accuracies": 0.7519841194152832,
+      "eval_rewards/chosen": -0.904190182685852,
+      "eval_rewards/margins": 0.8288835883140564,
+      "eval_rewards/rejected": -1.7330738306045532,
+      "eval_runtime": 211.7151,
+      "eval_samples_per_second": 9.447,
+      "eval_steps_per_second": 0.298,
+      "step": 200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": 0.21950086951255798,
+      "logits/rejected": 0.7312016487121582,
+      "logps/chosen": -394.24078369140625,
+      "logps/rejected": -439.376220703125,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -1.0525823831558228,
+      "rewards/margins": 0.7364203929901123,
+      "rewards/rejected": -1.789002776145935,
+      "step": 210
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.265574537815398e-07,
+      "logits/chosen": 0.034196797758340836,
+      "logits/rejected": 0.35187411308288574,
+      "logps/chosen": -332.6381530761719,
+      "logps/rejected": -382.64434814453125,
+      "loss": 0.5384,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": -0.7204285860061646,
+      "rewards/margins": 0.6646216511726379,
+      "rewards/rejected": -1.3850500583648682,
+      "step": 220
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0893973387735683e-07,
+      "logits/chosen": 0.3270489573478699,
+      "logits/rejected": 0.8940129280090332,
+      "logps/chosen": -341.1247253417969,
+      "logps/rejected": -408.52337646484375,
+      "loss": 0.5218,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.765308141708374,
+      "rewards/margins": 0.8177844882011414,
+      "rewards/rejected": -1.583092451095581,
+      "step": 230
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": 0.8036887049674988,
+      "logits/rejected": 1.329053521156311,
+      "logps/chosen": -384.66473388671875,
+      "logps/rejected": -448.1686096191406,
+      "loss": 0.5285,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1919479370117188,
+      "rewards/margins": 0.8430492281913757,
+      "rewards/rejected": -2.0349972248077393,
+      "step": 240
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7285261601056697e-07,
+      "logits/chosen": 0.51897132396698,
+      "logits/rejected": 1.1455881595611572,
+      "logps/chosen": -373.64044189453125,
+      "logps/rejected": -425.75494384765625,
+      "loss": 0.519,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -0.9999877214431763,
+      "rewards/margins": 0.7631025314331055,
+      "rewards/rejected": -1.7630901336669922,
+      "step": 250
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5457665670441937e-07,
+      "logits/chosen": 0.5749747157096863,
+      "logits/rejected": 1.220221757888794,
+      "logps/chosen": -371.4618225097656,
+      "logps/rejected": -424.69439697265625,
+      "loss": 0.4969,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.9122276306152344,
+      "rewards/margins": 0.8183482885360718,
+      "rewards/rejected": -1.7305759191513062,
+      "step": 260
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": 0.8650910258293152,
+      "logits/rejected": 1.3838642835617065,
+      "logps/chosen": -398.350341796875,
+      "logps/rejected": -484.79547119140625,
+      "loss": 0.5051,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.091962456703186,
+      "rewards/margins": 0.9698148965835571,
+      "rewards/rejected": -2.061777353286743,
+      "step": 270
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1804923757009882e-07,
+      "logits/chosen": 0.9545858502388,
+      "logits/rejected": 1.6210914850234985,
+      "logps/chosen": -365.74053955078125,
+      "logps/rejected": -422.39300537109375,
+      "loss": 0.5248,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": -0.9749383926391602,
+      "rewards/margins": 0.841883659362793,
+      "rewards/rejected": -1.8168220520019531,
+      "step": 280
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9999357655598891e-07,
+      "logits/chosen": 1.363952875137329,
+      "logits/rejected": 2.0388360023498535,
+      "logps/chosen": -385.5817565917969,
+      "logps/rejected": -436.33209228515625,
+      "loss": 0.5155,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": -1.1055189371109009,
+      "rewards/margins": 0.77390056848526,
+      "rewards/rejected": -1.8794195652008057,
+      "step": 290
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": 1.3142567873001099,
+      "logits/rejected": 1.9236018657684326,
+      "logps/chosen": -394.31964111328125,
+      "logps/rejected": -467.43756103515625,
+      "loss": 0.5027,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.9480918049812317,
+      "rewards/margins": 0.7692841291427612,
+      "rewards/rejected": -1.7173759937286377,
+      "step": 300
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": 1.1385366916656494,
+      "eval_logits/rejected": 1.822389006614685,
+      "eval_logps/chosen": -372.8834228515625,
+      "eval_logps/rejected": -436.4477844238281,
+      "eval_loss": 0.508425235748291,
+      "eval_rewards/accuracies": 0.7638888955116272,
+      "eval_rewards/chosen": -0.887736976146698,
+      "eval_rewards/margins": 0.8589555621147156,
+      "eval_rewards/rejected": -1.746692419052124,
+      "eval_runtime": 211.1789,
+      "eval_samples_per_second": 9.471,
+      "eval_steps_per_second": 0.298,
+      "step": 300
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.647817538357072e-07,
+      "logits/chosen": 1.0226597785949707,
+      "logits/rejected": 1.96248459815979,
+      "logps/chosen": -367.84130859375,
+      "logps/rejected": -404.55963134765625,
+      "loss": 0.4985,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.9431735277175903,
+      "rewards/margins": 0.8295876383781433,
+      "rewards/rejected": -1.7727611064910889,
+      "step": 310
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.478143389201113e-07,
+      "logits/chosen": 1.6019757986068726,
+      "logits/rejected": 2.321258068084717,
+      "logps/chosen": -374.2918701171875,
+      "logps/rejected": -456.11602783203125,
+      "loss": 0.5035,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": -1.0117809772491455,
+      "rewards/margins": 0.9862319231033325,
+      "rewards/rejected": -1.998012900352478,
+      "step": 320
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": 1.089303970336914,
+      "logits/rejected": 1.724713921546936,
+      "logps/chosen": -422.41314697265625,
+      "logps/rejected": -462.42999267578125,
+      "loss": 0.5162,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.184761881828308,
+      "rewards/margins": 0.7901886701583862,
+      "rewards/rejected": -1.9749505519866943,
+      "step": 330
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1561076868822755e-07,
+      "logits/chosen": 1.2003899812698364,
+      "logits/rejected": 1.7042887210845947,
+      "logps/chosen": -374.2175598144531,
+      "logps/rejected": -423.61114501953125,
+      "loss": 0.5141,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.0190467834472656,
+      "rewards/margins": 0.6825306415557861,
+      "rewards/rejected": -1.7015774250030518,
+      "step": 340
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0054723495346482e-07,
+      "logits/chosen": 1.183173418045044,
+      "logits/rejected": 2.0356898307800293,
+      "logps/chosen": -359.0115661621094,
+      "logps/rejected": -438.88336181640625,
+      "loss": 0.4766,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": -1.0118539333343506,
+      "rewards/margins": 0.9436109662055969,
+      "rewards/rejected": -1.9554650783538818,
+      "step": 350
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": 1.1831741333007812,
+      "logits/rejected": 1.9693758487701416,
+      "logps/chosen": -408.16705322265625,
+      "logps/rejected": -451.48370361328125,
+      "loss": 0.4836,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": -1.0999749898910522,
+      "rewards/margins": 0.8795393705368042,
+      "rewards/rejected": -1.9795143604278564,
+      "step": 360
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.289996455765748e-08,
+      "logits/chosen": 1.1307913064956665,
+      "logits/rejected": 2.096768856048584,
+      "logps/chosen": -413.69805908203125,
+      "logps/rejected": -458.0303649902344,
+      "loss": 0.499,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.0460116863250732,
+      "rewards/margins": 0.9790540933609009,
+      "rewards/rejected": -2.0250658988952637,
+      "step": 370
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.046442623320145e-08,
+      "logits/chosen": 0.9909412264823914,
+      "logits/rejected": 1.6467933654785156,
+      "logps/chosen": -381.5520324707031,
+      "logps/rejected": -453.1272888183594,
+      "loss": 0.4727,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0082350969314575,
+      "rewards/margins": 0.8770313262939453,
+      "rewards/rejected": -1.8852663040161133,
+      "step": 380
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": 1.356707215309143,
+      "logits/rejected": 1.9564104080200195,
+      "logps/chosen": -383.09393310546875,
+      "logps/rejected": -455.9044494628906,
+      "loss": 0.5061,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.0799481868743896,
+      "rewards/margins": 0.7995551228523254,
+      "rewards/rejected": -1.879503607749939,
+      "step": 390
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.8702478614051345e-08,
+      "logits/chosen": 1.2665964365005493,
+      "logits/rejected": 1.9937832355499268,
+      "logps/chosen": -441.0684509277344,
+      "logps/rejected": -508.45269775390625,
+      "loss": 0.4823,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.202941656112671,
+      "rewards/margins": 1.0496307611465454,
+      "rewards/rejected": -2.2525722980499268,
+      "step": 400
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": 1.3074696063995361,
+      "eval_logits/rejected": 1.9977563619613647,
+      "eval_logps/chosen": -403.6374816894531,
+      "eval_logps/rejected": -476.98516845703125,
+      "eval_loss": 0.5037237405776978,
+      "eval_rewards/accuracies": 0.761904776096344,
+      "eval_rewards/chosen": -1.195277214050293,
+      "eval_rewards/margins": 0.9567888975143433,
+      "eval_rewards/rejected": -2.152066230773926,
+      "eval_runtime": 212.3326,
+      "eval_samples_per_second": 9.419,
+      "eval_steps_per_second": 0.297,
+      "step": 400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9492720416985e-08,
+      "logits/chosen": 1.6189842224121094,
+      "logits/rejected": 2.477853775024414,
+      "logps/chosen": -386.3858947753906,
+      "logps/rejected": -442.0225524902344,
+      "loss": 0.5095,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": -1.2031550407409668,
+      "rewards/margins": 0.845988929271698,
+      "rewards/rejected": -2.0491440296173096,
+      "step": 410
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": 1.4740936756134033,
+      "logits/rejected": 1.9605319499969482,
+      "logps/chosen": -398.62274169921875,
+      "logps/rejected": -460.81201171875,
+      "loss": 0.5112,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -1.1762058734893799,
+      "rewards/margins": 0.8007314801216125,
+      "rewards/rejected": -1.9769372940063477,
+      "step": 420
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4662207078575684e-08,
+      "logits/chosen": 1.1320947408676147,
+      "logits/rejected": 1.8564393520355225,
+      "logps/chosen": -397.1152648925781,
+      "logps/rejected": -444.3277282714844,
+      "loss": 0.5032,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.07761549949646,
+      "rewards/margins": 0.886116623878479,
+      "rewards/rejected": -1.963732361793518,
+      "step": 430
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.12094829893642e-09,
+      "logits/chosen": 1.1670544147491455,
+      "logits/rejected": 1.9005542993545532,
+      "logps/chosen": -362.5390319824219,
+      "logps/rejected": -448.8984375,
+      "loss": 0.4901,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.019899606704712,
+      "rewards/margins": 0.9710233807563782,
+      "rewards/rejected": -1.9909226894378662,
+      "step": 440
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": 1.165825605392456,
+      "logits/rejected": 1.7851331233978271,
+      "logps/chosen": -393.0291442871094,
+      "logps/rejected": -466.2897033691406,
+      "loss": 0.5112,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.1060134172439575,
+      "rewards/margins": 0.8082348704338074,
+      "rewards/rejected": -1.9142482280731201,
+      "step": 450
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9347820230782295e-09,
+      "logits/chosen": 1.1381809711456299,
+      "logits/rejected": 2.048168897628784,
+      "logps/chosen": -380.725830078125,
+      "logps/rejected": -454.0580139160156,
+      "loss": 0.4878,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.0911556482315063,
+      "rewards/margins": 0.9514607191085815,
+      "rewards/rejected": -2.042616367340088,
+      "step": 460
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.2839470889836627e-10,
+      "logits/chosen": 0.9774567484855652,
+      "logits/rejected": 1.967919111251831,
+      "logps/chosen": -411.60504150390625,
+      "logps/rejected": -474.35650634765625,
+      "loss": 0.4835,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -1.1035670042037964,
+      "rewards/margins": 0.9632620811462402,
+      "rewards/rejected": -2.066829204559326,
+      "step": 470
+    },
+    {
+      "epoch": 1.0,
+      "step": 477,
+      "total_flos": 0.0,
+      "train_loss": 0.540120158305458,
+      "train_runtime": 18392.9814,
+      "train_samples_per_second": 3.324,
+      "train_steps_per_second": 0.026
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 477,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}