Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,79 @@
+---
+library_name: transformers
+license: gemma
+base_model: google/gemma-7b
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: gemma-7b-borpo-low-quality-v4
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gemma-7b-borpo-low-quality-v4
+
+This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.9080
+- Rewards/chosen: -0.6019
+- Rewards/rejected: -0.7507
+- Rewards/accuracies: 0.6259
+- Rewards/margins: 0.1488
+- Logps/rejected: -1.5015
+- Logps/chosen: -1.2038
+- Logits/rejected: 247.6263
+- Logits/chosen: 282.0085
+- Nll Loss: 1.5545
+- Log Odds Ratio: -0.6477
+- Log Odds Chosen: 0.4230
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 4
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Nll Loss | Log Odds Ratio | Log Odds Chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:--------------:|:---------------:|
+| 1.8238        | 0.9955 | 167  | 1.8176          | -0.5378        | -0.6319          | 0.5468             | 0.0941          | -1.2637        | -1.0755      | 293.2744        | 322.6454      | 1.4783   | -0.6631        | 0.2616          |
+| 1.3092        | 1.9970 | 335  | 1.7560          | -0.5202        | -0.6309          | 0.5324             | 0.1106          | -1.2617        | -1.0405      | 279.0659        | 309.3900      | 1.4054   | -0.6637        | 0.3224          |
+| 0.6827        | 2.9866 | 501  | 1.9080          | -0.6019        | -0.7507          | 0.6259             | 0.1488          | -1.5015        | -1.2038      | 247.6263        | 282.0085      | 1.5545   | -0.6477        | 0.4230          |
+
+
+### Framework versions
+
+- Transformers 4.44.2
+- Pytorch 2.4.1+cu121
+- Datasets 3.0.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9865871833084947,
+    "total_flos": 0.0,
+    "train_loss": 1.686337627812536,
+    "train_runtime": 31593.308,
+    "train_samples": 5364,
+    "train_samples_per_second": 0.509,
+    "train_steps_per_second": 0.016
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "google/gemma-7b",
+  "architectures": [
+    "GemmaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 106,
+  "eos_token_id": 107,
+  "head_dim": 256,
+  "hidden_act": "gelu",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 24576,
+  "max_position_embeddings": 8192,
+  "model_type": "gemma",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 16,
+  "pad_token_id": 107,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "use_cache": false,
+  "vocab_size": 256000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 106,
+  "eos_token_id": 107,
+  "pad_token_id": 107,
+  "transformers_version": "4.44.2"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7014c73861af23b57167bd5b78fd5da80aaf28da346b1537da15fcd99f35be90
+size 4995496656

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:474e7b757ab8f3d73b03132238cd45157a3836ee6c115882cc07a944d0ccad63
+size 4982953168

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d40c9daedb4f0dcb282736b5c1db225d948b8ec6e5c6def0c4a7c506cff1583b
+size 4982953200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7669d74a0d322dfc1fb3a8e2c6f0b2bc1bbe19141386e928f00901baf950b419
+size 2113988336

model.safetensors.index.json

@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 17075361792
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

runs/Sep22_17-15-08_zizgpu06.cpu.stats.ox.ac.uk/events.out.tfevents.1727021816.zizgpu06.cpu.stats.ox.ac.uk.3932700.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74ceadefd8f31214f7b970e4574a0cd983fdd61915f926193a5be4e24f21c5fa
+size 94140

special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:322a5f52ab5cab196761ab397a022d6fa3a2e1418585e532bb6efb2fedd2ae94
+size 17477501

tokenizer_config.json

@@ -0,0 +1,70 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "106": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": null,
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9865871833084947,
+    "total_flos": 0.0,
+    "train_loss": 1.686337627812536,
+    "train_runtime": 31593.308,
+    "train_samples": 5364,
+    "train_samples_per_second": 0.509,
+    "train_steps_per_second": 0.016
+}

trainer_state.json

@@ -0,0 +1,1899 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9865871833084947,
+  "eval_steps": 500,
+  "global_step": 501,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.029806259314456036,
+      "grad_norm": 1823.4444580078125,
+      "learning_rate": 2.5000000000000004e-07,
+      "log_odds_chosen": -0.22316808998584747,
+      "log_odds_ratio": -1.008633017539978,
+      "logits/chosen": 204.3894805908203,
+      "logits/rejected": 203.0862274169922,
+      "logps/chosen": -14.825190544128418,
+      "logps/rejected": -14.602023124694824,
+      "loss": 15.3951,
+      "nll_loss": 14.544248580932617,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -7.412595272064209,
+      "rewards/margins": -0.1115839034318924,
+      "rewards/rejected": -7.301011562347412,
+      "step": 5
+    },
+    {
+      "epoch": 0.05961251862891207,
+      "grad_norm": 1235.2249755859375,
+      "learning_rate": 5.000000000000001e-07,
+      "log_odds_chosen": 0.2599984109401703,
+      "log_odds_ratio": -0.7687832713127136,
+      "logits/chosen": 219.47842407226562,
+      "logits/rejected": 223.53195190429688,
+      "logps/chosen": -12.257360458374023,
+      "logps/rejected": -12.516705513000488,
+      "loss": 13.0935,
+      "nll_loss": 12.36180305480957,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -6.128680229187012,
+      "rewards/margins": 0.1296723335981369,
+      "rewards/rejected": -6.258352756500244,
+      "step": 10
+    },
+    {
+      "epoch": 0.08941877794336811,
+      "grad_norm": 800.8740844726562,
+      "learning_rate": 7.5e-07,
+      "log_odds_chosen": 0.07262952625751495,
+      "log_odds_ratio": -0.7626513242721558,
+      "logits/chosen": 284.1068115234375,
+      "logits/rejected": 263.359619140625,
+      "logps/chosen": -8.042943000793457,
+      "logps/rejected": -8.115506172180176,
+      "loss": 8.8161,
+      "nll_loss": 8.040520668029785,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.0214715003967285,
+      "rewards/margins": 0.03628172725439072,
+      "rewards/rejected": -4.057753086090088,
+      "step": 15
+    },
+    {
+      "epoch": 0.11922503725782414,
+      "grad_norm": 245.36756896972656,
+      "learning_rate": 1.0000000000000002e-06,
+      "log_odds_chosen": -0.1374012976884842,
+      "log_odds_ratio": -0.9441366195678711,
+      "logits/chosen": 284.14410400390625,
+      "logits/rejected": 279.113037109375,
+      "logps/chosen": -5.502886772155762,
+      "logps/rejected": -5.366313934326172,
+      "loss": 5.9511,
+      "nll_loss": 5.569659233093262,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -2.751443386077881,
+      "rewards/margins": -0.06828644871711731,
+      "rewards/rejected": -2.683156967163086,
+      "step": 20
+    },
+    {
+      "epoch": 0.14903129657228018,
+      "grad_norm": 150.3289337158203,
+      "learning_rate": 1.25e-06,
+      "log_odds_chosen": -0.0038304091431200504,
+      "log_odds_ratio": -0.8650141954421997,
+      "logits/chosen": 296.8591003417969,
+      "logits/rejected": 306.09613037109375,
+      "logps/chosen": -3.4051125049591064,
+      "logps/rejected": -3.383711338043213,
+      "loss": 3.9954,
+      "nll_loss": 3.5142741203308105,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.7025562524795532,
+      "rewards/margins": -0.010700652375817299,
+      "rewards/rejected": -1.6918556690216064,
+      "step": 25
+    },
+    {
+      "epoch": 0.17883755588673622,
+      "grad_norm": 95.30947875976562,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": -0.02694419026374817,
+      "log_odds_ratio": -0.8674762845039368,
+      "logits/chosen": 341.46405029296875,
+      "logits/rejected": 369.68597412109375,
+      "logps/chosen": -2.748025894165039,
+      "logps/rejected": -2.6941990852355957,
+      "loss": 3.0003,
+      "nll_loss": 2.7433438301086426,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3740129470825195,
+      "rewards/margins": -0.02691327966749668,
+      "rewards/rejected": -1.3470995426177979,
+      "step": 30
+    },
+    {
+      "epoch": 0.20864381520119224,
+      "grad_norm": 61.000022888183594,
+      "learning_rate": 1.75e-06,
+      "log_odds_chosen": 0.20579464733600616,
+      "log_odds_ratio": -0.7235999703407288,
+      "logits/chosen": 380.79266357421875,
+      "logits/rejected": 368.7811279296875,
+      "logps/chosen": -1.8658641576766968,
+      "logps/rejected": -2.010704517364502,
+      "loss": 2.4897,
+      "nll_loss": 2.063866376876831,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.9329320788383484,
+      "rewards/margins": 0.07242023944854736,
+      "rewards/rejected": -1.005352258682251,
+      "step": 35
+    },
+    {
+      "epoch": 0.23845007451564829,
+      "grad_norm": 52.54775619506836,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": 0.14920620620250702,
+      "log_odds_ratio": -0.7215272188186646,
+      "logits/chosen": 373.4493713378906,
+      "logits/rejected": 372.3648376464844,
+      "logps/chosen": -1.7074140310287476,
+      "logps/rejected": -1.8317168951034546,
+      "loss": 2.3294,
+      "nll_loss": 2.0935075283050537,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.8537070155143738,
+      "rewards/margins": 0.06215135008096695,
+      "rewards/rejected": -0.9158584475517273,
+      "step": 40
+    },
+    {
+      "epoch": 0.26825633383010433,
+      "grad_norm": 58.8853645324707,
+      "learning_rate": 2.25e-06,
+      "log_odds_chosen": 0.20898687839508057,
+      "log_odds_ratio": -0.666365385055542,
+      "logits/chosen": 390.11090087890625,
+      "logits/rejected": 399.29052734375,
+      "logps/chosen": -1.6940256357192993,
+      "logps/rejected": -1.859763503074646,
+      "loss": 2.2384,
+      "nll_loss": 1.81049382686615,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.8470128178596497,
+      "rewards/margins": 0.08286889642477036,
+      "rewards/rejected": -0.929881751537323,
+      "step": 45
+    },
+    {
+      "epoch": 0.29806259314456035,
+      "grad_norm": 53.57241439819336,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 0.4837431013584137,
+      "log_odds_ratio": -0.5640600919723511,
+      "logits/chosen": 397.261474609375,
+      "logits/rejected": 418.50787353515625,
+      "logps/chosen": -1.548592209815979,
+      "logps/rejected": -1.9497201442718506,
+      "loss": 2.2689,
+      "nll_loss": 1.9370243549346924,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7742961049079895,
+      "rewards/margins": 0.20056398212909698,
+      "rewards/rejected": -0.9748600721359253,
+      "step": 50
+    },
+    {
+      "epoch": 0.32786885245901637,
+      "grad_norm": 54.863670349121094,
+      "learning_rate": 2.7500000000000004e-06,
+      "log_odds_chosen": 0.10273619741201401,
+      "log_odds_ratio": -0.7500853538513184,
+      "logits/chosen": 393.6875,
+      "logits/rejected": 385.71246337890625,
+      "logps/chosen": -1.6378755569458008,
+      "logps/rejected": -1.7148265838623047,
+      "loss": 2.343,
+      "nll_loss": 2.059002637863159,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.8189377784729004,
+      "rewards/margins": 0.038475506007671356,
+      "rewards/rejected": -0.8574132919311523,
+      "step": 55
+    },
+    {
+      "epoch": 0.35767511177347244,
+      "grad_norm": 52.10759735107422,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": 0.20460805296897888,
+      "log_odds_ratio": -0.6378257870674133,
+      "logits/chosen": 393.1427917480469,
+      "logits/rejected": 384.66424560546875,
+      "logps/chosen": -1.5411105155944824,
+      "logps/rejected": -1.693114995956421,
+      "loss": 2.0643,
+      "nll_loss": 1.8742763996124268,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.7705552577972412,
+      "rewards/margins": 0.07600229978561401,
+      "rewards/rejected": -0.8465574979782104,
+      "step": 60
+    },
+    {
+      "epoch": 0.38748137108792846,
+      "grad_norm": 36.03993225097656,
+      "learning_rate": 3.2500000000000002e-06,
+      "log_odds_chosen": 0.31621289253234863,
+      "log_odds_ratio": -0.6504405736923218,
+      "logits/chosen": 391.6888427734375,
+      "logits/rejected": 392.4349365234375,
+      "logps/chosen": -1.3424034118652344,
+      "logps/rejected": -1.5777828693389893,
+      "loss": 1.9359,
+      "nll_loss": 1.5658212900161743,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6712017059326172,
+      "rewards/margins": 0.11768964678049088,
+      "rewards/rejected": -0.7888914346694946,
+      "step": 65
+    },
+    {
+      "epoch": 0.4172876304023845,
+      "grad_norm": 41.6033821105957,
+      "learning_rate": 3.5e-06,
+      "log_odds_chosen": 0.3797632157802582,
+      "log_odds_ratio": -0.6185951828956604,
+      "logits/chosen": 376.75579833984375,
+      "logits/rejected": 385.7332763671875,
+      "logps/chosen": -1.507440209388733,
+      "logps/rejected": -1.8190501928329468,
+      "loss": 2.0024,
+      "nll_loss": 1.7485978603363037,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7537201046943665,
+      "rewards/margins": 0.1558050811290741,
+      "rewards/rejected": -0.9095250964164734,
+      "step": 70
+    },
+    {
+      "epoch": 0.44709388971684055,
+      "grad_norm": 49.81876754760742,
+      "learning_rate": 3.7500000000000005e-06,
+      "log_odds_chosen": 0.5595852732658386,
+      "log_odds_ratio": -0.531262218952179,
+      "logits/chosen": 394.0195617675781,
+      "logits/rejected": 380.848388671875,
+      "logps/chosen": -1.278014898300171,
+      "logps/rejected": -1.7234604358673096,
+      "loss": 1.8859,
+      "nll_loss": 1.5626357793807983,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.6390074491500854,
+      "rewards/margins": 0.22272281348705292,
+      "rewards/rejected": -0.8617302179336548,
+      "step": 75
+    },
+    {
+      "epoch": 0.47690014903129657,
+      "grad_norm": 49.527286529541016,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": 0.4827095866203308,
+      "log_odds_ratio": -0.5827818512916565,
+      "logits/chosen": 383.16455078125,
+      "logits/rejected": 405.3050231933594,
+      "logps/chosen": -1.4663052558898926,
+      "logps/rejected": -1.8465086221694946,
+      "loss": 1.9594,
+      "nll_loss": 1.6925424337387085,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7331526279449463,
+      "rewards/margins": 0.19010166823863983,
+      "rewards/rejected": -0.9232543110847473,
+      "step": 80
+    },
+    {
+      "epoch": 0.5067064083457526,
+      "grad_norm": 35.50678634643555,
+      "learning_rate": 4.25e-06,
+      "log_odds_chosen": 0.32843467593193054,
+      "log_odds_ratio": -0.7204899787902832,
+      "logits/chosen": 409.7994689941406,
+      "logits/rejected": 394.08575439453125,
+      "logps/chosen": -1.5447129011154175,
+      "logps/rejected": -1.7741330862045288,
+      "loss": 1.9717,
+      "nll_loss": 1.7773849964141846,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7723564505577087,
+      "rewards/margins": 0.11471016705036163,
+      "rewards/rejected": -0.8870665431022644,
+      "step": 85
+    },
+    {
+      "epoch": 0.5365126676602087,
+      "grad_norm": 32.97898864746094,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": 1.082437515258789,
+      "log_odds_ratio": -0.48028555512428284,
+      "logits/chosen": 400.08160400390625,
+      "logits/rejected": 415.0059509277344,
+      "logps/chosen": -1.4097836017608643,
+      "logps/rejected": -2.338646411895752,
+      "loss": 1.7591,
+      "nll_loss": 1.5575084686279297,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7048918008804321,
+      "rewards/margins": 0.4644315242767334,
+      "rewards/rejected": -1.169323205947876,
+      "step": 90
+    },
+    {
+      "epoch": 0.5663189269746647,
+      "grad_norm": 59.56174850463867,
+      "learning_rate": 4.75e-06,
+      "log_odds_chosen": 0.43072813749313354,
+      "log_odds_ratio": -0.5813931226730347,
+      "logits/chosen": 370.78253173828125,
+      "logits/rejected": 383.03009033203125,
+      "logps/chosen": -1.1654596328735352,
+      "logps/rejected": -1.4328858852386475,
+      "loss": 1.8455,
+      "nll_loss": 1.4887888431549072,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5827298164367676,
+      "rewards/margins": 0.13371312618255615,
+      "rewards/rejected": -0.7164429426193237,
+      "step": 95
+    },
+    {
+      "epoch": 0.5961251862891207,
+      "grad_norm": 98.94036102294922,
+      "learning_rate": 5e-06,
+      "log_odds_chosen": 0.6077507138252258,
+      "log_odds_ratio": -0.5442172884941101,
+      "logits/chosen": 406.62945556640625,
+      "logits/rejected": 442.9092712402344,
+      "logps/chosen": -1.3020541667938232,
+      "logps/rejected": -1.8070911169052124,
+      "loss": 1.8887,
+      "nll_loss": 1.6196590662002563,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6510270833969116,
+      "rewards/margins": 0.2525184154510498,
+      "rewards/rejected": -0.9035455584526062,
+      "step": 100
+    },
+    {
+      "epoch": 0.6259314456035767,
+      "grad_norm": 64.72420501708984,
+      "learning_rate": 4.8795003647426654e-06,
+      "log_odds_chosen": 0.6622194051742554,
+      "log_odds_ratio": -0.5436784625053406,
+      "logits/chosen": 396.576416015625,
+      "logits/rejected": 409.37408447265625,
+      "logps/chosen": -1.239157795906067,
+      "logps/rejected": -1.7216756343841553,
+      "loss": 2.0086,
+      "nll_loss": 1.743311882019043,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6195788979530334,
+      "rewards/margins": 0.24125893414020538,
+      "rewards/rejected": -0.8608378171920776,
+      "step": 105
+    },
+    {
+      "epoch": 0.6557377049180327,
+      "grad_norm": 156.0756072998047,
+      "learning_rate": 4.767312946227961e-06,
+      "log_odds_chosen": 0.552756667137146,
+      "log_odds_ratio": -0.5492970943450928,
+      "logits/chosen": 376.5224304199219,
+      "logits/rejected": 374.7490234375,
+      "logps/chosen": -1.258983850479126,
+      "logps/rejected": -1.6897951364517212,
+      "loss": 1.8644,
+      "nll_loss": 1.6051260232925415,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.629491925239563,
+      "rewards/margins": 0.21540567278862,
+      "rewards/rejected": -0.8448975682258606,
+      "step": 110
+    },
+    {
+      "epoch": 0.6855439642324889,
+      "grad_norm": 27.666635513305664,
+      "learning_rate": 4.662524041201569e-06,
+      "log_odds_chosen": 0.7742821574211121,
+      "log_odds_ratio": -0.4874287545681,
+      "logits/chosen": 401.1915283203125,
+      "logits/rejected": 406.65716552734375,
+      "logps/chosen": -1.1674349308013916,
+      "logps/rejected": -1.7199904918670654,
+      "loss": 1.8546,
+      "nll_loss": 1.5520570278167725,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.5837174654006958,
+      "rewards/margins": 0.2762778401374817,
+      "rewards/rejected": -0.8599952459335327,
+      "step": 115
+    },
+    {
+      "epoch": 0.7153502235469449,
+      "grad_norm": 39.84739303588867,
+      "learning_rate": 4.564354645876385e-06,
+      "log_odds_chosen": 0.7211123108863831,
+      "log_odds_ratio": -0.5134823322296143,
+      "logits/chosen": 388.77838134765625,
+      "logits/rejected": 388.0323181152344,
+      "logps/chosen": -1.1216659545898438,
+      "logps/rejected": -1.614723801612854,
+      "loss": 1.8847,
+      "nll_loss": 1.5851175785064697,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5608329772949219,
+      "rewards/margins": 0.24652889370918274,
+      "rewards/rejected": -0.807361900806427,
+      "step": 120
+    },
+    {
+      "epoch": 0.7451564828614009,
+      "grad_norm": 29.12469482421875,
+      "learning_rate": 4.47213595499958e-06,
+      "log_odds_chosen": 0.3196253776550293,
+      "log_odds_ratio": -0.6582637429237366,
+      "logits/chosen": 386.79583740234375,
+      "logits/rejected": 398.7740783691406,
+      "logps/chosen": -1.351342797279358,
+      "logps/rejected": -1.5734379291534424,
+      "loss": 1.8179,
+      "nll_loss": 1.5471035242080688,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.675671398639679,
+      "rewards/margins": 0.11104752123355865,
+      "rewards/rejected": -0.7867189645767212,
+      "step": 125
+    },
+    {
+      "epoch": 0.7749627421758569,
+      "grad_norm": 38.485130310058594,
+      "learning_rate": 4.385290096535147e-06,
+      "log_odds_chosen": 0.6995328068733215,
+      "log_odds_ratio": -0.5530967712402344,
+      "logits/chosen": 409.95849609375,
+      "logits/rejected": 397.23114013671875,
+      "logps/chosen": -1.1655128002166748,
+      "logps/rejected": -1.6789804697036743,
+      "loss": 1.8506,
+      "nll_loss": 1.5426290035247803,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5827564001083374,
+      "rewards/margins": 0.2567337453365326,
+      "rewards/rejected": -0.8394902348518372,
+      "step": 130
+    },
+    {
+      "epoch": 0.8047690014903129,
+      "grad_norm": 52.03726577758789,
+      "learning_rate": 4.303314829119352e-06,
+      "log_odds_chosen": 0.42176657915115356,
+      "log_odds_ratio": -0.6203495264053345,
+      "logits/chosen": 415.891357421875,
+      "logits/rejected": 418.27984619140625,
+      "logps/chosen": -1.4386600255966187,
+      "logps/rejected": -1.7602298259735107,
+      "loss": 1.873,
+      "nll_loss": 1.672498106956482,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7193300127983093,
+      "rewards/margins": 0.16078472137451172,
+      "rewards/rejected": -0.8801149129867554,
+      "step": 135
+    },
+    {
+      "epoch": 0.834575260804769,
+      "grad_norm": 37.35191345214844,
+      "learning_rate": 4.2257712736425835e-06,
+      "log_odds_chosen": 0.30007433891296387,
+      "log_odds_ratio": -0.659325897693634,
+      "logits/chosen": 397.5603942871094,
+      "logits/rejected": 403.28192138671875,
+      "logps/chosen": -1.314620852470398,
+      "logps/rejected": -1.552886962890625,
+      "loss": 1.8804,
+      "nll_loss": 1.7158609628677368,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.657310426235199,
+      "rewards/margins": 0.11913303285837173,
+      "rewards/rejected": -0.7764434814453125,
+      "step": 140
+    },
+    {
+      "epoch": 0.8643815201192251,
+      "grad_norm": 40.69832992553711,
+      "learning_rate": 4.1522739926869985e-06,
+      "log_odds_chosen": 0.15929968655109406,
+      "log_odds_ratio": -0.6508444547653198,
+      "logits/chosen": 392.15386962890625,
+      "logits/rejected": 395.3309631347656,
+      "logps/chosen": -1.2985050678253174,
+      "logps/rejected": -1.4000908136367798,
+      "loss": 1.8468,
+      "nll_loss": 1.581002950668335,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.6492525339126587,
+      "rewards/margins": 0.0507928729057312,
+      "rewards/rejected": -0.7000454068183899,
+      "step": 145
+    },
+    {
+      "epoch": 0.8941877794336811,
+      "grad_norm": 33.805145263671875,
+      "learning_rate": 4.082482904638631e-06,
+      "log_odds_chosen": 0.7655292749404907,
+      "log_odds_ratio": -0.514534592628479,
+      "logits/chosen": 398.47711181640625,
+      "logits/rejected": 415.9359436035156,
+      "logps/chosen": -1.1850507259368896,
+      "logps/rejected": -1.7938915491104126,
+      "loss": 1.7734,
+      "nll_loss": 1.4344916343688965,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.5925253629684448,
+      "rewards/margins": 0.3044203519821167,
+      "rewards/rejected": -0.8969457745552063,
+      "step": 150
+    },
+    {
+      "epoch": 0.9239940387481371,
+      "grad_norm": 21.895387649536133,
+      "learning_rate": 4.016096644512495e-06,
+      "log_odds_chosen": 0.26020118594169617,
+      "log_odds_ratio": -0.6326924562454224,
+      "logits/chosen": 378.7313537597656,
+      "logits/rejected": 393.6383361816406,
+      "logps/chosen": -1.2521542310714722,
+      "logps/rejected": -1.4581738710403442,
+      "loss": 1.7575,
+      "nll_loss": 1.3882125616073608,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6260771155357361,
+      "rewards/margins": 0.10300980508327484,
+      "rewards/rejected": -0.7290869355201721,
+      "step": 155
+    },
+    {
+      "epoch": 0.9538002980625931,
+      "grad_norm": 65.63549041748047,
+      "learning_rate": 3.952847075210474e-06,
+      "log_odds_chosen": 0.28196483850479126,
+      "log_odds_ratio": -0.6715171337127686,
+      "logits/chosen": 386.80755615234375,
+      "logits/rejected": 431.31146240234375,
+      "logps/chosen": -1.1752804517745972,
+      "logps/rejected": -1.395171880722046,
+      "loss": 1.7333,
+      "nll_loss": 1.422628402709961,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.5876402258872986,
+      "rewards/margins": 0.1099456176161766,
+      "rewards/rejected": -0.697585940361023,
+      "step": 160
+    },
+    {
+      "epoch": 0.9836065573770492,
+      "grad_norm": 31.469694137573242,
+      "learning_rate": 3.892494720807615e-06,
+      "log_odds_chosen": 0.20551732182502747,
+      "log_odds_ratio": -0.6562041640281677,
+      "logits/chosen": 399.196533203125,
+      "logits/rejected": 412.1044006347656,
+      "logps/chosen": -1.233426809310913,
+      "logps/rejected": -1.369274377822876,
+      "loss": 1.8238,
+      "nll_loss": 1.516705870628357,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.6167134046554565,
+      "rewards/margins": 0.06792382895946503,
+      "rewards/rejected": -0.684637188911438,
+      "step": 165
+    },
+    {
+      "epoch": 0.9955290611028316,
+      "eval_log_odds_chosen": 0.2615682780742645,
+      "eval_log_odds_ratio": -0.6631397604942322,
+      "eval_logits/chosen": 322.6454162597656,
+      "eval_logits/rejected": 293.2744445800781,
+      "eval_logps/chosen": -1.0755056142807007,
+      "eval_logps/rejected": -1.263703465461731,
+      "eval_loss": 1.8175820112228394,
+      "eval_nll_loss": 1.4782638549804688,
+      "eval_rewards/accuracies": 0.5467625856399536,
+      "eval_rewards/chosen": -0.5377528071403503,
+      "eval_rewards/margins": 0.09409892559051514,
+      "eval_rewards/rejected": -0.6318517327308655,
+      "eval_runtime": 351.5652,
+      "eval_samples_per_second": 1.573,
+      "eval_steps_per_second": 0.395,
+      "step": 167
+    },
+    {
+      "epoch": 1.0134128166915053,
+      "grad_norm": 19.610740661621094,
+      "learning_rate": 3.834824944236852e-06,
+      "log_odds_chosen": 0.7168810963630676,
+      "log_odds_ratio": -0.48998212814331055,
+      "logits/chosen": 379.9788513183594,
+      "logits/rejected": 404.4870910644531,
+      "logps/chosen": -1.0245921611785889,
+      "logps/rejected": -1.525979995727539,
+      "loss": 1.6186,
+      "nll_loss": 1.2430717945098877,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.5122960805892944,
+      "rewards/margins": 0.2506938874721527,
+      "rewards/rejected": -0.7629899978637695,
+      "step": 170
+    },
+    {
+      "epoch": 1.0432190760059612,
+      "grad_norm": 27.745262145996094,
+      "learning_rate": 3.7796447300922724e-06,
+      "log_odds_chosen": 1.422102689743042,
+      "log_odds_ratio": -0.3321012854576111,
+      "logits/chosen": 357.04718017578125,
+      "logits/rejected": 398.559814453125,
+      "logps/chosen": -0.7521129846572876,
+      "logps/rejected": -1.7101812362670898,
+      "loss": 1.286,
+      "nll_loss": 1.213651418685913,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.3760564923286438,
+      "rewards/margins": 0.47903409600257874,
+      "rewards/rejected": -0.8550906181335449,
+      "step": 175
+    },
+    {
+      "epoch": 1.0730253353204173,
+      "grad_norm": 20.82083511352539,
+      "learning_rate": 3.72677996249965e-06,
+      "log_odds_chosen": 1.3344449996948242,
+      "log_odds_ratio": -0.3116641640663147,
+      "logits/chosen": 356.35601806640625,
+      "logits/rejected": 331.2690124511719,
+      "logps/chosen": -0.8246381878852844,
+      "logps/rejected": -1.7447946071624756,
+      "loss": 1.3455,
+      "nll_loss": 1.2647905349731445,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.4123190939426422,
+      "rewards/margins": 0.46007823944091797,
+      "rewards/rejected": -0.8723973035812378,
+      "step": 180
+    },
+    {
+      "epoch": 1.1028315946348732,
+      "grad_norm": 33.03415298461914,
+      "learning_rate": 3.6760731104690393e-06,
+      "log_odds_chosen": 1.7977418899536133,
+      "log_odds_ratio": -0.29093313217163086,
+      "logits/chosen": 388.28021240234375,
+      "logits/rejected": 380.4255065917969,
+      "logps/chosen": -0.8684544563293457,
+      "logps/rejected": -2.099797487258911,
+      "loss": 1.2411,
+      "nll_loss": 1.0944446325302124,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.43422722816467285,
+      "rewards/margins": 0.6156715750694275,
+      "rewards/rejected": -1.0498987436294556,
+      "step": 185
+    },
+    {
+      "epoch": 1.1326378539493294,
+      "grad_norm": 48.01770782470703,
+      "learning_rate": 3.6273812505500587e-06,
+      "log_odds_chosen": 1.2517069578170776,
+      "log_odds_ratio": -0.3731076717376709,
+      "logits/chosen": 356.5531005859375,
+      "logits/rejected": 402.0587463378906,
+      "logps/chosen": -0.8801576495170593,
+      "logps/rejected": -1.7424513101577759,
+      "loss": 1.326,
+      "nll_loss": 1.140998125076294,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.44007882475852966,
+      "rewards/margins": 0.4311468005180359,
+      "rewards/rejected": -0.8712256550788879,
+      "step": 190
+    },
+    {
+      "epoch": 1.1624441132637853,
+      "grad_norm": 25.167158126831055,
+      "learning_rate": 3.5805743701971648e-06,
+      "log_odds_chosen": 1.9819135665893555,
+      "log_odds_ratio": -0.29898396134376526,
+      "logits/chosen": 382.34100341796875,
+      "logits/rejected": 395.79425048828125,
+      "logps/chosen": -0.9413985013961792,
+      "logps/rejected": -2.448883533477783,
+      "loss": 1.3195,
+      "nll_loss": 1.2245818376541138,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.4706992506980896,
+      "rewards/margins": 0.7537423968315125,
+      "rewards/rejected": -1.2244417667388916,
+      "step": 195
+    },
+    {
+      "epoch": 1.1922503725782414,
+      "grad_norm": 29.544389724731445,
+      "learning_rate": 3.5355339059327378e-06,
+      "log_odds_chosen": 2.2445785999298096,
+      "log_odds_ratio": -0.27407315373420715,
+      "logits/chosen": 399.3606262207031,
+      "logits/rejected": 381.97125244140625,
+      "logps/chosen": -0.7390884757041931,
+      "logps/rejected": -2.445159435272217,
+      "loss": 1.3027,
+      "nll_loss": 1.0836920738220215,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.36954423785209656,
+      "rewards/margins": 0.853035569190979,
+      "rewards/rejected": -1.2225797176361084,
+      "step": 200
+    },
+    {
+      "epoch": 1.2220566318926975,
+      "grad_norm": 33.41192626953125,
+      "learning_rate": 3.4921514788478916e-06,
+      "log_odds_chosen": 2.2366576194763184,
+      "log_odds_ratio": -0.2573709785938263,
+      "logits/chosen": 359.3567810058594,
+      "logits/rejected": 353.2535400390625,
+      "logps/chosen": -0.7408849596977234,
+      "logps/rejected": -2.3824286460876465,
+      "loss": 1.2558,
+      "nll_loss": 1.1112185716629028,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.3704424798488617,
+      "rewards/margins": 0.8207718729972839,
+      "rewards/rejected": -1.1912143230438232,
+      "step": 205
+    },
+    {
+      "epoch": 1.2518628912071534,
+      "grad_norm": 18.833148956298828,
+      "learning_rate": 3.450327796711771e-06,
+      "log_odds_chosen": 2.773773431777954,
+      "log_odds_ratio": -0.24581894278526306,
+      "logits/chosen": 363.11090087890625,
+      "logits/rejected": 392.077880859375,
+      "logps/chosen": -0.7906020879745483,
+      "logps/rejected": -2.9855308532714844,
+      "loss": 1.2167,
+      "nll_loss": 1.0773674249649048,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.39530104398727417,
+      "rewards/margins": 1.0974645614624023,
+      "rewards/rejected": -1.4927654266357422,
+      "step": 210
+    },
+    {
+      "epoch": 1.2816691505216096,
+      "grad_norm": 32.97148895263672,
+      "learning_rate": 3.409971697352368e-06,
+      "log_odds_chosen": 2.409407138824463,
+      "log_odds_ratio": -0.2513101100921631,
+      "logits/chosen": 386.68804931640625,
+      "logits/rejected": 369.2494812011719,
+      "logps/chosen": -0.7668555378913879,
+      "logps/rejected": -2.570010185241699,
+      "loss": 1.232,
+      "nll_loss": 1.1109261512756348,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.38342776894569397,
+      "rewards/margins": 0.901577353477478,
+      "rewards/rejected": -1.2850050926208496,
+      "step": 215
+    },
+    {
+      "epoch": 1.3114754098360657,
+      "grad_norm": 27.4957218170166,
+      "learning_rate": 3.3709993123162106e-06,
+      "log_odds_chosen": 1.301114797592163,
+      "log_odds_ratio": -0.34698307514190674,
+      "logits/chosen": 377.33697509765625,
+      "logits/rejected": 371.03192138671875,
+      "logps/chosen": -0.8284207582473755,
+      "logps/rejected": -1.6063474416732788,
+      "loss": 1.2242,
+      "nll_loss": 1.0992552042007446,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.41421037912368774,
+      "rewards/margins": 0.3889634311199188,
+      "rewards/rejected": -0.8031737208366394,
+      "step": 220
+    },
+    {
+      "epoch": 1.3412816691505216,
+      "grad_norm": 27.05774688720703,
+      "learning_rate": 3.3333333333333333e-06,
+      "log_odds_chosen": 1.0342134237289429,
+      "log_odds_ratio": -0.3879423439502716,
+      "logits/chosen": 373.6796875,
+      "logits/rejected": 366.01123046875,
+      "logps/chosen": -0.9923228025436401,
+      "logps/rejected": -1.7060105800628662,
+      "loss": 1.2664,
+      "nll_loss": 1.197096586227417,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.49616140127182007,
+      "rewards/margins": 0.3568439185619354,
+      "rewards/rejected": -0.8530052900314331,
+      "step": 225
+    },
+    {
+      "epoch": 1.3710879284649775,
+      "grad_norm": 32.180084228515625,
+      "learning_rate": 3.296902366978936e-06,
+      "log_odds_chosen": 1.9371551275253296,
+      "log_odds_ratio": -0.265031635761261,
+      "logits/chosen": 354.0810546875,
+      "logits/rejected": 372.2239990234375,
+      "logps/chosen": -0.8565861582756042,
+      "logps/rejected": -2.277017116546631,
+      "loss": 1.2422,
+      "nll_loss": 0.9957939982414246,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.4282930791378021,
+      "rewards/margins": 0.7102155685424805,
+      "rewards/rejected": -1.1385085582733154,
+      "step": 230
+    },
+    {
+      "epoch": 1.4008941877794336,
+      "grad_norm": 20.605178833007812,
+      "learning_rate": 3.2616403652672114e-06,
+      "log_odds_chosen": 1.8511009216308594,
+      "log_odds_ratio": -0.2493407279253006,
+      "logits/chosen": 376.54779052734375,
+      "logits/rejected": 390.1648254394531,
+      "logps/chosen": -0.7664806246757507,
+      "logps/rejected": -2.081376075744629,
+      "loss": 1.1997,
+      "nll_loss": 1.0387040376663208,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.38324031233787537,
+      "rewards/margins": 0.6574475765228271,
+      "rewards/rejected": -1.0406880378723145,
+      "step": 235
+    },
+    {
+      "epoch": 1.4307004470938898,
+      "grad_norm": 24.71708106994629,
+      "learning_rate": 3.2274861218395142e-06,
+      "log_odds_chosen": 1.3407671451568604,
+      "log_odds_ratio": -0.33692649006843567,
+      "logits/chosen": 401.45428466796875,
+      "logits/rejected": 407.09161376953125,
+      "logps/chosen": -0.9054458737373352,
+      "logps/rejected": -1.8345670700073242,
+      "loss": 1.3273,
+      "nll_loss": 1.143675446510315,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.4527229368686676,
+      "rewards/margins": 0.46456044912338257,
+      "rewards/rejected": -0.9172835350036621,
+      "step": 240
+    },
+    {
+      "epoch": 1.4605067064083457,
+      "grad_norm": 37.8891487121582,
+      "learning_rate": 3.1943828249997e-06,
+      "log_odds_chosen": 1.8552640676498413,
+      "log_odds_ratio": -0.272559255361557,
+      "logits/chosen": 400.0021667480469,
+      "logits/rejected": 387.4122009277344,
+      "logps/chosen": -0.7997323274612427,
+      "logps/rejected": -2.1247220039367676,
+      "loss": 1.3455,
+      "nll_loss": 1.303070306777954,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.39986616373062134,
+      "rewards/margins": 0.6624947786331177,
+      "rewards/rejected": -1.0623610019683838,
+      "step": 245
+    },
+    {
+      "epoch": 1.4903129657228018,
+      "grad_norm": 35.78718948364258,
+      "learning_rate": 3.1622776601683796e-06,
+      "log_odds_chosen": 1.5998339653015137,
+      "log_odds_ratio": -0.3105994760990143,
+      "logits/chosen": 369.63177490234375,
+      "logits/rejected": 376.02960205078125,
+      "logps/chosen": -0.7897557020187378,
+      "logps/rejected": -1.8593695163726807,
+      "loss": 1.2365,
+      "nll_loss": 1.0359935760498047,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3948778510093689,
+      "rewards/margins": 0.5348068475723267,
+      "rewards/rejected": -0.9296847581863403,
+      "step": 250
+    },
+    {
+      "epoch": 1.520119225037258,
+      "grad_norm": 25.08989143371582,
+      "learning_rate": 3.131121455425748e-06,
+      "log_odds_chosen": 1.8597294092178345,
+      "log_odds_ratio": -0.21304909884929657,
+      "logits/chosen": 384.16290283203125,
+      "logits/rejected": 387.0345153808594,
+      "logps/chosen": -0.7654696702957153,
+      "logps/rejected": -2.019379138946533,
+      "loss": 1.2499,
+      "nll_loss": 1.030097246170044,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.38273483514785767,
+      "rewards/margins": 0.6269546747207642,
+      "rewards/rejected": -1.0096895694732666,
+      "step": 255
+    },
+    {
+      "epoch": 1.5499254843517138,
+      "grad_norm": 21.55389976501465,
+      "learning_rate": 3.1008683647302113e-06,
+      "log_odds_chosen": 1.4927784204483032,
+      "log_odds_ratio": -0.29883819818496704,
+      "logits/chosen": 362.99444580078125,
+      "logits/rejected": 404.9141845703125,
+      "logps/chosen": -0.8609813451766968,
+      "logps/rejected": -1.9159471988677979,
+      "loss": 1.2228,
+      "nll_loss": 1.09743070602417,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.4304906725883484,
+      "rewards/margins": 0.5274828672409058,
+      "rewards/rejected": -0.9579735994338989,
+      "step": 260
+    },
+    {
+      "epoch": 1.5797317436661698,
+      "grad_norm": 19.87468910217285,
+      "learning_rate": 3.0714755841697565e-06,
+      "log_odds_chosen": 1.570449709892273,
+      "log_odds_ratio": -0.3343026041984558,
+      "logits/chosen": 374.11920166015625,
+      "logits/rejected": 396.1978454589844,
+      "logps/chosen": -0.8117960691452026,
+      "logps/rejected": -1.8302360773086548,
+      "loss": 1.3153,
+      "nll_loss": 1.1015130281448364,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4058980345726013,
+      "rewards/margins": 0.5092200040817261,
+      "rewards/rejected": -0.9151180386543274,
+      "step": 265
+    },
+    {
+      "epoch": 1.6095380029806259,
+      "grad_norm": 20.458053588867188,
+      "learning_rate": 3.0429030972509227e-06,
+      "log_odds_chosen": 1.468200922012329,
+      "log_odds_ratio": -0.3185100853443146,
+      "logits/chosen": 360.3885192871094,
+      "logits/rejected": 371.0684509277344,
+      "logps/chosen": -0.948271632194519,
+      "logps/rejected": -1.9939063787460327,
+      "loss": 1.2987,
+      "nll_loss": 1.2175120115280151,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.4741358160972595,
+      "rewards/margins": 0.5228174328804016,
+      "rewards/rejected": -0.9969531893730164,
+      "step": 270
+    },
+    {
+      "epoch": 1.639344262295082,
+      "grad_norm": 21.885934829711914,
+      "learning_rate": 3.0151134457776365e-06,
+      "log_odds_chosen": 1.2957884073257446,
+      "log_odds_ratio": -0.34294062852859497,
+      "logits/chosen": 351.11566162109375,
+      "logits/rejected": 341.6363830566406,
+      "logps/chosen": -0.816031277179718,
+      "logps/rejected": -1.6235980987548828,
+      "loss": 1.3141,
+      "nll_loss": 1.0874848365783691,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.408015638589859,
+      "rewards/margins": 0.40378338098526,
+      "rewards/rejected": -0.8117990493774414,
+      "step": 275
+    },
+    {
+      "epoch": 1.669150521609538,
+      "grad_norm": 32.71710205078125,
+      "learning_rate": 2.988071523335984e-06,
+      "log_odds_chosen": 1.3881560564041138,
+      "log_odds_ratio": -0.44494813680648804,
+      "logits/chosen": 399.6920471191406,
+      "logits/rejected": 387.6843566894531,
+      "logps/chosen": -0.9040805697441101,
+      "logps/rejected": -1.8093194961547852,
+      "loss": 1.3021,
+      "nll_loss": 1.1397716999053955,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.45204028487205505,
+      "rewards/margins": 0.4526194930076599,
+      "rewards/rejected": -0.9046597480773926,
+      "step": 280
+    },
+    {
+      "epoch": 1.698956780923994,
+      "grad_norm": 31.200809478759766,
+      "learning_rate": 2.961744388795462e-06,
+      "log_odds_chosen": 1.7313816547393799,
+      "log_odds_ratio": -0.28469711542129517,
+      "logits/chosen": 359.5746154785156,
+      "logits/rejected": 362.81494140625,
+      "logps/chosen": -0.6776639223098755,
+      "logps/rejected": -1.7751470804214478,
+      "loss": 1.2089,
+      "nll_loss": 1.0214917659759521,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.33883196115493774,
+      "rewards/margins": 0.5487415790557861,
+      "rewards/rejected": -0.8875735402107239,
+      "step": 285
+    },
+    {
+      "epoch": 1.7287630402384502,
+      "grad_norm": 17.744766235351562,
+      "learning_rate": 2.9361010975735177e-06,
+      "log_odds_chosen": 1.5064774751663208,
+      "log_odds_ratio": -0.3122694492340088,
+      "logits/chosen": 376.41033935546875,
+      "logits/rejected": 415.8057556152344,
+      "logps/chosen": -0.8741256594657898,
+      "logps/rejected": -1.9100143909454346,
+      "loss": 1.259,
+      "nll_loss": 1.0986948013305664,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.4370628297328949,
+      "rewards/margins": 0.5179442167282104,
+      "rewards/rejected": -0.9550071954727173,
+      "step": 290
+    },
+    {
+      "epoch": 1.758569299552906,
+      "grad_norm": 22.985361099243164,
+      "learning_rate": 2.9111125486979104e-06,
+      "log_odds_chosen": 1.4937174320220947,
+      "log_odds_ratio": -0.3182430565357208,
+      "logits/chosen": 353.3531188964844,
+      "logits/rejected": 398.23382568359375,
+      "logps/chosen": -0.8985480070114136,
+      "logps/rejected": -1.8789575099945068,
+      "loss": 1.2814,
+      "nll_loss": 1.1132934093475342,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4492740035057068,
+      "rewards/margins": 0.4902048110961914,
+      "rewards/rejected": -0.9394787549972534,
+      "step": 295
+    },
+    {
+      "epoch": 1.788375558867362,
+      "grad_norm": 25.093547821044922,
+      "learning_rate": 2.8867513459481293e-06,
+      "log_odds_chosen": 1.7519241571426392,
+      "log_odds_ratio": -0.23565292358398438,
+      "logits/chosen": 396.87640380859375,
+      "logits/rejected": 374.09942626953125,
+      "logps/chosen": -0.6939581632614136,
+      "logps/rejected": -1.8357816934585571,
+      "loss": 1.201,
+      "nll_loss": 0.9846609234809875,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.3469790816307068,
+      "rewards/margins": 0.5709116458892822,
+      "rewards/rejected": -0.9178908467292786,
+      "step": 300
+    },
+    {
+      "epoch": 1.8181818181818183,
+      "grad_norm": 22.51293182373047,
+      "learning_rate": 2.862991671569341e-06,
+      "log_odds_chosen": 1.3752100467681885,
+      "log_odds_ratio": -0.39724329113960266,
+      "logits/chosen": 392.15093994140625,
+      "logits/rejected": 400.69677734375,
+      "logps/chosen": -0.9685919880867004,
+      "logps/rejected": -2.020549774169922,
+      "loss": 1.2557,
+      "nll_loss": 1.2231824398040771,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4842959940433502,
+      "rewards/margins": 0.5259788036346436,
+      "rewards/rejected": -1.010274887084961,
+      "step": 305
+    },
+    {
+      "epoch": 1.8479880774962743,
+      "grad_norm": 18.64189910888672,
+      "learning_rate": 2.839809171235324e-06,
+      "log_odds_chosen": 1.9540401697158813,
+      "log_odds_ratio": -0.28014156222343445,
+      "logits/chosen": 374.20074462890625,
+      "logits/rejected": 382.771240234375,
+      "logps/chosen": -0.823870837688446,
+      "logps/rejected": -2.250458002090454,
+      "loss": 1.3098,
+      "nll_loss": 1.1509441137313843,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.411935418844223,
+      "rewards/margins": 0.7132935523986816,
+      "rewards/rejected": -1.125229001045227,
+      "step": 310
+    },
+    {
+      "epoch": 1.8777943368107302,
+      "grad_norm": 20.656198501586914,
+      "learning_rate": 2.817180849095055e-06,
+      "log_odds_chosen": 0.9874370694160461,
+      "log_odds_ratio": -0.43443575501441956,
+      "logits/chosen": 348.67279052734375,
+      "logits/rejected": 367.7172546386719,
+      "logps/chosen": -1.063990831375122,
+      "logps/rejected": -1.7773240804672241,
+      "loss": 1.3159,
+      "nll_loss": 1.2891852855682373,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.531995415687561,
+      "rewards/margins": 0.3566665053367615,
+      "rewards/rejected": -0.8886620402336121,
+      "step": 315
+    },
+    {
+      "epoch": 1.9076005961251863,
+      "grad_norm": 22.823558807373047,
+      "learning_rate": 2.7950849718747376e-06,
+      "log_odds_chosen": 1.9457191228866577,
+      "log_odds_ratio": -0.23255281150341034,
+      "logits/chosen": 369.242431640625,
+      "logits/rejected": 389.74346923828125,
+      "logps/chosen": -0.7030061483383179,
+      "logps/rejected": -2.0039026737213135,
+      "loss": 1.1894,
+      "nll_loss": 1.0204839706420898,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.35150307416915894,
+      "rewards/margins": 0.650448203086853,
+      "rewards/rejected": -1.0019513368606567,
+      "step": 320
+    },
+    {
+      "epoch": 1.9374068554396424,
+      "grad_norm": 27.562149047851562,
+      "learning_rate": 2.773500981126146e-06,
+      "log_odds_chosen": 1.998150110244751,
+      "log_odds_ratio": -0.24417133629322052,
+      "logits/chosen": 365.95684814453125,
+      "logits/rejected": 397.9959411621094,
+      "logps/chosen": -0.7909399271011353,
+      "logps/rejected": -2.2285313606262207,
+      "loss": 1.179,
+      "nll_loss": 1.0059034824371338,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.3954699635505676,
+      "rewards/margins": 0.7187958359718323,
+      "rewards/rejected": -1.1142656803131104,
+      "step": 325
+    },
+    {
+      "epoch": 1.9672131147540983,
+      "grad_norm": 27.540767669677734,
+      "learning_rate": 2.752409412815902e-06,
+      "log_odds_chosen": 1.38077712059021,
+      "log_odds_ratio": -0.3091193735599518,
+      "logits/chosen": 361.24969482421875,
+      "logits/rejected": 369.6480407714844,
+      "logps/chosen": -0.8252070546150208,
+      "logps/rejected": -1.7504304647445679,
+      "loss": 1.2534,
+      "nll_loss": 0.9596601724624634,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.4126035273075104,
+      "rewards/margins": 0.46261176466941833,
+      "rewards/rejected": -0.8752152323722839,
+      "step": 330
+    },
+    {
+      "epoch": 1.9970193740685542,
+      "grad_norm": 35.79298782348633,
+      "learning_rate": 2.7317918235407652e-06,
+      "log_odds_chosen": 1.0996582508087158,
+      "log_odds_ratio": -0.4153285622596741,
+      "logits/chosen": 389.44366455078125,
+      "logits/rejected": 379.0640563964844,
+      "logps/chosen": -1.0918314456939697,
+      "logps/rejected": -1.8735624551773071,
+      "loss": 1.3092,
+      "nll_loss": 1.2989994287490845,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.5459157228469849,
+      "rewards/margins": 0.3908655047416687,
+      "rewards/rejected": -0.9367812275886536,
+      "step": 335
+    },
+    {
+      "epoch": 1.9970193740685542,
+      "eval_log_odds_chosen": 0.32236072421073914,
+      "eval_log_odds_ratio": -0.6637296676635742,
+      "eval_logits/chosen": 309.3900146484375,
+      "eval_logits/rejected": 279.0658874511719,
+      "eval_logps/chosen": -1.040497064590454,
+      "eval_logps/rejected": -1.2617474794387817,
+      "eval_loss": 1.7559822797775269,
+      "eval_nll_loss": 1.4054137468338013,
+      "eval_rewards/accuracies": 0.5323740839958191,
+      "eval_rewards/chosen": -0.520248532295227,
+      "eval_rewards/margins": 0.11062520742416382,
+      "eval_rewards/rejected": -0.6308737397193909,
+      "eval_runtime": 351.6864,
+      "eval_samples_per_second": 1.572,
+      "eval_steps_per_second": 0.395,
+      "step": 335
+    },
+    {
+      "epoch": 2.0268256333830106,
+      "grad_norm": 22.68450927734375,
+      "learning_rate": 2.711630722733202e-06,
+      "log_odds_chosen": 3.2875800132751465,
+      "log_odds_ratio": -0.10148422420024872,
+      "logits/chosen": 390.20111083984375,
+      "logits/rejected": 362.7218017578125,
+      "logps/chosen": -0.4825616478919983,
+      "logps/rejected": -2.6374728679656982,
+      "loss": 0.7684,
+      "nll_loss": 0.8495180010795593,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.24128082394599915,
+      "rewards/margins": 1.0774555206298828,
+      "rewards/rejected": -1.3187364339828491,
+      "step": 340
+    },
+    {
+      "epoch": 2.0566318926974665,
+      "grad_norm": 12.678057670593262,
+      "learning_rate": 2.691909510290828e-06,
+      "log_odds_chosen": 3.9607760906219482,
+      "log_odds_ratio": -0.06652864068746567,
+      "logits/chosen": 340.1753234863281,
+      "logits/rejected": 337.1167907714844,
+      "logps/chosen": -0.44624167680740356,
+      "logps/rejected": -3.1436822414398193,
+      "loss": 0.6642,
+      "nll_loss": 0.6589840650558472,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.22312083840370178,
+      "rewards/margins": 1.3487205505371094,
+      "rewards/rejected": -1.5718411207199097,
+      "step": 345
+    },
+    {
+      "epoch": 2.0864381520119224,
+      "grad_norm": 18.899171829223633,
+      "learning_rate": 2.6726124191242444e-06,
+      "log_odds_chosen": 3.980642318725586,
+      "log_odds_ratio": -0.04676336795091629,
+      "logits/chosen": 340.1651916503906,
+      "logits/rejected": 373.43426513671875,
+      "logps/chosen": -0.43081822991371155,
+      "logps/rejected": -3.3248093128204346,
+      "loss": 0.6728,
+      "nll_loss": 0.6396235227584839,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.21540911495685577,
+      "rewards/margins": 1.446995496749878,
+      "rewards/rejected": -1.6624046564102173,
+      "step": 350
+    },
+    {
+      "epoch": 2.1162444113263787,
+      "grad_norm": 15.860565185546875,
+      "learning_rate": 2.6537244621713765e-06,
+      "log_odds_chosen": 3.6839728355407715,
+      "log_odds_ratio": -0.07649212330579758,
+      "logits/chosen": 345.269287109375,
+      "logits/rejected": 359.37823486328125,
+      "logps/chosen": -0.45684829354286194,
+      "logps/rejected": -3.046020984649658,
+      "loss": 0.666,
+      "nll_loss": 0.7428454160690308,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.22842414677143097,
+      "rewards/margins": 1.294586420059204,
+      "rewards/rejected": -1.523010492324829,
+      "step": 355
+    },
+    {
+      "epoch": 2.1460506706408347,
+      "grad_norm": 13.459833145141602,
+      "learning_rate": 2.6352313834736496e-06,
+      "log_odds_chosen": 4.002976417541504,
+      "log_odds_ratio": -0.04872741550207138,
+      "logits/chosen": 340.22161865234375,
+      "logits/rejected": 384.5272521972656,
+      "logps/chosen": -0.433218777179718,
+      "logps/rejected": -3.1645781993865967,
+      "loss": 0.6602,
+      "nll_loss": 0.6463421583175659,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.216609388589859,
+      "rewards/margins": 1.3656798601150513,
+      "rewards/rejected": -1.5822890996932983,
+      "step": 360
+    },
+    {
+      "epoch": 2.1758569299552906,
+      "grad_norm": 14.115008354187012,
+      "learning_rate": 2.6171196129510684e-06,
+      "log_odds_chosen": 3.177072763442993,
+      "log_odds_ratio": -0.0845123901963234,
+      "logits/chosen": 326.1507568359375,
+      "logits/rejected": 306.5514831542969,
+      "logps/chosen": -0.41812819242477417,
+      "logps/rejected": -2.4115400314331055,
+      "loss": 0.6679,
+      "nll_loss": 0.6390963196754456,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.20906409621238708,
+      "rewards/margins": 0.9967058300971985,
+      "rewards/rejected": -1.2057700157165527,
+      "step": 365
+    },
+    {
+      "epoch": 2.2056631892697465,
+      "grad_norm": 15.614830017089844,
+      "learning_rate": 2.599376224550182e-06,
+      "log_odds_chosen": 3.3914806842803955,
+      "log_odds_ratio": -0.08955641835927963,
+      "logits/chosen": 287.53955078125,
+      "logits/rejected": 306.90667724609375,
+      "logps/chosen": -0.44138556718826294,
+      "logps/rejected": -2.749290943145752,
+      "loss": 0.6969,
+      "nll_loss": 0.7107062935829163,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.22069278359413147,
+      "rewards/margins": 1.153952717781067,
+      "rewards/rejected": -1.374645471572876,
+      "step": 370
+    },
+    {
+      "epoch": 2.235469448584203,
+      "grad_norm": 15.579936027526855,
+      "learning_rate": 2.5819888974716113e-06,
+      "log_odds_chosen": 3.3506603240966797,
+      "log_odds_ratio": -0.09304364025592804,
+      "logits/chosen": 339.9989929199219,
+      "logits/rejected": 357.97528076171875,
+      "logps/chosen": -0.5137821435928345,
+      "logps/rejected": -2.7970046997070312,
+      "loss": 0.6856,
+      "nll_loss": 0.7090636491775513,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.25689107179641724,
+      "rewards/margins": 1.1416113376617432,
+      "rewards/rejected": -1.3985023498535156,
+      "step": 375
+    },
+    {
+      "epoch": 2.2652757078986587,
+      "grad_norm": 17.770183563232422,
+      "learning_rate": 2.564945880212886e-06,
+      "log_odds_chosen": 3.6279399394989014,
+      "log_odds_ratio": -0.04611171409487724,
+      "logits/chosen": 343.6501159667969,
+      "logits/rejected": 322.5646057128906,
+      "logps/chosen": -0.35303473472595215,
+      "logps/rejected": -2.671903610229492,
+      "loss": 0.6689,
+      "nll_loss": 0.6217206120491028,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.17651736736297607,
+      "rewards/margins": 1.1594343185424805,
+      "rewards/rejected": -1.335951805114746,
+      "step": 380
+    },
+    {
+      "epoch": 2.2950819672131146,
+      "grad_norm": 12.41324520111084,
+      "learning_rate": 2.5482359571881276e-06,
+      "log_odds_chosen": 3.9699618816375732,
+      "log_odds_ratio": -0.04786146804690361,
+      "logits/chosen": 341.5869445800781,
+      "logits/rejected": 333.04608154296875,
+      "logps/chosen": -0.3405749499797821,
+      "logps/rejected": -2.8487496376037598,
+      "loss": 0.6364,
+      "nll_loss": 0.5744936466217041,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.17028747498989105,
+      "rewards/margins": 1.254087209701538,
+      "rewards/rejected": -1.4243748188018799,
+      "step": 385
+    },
+    {
+      "epoch": 2.3248882265275705,
+      "grad_norm": 14.415823936462402,
+      "learning_rate": 2.5318484177091667e-06,
+      "log_odds_chosen": 3.405970335006714,
+      "log_odds_ratio": -0.05260590463876724,
+      "logits/chosen": 360.901611328125,
+      "logits/rejected": 378.63616943359375,
+      "logps/chosen": -0.5018097162246704,
+      "logps/rejected": -2.8674509525299072,
+      "loss": 0.7074,
+      "nll_loss": 0.6968181729316711,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2509048581123352,
+      "rewards/margins": 1.1828205585479736,
+      "rewards/rejected": -1.4337254762649536,
+      "step": 390
+    },
+    {
+      "epoch": 2.354694485842027,
+      "grad_norm": 11.383499145507812,
+      "learning_rate": 2.515773027133138e-06,
+      "log_odds_chosen": 3.9227728843688965,
+      "log_odds_ratio": -0.05491284653544426,
+      "logits/chosen": 349.3078308105469,
+      "logits/rejected": 332.0811462402344,
+      "logps/chosen": -0.32619425654411316,
+      "logps/rejected": -2.7826080322265625,
+      "loss": 0.6264,
+      "nll_loss": 0.5774807929992676,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.16309712827205658,
+      "rewards/margins": 1.2282068729400635,
+      "rewards/rejected": -1.3913040161132812,
+      "step": 395
+    },
+    {
+      "epoch": 2.384500745156483,
+      "grad_norm": 15.452503204345703,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 3.7418270111083984,
+      "log_odds_ratio": -0.08289723098278046,
+      "logits/chosen": 353.244873046875,
+      "logits/rejected": 371.33941650390625,
+      "logps/chosen": -0.4247261583805084,
+      "logps/rejected": -3.0048041343688965,
+      "loss": 0.6678,
+      "nll_loss": 0.5866397619247437,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.2123630791902542,
+      "rewards/margins": 1.2900387048721313,
+      "rewards/rejected": -1.5024020671844482,
+      "step": 400
+    },
+    {
+      "epoch": 2.4143070044709387,
+      "grad_norm": 14.744851112365723,
+      "learning_rate": 2.484519974999767e-06,
+      "log_odds_chosen": 4.066527366638184,
+      "log_odds_ratio": -0.07095837593078613,
+      "logits/chosen": 405.4268493652344,
+      "logits/rejected": 363.4831237792969,
+      "logps/chosen": -0.4019082188606262,
+      "logps/rejected": -3.0571980476379395,
+      "loss": 0.6775,
+      "nll_loss": 0.6186262965202332,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.2009541094303131,
+      "rewards/margins": 1.3276448249816895,
+      "rewards/rejected": -1.5285990238189697,
+      "step": 405
+    },
+    {
+      "epoch": 2.444113263785395,
+      "grad_norm": 15.641300201416016,
+      "learning_rate": 2.4693239916239746e-06,
+      "log_odds_chosen": 3.6895904541015625,
+      "log_odds_ratio": -0.08013327419757843,
+      "logits/chosen": 352.7126770019531,
+      "logits/rejected": 362.0486145019531,
+      "logps/chosen": -0.4512965679168701,
+      "logps/rejected": -2.8881852626800537,
+      "loss": 0.6612,
+      "nll_loss": 0.6566362380981445,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.22564828395843506,
+      "rewards/margins": 1.2184442281723022,
+      "rewards/rejected": -1.4440926313400269,
+      "step": 410
+    },
+    {
+      "epoch": 2.473919523099851,
+      "grad_norm": 13.112720489501953,
+      "learning_rate": 2.4544034683690802e-06,
+      "log_odds_chosen": 4.032966613769531,
+      "log_odds_ratio": -0.03849425166845322,
+      "logits/chosen": 352.24176025390625,
+      "logits/rejected": 376.4973449707031,
+      "logps/chosen": -0.38949456810951233,
+      "logps/rejected": -3.1059412956237793,
+      "loss": 0.6593,
+      "nll_loss": 0.5457557439804077,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.19474728405475616,
+      "rewards/margins": 1.35822331905365,
+      "rewards/rejected": -1.5529706478118896,
+      "step": 415
+    },
+    {
+      "epoch": 2.503725782414307,
+      "grad_norm": 13.079460144042969,
+      "learning_rate": 2.4397501823713327e-06,
+      "log_odds_chosen": 3.7700283527374268,
+      "log_odds_ratio": -0.06273230165243149,
+      "logits/chosen": 351.4734802246094,
+      "logits/rejected": 317.71209716796875,
+      "logps/chosen": -0.3789517283439636,
+      "logps/rejected": -2.9251832962036133,
+      "loss": 0.6409,
+      "nll_loss": 0.7009339928627014,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.1894758641719818,
+      "rewards/margins": 1.273115634918213,
+      "rewards/rejected": -1.4625916481018066,
+      "step": 420
+    },
+    {
+      "epoch": 2.533532041728763,
+      "grad_norm": 20.27842903137207,
+      "learning_rate": 2.4253562503633297e-06,
+      "log_odds_chosen": 4.24957799911499,
+      "log_odds_ratio": -0.032464753836393356,
+      "logits/chosen": 348.3805847167969,
+      "logits/rejected": 337.718994140625,
+      "logps/chosen": -0.3897190988063812,
+      "logps/rejected": -3.382542848587036,
+      "loss": 0.6474,
+      "nll_loss": 0.6459823250770569,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1948595494031906,
+      "rewards/margins": 1.4964120388031006,
+      "rewards/rejected": -1.691271424293518,
+      "step": 425
+    },
+    {
+      "epoch": 2.563338301043219,
+      "grad_norm": 14.640429496765137,
+      "learning_rate": 2.411214110852061e-06,
+      "log_odds_chosen": 4.611870765686035,
+      "log_odds_ratio": -0.03490515798330307,
+      "logits/chosen": 347.74298095703125,
+      "logits/rejected": 351.59912109375,
+      "logps/chosen": -0.3038939833641052,
+      "logps/rejected": -3.4484775066375732,
+      "loss": 0.6441,
+      "nll_loss": 0.5448944568634033,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1519469916820526,
+      "rewards/margins": 1.572291612625122,
+      "rewards/rejected": -1.7242387533187866,
+      "step": 430
+    },
+    {
+      "epoch": 2.593144560357675,
+      "grad_norm": 15.337629318237305,
+      "learning_rate": 2.3973165074269213e-06,
+      "log_odds_chosen": 3.7105846405029297,
+      "log_odds_ratio": -0.04743606597185135,
+      "logits/chosen": 360.7786560058594,
+      "logits/rejected": 318.19854736328125,
+      "logps/chosen": -0.36348485946655273,
+      "logps/rejected": -2.7734932899475098,
+      "loss": 0.6432,
+      "nll_loss": 0.5634604692459106,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.18174242973327637,
+      "rewards/margins": 1.205004096031189,
+      "rewards/rejected": -1.3867466449737549,
+      "step": 435
+    },
+    {
+      "epoch": 2.6229508196721314,
+      "grad_norm": 13.271897315979004,
+      "learning_rate": 2.3836564731139807e-06,
+      "log_odds_chosen": 3.826213836669922,
+      "log_odds_ratio": -0.04420660808682442,
+      "logits/chosen": 342.50848388671875,
+      "logits/rejected": 348.86456298828125,
+      "logps/chosen": -0.30872535705566406,
+      "logps/rejected": -2.6104750633239746,
+      "loss": 0.666,
+      "nll_loss": 0.6212347745895386,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.15436267852783203,
+      "rewards/margins": 1.1508748531341553,
+      "rewards/rejected": -1.3052375316619873,
+      "step": 440
+    },
+    {
+      "epoch": 2.6527570789865873,
+      "grad_norm": 17.46467399597168,
+      "learning_rate": 2.3702273156998867e-06,
+      "log_odds_chosen": 4.261384963989258,
+      "log_odds_ratio": -0.03544224798679352,
+      "logits/chosen": 323.58251953125,
+      "logits/rejected": 354.87847900390625,
+      "logps/chosen": -0.4285755753517151,
+      "logps/rejected": -3.4066290855407715,
+      "loss": 0.6866,
+      "nll_loss": 0.6935943961143494,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.21428778767585754,
+      "rewards/margins": 1.4890267848968506,
+      "rewards/rejected": -1.7033145427703857,
+      "step": 445
+    },
+    {
+      "epoch": 2.682563338301043,
+      "grad_norm": 12.280491828918457,
+      "learning_rate": 2.357022603955159e-06,
+      "log_odds_chosen": 3.510754108428955,
+      "log_odds_ratio": -0.05934901908040047,
+      "logits/chosen": 356.21588134765625,
+      "logits/rejected": 349.2935485839844,
+      "logps/chosen": -0.4579055905342102,
+      "logps/rejected": -2.802351474761963,
+      "loss": 0.6947,
+      "nll_loss": 0.6444963216781616,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2289527952671051,
+      "rewards/margins": 1.1722228527069092,
+      "rewards/rejected": -1.4011757373809814,
+      "step": 450
+    },
+    {
+      "epoch": 2.712369597615499,
+      "grad_norm": 14.349300384521484,
+      "learning_rate": 2.3440361546924774e-06,
+      "log_odds_chosen": 3.9128804206848145,
+      "log_odds_ratio": -0.05345267057418823,
+      "logits/chosen": 379.7029724121094,
+      "logits/rejected": 353.4925842285156,
+      "logps/chosen": -0.4496696889400482,
+      "logps/rejected": -3.064148426055908,
+      "loss": 0.7279,
+      "nll_loss": 0.6569377779960632,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2248348444700241,
+      "rewards/margins": 1.307239294052124,
+      "rewards/rejected": -1.532074213027954,
+      "step": 455
+    },
+    {
+      "epoch": 2.742175856929955,
+      "grad_norm": 13.55716609954834,
+      "learning_rate": 2.3312620206007847e-06,
+      "log_odds_chosen": 4.291790962219238,
+      "log_odds_ratio": -0.044444940984249115,
+      "logits/chosen": 374.377197265625,
+      "logits/rejected": 386.43414306640625,
+      "logps/chosen": -0.3812227249145508,
+      "logps/rejected": -3.4395527839660645,
+      "loss": 0.6847,
+      "nll_loss": 0.7294474840164185,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1906113624572754,
+      "rewards/margins": 1.5291650295257568,
+      "rewards/rejected": -1.7197763919830322,
+      "step": 460
+    },
+    {
+      "epoch": 2.7719821162444114,
+      "grad_norm": 13.988215446472168,
+      "learning_rate": 2.3186944788008413e-06,
+      "log_odds_chosen": 4.281071186065674,
+      "log_odds_ratio": -0.04669572785496712,
+      "logits/chosen": 359.35101318359375,
+      "logits/rejected": 352.8810729980469,
+      "logps/chosen": -0.30659520626068115,
+      "logps/rejected": -3.1890709400177,
+      "loss": 0.6956,
+      "nll_loss": 0.6471156477928162,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.15329760313034058,
+      "rewards/margins": 1.4412376880645752,
+      "rewards/rejected": -1.59453547000885,
+      "step": 465
+    },
+    {
+      "epoch": 2.8017883755588673,
+      "grad_norm": 16.967275619506836,
+      "learning_rate": 2.3063280200722128e-06,
+      "log_odds_chosen": 3.476628065109253,
+      "log_odds_ratio": -0.07849601656198502,
+      "logits/chosen": 363.3095703125,
+      "logits/rejected": 323.81842041015625,
+      "logps/chosen": -0.43367767333984375,
+      "logps/rejected": -2.7612318992614746,
+      "loss": 0.6725,
+      "nll_loss": 0.6623119711875916,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.21683883666992188,
+      "rewards/margins": 1.1637769937515259,
+      "rewards/rejected": -1.3806159496307373,
+      "step": 470
+    },
+    {
+      "epoch": 2.8315946348733236,
+      "grad_norm": 19.491451263427734,
+      "learning_rate": 2.2941573387056174e-06,
+      "log_odds_chosen": 3.7874722480773926,
+      "log_odds_ratio": -0.04359927773475647,
+      "logits/chosen": 332.71844482421875,
+      "logits/rejected": 350.5658264160156,
+      "logps/chosen": -0.39931654930114746,
+      "logps/rejected": -2.8959929943084717,
+      "loss": 0.6421,
+      "nll_loss": 0.5615902543067932,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.19965827465057373,
+      "rewards/margins": 1.248338222503662,
+      "rewards/rejected": -1.4479964971542358,
+      "step": 475
+    },
+    {
+      "epoch": 2.8614008941877795,
+      "grad_norm": 18.62973976135254,
+      "learning_rate": 2.2821773229381924e-06,
+      "log_odds_chosen": 4.226737976074219,
+      "log_odds_ratio": -0.04461986571550369,
+      "logits/chosen": 344.9051208496094,
+      "logits/rejected": 380.61822509765625,
+      "logps/chosen": -0.3830781877040863,
+      "logps/rejected": -3.3659965991973877,
+      "loss": 0.6189,
+      "nll_loss": 0.5661858320236206,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.19153909385204315,
+      "rewards/margins": 1.4914592504501343,
+      "rewards/rejected": -1.6829982995986938,
+      "step": 480
+    },
+    {
+      "epoch": 2.8912071535022354,
+      "grad_norm": 14.639021873474121,
+      "learning_rate": 2.270383045932499e-06,
+      "log_odds_chosen": 3.5930495262145996,
+      "log_odds_ratio": -0.06206268072128296,
+      "logits/chosen": 341.22650146484375,
+      "logits/rejected": 361.0518493652344,
+      "logps/chosen": -0.48062896728515625,
+      "logps/rejected": -2.968130111694336,
+      "loss": 0.6645,
+      "nll_loss": 0.6335129737854004,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.24031448364257812,
+      "rewards/margins": 1.2437504529953003,
+      "rewards/rejected": -1.484065055847168,
+      "step": 485
+    },
+    {
+      "epoch": 2.9210134128166914,
+      "grad_norm": 13.041991233825684,
+      "learning_rate": 2.2587697572631284e-06,
+      "log_odds_chosen": 3.418093204498291,
+      "log_odds_ratio": -0.06181482598185539,
+      "logits/chosen": 360.23419189453125,
+      "logits/rejected": 314.44183349609375,
+      "logps/chosen": -0.4381326138973236,
+      "logps/rejected": -2.6530632972717285,
+      "loss": 0.7257,
+      "nll_loss": 0.6311744451522827,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2190663069486618,
+      "rewards/margins": 1.107465386390686,
+      "rewards/rejected": -1.3265316486358643,
+      "step": 490
+    },
+    {
+      "epoch": 2.9508196721311473,
+      "grad_norm": 13.917710304260254,
+      "learning_rate": 2.2473328748774737e-06,
+      "log_odds_chosen": 3.9481112957000732,
+      "log_odds_ratio": -0.07351940125226974,
+      "logits/chosen": 354.31781005859375,
+      "logits/rejected": 374.8128356933594,
+      "logps/chosen": -0.5134202837944031,
+      "logps/rejected": -3.2178916931152344,
+      "loss": 0.6495,
+      "nll_loss": 0.6444075703620911,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.25671014189720154,
+      "rewards/margins": 1.3522356748580933,
+      "rewards/rejected": -1.6089458465576172,
+      "step": 495
+    },
+    {
+      "epoch": 2.9806259314456036,
+      "grad_norm": 13.279515266418457,
+      "learning_rate": 2.23606797749979e-06,
+      "log_odds_chosen": 3.631924867630005,
+      "log_odds_ratio": -0.05388236045837402,
+      "logits/chosen": 363.3906555175781,
+      "logits/rejected": 350.91546630859375,
+      "logps/chosen": -0.3526715636253357,
+      "logps/rejected": -2.695366382598877,
+      "loss": 0.6827,
+      "nll_loss": 0.5351300835609436,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.17633578181266785,
+      "rewards/margins": 1.1713473796844482,
+      "rewards/rejected": -1.3476831912994385,
+      "step": 500
+    },
+    {
+      "epoch": 2.9865871833084947,
+      "eval_log_odds_chosen": 0.42297032475471497,
+      "eval_log_odds_ratio": -0.6477048993110657,
+      "eval_logits/chosen": 282.00848388671875,
+      "eval_logits/rejected": 247.6262664794922,
+      "eval_logps/chosen": -1.2038370370864868,
+      "eval_logps/rejected": -1.5014722347259521,
+      "eval_loss": 1.9079890251159668,
+      "eval_nll_loss": 1.5545235872268677,
+      "eval_rewards/accuracies": 0.6258992552757263,
+      "eval_rewards/chosen": -0.6019185185432434,
+      "eval_rewards/margins": 0.14881758391857147,
+      "eval_rewards/rejected": -0.7507361173629761,
+      "eval_runtime": 351.9872,
+      "eval_samples_per_second": 1.571,
+      "eval_steps_per_second": 0.395,
+      "step": 501
+    },
+    {
+      "epoch": 2.9865871833084947,
+      "step": 501,
+      "total_flos": 0.0,
+      "train_loss": 1.686337627812536,
+      "train_runtime": 31593.308,
+      "train_samples_per_second": 0.509,
+      "train_steps_per_second": 0.016
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 501,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be1d8a27d48554cddc963c8d0b2118d650c144ce868f0fed3e7f108c0b781389
+size 6776