Model save

README.md

@@ -0,0 +1,61 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-sft-full-orpo
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/statking/huggingface/runs/lw7rbi20)
+# zephyr-7b-sft-full-orpo
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.41.0.dev0
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9994756161510225,
+    "total_flos": 0.0,
+    "train_loss": 0.56347276506494,
+    "train_runtime": 19079.6454,
+    "train_samples": 61005,
+    "train_samples_per_second": 3.197,
+    "train_steps_per_second": 0.05
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.41.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8de661845ca4af8b34b8ce2a45722bcc4b2834c916890a9672a109c08517fd6
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41d93ae590bd8fedc2ce9c0d3406d28032086aa93efb3b8bd7351d4fc1684cda
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:327daa62025779f1d2ac5100edbf25a4b7774225203e30efa878921815a519f4
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9994756161510225,
+    "total_flos": 0.0,
+    "train_loss": 0.56347276506494,
+    "train_runtime": 19079.6454,
+    "train_samples": 61005,
+    "train_samples_per_second": 3.197,
+    "train_steps_per_second": 0.05
+}

trainer_state.json

@@ -0,0 +1,1752 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9994756161510225,
+  "eval_steps": 500,
+  "global_step": 953,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01048767697954903,
+      "grad_norm": 12.504458138350461,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": 0.1660214066505432,
+      "log_odds_ratio": -0.6960338354110718,
+      "logits/chosen": -2.542905330657959,
+      "logits/rejected": -2.5316882133483887,
+      "logps/chosen": -0.9998037219047546,
+      "logps/rejected": -1.0999689102172852,
+      "loss": 2.7433,
+      "nll_loss": 2.6550583839416504,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.04999018833041191,
+      "rewards/margins": 0.005008256994187832,
+      "rewards/rejected": -0.05499844625592232,
+      "step": 10
+    },
+    {
+      "epoch": 0.02097535395909806,
+      "grad_norm": 3.296398746092505,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": 0.1942831575870514,
+      "log_odds_ratio": -0.6660380959510803,
+      "logits/chosen": -3.148456335067749,
+      "logits/rejected": -3.171660900115967,
+      "logps/chosen": -0.7626909613609314,
+      "logps/rejected": -0.8731427192687988,
+      "loss": 0.563,
+      "nll_loss": 0.5225270986557007,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.03813454881310463,
+      "rewards/margins": 0.00552258500829339,
+      "rewards/rejected": -0.04365713149309158,
+      "step": 20
+    },
+    {
+      "epoch": 0.03146303093864709,
+      "grad_norm": 2.4400188978085695,
+      "learning_rate": 6e-06,
+      "log_odds_chosen": 0.2339784801006317,
+      "log_odds_ratio": -0.6537522673606873,
+      "logits/chosen": -2.9630327224731445,
+      "logits/rejected": -2.9368481636047363,
+      "logps/chosen": -0.8345462679862976,
+      "logps/rejected": -0.9655241966247559,
+      "loss": 0.5355,
+      "nll_loss": 0.4940575659275055,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04172731190919876,
+      "rewards/margins": 0.0065488978289067745,
+      "rewards/rejected": -0.04827621206641197,
+      "step": 30
+    },
+    {
+      "epoch": 0.04195070791819612,
+      "grad_norm": 2.765802378357493,
+      "learning_rate": 8.000000000000001e-06,
+      "log_odds_chosen": 0.15870003402233124,
+      "log_odds_ratio": -0.6969180107116699,
+      "logits/chosen": -2.8065195083618164,
+      "logits/rejected": -2.7910008430480957,
+      "logps/chosen": -0.8027766346931458,
+      "logps/rejected": -0.9165509343147278,
+      "loss": 0.5199,
+      "nll_loss": 0.48035889863967896,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.04013883322477341,
+      "rewards/margins": 0.005688714794814587,
+      "rewards/rejected": -0.04582754150032997,
+      "step": 40
+    },
+    {
+      "epoch": 0.05243838489774515,
+      "grad_norm": 2.7404814506796704,
+      "learning_rate": 1e-05,
+      "log_odds_chosen": 0.24872338771820068,
+      "log_odds_ratio": -0.680080771446228,
+      "logits/chosen": -2.7704856395721436,
+      "logits/rejected": -2.77298641204834,
+      "logps/chosen": -0.7987793684005737,
+      "logps/rejected": -0.9668463468551636,
+      "loss": 0.5424,
+      "nll_loss": 0.48421746492385864,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.03993896767497063,
+      "rewards/margins": 0.00840335339307785,
+      "rewards/rejected": -0.048342324793338776,
+      "step": 50
+    },
+    {
+      "epoch": 0.06292606187729417,
+      "grad_norm": 2.7601739927853473,
+      "learning_rate": 1.2e-05,
+      "log_odds_chosen": 0.21160352230072021,
+      "log_odds_ratio": -0.6764382123947144,
+      "logits/chosen": -3.0032615661621094,
+      "logits/rejected": -2.9960169792175293,
+      "logps/chosen": -0.7965995669364929,
+      "logps/rejected": -0.917363166809082,
+      "loss": 0.5463,
+      "nll_loss": 0.516124427318573,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.039829984307289124,
+      "rewards/margins": 0.006038171239197254,
+      "rewards/rejected": -0.045868150889873505,
+      "step": 60
+    },
+    {
+      "epoch": 0.07341373885684321,
+      "grad_norm": 3.2123267767300128,
+      "learning_rate": 1.4e-05,
+      "log_odds_chosen": 0.19886036217212677,
+      "log_odds_ratio": -0.690485417842865,
+      "logits/chosen": -2.978163719177246,
+      "logits/rejected": -3.0078656673431396,
+      "logps/chosen": -0.8206535577774048,
+      "logps/rejected": -0.9310994148254395,
+      "loss": 0.5403,
+      "nll_loss": 0.530234694480896,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.04103267565369606,
+      "rewards/margins": 0.0055222949013113976,
+      "rewards/rejected": -0.046554967761039734,
+      "step": 70
+    },
+    {
+      "epoch": 0.08390141583639224,
+      "grad_norm": 3.267750524500123,
+      "learning_rate": 1.6000000000000003e-05,
+      "log_odds_chosen": 0.1725669652223587,
+      "log_odds_ratio": -0.689757764339447,
+      "logits/chosen": -2.963442087173462,
+      "logits/rejected": -2.953914165496826,
+      "logps/chosen": -0.8903671503067017,
+      "logps/rejected": -1.0184500217437744,
+      "loss": 0.5632,
+      "nll_loss": 0.48384732007980347,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.0445183590054512,
+      "rewards/margins": 0.006404136773198843,
+      "rewards/rejected": -0.050922494381666183,
+      "step": 80
+    },
+    {
+      "epoch": 0.09438909281594127,
+      "grad_norm": 6.338896835312273,
+      "learning_rate": 1.8e-05,
+      "log_odds_chosen": 0.2590278387069702,
+      "log_odds_ratio": -0.6696828603744507,
+      "logits/chosen": -2.7556283473968506,
+      "logits/rejected": -2.759223461151123,
+      "logps/chosen": -0.8806008100509644,
+      "logps/rejected": -1.0427037477493286,
+      "loss": 0.5599,
+      "nll_loss": 0.49117976427078247,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.044030044227838516,
+      "rewards/margins": 0.008105142042040825,
+      "rewards/rejected": -0.05213518068194389,
+      "step": 90
+    },
+    {
+      "epoch": 0.1048767697954903,
+      "grad_norm": 2.844482964932932,
+      "learning_rate": 2e-05,
+      "log_odds_chosen": 0.20001336932182312,
+      "log_odds_ratio": -0.6672823429107666,
+      "logits/chosen": -2.836613178253174,
+      "logits/rejected": -2.826347827911377,
+      "logps/chosen": -0.8816211819648743,
+      "logps/rejected": -1.0050264596939087,
+      "loss": 0.5675,
+      "nll_loss": 0.5239149332046509,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.044081058353185654,
+      "rewards/margins": 0.006170268170535564,
+      "rewards/rejected": -0.05025132745504379,
+      "step": 100
+    },
+    {
+      "epoch": 0.11536444677503933,
+      "grad_norm": 2.717573270122186,
+      "learning_rate": 1.9069251784911845e-05,
+      "log_odds_chosen": 0.26770642399787903,
+      "log_odds_ratio": -0.6399692296981812,
+      "logits/chosen": -2.8041529655456543,
+      "logits/rejected": -2.828374147415161,
+      "logps/chosen": -0.8482567071914673,
+      "logps/rejected": -1.021328330039978,
+      "loss": 0.568,
+      "nll_loss": 0.5094035863876343,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.042412832379341125,
+      "rewards/margins": 0.008653589524328709,
+      "rewards/rejected": -0.05106641724705696,
+      "step": 110
+    },
+    {
+      "epoch": 0.12585212375458835,
+      "grad_norm": 2.3522582585650906,
+      "learning_rate": 1.825741858350554e-05,
+      "log_odds_chosen": 0.2770318388938904,
+      "log_odds_ratio": -0.6538770198822021,
+      "logits/chosen": -2.9046432971954346,
+      "logits/rejected": -2.921250343322754,
+      "logps/chosen": -0.8698671460151672,
+      "logps/rejected": -1.0593181848526,
+      "loss": 0.6048,
+      "nll_loss": 0.5620476007461548,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0434933602809906,
+      "rewards/margins": 0.009472550824284554,
+      "rewards/rejected": -0.05296590179204941,
+      "step": 120
+    },
+    {
+      "epoch": 0.1363398007341374,
+      "grad_norm": 2.3512564845307704,
+      "learning_rate": 1.7541160386140587e-05,
+      "log_odds_chosen": 0.213302880525589,
+      "log_odds_ratio": -0.6861675977706909,
+      "logits/chosen": -2.926781177520752,
+      "logits/rejected": -2.930361747741699,
+      "logps/chosen": -0.9192083477973938,
+      "logps/rejected": -1.06519615650177,
+      "loss": 0.5923,
+      "nll_loss": 0.5574383735656738,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.04596042260527611,
+      "rewards/margins": 0.007299385964870453,
+      "rewards/rejected": -0.05325980857014656,
+      "step": 130
+    },
+    {
+      "epoch": 0.14682747771368643,
+      "grad_norm": 2.2489368047485705,
+      "learning_rate": 1.6903085094570334e-05,
+      "log_odds_chosen": 0.24789170920848846,
+      "log_odds_ratio": -0.655090868473053,
+      "logits/chosen": -2.9084389209747314,
+      "logits/rejected": -2.9173099994659424,
+      "logps/chosen": -0.9441210031509399,
+      "logps/rejected": -1.1045926809310913,
+      "loss": 0.5882,
+      "nll_loss": 0.5544429421424866,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.047206051647663116,
+      "rewards/margins": 0.008023588918149471,
+      "rewards/rejected": -0.05522964149713516,
+      "step": 140
+    },
+    {
+      "epoch": 0.15731515469323545,
+      "grad_norm": 2.6715309670512903,
+      "learning_rate": 1.6329931618554523e-05,
+      "log_odds_chosen": 0.14654028415679932,
+      "log_odds_ratio": -0.7416929006576538,
+      "logits/chosen": -2.8286139965057373,
+      "logits/rejected": -2.842860698699951,
+      "logps/chosen": -0.9699670672416687,
+      "logps/rejected": -1.0669214725494385,
+      "loss": 0.5441,
+      "nll_loss": 0.5359360575675964,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.048498354852199554,
+      "rewards/margins": 0.004847715608775616,
+      "rewards/rejected": -0.053346067667007446,
+      "step": 150
+    },
+    {
+      "epoch": 0.16780283167278448,
+      "grad_norm": 2.4917874181934616,
+      "learning_rate": 1.5811388300841898e-05,
+      "log_odds_chosen": 0.19475655257701874,
+      "log_odds_ratio": -0.664051353931427,
+      "logits/chosen": -2.8252522945404053,
+      "logits/rejected": -2.839994192123413,
+      "logps/chosen": -0.9179447889328003,
+      "logps/rejected": -1.0352815389633179,
+      "loss": 0.6078,
+      "nll_loss": 0.5540346503257751,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.045897237956523895,
+      "rewards/margins": 0.005866840481758118,
+      "rewards/rejected": -0.05176408216357231,
+      "step": 160
+    },
+    {
+      "epoch": 0.1782905086523335,
+      "grad_norm": 2.493896039254152,
+      "learning_rate": 1.533929977694741e-05,
+      "log_odds_chosen": 0.25445470213890076,
+      "log_odds_ratio": -0.6574397087097168,
+      "logits/chosen": -2.895998477935791,
+      "logits/rejected": -2.9125123023986816,
+      "logps/chosen": -0.8917832374572754,
+      "logps/rejected": -1.0586717128753662,
+      "loss": 0.5884,
+      "nll_loss": 0.5544494986534119,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04458915814757347,
+      "rewards/margins": 0.008344428613781929,
+      "rewards/rejected": -0.05293358489871025,
+      "step": 170
+    },
+    {
+      "epoch": 0.18877818563188253,
+      "grad_norm": 2.368451448201635,
+      "learning_rate": 1.49071198499986e-05,
+      "log_odds_chosen": 0.2552924156188965,
+      "log_odds_ratio": -0.6543556451797485,
+      "logits/chosen": -2.8886399269104004,
+      "logits/rejected": -2.905686378479004,
+      "logps/chosen": -0.9206914901733398,
+      "logps/rejected": -1.091048240661621,
+      "loss": 0.5686,
+      "nll_loss": 0.551173985004425,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04603457450866699,
+      "rewards/margins": 0.008517834357917309,
+      "rewards/rejected": -0.054552413523197174,
+      "step": 180
+    },
+    {
+      "epoch": 0.19926586261143156,
+      "grad_norm": 4.734046585912702,
+      "learning_rate": 1.4509525002200235e-05,
+      "log_odds_chosen": 0.21173310279846191,
+      "log_odds_ratio": -0.6579927206039429,
+      "logits/chosen": -2.9355111122131348,
+      "logits/rejected": -2.952430009841919,
+      "logps/chosen": -0.9388859868049622,
+      "logps/rejected": -1.0733187198638916,
+      "loss": 0.5936,
+      "nll_loss": 0.6142745018005371,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04694430157542229,
+      "rewards/margins": 0.006721635349094868,
+      "rewards/rejected": -0.05366594344377518,
+      "step": 190
+    },
+    {
+      "epoch": 0.2097535395909806,
+      "grad_norm": 2.2391424397427073,
+      "learning_rate": 1.4142135623730951e-05,
+      "log_odds_chosen": 0.28418153524398804,
+      "log_odds_ratio": -0.6668760180473328,
+      "logits/chosen": -2.873599052429199,
+      "logits/rejected": -2.9066414833068848,
+      "logps/chosen": -0.9204713702201843,
+      "logps/rejected": -1.128112554550171,
+      "loss": 0.5689,
+      "nll_loss": 0.5723541975021362,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.046023570001125336,
+      "rewards/margins": 0.010382059030234814,
+      "rewards/rejected": -0.056405626237392426,
+      "step": 200
+    },
+    {
+      "epoch": 0.22024121657052964,
+      "grad_norm": 2.1684330770876152,
+      "learning_rate": 1.3801311186847084e-05,
+      "log_odds_chosen": 0.11919783055782318,
+      "log_odds_ratio": -0.7173447012901306,
+      "logits/chosen": -2.884079933166504,
+      "logits/rejected": -2.8981668949127197,
+      "logps/chosen": -0.8726099729537964,
+      "logps/rejected": -0.9488958120346069,
+      "loss": 0.5693,
+      "nll_loss": 0.5325449109077454,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04363049939274788,
+      "rewards/margins": 0.0038142912089824677,
+      "rewards/rejected": -0.04744479060173035,
+      "step": 210
+    },
+    {
+      "epoch": 0.23072889355007867,
+      "grad_norm": 2.510753834710904,
+      "learning_rate": 1.3483997249264842e-05,
+      "log_odds_chosen": 0.18100012838840485,
+      "log_odds_ratio": -0.7047401666641235,
+      "logits/chosen": -2.8885810375213623,
+      "logits/rejected": -2.8980116844177246,
+      "logps/chosen": -0.8880792856216431,
+      "logps/rejected": -1.0071966648101807,
+      "loss": 0.5589,
+      "nll_loss": 0.5211626291275024,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.044403962790966034,
+      "rewards/margins": 0.005955878179520369,
+      "rewards/rejected": -0.05035984516143799,
+      "step": 220
+    },
+    {
+      "epoch": 0.2412165705296277,
+      "grad_norm": 2.0148191421861705,
+      "learning_rate": 1.3187609467915744e-05,
+      "log_odds_chosen": 0.2717307209968567,
+      "log_odds_ratio": -0.6763201951980591,
+      "logits/chosen": -2.829516887664795,
+      "logits/rejected": -2.842909574508667,
+      "logps/chosen": -0.9367680549621582,
+      "logps/rejected": -1.1125657558441162,
+      "loss": 0.5701,
+      "nll_loss": 0.5263533592224121,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.04683841019868851,
+      "rewards/margins": 0.008789879269897938,
+      "rewards/rejected": -0.05562828853726387,
+      "step": 230
+    },
+    {
+      "epoch": 0.2517042475091767,
+      "grad_norm": 2.286828850039024,
+      "learning_rate": 1.2909944487358057e-05,
+      "log_odds_chosen": 0.2564060091972351,
+      "log_odds_ratio": -0.651031494140625,
+      "logits/chosen": -2.979280471801758,
+      "logits/rejected": -3.0063037872314453,
+      "logps/chosen": -0.9010913968086243,
+      "logps/rejected": -1.065353512763977,
+      "loss": 0.5799,
+      "nll_loss": 0.5546143054962158,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04505457356572151,
+      "rewards/margins": 0.008213100023567677,
+      "rewards/rejected": -0.053267668932676315,
+      "step": 240
+    },
+    {
+      "epoch": 0.26219192448872575,
+      "grad_norm": 3.959216899336302,
+      "learning_rate": 1.2649110640673518e-05,
+      "log_odds_chosen": 0.2661912143230438,
+      "log_odds_ratio": -0.6746715307235718,
+      "logits/chosen": -2.9726908206939697,
+      "logits/rejected": -2.974113941192627,
+      "logps/chosen": -0.8829942941665649,
+      "logps/rejected": -1.0264866352081299,
+      "loss": 0.5502,
+      "nll_loss": 0.5201153755187988,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.04414971172809601,
+      "rewards/margins": 0.007174622267484665,
+      "rewards/rejected": -0.05132433772087097,
+      "step": 250
+    },
+    {
+      "epoch": 0.2726796014682748,
+      "grad_norm": 2.2699181039817,
+      "learning_rate": 1.2403473458920845e-05,
+      "log_odds_chosen": 0.2342940866947174,
+      "log_odds_ratio": -0.6783974766731262,
+      "logits/chosen": -2.9759726524353027,
+      "logits/rejected": -2.9923360347747803,
+      "logps/chosen": -0.9042210578918457,
+      "logps/rejected": -1.0481539964675903,
+      "loss": 0.5304,
+      "nll_loss": 0.45657747983932495,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0452110581099987,
+      "rewards/margins": 0.007196647580713034,
+      "rewards/rejected": -0.052407700568437576,
+      "step": 260
+    },
+    {
+      "epoch": 0.2831672784478238,
+      "grad_norm": 2.380998150273162,
+      "learning_rate": 1.2171612389003691e-05,
+      "log_odds_chosen": 0.17961958050727844,
+      "log_odds_ratio": -0.6983593702316284,
+      "logits/chosen": -2.938765525817871,
+      "logits/rejected": -2.965757369995117,
+      "logps/chosen": -0.9548166990280151,
+      "logps/rejected": -1.0895111560821533,
+      "loss": 0.5673,
+      "nll_loss": 0.5430372357368469,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.04774082824587822,
+      "rewards/margins": 0.006734730210155249,
+      "rewards/rejected": -0.0544755645096302,
+      "step": 270
+    },
+    {
+      "epoch": 0.29365495542737285,
+      "grad_norm": 2.0870887262121323,
+      "learning_rate": 1.1952286093343936e-05,
+      "log_odds_chosen": 0.2291949987411499,
+      "log_odds_ratio": -0.6750219464302063,
+      "logits/chosen": -2.928527355194092,
+      "logits/rejected": -2.9543163776397705,
+      "logps/chosen": -0.9355181455612183,
+      "logps/rejected": -1.0729036331176758,
+      "loss": 0.5434,
+      "nll_loss": 0.47713321447372437,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04677591472864151,
+      "rewards/margins": 0.006869266740977764,
+      "rewards/rejected": -0.05364518240094185,
+      "step": 280
+    },
+    {
+      "epoch": 0.30414263240692185,
+      "grad_norm": 2.661552133228645,
+      "learning_rate": 1.1744404390294071e-05,
+      "log_odds_chosen": 0.36491650342941284,
+      "log_odds_ratio": -0.620793879032135,
+      "logits/chosen": -2.880122661590576,
+      "logits/rejected": -2.8935391902923584,
+      "logps/chosen": -0.836012065410614,
+      "logps/rejected": -1.05286705493927,
+      "loss": 0.5596,
+      "nll_loss": 0.4885989725589752,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0418006032705307,
+      "rewards/margins": 0.010842744261026382,
+      "rewards/rejected": -0.05264334753155708,
+      "step": 290
+    },
+    {
+      "epoch": 0.3146303093864709,
+      "grad_norm": 3.127285518362044,
+      "learning_rate": 1.1547005383792517e-05,
+      "log_odds_chosen": 0.255328893661499,
+      "log_odds_ratio": -0.6939107179641724,
+      "logits/chosen": -2.9603378772735596,
+      "logits/rejected": -2.992128372192383,
+      "logps/chosen": -0.8731514811515808,
+      "logps/rejected": -1.0526010990142822,
+      "loss": 0.5835,
+      "nll_loss": 0.5112031102180481,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.0436575748026371,
+      "rewards/margins": 0.008972481824457645,
+      "rewards/rejected": -0.052630048245191574,
+      "step": 300
+    },
+    {
+      "epoch": 0.3251179863660199,
+      "grad_norm": 2.013637214040506,
+      "learning_rate": 1.1359236684941297e-05,
+      "log_odds_chosen": 0.21040907502174377,
+      "log_odds_ratio": -0.688109278678894,
+      "logits/chosen": -2.9860305786132812,
+      "logits/rejected": -2.9820261001586914,
+      "logps/chosen": -0.9089478254318237,
+      "logps/rejected": -1.0382112264633179,
+      "loss": 0.585,
+      "nll_loss": 0.5399721264839172,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.045447397977113724,
+      "rewards/margins": 0.006463165394961834,
+      "rewards/rejected": -0.051910560578107834,
+      "step": 310
+    },
+    {
+      "epoch": 0.33560566334556896,
+      "grad_norm": 2.1577553752792995,
+      "learning_rate": 1.118033988749895e-05,
+      "log_odds_chosen": 0.27985960245132446,
+      "log_odds_ratio": -0.6601210832595825,
+      "logits/chosen": -3.0387003421783447,
+      "logits/rejected": -3.0464096069335938,
+      "logps/chosen": -0.9086373448371887,
+      "logps/rejected": -1.0836986303329468,
+      "loss": 0.5243,
+      "nll_loss": 0.4922841191291809,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.045431867241859436,
+      "rewards/margins": 0.008753069676458836,
+      "rewards/rejected": -0.0541849359869957,
+      "step": 320
+    },
+    {
+      "epoch": 0.34609334032511796,
+      "grad_norm": 2.422690319169778,
+      "learning_rate": 1.1009637651263608e-05,
+      "log_odds_chosen": 0.28255337476730347,
+      "log_odds_ratio": -0.6909259557723999,
+      "logits/chosen": -2.950887441635132,
+      "logits/rejected": -2.9948947429656982,
+      "logps/chosen": -0.9054603576660156,
+      "logps/rejected": -1.0888211727142334,
+      "loss": 0.5544,
+      "nll_loss": 0.5376341342926025,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04527302458882332,
+      "rewards/margins": 0.009168041869997978,
+      "rewards/rejected": -0.05444106459617615,
+      "step": 330
+    },
+    {
+      "epoch": 0.356581017304667,
+      "grad_norm": 2.2975046406882798,
+      "learning_rate": 1.0846522890932809e-05,
+      "log_odds_chosen": 0.2153971642255783,
+      "log_odds_ratio": -0.6926898956298828,
+      "logits/chosen": -2.9686572551727295,
+      "logits/rejected": -3.0199432373046875,
+      "logps/chosen": -0.8590608835220337,
+      "logps/rejected": -1.00636887550354,
+      "loss": 0.5708,
+      "nll_loss": 0.5127817392349243,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.042953044176101685,
+      "rewards/margins": 0.007365405559539795,
+      "rewards/rejected": -0.05031844973564148,
+      "step": 340
+    },
+    {
+      "epoch": 0.36706869428421607,
+      "grad_norm": 2.135727653321979,
+      "learning_rate": 1.0690449676496977e-05,
+      "log_odds_chosen": 0.2665565609931946,
+      "log_odds_ratio": -0.6829238533973694,
+      "logits/chosen": -3.044860363006592,
+      "logits/rejected": -3.0616378784179688,
+      "logps/chosen": -0.8791500329971313,
+      "logps/rejected": -1.0402672290802002,
+      "loss": 0.5495,
+      "nll_loss": 0.5228344202041626,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.04395749792456627,
+      "rewards/margins": 0.00805586390197277,
+      "rewards/rejected": -0.05201335996389389,
+      "step": 350
+    },
+    {
+      "epoch": 0.37755637126376507,
+      "grad_norm": 3.150177435714442,
+      "learning_rate": 1.0540925533894598e-05,
+      "log_odds_chosen": 0.4033277928829193,
+      "log_odds_ratio": -0.602225124835968,
+      "logits/chosen": -2.9472672939300537,
+      "logits/rejected": -2.975858211517334,
+      "logps/chosen": -0.8669608235359192,
+      "logps/rejected": -1.110353708267212,
+      "loss": 0.5494,
+      "nll_loss": 0.5087054371833801,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0433480478823185,
+      "rewards/margins": 0.01216964516788721,
+      "rewards/rejected": -0.05551769211888313,
+      "step": 360
+    },
+    {
+      "epoch": 0.3880440482433141,
+      "grad_norm": 2.130197231019511,
+      "learning_rate": 1.0397504898200728e-05,
+      "log_odds_chosen": 0.3966829478740692,
+      "log_odds_ratio": -0.6142522096633911,
+      "logits/chosen": -3.0528526306152344,
+      "logits/rejected": -3.0623490810394287,
+      "logps/chosen": -0.8640265464782715,
+      "logps/rejected": -1.1243717670440674,
+      "loss": 0.5232,
+      "nll_loss": 0.5101068615913391,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.043201327323913574,
+      "rewards/margins": 0.013017257675528526,
+      "rewards/rejected": -0.05621858313679695,
+      "step": 370
+    },
+    {
+      "epoch": 0.3985317252228631,
+      "grad_norm": 2.415549044992692,
+      "learning_rate": 1.0259783520851543e-05,
+      "log_odds_chosen": 0.46208301186561584,
+      "log_odds_ratio": -0.5873923301696777,
+      "logits/chosen": -3.055903196334839,
+      "logits/rejected": -3.089763879776001,
+      "logps/chosen": -0.8685981035232544,
+      "logps/rejected": -1.1247217655181885,
+      "loss": 0.5376,
+      "nll_loss": 0.5167646408081055,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.0434299036860466,
+      "rewards/margins": 0.01280617993324995,
+      "rewards/rejected": -0.056236088275909424,
+      "step": 380
+    },
+    {
+      "epoch": 0.4090194022024122,
+      "grad_norm": 2.4197618087673036,
+      "learning_rate": 1.0127393670836667e-05,
+      "log_odds_chosen": 0.08936772495508194,
+      "log_odds_ratio": -0.7186132073402405,
+      "logits/chosen": -2.998857021331787,
+      "logits/rejected": -3.021352529525757,
+      "logps/chosen": -0.9128287434577942,
+      "logps/rejected": -0.9754525423049927,
+      "loss": 0.5571,
+      "nll_loss": 0.5319759845733643,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.04564143717288971,
+      "rewards/margins": 0.0031311833299696445,
+      "rewards/rejected": -0.048772621899843216,
+      "step": 390
+    },
+    {
+      "epoch": 0.4195070791819612,
+      "grad_norm": 2.0748995530757424,
+      "learning_rate": 1e-05,
+      "log_odds_chosen": 0.23965713381767273,
+      "log_odds_ratio": -0.6899853348731995,
+      "logits/chosen": -2.883575201034546,
+      "logits/rejected": -2.908125400543213,
+      "logps/chosen": -0.9490350484848022,
+      "logps/rejected": -1.1106139421463013,
+      "loss": 0.5725,
+      "nll_loss": 0.5262094736099243,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.04745175316929817,
+      "rewards/margins": 0.00807894580066204,
+      "rewards/rejected": -0.05553068965673447,
+      "step": 400
+    },
+    {
+      "epoch": 0.4299947561615102,
+      "grad_norm": 2.0498490112152026,
+      "learning_rate": 9.877295966495898e-06,
+      "log_odds_chosen": 0.14244404435157776,
+      "log_odds_ratio": -0.7278560996055603,
+      "logits/chosen": -2.988100051879883,
+      "logits/rejected": -2.9914164543151855,
+      "logps/chosen": -0.8709594011306763,
+      "logps/rejected": -0.9773006439208984,
+      "loss": 0.5455,
+      "nll_loss": 0.4832683503627777,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.04354798048734665,
+      "rewards/margins": 0.0053170593455433846,
+      "rewards/rejected": -0.04886503517627716,
+      "step": 410
+    },
+    {
+      "epoch": 0.4404824331410593,
+      "grad_norm": 1.9311064341389872,
+      "learning_rate": 9.759000729485331e-06,
+      "log_odds_chosen": 0.30063071846961975,
+      "log_odds_ratio": -0.643203854560852,
+      "logits/chosen": -2.9488558769226074,
+      "logits/rejected": -2.9841551780700684,
+      "logps/chosen": -0.8707404136657715,
+      "logps/rejected": -1.0532442331314087,
+      "loss": 0.5355,
+      "nll_loss": 0.474843829870224,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04353701323270798,
+      "rewards/margins": 0.009125196374952793,
+      "rewards/rejected": -0.05266221612691879,
+      "step": 420
+    },
+    {
+      "epoch": 0.4509701101206083,
+      "grad_norm": 2.119895291758326,
+      "learning_rate": 9.644856443408244e-06,
+      "log_odds_chosen": 0.2837393879890442,
+      "log_odds_ratio": -0.6551750898361206,
+      "logits/chosen": -2.9840757846832275,
+      "logits/rejected": -2.9921929836273193,
+      "logps/chosen": -0.8468173146247864,
+      "logps/rejected": -1.0135347843170166,
+      "loss": 0.5557,
+      "nll_loss": 0.5443450212478638,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04234086349606514,
+      "rewards/margins": 0.00833587534725666,
+      "rewards/rejected": -0.05067674070596695,
+      "step": 430
+    },
+    {
+      "epoch": 0.46145778710015734,
+      "grad_norm": 2.095435518308805,
+      "learning_rate": 9.534625892455923e-06,
+      "log_odds_chosen": 0.2355252504348755,
+      "log_odds_ratio": -0.6598283648490906,
+      "logits/chosen": -3.0252740383148193,
+      "logits/rejected": -3.045849323272705,
+      "logps/chosen": -0.8709392547607422,
+      "logps/rejected": -1.0179613828659058,
+      "loss": 0.5508,
+      "nll_loss": 0.5189236998558044,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.04354696720838547,
+      "rewards/margins": 0.0073511130176484585,
+      "rewards/rejected": -0.050898075103759766,
+      "step": 440
+    },
+    {
+      "epoch": 0.47194546407970633,
+      "grad_norm": 1.9017756846669818,
+      "learning_rate": 9.428090415820635e-06,
+      "log_odds_chosen": 0.34075412154197693,
+      "log_odds_ratio": -0.6583858728408813,
+      "logits/chosen": -3.0218703746795654,
+      "logits/rejected": -3.0481696128845215,
+      "logps/chosen": -0.8293315768241882,
+      "logps/rejected": -1.047191858291626,
+      "loss": 0.5286,
+      "nll_loss": 0.4964592456817627,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04146658256649971,
+      "rewards/margins": 0.010893006809055805,
+      "rewards/rejected": -0.05235959216952324,
+      "step": 450
+    },
+    {
+      "epoch": 0.4824331410592554,
+      "grad_norm": 2.079766146123277,
+      "learning_rate": 9.325048082403139e-06,
+      "log_odds_chosen": 0.16855968534946442,
+      "log_odds_ratio": -0.711928129196167,
+      "logits/chosen": -3.0086510181427,
+      "logits/rejected": -3.0489156246185303,
+      "logps/chosen": -0.9442957043647766,
+      "logps/rejected": -1.072997808456421,
+      "loss": 0.5326,
+      "nll_loss": 0.5338221788406372,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.04721478372812271,
+      "rewards/margins": 0.00643510278314352,
+      "rewards/rejected": -0.05364988371729851,
+      "step": 460
+    },
+    {
+      "epoch": 0.4929208180388044,
+      "grad_norm": 2.4868491558153085,
+      "learning_rate": 9.225312080288851e-06,
+      "log_odds_chosen": 0.23586861789226532,
+      "log_odds_ratio": -0.6902174949645996,
+      "logits/chosen": -2.986264705657959,
+      "logits/rejected": -3.0127644538879395,
+      "logps/chosen": -0.8882457613945007,
+      "logps/rejected": -1.034985899925232,
+      "loss": 0.5413,
+      "nll_loss": 0.5090312361717224,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.044412292540073395,
+      "rewards/margins": 0.007337009999901056,
+      "rewards/rejected": -0.051749296486377716,
+      "step": 470
+    },
+    {
+      "epoch": 0.5034084950183534,
+      "grad_norm": 2.0043501739666882,
+      "learning_rate": 9.12870929175277e-06,
+      "log_odds_chosen": 0.17604230344295502,
+      "log_odds_ratio": -0.707550048828125,
+      "logits/chosen": -3.088604211807251,
+      "logits/rejected": -3.12184476852417,
+      "logps/chosen": -0.8456010818481445,
+      "logps/rejected": -0.9586717486381531,
+      "loss": 0.5178,
+      "nll_loss": 0.5126105546951294,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.042280055582523346,
+      "rewards/margins": 0.005653535481542349,
+      "rewards/rejected": -0.047933585941791534,
+      "step": 480
+    },
+    {
+      "epoch": 0.5138961719979025,
+      "grad_norm": 1.9415978406566505,
+      "learning_rate": 9.035079029052514e-06,
+      "log_odds_chosen": 0.22476902604103088,
+      "log_odds_ratio": -0.6716736555099487,
+      "logits/chosen": -3.003417491912842,
+      "logits/rejected": -3.0048608779907227,
+      "logps/chosen": -0.9196673631668091,
+      "logps/rejected": -1.0358223915100098,
+      "loss": 0.5397,
+      "nll_loss": 0.5024985671043396,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.045983362942934036,
+      "rewards/margins": 0.005807754583656788,
+      "rewards/rejected": -0.051791124045848846,
+      "step": 490
+    },
+    {
+      "epoch": 0.5243838489774515,
+      "grad_norm": 2.2353701695425423,
+      "learning_rate": 8.94427190999916e-06,
+      "log_odds_chosen": 0.20684054493904114,
+      "log_odds_ratio": -0.698712944984436,
+      "logits/chosen": -3.0111751556396484,
+      "logits/rejected": -3.0036330223083496,
+      "logps/chosen": -0.8826943635940552,
+      "logps/rejected": -1.0074814558029175,
+      "loss": 0.548,
+      "nll_loss": 0.5235316157341003,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0441347137093544,
+      "rewards/margins": 0.006239361595362425,
+      "rewards/rejected": -0.050374072045087814,
+      "step": 500
+    },
+    {
+      "epoch": 0.5348715259570005,
+      "grad_norm": 1.742537477144132,
+      "learning_rate": 8.856148855400955e-06,
+      "log_odds_chosen": 0.3066679835319519,
+      "log_odds_ratio": -0.6453306674957275,
+      "logits/chosen": -2.9636032581329346,
+      "logits/rejected": -2.97407865524292,
+      "logps/chosen": -0.8404191136360168,
+      "logps/rejected": -1.0267155170440674,
+      "loss": 0.5264,
+      "nll_loss": 0.5354185104370117,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04202095791697502,
+      "rewards/margins": 0.009314822033047676,
+      "rewards/rejected": -0.05133577436208725,
+      "step": 510
+    },
+    {
+      "epoch": 0.5453592029365496,
+      "grad_norm": 1.6799388590726438,
+      "learning_rate": 8.770580193070294e-06,
+      "log_odds_chosen": 0.24468369781970978,
+      "log_odds_ratio": -0.6710330247879028,
+      "logits/chosen": -2.959213972091675,
+      "logits/rejected": -2.966728687286377,
+      "logps/chosen": -0.9035038948059082,
+      "logps/rejected": -1.0690029859542847,
+      "loss": 0.5366,
+      "nll_loss": 0.47406935691833496,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04517520219087601,
+      "rewards/margins": 0.008274954743683338,
+      "rewards/rejected": -0.053450148552656174,
+      "step": 520
+    },
+    {
+      "epoch": 0.5558468799160986,
+      "grad_norm": 1.8707354612150964,
+      "learning_rate": 8.687444855261389e-06,
+      "log_odds_chosen": 0.4215427339076996,
+      "log_odds_ratio": -0.6489927172660828,
+      "logits/chosen": -3.0756938457489014,
+      "logits/rejected": -3.0923542976379395,
+      "logps/chosen": -0.8253329992294312,
+      "logps/rejected": -1.1108949184417725,
+      "loss": 0.5365,
+      "nll_loss": 0.45042163133621216,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.04126664996147156,
+      "rewards/margins": 0.014278100803494453,
+      "rewards/rejected": -0.05554475262761116,
+      "step": 530
+    },
+    {
+      "epoch": 0.5663345568956476,
+      "grad_norm": 1.922705947748225,
+      "learning_rate": 8.606629658238705e-06,
+      "log_odds_chosen": 0.1879667341709137,
+      "log_odds_ratio": -0.6903280019760132,
+      "logits/chosen": -2.975130796432495,
+      "logits/rejected": -3.0028696060180664,
+      "logps/chosen": -0.8695458173751831,
+      "logps/rejected": -0.9805169105529785,
+      "loss": 0.5535,
+      "nll_loss": 0.5275255441665649,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.04347729682922363,
+      "rewards/margins": 0.005548550747334957,
+      "rewards/rejected": -0.049025844782590866,
+      "step": 540
+    },
+    {
+      "epoch": 0.5768222338751966,
+      "grad_norm": 1.9089385183272836,
+      "learning_rate": 8.528028654224417e-06,
+      "log_odds_chosen": 0.42722567915916443,
+      "log_odds_ratio": -0.6043616533279419,
+      "logits/chosen": -2.9973807334899902,
+      "logits/rejected": -3.0049965381622314,
+      "logps/chosen": -0.8592002987861633,
+      "logps/rejected": -1.1192405223846436,
+      "loss": 0.537,
+      "nll_loss": 0.5372708439826965,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.042960021644830704,
+      "rewards/margins": 0.013002010062336922,
+      "rewards/rejected": -0.05596202611923218,
+      "step": 550
+    },
+    {
+      "epoch": 0.5873099108547457,
+      "grad_norm": 1.9519454661958895,
+      "learning_rate": 8.451542547285167e-06,
+      "log_odds_chosen": 0.23686861991882324,
+      "log_odds_ratio": -0.679013192653656,
+      "logits/chosen": -3.0309016704559326,
+      "logits/rejected": -3.0620574951171875,
+      "logps/chosen": -0.8845365643501282,
+      "logps/rejected": -1.0314432382583618,
+      "loss": 0.5215,
+      "nll_loss": 0.5018130540847778,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04422682151198387,
+      "rewards/margins": 0.0073453388176858425,
+      "rewards/rejected": -0.05157216265797615,
+      "step": 560
+    },
+    {
+      "epoch": 0.5977975878342947,
+      "grad_norm": 1.902474576616517,
+      "learning_rate": 8.37707816583391e-06,
+      "log_odds_chosen": 0.157462477684021,
+      "log_odds_ratio": -0.7165660858154297,
+      "logits/chosen": -2.971592903137207,
+      "logits/rejected": -2.9932913780212402,
+      "logps/chosen": -0.8898121118545532,
+      "logps/rejected": -0.9948716163635254,
+      "loss": 0.5041,
+      "nll_loss": 0.5276492834091187,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.044490598142147064,
+      "rewards/margins": 0.005252980627119541,
+      "rewards/rejected": -0.04974358528852463,
+      "step": 570
+    },
+    {
+      "epoch": 0.6082852648138437,
+      "grad_norm": 1.9526588876095308,
+      "learning_rate": 8.304547985373997e-06,
+      "log_odds_chosen": 0.27767136693000793,
+      "log_odds_ratio": -0.6578360199928284,
+      "logits/chosen": -3.0485613346099854,
+      "logits/rejected": -3.061281204223633,
+      "logps/chosen": -0.8733240962028503,
+      "logps/rejected": -1.0594861507415771,
+      "loss": 0.5456,
+      "nll_loss": 0.48286086320877075,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.043666206300258636,
+      "rewards/margins": 0.009308096952736378,
+      "rewards/rejected": -0.05297430604696274,
+      "step": 580
+    },
+    {
+      "epoch": 0.6187729417933928,
+      "grad_norm": 1.963515177379308,
+      "learning_rate": 8.233869695926184e-06,
+      "log_odds_chosen": 0.32016056776046753,
+      "log_odds_ratio": -0.6649240255355835,
+      "logits/chosen": -3.0834898948669434,
+      "logits/rejected": -3.123967409133911,
+      "logps/chosen": -0.8281318545341492,
+      "logps/rejected": -1.021436095237732,
+      "loss": 0.5124,
+      "nll_loss": 0.5498961210250854,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04140659421682358,
+      "rewards/margins": 0.009665210731327534,
+      "rewards/rejected": -0.05107180029153824,
+      "step": 590
+    },
+    {
+      "epoch": 0.6292606187729418,
+      "grad_norm": 2.1416673571833584,
+      "learning_rate": 8.164965809277262e-06,
+      "log_odds_chosen": 0.3141978085041046,
+      "log_odds_ratio": -0.6486893892288208,
+      "logits/chosen": -3.1147074699401855,
+      "logits/rejected": -3.11454176902771,
+      "logps/chosen": -0.8215556144714355,
+      "logps/rejected": -1.009476661682129,
+      "loss": 0.5144,
+      "nll_loss": 0.4836875796318054,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04107777774333954,
+      "rewards/margins": 0.009396053850650787,
+      "rewards/rejected": -0.05047383904457092,
+      "step": 600
+    },
+    {
+      "epoch": 0.6397482957524908,
+      "grad_norm": 2.03894912155955,
+      "learning_rate": 8.097763301789162e-06,
+      "log_odds_chosen": 0.1958848237991333,
+      "log_odds_ratio": -0.6933802366256714,
+      "logits/chosen": -3.016098737716675,
+      "logits/rejected": -3.046642780303955,
+      "logps/chosen": -0.8733209371566772,
+      "logps/rejected": -0.9883171916007996,
+      "loss": 0.526,
+      "nll_loss": 0.4880569875240326,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0436660535633564,
+      "rewards/margins": 0.005749809555709362,
+      "rewards/rejected": -0.049415864050388336,
+      "step": 610
+    },
+    {
+      "epoch": 0.6502359727320398,
+      "grad_norm": 2.068974001178546,
+      "learning_rate": 8.03219328902499e-06,
+      "log_odds_chosen": 0.17991718649864197,
+      "log_odds_ratio": -0.7055822610855103,
+      "logits/chosen": -3.045403003692627,
+      "logits/rejected": -3.0644798278808594,
+      "logps/chosen": -0.8806620836257935,
+      "logps/rejected": -1.0145095586776733,
+      "loss": 0.5295,
+      "nll_loss": 0.5151625275611877,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.04403311014175415,
+      "rewards/margins": 0.006692370865494013,
+      "rewards/rejected": -0.05072547867894173,
+      "step": 620
+    },
+    {
+      "epoch": 0.6607236497115889,
+      "grad_norm": 1.9705491215328443,
+      "learning_rate": 7.968190728895958e-06,
+      "log_odds_chosen": 0.23948292434215546,
+      "log_odds_ratio": -0.6947344541549683,
+      "logits/chosen": -3.016519546508789,
+      "logits/rejected": -3.042133331298828,
+      "logps/chosen": -0.8557758331298828,
+      "logps/rejected": -1.0029237270355225,
+      "loss": 0.5331,
+      "nll_loss": 0.5245988368988037,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0427887924015522,
+      "rewards/margins": 0.007357400842010975,
+      "rewards/rejected": -0.0501461923122406,
+      "step": 630
+    },
+    {
+      "epoch": 0.6712113266911379,
+      "grad_norm": 2.664256522681278,
+      "learning_rate": 7.905694150420949e-06,
+      "log_odds_chosen": 0.3717094659805298,
+      "log_odds_ratio": -0.6480633020401001,
+      "logits/chosen": -3.0543761253356934,
+      "logits/rejected": -3.0751733779907227,
+      "logps/chosen": -0.8645519018173218,
+      "logps/rejected": -1.102386713027954,
+      "loss": 0.5149,
+      "nll_loss": 0.46133953332901,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.04322759807109833,
+      "rewards/margins": 0.011891739442944527,
+      "rewards/rejected": -0.0551193431019783,
+      "step": 640
+    },
+    {
+      "epoch": 0.6816990036706869,
+      "grad_norm": 1.878621524799117,
+      "learning_rate": 7.844645405527363e-06,
+      "log_odds_chosen": 0.1861819326877594,
+      "log_odds_ratio": -0.7022497057914734,
+      "logits/chosen": -3.0863146781921387,
+      "logits/rejected": -3.113098621368408,
+      "logps/chosen": -0.8403372764587402,
+      "logps/rejected": -0.9548438191413879,
+      "loss": 0.5336,
+      "nll_loss": 0.5122831463813782,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.04201686754822731,
+      "rewards/margins": 0.0057253288105130196,
+      "rewards/rejected": -0.047742195427417755,
+      "step": 650
+    },
+    {
+      "epoch": 0.6921866806502359,
+      "grad_norm": 1.8977100039056058,
+      "learning_rate": 7.78498944161523e-06,
+      "log_odds_chosen": 0.2854728400707245,
+      "log_odds_ratio": -0.6552462577819824,
+      "logits/chosen": -3.052263021469116,
+      "logits/rejected": -3.0898962020874023,
+      "logps/chosen": -0.8826674222946167,
+      "logps/rejected": -1.0711818933486938,
+      "loss": 0.5304,
+      "nll_loss": 0.4874996542930603,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.044133372604846954,
+      "rewards/margins": 0.009425725787878036,
+      "rewards/rejected": -0.05355909466743469,
+      "step": 660
+    },
+    {
+      "epoch": 0.702674357629785,
+      "grad_norm": 1.8195731091765575,
+      "learning_rate": 7.726674092862559e-06,
+      "log_odds_chosen": 0.4364054203033447,
+      "log_odds_ratio": -0.6321254968643188,
+      "logits/chosen": -2.9931445121765137,
+      "logits/rejected": -3.025317907333374,
+      "logps/chosen": -0.8416171073913574,
+      "logps/rejected": -1.1292223930358887,
+      "loss": 0.5237,
+      "nll_loss": 0.46936100721359253,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.04208085685968399,
+      "rewards/margins": 0.014380265958607197,
+      "rewards/rejected": -0.05646112561225891,
+      "step": 670
+    },
+    {
+      "epoch": 0.713162034609334,
+      "grad_norm": 2.0599075037830192,
+      "learning_rate": 7.669649888473705e-06,
+      "log_odds_chosen": 0.31395241618156433,
+      "log_odds_ratio": -0.650139570236206,
+      "logits/chosen": -2.9855525493621826,
+      "logits/rejected": -2.9897267818450928,
+      "logps/chosen": -0.8750125169754028,
+      "logps/rejected": -1.0669299364089966,
+      "loss": 0.5075,
+      "nll_loss": 0.4943002760410309,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04375062882900238,
+      "rewards/margins": 0.009595867246389389,
+      "rewards/rejected": -0.05334649235010147,
+      "step": 680
+    },
+    {
+      "epoch": 0.723649711588883,
+      "grad_norm": 1.8347271674417223,
+      "learning_rate": 7.61386987626881e-06,
+      "log_odds_chosen": 0.18291696906089783,
+      "log_odds_ratio": -0.7239105701446533,
+      "logits/chosen": -2.97595477104187,
+      "logits/rejected": -2.991725444793701,
+      "logps/chosen": -0.8641953468322754,
+      "logps/rejected": -0.9991108179092407,
+      "loss": 0.5304,
+      "nll_loss": 0.5499680638313293,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.04320976510643959,
+      "rewards/margins": 0.006745772901922464,
+      "rewards/rejected": -0.04995553940534592,
+      "step": 690
+    },
+    {
+      "epoch": 0.7341373885684321,
+      "grad_norm": 2.2852704943230915,
+      "learning_rate": 7.559289460184545e-06,
+      "log_odds_chosen": 0.3105728030204773,
+      "log_odds_ratio": -0.6319602727890015,
+      "logits/chosen": -2.985989809036255,
+      "logits/rejected": -3.0209579467773438,
+      "logps/chosen": -0.8320032358169556,
+      "logps/rejected": -1.0303562879562378,
+      "loss": 0.5296,
+      "nll_loss": 0.5422422885894775,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04160016402602196,
+      "rewards/margins": 0.009917653165757656,
+      "rewards/rejected": -0.05151782184839249,
+      "step": 700
+    },
+    {
+      "epoch": 0.7446250655479811,
+      "grad_norm": 1.9768197452256755,
+      "learning_rate": 7.505866250408016e-06,
+      "log_odds_chosen": 0.2948063015937805,
+      "log_odds_ratio": -0.6451742649078369,
+      "logits/chosen": -3.1170597076416016,
+      "logits/rejected": -3.136089324951172,
+      "logps/chosen": -0.8415013551712036,
+      "logps/rejected": -1.0454984903335571,
+      "loss": 0.5237,
+      "nll_loss": 0.47949719429016113,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04207506403326988,
+      "rewards/margins": 0.01019985694438219,
+      "rewards/rejected": -0.0522749237716198,
+      "step": 710
+    },
+    {
+      "epoch": 0.7551127425275301,
+      "grad_norm": 1.905599119477425,
+      "learning_rate": 7.4535599249993e-06,
+      "log_odds_chosen": 0.40306347608566284,
+      "log_odds_ratio": -0.6352882385253906,
+      "logits/chosen": -3.064483642578125,
+      "logits/rejected": -3.087808847427368,
+      "logps/chosen": -0.7972971200942993,
+      "logps/rejected": -1.046507477760315,
+      "loss": 0.5304,
+      "nll_loss": 0.4636651873588562,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.039864856749773026,
+      "rewards/margins": 0.012460513040423393,
+      "rewards/rejected": -0.05232536792755127,
+      "step": 720
+    },
+    {
+      "epoch": 0.7656004195070791,
+      "grad_norm": 2.19124615484763,
+      "learning_rate": 7.402332101976053e-06,
+      "log_odds_chosen": 0.12367966026067734,
+      "log_odds_ratio": -0.7226089239120483,
+      "logits/chosen": -3.0835583209991455,
+      "logits/rejected": -3.0826332569122314,
+      "logps/chosen": -0.8365408778190613,
+      "logps/rejected": -0.9029885530471802,
+      "loss": 0.5374,
+      "nll_loss": 0.5031268000602722,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.04182704538106918,
+      "rewards/margins": 0.0033223754726350307,
+      "rewards/rejected": -0.04514942690730095,
+      "step": 730
+    },
+    {
+      "epoch": 0.7760880964866282,
+      "grad_norm": 2.0835998895674837,
+      "learning_rate": 7.352146220938079e-06,
+      "log_odds_chosen": 0.33691075444221497,
+      "log_odds_ratio": -0.6264201402664185,
+      "logits/chosen": -3.1278512477874756,
+      "logits/rejected": -3.139995574951172,
+      "logps/chosen": -0.8067742586135864,
+      "logps/rejected": -1.0221493244171143,
+      "loss": 0.5312,
+      "nll_loss": 0.4790155291557312,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.040338706225156784,
+      "rewards/margins": 0.010768752545118332,
+      "rewards/rejected": -0.051107458770275116,
+      "step": 740
+    },
+    {
+      "epoch": 0.7865757734661772,
+      "grad_norm": 1.9667031119071154,
+      "learning_rate": 7.3029674334022146e-06,
+      "log_odds_chosen": 0.23670358955860138,
+      "log_odds_ratio": -0.6752098202705383,
+      "logits/chosen": -3.1056113243103027,
+      "logits/rejected": -3.1298460960388184,
+      "logps/chosen": -0.8614869117736816,
+      "logps/rejected": -0.9949930310249329,
+      "loss": 0.5426,
+      "nll_loss": 0.4975660443305969,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0430743470788002,
+      "rewards/margins": 0.006675302051007748,
+      "rewards/rejected": -0.049749650061130524,
+      "step": 750
+    },
+    {
+      "epoch": 0.7970634504457262,
+      "grad_norm": 1.8638714551633075,
+      "learning_rate": 7.254762501100117e-06,
+      "log_odds_chosen": 0.2394195795059204,
+      "log_odds_ratio": -0.6686865091323853,
+      "logits/chosen": -3.092322826385498,
+      "logits/rejected": -3.0998446941375732,
+      "logps/chosen": -0.8189753293991089,
+      "logps/rejected": -0.9735254049301147,
+      "loss": 0.5115,
+      "nll_loss": 0.4049908220767975,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.0409487709403038,
+      "rewards/margins": 0.007727508433163166,
+      "rewards/rejected": -0.048676274716854095,
+      "step": 760
+    },
+    {
+      "epoch": 0.8075511274252754,
+      "grad_norm": 2.098087236150393,
+      "learning_rate": 7.207499701564472e-06,
+      "log_odds_chosen": 0.21572642028331757,
+      "log_odds_ratio": -0.7029857635498047,
+      "logits/chosen": -3.0059127807617188,
+      "logits/rejected": -3.0258781909942627,
+      "logps/chosen": -0.8941653370857239,
+      "logps/rejected": -1.0438942909240723,
+      "loss": 0.5343,
+      "nll_loss": 0.5011810064315796,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.044708263128995895,
+      "rewards/margins": 0.007486448623239994,
+      "rewards/rejected": -0.05219471454620361,
+      "step": 770
+    },
+    {
+      "epoch": 0.8180388044048243,
+      "grad_norm": 1.908201970451478,
+      "learning_rate": 7.1611487403943295e-06,
+      "log_odds_chosen": 0.22588184475898743,
+      "log_odds_ratio": -0.6703106164932251,
+      "logits/chosen": -3.0057101249694824,
+      "logits/rejected": -3.0319108963012695,
+      "logps/chosen": -0.8802768588066101,
+      "logps/rejected": -0.997613787651062,
+      "loss": 0.5466,
+      "nll_loss": 0.5490036606788635,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0440138503909111,
+      "rewards/margins": 0.005866837687790394,
+      "rewards/rejected": -0.04988069087266922,
+      "step": 780
+    },
+    {
+      "epoch": 0.8285264813843733,
+      "grad_norm": 1.8452821315553456,
+      "learning_rate": 7.115680669648201e-06,
+      "log_odds_chosen": 0.32251420617103577,
+      "log_odds_ratio": -0.6489396691322327,
+      "logits/chosen": -2.991415500640869,
+      "logits/rejected": -3.0075478553771973,
+      "logps/chosen": -0.8143788576126099,
+      "logps/rejected": -1.0171436071395874,
+      "loss": 0.5052,
+      "nll_loss": 0.4423222541809082,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.04071894288063049,
+      "rewards/margins": 0.010138243436813354,
+      "rewards/rejected": -0.05085718631744385,
+      "step": 790
+    },
+    {
+      "epoch": 0.8390141583639223,
+      "grad_norm": 2.099723593564682,
+      "learning_rate": 7.0710678118654756e-06,
+      "log_odds_chosen": 0.4498319625854492,
+      "log_odds_ratio": -0.5986544489860535,
+      "logits/chosen": -2.9999208450317383,
+      "logits/rejected": -2.9963490962982178,
+      "logps/chosen": -0.782555341720581,
+      "logps/rejected": -1.068285584449768,
+      "loss": 0.5173,
+      "nll_loss": 0.4201901853084564,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.03912776708602905,
+      "rewards/margins": 0.014286505989730358,
+      "rewards/rejected": -0.053414274007081985,
+      "step": 800
+    },
+    {
+      "epoch": 0.8495018353434715,
+      "grad_norm": 1.9010573028789273,
+      "learning_rate": 7.027283689263066e-06,
+      "log_odds_chosen": 0.34422335028648376,
+      "log_odds_ratio": -0.6322020292282104,
+      "logits/chosen": -3.0011842250823975,
+      "logits/rejected": -2.9966137409210205,
+      "logps/chosen": -0.8086786270141602,
+      "logps/rejected": -1.0155996084213257,
+      "loss": 0.5132,
+      "nll_loss": 0.4740920066833496,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.040433935821056366,
+      "rewards/margins": 0.010346042923629284,
+      "rewards/rejected": -0.05077998712658882,
+      "step": 810
+    },
+    {
+      "epoch": 0.8599895123230205,
+      "grad_norm": 2.3144073315770353,
+      "learning_rate": 6.984302957695783e-06,
+      "log_odds_chosen": 0.29515784978866577,
+      "log_odds_ratio": -0.6521409749984741,
+      "logits/chosen": -2.943692445755005,
+      "logits/rejected": -2.9414219856262207,
+      "logps/chosen": -0.8414862751960754,
+      "logps/rejected": -1.0143965482711792,
+      "loss": 0.504,
+      "nll_loss": 0.4271189570426941,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04207431524991989,
+      "rewards/margins": 0.008645516820251942,
+      "rewards/rejected": -0.05071982741355896,
+      "step": 820
+    },
+    {
+      "epoch": 0.8704771893025695,
+      "grad_norm": 2.371001107698096,
+      "learning_rate": 6.942101345006233e-06,
+      "log_odds_chosen": 0.2455742061138153,
+      "log_odds_ratio": -0.7013689279556274,
+      "logits/chosen": -2.933568239212036,
+      "logits/rejected": -2.977832794189453,
+      "logps/chosen": -0.8553229570388794,
+      "logps/rejected": -1.0332233905792236,
+      "loss": 0.5251,
+      "nll_loss": 0.46586036682128906,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.04276614636182785,
+      "rewards/margins": 0.008895025588572025,
+      "rewards/rejected": -0.0516611710190773,
+      "step": 830
+    },
+    {
+      "epoch": 0.8809648662821186,
+      "grad_norm": 1.977587507180873,
+      "learning_rate": 6.900655593423542e-06,
+      "log_odds_chosen": 0.19387319684028625,
+      "log_odds_ratio": -0.6939007639884949,
+      "logits/chosen": -2.9483094215393066,
+      "logits/rejected": -2.966421365737915,
+      "logps/chosen": -0.8696029782295227,
+      "logps/rejected": -1.0034617185592651,
+      "loss": 0.5136,
+      "nll_loss": 0.48451894521713257,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.043480150401592255,
+      "rewards/margins": 0.006692938506603241,
+      "rewards/rejected": -0.050173092633485794,
+      "step": 840
+    },
+    {
+      "epoch": 0.8914525432616676,
+      "grad_norm": 2.0931872980265527,
+      "learning_rate": 6.859943405700353e-06,
+      "log_odds_chosen": 0.27469760179519653,
+      "log_odds_ratio": -0.6496983170509338,
+      "logits/chosen": -2.882544994354248,
+      "logits/rejected": -2.907102584838867,
+      "logps/chosen": -0.8309645652770996,
+      "logps/rejected": -0.9983605146408081,
+      "loss": 0.5054,
+      "nll_loss": 0.4892002046108246,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0415482297539711,
+      "rewards/margins": 0.008369805291295052,
+      "rewards/rejected": -0.049918033182621,
+      "step": 850
+    },
+    {
+      "epoch": 0.9019402202412166,
+      "grad_norm": 1.9059523373512675,
+      "learning_rate": 6.819943394704736e-06,
+      "log_odds_chosen": 0.2372780740261078,
+      "log_odds_ratio": -0.6811105012893677,
+      "logits/chosen": -2.9579243659973145,
+      "logits/rejected": -2.9706907272338867,
+      "logps/chosen": -0.8282278180122375,
+      "logps/rejected": -0.982342541217804,
+      "loss": 0.5277,
+      "nll_loss": 0.4725598692893982,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.0414113886654377,
+      "rewards/margins": 0.007705743424594402,
+      "rewards/rejected": -0.049117133021354675,
+      "step": 860
+    },
+    {
+      "epoch": 0.9124278972207656,
+      "grad_norm": 1.892543797666968,
+      "learning_rate": 6.780635036208105e-06,
+      "log_odds_chosen": 0.287548691034317,
+      "log_odds_ratio": -0.6644268035888672,
+      "logits/chosen": -3.0049710273742676,
+      "logits/rejected": -3.0431902408599854,
+      "logps/chosen": -0.8620280027389526,
+      "logps/rejected": -1.0551369190216064,
+      "loss": 0.4935,
+      "nll_loss": 0.4828346371650696,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.04310140386223793,
+      "rewards/margins": 0.009655444882810116,
+      "rewards/rejected": -0.05275684595108032,
+      "step": 870
+    },
+    {
+      "epoch": 0.9229155742003147,
+      "grad_norm": 1.6128728864363475,
+      "learning_rate": 6.741998624632421e-06,
+      "log_odds_chosen": 0.2844703197479248,
+      "log_odds_ratio": -0.6617631316184998,
+      "logits/chosen": -3.044353723526001,
+      "logits/rejected": -3.0480034351348877,
+      "logps/chosen": -0.808245837688446,
+      "logps/rejected": -0.990073561668396,
+      "loss": 0.4881,
+      "nll_loss": 0.43747878074645996,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0404122956097126,
+      "rewards/margins": 0.009091392159461975,
+      "rewards/rejected": -0.04950368404388428,
+      "step": 880
+    },
+    {
+      "epoch": 0.9334032511798637,
+      "grad_norm": 2.329046484142618,
+      "learning_rate": 6.70401523153991e-06,
+      "log_odds_chosen": 0.32051050662994385,
+      "log_odds_ratio": -0.6461818218231201,
+      "logits/chosen": -3.0071539878845215,
+      "logits/rejected": -3.0232186317443848,
+      "logps/chosen": -0.8105939030647278,
+      "logps/rejected": -0.993729293346405,
+      "loss": 0.4935,
+      "nll_loss": 0.46434158086776733,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.04052969440817833,
+      "rewards/margins": 0.00915677472949028,
+      "rewards/rejected": -0.04968646913766861,
+      "step": 890
+    },
+    {
+      "epoch": 0.9438909281594127,
+      "grad_norm": 2.0086740635642073,
+      "learning_rate": 6.666666666666667e-06,
+      "log_odds_chosen": 0.2798821032047272,
+      "log_odds_ratio": -0.664302408695221,
+      "logits/chosen": -2.9259209632873535,
+      "logits/rejected": -2.9381814002990723,
+      "logps/chosen": -0.7818757891654968,
+      "logps/rejected": -0.9571603536605835,
+      "loss": 0.5239,
+      "nll_loss": 0.4661863446235657,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.03909378498792648,
+      "rewards/margins": 0.008764232508838177,
+      "rewards/rejected": -0.04785802215337753,
+      "step": 900
+    },
+    {
+      "epoch": 0.9543786051389617,
+      "grad_norm": 2.068822950454407,
+      "learning_rate": 6.629935441317959e-06,
+      "log_odds_chosen": 0.479647159576416,
+      "log_odds_ratio": -0.6314842700958252,
+      "logits/chosen": -2.974902629852295,
+      "logits/rejected": -2.9787256717681885,
+      "logps/chosen": -0.8285977244377136,
+      "logps/rejected": -1.1534996032714844,
+      "loss": 0.5142,
+      "nll_loss": 0.46572408080101013,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04142988473176956,
+      "rewards/margins": 0.016245096921920776,
+      "rewards/rejected": -0.05767498165369034,
+      "step": 910
+    },
+    {
+      "epoch": 0.9648662821185108,
+      "grad_norm": 1.9606527520032064,
+      "learning_rate": 6.593804733957872e-06,
+      "log_odds_chosen": 0.3219223618507385,
+      "log_odds_ratio": -0.649006187915802,
+      "logits/chosen": -2.895038604736328,
+      "logits/rejected": -2.9138269424438477,
+      "logps/chosen": -0.7895429134368896,
+      "logps/rejected": -0.9961126446723938,
+      "loss": 0.4837,
+      "nll_loss": 0.43109196424484253,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0394771471619606,
+      "rewards/margins": 0.010328484699130058,
+      "rewards/rejected": -0.04980562627315521,
+      "step": 920
+    },
+    {
+      "epoch": 0.9753539590980598,
+      "grad_norm": 2.2191050074705405,
+      "learning_rate": 6.55825835783953e-06,
+      "log_odds_chosen": 0.21952304244041443,
+      "log_odds_ratio": -0.6805615425109863,
+      "logits/chosen": -2.8973617553710938,
+      "logits/rejected": -2.900251865386963,
+      "logps/chosen": -0.8730388879776001,
+      "logps/rejected": -1.0255097150802612,
+      "loss": 0.5135,
+      "nll_loss": 0.5237925052642822,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04365193843841553,
+      "rewards/margins": 0.007623549550771713,
+      "rewards/rejected": -0.05127548426389694,
+      "step": 930
+    },
+    {
+      "epoch": 0.9858416360776088,
+      "grad_norm": 1.9816052115352747,
+      "learning_rate": 6.523280730534423e-06,
+      "log_odds_chosen": 0.2554723024368286,
+      "log_odds_ratio": -0.6887288689613342,
+      "logits/chosen": -2.93623685836792,
+      "logits/rejected": -2.9283607006073,
+      "logps/chosen": -0.7786284685134888,
+      "logps/rejected": -0.9273189306259155,
+      "loss": 0.5095,
+      "nll_loss": 0.4773116111755371,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.03893141821026802,
+      "rewards/margins": 0.007434530649334192,
+      "rewards/rejected": -0.046365950256586075,
+      "step": 940
+    },
+    {
+      "epoch": 0.9963293130571579,
+      "grad_norm": 2.074452011854083,
+      "learning_rate": 6.488856845230502e-06,
+      "log_odds_chosen": 0.2605803310871124,
+      "log_odds_ratio": -0.6914502382278442,
+      "logits/chosen": -2.9090209007263184,
+      "logits/rejected": -2.9163012504577637,
+      "logps/chosen": -0.8585780262947083,
+      "logps/rejected": -1.0175925493240356,
+      "loss": 0.5383,
+      "nll_loss": 0.503527045249939,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.04292890429496765,
+      "rewards/margins": 0.007950720377266407,
+      "rewards/rejected": -0.050879620015621185,
+      "step": 950
+    },
+    {
+      "epoch": 0.9994756161510225,
+      "step": 953,
+      "total_flos": 0.0,
+      "train_loss": 0.56347276506494,
+      "train_runtime": 19079.6454,
+      "train_samples_per_second": 3.197,
+      "train_steps_per_second": 0.05
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 953,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bf035813fd07976aa0220cc580abeb9837a63f23c3331acf22c16aa3d9e2647
+size 6648