Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: KL_shift-zephyr-7b-sft-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# KL_shift-zephyr-7b-sft-full
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 6
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 48
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.41.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.9191367263135206,
+    "train_runtime": 14436.6556,
+    "train_samples": 61135,
+    "train_samples_per_second": 4.235,
+    "train_steps_per_second": 0.088
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.2",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.41.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59d0c0e5a6f0316be761f523666c7be13d574dd79d08dc3d057c5fa39fe6098a
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:123f8605977eb983d376e7360ea585c91d084ee01748f316dcb76d8148bec1b0
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c1aa1958e870a4d6c5df621638a2ad7814775c7af80f14bdf150c5aa6600e31
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.9191367263135206,
+    "train_runtime": 14436.6556,
+    "train_samples": 61135,
+    "train_samples_per_second": 4.235,
+    "train_steps_per_second": 0.088
+}

trainer_state.json

@@ -0,0 +1,1962 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1274,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0007849293563579278,
+      "grad_norm": 15.217127460584273,
+      "learning_rate": 3.90625e-09,
+      "logits/chosen": 5881.4375,
+      "logits/rejected": 2834.66162109375,
+      "logps/chosen": -257.5969543457031,
+      "logps/rejected": -120.09489440917969,
+      "loss": 1.0,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.007849293563579277,
+      "grad_norm": 16.776894624882384,
+      "learning_rate": 3.9062499999999997e-08,
+      "logits/chosen": 4945.97705078125,
+      "logits/rejected": 4332.49072265625,
+      "logps/chosen": -248.80799865722656,
+      "logps/rejected": -214.25498962402344,
+      "loss": 1.0,
+      "rewards/accuracies": 0.46296295523643494,
+      "rewards/chosen": 0.008178697898983955,
+      "rewards/margins": -0.02892056480050087,
+      "rewards/rejected": 0.037099260836839676,
+      "step": 10
+    },
+    {
+      "epoch": 0.015698587127158554,
+      "grad_norm": 13.99592115814451,
+      "learning_rate": 7.812499999999999e-08,
+      "logits/chosen": 6023.15771484375,
+      "logits/rejected": 4842.72216796875,
+      "logps/chosen": -266.2255859375,
+      "logps/rejected": -232.41552734375,
+      "loss": 1.0001,
+      "rewards/accuracies": 0.5416666269302368,
+      "rewards/chosen": 0.08771798014640808,
+      "rewards/margins": 0.051151834428310394,
+      "rewards/rejected": 0.03656615689396858,
+      "step": 20
+    },
+    {
+      "epoch": 0.023547880690737835,
+      "grad_norm": 13.81414691081162,
+      "learning_rate": 1.1718749999999999e-07,
+      "logits/chosen": 6032.67578125,
+      "logits/rejected": 5038.6357421875,
+      "logps/chosen": -310.4226989746094,
+      "logps/rejected": -262.90899658203125,
+      "loss": 0.9997,
+      "rewards/accuracies": 0.5000000596046448,
+      "rewards/chosen": 0.18663737177848816,
+      "rewards/margins": 0.0590377077460289,
+      "rewards/rejected": 0.12759965658187866,
+      "step": 30
+    },
+    {
+      "epoch": 0.03139717425431711,
+      "grad_norm": 14.509114343975037,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": 5242.150390625,
+      "logits/rejected": 4343.94189453125,
+      "logps/chosen": -263.48541259765625,
+      "logps/rejected": -224.81298828125,
+      "loss": 0.9982,
+      "rewards/accuracies": 0.6583333611488342,
+      "rewards/chosen": 0.5348619222640991,
+      "rewards/margins": 0.2838565707206726,
+      "rewards/rejected": 0.2510053813457489,
+      "step": 40
+    },
+    {
+      "epoch": 0.03924646781789639,
+      "grad_norm": 15.528115845986795,
+      "learning_rate": 1.9531249999999998e-07,
+      "logits/chosen": 6364.38134765625,
+      "logits/rejected": 4975.55712890625,
+      "logps/chosen": -324.6205749511719,
+      "logps/rejected": -258.08489990234375,
+      "loss": 0.9964,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 1.489025354385376,
+      "rewards/margins": 1.024138331413269,
+      "rewards/rejected": 0.46488696336746216,
+      "step": 50
+    },
+    {
+      "epoch": 0.04709576138147567,
+      "grad_norm": 15.841524244504972,
+      "learning_rate": 2.3437499999999998e-07,
+      "logits/chosen": 5383.0732421875,
+      "logits/rejected": 4504.74755859375,
+      "logps/chosen": -268.93963623046875,
+      "logps/rejected": -262.5002136230469,
+      "loss": 0.9928,
+      "rewards/accuracies": 0.6583333611488342,
+      "rewards/chosen": 0.9874664545059204,
+      "rewards/margins": 1.2968491315841675,
+      "rewards/rejected": -0.30938273668289185,
+      "step": 60
+    },
+    {
+      "epoch": 0.054945054945054944,
+      "grad_norm": 12.412319893385746,
+      "learning_rate": 2.734375e-07,
+      "logits/chosen": 5106.55615234375,
+      "logits/rejected": 4840.5986328125,
+      "logps/chosen": -232.88815307617188,
+      "logps/rejected": -234.45919799804688,
+      "loss": 0.9892,
+      "rewards/accuracies": 0.6583333611488342,
+      "rewards/chosen": 0.3698170781135559,
+      "rewards/margins": 2.176013708114624,
+      "rewards/rejected": -1.8061968088150024,
+      "step": 70
+    },
+    {
+      "epoch": 0.06279434850863422,
+      "grad_norm": 12.80231150764336,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": 5693.59423828125,
+      "logits/rejected": 5163.55810546875,
+      "logps/chosen": -255.7566375732422,
+      "logps/rejected": -245.6920166015625,
+      "loss": 0.9821,
+      "rewards/accuracies": 0.6583333611488342,
+      "rewards/chosen": 0.22178272902965546,
+      "rewards/margins": 3.601436138153076,
+      "rewards/rejected": -3.379653215408325,
+      "step": 80
+    },
+    {
+      "epoch": 0.0706436420722135,
+      "grad_norm": 14.016716726890994,
+      "learning_rate": 3.5156249999999997e-07,
+      "logits/chosen": 5872.6611328125,
+      "logits/rejected": 5053.22900390625,
+      "logps/chosen": -282.4061584472656,
+      "logps/rejected": -248.04946899414062,
+      "loss": 0.9734,
+      "rewards/accuracies": 0.7416666746139526,
+      "rewards/chosen": 2.7849347591400146,
+      "rewards/margins": 7.367936611175537,
+      "rewards/rejected": -4.583001136779785,
+      "step": 90
+    },
+    {
+      "epoch": 0.07849293563579278,
+      "grad_norm": 14.456274236246015,
+      "learning_rate": 3.9062499999999997e-07,
+      "logits/chosen": 5763.5947265625,
+      "logits/rejected": 5547.8154296875,
+      "logps/chosen": -277.15655517578125,
+      "logps/rejected": -276.28717041015625,
+      "loss": 0.9719,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 1.1185591220855713,
+      "rewards/margins": 9.217432022094727,
+      "rewards/rejected": -8.098873138427734,
+      "step": 100
+    },
+    {
+      "epoch": 0.08634222919937205,
+      "grad_norm": 15.965712184122198,
+      "learning_rate": 4.2968749999999996e-07,
+      "logits/chosen": 6198.5576171875,
+      "logits/rejected": 5092.955078125,
+      "logps/chosen": -256.78546142578125,
+      "logps/rejected": -254.78933715820312,
+      "loss": 0.9584,
+      "rewards/accuracies": 0.7166667580604553,
+      "rewards/chosen": 0.352399080991745,
+      "rewards/margins": 11.528807640075684,
+      "rewards/rejected": -11.176408767700195,
+      "step": 110
+    },
+    {
+      "epoch": 0.09419152276295134,
+      "grad_norm": 20.370249209275958,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": 6045.43408203125,
+      "logits/rejected": 5060.17822265625,
+      "logps/chosen": -277.0526428222656,
+      "logps/rejected": -283.88458251953125,
+      "loss": 0.9454,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.5342214107513428,
+      "rewards/margins": 16.816547393798828,
+      "rewards/rejected": -14.282328605651855,
+      "step": 120
+    },
+    {
+      "epoch": 0.10204081632653061,
+      "grad_norm": 14.630467472384158,
+      "learning_rate": 4.999962424962166e-07,
+      "logits/chosen": 6011.037109375,
+      "logits/rejected": 5550.65185546875,
+      "logps/chosen": -289.7455749511719,
+      "logps/rejected": -285.1080627441406,
+      "loss": 0.9568,
+      "rewards/accuracies": 0.658333420753479,
+      "rewards/chosen": 0.3616622984409332,
+      "rewards/margins": 12.038993835449219,
+      "rewards/rejected": -11.677331924438477,
+      "step": 130
+    },
+    {
+      "epoch": 0.10989010989010989,
+      "grad_norm": 17.081001015208503,
+      "learning_rate": 4.998647417232375e-07,
+      "logits/chosen": 5865.60693359375,
+      "logits/rejected": 5142.06005859375,
+      "logps/chosen": -261.2616882324219,
+      "logps/rejected": -265.9956359863281,
+      "loss": 0.9585,
+      "rewards/accuracies": 0.6583333015441895,
+      "rewards/chosen": -1.0194756984710693,
+      "rewards/margins": 14.62031364440918,
+      "rewards/rejected": -15.639788627624512,
+      "step": 140
+    },
+    {
+      "epoch": 0.11773940345368916,
+      "grad_norm": 28.23881440069633,
+      "learning_rate": 4.995454786965036e-07,
+      "logits/chosen": 5954.14501953125,
+      "logits/rejected": 4962.21630859375,
+      "logps/chosen": -278.65301513671875,
+      "logps/rejected": -264.0430908203125,
+      "loss": 0.9532,
+      "rewards/accuracies": 0.6833333969116211,
+      "rewards/chosen": -0.7810039520263672,
+      "rewards/margins": 21.372682571411133,
+      "rewards/rejected": -22.153682708740234,
+      "step": 150
+    },
+    {
+      "epoch": 0.12558869701726844,
+      "grad_norm": 17.275009395044158,
+      "learning_rate": 4.990386933279972e-07,
+      "logits/chosen": 5890.1767578125,
+      "logits/rejected": 5239.57373046875,
+      "logps/chosen": -276.5395812988281,
+      "logps/rejected": -283.18988037109375,
+      "loss": 0.9486,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": -2.9753592014312744,
+      "rewards/margins": 14.08137321472168,
+      "rewards/rejected": -17.056734085083008,
+      "step": 160
+    },
+    {
+      "epoch": 0.13343799058084774,
+      "grad_norm": 19.7776157223108,
+      "learning_rate": 4.983447664444096e-07,
+      "logits/chosen": 6131.412109375,
+      "logits/rejected": 5464.201171875,
+      "logps/chosen": -292.5121765136719,
+      "logps/rejected": -288.33782958984375,
+      "loss": 0.947,
+      "rewards/accuracies": 0.658333420753479,
+      "rewards/chosen": -2.714707136154175,
+      "rewards/margins": 16.617053985595703,
+      "rewards/rejected": -19.33176040649414,
+      "step": 170
+    },
+    {
+      "epoch": 0.141287284144427,
+      "grad_norm": 16.27120763569239,
+      "learning_rate": 4.97464219500968e-07,
+      "logits/chosen": 5405.88671875,
+      "logits/rejected": 4714.19677734375,
+      "logps/chosen": -262.1574401855469,
+      "logps/rejected": -261.7540283203125,
+      "loss": 0.9482,
+      "rewards/accuracies": 0.6416666507720947,
+      "rewards/chosen": 1.2868962287902832,
+      "rewards/margins": 15.773916244506836,
+      "rewards/rejected": -14.487022399902344,
+      "step": 180
+    },
+    {
+      "epoch": 0.14913657770800628,
+      "grad_norm": 19.668914928097266,
+      "learning_rate": 4.963977141895843e-07,
+      "logits/chosen": 5469.93505859375,
+      "logits/rejected": 4702.2255859375,
+      "logps/chosen": -289.6636047363281,
+      "logps/rejected": -291.77496337890625,
+      "loss": 0.9296,
+      "rewards/accuracies": 0.7499999403953552,
+      "rewards/chosen": -2.378950595855713,
+      "rewards/margins": 27.93460464477539,
+      "rewards/rejected": -30.313556671142578,
+      "step": 190
+    },
+    {
+      "epoch": 0.15698587127158556,
+      "grad_norm": 20.842473477340224,
+      "learning_rate": 4.951460519416227e-07,
+      "logits/chosen": 5406.935546875,
+      "logits/rejected": 4991.7724609375,
+      "logps/chosen": -249.13339233398438,
+      "logps/rejected": -282.2489013671875,
+      "loss": 0.9367,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 2.116595983505249,
+      "rewards/margins": 20.891857147216797,
+      "rewards/rejected": -18.775264739990234,
+      "step": 200
+    },
+    {
+      "epoch": 0.16483516483516483,
+      "grad_norm": 33.15523072801943,
+      "learning_rate": 4.937101733256606e-07,
+      "logits/chosen": 4901.9677734375,
+      "logits/rejected": 4344.95947265625,
+      "logps/chosen": -224.97476196289062,
+      "logps/rejected": -243.6951446533203,
+      "loss": 0.9347,
+      "rewards/accuracies": 0.6916666030883789,
+      "rewards/chosen": -0.2618327736854553,
+      "rewards/margins": 20.972537994384766,
+      "rewards/rejected": -21.2343692779541,
+      "step": 210
+    },
+    {
+      "epoch": 0.1726844583987441,
+      "grad_norm": 21.17945225425017,
+      "learning_rate": 4.920911573406924e-07,
+      "logits/chosen": 5983.14404296875,
+      "logits/rejected": 5075.22900390625,
+      "logps/chosen": -287.1895751953125,
+      "logps/rejected": -258.6339416503906,
+      "loss": 0.939,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -4.556641101837158,
+      "rewards/margins": 20.536806106567383,
+      "rewards/rejected": -25.09344482421875,
+      "step": 220
+    },
+    {
+      "epoch": 0.18053375196232338,
+      "grad_norm": 15.05328515456915,
+      "learning_rate": 4.902902206053098e-07,
+      "logits/chosen": 5476.01611328125,
+      "logits/rejected": 4954.21630859375,
+      "logps/chosen": -268.1181945800781,
+      "logps/rejected": -276.8870544433594,
+      "loss": 0.9401,
+      "rewards/accuracies": 0.6583333611488342,
+      "rewards/chosen": 2.1590466499328613,
+      "rewards/margins": 21.023910522460938,
+      "rewards/rejected": -18.864864349365234,
+      "step": 230
+    },
+    {
+      "epoch": 0.18838304552590268,
+      "grad_norm": 18.01659982755688,
+      "learning_rate": 4.883087164434672e-07,
+      "logits/chosen": 4976.4541015625,
+      "logits/rejected": 3982.62255859375,
+      "logps/chosen": -239.6964111328125,
+      "logps/rejected": -237.77529907226562,
+      "loss": 0.9329,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": -0.002855014754459262,
+      "rewards/margins": 24.846181869506836,
+      "rewards/rejected": -24.849035263061523,
+      "step": 240
+    },
+    {
+      "epoch": 0.19623233908948196,
+      "grad_norm": 19.701799187597075,
+      "learning_rate": 4.861481338675183e-07,
+      "logits/chosen": 5823.4638671875,
+      "logits/rejected": 5177.93896484375,
+      "logps/chosen": -241.93338012695312,
+      "logps/rejected": -285.5468444824219,
+      "loss": 0.9424,
+      "rewards/accuracies": 0.6833333373069763,
+      "rewards/chosen": -6.285933017730713,
+      "rewards/margins": 24.868818283081055,
+      "rewards/rejected": -31.15475082397461,
+      "step": 250
+    },
+    {
+      "epoch": 0.20408163265306123,
+      "grad_norm": 715.9229177986646,
+      "learning_rate": 4.838100964592904e-07,
+      "logits/chosen": 5995.43701171875,
+      "logits/rejected": 4819.33056640625,
+      "logps/chosen": -283.37677001953125,
+      "logps/rejected": -258.4510192871094,
+      "loss": 0.991,
+      "rewards/accuracies": 0.7416667342185974,
+      "rewards/chosen": 1.375902771949768,
+      "rewards/margins": 21.302745819091797,
+      "rewards/rejected": -19.92684555053711,
+      "step": 260
+    },
+    {
+      "epoch": 0.2119309262166405,
+      "grad_norm": 30.214986071208994,
+      "learning_rate": 4.812963611500339e-07,
+      "logits/chosen": 5903.2216796875,
+      "logits/rejected": 5689.96435546875,
+      "logps/chosen": -275.84600830078125,
+      "logps/rejected": -287.48284912109375,
+      "loss": 0.9176,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.337425708770752,
+      "rewards/margins": 19.31707763671875,
+      "rewards/rejected": -20.654502868652344,
+      "step": 270
+    },
+    {
+      "epoch": 0.21978021978021978,
+      "grad_norm": 21.071420758528312,
+      "learning_rate": 4.786088169001671e-07,
+      "logits/chosen": 5051.4794921875,
+      "logits/rejected": 4381.0185546875,
+      "logps/chosen": -228.6282196044922,
+      "logps/rejected": -264.83636474609375,
+      "loss": 0.9491,
+      "rewards/accuracies": 0.7583333253860474,
+      "rewards/chosen": 0.08709268271923065,
+      "rewards/margins": 30.51346778869629,
+      "rewards/rejected": -30.426376342773438,
+      "step": 280
+    },
+    {
+      "epoch": 0.22762951334379905,
+      "grad_norm": 20.756738551757568,
+      "learning_rate": 4.7574948327980567e-07,
+      "logits/chosen": 6978.69384765625,
+      "logits/rejected": 5147.8818359375,
+      "logps/chosen": -318.50439453125,
+      "logps/rejected": -284.42498779296875,
+      "loss": 0.9255,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": 6.286134243011475,
+      "rewards/margins": 32.94474411010742,
+      "rewards/rejected": -26.65860939025879,
+      "step": 290
+    },
+    {
+      "epoch": 0.23547880690737832,
+      "grad_norm": 24.77524846092873,
+      "learning_rate": 4.727205089511466e-07,
+      "logits/chosen": 5091.29833984375,
+      "logits/rejected": 5012.20361328125,
+      "logps/chosen": -240.96395874023438,
+      "logps/rejected": -272.696533203125,
+      "loss": 0.9404,
+      "rewards/accuracies": 0.6583333611488342,
+      "rewards/chosen": -4.223406791687012,
+      "rewards/margins": 24.546117782592773,
+      "rewards/rejected": -28.769527435302734,
+      "step": 300
+    },
+    {
+      "epoch": 0.24332810047095763,
+      "grad_norm": 17.46534329970633,
+      "learning_rate": 4.6952417005384247e-07,
+      "logits/chosen": 5712.3994140625,
+      "logits/rejected": 5113.927734375,
+      "logps/chosen": -251.79891967773438,
+      "logps/rejected": -256.73321533203125,
+      "loss": 0.9389,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": 3.8696587085723877,
+      "rewards/margins": 18.93717384338379,
+      "rewards/rejected": -15.06751537322998,
+      "step": 310
+    },
+    {
+      "epoch": 0.25117739403453687,
+      "grad_norm": 23.489605989743538,
+      "learning_rate": 4.661628684945851e-07,
+      "logits/chosen": 5749.4189453125,
+      "logits/rejected": 4976.5224609375,
+      "logps/chosen": -272.5289001464844,
+      "logps/rejected": -300.0706481933594,
+      "loss": 0.9306,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 4.758330345153809,
+      "rewards/margins": 32.90387725830078,
+      "rewards/rejected": -28.145549774169922,
+      "step": 320
+    },
+    {
+      "epoch": 0.25902668759811615,
+      "grad_norm": 27.939618074872772,
+      "learning_rate": 4.626391301421782e-07,
+      "logits/chosen": 5560.0283203125,
+      "logits/rejected": 5054.7841796875,
+      "logps/chosen": -270.5875549316406,
+      "logps/rejected": -273.18206787109375,
+      "loss": 0.9383,
+      "rewards/accuracies": 0.6833333969116211,
+      "rewards/chosen": -0.4894745945930481,
+      "rewards/margins": 23.300487518310547,
+      "rewards/rejected": -23.789962768554688,
+      "step": 330
+    },
+    {
+      "epoch": 0.2668759811616955,
+      "grad_norm": 23.064661533697098,
+      "learning_rate": 4.5895560292945996e-07,
+      "logits/chosen": 5838.73681640625,
+      "logits/rejected": 5922.8486328125,
+      "logps/chosen": -264.6275329589844,
+      "logps/rejected": -323.1180725097656,
+      "loss": 0.9292,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": 3.4555885791778564,
+      "rewards/margins": 24.69220542907715,
+      "rewards/rejected": -21.236614227294922,
+      "step": 340
+    },
+    {
+      "epoch": 0.27472527472527475,
+      "grad_norm": 26.7902596796826,
+      "learning_rate": 4.5511505486349865e-07,
+      "logits/chosen": 6143.82421875,
+      "logits/rejected": 5557.6533203125,
+      "logps/chosen": -267.94775390625,
+      "logps/rejected": -314.7910461425781,
+      "loss": 0.9275,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": 4.959996223449707,
+      "rewards/margins": 36.25249481201172,
+      "rewards/rejected": -31.29250144958496,
+      "step": 350
+    },
+    {
+      "epoch": 0.282574568288854,
+      "grad_norm": 22.401607581910273,
+      "learning_rate": 4.5112037194555876e-07,
+      "logits/chosen": 5670.1220703125,
+      "logits/rejected": 5525.7197265625,
+      "logps/chosen": -243.54159545898438,
+      "logps/rejected": -309.3414001464844,
+      "loss": 0.9155,
+      "rewards/accuracies": 0.7500000596046448,
+      "rewards/chosen": 0.14337244629859924,
+      "rewards/margins": 35.511756896972656,
+      "rewards/rejected": -35.36838150024414,
+      "step": 360
+    },
+    {
+      "epoch": 0.2904238618524333,
+      "grad_norm": 25.673874629204995,
+      "learning_rate": 4.4697455600239863e-07,
+      "logits/chosen": 5165.1650390625,
+      "logits/rejected": 4833.2763671875,
+      "logps/chosen": -255.64859008789062,
+      "logps/rejected": -257.18560791015625,
+      "loss": 0.942,
+      "rewards/accuracies": 0.7166666388511658,
+      "rewards/chosen": 3.253772735595703,
+      "rewards/margins": 22.0217342376709,
+      "rewards/rejected": -18.767959594726562,
+      "step": 370
+    },
+    {
+      "epoch": 0.29827315541601257,
+      "grad_norm": 16.59204251418475,
+      "learning_rate": 4.426807224305315e-07,
+      "logits/chosen": 6208.30322265625,
+      "logits/rejected": 5111.9501953125,
+      "logps/chosen": -301.24530029296875,
+      "logps/rejected": -280.9383850097656,
+      "loss": 0.9236,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": 6.731601715087891,
+      "rewards/margins": 31.761554718017578,
+      "rewards/rejected": -25.029956817626953,
+      "step": 380
+    },
+    {
+      "epoch": 0.30612244897959184,
+      "grad_norm": 18.775548887189014,
+      "learning_rate": 4.3824209785514326e-07,
+      "logits/chosen": 6327.4404296875,
+      "logits/rejected": 4821.05029296875,
+      "logps/chosen": -285.92803955078125,
+      "logps/rejected": -288.6047058105469,
+      "loss": 0.9076,
+      "rewards/accuracies": 0.7916667461395264,
+      "rewards/chosen": 5.598370552062988,
+      "rewards/margins": 48.711387634277344,
+      "rewards/rejected": -43.113014221191406,
+      "step": 390
+    },
+    {
+      "epoch": 0.3139717425431711,
+      "grad_norm": 18.55961719941118,
+      "learning_rate": 4.3366201770542687e-07,
+      "logits/chosen": 5379.22021484375,
+      "logits/rejected": 5216.08642578125,
+      "logps/chosen": -281.50048828125,
+      "logps/rejected": -329.05059814453125,
+      "loss": 0.9521,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -6.883020877838135,
+      "rewards/margins": 35.55707931518555,
+      "rewards/rejected": -42.440101623535156,
+      "step": 400
+    },
+    {
+      "epoch": 0.3218210361067504,
+      "grad_norm": 27.884929639357004,
+      "learning_rate": 4.2894392370815567e-07,
+      "logits/chosen": 5830.39111328125,
+      "logits/rejected": 5225.48681640625,
+      "logps/chosen": -288.3466491699219,
+      "logps/rejected": -321.122314453125,
+      "loss": 0.8869,
+      "rewards/accuracies": 0.7833333611488342,
+      "rewards/chosen": 3.522181749343872,
+      "rewards/margins": 42.019920349121094,
+      "rewards/rejected": -38.49774169921875,
+      "step": 410
+    },
+    {
+      "epoch": 0.32967032967032966,
+      "grad_norm": 20.26058977803537,
+      "learning_rate": 4.2409136130137845e-07,
+      "logits/chosen": 5500.486328125,
+      "logits/rejected": 4993.22412109375,
+      "logps/chosen": -286.73968505859375,
+      "logps/rejected": -286.5626525878906,
+      "loss": 0.9178,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8464605212211609,
+      "rewards/margins": 32.63377380371094,
+      "rewards/rejected": -33.4802360534668,
+      "step": 420
+    },
+    {
+      "epoch": 0.33751962323390894,
+      "grad_norm": 23.259074401678372,
+      "learning_rate": 4.1910797697018017e-07,
+      "logits/chosen": 5319.9033203125,
+      "logits/rejected": 4466.1845703125,
+      "logps/chosen": -253.0988006591797,
+      "logps/rejected": -259.62548828125,
+      "loss": 0.9059,
+      "rewards/accuracies": 0.7750000953674316,
+      "rewards/chosen": 3.696092128753662,
+      "rewards/margins": 38.31728744506836,
+      "rewards/rejected": -34.621192932128906,
+      "step": 430
+    },
+    {
+      "epoch": 0.3453689167974882,
+      "grad_norm": 23.635617682728697,
+      "learning_rate": 4.1399751550651084e-07,
+      "logits/chosen": 5703.9736328125,
+      "logits/rejected": 5649.138671875,
+      "logps/chosen": -251.727783203125,
+      "logps/rejected": -292.34222412109375,
+      "loss": 0.9191,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": -0.9597722887992859,
+      "rewards/margins": 31.166919708251953,
+      "rewards/rejected": -32.12669372558594,
+      "step": 440
+    },
+    {
+      "epoch": 0.3532182103610675,
+      "grad_norm": 19.26341973190091,
+      "learning_rate": 4.087638171951401e-07,
+      "logits/chosen": 6566.53125,
+      "logits/rejected": 4723.1767578125,
+      "logps/chosen": -286.6239318847656,
+      "logps/rejected": -269.2875671386719,
+      "loss": 0.9225,
+      "rewards/accuracies": 0.7500000596046448,
+      "rewards/chosen": 0.43297329545021057,
+      "rewards/margins": 43.72136306762695,
+      "rewards/rejected": -43.28838348388672,
+      "step": 450
+    },
+    {
+      "epoch": 0.36106750392464676,
+      "grad_norm": 24.15018784175595,
+      "learning_rate": 4.034108149278543e-07,
+      "logits/chosen": 6809.2626953125,
+      "logits/rejected": 5260.38818359375,
+      "logps/chosen": -328.55633544921875,
+      "logps/rejected": -295.267578125,
+      "loss": 0.9139,
+      "rewards/accuracies": 0.8166667222976685,
+      "rewards/chosen": 4.154780864715576,
+      "rewards/margins": 34.77088928222656,
+      "rewards/rejected": -30.616104125976562,
+      "step": 460
+    },
+    {
+      "epoch": 0.36891679748822603,
+      "grad_norm": 19.930362113206062,
+      "learning_rate": 3.979425312480629e-07,
+      "logits/chosen": 5824.78857421875,
+      "logits/rejected": 5113.4462890625,
+      "logps/chosen": -287.6418762207031,
+      "logps/rejected": -312.6920471191406,
+      "loss": 0.916,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.1438978612422943,
+      "rewards/margins": 34.73716735839844,
+      "rewards/rejected": -34.5932731628418,
+      "step": 470
+    },
+    {
+      "epoch": 0.37676609105180536,
+      "grad_norm": 17.674222833195227,
+      "learning_rate": 3.923630753280357e-07,
+      "logits/chosen": 6285.4208984375,
+      "logits/rejected": 5395.0048828125,
+      "logps/chosen": -281.23046875,
+      "logps/rejected": -289.922119140625,
+      "loss": 0.91,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.4238802194595337,
+      "rewards/margins": 41.704959869384766,
+      "rewards/rejected": -43.128841400146484,
+      "step": 480
+    },
+    {
+      "epoch": 0.38461538461538464,
+      "grad_norm": 19.792313593032937,
+      "learning_rate": 3.866766398810424e-07,
+      "logits/chosen": 5899.3759765625,
+      "logits/rejected": 5611.10400390625,
+      "logps/chosen": -232.70321655273438,
+      "logps/rejected": -309.1105651855469,
+      "loss": 0.8918,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 8.237863540649414,
+      "rewards/margins": 44.7490348815918,
+      "rewards/rejected": -36.511173248291016,
+      "step": 490
+    },
+    {
+      "epoch": 0.3924646781789639,
+      "grad_norm": 19.90809886153986,
+      "learning_rate": 3.8088749801071496e-07,
+      "logits/chosen": 6366.142578125,
+      "logits/rejected": 4952.5068359375,
+      "logps/chosen": -303.6125793457031,
+      "logps/rejected": -313.29315185546875,
+      "loss": 0.912,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -5.123102188110352,
+      "rewards/margins": 43.737327575683594,
+      "rewards/rejected": -48.86042785644531,
+      "step": 500
+    },
+    {
+      "epoch": 0.4003139717425432,
+      "grad_norm": 15.360327222439452,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": 5147.7861328125,
+      "logits/rejected": 4536.16455078125,
+      "logps/chosen": -252.94473266601562,
+      "logps/rejected": -273.7191162109375,
+      "loss": 0.911,
+      "rewards/accuracies": 0.7500001192092896,
+      "rewards/chosen": 3.915085554122925,
+      "rewards/margins": 48.51652526855469,
+      "rewards/rejected": -44.601436614990234,
+      "step": 510
+    },
+    {
+      "epoch": 0.40816326530612246,
+      "grad_norm": 22.726045624861126,
+      "learning_rate": 3.6901857004211443e-07,
+      "logits/chosen": 5440.7919921875,
+      "logits/rejected": 5028.6923828125,
+      "logps/chosen": -270.27691650390625,
+      "logps/rejected": -308.1585693359375,
+      "loss": 0.9365,
+      "rewards/accuracies": 0.6833332777023315,
+      "rewards/chosen": -4.1040358543396,
+      "rewards/margins": 34.928199768066406,
+      "rewards/rejected": -39.03223419189453,
+      "step": 520
+    },
+    {
+      "epoch": 0.41601255886970173,
+      "grad_norm": 22.1271600593506,
+      "learning_rate": 3.6294770291596076e-07,
+      "logits/chosen": 6135.3037109375,
+      "logits/rejected": 5045.927734375,
+      "logps/chosen": -278.6271057128906,
+      "logps/rejected": -283.05340576171875,
+      "loss": 0.9054,
+      "rewards/accuracies": 0.7666667699813843,
+      "rewards/chosen": 5.665005683898926,
+      "rewards/margins": 24.647602081298828,
+      "rewards/rejected": -18.98259925842285,
+      "step": 530
+    },
+    {
+      "epoch": 0.423861852433281,
+      "grad_norm": 41.55555763430692,
+      "learning_rate": 3.5679196060850034e-07,
+      "logits/chosen": 5851.7666015625,
+      "logits/rejected": 5219.24609375,
+      "logps/chosen": -277.853271484375,
+      "logps/rejected": -295.34912109375,
+      "loss": 0.9161,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.46962302923202515,
+      "rewards/margins": 39.48213577270508,
+      "rewards/rejected": -39.951759338378906,
+      "step": 540
+    },
+    {
+      "epoch": 0.4317111459968603,
+      "grad_norm": 20.618281611990163,
+      "learning_rate": 3.505559688866229e-07,
+      "logits/chosen": 5663.544921875,
+      "logits/rejected": 5269.53955078125,
+      "logps/chosen": -283.8811340332031,
+      "logps/rejected": -334.2888488769531,
+      "loss": 0.9078,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": -2.544511556625366,
+      "rewards/margins": 40.56104278564453,
+      "rewards/rejected": -43.10555648803711,
+      "step": 550
+    },
+    {
+      "epoch": 0.43956043956043955,
+      "grad_norm": 19.391598225192343,
+      "learning_rate": 3.4424441382108826e-07,
+      "logits/chosen": 5656.9873046875,
+      "logits/rejected": 5293.77392578125,
+      "logps/chosen": -279.07427978515625,
+      "logps/rejected": -302.99371337890625,
+      "loss": 0.935,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.946714162826538,
+      "rewards/margins": 33.268821716308594,
+      "rewards/rejected": -36.215538024902344,
+      "step": 560
+    },
+    {
+      "epoch": 0.4474097331240188,
+      "grad_norm": 27.492150331098347,
+      "learning_rate": 3.378620382651523e-07,
+      "logits/chosen": 6007.7392578125,
+      "logits/rejected": 5555.06689453125,
+      "logps/chosen": -315.08062744140625,
+      "logps/rejected": -331.8717956542969,
+      "loss": 0.9104,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.5156325101852417,
+      "rewards/margins": 39.41523742675781,
+      "rewards/rejected": -39.930870056152344,
+      "step": 570
+    },
+    {
+      "epoch": 0.4552590266875981,
+      "grad_norm": 24.81865808929877,
+      "learning_rate": 3.314136382905234e-07,
+      "logits/chosen": 5973.07958984375,
+      "logits/rejected": 5394.1357421875,
+      "logps/chosen": -284.0478515625,
+      "logps/rejected": -333.41473388671875,
+      "loss": 0.9205,
+      "rewards/accuracies": 0.7916666865348816,
+      "rewards/chosen": 3.1989758014678955,
+      "rewards/margins": 51.7633056640625,
+      "rewards/rejected": -48.564327239990234,
+      "step": 580
+    },
+    {
+      "epoch": 0.4631083202511774,
+      "grad_norm": 25.774824989224687,
+      "learning_rate": 3.249040595833274e-07,
+      "logits/chosen": 6467.61328125,
+      "logits/rejected": 5461.263671875,
+      "logps/chosen": -303.5272216796875,
+      "logps/rejected": -285.9855041503906,
+      "loss": 0.9112,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": 2.6262047290802,
+      "rewards/margins": 42.771522521972656,
+      "rewards/rejected": -40.14531707763672,
+      "step": 590
+    },
+    {
+      "epoch": 0.47095761381475665,
+      "grad_norm": 27.67394815870794,
+      "learning_rate": 3.1833819380279023e-07,
+      "logits/chosen": 6110.16455078125,
+      "logits/rejected": 5210.61865234375,
+      "logps/chosen": -247.8615264892578,
+      "logps/rejected": -286.87371826171875,
+      "loss": 0.9008,
+      "rewards/accuracies": 0.8083332777023315,
+      "rewards/chosen": 4.629504203796387,
+      "rewards/margins": 36.539493560791016,
+      "rewards/rejected": -31.909988403320312,
+      "step": 600
+    },
+    {
+      "epoch": 0.478806907378336,
+      "grad_norm": 20.718470452802894,
+      "learning_rate": 3.11720974905373e-07,
+      "logits/chosen": 5875.37109375,
+      "logits/rejected": 5125.77099609375,
+      "logps/chosen": -269.7516784667969,
+      "logps/rejected": -293.9992370605469,
+      "loss": 0.8922,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": 3.0898051261901855,
+      "rewards/margins": 46.59734344482422,
+      "rewards/rejected": -43.50753402709961,
+      "step": 610
+    },
+    {
+      "epoch": 0.48665620094191525,
+      "grad_norm": 20.359862853723154,
+      "learning_rate": 3.0505737543712275e-07,
+      "logits/chosen": 4997.51416015625,
+      "logits/rejected": 4120.1025390625,
+      "logps/chosen": -248.70364379882812,
+      "logps/rejected": -253.5068817138672,
+      "loss": 0.9011,
+      "rewards/accuracies": 0.7833333015441895,
+      "rewards/chosen": -0.8777009844779968,
+      "rewards/margins": 39.05192947387695,
+      "rewards/rejected": -39.929630279541016,
+      "step": 620
+    },
+    {
+      "epoch": 0.4945054945054945,
+      "grad_norm": 24.410845424528254,
+      "learning_rate": 2.9835240279702513e-07,
+      "logits/chosen": 6503.3544921875,
+      "logits/rejected": 5521.38134765625,
+      "logps/chosen": -311.4924621582031,
+      "logps/rejected": -310.57275390625,
+      "loss": 0.8996,
+      "rewards/accuracies": 0.7583333849906921,
+      "rewards/chosen": 5.219082832336426,
+      "rewards/margins": 49.75696563720703,
+      "rewards/rejected": -44.53789138793945,
+      "step": 630
+    },
+    {
+      "epoch": 0.5023547880690737,
+      "grad_norm": 20.26453021100001,
+      "learning_rate": 2.9161109547416667e-07,
+      "logits/chosen": 6199.2998046875,
+      "logits/rejected": 5321.87744140625,
+      "logps/chosen": -277.9691162109375,
+      "logps/rejected": -303.0850524902344,
+      "loss": 0.9023,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.7213143110275269,
+      "rewards/margins": 25.9655704498291,
+      "rewards/rejected": -25.24425506591797,
+      "step": 640
+    },
+    {
+      "epoch": 0.5102040816326531,
+      "grad_norm": 25.607967393691553,
+      "learning_rate": 2.848385192615339e-07,
+      "logits/chosen": 5318.2783203125,
+      "logits/rejected": 4340.18603515625,
+      "logps/chosen": -256.1814270019531,
+      "logps/rejected": -262.76361083984375,
+      "loss": 0.9022,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": 1.3570995330810547,
+      "rewards/margins": 37.01885223388672,
+      "rewards/rejected": -35.66175842285156,
+      "step": 650
+    },
+    {
+      "epoch": 0.5180533751962323,
+      "grad_norm": 28.57773283940273,
+      "learning_rate": 2.780397634492949e-07,
+      "logits/chosen": 5933.3408203125,
+      "logits/rejected": 4759.22509765625,
+      "logps/chosen": -289.2817077636719,
+      "logps/rejected": -310.09234619140625,
+      "loss": 0.9164,
+      "rewards/accuracies": 0.8166667222976685,
+      "rewards/chosen": 1.0365993976593018,
+      "rewards/margins": 58.85808181762695,
+      "rewards/rejected": -57.82147979736328,
+      "step": 660
+    },
+    {
+      "epoch": 0.5259026687598116,
+      "grad_norm": 23.773604822565275,
+      "learning_rate": 2.71219937000424e-07,
+      "logits/chosen": 5936.36865234375,
+      "logits/rejected": 4900.83056640625,
+      "logps/chosen": -274.3346862792969,
+      "logps/rejected": -285.41607666015625,
+      "loss": 0.9161,
+      "rewards/accuracies": 0.7166666388511658,
+      "rewards/chosen": 1.8758052587509155,
+      "rewards/margins": 36.20291519165039,
+      "rewards/rejected": -34.327110290527344,
+      "step": 670
+    },
+    {
+      "epoch": 0.533751962323391,
+      "grad_norm": 23.006395031938606,
+      "learning_rate": 2.6438416471154273e-07,
+      "logits/chosen": 5816.591796875,
+      "logits/rejected": 4883.8837890625,
+      "logps/chosen": -274.1582946777344,
+      "logps/rejected": -274.8831481933594,
+      "loss": 0.9049,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 3.6695594787597656,
+      "rewards/margins": 43.54711151123047,
+      "rewards/rejected": -39.87754821777344,
+      "step": 680
+    },
+    {
+      "epoch": 0.5416012558869702,
+      "grad_norm": 26.525872068105123,
+      "learning_rate": 2.5753758336186326e-07,
+      "logits/chosen": 5734.0380859375,
+      "logits/rejected": 5293.50341796875,
+      "logps/chosen": -276.3390197753906,
+      "logps/rejected": -324.45611572265625,
+      "loss": 0.8992,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2499253749847412,
+      "rewards/margins": 45.74732971191406,
+      "rewards/rejected": -46.997257232666016,
+      "step": 690
+    },
+    {
+      "epoch": 0.5494505494505495,
+      "grad_norm": 26.541363934518053,
+      "learning_rate": 2.5068533785312666e-07,
+      "logits/chosen": 5495.10595703125,
+      "logits/rejected": 5287.18896484375,
+      "logps/chosen": -251.01681518554688,
+      "logps/rejected": -289.4714660644531,
+      "loss": 0.9318,
+      "rewards/accuracies": 0.7166666388511658,
+      "rewards/chosen": -0.9005786776542664,
+      "rewards/margins": 43.11927032470703,
+      "rewards/rejected": -44.01984405517578,
+      "step": 700
+    },
+    {
+      "epoch": 0.5572998430141287,
+      "grad_norm": 24.46002490835382,
+      "learning_rate": 2.4383257734343794e-07,
+      "logits/chosen": 5464.05712890625,
+      "logits/rejected": 5455.001953125,
+      "logps/chosen": -250.55587768554688,
+      "logps/rejected": -300.8583679199219,
+      "loss": 0.9064,
+      "rewards/accuracies": 0.6916667222976685,
+      "rewards/chosen": -1.3100314140319824,
+      "rewards/margins": 35.45912551879883,
+      "rewards/rejected": -36.76915740966797,
+      "step": 710
+    },
+    {
+      "epoch": 0.565149136577708,
+      "grad_norm": 21.147359750230834,
+      "learning_rate": 2.3698445137790258e-07,
+      "logits/chosen": 5864.6064453125,
+      "logits/rejected": 5073.22607421875,
+      "logps/chosen": -277.67425537109375,
+      "logps/rejected": -293.50439453125,
+      "loss": 0.9079,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 7.0904951095581055,
+      "rewards/margins": 42.4764289855957,
+      "rewards/rejected": -35.385929107666016,
+      "step": 720
+    },
+    {
+      "epoch": 0.5729984301412873,
+      "grad_norm": 22.688465462752504,
+      "learning_rate": 2.3014610601897157e-07,
+      "logits/chosen": 6338.6025390625,
+      "logits/rejected": 4882.97021484375,
+      "logps/chosen": -295.96942138671875,
+      "logps/rejected": -287.09844970703125,
+      "loss": 0.8906,
+      "rewards/accuracies": 0.7416667342185974,
+      "rewards/chosen": -0.9103342294692993,
+      "rewards/margins": 45.903358459472656,
+      "rewards/rejected": -46.81369400024414,
+      "step": 730
+    },
+    {
+      "epoch": 0.5808477237048666,
+      "grad_norm": 24.937006724806338,
+      "learning_rate": 2.2332267997940513e-07,
+      "logits/chosen": 5288.89111328125,
+      "logits/rejected": 4503.5400390625,
+      "logps/chosen": -243.6168975830078,
+      "logps/rejected": -258.78863525390625,
+      "loss": 0.8925,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 4.391143321990967,
+      "rewards/margins": 45.62685775756836,
+      "rewards/rejected": -41.2357177734375,
+      "step": 740
+    },
+    {
+      "epoch": 0.5886970172684458,
+      "grad_norm": 25.087199062301373,
+      "learning_rate": 2.1651930076075723e-07,
+      "logits/chosen": 5745.19140625,
+      "logits/rejected": 5214.74365234375,
+      "logps/chosen": -238.671875,
+      "logps/rejected": -253.86264038085938,
+      "loss": 0.9096,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": 1.5903170108795166,
+      "rewards/margins": 32.90300750732422,
+      "rewards/rejected": -31.31268882751465,
+      "step": 750
+    },
+    {
+      "epoch": 0.5965463108320251,
+      "grad_norm": 22.6004674707504,
+      "learning_rate": 2.0974108080028692e-07,
+      "logits/chosen": 6041.4853515625,
+      "logits/rejected": 4772.58740234375,
+      "logps/chosen": -259.30780029296875,
+      "logps/rejected": -272.39886474609375,
+      "loss": 0.9056,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": 4.05280065536499,
+      "rewards/margins": 38.531856536865234,
+      "rewards/rejected": -34.47905731201172,
+      "step": 760
+    },
+    {
+      "epoch": 0.6043956043956044,
+      "grad_norm": 32.07154020812047,
+      "learning_rate": 2.0299311362918773e-07,
+      "logits/chosen": 6262.0263671875,
+      "logits/rejected": 5383.724609375,
+      "logps/chosen": -292.2076110839844,
+      "logps/rejected": -327.34906005859375,
+      "loss": 0.9143,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": 2.43733549118042,
+      "rewards/margins": 44.98815155029297,
+      "rewards/rejected": -42.550819396972656,
+      "step": 770
+    },
+    {
+      "epoch": 0.6122448979591837,
+      "grad_norm": 23.504239493179618,
+      "learning_rate": 1.962804700450265e-07,
+      "logits/chosen": 6072.6123046875,
+      "logits/rejected": 5769.93896484375,
+      "logps/chosen": -277.5558776855469,
+      "logps/rejected": -344.66473388671875,
+      "loss": 0.9245,
+      "rewards/accuracies": 0.7583333849906921,
+      "rewards/chosen": 5.5767011642456055,
+      "rewards/margins": 35.678287506103516,
+      "rewards/rejected": -30.10158348083496,
+      "step": 780
+    },
+    {
+      "epoch": 0.6200941915227629,
+      "grad_norm": 23.391365758751537,
+      "learning_rate": 1.8960819430126334e-07,
+      "logits/chosen": 5679.32666015625,
+      "logits/rejected": 5026.5771484375,
+      "logps/chosen": -254.663330078125,
+      "logps/rejected": -304.33038330078125,
+      "loss": 0.8986,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.2119230329990387,
+      "rewards/margins": 48.772735595703125,
+      "rewards/rejected": -48.98466110229492,
+      "step": 790
+    },
+    {
+      "epoch": 0.6279434850863422,
+      "grad_norm": 24.446664309226747,
+      "learning_rate": 1.8298130031671972e-07,
+      "logits/chosen": 5691.87255859375,
+      "logits/rejected": 4992.9775390625,
+      "logps/chosen": -277.36175537109375,
+      "logps/rejected": -307.2684020996094,
+      "loss": 0.9219,
+      "rewards/accuracies": 0.7583333253860474,
+      "rewards/chosen": -3.6252803802490234,
+      "rewards/margins": 37.125953674316406,
+      "rewards/rejected": -40.75123596191406,
+      "step": 800
+    },
+    {
+      "epoch": 0.6357927786499215,
+      "grad_norm": 24.78783701861456,
+      "learning_rate": 1.7640476790784075e-07,
+      "logits/chosen": 5191.9833984375,
+      "logits/rejected": 4733.341796875,
+      "logps/chosen": -273.46575927734375,
+      "logps/rejected": -316.88641357421875,
+      "loss": 0.9032,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.3641335070133209,
+      "rewards/margins": 43.47520446777344,
+      "rewards/rejected": -43.11106872558594,
+      "step": 810
+    },
+    {
+      "epoch": 0.6436420722135008,
+      "grad_norm": 22.35233438036302,
+      "learning_rate": 1.6988353904658492e-07,
+      "logits/chosen": 5694.0908203125,
+      "logits/rejected": 4441.0947265625,
+      "logps/chosen": -278.25262451171875,
+      "logps/rejected": -253.1114044189453,
+      "loss": 0.8889,
+      "rewards/accuracies": 0.8083332777023315,
+      "rewards/chosen": 4.820112228393555,
+      "rewards/margins": 42.1064453125,
+      "rewards/rejected": -37.28633499145508,
+      "step": 820
+    },
+    {
+      "epoch": 0.6514913657770801,
+      "grad_norm": 33.971535716829386,
+      "learning_rate": 1.634225141467513e-07,
+      "logits/chosen": 5630.9619140625,
+      "logits/rejected": 5059.73388671875,
+      "logps/chosen": -262.0240173339844,
+      "logps/rejected": -293.10284423828125,
+      "loss": 0.9036,
+      "rewards/accuracies": 0.7916666865348816,
+      "rewards/chosen": 1.4304834604263306,
+      "rewards/margins": 46.748069763183594,
+      "rewards/rejected": -45.317588806152344,
+      "step": 830
+    },
+    {
+      "epoch": 0.6593406593406593,
+      "grad_norm": 22.058744826701787,
+      "learning_rate": 1.570265483815364e-07,
+      "logits/chosen": 6187.8076171875,
+      "logits/rejected": 5085.29052734375,
+      "logps/chosen": -284.2825927734375,
+      "logps/rejected": -312.46990966796875,
+      "loss": 0.8997,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": 1.250636339187622,
+      "rewards/margins": 39.52533721923828,
+      "rewards/rejected": -38.27470016479492,
+      "step": 840
+    },
+    {
+      "epoch": 0.6671899529042387,
+      "grad_norm": 24.57030833624564,
+      "learning_rate": 1.5070044803508691e-07,
+      "logits/chosen": 5731.578125,
+      "logits/rejected": 5200.16650390625,
+      "logps/chosen": -273.11700439453125,
+      "logps/rejected": -295.6126403808594,
+      "loss": 0.8764,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.5748471021652222,
+      "rewards/margins": 49.07981872558594,
+      "rewards/rejected": -49.65467071533203,
+      "step": 850
+    },
+    {
+      "epoch": 0.6750392464678179,
+      "grad_norm": 29.643861873247765,
+      "learning_rate": 1.444489668907914e-07,
+      "logits/chosen": 6211.44091796875,
+      "logits/rejected": 5296.041015625,
+      "logps/chosen": -299.6576232910156,
+      "logps/rejected": -300.2407531738281,
+      "loss": 0.9124,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": 3.2107672691345215,
+      "rewards/margins": 49.146392822265625,
+      "rewards/rejected": -45.93562698364258,
+      "step": 860
+    },
+    {
+      "epoch": 0.6828885400313972,
+      "grad_norm": 19.18443294400448,
+      "learning_rate": 1.3827680265902232e-07,
+      "logits/chosen": 6146.9755859375,
+      "logits/rejected": 5104.4638671875,
+      "logps/chosen": -284.4742736816406,
+      "logps/rejected": -296.0386962890625,
+      "loss": 0.8972,
+      "rewards/accuracies": 0.7416666746139526,
+      "rewards/chosen": 4.938233375549316,
+      "rewards/margins": 46.767799377441406,
+      "rewards/rejected": -41.829566955566406,
+      "step": 870
+    },
+    {
+      "epoch": 0.6907378335949764,
+      "grad_norm": 48.40221110161389,
+      "learning_rate": 1.3218859344701632e-07,
+      "logits/chosen": 5446.45849609375,
+      "logits/rejected": 5177.0341796875,
+      "logps/chosen": -252.7174835205078,
+      "logps/rejected": -326.0343322753906,
+      "loss": 0.9097,
+      "rewards/accuracies": 0.8083333969116211,
+      "rewards/chosen": 6.035229206085205,
+      "rewards/margins": 43.76191711425781,
+      "rewards/rejected": -37.726688385009766,
+      "step": 880
+    },
+    {
+      "epoch": 0.6985871271585558,
+      "grad_norm": 27.2644482399342,
+      "learning_rate": 1.2618891427354172e-07,
+      "logits/chosen": 6387.37939453125,
+      "logits/rejected": 5229.87353515625,
+      "logps/chosen": -305.6217346191406,
+      "logps/rejected": -304.56756591796875,
+      "loss": 0.9068,
+      "rewards/accuracies": 0.7833333015441895,
+      "rewards/chosen": 6.906777858734131,
+      "rewards/margins": 49.44459915161133,
+      "rewards/rejected": -42.537818908691406,
+      "step": 890
+    },
+    {
+      "epoch": 0.706436420722135,
+      "grad_norm": 22.953604786244295,
+      "learning_rate": 1.202822736309758e-07,
+      "logits/chosen": 5421.3154296875,
+      "logits/rejected": 5001.1025390625,
+      "logps/chosen": -253.27780151367188,
+      "logps/rejected": -305.64788818359375,
+      "loss": 0.91,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 4.763638019561768,
+      "rewards/margins": 45.9899787902832,
+      "rewards/rejected": -41.226341247558594,
+      "step": 900
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 27.91344694605715,
+      "learning_rate": 1.1447311009737299e-07,
+      "logits/chosen": 5335.6162109375,
+      "logits/rejected": 5072.3359375,
+      "logps/chosen": -258.45941162109375,
+      "logps/rejected": -303.97113037109375,
+      "loss": 0.9052,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": -0.8920276761054993,
+      "rewards/margins": 45.10370635986328,
+      "rewards/rejected": -45.9957389831543,
+      "step": 910
+    },
+    {
+      "epoch": 0.7221350078492935,
+      "grad_norm": 23.26853190795506,
+      "learning_rate": 1.0876578900107053e-07,
+      "logits/chosen": 5871.88720703125,
+      "logits/rejected": 4929.97900390625,
+      "logps/chosen": -283.12799072265625,
+      "logps/rejected": -281.83013916015625,
+      "loss": 0.9041,
+      "rewards/accuracies": 0.7916666865348816,
+      "rewards/chosen": 3.5937552452087402,
+      "rewards/margins": 40.470191955566406,
+      "rewards/rejected": -36.876434326171875,
+      "step": 920
+    },
+    {
+      "epoch": 0.7299843014128728,
+      "grad_norm": 19.12047364944922,
+      "learning_rate": 1.0316459914033793e-07,
+      "logits/chosen": 5825.18115234375,
+      "logits/rejected": 4393.8466796875,
+      "logps/chosen": -289.2558898925781,
+      "logps/rejected": -272.1807861328125,
+      "loss": 0.9068,
+      "rewards/accuracies": 0.7500000596046448,
+      "rewards/chosen": -1.3023111820220947,
+      "rewards/margins": 37.931583404541016,
+      "rewards/rejected": -39.233890533447266,
+      "step": 930
+    },
+    {
+      "epoch": 0.7378335949764521,
+      "grad_norm": 20.651437164552735,
+      "learning_rate": 9.767374956053584e-08,
+      "logits/chosen": 5656.6083984375,
+      "logits/rejected": 4979.91943359375,
+      "logps/chosen": -261.6556396484375,
+      "logps/rejected": -294.62158203125,
+      "loss": 0.9014,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 3.210376024246216,
+      "rewards/margins": 50.671321868896484,
+      "rewards/rejected": -47.46094512939453,
+      "step": 940
+    },
+    {
+      "epoch": 0.7456828885400314,
+      "grad_norm": 21.44755274256472,
+      "learning_rate": 9.229736639120561e-08,
+      "logits/chosen": 5831.8720703125,
+      "logits/rejected": 5376.9619140625,
+      "logps/chosen": -261.1865539550781,
+      "logps/rejected": -289.8360900878906,
+      "loss": 0.9157,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": 2.373589277267456,
+      "rewards/margins": 29.416553497314453,
+      "rewards/rejected": -27.042964935302734,
+      "step": 950
+    },
+    {
+      "epoch": 0.7535321821036107,
+      "grad_norm": 34.123365725576235,
+      "learning_rate": 8.70394897454659e-08,
+      "logits/chosen": 5702.736328125,
+      "logits/rejected": 5105.47802734375,
+      "logps/chosen": -254.7130126953125,
+      "logps/rejected": -283.2959289550781,
+      "loss": 0.8878,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 9.300559997558594,
+      "rewards/margins": 46.2738151550293,
+      "rewards/rejected": -36.97325897216797,
+      "step": 960
+    },
+    {
+      "epoch": 0.7613814756671899,
+      "grad_norm": 30.293238410586564,
+      "learning_rate": 8.19040706840472e-08,
+      "logits/chosen": 5806.1142578125,
+      "logits/rejected": 4896.1630859375,
+      "logps/chosen": -283.5286560058594,
+      "logps/rejected": -292.33721923828125,
+      "loss": 0.885,
+      "rewards/accuracies": 0.7500000596046448,
+      "rewards/chosen": 5.602731227874756,
+      "rewards/margins": 51.605499267578125,
+      "rewards/rejected": -46.002769470214844,
+      "step": 970
+    },
+    {
+      "epoch": 0.7692307692307693,
+      "grad_norm": 23.491592765262904,
+      "learning_rate": 7.689496824624525e-08,
+      "logits/chosen": 5516.03369140625,
+      "logits/rejected": 4427.40185546875,
+      "logps/chosen": -270.09564208984375,
+      "logps/rejected": -297.0606689453125,
+      "loss": 0.8801,
+      "rewards/accuracies": 0.7916666269302368,
+      "rewards/chosen": 6.5463409423828125,
+      "rewards/margins": 64.91987609863281,
+      "rewards/rejected": -58.37353515625,
+      "step": 980
+    },
+    {
+      "epoch": 0.7770800627943485,
+      "grad_norm": 36.05397286248155,
+      "learning_rate": 7.201594655002458e-08,
+      "logits/chosen": 5858.2021484375,
+      "logits/rejected": 4895.6455078125,
+      "logps/chosen": -267.0971984863281,
+      "logps/rejected": -288.00775146484375,
+      "loss": 0.8929,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": 1.010213851928711,
+      "rewards/margins": 52.7333869934082,
+      "rewards/rejected": -51.723175048828125,
+      "step": 990
+    },
+    {
+      "epoch": 0.7849293563579278,
+      "grad_norm": 35.53234033842205,
+      "learning_rate": 6.727067196345099e-08,
+      "logits/chosen": 5515.2802734375,
+      "logits/rejected": 4669.0283203125,
+      "logps/chosen": -259.7113952636719,
+      "logps/rejected": -264.55084228515625,
+      "loss": 0.9068,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": 0.6101600527763367,
+      "rewards/margins": 39.48523712158203,
+      "rewards/rejected": -38.875083923339844,
+      "step": 1000
+    },
+    {
+      "epoch": 0.792778649921507,
+      "grad_norm": 25.616049738852727,
+      "learning_rate": 6.26627103495786e-08,
+      "logits/chosen": 5715.3486328125,
+      "logits/rejected": 4768.5458984375,
+      "logps/chosen": -254.9511260986328,
+      "logps/rejected": -275.54779052734375,
+      "loss": 0.9084,
+      "rewards/accuracies": 0.7166666388511658,
+      "rewards/chosen": 4.464234828948975,
+      "rewards/margins": 44.496917724609375,
+      "rewards/rejected": -40.032676696777344,
+      "step": 1010
+    },
+    {
+      "epoch": 0.8006279434850864,
+      "grad_norm": 22.502131941962762,
+      "learning_rate": 5.8195524386862374e-08,
+      "logits/chosen": 5797.1455078125,
+      "logits/rejected": 5151.40771484375,
+      "logps/chosen": -287.3981018066406,
+      "logps/rejected": -309.9508361816406,
+      "loss": 0.902,
+      "rewards/accuracies": 0.7916666269302368,
+      "rewards/chosen": 10.498518943786621,
+      "rewards/margins": 46.848793029785156,
+      "rewards/rejected": -36.35027313232422,
+      "step": 1020
+    },
+    {
+      "epoch": 0.8084772370486656,
+      "grad_norm": 20.045575816512876,
+      "learning_rate": 5.38724709671092e-08,
+      "logits/chosen": 6191.720703125,
+      "logits/rejected": 5863.8681640625,
+      "logps/chosen": -272.2727966308594,
+      "logps/rejected": -315.62384033203125,
+      "loss": 0.8982,
+      "rewards/accuracies": 0.7500000596046448,
+      "rewards/chosen": 5.527390480041504,
+      "rewards/margins": 46.04096603393555,
+      "rewards/rejected": -40.513572692871094,
+      "step": 1030
+    },
+    {
+      "epoch": 0.8163265306122449,
+      "grad_norm": 25.302444772287263,
+      "learning_rate": 4.969679867292276e-08,
+      "logits/chosen": 5513.0185546875,
+      "logits/rejected": 5018.1357421875,
+      "logps/chosen": -264.01837158203125,
+      "logps/rejected": -301.9166564941406,
+      "loss": 0.8943,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": 1.22486412525177,
+      "rewards/margins": 49.567771911621094,
+      "rewards/rejected": -48.34290313720703,
+      "step": 1040
+    },
+    {
+      "epoch": 0.8241758241758241,
+      "grad_norm": 26.971621245905506,
+      "learning_rate": 4.5671645336537416e-08,
+      "logits/chosen": 5564.62939453125,
+      "logits/rejected": 5052.43994140625,
+      "logps/chosen": -279.4435119628906,
+      "logps/rejected": -315.63885498046875,
+      "loss": 0.8897,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 2.089627504348755,
+      "rewards/margins": 52.004425048828125,
+      "rewards/rejected": -49.914798736572266,
+      "step": 1050
+    },
+    {
+      "epoch": 0.8320251177394035,
+      "grad_norm": 33.760753801014786,
+      "learning_rate": 4.180003568187776e-08,
+      "logits/chosen": 6866.97265625,
+      "logits/rejected": 5416.5302734375,
+      "logps/chosen": -308.72735595703125,
+      "logps/rejected": -301.6792907714844,
+      "loss": 0.9172,
+      "rewards/accuracies": 0.6833333373069763,
+      "rewards/chosen": -0.8046684265136719,
+      "rewards/margins": 36.33965301513672,
+      "rewards/rejected": -37.14432144165039,
+      "step": 1060
+    },
+    {
+      "epoch": 0.8398744113029827,
+      "grad_norm": 42.68076165248159,
+      "learning_rate": 3.8084879051612144e-08,
+      "logits/chosen": 5713.95068359375,
+      "logits/rejected": 5230.85009765625,
+      "logps/chosen": -266.6412048339844,
+      "logps/rejected": -279.85150146484375,
+      "loss": 0.8971,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": 6.2511186599731445,
+      "rewards/margins": 48.867332458496094,
+      "rewards/rejected": -42.6162109375,
+      "step": 1070
+    },
+    {
+      "epoch": 0.847723704866562,
+      "grad_norm": 26.287897922478123,
+      "learning_rate": 3.452896722091128e-08,
+      "logits/chosen": 6276.97119140625,
+      "logits/rejected": 4863.9853515625,
+      "logps/chosen": -304.9378967285156,
+      "logps/rejected": -293.0082092285156,
+      "loss": 0.8843,
+      "rewards/accuracies": 0.8250001072883606,
+      "rewards/chosen": 7.549924373626709,
+      "rewards/margins": 59.31702423095703,
+      "rewards/rejected": -51.76709747314453,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8555729984301413,
+      "grad_norm": 28.443568850519753,
+      "learning_rate": 3.11349722995527e-08,
+      "logits/chosen": 6319.14599609375,
+      "logits/rejected": 4778.30322265625,
+      "logps/chosen": -271.9593505859375,
+      "logps/rejected": -292.22821044921875,
+      "loss": 0.9029,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": 1.8383331298828125,
+      "rewards/margins": 44.397727966308594,
+      "rewards/rejected": -42.55939865112305,
+      "step": 1090
+    },
+    {
+      "epoch": 0.8634222919937206,
+      "grad_norm": 32.28685337112104,
+      "learning_rate": 2.7905444723949762e-08,
+      "logits/chosen": 6111.0732421875,
+      "logits/rejected": 5072.91796875,
+      "logps/chosen": -282.01324462890625,
+      "logps/rejected": -285.5963134765625,
+      "loss": 0.908,
+      "rewards/accuracies": 0.7833333611488342,
+      "rewards/chosen": 1.209522008895874,
+      "rewards/margins": 54.696800231933594,
+      "rewards/rejected": -53.487281799316406,
+      "step": 1100
+    },
+    {
+      "epoch": 0.8712715855572999,
+      "grad_norm": 25.46638694280487,
+      "learning_rate": 2.484281134061142e-08,
+      "logits/chosen": 6478.18701171875,
+      "logits/rejected": 5250.2666015625,
+      "logps/chosen": -308.4444580078125,
+      "logps/rejected": -311.63800048828125,
+      "loss": 0.9019,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.8299369812011719,
+      "rewards/margins": 49.747154235839844,
+      "rewards/rejected": -48.917213439941406,
+      "step": 1110
+    },
+    {
+      "epoch": 0.8791208791208791,
+      "grad_norm": 33.894472996102174,
+      "learning_rate": 2.194937358247506e-08,
+      "logits/chosen": 6344.24462890625,
+      "logits/rejected": 5178.0888671875,
+      "logps/chosen": -291.0325622558594,
+      "logps/rejected": -310.59222412109375,
+      "loss": 0.89,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.3055509626865387,
+      "rewards/margins": 51.41431427001953,
+      "rewards/rejected": -51.108768463134766,
+      "step": 1120
+    },
+    {
+      "epoch": 0.8869701726844584,
+      "grad_norm": 35.09428216666554,
+      "learning_rate": 1.9227305739481612e-08,
+      "logits/chosen": 5788.7275390625,
+      "logits/rejected": 4550.9697265625,
+      "logps/chosen": -270.097412109375,
+      "logps/rejected": -266.88616943359375,
+      "loss": 0.8938,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": 8.472616195678711,
+      "rewards/margins": 50.75660705566406,
+      "rewards/rejected": -42.28398895263672,
+      "step": 1130
+    },
+    {
+      "epoch": 0.8948194662480377,
+      "grad_norm": 24.4757646274536,
+      "learning_rate": 1.6678653324693787e-08,
+      "logits/chosen": 6356.0703125,
+      "logits/rejected": 5166.41259765625,
+      "logps/chosen": -299.87261962890625,
+      "logps/rejected": -303.05859375,
+      "loss": 0.8891,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 5.566819667816162,
+      "rewards/margins": 47.01850128173828,
+      "rewards/rejected": -41.45167922973633,
+      "step": 1140
+    },
+    {
+      "epoch": 0.902668759811617,
+      "grad_norm": 24.046707084849317,
+      "learning_rate": 1.4305331537183384e-08,
+      "logits/chosen": 5655.9697265625,
+      "logits/rejected": 5174.4541015625,
+      "logps/chosen": -265.05810546875,
+      "logps/rejected": -304.6141357421875,
+      "loss": 0.8896,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 2.137265682220459,
+      "rewards/margins": 43.28087615966797,
+      "rewards/rejected": -41.14360809326172,
+      "step": 1150
+    },
+    {
+      "epoch": 0.9105180533751962,
+      "grad_norm": 23.734757870906822,
+      "learning_rate": 1.2109123822844653e-08,
+      "logits/chosen": 5815.59423828125,
+      "logits/rejected": 4604.97900390625,
+      "logps/chosen": -269.80438232421875,
+      "logps/rejected": -275.7255859375,
+      "loss": 0.8999,
+      "rewards/accuracies": 0.7500000596046448,
+      "rewards/chosen": 0.8734811544418335,
+      "rewards/margins": 42.580562591552734,
+      "rewards/rejected": -41.70708465576172,
+      "step": 1160
+    },
+    {
+      "epoch": 0.9183673469387755,
+      "grad_norm": 31.83145273716174,
+      "learning_rate": 1.0091680534213387e-08,
+      "logits/chosen": 6320.2197265625,
+      "logits/rejected": 6094.9072265625,
+      "logps/chosen": -290.4259033203125,
+      "logps/rejected": -343.50604248046875,
+      "loss": 0.9088,
+      "rewards/accuracies": 0.6916667222976685,
+      "rewards/chosen": 1.7011464834213257,
+      "rewards/margins": 41.39912796020508,
+      "rewards/rejected": -39.6979866027832,
+      "step": 1170
+    },
+    {
+      "epoch": 0.9262166405023547,
+      "grad_norm": 31.116419015526656,
+      "learning_rate": 8.254517690300944e-09,
+      "logits/chosen": 5599.1767578125,
+      "logits/rejected": 5051.66455078125,
+      "logps/chosen": -274.82489013671875,
+      "logps/rejected": -302.0051574707031,
+      "loss": 0.9037,
+      "rewards/accuracies": 0.7666667699813843,
+      "rewards/chosen": 4.126657009124756,
+      "rewards/margins": 47.318702697753906,
+      "rewards/rejected": -43.19204330444336,
+      "step": 1180
+    },
+    {
+      "epoch": 0.9340659340659341,
+      "grad_norm": 25.672415270441174,
+      "learning_rate": 6.599015837372907e-09,
+      "logits/chosen": 6093.4912109375,
+      "logits/rejected": 5290.40185546875,
+      "logps/chosen": -295.9696044921875,
+      "logps/rejected": -312.8633728027344,
+      "loss": 0.8923,
+      "rewards/accuracies": 0.7166666984558105,
+      "rewards/chosen": -3.6509156227111816,
+      "rewards/margins": 45.37594985961914,
+      "rewards/rejected": -49.02686309814453,
+      "step": 1190
+    },
+    {
+      "epoch": 0.9419152276295133,
+      "grad_norm": 32.1293848999246,
+      "learning_rate": 5.126419011529992e-09,
+      "logits/chosen": 6286.7607421875,
+      "logits/rejected": 5351.490234375,
+      "logps/chosen": -299.1442565917969,
+      "logps/rejected": -305.995361328125,
+      "loss": 0.8891,
+      "rewards/accuracies": 0.8499999046325684,
+      "rewards/chosen": 5.603835582733154,
+      "rewards/margins": 53.45023727416992,
+      "rewards/rejected": -47.84640121459961,
+      "step": 1200
+    },
+    {
+      "epoch": 0.9497645211930926,
+      "grad_norm": 37.72170489863353,
+      "learning_rate": 3.837833803870177e-09,
+      "logits/chosen": 5903.0517578125,
+      "logits/rejected": 5179.5322265625,
+      "logps/chosen": -275.3850402832031,
+      "logps/rejected": -299.48687744140625,
+      "loss": 0.895,
+      "rewards/accuracies": 0.783333420753479,
+      "rewards/chosen": 4.417906284332275,
+      "rewards/margins": 51.5880126953125,
+      "rewards/rejected": -47.17010498046875,
+      "step": 1210
+    },
+    {
+      "epoch": 0.957613814756672,
+      "grad_norm": 23.85369498972877,
+      "learning_rate": 2.734228528934679e-09,
+      "logits/chosen": 7313.7734375,
+      "logits/rejected": 5392.05517578125,
+      "logps/chosen": -346.1097106933594,
+      "logps/rejected": -333.2154541015625,
+      "loss": 0.9148,
+      "rewards/accuracies": 0.7583333849906921,
+      "rewards/chosen": 0.30585241317749023,
+      "rewards/margins": 48.22237014770508,
+      "rewards/rejected": -47.9165153503418,
+      "step": 1220
+    },
+    {
+      "epoch": 0.9654631083202512,
+      "grad_norm": 26.200635812141893,
+      "learning_rate": 1.8164324970625645e-09,
+      "logits/chosen": 6493.0498046875,
+      "logits/rejected": 5102.583984375,
+      "logps/chosen": -301.46197509765625,
+      "logps/rejected": -308.8186950683594,
+      "loss": 0.9018,
+      "rewards/accuracies": 0.7083333730697632,
+      "rewards/chosen": 6.893446445465088,
+      "rewards/margins": 50.65166091918945,
+      "rewards/rejected": -43.758216857910156,
+      "step": 1230
+    },
+    {
+      "epoch": 0.9733124018838305,
+      "grad_norm": 22.483038203329606,
+      "learning_rate": 1.0851353912008642e-09,
+      "logits/chosen": 5596.9306640625,
+      "logits/rejected": 5155.1064453125,
+      "logps/chosen": -273.6884460449219,
+      "logps/rejected": -323.09979248046875,
+      "loss": 0.8851,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7349063754081726,
+      "rewards/margins": 46.14875030517578,
+      "rewards/rejected": -46.88364791870117,
+      "step": 1240
+    },
+    {
+      "epoch": 0.9811616954474097,
+      "grad_norm": 20.587079205487154,
+      "learning_rate": 5.408867486384471e-10,
+      "logits/chosen": 5712.6357421875,
+      "logits/rejected": 4821.349609375,
+      "logps/chosen": -263.42022705078125,
+      "logps/rejected": -261.34967041015625,
+      "loss": 0.8966,
+      "rewards/accuracies": 0.7833333015441895,
+      "rewards/chosen": 7.018378257751465,
+      "rewards/margins": 45.015357971191406,
+      "rewards/rejected": -37.996986389160156,
+      "step": 1250
+    },
+    {
+      "epoch": 0.989010989010989,
+      "grad_norm": 24.191259232021025,
+      "learning_rate": 1.840955480532924e-10,
+      "logits/chosen": 5454.1728515625,
+      "logits/rejected": 5137.4345703125,
+      "logps/chosen": -264.6805725097656,
+      "logps/rejected": -294.5889587402344,
+      "loss": 0.8915,
+      "rewards/accuracies": 0.7833333611488342,
+      "rewards/chosen": 1.9722245931625366,
+      "rewards/margins": 38.85475540161133,
+      "rewards/rejected": -36.882530212402344,
+      "step": 1260
+    },
+    {
+      "epoch": 0.9968602825745683,
+      "grad_norm": 27.833416743925994,
+      "learning_rate": 1.502990218302247e-11,
+      "logits/chosen": 5687.4814453125,
+      "logits/rejected": 4611.4326171875,
+      "logps/chosen": -260.2027282714844,
+      "logps/rejected": -269.2962951660156,
+      "loss": 0.8907,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": 1.270787000656128,
+      "rewards/margins": 42.4088020324707,
+      "rewards/rejected": -41.13801193237305,
+      "step": 1270
+    },
+    {
+      "epoch": 1.0,
+      "step": 1274,
+      "total_flos": 0.0,
+      "train_loss": 0.9191367263135206,
+      "train_runtime": 14436.6556,
+      "train_samples_per_second": 4.235,
+      "train_steps_per_second": 0.088
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1274,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 2000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 6,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eef5190a909e88574e2c647d150bf363f6b76f860b310dc15ecd7609f1023d3e
+size 7288