Model first version

config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "UCLA-AGI/zephyr-7b-sft-full-SPIN-iter0",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.35.0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.35.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2515f61594b642dfd75582db9d92f54be8abf4e7ff701c4960f8f0b7db26003e
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dc433836aafde71b58f7eddab07652674034f3ba508146425b4a4f5e08ee582
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2632184e8f07705e4e1f3e9ee3d3377c9ec1985941d29c24533229cde030efc
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,41 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}

trainer_state.json

@@ -0,0 +1,2443 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 100,
+  "global_step": 1556,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.0706638115631692e-09,
+      "logits/chosen": -3.0633435249328613,
+      "logits/rejected": -3.0370049476623535,
+      "logps/chosen": -237.29315185546875,
+      "logps/rejected": -251.69747924804688,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 1.070663811563169e-08,
+      "logits/chosen": -2.990461826324463,
+      "logits/rejected": -3.0024797916412354,
+      "logps/chosen": -356.6201171875,
+      "logps/rejected": -390.87042236328125,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5555555820465088,
+      "rewards/chosen": -0.004924382548779249,
+      "rewards/margins": 0.009135871194303036,
+      "rewards/rejected": -0.014060255140066147,
+      "step": 10
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 2.141327623126338e-08,
+      "logits/chosen": -3.002528429031372,
+      "logits/rejected": -3.0017483234405518,
+      "logps/chosen": -350.7555847167969,
+      "logps/rejected": -393.46014404296875,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.050124846398830414,
+      "rewards/margins": 0.028588850051164627,
+      "rewards/rejected": 0.02153599075973034,
+      "step": 20
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 3.2119914346895076e-08,
+      "logits/chosen": -2.975447416305542,
+      "logits/rejected": -3.0126380920410156,
+      "logps/chosen": -375.95391845703125,
+      "logps/rejected": -432.83587646484375,
+      "loss": 0.6435,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.2032477855682373,
+      "rewards/margins": 0.1010356992483139,
+      "rewards/rejected": 0.1022120863199234,
+      "step": 30
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 4.282655246252676e-08,
+      "logits/chosen": -3.0026869773864746,
+      "logits/rejected": -2.9945485591888428,
+      "logps/chosen": -383.3456115722656,
+      "logps/rejected": -392.7911376953125,
+      "loss": 0.5784,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.4273909628391266,
+      "rewards/margins": 0.30088725686073303,
+      "rewards/rejected": 0.12650372087955475,
+      "step": 40
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 5.353319057815846e-08,
+      "logits/chosen": -2.989891529083252,
+      "logits/rejected": -2.996675968170166,
+      "logps/chosen": -339.07513427734375,
+      "logps/rejected": -373.727783203125,
+      "loss": 0.5345,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.6149066686630249,
+      "rewards/margins": 0.39920732378959656,
+      "rewards/rejected": 0.21569931507110596,
+      "step": 50
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 6.423982869379015e-08,
+      "logits/chosen": -3.026094913482666,
+      "logits/rejected": -2.9982128143310547,
+      "logps/chosen": -327.8692321777344,
+      "logps/rejected": -375.9877624511719,
+      "loss": 0.4485,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.6913961172103882,
+      "rewards/margins": 0.7190420031547546,
+      "rewards/rejected": -0.02764584682881832,
+      "step": 60
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 7.494646680942184e-08,
+      "logits/chosen": -2.974823474884033,
+      "logits/rejected": -2.980032444000244,
+      "logps/chosen": -351.2728576660156,
+      "logps/rejected": -395.68609619140625,
+      "loss": 0.3966,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.8828132748603821,
+      "rewards/margins": 0.9640719294548035,
+      "rewards/rejected": -0.08125858008861542,
+      "step": 70
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 8.565310492505352e-08,
+      "logits/chosen": -2.977529287338257,
+      "logits/rejected": -2.9725558757781982,
+      "logps/chosen": -359.2842712402344,
+      "logps/rejected": -405.7890625,
+      "loss": 0.3519,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.091180682182312,
+      "rewards/margins": 1.2520115375518799,
+      "rewards/rejected": -0.16083075106143951,
+      "step": 80
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 9.635974304068522e-08,
+      "logits/chosen": -2.979015827178955,
+      "logits/rejected": -2.9813497066497803,
+      "logps/chosen": -309.3511047363281,
+      "logps/rejected": -358.91607666015625,
+      "loss": 0.3201,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 1.2030521631240845,
+      "rewards/margins": 1.6773903369903564,
+      "rewards/rejected": -0.4743381440639496,
+      "step": 90
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.0706638115631692e-07,
+      "logits/chosen": -2.941194534301758,
+      "logits/rejected": -2.9548678398132324,
+      "logps/chosen": -343.6178894042969,
+      "logps/rejected": -463.1512145996094,
+      "loss": 0.2696,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.2106283903121948,
+      "rewards/margins": 1.9713561534881592,
+      "rewards/rejected": -0.7607278823852539,
+      "step": 100
+    },
+    {
+      "epoch": 0.13,
+      "eval_logits/chosen": -2.977161169052124,
+      "eval_logits/rejected": -2.957442045211792,
+      "eval_logps/chosen": -296.8330383300781,
+      "eval_logps/rejected": -349.66558837890625,
+      "eval_loss": 0.2511790990829468,
+      "eval_rewards/accuracies": 0.921875,
+      "eval_rewards/chosen": 1.1878268718719482,
+      "eval_rewards/margins": 1.8798556327819824,
+      "eval_rewards/rejected": -0.6920287609100342,
+      "eval_runtime": 38.7534,
+      "eval_samples_per_second": 12.902,
+      "eval_steps_per_second": 0.413,
+      "step": 100
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.177730192719486e-07,
+      "logits/chosen": -2.9442899227142334,
+      "logits/rejected": -2.9481866359710693,
+      "logps/chosen": -346.63873291015625,
+      "logps/rejected": -406.31964111328125,
+      "loss": 0.2493,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 1.2421057224273682,
+      "rewards/margins": 2.18147873878479,
+      "rewards/rejected": -0.9393728971481323,
+      "step": 110
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.284796573875803e-07,
+      "logits/chosen": -2.94069242477417,
+      "logits/rejected": -2.9417574405670166,
+      "logps/chosen": -351.788330078125,
+      "logps/rejected": -379.61065673828125,
+      "loss": 0.2406,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.7772680521011353,
+      "rewards/margins": 1.8036502599716187,
+      "rewards/rejected": -1.0263820886611938,
+      "step": 120
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.3918629550321198e-07,
+      "logits/chosen": -2.926699638366699,
+      "logits/rejected": -2.911668300628662,
+      "logps/chosen": -327.4112548828125,
+      "logps/rejected": -408.2745361328125,
+      "loss": 0.2073,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.5646601915359497,
+      "rewards/margins": 2.2064461708068848,
+      "rewards/rejected": -1.6417862176895142,
+      "step": 130
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.4989293361884367e-07,
+      "logits/chosen": -2.904219150543213,
+      "logits/rejected": -2.921232223510742,
+      "logps/chosen": -311.6190185546875,
+      "logps/rejected": -411.2701110839844,
+      "loss": 0.1967,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.46902722120285034,
+      "rewards/margins": 2.7694640159606934,
+      "rewards/rejected": -2.3004367351531982,
+      "step": 140
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.6059957173447535e-07,
+      "logits/chosen": -2.901981830596924,
+      "logits/rejected": -2.9112467765808105,
+      "logps/chosen": -301.6145324707031,
+      "logps/rejected": -391.1957092285156,
+      "loss": 0.1723,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.218230202794075,
+      "rewards/margins": 3.2492637634277344,
+      "rewards/rejected": -3.031033992767334,
+      "step": 150
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.7130620985010704e-07,
+      "logits/chosen": -2.8996052742004395,
+      "logits/rejected": -2.8838694095611572,
+      "logps/chosen": -312.6499938964844,
+      "logps/rejected": -447.8002014160156,
+      "loss": 0.1554,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.3209637701511383,
+      "rewards/margins": 4.501524925231934,
+      "rewards/rejected": -4.180561065673828,
+      "step": 160
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.8201284796573874e-07,
+      "logits/chosen": -2.8928513526916504,
+      "logits/rejected": -2.9001543521881104,
+      "logps/chosen": -329.20953369140625,
+      "logps/rejected": -423.6446228027344,
+      "loss": 0.1566,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.1461164504289627,
+      "rewards/margins": 4.050145626068115,
+      "rewards/rejected": -3.904029130935669,
+      "step": 170
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.9271948608137044e-07,
+      "logits/chosen": -2.8557610511779785,
+      "logits/rejected": -2.855731725692749,
+      "logps/chosen": -338.60076904296875,
+      "logps/rejected": -448.8922424316406,
+      "loss": 0.1421,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.21355919539928436,
+      "rewards/margins": 4.191808223724365,
+      "rewards/rejected": -3.9782490730285645,
+      "step": 180
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 2.0342612419700214e-07,
+      "logits/chosen": -2.8638434410095215,
+      "logits/rejected": -2.877293825149536,
+      "logps/chosen": -347.19573974609375,
+      "logps/rejected": -469.17755126953125,
+      "loss": 0.1381,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.14256651699543,
+      "rewards/margins": 3.96684193611145,
+      "rewards/rejected": -4.109408378601074,
+      "step": 190
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 2.1413276231263384e-07,
+      "logits/chosen": -2.829555034637451,
+      "logits/rejected": -2.85453462600708,
+      "logps/chosen": -364.0372009277344,
+      "logps/rejected": -442.7489318847656,
+      "loss": 0.1427,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.18970072269439697,
+      "rewards/margins": 5.029218673706055,
+      "rewards/rejected": -4.839517593383789,
+      "step": 200
+    },
+    {
+      "epoch": 0.26,
+      "eval_logits/chosen": -2.8512933254241943,
+      "eval_logits/rejected": -2.8302505016326904,
+      "eval_logps/chosen": -305.8147888183594,
+      "eval_logps/rejected": -387.1728210449219,
+      "eval_loss": 0.12157174944877625,
+      "eval_rewards/accuracies": 0.96875,
+      "eval_rewards/chosen": 0.28965064883232117,
+      "eval_rewards/margins": 4.73240327835083,
+      "eval_rewards/rejected": -4.442752361297607,
+      "eval_runtime": 38.702,
+      "eval_samples_per_second": 12.919,
+      "eval_steps_per_second": 0.413,
+      "step": 200
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 2.248394004282655e-07,
+      "logits/chosen": -2.817666530609131,
+      "logits/rejected": -2.8465371131896973,
+      "logps/chosen": -325.3854675292969,
+      "logps/rejected": -439.5003356933594,
+      "loss": 0.1413,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.21560493111610413,
+      "rewards/margins": 4.514598369598389,
+      "rewards/rejected": -4.298993110656738,
+      "step": 210
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 2.355460385438972e-07,
+      "logits/chosen": -2.7650692462921143,
+      "logits/rejected": -2.7801504135131836,
+      "logps/chosen": -326.321533203125,
+      "logps/rejected": -456.98663330078125,
+      "loss": 0.1332,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.027444612234830856,
+      "rewards/margins": 5.077801704406738,
+      "rewards/rejected": -5.050357818603516,
+      "step": 220
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 2.462526766595289e-07,
+      "logits/chosen": -2.788020610809326,
+      "logits/rejected": -2.7895946502685547,
+      "logps/chosen": -324.4822998046875,
+      "logps/rejected": -439.76397705078125,
+      "loss": 0.1356,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.29427874088287354,
+      "rewards/margins": 5.166212558746338,
+      "rewards/rejected": -4.871933460235596,
+      "step": 230
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 2.569593147751606e-07,
+      "logits/chosen": -2.6995949745178223,
+      "logits/rejected": -2.7345399856567383,
+      "logps/chosen": -356.4814758300781,
+      "logps/rejected": -490.60931396484375,
+      "loss": 0.1074,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.35535210371017456,
+      "rewards/margins": 6.1955766677856445,
+      "rewards/rejected": -6.550928592681885,
+      "step": 240
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 2.676659528907923e-07,
+      "logits/chosen": -2.6892549991607666,
+      "logits/rejected": -2.694087505340576,
+      "logps/chosen": -305.6263122558594,
+      "logps/rejected": -387.88543701171875,
+      "loss": 0.0979,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.6652821898460388,
+      "rewards/margins": 4.929129600524902,
+      "rewards/rejected": -5.5944108963012695,
+      "step": 250
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 2.7837259100642395e-07,
+      "logits/chosen": -2.73167085647583,
+      "logits/rejected": -2.7620654106140137,
+      "logps/chosen": -408.2175598144531,
+      "logps/rejected": -449.8201599121094,
+      "loss": 0.1298,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.9623678922653198,
+      "rewards/margins": 5.48039436340332,
+      "rewards/rejected": -6.4427618980407715,
+      "step": 260
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 2.890792291220557e-07,
+      "logits/chosen": -2.7657806873321533,
+      "logits/rejected": -2.802060604095459,
+      "logps/chosen": -384.2090148925781,
+      "logps/rejected": -481.82696533203125,
+      "loss": 0.1181,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.16663847863674164,
+      "rewards/margins": 5.502591133117676,
+      "rewards/rejected": -5.335952281951904,
+      "step": 270
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 2.9978586723768735e-07,
+      "logits/chosen": -2.673283815383911,
+      "logits/rejected": -2.707296848297119,
+      "logps/chosen": -312.5271911621094,
+      "logps/rejected": -411.64031982421875,
+      "loss": 0.0947,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.9499552845954895,
+      "rewards/margins": 4.852605819702148,
+      "rewards/rejected": -5.802561283111572,
+      "step": 280
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.1049250535331905e-07,
+      "logits/chosen": -2.623725175857544,
+      "logits/rejected": -2.7073614597320557,
+      "logps/chosen": -391.2462158203125,
+      "logps/rejected": -474.2684631347656,
+      "loss": 0.1168,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.0791637897491455,
+      "rewards/margins": 7.065374851226807,
+      "rewards/rejected": -8.144537925720215,
+      "step": 290
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.211991434689507e-07,
+      "logits/chosen": -2.6202073097229004,
+      "logits/rejected": -2.652608633041382,
+      "logps/chosen": -341.9140319824219,
+      "logps/rejected": -462.9012145996094,
+      "loss": 0.0944,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.275757372379303,
+      "rewards/margins": 5.93372106552124,
+      "rewards/rejected": -6.20947790145874,
+      "step": 300
+    },
+    {
+      "epoch": 0.39,
+      "eval_logits/chosen": -2.6932637691497803,
+      "eval_logits/rejected": -2.6872053146362305,
+      "eval_logps/chosen": -311.619873046875,
+      "eval_logps/rejected": -409.2980041503906,
+      "eval_loss": 0.11095032095909119,
+      "eval_rewards/accuracies": 0.90625,
+      "eval_rewards/chosen": -0.29085665941238403,
+      "eval_rewards/margins": 6.364411354064941,
+      "eval_rewards/rejected": -6.65526819229126,
+      "eval_runtime": 38.7504,
+      "eval_samples_per_second": 12.903,
+      "eval_steps_per_second": 0.413,
+      "step": 300
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.3190578158458244e-07,
+      "logits/chosen": -2.6386542320251465,
+      "logits/rejected": -2.7159385681152344,
+      "logps/chosen": -368.5979919433594,
+      "logps/rejected": -466.84783935546875,
+      "loss": 0.131,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.8093490600585938,
+      "rewards/margins": 7.050684452056885,
+      "rewards/rejected": -7.8600335121154785,
+      "step": 310
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.426124197002141e-07,
+      "logits/chosen": -2.571882486343384,
+      "logits/rejected": -2.6551308631896973,
+      "logps/chosen": -361.48394775390625,
+      "logps/rejected": -489.70989990234375,
+      "loss": 0.0905,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.7510203123092651,
+      "rewards/margins": 8.015697479248047,
+      "rewards/rejected": -8.766717910766602,
+      "step": 320
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.533190578158458e-07,
+      "logits/chosen": -2.5930895805358887,
+      "logits/rejected": -2.6723227500915527,
+      "logps/chosen": -384.87664794921875,
+      "logps/rejected": -509.010986328125,
+      "loss": 0.1232,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.44466814398765564,
+      "rewards/margins": 6.863368988037109,
+      "rewards/rejected": -7.308036804199219,
+      "step": 330
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.640256959314775e-07,
+      "logits/chosen": -2.5658717155456543,
+      "logits/rejected": -2.62716007232666,
+      "logps/chosen": -304.2865295410156,
+      "logps/rejected": -435.2959899902344,
+      "loss": 0.0874,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.5686666369438171,
+      "rewards/margins": 6.669247627258301,
+      "rewards/rejected": -7.237914085388184,
+      "step": 340
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.747323340471092e-07,
+      "logits/chosen": -2.584165096282959,
+      "logits/rejected": -2.70393967628479,
+      "logps/chosen": -364.13262939453125,
+      "logps/rejected": -477.5604553222656,
+      "loss": 0.1015,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.5823951363563538,
+      "rewards/margins": 7.484101295471191,
+      "rewards/rejected": -8.066494941711426,
+      "step": 350
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.854389721627409e-07,
+      "logits/chosen": -2.5895907878875732,
+      "logits/rejected": -2.646876573562622,
+      "logps/chosen": -355.0018005371094,
+      "logps/rejected": -442.65948486328125,
+      "loss": 0.0896,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.8058759570121765,
+      "rewards/margins": 8.065896987915039,
+      "rewards/rejected": -8.871771812438965,
+      "step": 360
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.961456102783726e-07,
+      "logits/chosen": -2.615499973297119,
+      "logits/rejected": -2.6612184047698975,
+      "logps/chosen": -308.342041015625,
+      "logps/rejected": -432.08319091796875,
+      "loss": 0.0821,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.5296161770820618,
+      "rewards/margins": 7.243483066558838,
+      "rewards/rejected": -7.773098945617676,
+      "step": 370
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 4.068522483940043e-07,
+      "logits/chosen": -2.6956448554992676,
+      "logits/rejected": -2.7061805725097656,
+      "logps/chosen": -346.4541931152344,
+      "logps/rejected": -481.19989013671875,
+      "loss": 0.1104,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.6448992490768433,
+      "rewards/margins": 7.711002349853516,
+      "rewards/rejected": -8.355902671813965,
+      "step": 380
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 4.175588865096359e-07,
+      "logits/chosen": -2.6077234745025635,
+      "logits/rejected": -2.6278557777404785,
+      "logps/chosen": -353.8262634277344,
+      "logps/rejected": -447.3440856933594,
+      "loss": 0.0958,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.5978514552116394,
+      "rewards/margins": 7.370479583740234,
+      "rewards/rejected": -7.968331336975098,
+      "step": 390
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.282655246252677e-07,
+      "logits/chosen": -2.603065252304077,
+      "logits/rejected": -2.675497531890869,
+      "logps/chosen": -355.2611999511719,
+      "logps/rejected": -411.75732421875,
+      "loss": 0.1039,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.2750840187072754,
+      "rewards/margins": 7.0222907066345215,
+      "rewards/rejected": -7.2973737716674805,
+      "step": 400
+    },
+    {
+      "epoch": 0.51,
+      "eval_logits/chosen": -2.6301259994506836,
+      "eval_logits/rejected": -2.6286230087280273,
+      "eval_logps/chosen": -315.64288330078125,
+      "eval_logps/rejected": -421.1318359375,
+      "eval_loss": 0.07803654670715332,
+      "eval_rewards/accuracies": 0.984375,
+      "eval_rewards/chosen": -0.6931607723236084,
+      "eval_rewards/margins": 7.145491600036621,
+      "eval_rewards/rejected": -7.83865213394165,
+      "eval_runtime": 38.7861,
+      "eval_samples_per_second": 12.891,
+      "eval_steps_per_second": 0.413,
+      "step": 400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.389721627408993e-07,
+      "logits/chosen": -2.5576305389404297,
+      "logits/rejected": -2.602813243865967,
+      "logps/chosen": -361.10797119140625,
+      "logps/rejected": -468.213134765625,
+      "loss": 0.1042,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.9836179614067078,
+      "rewards/margins": 6.6080522537231445,
+      "rewards/rejected": -7.591670989990234,
+      "step": 410
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.49678800856531e-07,
+      "logits/chosen": -2.521080732345581,
+      "logits/rejected": -2.5644307136535645,
+      "logps/chosen": -325.7511901855469,
+      "logps/rejected": -407.7994384765625,
+      "loss": 0.1057,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.45771685242652893,
+      "rewards/margins": 7.0977678298950195,
+      "rewards/rejected": -7.555483818054199,
+      "step": 420
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.603854389721627e-07,
+      "logits/chosen": -2.5245959758758545,
+      "logits/rejected": -2.559770107269287,
+      "logps/chosen": -340.15087890625,
+      "logps/rejected": -485.052490234375,
+      "loss": 0.084,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.994246780872345,
+      "rewards/margins": 7.357940673828125,
+      "rewards/rejected": -8.35218620300293,
+      "step": 430
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.710920770877944e-07,
+      "logits/chosen": -2.401303768157959,
+      "logits/rejected": -2.548125743865967,
+      "logps/chosen": -358.9648742675781,
+      "logps/rejected": -462.87890625,
+      "loss": 0.1172,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.293526530265808,
+      "rewards/margins": 7.095101833343506,
+      "rewards/rejected": -8.388628005981445,
+      "step": 440
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.817987152034261e-07,
+      "logits/chosen": -2.4654183387756348,
+      "logits/rejected": -2.560048818588257,
+      "logps/chosen": -291.2701721191406,
+      "logps/rejected": -362.7830505371094,
+      "loss": 0.0959,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.4040035009384155,
+      "rewards/margins": 5.726696968078613,
+      "rewards/rejected": -7.130700588226318,
+      "step": 450
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.925053533190578e-07,
+      "logits/chosen": -2.489262104034424,
+      "logits/rejected": -2.5457305908203125,
+      "logps/chosen": -356.9480285644531,
+      "logps/rejected": -435.594970703125,
+      "loss": 0.1132,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.3584586381912231,
+      "rewards/margins": 6.3141889572143555,
+      "rewards/rejected": -7.672647953033447,
+      "step": 460
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.996429421566293e-07,
+      "logits/chosen": -2.5229034423828125,
+      "logits/rejected": -2.565725326538086,
+      "logps/chosen": -326.0317077636719,
+      "logps/rejected": -448.7723083496094,
+      "loss": 0.1051,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.5931789875030518,
+      "rewards/margins": 7.049294471740723,
+      "rewards/rejected": -8.642473220825195,
+      "step": 470
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 4.98452749345394e-07,
+      "logits/chosen": -2.5022709369659424,
+      "logits/rejected": -2.555453062057495,
+      "logps/chosen": -361.46563720703125,
+      "logps/rejected": -498.7660217285156,
+      "loss": 0.1386,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.8986074924468994,
+      "rewards/margins": 6.340726375579834,
+      "rewards/rejected": -8.239333152770996,
+      "step": 480
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 4.972625565341585e-07,
+      "logits/chosen": -2.4549243450164795,
+      "logits/rejected": -2.5045337677001953,
+      "logps/chosen": -320.4005432128906,
+      "logps/rejected": -437.33612060546875,
+      "loss": 0.0958,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.8746875524520874,
+      "rewards/margins": 6.6805620193481445,
+      "rewards/rejected": -8.555249214172363,
+      "step": 490
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 4.960723637229232e-07,
+      "logits/chosen": -2.448908567428589,
+      "logits/rejected": -2.458101272583008,
+      "logps/chosen": -355.0153503417969,
+      "logps/rejected": -504.32330322265625,
+      "loss": 0.0762,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.4554470777511597,
+      "rewards/margins": 8.177068710327148,
+      "rewards/rejected": -9.632516860961914,
+      "step": 500
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/chosen": -2.504735231399536,
+      "eval_logits/rejected": -2.5092720985412598,
+      "eval_logps/chosen": -323.16851806640625,
+      "eval_logps/rejected": -433.9158020019531,
+      "eval_loss": 0.08059512078762054,
+      "eval_rewards/accuracies": 0.953125,
+      "eval_rewards/chosen": -1.4457205533981323,
+      "eval_rewards/margins": 7.671328544616699,
+      "eval_rewards/rejected": -9.117048263549805,
+      "eval_runtime": 38.7512,
+      "eval_samples_per_second": 12.903,
+      "eval_steps_per_second": 0.413,
+      "step": 500
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 4.948821709116876e-07,
+      "logits/chosen": -2.376183032989502,
+      "logits/rejected": -2.455298900604248,
+      "logps/chosen": -485.12603759765625,
+      "logps/rejected": -551.7554931640625,
+      "loss": 0.1056,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.5901005268096924,
+      "rewards/margins": 8.771623611450195,
+      "rewards/rejected": -10.361722946166992,
+      "step": 510
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 4.936919781004522e-07,
+      "logits/chosen": -2.470151424407959,
+      "logits/rejected": -2.5587172508239746,
+      "logps/chosen": -377.3062438964844,
+      "logps/rejected": -507.6141052246094,
+      "loss": 0.0955,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.233569860458374,
+      "rewards/margins": 8.123286247253418,
+      "rewards/rejected": -9.356857299804688,
+      "step": 520
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.925017852892168e-07,
+      "logits/chosen": -2.5230183601379395,
+      "logits/rejected": -2.603940725326538,
+      "logps/chosen": -362.92333984375,
+      "logps/rejected": -481.7613220214844,
+      "loss": 0.0683,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -1.9068357944488525,
+      "rewards/margins": 7.921334743499756,
+      "rewards/rejected": -9.828168869018555,
+      "step": 530
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.913115924779814e-07,
+      "logits/chosen": -2.438596248626709,
+      "logits/rejected": -2.562830924987793,
+      "logps/chosen": -386.5306701660156,
+      "logps/rejected": -499.86444091796875,
+      "loss": 0.0677,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2040196657180786,
+      "rewards/margins": 8.880427360534668,
+      "rewards/rejected": -10.084446907043457,
+      "step": 540
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.90121399666746e-07,
+      "logits/chosen": -2.4589312076568604,
+      "logits/rejected": -2.524345874786377,
+      "logps/chosen": -332.1251220703125,
+      "logps/rejected": -433.63787841796875,
+      "loss": 0.1309,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.1124681234359741,
+      "rewards/margins": 7.2715253829956055,
+      "rewards/rejected": -8.383993148803711,
+      "step": 550
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 4.889312068555106e-07,
+      "logits/chosen": -2.58622407913208,
+      "logits/rejected": -2.60271954536438,
+      "logps/chosen": -271.59014892578125,
+      "logps/rejected": -417.29833984375,
+      "loss": 0.1275,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -1.1244533061981201,
+      "rewards/margins": 6.384497165679932,
+      "rewards/rejected": -7.508950710296631,
+      "step": 560
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 4.877410140442752e-07,
+      "logits/chosen": -2.4364261627197266,
+      "logits/rejected": -2.4858317375183105,
+      "logps/chosen": -350.3711853027344,
+      "logps/rejected": -449.4051818847656,
+      "loss": 0.0982,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.0685746669769287,
+      "rewards/margins": 7.424908638000488,
+      "rewards/rejected": -8.49348258972168,
+      "step": 570
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 4.865508212330398e-07,
+      "logits/chosen": -2.441240072250366,
+      "logits/rejected": -2.527020215988159,
+      "logps/chosen": -366.98150634765625,
+      "logps/rejected": -525.4156494140625,
+      "loss": 0.0867,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9126319885253906,
+      "rewards/margins": 9.27831745147705,
+      "rewards/rejected": -10.190949440002441,
+      "step": 580
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 4.853606284218044e-07,
+      "logits/chosen": -2.3090662956237793,
+      "logits/rejected": -2.3255538940429688,
+      "logps/chosen": -371.3923034667969,
+      "logps/rejected": -526.1776123046875,
+      "loss": 0.1095,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -2.190389633178711,
+      "rewards/margins": 9.64104175567627,
+      "rewards/rejected": -11.831432342529297,
+      "step": 590
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 4.841704356105689e-07,
+      "logits/chosen": -2.334197521209717,
+      "logits/rejected": -2.423285484313965,
+      "logps/chosen": -369.0033264160156,
+      "logps/rejected": -506.4518127441406,
+      "loss": 0.0959,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.7217298746109009,
+      "rewards/margins": 8.07056999206543,
+      "rewards/rejected": -9.7923002243042,
+      "step": 600
+    },
+    {
+      "epoch": 0.77,
+      "eval_logits/chosen": -2.467820405960083,
+      "eval_logits/rejected": -2.440288782119751,
+      "eval_logps/chosen": -318.6737060546875,
+      "eval_logps/rejected": -428.9326171875,
+      "eval_loss": 0.07413332909345627,
+      "eval_rewards/accuracies": 0.984375,
+      "eval_rewards/chosen": -0.9962404370307922,
+      "eval_rewards/margins": 7.622487545013428,
+      "eval_rewards/rejected": -8.618727684020996,
+      "eval_runtime": 38.7439,
+      "eval_samples_per_second": 12.905,
+      "eval_steps_per_second": 0.413,
+      "step": 600
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.829802427993334e-07,
+      "logits/chosen": -2.3268227577209473,
+      "logits/rejected": -2.3746628761291504,
+      "logps/chosen": -404.0111083984375,
+      "logps/rejected": -492.5167541503906,
+      "loss": 0.0859,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.8345616459846497,
+      "rewards/margins": 8.07560920715332,
+      "rewards/rejected": -8.910171508789062,
+      "step": 610
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 4.81790049988098e-07,
+      "logits/chosen": -2.415301561355591,
+      "logits/rejected": -2.4919333457946777,
+      "logps/chosen": -388.5622253417969,
+      "logps/rejected": -531.6051025390625,
+      "loss": 0.0631,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.3940558433532715,
+      "rewards/margins": 7.842892646789551,
+      "rewards/rejected": -9.23694896697998,
+      "step": 620
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 4.805998571768626e-07,
+      "logits/chosen": -2.310925245285034,
+      "logits/rejected": -2.42446231842041,
+      "logps/chosen": -342.0956115722656,
+      "logps/rejected": -516.9351196289062,
+      "loss": 0.1142,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.3800750970840454,
+      "rewards/margins": 8.762998580932617,
+      "rewards/rejected": -10.143075942993164,
+      "step": 630
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.794096643656272e-07,
+      "logits/chosen": -2.280027151107788,
+      "logits/rejected": -2.31703782081604,
+      "logps/chosen": -409.70379638671875,
+      "logps/rejected": -529.5406494140625,
+      "loss": 0.0723,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.3363559246063232,
+      "rewards/margins": 10.320574760437012,
+      "rewards/rejected": -11.656930923461914,
+      "step": 640
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 4.782194715543918e-07,
+      "logits/chosen": -2.276779890060425,
+      "logits/rejected": -2.343441963195801,
+      "logps/chosen": -348.50531005859375,
+      "logps/rejected": -521.2000122070312,
+      "loss": 0.0902,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -1.5391457080841064,
+      "rewards/margins": 9.673690795898438,
+      "rewards/rejected": -11.212836265563965,
+      "step": 650
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 4.770292787431564e-07,
+      "logits/chosen": -2.3436553478240967,
+      "logits/rejected": -2.3175175189971924,
+      "logps/chosen": -386.4251403808594,
+      "logps/rejected": -530.1958618164062,
+      "loss": 0.0787,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -2.0843443870544434,
+      "rewards/margins": 9.766562461853027,
+      "rewards/rejected": -11.850906372070312,
+      "step": 660
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 4.7583908593192097e-07,
+      "logits/chosen": -2.2515616416931152,
+      "logits/rejected": -2.2762718200683594,
+      "logps/chosen": -396.88751220703125,
+      "logps/rejected": -541.3609619140625,
+      "loss": 0.0841,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.3059911727905273,
+      "rewards/margins": 9.442736625671387,
+      "rewards/rejected": -11.748727798461914,
+      "step": 670
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 4.746488931206855e-07,
+      "logits/chosen": -2.304055690765381,
+      "logits/rejected": -2.3429813385009766,
+      "logps/chosen": -353.8645935058594,
+      "logps/rejected": -520.8157348632812,
+      "loss": 0.0793,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.6302597522735596,
+      "rewards/margins": 10.208868980407715,
+      "rewards/rejected": -12.839129447937012,
+      "step": 680
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 4.734587003094501e-07,
+      "logits/chosen": -2.326953887939453,
+      "logits/rejected": -2.4166040420532227,
+      "logps/chosen": -377.34356689453125,
+      "logps/rejected": -494.58782958984375,
+      "loss": 0.1041,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -2.1931746006011963,
+      "rewards/margins": 9.344148635864258,
+      "rewards/rejected": -11.537323951721191,
+      "step": 690
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.722685074982147e-07,
+      "logits/chosen": -2.3279807567596436,
+      "logits/rejected": -2.38569974899292,
+      "logps/chosen": -320.0870056152344,
+      "logps/rejected": -498.17706298828125,
+      "loss": 0.0814,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5494163036346436,
+      "rewards/margins": 10.105340957641602,
+      "rewards/rejected": -11.654756546020508,
+      "step": 700
+    },
+    {
+      "epoch": 0.9,
+      "eval_logits/chosen": -2.498293399810791,
+      "eval_logits/rejected": -2.4712274074554443,
+      "eval_logps/chosen": -323.183837890625,
+      "eval_logps/rejected": -441.4797058105469,
+      "eval_loss": 0.055789634585380554,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -1.4472523927688599,
+      "eval_rewards/margins": 8.426188468933105,
+      "eval_rewards/rejected": -9.87343978881836,
+      "eval_runtime": 38.7758,
+      "eval_samples_per_second": 12.895,
+      "eval_steps_per_second": 0.413,
+      "step": 700
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.710783146869793e-07,
+      "logits/chosen": -2.3991808891296387,
+      "logits/rejected": -2.4218363761901855,
+      "logps/chosen": -314.1746520996094,
+      "logps/rejected": -519.7462768554688,
+      "loss": 0.0819,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.8268877267837524,
+      "rewards/margins": 10.331625938415527,
+      "rewards/rejected": -12.158514022827148,
+      "step": 710
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 4.698881218757438e-07,
+      "logits/chosen": -2.363438606262207,
+      "logits/rejected": -2.3997836112976074,
+      "logps/chosen": -305.2399597167969,
+      "logps/rejected": -481.65582275390625,
+      "loss": 0.0786,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.5438249111175537,
+      "rewards/margins": 8.62690544128418,
+      "rewards/rejected": -10.17072868347168,
+      "step": 720
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.6869792906450845e-07,
+      "logits/chosen": -2.3670878410339355,
+      "logits/rejected": -2.4363322257995605,
+      "logps/chosen": -342.06622314453125,
+      "logps/rejected": -468.9805603027344,
+      "loss": 0.0719,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.5114291906356812,
+      "rewards/margins": 8.608851432800293,
+      "rewards/rejected": -10.120282173156738,
+      "step": 730
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.67507736253273e-07,
+      "logits/chosen": -2.2785589694976807,
+      "logits/rejected": -2.3089492321014404,
+      "logps/chosen": -407.75048828125,
+      "logps/rejected": -557.4127197265625,
+      "loss": 0.0903,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.8981235027313232,
+      "rewards/margins": 10.704629898071289,
+      "rewards/rejected": -12.602753639221191,
+      "step": 740
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.6631754344203763e-07,
+      "logits/chosen": -2.3073747158050537,
+      "logits/rejected": -2.383291244506836,
+      "logps/chosen": -357.61492919921875,
+      "logps/rejected": -522.1990356445312,
+      "loss": 0.1043,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -2.5501503944396973,
+      "rewards/margins": 8.703204154968262,
+      "rewards/rejected": -11.253355026245117,
+      "step": 750
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.6512735063080217e-07,
+      "logits/chosen": -2.492027521133423,
+      "logits/rejected": -2.534536361694336,
+      "logps/chosen": -430.7220764160156,
+      "logps/rejected": -559.482666015625,
+      "loss": 0.0971,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.385508418083191,
+      "rewards/margins": 9.584807395935059,
+      "rewards/rejected": -10.970315933227539,
+      "step": 760
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.6393715781956676e-07,
+      "logits/chosen": -2.3780312538146973,
+      "logits/rejected": -2.37473201751709,
+      "logps/chosen": -326.2506103515625,
+      "logps/rejected": -496.7969665527344,
+      "loss": 0.0865,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.0338951349258423,
+      "rewards/margins": 9.33600902557373,
+      "rewards/rejected": -10.369903564453125,
+      "step": 770
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.6274696500833135e-07,
+      "logits/chosen": -2.4264612197875977,
+      "logits/rejected": -2.45288348197937,
+      "logps/chosen": -368.6007385253906,
+      "logps/rejected": -534.6527709960938,
+      "loss": 0.0645,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.9812146425247192,
+      "rewards/margins": 9.224861145019531,
+      "rewards/rejected": -10.206075668334961,
+      "step": 780
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 4.6155677219709594e-07,
+      "logits/chosen": -2.383737087249756,
+      "logits/rejected": -2.4557416439056396,
+      "logps/chosen": -401.9710388183594,
+      "logps/rejected": -555.4797973632812,
+      "loss": 0.0216,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5994548797607422,
+      "rewards/margins": 12.170892715454102,
+      "rewards/rejected": -13.770347595214844,
+      "step": 790
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 4.603665793858605e-07,
+      "logits/chosen": -2.4060428142547607,
+      "logits/rejected": -2.4426844120025635,
+      "logps/chosen": -366.8950500488281,
+      "logps/rejected": -558.5940551757812,
+      "loss": 0.0164,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.232177972793579,
+      "rewards/margins": 12.297248840332031,
+      "rewards/rejected": -13.529426574707031,
+      "step": 800
+    },
+    {
+      "epoch": 1.03,
+      "eval_logits/chosen": -2.492385149002075,
+      "eval_logits/rejected": -2.468630313873291,
+      "eval_logps/chosen": -324.3902587890625,
+      "eval_logps/rejected": -453.6976623535156,
+      "eval_loss": 0.06341304630041122,
+      "eval_rewards/accuracies": 0.984375,
+      "eval_rewards/chosen": -1.5678963661193848,
+      "eval_rewards/margins": 9.527338981628418,
+      "eval_rewards/rejected": -11.095235824584961,
+      "eval_runtime": 38.5408,
+      "eval_samples_per_second": 12.973,
+      "eval_steps_per_second": 0.415,
+      "step": 800
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 4.5917638657462507e-07,
+      "logits/chosen": -2.33616042137146,
+      "logits/rejected": -2.3640098571777344,
+      "logps/chosen": -373.46905517578125,
+      "logps/rejected": -514.2394409179688,
+      "loss": 0.0209,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4009530544281006,
+      "rewards/margins": 10.871899604797363,
+      "rewards/rejected": -12.272851943969727,
+      "step": 810
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 4.5798619376338966e-07,
+      "logits/chosen": -2.4044508934020996,
+      "logits/rejected": -2.420480966567993,
+      "logps/chosen": -347.3623962402344,
+      "logps/rejected": -556.5758056640625,
+      "loss": 0.0227,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.055593490600586,
+      "rewards/margins": 11.897196769714355,
+      "rewards/rejected": -12.952789306640625,
+      "step": 820
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 4.567960009521542e-07,
+      "logits/chosen": -2.359771490097046,
+      "logits/rejected": -2.4249939918518066,
+      "logps/chosen": -370.0980529785156,
+      "logps/rejected": -567.7897338867188,
+      "loss": 0.0131,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.0004942417144775,
+      "rewards/margins": 12.14315414428711,
+      "rewards/rejected": -14.143648147583008,
+      "step": 830
+    },
+    {
+      "epoch": 1.08,
+      "learning_rate": 4.5560580814091884e-07,
+      "logits/chosen": -2.3424394130706787,
+      "logits/rejected": -2.342963457107544,
+      "logps/chosen": -385.192626953125,
+      "logps/rejected": -510.11749267578125,
+      "loss": 0.0098,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.1284375190734863,
+      "rewards/margins": 11.841325759887695,
+      "rewards/rejected": -13.969762802124023,
+      "step": 840
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 4.5441561532968337e-07,
+      "logits/chosen": -2.3772830963134766,
+      "logits/rejected": -2.414663791656494,
+      "logps/chosen": -375.8727722167969,
+      "logps/rejected": -580.7897338867188,
+      "loss": 0.0093,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.0805163383483887,
+      "rewards/margins": 12.892430305480957,
+      "rewards/rejected": -15.972944259643555,
+      "step": 850
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 4.5322542251844796e-07,
+      "logits/chosen": -2.3776564598083496,
+      "logits/rejected": -2.409484386444092,
+      "logps/chosen": -331.92431640625,
+      "logps/rejected": -500.89739990234375,
+      "loss": 0.0143,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.3345754146575928,
+      "rewards/margins": 11.422739028930664,
+      "rewards/rejected": -13.757314682006836,
+      "step": 860
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 4.5203522970721255e-07,
+      "logits/chosen": -2.3700737953186035,
+      "logits/rejected": -2.397162914276123,
+      "logps/chosen": -340.53094482421875,
+      "logps/rejected": -506.8477478027344,
+      "loss": 0.0146,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.4118890166282654,
+      "rewards/margins": 12.948440551757812,
+      "rewards/rejected": -13.360328674316406,
+      "step": 870
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 4.5084503689597714e-07,
+      "logits/chosen": -2.41035795211792,
+      "logits/rejected": -2.4271979331970215,
+      "logps/chosen": -329.87933349609375,
+      "logps/rejected": -537.0123291015625,
+      "loss": 0.0135,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4996788501739502,
+      "rewards/margins": 11.888396263122559,
+      "rewards/rejected": -13.388073921203613,
+      "step": 880
+    },
+    {
+      "epoch": 1.14,
+      "learning_rate": 4.496548440847417e-07,
+      "logits/chosen": -2.401721477508545,
+      "logits/rejected": -2.447669506072998,
+      "logps/chosen": -366.2709045410156,
+      "logps/rejected": -519.80224609375,
+      "loss": 0.0139,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3627954721450806,
+      "rewards/margins": 12.356982231140137,
+      "rewards/rejected": -13.71977710723877,
+      "step": 890
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 4.484646512735063e-07,
+      "logits/chosen": -2.4436986446380615,
+      "logits/rejected": -2.5449397563934326,
+      "logps/chosen": -384.5765686035156,
+      "logps/rejected": -555.2340087890625,
+      "loss": 0.0172,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.7870714068412781,
+      "rewards/margins": 11.903576850891113,
+      "rewards/rejected": -12.690648078918457,
+      "step": 900
+    },
+    {
+      "epoch": 1.16,
+      "eval_logits/chosen": -2.5417840480804443,
+      "eval_logits/rejected": -2.5121681690216064,
+      "eval_logps/chosen": -326.2882080078125,
+      "eval_logps/rejected": -464.37054443359375,
+      "eval_loss": 0.06124735251069069,
+      "eval_rewards/accuracies": 0.984375,
+      "eval_rewards/chosen": -1.7576879262924194,
+      "eval_rewards/margins": 10.404834747314453,
+      "eval_rewards/rejected": -12.162521362304688,
+      "eval_runtime": 38.6563,
+      "eval_samples_per_second": 12.934,
+      "eval_steps_per_second": 0.414,
+      "step": 900
+    },
+    {
+      "epoch": 1.17,
+      "learning_rate": 4.4727445846227086e-07,
+      "logits/chosen": -2.438345432281494,
+      "logits/rejected": -2.4737024307250977,
+      "logps/chosen": -369.38397216796875,
+      "logps/rejected": -519.6220703125,
+      "loss": 0.011,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.9280792474746704,
+      "rewards/margins": 12.675816535949707,
+      "rewards/rejected": -14.60389518737793,
+      "step": 910
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 4.4608426565103545e-07,
+      "logits/chosen": -2.450275182723999,
+      "logits/rejected": -2.462500810623169,
+      "logps/chosen": -343.4928283691406,
+      "logps/rejected": -515.9462280273438,
+      "loss": 0.0221,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.3710033893585205,
+      "rewards/margins": 13.644805908203125,
+      "rewards/rejected": -16.015810012817383,
+      "step": 920
+    },
+    {
+      "epoch": 1.2,
+      "learning_rate": 4.4489407283980004e-07,
+      "logits/chosen": -2.423760414123535,
+      "logits/rejected": -2.385545253753662,
+      "logps/chosen": -370.15985107421875,
+      "logps/rejected": -515.8549194335938,
+      "loss": 0.0097,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.4730286598205566,
+      "rewards/margins": 12.967801094055176,
+      "rewards/rejected": -15.440831184387207,
+      "step": 930
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 4.437038800285646e-07,
+      "logits/chosen": -2.399423360824585,
+      "logits/rejected": -2.418363094329834,
+      "logps/chosen": -384.27984619140625,
+      "logps/rejected": -549.5245971679688,
+      "loss": 0.0156,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -3.72330904006958,
+      "rewards/margins": 12.818862915039062,
+      "rewards/rejected": -16.542171478271484,
+      "step": 940
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 4.4251368721732916e-07,
+      "logits/chosen": -2.5278353691101074,
+      "logits/rejected": -2.5364837646484375,
+      "logps/chosen": -329.5386657714844,
+      "logps/rejected": -519.6696166992188,
+      "loss": 0.0223,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4351348876953125,
+      "rewards/margins": 11.446606636047363,
+      "rewards/rejected": -12.881741523742676,
+      "step": 950
+    },
+    {
+      "epoch": 1.23,
+      "learning_rate": 4.413234944060938e-07,
+      "logits/chosen": -2.527299165725708,
+      "logits/rejected": -2.5759024620056152,
+      "logps/chosen": -403.71063232421875,
+      "logps/rejected": -589.4862670898438,
+      "loss": 0.0147,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.19414202868938446,
+      "rewards/margins": 12.035063743591309,
+      "rewards/rejected": -12.229207038879395,
+      "step": 960
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 4.4013330159485834e-07,
+      "logits/chosen": -2.4672398567199707,
+      "logits/rejected": -2.4999210834503174,
+      "logps/chosen": -334.6300048828125,
+      "logps/rejected": -534.4932250976562,
+      "loss": 0.0255,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.7743580341339111,
+      "rewards/margins": 12.416712760925293,
+      "rewards/rejected": -14.191072463989258,
+      "step": 970
+    },
+    {
+      "epoch": 1.26,
+      "learning_rate": 4.3894310878362293e-07,
+      "logits/chosen": -2.447817087173462,
+      "logits/rejected": -2.5005249977111816,
+      "logps/chosen": -338.5157470703125,
+      "logps/rejected": -544.09423828125,
+      "loss": 0.0229,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.8672630786895752,
+      "rewards/margins": 12.040175437927246,
+      "rewards/rejected": -13.907438278198242,
+      "step": 980
+    },
+    {
+      "epoch": 1.27,
+      "learning_rate": 4.377529159723875e-07,
+      "logits/chosen": -2.4685416221618652,
+      "logits/rejected": -2.49491548538208,
+      "logps/chosen": -366.1611022949219,
+      "logps/rejected": -518.9093627929688,
+      "loss": 0.0079,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9218127727508545,
+      "rewards/margins": 11.573265075683594,
+      "rewards/rejected": -13.495076179504395,
+      "step": 990
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 4.365627231611521e-07,
+      "logits/chosen": -2.470853805541992,
+      "logits/rejected": -2.497331380844116,
+      "logps/chosen": -405.1899719238281,
+      "logps/rejected": -591.7445068359375,
+      "loss": 0.0057,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9404414892196655,
+      "rewards/margins": 13.470489501953125,
+      "rewards/rejected": -15.410931587219238,
+      "step": 1000
+    },
+    {
+      "epoch": 1.29,
+      "eval_logits/chosen": -2.5345709323883057,
+      "eval_logits/rejected": -2.507004737854004,
+      "eval_logps/chosen": -336.10919189453125,
+      "eval_logps/rejected": -476.1966552734375,
+      "eval_loss": 0.0556936077773571,
+      "eval_rewards/accuracies": 0.984375,
+      "eval_rewards/chosen": -2.7397918701171875,
+      "eval_rewards/margins": 10.605344772338867,
+      "eval_rewards/rejected": -13.345136642456055,
+      "eval_runtime": 38.7118,
+      "eval_samples_per_second": 12.916,
+      "eval_steps_per_second": 0.413,
+      "step": 1000
+    },
+    {
+      "epoch": 1.3,
+      "learning_rate": 4.3537253034991665e-07,
+      "logits/chosen": -2.441990852355957,
+      "logits/rejected": -2.4507715702056885,
+      "logps/chosen": -329.62542724609375,
+      "logps/rejected": -574.9547729492188,
+      "loss": 0.0214,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -3.120880603790283,
+      "rewards/margins": 13.88032054901123,
+      "rewards/rejected": -17.001201629638672,
+      "step": 1010
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 4.3418233753868124e-07,
+      "logits/chosen": -2.3679394721984863,
+      "logits/rejected": -2.410681962966919,
+      "logps/chosen": -341.8808898925781,
+      "logps/rejected": -532.3084106445312,
+      "loss": 0.0303,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.787487030029297,
+      "rewards/margins": 11.951956748962402,
+      "rewards/rejected": -14.739442825317383,
+      "step": 1020
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 4.3299214472744583e-07,
+      "logits/chosen": -2.4356143474578857,
+      "logits/rejected": -2.484920024871826,
+      "logps/chosen": -378.17376708984375,
+      "logps/rejected": -561.7147216796875,
+      "loss": 0.0212,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.4539060592651367,
+      "rewards/margins": 12.572771072387695,
+      "rewards/rejected": -15.026677131652832,
+      "step": 1030
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 4.3180195191621036e-07,
+      "logits/chosen": -2.4165291786193848,
+      "logits/rejected": -2.3931941986083984,
+      "logps/chosen": -377.8540344238281,
+      "logps/rejected": -555.7592163085938,
+      "loss": 0.0254,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2512832880020142,
+      "rewards/margins": 12.33320426940918,
+      "rewards/rejected": -13.58448600769043,
+      "step": 1040
+    },
+    {
+      "epoch": 1.35,
+      "learning_rate": 4.30611759104975e-07,
+      "logits/chosen": -2.3533992767333984,
+      "logits/rejected": -2.3296687602996826,
+      "logps/chosen": -418.5027770996094,
+      "logps/rejected": -600.8396606445312,
+      "loss": 0.0201,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.496593952178955,
+      "rewards/margins": 13.320207595825195,
+      "rewards/rejected": -15.816801071166992,
+      "step": 1050
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 4.2942156629373954e-07,
+      "logits/chosen": -2.246854782104492,
+      "logits/rejected": -2.3130173683166504,
+      "logps/chosen": -396.1013488769531,
+      "logps/rejected": -553.8746337890625,
+      "loss": 0.0209,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.86772084236145,
+      "rewards/margins": 13.22656536102295,
+      "rewards/rejected": -16.094287872314453,
+      "step": 1060
+    },
+    {
+      "epoch": 1.38,
+      "learning_rate": 4.2823137348250413e-07,
+      "logits/chosen": -2.1099252700805664,
+      "logits/rejected": -2.1625306606292725,
+      "logps/chosen": -439.188232421875,
+      "logps/rejected": -567.4981689453125,
+      "loss": 0.0195,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -3.4562058448791504,
+      "rewards/margins": 11.824674606323242,
+      "rewards/rejected": -15.280881881713867,
+      "step": 1070
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 4.270411806712687e-07,
+      "logits/chosen": -2.182868480682373,
+      "logits/rejected": -2.140045642852783,
+      "logps/chosen": -414.1625061035156,
+      "logps/rejected": -590.7791748046875,
+      "loss": 0.0203,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.0633182525634766,
+      "rewards/margins": 13.505340576171875,
+      "rewards/rejected": -15.568659782409668,
+      "step": 1080
+    },
+    {
+      "epoch": 1.4,
+      "learning_rate": 4.258509878600333e-07,
+      "logits/chosen": -2.301701068878174,
+      "logits/rejected": -2.3724331855773926,
+      "logps/chosen": -318.6136779785156,
+      "logps/rejected": -549.11572265625,
+      "loss": 0.0162,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7360296249389648,
+      "rewards/margins": 12.463074684143066,
+      "rewards/rejected": -14.199106216430664,
+      "step": 1090
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 4.2466079504879785e-07,
+      "logits/chosen": -2.3375637531280518,
+      "logits/rejected": -2.371568202972412,
+      "logps/chosen": -355.43218994140625,
+      "logps/rejected": -497.6923828125,
+      "loss": 0.0296,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.8382488489151,
+      "rewards/margins": 11.133204460144043,
+      "rewards/rejected": -12.971455574035645,
+      "step": 1100
+    },
+    {
+      "epoch": 1.41,
+      "eval_logits/chosen": -2.422253131866455,
+      "eval_logits/rejected": -2.3856472969055176,
+      "eval_logps/chosen": -327.49688720703125,
+      "eval_logps/rejected": -458.99761962890625,
+      "eval_loss": 0.0712868794798851,
+      "eval_rewards/accuracies": 0.953125,
+      "eval_rewards/chosen": -1.8785579204559326,
+      "eval_rewards/margins": 9.746674537658691,
+      "eval_rewards/rejected": -11.625232696533203,
+      "eval_runtime": 38.5688,
+      "eval_samples_per_second": 12.964,
+      "eval_steps_per_second": 0.415,
+      "step": 1100
+    },
+    {
+      "epoch": 1.43,
+      "learning_rate": 4.234706022375625e-07,
+      "logits/chosen": -2.335549831390381,
+      "logits/rejected": -2.3633885383605957,
+      "logps/chosen": -334.0445251464844,
+      "logps/rejected": -532.0367431640625,
+      "loss": 0.0173,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.1568909883499146,
+      "rewards/margins": 12.151830673217773,
+      "rewards/rejected": -13.308721542358398,
+      "step": 1110
+    },
+    {
+      "epoch": 1.44,
+      "learning_rate": 4.2228040942632703e-07,
+      "logits/chosen": -2.2730376720428467,
+      "logits/rejected": -2.279794931411743,
+      "logps/chosen": -372.47711181640625,
+      "logps/rejected": -565.377197265625,
+      "loss": 0.0135,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.948188066482544,
+      "rewards/margins": 12.926470756530762,
+      "rewards/rejected": -14.874661445617676,
+      "step": 1120
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 4.210902166150916e-07,
+      "logits/chosen": -2.1850171089172363,
+      "logits/rejected": -2.2554237842559814,
+      "logps/chosen": -330.89398193359375,
+      "logps/rejected": -572.4408569335938,
+      "loss": 0.0152,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.2409050464630127,
+      "rewards/margins": 15.152783393859863,
+      "rewards/rejected": -17.393688201904297,
+      "step": 1130
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 4.199000238038562e-07,
+      "logits/chosen": -2.2348155975341797,
+      "logits/rejected": -2.276552200317383,
+      "logps/chosen": -391.0440673828125,
+      "logps/rejected": -562.8758544921875,
+      "loss": 0.0083,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.518620014190674,
+      "rewards/margins": 13.422780990600586,
+      "rewards/rejected": -15.941401481628418,
+      "step": 1140
+    },
+    {
+      "epoch": 1.48,
+      "learning_rate": 4.187098309926208e-07,
+      "logits/chosen": -2.234314441680908,
+      "logits/rejected": -2.273665428161621,
+      "logps/chosen": -379.77752685546875,
+      "logps/rejected": -609.7650146484375,
+      "loss": 0.0167,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.2126364707946777,
+      "rewards/margins": 14.33509635925293,
+      "rewards/rejected": -16.547733306884766,
+      "step": 1150
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 4.1751963818138534e-07,
+      "logits/chosen": -2.2460713386535645,
+      "logits/rejected": -2.28529953956604,
+      "logps/chosen": -391.7981872558594,
+      "logps/rejected": -584.82373046875,
+      "loss": 0.0106,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.583667278289795,
+      "rewards/margins": 13.928072929382324,
+      "rewards/rejected": -16.511741638183594,
+      "step": 1160
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 4.1632944537015e-07,
+      "logits/chosen": -2.312187671661377,
+      "logits/rejected": -2.313152313232422,
+      "logps/chosen": -332.22418212890625,
+      "logps/rejected": -550.9510498046875,
+      "loss": 0.0151,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.8060202598571777,
+      "rewards/margins": 13.428415298461914,
+      "rewards/rejected": -16.23443603515625,
+      "step": 1170
+    },
+    {
+      "epoch": 1.52,
+      "learning_rate": 4.151392525589145e-07,
+      "logits/chosen": -2.269207715988159,
+      "logits/rejected": -2.2718236446380615,
+      "logps/chosen": -332.3182067871094,
+      "logps/rejected": -509.44085693359375,
+      "loss": 0.0267,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -3.1180636882781982,
+      "rewards/margins": 12.261663436889648,
+      "rewards/rejected": -15.379727363586426,
+      "step": 1180
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 4.139490597476791e-07,
+      "logits/chosen": -2.2478084564208984,
+      "logits/rejected": -2.3000128269195557,
+      "logps/chosen": -337.1382141113281,
+      "logps/rejected": -537.2418212890625,
+      "loss": 0.0108,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.009748935699463,
+      "rewards/margins": 12.527368545532227,
+      "rewards/rejected": -14.537118911743164,
+      "step": 1190
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 4.127588669364437e-07,
+      "logits/chosen": -2.313680648803711,
+      "logits/rejected": -2.327012538909912,
+      "logps/chosen": -291.6064758300781,
+      "logps/rejected": -546.3372802734375,
+      "loss": 0.0148,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -3.4880402088165283,
+      "rewards/margins": 14.52784252166748,
+      "rewards/rejected": -18.01588249206543,
+      "step": 1200
+    },
+    {
+      "epoch": 1.54,
+      "eval_logits/chosen": -2.3877577781677246,
+      "eval_logits/rejected": -2.35882568359375,
+      "eval_logps/chosen": -347.202880859375,
+      "eval_logps/rejected": -496.7171325683594,
+      "eval_loss": 0.07778895646333694,
+      "eval_rewards/accuracies": 0.953125,
+      "eval_rewards/chosen": -3.8491578102111816,
+      "eval_rewards/margins": 11.548023223876953,
+      "eval_rewards/rejected": -15.397181510925293,
+      "eval_runtime": 38.6215,
+      "eval_samples_per_second": 12.946,
+      "eval_steps_per_second": 0.414,
+      "step": 1200
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 4.115686741252083e-07,
+      "logits/chosen": -2.292132616043091,
+      "logits/rejected": -2.347907781600952,
+      "logps/chosen": -362.74481201171875,
+      "logps/rejected": -558.0933837890625,
+      "loss": 0.0163,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.724060297012329,
+      "rewards/margins": 13.57036304473877,
+      "rewards/rejected": -16.294422149658203,
+      "step": 1210
+    },
+    {
+      "epoch": 1.57,
+      "learning_rate": 4.103784813139728e-07,
+      "logits/chosen": -2.3167264461517334,
+      "logits/rejected": -2.3449079990386963,
+      "logps/chosen": -369.4256591796875,
+      "logps/rejected": -566.0360107421875,
+      "loss": 0.0155,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -3.4625415802001953,
+      "rewards/margins": 13.401751518249512,
+      "rewards/rejected": -16.86429214477539,
+      "step": 1220
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 4.091882885027374e-07,
+      "logits/chosen": -2.3674325942993164,
+      "logits/rejected": -2.455508232116699,
+      "logps/chosen": -381.26068115234375,
+      "logps/rejected": -550.90625,
+      "loss": 0.0244,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -3.13775897026062,
+      "rewards/margins": 13.795980453491211,
+      "rewards/rejected": -16.933740615844727,
+      "step": 1230
+    },
+    {
+      "epoch": 1.59,
+      "learning_rate": 4.07998095691502e-07,
+      "logits/chosen": -2.3083391189575195,
+      "logits/rejected": -2.330939769744873,
+      "logps/chosen": -362.44171142578125,
+      "logps/rejected": -523.51171875,
+      "loss": 0.0124,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.1269755363464355,
+      "rewards/margins": 12.574740409851074,
+      "rewards/rejected": -15.701716423034668,
+      "step": 1240
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 4.0680790288026654e-07,
+      "logits/chosen": -2.3918001651763916,
+      "logits/rejected": -2.426542282104492,
+      "logps/chosen": -420.2566833496094,
+      "logps/rejected": -605.3551025390625,
+      "loss": 0.0202,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.9122402667999268,
+      "rewards/margins": 13.67309856414795,
+      "rewards/rejected": -16.585338592529297,
+      "step": 1250
+    },
+    {
+      "epoch": 1.62,
+      "learning_rate": 4.056177100690312e-07,
+      "logits/chosen": -2.2674708366394043,
+      "logits/rejected": -2.2906508445739746,
+      "logps/chosen": -390.3266296386719,
+      "logps/rejected": -587.2613525390625,
+      "loss": 0.011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -4.047337532043457,
+      "rewards/margins": 13.966493606567383,
+      "rewards/rejected": -18.013832092285156,
+      "step": 1260
+    },
+    {
+      "epoch": 1.63,
+      "learning_rate": 4.044275172577957e-07,
+      "logits/chosen": -2.256685733795166,
+      "logits/rejected": -2.283980131149292,
+      "logps/chosen": -307.6758728027344,
+      "logps/rejected": -536.929931640625,
+      "loss": 0.0251,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.8640975952148438,
+      "rewards/margins": 14.663250923156738,
+      "rewards/rejected": -17.527347564697266,
+      "step": 1270
+    },
+    {
+      "epoch": 1.65,
+      "learning_rate": 4.0323732444656036e-07,
+      "logits/chosen": -2.2302117347717285,
+      "logits/rejected": -2.319187641143799,
+      "logps/chosen": -377.6014099121094,
+      "logps/rejected": -592.4954223632812,
+      "loss": 0.0208,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.1300606727600098,
+      "rewards/margins": 14.494562149047852,
+      "rewards/rejected": -16.624622344970703,
+      "step": 1280
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 4.020471316353249e-07,
+      "logits/chosen": -2.3077661991119385,
+      "logits/rejected": -2.34450364112854,
+      "logps/chosen": -384.89007568359375,
+      "logps/rejected": -577.9298095703125,
+      "loss": 0.0126,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.4994373321533203,
+      "rewards/margins": 12.733844757080078,
+      "rewards/rejected": -14.233282089233398,
+      "step": 1290
+    },
+    {
+      "epoch": 1.67,
+      "learning_rate": 4.008569388240895e-07,
+      "logits/chosen": -2.230447292327881,
+      "logits/rejected": -2.283294677734375,
+      "logps/chosen": -346.1694641113281,
+      "logps/rejected": -534.3992919921875,
+      "loss": 0.019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3882415294647217,
+      "rewards/margins": 14.5983247756958,
+      "rewards/rejected": -15.986566543579102,
+      "step": 1300
+    },
+    {
+      "epoch": 1.67,
+      "eval_logits/chosen": -2.403440475463867,
+      "eval_logits/rejected": -2.378675699234009,
+      "eval_logps/chosen": -332.9962463378906,
+      "eval_logps/rejected": -477.9118957519531,
+      "eval_loss": 0.07047431915998459,
+      "eval_rewards/accuracies": 0.9375,
+      "eval_rewards/chosen": -2.4284939765930176,
+      "eval_rewards/margins": 11.088165283203125,
+      "eval_rewards/rejected": -13.516657829284668,
+      "eval_runtime": 38.6695,
+      "eval_samples_per_second": 12.93,
+      "eval_steps_per_second": 0.414,
+      "step": 1300
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 3.996667460128541e-07,
+      "logits/chosen": -2.31799578666687,
+      "logits/rejected": -2.3302206993103027,
+      "logps/chosen": -333.87261962890625,
+      "logps/rejected": -506.0113220214844,
+      "loss": 0.0166,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.8233000040054321,
+      "rewards/margins": 13.524618148803711,
+      "rewards/rejected": -15.347920417785645,
+      "step": 1310
+    },
+    {
+      "epoch": 1.7,
+      "learning_rate": 3.9847655320161867e-07,
+      "logits/chosen": -2.3380274772644043,
+      "logits/rejected": -2.3655543327331543,
+      "logps/chosen": -330.939453125,
+      "logps/rejected": -566.5387573242188,
+      "loss": 0.0211,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.293247938156128,
+      "rewards/margins": 13.109285354614258,
+      "rewards/rejected": -16.402530670166016,
+      "step": 1320
+    },
+    {
+      "epoch": 1.71,
+      "learning_rate": 3.972863603903832e-07,
+      "logits/chosen": -2.4296791553497314,
+      "logits/rejected": -2.395019054412842,
+      "logps/chosen": -368.58843994140625,
+      "logps/rejected": -550.57177734375,
+      "loss": 0.0147,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.1659107208251953,
+      "rewards/margins": 14.171772956848145,
+      "rewards/rejected": -16.337684631347656,
+      "step": 1330
+    },
+    {
+      "epoch": 1.72,
+      "learning_rate": 3.9609616757914784e-07,
+      "logits/chosen": -2.386429786682129,
+      "logits/rejected": -2.401638984680176,
+      "logps/chosen": -347.26214599609375,
+      "logps/rejected": -538.3074951171875,
+      "loss": 0.0162,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.806589126586914,
+      "rewards/margins": 12.520380973815918,
+      "rewards/rejected": -15.326970100402832,
+      "step": 1340
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 3.949059747679124e-07,
+      "logits/chosen": -2.3784899711608887,
+      "logits/rejected": -2.42669939994812,
+      "logps/chosen": -364.69512939453125,
+      "logps/rejected": -592.1053466796875,
+      "loss": 0.0159,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -3.1103992462158203,
+      "rewards/margins": 15.538830757141113,
+      "rewards/rejected": -18.649229049682617,
+      "step": 1350
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 3.9371578195667697e-07,
+      "logits/chosen": -2.4179718494415283,
+      "logits/rejected": -2.4337425231933594,
+      "logps/chosen": -338.0289001464844,
+      "logps/rejected": -522.47412109375,
+      "loss": 0.0343,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.291858196258545,
+      "rewards/margins": 14.658024787902832,
+      "rewards/rejected": -16.949880599975586,
+      "step": 1360
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 3.9252558914544156e-07,
+      "logits/chosen": -2.37274169921875,
+      "logits/rejected": -2.376906633377075,
+      "logps/chosen": -371.0089111328125,
+      "logps/rejected": -562.0587158203125,
+      "loss": 0.0236,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.699599504470825,
+      "rewards/margins": 14.066309928894043,
+      "rewards/rejected": -16.76590919494629,
+      "step": 1370
+    },
+    {
+      "epoch": 1.77,
+      "learning_rate": 3.9133539633420615e-07,
+      "logits/chosen": -2.3570303916931152,
+      "logits/rejected": -2.4414098262786865,
+      "logps/chosen": -347.50531005859375,
+      "logps/rejected": -606.2113647460938,
+      "loss": 0.015,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.3379924297332764,
+      "rewards/margins": 14.867982864379883,
+      "rewards/rejected": -17.205974578857422,
+      "step": 1380
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 3.901452035229707e-07,
+      "logits/chosen": -2.373347043991089,
+      "logits/rejected": -2.4218459129333496,
+      "logps/chosen": -421.48187255859375,
+      "logps/rejected": -606.8762817382812,
+      "loss": 0.0132,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.6556594371795654,
+      "rewards/margins": 14.492483139038086,
+      "rewards/rejected": -17.148143768310547,
+      "step": 1390
+    },
+    {
+      "epoch": 1.8,
+      "learning_rate": 3.8895501071173533e-07,
+      "logits/chosen": -2.3142504692077637,
+      "logits/rejected": -2.3538806438446045,
+      "logps/chosen": -325.9708557128906,
+      "logps/rejected": -511.67449951171875,
+      "loss": 0.0214,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.7798726558685303,
+      "rewards/margins": 13.109631538391113,
+      "rewards/rejected": -15.889503479003906,
+      "step": 1400
+    },
+    {
+      "epoch": 1.8,
+      "eval_logits/chosen": -2.3960964679718018,
+      "eval_logits/rejected": -2.3517098426818848,
+      "eval_logps/chosen": -346.35821533203125,
+      "eval_logps/rejected": -495.85186767578125,
+      "eval_loss": 0.07910314947366714,
+      "eval_rewards/accuracies": 0.96875,
+      "eval_rewards/chosen": -3.7646918296813965,
+      "eval_rewards/margins": 11.545960426330566,
+      "eval_rewards/rejected": -15.310651779174805,
+      "eval_runtime": 38.7173,
+      "eval_samples_per_second": 12.914,
+      "eval_steps_per_second": 0.413,
+      "step": 1400
+    },
+    {
+      "epoch": 1.81,
+      "learning_rate": 3.8776481790049987e-07,
+      "logits/chosen": -2.3062312602996826,
+      "logits/rejected": -2.3327383995056152,
+      "logps/chosen": -354.59381103515625,
+      "logps/rejected": -503.6541442871094,
+      "loss": 0.0196,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -3.5312271118164062,
+      "rewards/margins": 13.276026725769043,
+      "rewards/rejected": -16.807254791259766,
+      "step": 1410
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 3.865746250892644e-07,
+      "logits/chosen": -2.396146774291992,
+      "logits/rejected": -2.3744444847106934,
+      "logps/chosen": -397.74609375,
+      "logps/rejected": -583.1174926757812,
+      "loss": 0.0162,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.2743606567382812,
+      "rewards/margins": 15.37347412109375,
+      "rewards/rejected": -18.647836685180664,
+      "step": 1420
+    },
+    {
+      "epoch": 1.84,
+      "learning_rate": 3.8538443227802905e-07,
+      "logits/chosen": -2.3621578216552734,
+      "logits/rejected": -2.3470935821533203,
+      "logps/chosen": -374.19757080078125,
+      "logps/rejected": -564.0121459960938,
+      "loss": 0.022,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -4.340083122253418,
+      "rewards/margins": 13.78313159942627,
+      "rewards/rejected": -18.123212814331055,
+      "step": 1430
+    },
+    {
+      "epoch": 1.85,
+      "learning_rate": 3.841942394667936e-07,
+      "logits/chosen": -2.304884672164917,
+      "logits/rejected": -2.4029793739318848,
+      "logps/chosen": -369.39898681640625,
+      "logps/rejected": -578.387451171875,
+      "loss": 0.0146,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.7867379188537598,
+      "rewards/margins": 14.443509101867676,
+      "rewards/rejected": -17.23024559020996,
+      "step": 1440
+    },
+    {
+      "epoch": 1.86,
+      "learning_rate": 3.8300404665555817e-07,
+      "logits/chosen": -2.2816107273101807,
+      "logits/rejected": -2.2829480171203613,
+      "logps/chosen": -374.7585144042969,
+      "logps/rejected": -540.5015869140625,
+      "loss": 0.0164,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.205556869506836,
+      "rewards/margins": 14.464788436889648,
+      "rewards/rejected": -16.670345306396484,
+      "step": 1450
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 3.8181385384432276e-07,
+      "logits/chosen": -2.282743453979492,
+      "logits/rejected": -2.2942354679107666,
+      "logps/chosen": -394.46502685546875,
+      "logps/rejected": -594.6571044921875,
+      "loss": 0.0112,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.892620325088501,
+      "rewards/margins": 14.386013984680176,
+      "rewards/rejected": -17.27863311767578,
+      "step": 1460
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 3.8062366103308735e-07,
+      "logits/chosen": -2.2720725536346436,
+      "logits/rejected": -2.245262622833252,
+      "logps/chosen": -342.9836730957031,
+      "logps/rejected": -546.7418212890625,
+      "loss": 0.0365,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.7557284832000732,
+      "rewards/margins": 14.667689323425293,
+      "rewards/rejected": -17.423416137695312,
+      "step": 1470
+    },
+    {
+      "epoch": 1.9,
+      "learning_rate": 3.794334682218519e-07,
+      "logits/chosen": -2.295213222503662,
+      "logits/rejected": -2.3375067710876465,
+      "logps/chosen": -370.61798095703125,
+      "logps/rejected": -474.4059143066406,
+      "loss": 0.0237,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -3.196665048599243,
+      "rewards/margins": 12.084269523620605,
+      "rewards/rejected": -15.28093433380127,
+      "step": 1480
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 3.7824327541061653e-07,
+      "logits/chosen": -2.4100170135498047,
+      "logits/rejected": -2.4586007595062256,
+      "logps/chosen": -358.7035217285156,
+      "logps/rejected": -547.9478149414062,
+      "loss": 0.0184,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.731393337249756,
+      "rewards/margins": 12.000238418579102,
+      "rewards/rejected": -15.73162841796875,
+      "step": 1490
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 3.7705308259938107e-07,
+      "logits/chosen": -2.432584047317505,
+      "logits/rejected": -2.430572032928467,
+      "logps/chosen": -400.4476318359375,
+      "logps/rejected": -589.388427734375,
+      "loss": 0.0124,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.0731418132781982,
+      "rewards/margins": 13.324457168579102,
+      "rewards/rejected": -16.397600173950195,
+      "step": 1500
+    },
+    {
+      "epoch": 1.93,
+      "eval_logits/chosen": -2.4233508110046387,
+      "eval_logits/rejected": -2.3732004165649414,
+      "eval_logps/chosen": -345.49517822265625,
+      "eval_logps/rejected": -491.72662353515625,
+      "eval_loss": 0.08803335577249527,
+      "eval_rewards/accuracies": 0.9375,
+      "eval_rewards/chosen": -3.678384304046631,
+      "eval_rewards/margins": 11.219746589660645,
+      "eval_rewards/rejected": -14.89813232421875,
+      "eval_runtime": 38.608,
+      "eval_samples_per_second": 12.951,
+      "eval_steps_per_second": 0.414,
+      "step": 1500
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 3.7586288978814566e-07,
+      "logits/chosen": -2.335282564163208,
+      "logits/rejected": -2.330732583999634,
+      "logps/chosen": -388.20806884765625,
+      "logps/rejected": -580.2225341796875,
+      "loss": 0.0118,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.1078379154205322,
+      "rewards/margins": 13.242405891418457,
+      "rewards/rejected": -16.350242614746094,
+      "step": 1510
+    },
+    {
+      "epoch": 1.95,
+      "learning_rate": 3.7467269697691025e-07,
+      "logits/chosen": -2.3464579582214355,
+      "logits/rejected": -2.3436694145202637,
+      "logps/chosen": -335.885986328125,
+      "logps/rejected": -532.0635986328125,
+      "loss": 0.0328,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.1488466262817383,
+      "rewards/margins": 13.591397285461426,
+      "rewards/rejected": -16.740243911743164,
+      "step": 1520
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 3.7348250416567484e-07,
+      "logits/chosen": -2.2621750831604004,
+      "logits/rejected": -2.2600533962249756,
+      "logps/chosen": -415.00982666015625,
+      "logps/rejected": -549.5345458984375,
+      "loss": 0.0264,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.42472767829895,
+      "rewards/margins": 13.469167709350586,
+      "rewards/rejected": -15.893896102905273,
+      "step": 1530
+    },
+    {
+      "epoch": 1.98,
+      "learning_rate": 3.722923113544394e-07,
+      "logits/chosen": -2.361262559890747,
+      "logits/rejected": -2.315338611602783,
+      "logps/chosen": -394.708740234375,
+      "logps/rejected": -578.1019287109375,
+      "loss": 0.0251,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.082348585128784,
+      "rewards/margins": 15.23118782043457,
+      "rewards/rejected": -17.31353759765625,
+      "step": 1540
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 3.71102118543204e-07,
+      "logits/chosen": -2.315455913543701,
+      "logits/rejected": -2.284585952758789,
+      "logps/chosen": -367.0815734863281,
+      "logps/rejected": -577.2198486328125,
+      "loss": 0.0113,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.139265537261963,
+      "rewards/margins": 14.051069259643555,
+      "rewards/rejected": -16.19033432006836,
+      "step": 1550
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 4668,
+  "num_train_epochs": 6,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f09f6ebae60f5cf7aa9bc7b8e6d25455a48190b29ccdc77065805f0e5acf97ca
+size 6008