Model save

README.md

@@ -1,17 +1,11 @@
 ---
 license: other
-base_model: Columbia-NLP/gemma-2b-zephyr-sft
+base_model: HuggingFaceH4/zephyr-7b-gemma-sft-v0.1
 tags:
-- alignment-handbook
-- trl
-- dpo
-- generated_from_trainer
 - trl
 - dpo
 - alignment-handbook
 - generated_from_trainer
-datasets:
-- argilla/dpo-mix-7k
 model-index:
 - name: zephyr-7b-gemma-dpo
   results: []
@@ -22,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-gemma-dpo
 
-This model is a fine-tuned version of [Columbia-NLP/gemma-2b-zephyr-sft](https://huggingface.co/Columbia-NLP/gemma-2b-zephyr-sft) on the argilla/dpo-mix-7k dataset.
+This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5964
-- Rewards/chosen: 0.3541
-- Rewards/rejected: 0.1118
-- Rewards/accuracies: 0.6562
-- Rewards/margins: 0.2424
-- Logps/rejected: -373.3665
-- Logps/chosen: -371.9911
-- Logits/rejected: -12.6656
-- Logits/chosen: -13.2953
+- Loss: 0.4673
+- Rewards/chosen: -4.1293
+- Rewards/rejected: -5.7148
+- Rewards/accuracies: 0.6979
+- Rewards/margins: 1.5855
+- Logps/rejected: -476.3664
+- Logps/chosen: -446.2537
+- Logits/rejected: 91.1323
+- Logits/chosen: 96.7315
 
 ## Model description
 
@@ -69,7 +63,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5732        | 1.8957 | 100  | 0.5972          | 0.3534         | 0.1096           | 0.6771             | 0.2437          | -373.4094      | -372.0067    | -12.6654        | -13.2926      |
+| 0.1585        | 1.8957 | 100  | 0.4673          | -4.1293        | -5.7148          | 0.6979             | 1.5855          | -476.3664      | -446.2537    | 91.1323         | 96.7315       |
 
 
 ### Framework versions

all_results.json

@@ -14,9 +14,9 @@
     "eval_samples_per_second": 51.65,
     "eval_steps_per_second": 1.653,
     "total_flos": 0.0,
-    "train_loss": 0.627926590350958,
-    "train_runtime": 756.4701,
+    "train_loss": 0.39153398688022906,
+    "train_runtime": 2311.0387,
     "train_samples": 6750,
-    "train_samples_per_second": 17.846,
-    "train_steps_per_second": 0.137
+    "train_samples_per_second": 5.842,
+    "train_steps_per_second": 0.045
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "Columbia-NLP/gemma-2b-zephyr-sft",
+  "_name_or_path": "HuggingFaceH4/zephyr-7b-gemma-sft-v0.1",
   "architectures": [
     "GemmaForCausalLM"
   ],
@@ -10,20 +10,20 @@
   "head_dim": 256,
   "hidden_act": "gelu",
   "hidden_activation": null,
-  "hidden_size": 2048,
+  "hidden_size": 3072,
   "initializer_range": 0.02,
-  "intermediate_size": 16384,
+  "intermediate_size": 24576,
   "max_position_embeddings": 8192,
   "model_type": "gemma",
-  "num_attention_heads": 8,
-  "num_hidden_layers": 18,
-  "num_key_value_heads": 1,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 16,
   "pad_token_id": 0,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.1",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 256000
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c5a7396241c9049d5aa615a0081a16f922ac27a595fad308da7be35b297a132
+oid sha256:b26faf7dcff7b7ca7bbf4ebc9d54968ab56cd1bbe5b3f4059d0ec34b7b1ccdd2
 size 4995496656

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd5dd371acf66dbd26a13036c2b42efee4bf540edd178d9cb3a225eccd87d21c
+oid sha256:98a41992ae6f80c80eaa24e7d8dbecab5d07c2802028c109568fe70565b4c6d8
 size 4982953168

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:836f574fd62d28f4d282b81d3712708535d3eadc45ddd0509c932611129327c5
+oid sha256:5d1d7ab5de3f2e26234060bf0c99e343d3a84489614f455b267bd22f059fc862
 size 4982953200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85fb13399e2932fccc4212b3907e81ae00bcdf3dbbb785a234bceaa501d29091
+oid sha256:07f5983e107d05b629942a14afa7af7fe7e3836b05bc872e472789542c0f95b6
 size 2113988336

model.safetensors.index.json

@@ -1,171 +1,261 @@
 {
   "metadata": {
-    "total_size": 5012344832
+    "total_size": 17075361792
   },
   "weight_map": {
-    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.norm.weight": "model-00002-of-00002.safetensors"
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
   }
 }

runs/Apr27_16-12-58_660111d13776/events.out.tfevents.1714230903.660111d13776.64079.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b97fd433bfa6de3447258f58730d6d4afae52966b1f8d0359a70e2097f3673e6
+size 9663

runs/Apr27_19-16-30_660111d13776/events.out.tfevents.1714241901.660111d13776.66670.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:945218278ec2aa200d3ae01a36d4e1429890a97741bbdfd88e455df07f081f0c
+size 13446

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.971563981042654,
     "total_flos": 0.0,
-    "train_loss": 0.627926590350958,
-    "train_runtime": 756.4701,
+    "train_loss": 0.39153398688022906,
+    "train_runtime": 2311.0387,
     "train_samples": 6750,
-    "train_samples_per_second": 17.846,
-    "train_steps_per_second": 0.137
+    "train_samples_per_second": 5.842,
+    "train_steps_per_second": 0.045
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.018957345971563982,
-      "grad_norm": 15.786988646394411,
+      "grad_norm": 132.15360444004384,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": -13.905267715454102,
-      "logits/rejected": -14.118387222290039,
-      "logps/chosen": -350.8895263671875,
-      "logps/rejected": -446.6286926269531,
+      "logits/chosen": 117.53560638427734,
+      "logits/rejected": 126.8960952758789,
+      "logps/chosen": -335.40118408203125,
+      "logps/rejected": -439.16552734375,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,178 +25,178 @@
     },
     {
       "epoch": 0.1895734597156398,
-      "grad_norm": 15.908099576913655,
+      "grad_norm": 132.3674027987073,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": -14.040081024169922,
-      "logits/rejected": -14.157392501831055,
-      "logps/chosen": -416.2701416015625,
-      "logps/rejected": -449.4697265625,
-      "loss": 0.693,
-      "rewards/accuracies": 0.5138888955116272,
-      "rewards/chosen": 0.0006088384543545544,
-      "rewards/margins": 0.008209776133298874,
-      "rewards/rejected": -0.007600938435643911,
+      "logits/chosen": 135.01699829101562,
+      "logits/rejected": 138.37664794921875,
+      "logps/chosen": -396.05718994140625,
+      "logps/rejected": -439.1203918457031,
+      "loss": 0.7127,
+      "rewards/accuracies": 0.4583333432674408,
+      "rewards/chosen": -0.0030322629027068615,
+      "rewards/margins": -0.013390823267400265,
+      "rewards/rejected": 0.010358559899032116,
       "step": 10
     },
     {
       "epoch": 0.3791469194312796,
-      "grad_norm": 14.229474825008781,
+      "grad_norm": 131.21733523095625,
       "learning_rate": 4.885348141000122e-07,
-      "logits/chosen": -13.39338207244873,
-      "logits/rejected": -13.542058944702148,
-      "logps/chosen": -392.9753723144531,
-      "logps/rejected": -427.68096923828125,
-      "loss": 0.6892,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": 0.010071685537695885,
-      "rewards/margins": 0.003802267834544182,
-      "rewards/rejected": 0.006269416771829128,
+      "logits/chosen": 121.60444641113281,
+      "logits/rejected": 125.29842376708984,
+      "logps/chosen": -370.2664489746094,
+      "logps/rejected": -422.78851318359375,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.10727670043706894,
+      "rewards/margins": 0.247134730219841,
+      "rewards/rejected": -0.13985800743103027,
       "step": 20
     },
     {
       "epoch": 0.5687203791469194,
-      "grad_norm": 15.853985724357454,
+      "grad_norm": 117.90232463642135,
       "learning_rate": 4.5025027361734613e-07,
-      "logits/chosen": -14.269427299499512,
-      "logits/rejected": -13.808093070983887,
-      "logps/chosen": -412.9443359375,
-      "logps/rejected": -428.38494873046875,
-      "loss": 0.674,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.04771440848708153,
-      "rewards/margins": 0.035354893654584885,
-      "rewards/rejected": 0.012359511107206345,
+      "logits/chosen": 142.974853515625,
+      "logits/rejected": 136.52386474609375,
+      "logps/chosen": -424.7781677246094,
+      "logps/rejected": -469.64813232421875,
+      "loss": 0.5746,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6156466007232666,
+      "rewards/margins": 0.8666501045227051,
+      "rewards/rejected": -2.4822967052459717,
       "step": 30
     },
     {
       "epoch": 0.7582938388625592,
-      "grad_norm": 14.687978809678542,
+      "grad_norm": 104.91283452119073,
       "learning_rate": 3.893311157806091e-07,
-      "logits/chosen": -13.886492729187012,
-      "logits/rejected": -13.28197956085205,
-      "logps/chosen": -374.98211669921875,
-      "logps/rejected": -366.5968322753906,
-      "loss": 0.657,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.13442906737327576,
-      "rewards/margins": 0.07902240008115768,
-      "rewards/rejected": 0.05540664866566658,
+      "logits/chosen": 126.9936752319336,
+      "logits/rejected": 115.53365325927734,
+      "logps/chosen": -399.81353759765625,
+      "logps/rejected": -426.99853515625,
+      "loss": 0.5456,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.2809689044952393,
+      "rewards/margins": 1.1751956939697266,
+      "rewards/rejected": -3.456164598464966,
       "step": 40
     },
     {
       "epoch": 0.9478672985781991,
-      "grad_norm": 15.872142673244408,
+      "grad_norm": 123.57780236639618,
       "learning_rate": 3.126631330646801e-07,
-      "logits/chosen": -14.917936325073242,
-      "logits/rejected": -14.90648078918457,
-      "logps/chosen": -429.6836853027344,
-      "logps/rejected": -480.3504943847656,
-      "loss": 0.6344,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.24091288447380066,
-      "rewards/margins": 0.1330389827489853,
-      "rewards/rejected": 0.10787389427423477,
+      "logits/chosen": 142.1190643310547,
+      "logits/rejected": 146.2515411376953,
+      "logps/chosen": -456.97979736328125,
+      "logps/rejected": -540.1392822265625,
+      "loss": 0.489,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.891798734664917,
+      "rewards/margins": 1.2988468408584595,
+      "rewards/rejected": -3.190645456314087,
       "step": 50
     },
     {
       "epoch": 1.1374407582938388,
-      "grad_norm": 14.061428605486398,
+      "grad_norm": 67.1680971334559,
       "learning_rate": 2.2891223348923882e-07,
-      "logits/chosen": -14.622962951660156,
-      "logits/rejected": -14.403157234191895,
-      "logps/chosen": -415.7464904785156,
-      "logps/rejected": -441.731201171875,
-      "loss": 0.6063,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.3395090103149414,
-      "rewards/margins": 0.22218124568462372,
-      "rewards/rejected": 0.11732780933380127,
+      "logits/chosen": 133.56114196777344,
+      "logits/rejected": 137.20738220214844,
+      "logps/chosen": -449.55303955078125,
+      "logps/rejected": -534.8367919921875,
+      "loss": 0.3117,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -2.5773684978485107,
+      "rewards/margins": 2.346193790435791,
+      "rewards/rejected": -4.923562049865723,
       "step": 60
     },
     {
       "epoch": 1.3270142180094786,
-      "grad_norm": 12.963152293888875,
+      "grad_norm": 48.54475300946312,
       "learning_rate": 1.4754491880085317e-07,
-      "logits/chosen": -14.022384643554688,
-      "logits/rejected": -13.828951835632324,
-      "logps/chosen": -382.23468017578125,
-      "logps/rejected": -418.2818908691406,
-      "loss": 0.6011,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.3396778702735901,
-      "rewards/margins": 0.22157195210456848,
-      "rewards/rejected": 0.118105947971344,
+      "logits/chosen": 125.71492004394531,
+      "logits/rejected": 127.68719482421875,
+      "logps/chosen": -426.90228271484375,
+      "logps/rejected": -528.0679321289062,
+      "loss": 0.195,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -2.9879212379455566,
+      "rewards/margins": 2.7397806644439697,
+      "rewards/rejected": -5.727701663970947,
       "step": 70
     },
     {
       "epoch": 1.5165876777251186,
-      "grad_norm": 12.394681314131397,
+      "grad_norm": 55.371866892062,
       "learning_rate": 7.775827023107834e-08,
-      "logits/chosen": -13.705121040344238,
-      "logits/rejected": -14.205709457397461,
-      "logps/chosen": -367.263427734375,
-      "logps/rejected": -423.30841064453125,
-      "loss": 0.5788,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.36119210720062256,
-      "rewards/margins": 0.3365553319454193,
-      "rewards/rejected": 0.024636749178171158,
+      "logits/chosen": 111.7248306274414,
+      "logits/rejected": 128.3420867919922,
+      "logps/chosen": -427.53106689453125,
+      "logps/rejected": -546.7640991210938,
+      "loss": 0.1651,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -3.693999767303467,
+      "rewards/margins": 2.9823195934295654,
+      "rewards/rejected": -6.676319122314453,
       "step": 80
     },
     {
       "epoch": 1.7061611374407581,
-      "grad_norm": 14.456589635016153,
+      "grad_norm": 40.99464664899818,
       "learning_rate": 2.7440387297912122e-08,
-      "logits/chosen": -13.98394775390625,
-      "logits/rejected": -14.161648750305176,
-      "logps/chosen": -399.45458984375,
-      "logps/rejected": -447.48828125,
-      "loss": 0.5766,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.3995341658592224,
-      "rewards/margins": 0.34082064032554626,
-      "rewards/rejected": 0.05871356278657913,
+      "logits/chosen": 110.8941879272461,
+      "logits/rejected": 123.70848083496094,
+      "logps/chosen": -457.2183532714844,
+      "logps/rejected": -575.8634033203125,
+      "loss": 0.1557,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -3.5393664836883545,
+      "rewards/margins": 3.369715929031372,
+      "rewards/rejected": -6.909082889556885,
       "step": 90
     },
     {
       "epoch": 1.8957345971563981,
-      "grad_norm": 13.44211674398592,
+      "grad_norm": 45.385328063823785,
       "learning_rate": 2.27878296044029e-09,
-      "logits/chosen": -14.160197257995605,
-      "logits/rejected": -14.141824722290039,
-      "logps/chosen": -392.3072509765625,
-      "logps/rejected": -421.604248046875,
-      "loss": 0.5732,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.4029002785682678,
-      "rewards/margins": 0.27652695775032043,
-      "rewards/rejected": 0.1263733208179474,
+      "logits/chosen": 117.1551284790039,
+      "logits/rejected": 117.0487060546875,
+      "logps/chosen": -446.9934997558594,
+      "logps/rejected": -541.2728881835938,
+      "loss": 0.1585,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -3.250919818878174,
+      "rewards/margins": 2.9745240211486816,
+      "rewards/rejected": -6.2254438400268555,
       "step": 100
     },
     {
       "epoch": 1.8957345971563981,
-      "eval_logits/chosen": -13.292621612548828,
-      "eval_logits/rejected": -12.66539478302002,
-      "eval_logps/chosen": -372.0066833496094,
-      "eval_logps/rejected": -373.4093933105469,
-      "eval_loss": 0.5971602201461792,
-      "eval_rewards/accuracies": 0.6770833134651184,
-      "eval_rewards/chosen": 0.3533553183078766,
-      "eval_rewards/margins": 0.24372106790542603,
-      "eval_rewards/rejected": 0.10963428020477295,
-      "eval_runtime": 20.0916,
-      "eval_samples_per_second": 37.329,
-      "eval_steps_per_second": 1.195,
+      "eval_logits/chosen": 96.73149871826172,
+      "eval_logits/rejected": 91.1323013305664,
+      "eval_logps/chosen": -446.253662109375,
+      "eval_logps/rejected": -476.3663635253906,
+      "eval_loss": 0.46732592582702637,
+      "eval_rewards/accuracies": 0.6979166865348816,
+      "eval_rewards/chosen": -4.1292724609375,
+      "eval_rewards/margins": 1.5854991674423218,
+      "eval_rewards/rejected": -5.714771270751953,
+      "eval_runtime": 120.4793,
+      "eval_samples_per_second": 6.225,
+      "eval_steps_per_second": 0.199,
       "step": 100
     },
     {
       "epoch": 1.971563981042654,
       "step": 104,
       "total_flos": 0.0,
-      "train_loss": 0.627926590350958,
-      "train_runtime": 756.4701,
-      "train_samples_per_second": 17.846,
-      "train_steps_per_second": 0.137
+      "train_loss": 0.39153398688022906,
+      "train_runtime": 2311.0387,
+      "train_samples_per_second": 5.842,
+      "train_steps_per_second": 0.045
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c83a4cd67c56c86e8779774ef2b3c0c2d20d775dd7b0aa4eba03778d916c3903
+oid sha256:79ee78a4306adfc04ffa07fc0ca8acbb9d3417b9d7c9f4adaf815a8d83ea6a24
 size 6264