Model save

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+base_model: Minbyul/mistral-7b-wo-medication_qa-sft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: mistral-7b-dpo-full-sft-wo-medication_qa
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral-7b-dpo-full-sft-wo-medication_qa
+
+This model is a fine-tuned version of [Minbyul/mistral-7b-wo-medication_qa-sft](https://huggingface.co/Minbyul/mistral-7b-wo-medication_qa-sft) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Logits/chosen: -2.5877
+- Logits/rejected: -2.3930
+- Logps/chosen: -803.1033
+- Logps/rejected: -1661.4266
+- Loss: 0.0753
+- Rewards/accuracies: 0.9531
+- Rewards/chosen: -3.7097
+- Rewards/margins: 7.4753
+- Rewards/rejected: -11.1850
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
+|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
+| 0.2799        | 0.31  | 100  | -3.0348       | -3.0868         | -584.1479    | -794.0103      | 0.5261          | 0.75               | -1.5202        | 0.9907          | -2.5108          |
+| 0.154         | 0.62  | 200  | -2.6948       | -2.5547         | -742.1359    | -1446.8754     | 0.0923          | 0.9375             | -3.1001        | 5.9394          | -9.0395          |
+| 0.0948        | 0.92  | 300  | -2.5877       | -2.3930         | -803.1033    | -1661.4266     | 0.0753          | 0.9531             | -3.7097        | 7.4753          | -11.1850         |
+
+
+### Framework versions
+
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.2
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.007740737387427577,
+    "train_runtime": 396.513,
+    "train_samples": 20740,
+    "train_samples_per_second": 52.306,
+    "train_steps_per_second": 0.817
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:184b5890e8da33b600c1b7a61a2b5268b4d38296f7a4649d50e71b9b07514f26
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f970eda41d16e3f7da2627eb2cd794dd83ab887271d5189e375fff055a8975b
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1088dd914ef4b5b1ccc8c5cea11d45cf04dc8138fb88cc5200d5656a9079256
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.007740737387427577,
+    "train_runtime": 396.513,
+    "train_samples": 20740,
+    "train_samples_per_second": 52.306,
+    "train_steps_per_second": 0.817
+}

trainer_state.json

@@ -0,0 +1,573 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9984591679506933,
+  "eval_steps": 100,
+  "global_step": 324,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 39.87631410320537,
+      "learning_rate": 1.5151515151515152e-08,
+      "logits/chosen": -3.1684141159057617,
+      "logits/rejected": -3.1765036582946777,
+      "logps/chosen": -1262.7908935546875,
+      "logps/rejected": -1304.270263671875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 44.090928533988176,
+      "learning_rate": 1.5151515151515152e-07,
+      "logits/chosen": -3.145017385482788,
+      "logits/rejected": -3.17344069480896,
+      "logps/chosen": -1035.2520751953125,
+      "logps/rejected": -1331.3636474609375,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.5277777910232544,
+      "rewards/chosen": 0.003002108307555318,
+      "rewards/margins": 0.0036764023825526237,
+      "rewards/rejected": -0.0006742942496202886,
+      "step": 10
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 32.37744800941447,
+      "learning_rate": 3.0303030303030305e-07,
+      "logits/chosen": -3.1358423233032227,
+      "logits/rejected": -3.18705415725708,
+      "logps/chosen": -968.2097778320312,
+      "logps/rejected": -1354.069580078125,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.03913033753633499,
+      "rewards/margins": 0.06312780827283859,
+      "rewards/rejected": -0.023997480049729347,
+      "step": 20
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 30.93962012271263,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": -3.2511069774627686,
+      "logits/rejected": -3.244719982147217,
+      "logps/chosen": -1036.7672119140625,
+      "logps/rejected": -1373.1820068359375,
+      "loss": 0.5896,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.06491168588399887,
+      "rewards/margins": 0.31840670108795166,
+      "rewards/rejected": -0.2534949779510498,
+      "step": 30
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 31.51946350483435,
+      "learning_rate": 4.992864684782648e-07,
+      "logits/chosen": -3.3422675132751465,
+      "logits/rejected": -3.370623826980591,
+      "logps/chosen": -1083.17431640625,
+      "logps/rejected": -1492.5845947265625,
+      "loss": 0.5125,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.07627250999212265,
+      "rewards/margins": 0.8225336074829102,
+      "rewards/rejected": -0.898806095123291,
+      "step": 40
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 29.782679812110892,
+      "learning_rate": 4.958014217656854e-07,
+      "logits/chosen": -3.3696506023406982,
+      "logits/rejected": -3.4038467407226562,
+      "logps/chosen": -1081.7869873046875,
+      "logps/rejected": -1461.259033203125,
+      "loss": 0.4171,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.1066322773694992,
+      "rewards/margins": 1.0666204690933228,
+      "rewards/rejected": -1.173252820968628,
+      "step": 50
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 30.594547647279217,
+      "learning_rate": 4.894543310469967e-07,
+      "logits/chosen": -3.352465867996216,
+      "logits/rejected": -3.3652706146240234,
+      "logps/chosen": -1111.260009765625,
+      "logps/rejected": -1504.715087890625,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.2318076640367508,
+      "rewards/margins": 1.2468664646148682,
+      "rewards/rejected": -1.4786741733551025,
+      "step": 60
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 31.824817232007625,
+      "learning_rate": 4.803191000971128e-07,
+      "logits/chosen": -3.3312506675720215,
+      "logits/rejected": -3.355130434036255,
+      "logps/chosen": -968.1290893554688,
+      "logps/rejected": -1600.333251953125,
+      "loss": 0.3874,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.22658078372478485,
+      "rewards/margins": 2.3517754077911377,
+      "rewards/rejected": -2.5783562660217285,
+      "step": 70
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 25.823223858100576,
+      "learning_rate": 4.685020970273189e-07,
+      "logits/chosen": -3.2700467109680176,
+      "logits/rejected": -3.3080413341522217,
+      "logps/chosen": -1002.4366455078125,
+      "logps/rejected": -1593.41796875,
+      "loss": 0.3546,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.07068847864866257,
+      "rewards/margins": 2.2586522102355957,
+      "rewards/rejected": -2.329341173171997,
+      "step": 80
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 43.928369861559965,
+      "learning_rate": 4.541409157643027e-07,
+      "logits/chosen": -3.235419511795044,
+      "logits/rejected": -3.2496044635772705,
+      "logps/chosen": -956.4049072265625,
+      "logps/rejected": -1599.0389404296875,
+      "loss": 0.3143,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 0.038097791373729706,
+      "rewards/margins": 2.2678751945495605,
+      "rewards/rejected": -2.2297775745391846,
+      "step": 90
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 30.820532733997354,
+      "learning_rate": 4.374027739443952e-07,
+      "logits/chosen": -3.204524517059326,
+      "logits/rejected": -3.163343906402588,
+      "logps/chosen": -1068.4237060546875,
+      "logps/rejected": -1704.1986083984375,
+      "loss": 0.2799,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.3904297947883606,
+      "rewards/margins": 3.4663283824920654,
+      "rewards/rejected": -3.8567581176757812,
+      "step": 100
+    },
+    {
+      "epoch": 0.31,
+      "eval_logits/chosen": -3.0348000526428223,
+      "eval_logits/rejected": -3.0867843627929688,
+      "eval_logps/chosen": -584.1478881835938,
+      "eval_logps/rejected": -794.01025390625,
+      "eval_loss": 0.5261008143424988,
+      "eval_rewards/accuracies": 0.75,
+      "eval_rewards/chosen": -1.5201810598373413,
+      "eval_rewards/margins": 0.9906590580940247,
+      "eval_rewards/rejected": -2.5108399391174316,
+      "eval_runtime": 34.7053,
+      "eval_samples_per_second": 7.261,
+      "eval_steps_per_second": 0.231,
+      "step": 100
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 47.845448282397456,
+      "learning_rate": 4.184825658775027e-07,
+      "logits/chosen": -3.128324031829834,
+      "logits/rejected": -3.134152889251709,
+      "logps/chosen": -1042.473388671875,
+      "logps/rejected": -1787.997802734375,
+      "loss": 0.2816,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.6833322644233704,
+      "rewards/margins": 3.4197134971618652,
+      "rewards/rejected": -4.10304594039917,
+      "step": 110
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 24.2664669682948,
+      "learning_rate": 3.9760059325148063e-07,
+      "logits/chosen": -3.1436760425567627,
+      "logits/rejected": -3.091614246368408,
+      "logps/chosen": -1067.834716796875,
+      "logps/rejected": -1788.0120849609375,
+      "loss": 0.2536,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.610935389995575,
+      "rewards/margins": 4.445748329162598,
+      "rewards/rejected": -5.056683540344238,
+      "step": 120
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 22.23462347593175,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -3.1414103507995605,
+      "logits/rejected": -3.0941264629364014,
+      "logps/chosen": -1100.4937744140625,
+      "logps/rejected": -1801.8560791015625,
+      "loss": 0.2298,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.540351152420044,
+      "rewards/margins": 3.7757785320281982,
+      "rewards/rejected": -4.3161301612854,
+      "step": 130
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 28.613362043744857,
+      "learning_rate": 3.509439412016004e-07,
+      "logits/chosen": -3.0641441345214844,
+      "logits/rejected": -3.0451717376708984,
+      "logps/chosen": -1098.5340576171875,
+      "logps/rejected": -1918.6890869140625,
+      "loss": 0.2135,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.7187612056732178,
+      "rewards/margins": 5.323573112487793,
+      "rewards/rejected": -6.04233455657959,
+      "step": 140
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 30.54616548038225,
+      "learning_rate": 3.2571251897448763e-07,
+      "logits/chosen": -2.992375135421753,
+      "logits/rejected": -2.95180606842041,
+      "logps/chosen": -1197.9376220703125,
+      "logps/rejected": -2077.058349609375,
+      "loss": 0.1801,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.5255634784698486,
+      "rewards/margins": 5.869881629943848,
+      "rewards/rejected": -7.395445346832275,
+      "step": 150
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 36.74589910484145,
+      "learning_rate": 2.9959952104467243e-07,
+      "logits/chosen": -2.9339356422424316,
+      "logits/rejected": -2.85386323928833,
+      "logps/chosen": -1247.737060546875,
+      "logps/rejected": -2310.10205078125,
+      "loss": 0.1778,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -2.0261478424072266,
+      "rewards/margins": 7.8707451820373535,
+      "rewards/rejected": -9.896891593933105,
+      "step": 160
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 32.132075393104884,
+      "learning_rate": 2.729089999626637e-07,
+      "logits/chosen": -2.980856418609619,
+      "logits/rejected": -2.856822967529297,
+      "logps/chosen": -1185.372802734375,
+      "logps/rejected": -2347.78076171875,
+      "loss": 0.1698,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -1.51887047290802,
+      "rewards/margins": 8.371113777160645,
+      "rewards/rejected": -9.889985084533691,
+      "step": 170
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 26.839790210789428,
+      "learning_rate": 2.459517327993746e-07,
+      "logits/chosen": -2.962564468383789,
+      "logits/rejected": -2.8451316356658936,
+      "logps/chosen": -1266.3397216796875,
+      "logps/rejected": -2263.588623046875,
+      "loss": 0.141,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -1.7179988622665405,
+      "rewards/margins": 7.524572849273682,
+      "rewards/rejected": -9.242570877075195,
+      "step": 180
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 33.374896332465084,
+      "learning_rate": 2.1904160254356748e-07,
+      "logits/chosen": -2.881953477859497,
+      "logits/rejected": -2.7538435459136963,
+      "logps/chosen": -1169.200927734375,
+      "logps/rejected": -2476.24072265625,
+      "loss": 0.1207,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -1.6462358236312866,
+      "rewards/margins": 9.677408218383789,
+      "rewards/rejected": -11.323644638061523,
+      "step": 190
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 44.90567050679125,
+      "learning_rate": 1.9249194333484563e-07,
+      "logits/chosen": -2.8342463970184326,
+      "logits/rejected": -2.730264902114868,
+      "logps/chosen": -1229.9298095703125,
+      "logps/rejected": -2291.61181640625,
+      "loss": 0.154,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -2.0603644847869873,
+      "rewards/margins": 7.22509765625,
+      "rewards/rejected": -9.285462379455566,
+      "step": 200
+    },
+    {
+      "epoch": 0.62,
+      "eval_logits/chosen": -2.694772720336914,
+      "eval_logits/rejected": -2.554710626602173,
+      "eval_logps/chosen": -742.1358642578125,
+      "eval_logps/rejected": -1446.8753662109375,
+      "eval_loss": 0.09226308017969131,
+      "eval_rewards/accuracies": 0.9375,
+      "eval_rewards/chosen": -3.100059986114502,
+      "eval_rewards/margins": 5.939432144165039,
+      "eval_rewards/rejected": -9.0394926071167,
+      "eval_runtime": 34.5072,
+      "eval_samples_per_second": 7.303,
+      "eval_steps_per_second": 0.232,
+      "step": 200
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 29.94298857126716,
+      "learning_rate": 1.6661189208729489e-07,
+      "logits/chosen": -2.786771059036255,
+      "logits/rejected": -2.6327857971191406,
+      "logps/chosen": -1283.205810546875,
+      "logps/rejected": -2401.399169921875,
+      "loss": 0.1526,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -2.189497709274292,
+      "rewards/margins": 8.632684707641602,
+      "rewards/rejected": -10.822182655334473,
+      "step": 210
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 33.13137270748439,
+      "learning_rate": 1.4170278898446175e-07,
+      "logits/chosen": -2.828369379043579,
+      "logits/rejected": -2.650123119354248,
+      "logps/chosen": -1175.46826171875,
+      "logps/rejected": -2410.84326171875,
+      "loss": 0.1252,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.8023840188980103,
+      "rewards/margins": 8.576199531555176,
+      "rewards/rejected": -10.378583908081055,
+      "step": 220
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 30.216897410019698,
+      "learning_rate": 1.1805466875731276e-07,
+      "logits/chosen": -2.820298671722412,
+      "logits/rejected": -2.622697591781616,
+      "logps/chosen": -1135.1295166015625,
+      "logps/rejected": -2719.731201171875,
+      "loss": 0.1353,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -1.7641971111297607,
+      "rewards/margins": 11.304890632629395,
+      "rewards/rejected": -13.06908893585205,
+      "step": 230
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 28.800219923929006,
+      "learning_rate": 9.594288359976815e-08,
+      "logits/chosen": -2.815680742263794,
+      "logits/rejected": -2.6530845165252686,
+      "logps/chosen": -1304.4205322265625,
+      "logps/rejected": -2253.841064453125,
+      "loss": 0.1093,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -1.9788525104522705,
+      "rewards/margins": 7.066276550292969,
+      "rewards/rejected": -9.045129776000977,
+      "step": 240
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 28.53520064286844,
+      "learning_rate": 7.56248970436493e-08,
+      "logits/chosen": -2.7308030128479004,
+      "logits/rejected": -2.56375789642334,
+      "logps/chosen": -1186.9593505859375,
+      "logps/rejected": -2636.3701171875,
+      "loss": 0.0913,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -2.045689105987549,
+      "rewards/margins": 10.186556816101074,
+      "rewards/rejected": -12.232245445251465,
+      "step": 250
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 21.7882636792373,
+      "learning_rate": 5.733728612427771e-08,
+      "logits/chosen": -2.7645225524902344,
+      "logits/rejected": -2.5233638286590576,
+      "logps/chosen": -1321.7745361328125,
+      "logps/rejected": -2658.189453125,
+      "loss": 0.1155,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.5851030349731445,
+      "rewards/margins": 10.245210647583008,
+      "rewards/rejected": -12.830312728881836,
+      "step": 260
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 34.32906854711248,
+      "learning_rate": 4.1292986742682254e-08,
+      "logits/chosen": -2.668457508087158,
+      "logits/rejected": -2.500288963317871,
+      "logps/chosen": -1262.6650390625,
+      "logps/rejected": -2782.10009765625,
+      "loss": 0.0989,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -2.7102155685424805,
+      "rewards/margins": 11.686820983886719,
+      "rewards/rejected": -14.3970365524292,
+      "step": 270
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 42.763438453906815,
+      "learning_rate": 2.7678814298657732e-08,
+      "logits/chosen": -2.6972427368164062,
+      "logits/rejected": -2.4791617393493652,
+      "logps/chosen": -1353.071533203125,
+      "logps/rejected": -2790.54052734375,
+      "loss": 0.1022,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.827423095703125,
+      "rewards/margins": 11.204734802246094,
+      "rewards/rejected": -14.032157897949219,
+      "step": 280
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 26.418126829556318,
+      "learning_rate": 1.6653288463741062e-08,
+      "logits/chosen": -2.689786672592163,
+      "logits/rejected": -2.518730401992798,
+      "logps/chosen": -1242.576416015625,
+      "logps/rejected": -2554.541748046875,
+      "loss": 0.1067,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -2.544224262237549,
+      "rewards/margins": 9.17101001739502,
+      "rewards/rejected": -11.715234756469727,
+      "step": 290
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 40.4422866987403,
+      "learning_rate": 8.344787421847216e-09,
+      "logits/chosen": -2.65974497795105,
+      "logits/rejected": -2.4722535610198975,
+      "logps/chosen": -1281.4610595703125,
+      "logps/rejected": -2672.197265625,
+      "loss": 0.0948,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -2.5506279468536377,
+      "rewards/margins": 10.449880599975586,
+      "rewards/rejected": -13.000508308410645,
+      "step": 300
+    },
+    {
+      "epoch": 0.92,
+      "eval_logits/chosen": -2.5877139568328857,
+      "eval_logits/rejected": -2.3930397033691406,
+      "eval_logps/chosen": -803.1033325195312,
+      "eval_logps/rejected": -1661.4266357421875,
+      "eval_loss": 0.07533077150583267,
+      "eval_rewards/accuracies": 0.953125,
+      "eval_rewards/chosen": -3.70973539352417,
+      "eval_rewards/margins": 7.4752678871154785,
+      "eval_rewards/rejected": -11.185002326965332,
+      "eval_runtime": 34.126,
+      "eval_samples_per_second": 7.384,
+      "eval_steps_per_second": 0.234,
+      "step": 300
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 38.70439473653127,
+      "learning_rate": 2.850053069080344e-09,
+      "logits/chosen": -2.730034351348877,
+      "logits/rejected": -2.4978787899017334,
+      "logps/chosen": -1227.954345703125,
+      "logps/rejected": -2714.13623046875,
+      "loss": 0.1114,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -2.477766990661621,
+      "rewards/margins": 11.021059036254883,
+      "rewards/rejected": -13.498825073242188,
+      "step": 310
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 25.381541507877866,
+      "learning_rate": 2.3306457775981727e-10,
+      "logits/chosen": -2.6724932193756104,
+      "logits/rejected": -2.4461209774017334,
+      "logps/chosen": -1288.38623046875,
+      "logps/rejected": -2817.382568359375,
+      "loss": 0.0963,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.566399097442627,
+      "rewards/margins": 11.983835220336914,
+      "rewards/rejected": -14.550233840942383,
+      "step": 320
+    },
+    {
+      "epoch": 1.0,
+      "step": 324,
+      "total_flos": 0.0,
+      "train_loss": 0.007740737387427577,
+      "train_runtime": 396.513,
+      "train_samples_per_second": 52.306,
+      "train_steps_per_second": 0.817
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 324,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71241e82b1ef8d44a986e86ae7e9ecb396f96d30d47b4753dc5d68d7cd43bccd
+oid sha256:99796805fd9ed367cb8af34e6aa4b34288f21199d46d556b02001fe5444b8f52
 size 6264