Model save

README.md

@@ -0,0 +1,75 @@
+---
+license: apache-2.0
+base_model: Minbyul/mistral-7b-wo-live_qa-sft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: mistral-7b-dpo-full-sft-wo-live_qa
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral-7b-dpo-full-sft-wo-live_qa
+
+This model is a fine-tuned version of [Minbyul/mistral-7b-wo-live_qa-sft](https://huggingface.co/Minbyul/mistral-7b-wo-live_qa-sft) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0468
+- Rewards/chosen: -0.7745
+- Rewards/rejected: -14.5125
+- Rewards/accuracies: 0.875
+- Rewards/margins: 13.7379
+- Logps/rejected: -2051.7014
+- Logps/chosen: -168.0429
+- Logits/rejected: -1.5809
+- Logits/chosen: -1.9182
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.2894        | 0.3   | 100  | 0.2025          | -0.2133        | -2.0069          | 0.875              | 1.7935          | -801.1411      | -111.9223    | -2.6944         | -2.4093       |
+| 0.1355        | 0.61  | 200  | 0.0506          | -0.8364        | -12.1329         | 0.875              | 11.2965         | -1813.7421     | -174.2286    | -1.5654         | -1.9005       |
+| 0.0848        | 0.91  | 300  | 0.0468          | -0.7745        | -14.5125         | 0.875              | 13.7379         | -2051.7014     | -168.0429    | -1.5809         | -1.9182       |
+
+
+### Framework versions
+
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.2
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.23545521443209996,
+    "train_runtime": 6883.5239,
+    "train_samples": 20968,
+    "train_samples_per_second": 3.046,
+    "train_steps_per_second": 0.048
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.0.dev0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c15b606ebb6aac527a3da9729442b9999afcd2a5782fe307fe6b16404fdc9239
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b004d5984c47dac0b787132736d5f25d55a5808712e42867118cd7db6fbc020
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2070d7c660e8373fd5d64c8f25909c54d83d9dda95c36560351fb8b4c07150cc
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.23545521443209996,
+    "train_runtime": 6883.5239,
+    "train_samples": 20968,
+    "train_samples_per_second": 3.046,
+    "train_steps_per_second": 0.048
+}

trainer_state.json

@@ -0,0 +1,573 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 328,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 52.10260886950557,
+      "learning_rate": 1.5151515151515152e-08,
+      "logits/chosen": -2.9049293994903564,
+      "logits/rejected": -3.137899398803711,
+      "logps/chosen": -848.0860595703125,
+      "logps/rejected": -1267.928466796875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 54.32742967797558,
+      "learning_rate": 1.5151515151515152e-07,
+      "logits/chosen": -3.1098616123199463,
+      "logits/rejected": -3.1124260425567627,
+      "logps/chosen": -1029.3779296875,
+      "logps/rejected": -1345.0582275390625,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.4930555522441864,
+      "rewards/chosen": 0.00401829881593585,
+      "rewards/margins": 0.004102306440472603,
+      "rewards/rejected": -8.400815568165854e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 31.172998615351542,
+      "learning_rate": 3.0303030303030305e-07,
+      "logits/chosen": -3.117793560028076,
+      "logits/rejected": -3.1491177082061768,
+      "logps/chosen": -1071.2353515625,
+      "logps/rejected": -1354.4710693359375,
+      "loss": 0.6537,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.07634462416172028,
+      "rewards/margins": 0.08821593225002289,
+      "rewards/rejected": -0.011871310882270336,
+      "step": 20
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 28.792134111609187,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": -3.1649086475372314,
+      "logits/rejected": -3.2152061462402344,
+      "logps/chosen": -1064.19970703125,
+      "logps/rejected": -1391.3843994140625,
+      "loss": 0.5897,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.1755533069372177,
+      "rewards/margins": 0.30122771859169006,
+      "rewards/rejected": -0.12567442655563354,
+      "step": 30
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 26.037250152730635,
+      "learning_rate": 4.993056783981012e-07,
+      "logits/chosen": -3.338702440261841,
+      "logits/rejected": -3.3210806846618652,
+      "logps/chosen": -1098.9052734375,
+      "logps/rejected": -1450.2767333984375,
+      "loss": 0.4997,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 0.13215045630931854,
+      "rewards/margins": 0.7883480787277222,
+      "rewards/rejected": -0.6561976075172424,
+      "step": 40
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 27.996786342247532,
+      "learning_rate": 4.95914200522199e-07,
+      "logits/chosen": -3.2485861778259277,
+      "logits/rejected": -3.324843168258667,
+      "logps/chosen": -996.6271362304688,
+      "logps/rejected": -1481.156982421875,
+      "loss": 0.467,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.08611873537302017,
+      "rewards/margins": 1.149458646774292,
+      "rewards/rejected": -1.0633398294448853,
+      "step": 50
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 25.09282538012874,
+      "learning_rate": 4.897364164920514e-07,
+      "logits/chosen": -3.313831329345703,
+      "logits/rejected": -3.2785778045654297,
+      "logps/chosen": -959.1129150390625,
+      "logps/rejected": -1587.156982421875,
+      "loss": 0.3729,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 0.13842354714870453,
+      "rewards/margins": 1.846784234046936,
+      "rewards/rejected": -1.7083606719970703,
+      "step": 60
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 30.921162958697664,
+      "learning_rate": 4.808423230692373e-07,
+      "logits/chosen": -3.2474944591522217,
+      "logits/rejected": -3.291019916534424,
+      "logps/chosen": -1021.1806640625,
+      "logps/rejected": -1590.413818359375,
+      "loss": 0.3752,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.16332893073558807,
+      "rewards/margins": 2.0271217823028564,
+      "rewards/rejected": -2.190450429916382,
+      "step": 70
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 31.254868204859726,
+      "learning_rate": 4.693326938861366e-07,
+      "logits/chosen": -3.301940441131592,
+      "logits/rejected": -3.3018264770507812,
+      "logps/chosen": -1133.804443359375,
+      "logps/rejected": -1672.321533203125,
+      "loss": 0.3439,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.7101578712463379,
+      "rewards/margins": 2.033524751663208,
+      "rewards/rejected": -2.743682861328125,
+      "step": 80
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 34.55073034392427,
+      "learning_rate": 4.553379376404084e-07,
+      "logits/chosen": -3.212740421295166,
+      "logits/rejected": -3.263364791870117,
+      "logps/chosen": -1189.436279296875,
+      "logps/rejected": -1603.1971435546875,
+      "loss": 0.3203,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.9354013204574585,
+      "rewards/margins": 2.177839517593384,
+      "rewards/rejected": -3.113241195678711,
+      "step": 90
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 44.68710091171008,
+      "learning_rate": 4.3901662051233745e-07,
+      "logits/chosen": -3.138298511505127,
+      "logits/rejected": -3.148540496826172,
+      "logps/chosen": -1065.9473876953125,
+      "logps/rejected": -1705.128173828125,
+      "loss": 0.2894,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.9562646150588989,
+      "rewards/margins": 2.6660516262054443,
+      "rewards/rejected": -3.622316360473633,
+      "step": 100
+    },
+    {
+      "epoch": 0.3,
+      "eval_logits/chosen": -2.4093375205993652,
+      "eval_logits/rejected": -2.694366216659546,
+      "eval_logps/chosen": -111.92229461669922,
+      "eval_logps/rejected": -801.1410522460938,
+      "eval_loss": 0.2025093287229538,
+      "eval_rewards/accuracies": 0.875,
+      "eval_rewards/chosen": -0.21333497762680054,
+      "eval_rewards/margins": 1.7935402393341064,
+      "eval_rewards/rejected": -2.0068752765655518,
+      "eval_runtime": 5.1839,
+      "eval_samples_per_second": 4.63,
+      "eval_steps_per_second": 0.193,
+      "step": 100
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 38.50657095083863,
+      "learning_rate": 4.2055366954665237e-07,
+      "logits/chosen": -3.1230928897857666,
+      "logits/rejected": -3.1047117710113525,
+      "logps/chosen": -1150.9886474609375,
+      "logps/rejected": -1801.260009765625,
+      "loss": 0.269,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -1.3049598932266235,
+      "rewards/margins": 2.929929494857788,
+      "rewards/rejected": -4.234889030456543,
+      "step": 110
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 29.424331589339463,
+      "learning_rate": 4.0015827735521523e-07,
+      "logits/chosen": -3.0130834579467773,
+      "logits/rejected": -3.0237557888031006,
+      "logps/chosen": -1210.0142822265625,
+      "logps/rejected": -1803.6591796875,
+      "loss": 0.2426,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -1.3954451084136963,
+      "rewards/margins": 3.6785919666290283,
+      "rewards/rejected": -5.074037551879883,
+      "step": 120
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 61.571819091215445,
+      "learning_rate": 3.780615318811402e-07,
+      "logits/chosen": -3.051529884338379,
+      "logits/rejected": -2.9141249656677246,
+      "logps/chosen": -1176.2293701171875,
+      "logps/rejected": -2026.753173828125,
+      "loss": 0.2321,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -1.6401822566986084,
+      "rewards/margins": 4.810054779052734,
+      "rewards/rejected": -6.450237274169922,
+      "step": 130
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 53.20520696391761,
+      "learning_rate": 3.545137980800601e-07,
+      "logits/chosen": -3.0094645023345947,
+      "logits/rejected": -2.877675771713257,
+      "logps/chosen": -1233.012451171875,
+      "logps/rejected": -2089.841552734375,
+      "loss": 0.1758,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -1.5989583730697632,
+      "rewards/margins": 5.455811500549316,
+      "rewards/rejected": -7.054769039154053,
+      "step": 140
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 39.12953948720105,
+      "learning_rate": 3.2978188118513814e-07,
+      "logits/chosen": -2.921603202819824,
+      "logits/rejected": -2.7563977241516113,
+      "logps/chosen": -1218.6588134765625,
+      "logps/rejected": -2079.738037109375,
+      "loss": 0.1843,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.8553876876831055,
+      "rewards/margins": 5.326550483703613,
+      "rewards/rejected": -7.181939601898193,
+      "step": 150
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 51.519512868650644,
+      "learning_rate": 3.041460036971664e-07,
+      "logits/chosen": -2.8195321559906006,
+      "logits/rejected": -2.6623053550720215,
+      "logps/chosen": -1251.6900634765625,
+      "logps/rejected": -2145.56591796875,
+      "loss": 0.171,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -2.4222846031188965,
+      "rewards/margins": 5.688368797302246,
+      "rewards/rejected": -8.110652923583984,
+      "step": 160
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 34.8732410879268,
+      "learning_rate": 2.778966303516603e-07,
+      "logits/chosen": -2.7237300872802734,
+      "logits/rejected": -2.5120363235473633,
+      "logps/chosen": -1312.8253173828125,
+      "logps/rejected": -2203.4736328125,
+      "loss": 0.1537,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -2.7755582332611084,
+      "rewards/margins": 5.876898288726807,
+      "rewards/rejected": -8.652456283569336,
+      "step": 170
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 38.69272613867567,
+      "learning_rate": 2.513311770373421e-07,
+      "logits/chosen": -2.8833096027374268,
+      "logits/rejected": -2.700117349624634,
+      "logps/chosen": -1203.2437744140625,
+      "logps/rejected": -2128.73876953125,
+      "loss": 0.164,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -1.8221490383148193,
+      "rewards/margins": 5.741241455078125,
+      "rewards/rejected": -7.563390254974365,
+      "step": 180
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 43.40638062311321,
+      "learning_rate": 2.2475064095527946e-07,
+      "logits/chosen": -2.8215672969818115,
+      "logits/rejected": -2.522494316101074,
+      "logps/chosen": -1261.7806396484375,
+      "logps/rejected": -2357.189453125,
+      "loss": 0.1105,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -2.5550084114074707,
+      "rewards/margins": 7.024233341217041,
+      "rewards/rejected": -9.579240798950195,
+      "step": 190
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 46.47610329796461,
+      "learning_rate": 1.984561902003255e-07,
+      "logits/chosen": -2.600424289703369,
+      "logits/rejected": -2.3879284858703613,
+      "logps/chosen": -1387.474365234375,
+      "logps/rejected": -2327.524169921875,
+      "loss": 0.1355,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -3.2148842811584473,
+      "rewards/margins": 6.671014308929443,
+      "rewards/rejected": -9.885897636413574,
+      "step": 200
+    },
+    {
+      "epoch": 0.61,
+      "eval_logits/chosen": -1.9005341529846191,
+      "eval_logits/rejected": -1.5654257535934448,
+      "eval_logps/chosen": -174.2285614013672,
+      "eval_logps/rejected": -1813.7420654296875,
+      "eval_loss": 0.05062888190150261,
+      "eval_rewards/accuracies": 0.875,
+      "eval_rewards/chosen": -0.8363977074623108,
+      "eval_rewards/margins": 11.296485900878906,
+      "eval_rewards/rejected": -12.13288402557373,
+      "eval_runtime": 4.9098,
+      "eval_samples_per_second": 4.888,
+      "eval_steps_per_second": 0.204,
+      "step": 200
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 31.32678074756829,
+      "learning_rate": 1.7274575140626315e-07,
+      "logits/chosen": -2.630878448486328,
+      "logits/rejected": -2.360283136367798,
+      "logps/chosen": -1274.6114501953125,
+      "logps/rejected": -2422.227294921875,
+      "loss": 0.107,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -2.9166343212127686,
+      "rewards/margins": 7.893813133239746,
+      "rewards/rejected": -10.810447692871094,
+      "step": 210
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 48.30372175049827,
+      "learning_rate": 1.4791063411799937e-07,
+      "logits/chosen": -2.758852481842041,
+      "logits/rejected": -2.4495952129364014,
+      "logps/chosen": -1242.4476318359375,
+      "logps/rejected": -2313.494384765625,
+      "loss": 0.1167,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -2.645324230194092,
+      "rewards/margins": 7.094490051269531,
+      "rewards/rejected": -9.739813804626465,
+      "step": 220
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 31.566932885659007,
+      "learning_rate": 1.2423223013801945e-07,
+      "logits/chosen": -2.7269630432128906,
+      "logits/rejected": -2.492689609527588,
+      "logps/chosen": -1234.9451904296875,
+      "logps/rejected": -2318.334228515625,
+      "loss": 0.1129,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.171018123626709,
+      "rewards/margins": 7.9699273109436035,
+      "rewards/rejected": -10.140945434570312,
+      "step": 230
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 34.83394111035656,
+      "learning_rate": 1.0197882524482667e-07,
+      "logits/chosen": -2.7251534461975098,
+      "logits/rejected": -2.4403250217437744,
+      "logps/chosen": -1231.4935302734375,
+      "logps/rejected": -2440.34912109375,
+      "loss": 0.1418,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -2.6753926277160645,
+      "rewards/margins": 7.822265625,
+      "rewards/rejected": -10.497659683227539,
+      "step": 240
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 24.25621896047951,
+      "learning_rate": 8.140255940787058e-08,
+      "logits/chosen": -2.695079803466797,
+      "logits/rejected": -2.338381290435791,
+      "logps/chosen": -1329.83642578125,
+      "logps/rejected": -2476.803955078125,
+      "loss": 0.1016,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -2.7425146102905273,
+      "rewards/margins": 8.260651588439941,
+      "rewards/rejected": -11.003166198730469,
+      "step": 250
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 27.532348797533775,
+      "learning_rate": 6.273656994094231e-08,
+      "logits/chosen": -2.669456958770752,
+      "logits/rejected": -2.3488214015960693,
+      "logps/chosen": -1346.739013671875,
+      "logps/rejected": -2527.728759765625,
+      "loss": 0.0974,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -3.164280891418457,
+      "rewards/margins": 8.543807983398438,
+      "rewards/rejected": -11.708089828491211,
+      "step": 260
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 32.33656052797523,
+      "learning_rate": 4.6192349963253137e-08,
+      "logits/chosen": -2.675199031829834,
+      "logits/rejected": -2.319917678833008,
+      "logps/chosen": -1294.7154541015625,
+      "logps/rejected": -2497.270263671875,
+      "loss": 0.0645,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -3.0818674564361572,
+      "rewards/margins": 7.933001518249512,
+      "rewards/rejected": -11.014870643615723,
+      "step": 270
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 29.09098674295539,
+      "learning_rate": 3.195735209788528e-08,
+      "logits/chosen": -2.6060080528259277,
+      "logits/rejected": -2.3255152702331543,
+      "logps/chosen": -1324.2171630859375,
+      "logps/rejected": -2495.132080078125,
+      "loss": 0.0796,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -3.3014044761657715,
+      "rewards/margins": 7.963587760925293,
+      "rewards/rejected": -11.264991760253906,
+      "step": 280
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 50.71746791104581,
+      "learning_rate": 2.019286455866981e-08,
+      "logits/chosen": -2.5728938579559326,
+      "logits/rejected": -2.3195109367370605,
+      "logps/chosen": -1388.82861328125,
+      "logps/rejected": -2512.4306640625,
+      "loss": 0.0931,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -3.2404701709747314,
+      "rewards/margins": 8.0997953414917,
+      "rewards/rejected": -11.340266227722168,
+      "step": 290
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 52.710422459009,
+      "learning_rate": 1.1032183690276753e-08,
+      "logits/chosen": -2.529834270477295,
+      "logits/rejected": -2.2143425941467285,
+      "logps/chosen": -1260.817626953125,
+      "logps/rejected": -2584.9921875,
+      "loss": 0.0848,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -3.3079543113708496,
+      "rewards/margins": 9.132894515991211,
+      "rewards/rejected": -12.440850257873535,
+      "step": 300
+    },
+    {
+      "epoch": 0.91,
+      "eval_logits/chosen": -1.9181619882583618,
+      "eval_logits/rejected": -1.5808876752853394,
+      "eval_logps/chosen": -168.04290771484375,
+      "eval_logps/rejected": -2051.701416015625,
+      "eval_loss": 0.04682515189051628,
+      "eval_rewards/accuracies": 0.875,
+      "eval_rewards/chosen": -0.7745410203933716,
+      "eval_rewards/margins": 13.737937927246094,
+      "eval_rewards/rejected": -14.512479782104492,
+      "eval_runtime": 4.9321,
+      "eval_samples_per_second": 4.866,
+      "eval_steps_per_second": 0.203,
+      "step": 300
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 72.2830720687719,
+      "learning_rate": 4.579103667367384e-09,
+      "logits/chosen": -2.5878891944885254,
+      "logits/rejected": -2.2834665775299072,
+      "logps/chosen": -1402.7425537109375,
+      "logps/rejected": -2531.157958984375,
+      "loss": 0.0942,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -3.4184463024139404,
+      "rewards/margins": 8.603601455688477,
+      "rewards/rejected": -12.022048950195312,
+      "step": 310
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 30.83428232415201,
+      "learning_rate": 9.067404651211807e-10,
+      "logits/chosen": -2.603289842605591,
+      "logits/rejected": -2.298339366912842,
+      "logps/chosen": -1401.251708984375,
+      "logps/rejected": -2568.3681640625,
+      "loss": 0.1105,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -3.375328779220581,
+      "rewards/margins": 8.309103965759277,
+      "rewards/rejected": -11.684433937072754,
+      "step": 320
+    },
+    {
+      "epoch": 1.0,
+      "step": 328,
+      "total_flos": 0.0,
+      "train_loss": 0.23545521443209996,
+      "train_runtime": 6883.5239,
+      "train_samples_per_second": 3.046,
+      "train_steps_per_second": 0.048
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 328,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}