Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: mit
+base_model: microsoft/Phi-3-mini-4k-instruct
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: 0.001_SELM-Phi-3-mini-4k-instruct_iter_1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 0.001_SELM-Phi-3-mini-4k-instruct_iter_1
+
+This model is a fine-tuned version of [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.40.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9984301412872841,
+    "total_flos": 0.0,
+    "train_loss": 0.5863969910819575,
+    "train_runtime": 3000.191,
+    "train_samples": 20378,
+    "train_samples_per_second": 6.792,
+    "train_steps_per_second": 0.053
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "pad_token_id": 32000,
+  "transformers_version": "4.40.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6fb7b806e1a46bd26e706818014ef451d02284036e79a54cf4b19d1237acd98
+size 4972489328

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:278bcd1e0003d997302458f019156ea8be70c0e669dfabdfe701b93c450dcb1a
+size 2669692552

model.safetensors.index.json

@@ -0,0 +1,202 @@
+{
+  "metadata": {
+    "total_size": 7642159104
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9984301412872841,
+    "total_flos": 0.0,
+    "train_loss": 0.5863969910819575,
+    "train_runtime": 3000.191,
+    "train_samples": 20378,
+    "train_samples_per_second": 6.792,
+    "train_steps_per_second": 0.053
+}

trainer_state.json

@@ -0,0 +1,302 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9984301412872841,
+  "eval_steps": 500,
+  "global_step": 159,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.006279434850863423,
+      "grad_norm": 6.26599976561471,
+      "learning_rate": 3.125e-08,
+      "logits/chosen": 2.506397247314453,
+      "logits/rejected": 2.9004628658294678,
+      "logps/chosen": -304.1236877441406,
+      "logps/pi_response": -206.54444885253906,
+      "logps/ref_response": -206.54444885253906,
+      "logps/rejected": -435.8030700683594,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.06279434850863422,
+      "grad_norm": 6.5208769318369715,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": 2.331634759902954,
+      "logits/rejected": 2.5180442333221436,
+      "logps/chosen": -271.9783935546875,
+      "logps/pi_response": -178.7827606201172,
+      "logps/ref_response": -178.90916442871094,
+      "logps/rejected": -350.5011901855469,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": 0.0012251290027052164,
+      "rewards/margins": 0.0004269891942385584,
+      "rewards/rejected": 0.0007981394301168621,
+      "step": 10
+    },
+    {
+      "epoch": 0.12558869701726844,
+      "grad_norm": 6.149140437491673,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": 2.4192910194396973,
+      "logits/rejected": 2.5796756744384766,
+      "logps/chosen": -264.28521728515625,
+      "logps/pi_response": -169.31643676757812,
+      "logps/ref_response": -172.20350646972656,
+      "logps/rejected": -353.85626220703125,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.020391147583723068,
+      "rewards/margins": 0.01745672896504402,
+      "rewards/rejected": 0.002934417687356472,
+      "step": 20
+    },
+    {
+      "epoch": 0.18838304552590268,
+      "grad_norm": 5.838815655099781,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": 2.2606046199798584,
+      "logits/rejected": 2.399362087249756,
+      "logps/chosen": -261.7556457519531,
+      "logps/pi_response": -151.37615966796875,
+      "logps/ref_response": -163.4231719970703,
+      "logps/rejected": -365.0315246582031,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.08448224514722824,
+      "rewards/margins": 0.08167780935764313,
+      "rewards/rejected": 0.002804456278681755,
+      "step": 30
+    },
+    {
+      "epoch": 0.25117739403453687,
+      "grad_norm": 6.251630948090285,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": 2.2731361389160156,
+      "logits/rejected": 2.4390079975128174,
+      "logps/chosen": -283.78436279296875,
+      "logps/pi_response": -162.61363220214844,
+      "logps/ref_response": -179.4296417236328,
+      "logps/rejected": -353.84832763671875,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.11685456335544586,
+      "rewards/margins": 0.13428333401679993,
+      "rewards/rejected": -0.01742876134812832,
+      "step": 40
+    },
+    {
+      "epoch": 0.3139717425431711,
+      "grad_norm": 6.204506278337755,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": 1.9736655950546265,
+      "logits/rejected": 2.127274513244629,
+      "logps/chosen": -262.3057861328125,
+      "logps/pi_response": -151.85891723632812,
+      "logps/ref_response": -168.13865661621094,
+      "logps/rejected": -346.0932922363281,
+      "loss": 0.6134,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.028540844097733498,
+      "rewards/margins": 0.18684375286102295,
+      "rewards/rejected": -0.1583028882741928,
+      "step": 50
+    },
+    {
+      "epoch": 0.37676609105180536,
+      "grad_norm": 5.062174246506226,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": 1.7557458877563477,
+      "logits/rejected": 1.9249584674835205,
+      "logps/chosen": -270.45989990234375,
+      "logps/pi_response": -142.9950714111328,
+      "logps/ref_response": -163.70513916015625,
+      "logps/rejected": -408.8035583496094,
+      "loss": 0.5803,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.011763873510062695,
+      "rewards/margins": 0.3801829516887665,
+      "rewards/rejected": -0.39194679260253906,
+      "step": 60
+    },
+    {
+      "epoch": 0.43956043956043955,
+      "grad_norm": 4.514633956267623,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": 1.6569864749908447,
+      "logits/rejected": 1.8295177221298218,
+      "logps/chosen": -254.6250457763672,
+      "logps/pi_response": -151.6630859375,
+      "logps/ref_response": -169.66830444335938,
+      "logps/rejected": -397.61846923828125,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.03449697047472,
+      "rewards/margins": 0.402763307094574,
+      "rewards/rejected": -0.437260240316391,
+      "step": 70
+    },
+    {
+      "epoch": 0.5023547880690737,
+      "grad_norm": 5.51673658153614,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": 1.7929670810699463,
+      "logits/rejected": 1.9936065673828125,
+      "logps/chosen": -301.0906677246094,
+      "logps/pi_response": -161.8425750732422,
+      "logps/ref_response": -179.93528747558594,
+      "logps/rejected": -400.73992919921875,
+      "loss": 0.5737,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.09138133376836777,
+      "rewards/margins": 0.43019238114356995,
+      "rewards/rejected": -0.5215736627578735,
+      "step": 80
+    },
+    {
+      "epoch": 0.565149136577708,
+      "grad_norm": 4.60951328858753,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": 1.629387617111206,
+      "logits/rejected": 1.7540788650512695,
+      "logps/chosen": -322.956787109375,
+      "logps/pi_response": -162.01455688476562,
+      "logps/ref_response": -183.2974853515625,
+      "logps/rejected": -443.8954162597656,
+      "loss": 0.5516,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.10067585855722427,
+      "rewards/margins": 0.5000637769699097,
+      "rewards/rejected": -0.6007396578788757,
+      "step": 90
+    },
+    {
+      "epoch": 0.6279434850863422,
+      "grad_norm": 5.912396924653729,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": 1.504276990890503,
+      "logits/rejected": 1.6940498352050781,
+      "logps/chosen": -276.9794921875,
+      "logps/pi_response": -147.35867309570312,
+      "logps/ref_response": -169.77037048339844,
+      "logps/rejected": -378.74542236328125,
+      "loss": 0.5559,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.08432722836732864,
+      "rewards/margins": 0.4150608479976654,
+      "rewards/rejected": -0.4993881285190582,
+      "step": 100
+    },
+    {
+      "epoch": 0.6907378335949764,
+      "grad_norm": 6.153587429768731,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": 1.5478519201278687,
+      "logits/rejected": 1.7215379476547241,
+      "logps/chosen": -282.90142822265625,
+      "logps/pi_response": -164.94143676757812,
+      "logps/ref_response": -186.05224609375,
+      "logps/rejected": -429.9634704589844,
+      "loss": 0.5423,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.03883028030395508,
+      "rewards/margins": 0.512156069278717,
+      "rewards/rejected": -0.5509864091873169,
+      "step": 110
+    },
+    {
+      "epoch": 0.7535321821036107,
+      "grad_norm": 5.201928644747849,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": 1.4170118570327759,
+      "logits/rejected": 1.6051431894302368,
+      "logps/chosen": -289.0984802246094,
+      "logps/pi_response": -153.79116821289062,
+      "logps/ref_response": -173.28074645996094,
+      "logps/rejected": -421.106689453125,
+      "loss": 0.5484,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.0854920819401741,
+      "rewards/margins": 0.5487265586853027,
+      "rewards/rejected": -0.6342186331748962,
+      "step": 120
+    },
+    {
+      "epoch": 0.8163265306122449,
+      "grad_norm": 5.723480822818505,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": 1.4594230651855469,
+      "logits/rejected": 1.6405918598175049,
+      "logps/chosen": -275.15325927734375,
+      "logps/pi_response": -150.0062713623047,
+      "logps/ref_response": -171.3158721923828,
+      "logps/rejected": -406.230224609375,
+      "loss": 0.5302,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.06263308972120285,
+      "rewards/margins": 0.5518540740013123,
+      "rewards/rejected": -0.6144872307777405,
+      "step": 130
+    },
+    {
+      "epoch": 0.8791208791208791,
+      "grad_norm": 6.184911001401629,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": 1.5727994441986084,
+      "logits/rejected": 1.7528778314590454,
+      "logps/chosen": -298.0870056152344,
+      "logps/pi_response": -154.67698669433594,
+      "logps/ref_response": -175.67694091796875,
+      "logps/rejected": -445.009521484375,
+      "loss": 0.5339,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.09744294732809067,
+      "rewards/margins": 0.645055890083313,
+      "rewards/rejected": -0.7424989342689514,
+      "step": 140
+    },
+    {
+      "epoch": 0.9419152276295133,
+      "grad_norm": 4.863112258557619,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": 1.399854302406311,
+      "logits/rejected": 1.5590882301330566,
+      "logps/chosen": -294.4033203125,
+      "logps/pi_response": -164.02032470703125,
+      "logps/ref_response": -185.34927368164062,
+      "logps/rejected": -422.95574951171875,
+      "loss": 0.5319,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.10503934323787689,
+      "rewards/margins": 0.6324479579925537,
+      "rewards/rejected": -0.7374873757362366,
+      "step": 150
+    },
+    {
+      "epoch": 0.9984301412872841,
+      "step": 159,
+      "total_flos": 0.0,
+      "train_loss": 0.5863969910819575,
+      "train_runtime": 3000.191,
+      "train_samples_per_second": 6.792,
+      "train_steps_per_second": 0.053
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 159,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}