Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: mit
+base_model: HuggingFaceH4/mistral-7b-sft-beta
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: 0.001_idpo_same_scratch_iter_2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 0.001_idpo_same_scratch_iter_2
+
+This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5994115610542537,
+    "train_runtime": 8245.2497,
+    "train_samples": 40756,
+    "train_samples_per_second": 4.943,
+    "train_steps_per_second": 0.039
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc0e27d5aaf8da4a95f4dc290a3ee6f9b5ce09f577300da1851163df13cc624a
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce5555662492ee502280d8086b9fc25076610c0f346e51e82e0a12f646b43731
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:327f86e9c42bd97034a88b9ab32c1d7d2e43657f9b929738c5603be4468645bf
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.5994115610542537,
+    "train_runtime": 8245.2497,
+    "train_samples": 40756,
+    "train_samples_per_second": 4.943,
+    "train_steps_per_second": 0.039
+}

trainer_state.json

@@ -0,0 +1,542 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9984301412872841,
+  "eval_steps": 500,
+  "global_step": 318,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5625e-08,
+      "logits/chosen": -2.8509469032287598,
+      "logits/rejected": -2.833181858062744,
+      "logps/chosen": -133.73171997070312,
+      "logps/pi_response": -47.9057502746582,
+      "logps/ref_response": -47.9057502746582,
+      "logps/rejected": -197.21273803710938,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -2.82759690284729,
+      "logits/rejected": -2.798935890197754,
+      "logps/chosen": -253.7941131591797,
+      "logps/pi_response": -71.65193939208984,
+      "logps/ref_response": -71.66093444824219,
+      "logps/rejected": -215.510009765625,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4166666567325592,
+      "rewards/chosen": 2.8001426471746527e-05,
+      "rewards/margins": -0.00033620299655012786,
+      "rewards/rejected": 0.0003642044321168214,
+      "step": 10
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.7570412158966064,
+      "logits/rejected": -2.760845422744751,
+      "logps/chosen": -217.1673126220703,
+      "logps/pi_response": -74.27923583984375,
+      "logps/ref_response": -74.31416320800781,
+      "logps/rejected": -202.97946166992188,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.005049519240856171,
+      "rewards/margins": 0.0024197015445679426,
+      "rewards/rejected": 0.0026298172306269407,
+      "step": 20
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -2.7637505531311035,
+      "logits/rejected": -2.7554142475128174,
+      "logps/chosen": -234.3155059814453,
+      "logps/pi_response": -68.43399047851562,
+      "logps/ref_response": -68.40460205078125,
+      "logps/rejected": -203.23374938964844,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.028325339779257774,
+      "rewards/margins": 0.01702897995710373,
+      "rewards/rejected": 0.01129635889083147,
+      "step": 30
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": -2.8049890995025635,
+      "logits/rejected": -2.780224084854126,
+      "logps/chosen": -207.7467041015625,
+      "logps/pi_response": -68.51658630371094,
+      "logps/ref_response": -64.04513549804688,
+      "logps/rejected": -203.04983520507812,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.04143567755818367,
+      "rewards/margins": 0.02963915839791298,
+      "rewards/rejected": 0.011796516366302967,
+      "step": 40
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.951291206355559e-07,
+      "logits/chosen": -2.7561471462249756,
+      "logits/rejected": -2.7407054901123047,
+      "logps/chosen": -256.8748474121094,
+      "logps/pi_response": -92.87406921386719,
+      "logps/ref_response": -74.82122039794922,
+      "logps/rejected": -236.1123504638672,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0026867527049034834,
+      "rewards/margins": 0.06110120937228203,
+      "rewards/rejected": -0.05841444805264473,
+      "step": 50
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": -2.679486036300659,
+      "logits/rejected": -2.681763172149658,
+      "logps/chosen": -266.70977783203125,
+      "logps/pi_response": -106.77949523925781,
+      "logps/ref_response": -76.94175720214844,
+      "logps/rejected": -241.783447265625,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.05337841436266899,
+      "rewards/margins": 0.11435987800359726,
+      "rewards/rejected": -0.16773828864097595,
+      "step": 60
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.785350472409791e-07,
+      "logits/chosen": -2.6562085151672363,
+      "logits/rejected": -2.638962984085083,
+      "logps/chosen": -262.48358154296875,
+      "logps/pi_response": -129.61695861816406,
+      "logps/ref_response": -82.19517517089844,
+      "logps/rejected": -264.07568359375,
+      "loss": 0.6444,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.20274746417999268,
+      "rewards/margins": 0.09253005683422089,
+      "rewards/rejected": -0.29527753591537476,
+      "step": 70
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": -2.6726343631744385,
+      "logits/rejected": -2.667086601257324,
+      "logps/chosen": -273.891845703125,
+      "logps/pi_response": -138.45387268066406,
+      "logps/ref_response": -72.91886901855469,
+      "logps/rejected": -269.2601318359375,
+      "loss": 0.6287,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.2682049870491028,
+      "rewards/margins": 0.1169251948595047,
+      "rewards/rejected": -0.3851301670074463,
+      "step": 80
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.5095513994085974e-07,
+      "logits/chosen": -2.535644292831421,
+      "logits/rejected": -2.532330274581909,
+      "logps/chosen": -239.97012329101562,
+      "logps/pi_response": -127.95100402832031,
+      "logps/ref_response": -69.45870208740234,
+      "logps/rejected": -257.403564453125,
+      "loss": 0.6109,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.14788806438446045,
+      "rewards/margins": 0.2313682734966278,
+      "rewards/rejected": -0.37925636768341064,
+      "step": 90
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": -2.5136990547180176,
+      "logits/rejected": -2.509000062942505,
+      "logps/chosen": -240.0645294189453,
+      "logps/pi_response": -125.2841796875,
+      "logps/ref_response": -70.416259765625,
+      "logps/rejected": -254.03012084960938,
+      "loss": 0.607,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2124701738357544,
+      "rewards/margins": 0.27277401089668274,
+      "rewards/rejected": -0.4852442145347595,
+      "step": 100
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.137151834863213e-07,
+      "logits/chosen": -2.480569362640381,
+      "logits/rejected": -2.443432331085205,
+      "logps/chosen": -275.77630615234375,
+      "logps/pi_response": -136.15298461914062,
+      "logps/ref_response": -71.45120239257812,
+      "logps/rejected": -269.72015380859375,
+      "loss": 0.6071,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.2312408983707428,
+      "rewards/margins": 0.23492033779621124,
+      "rewards/rejected": -0.46616125106811523,
+      "step": 110
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": -2.439021348953247,
+      "logits/rejected": -2.4340755939483643,
+      "logps/chosen": -268.33770751953125,
+      "logps/pi_response": -167.921630859375,
+      "logps/ref_response": -77.01984405517578,
+      "logps/rejected": -322.13433837890625,
+      "loss": 0.5893,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.35008108615875244,
+      "rewards/margins": 0.42321157455444336,
+      "rewards/rejected": -0.7732926607131958,
+      "step": 120
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6860532770864005e-07,
+      "logits/chosen": -2.347311496734619,
+      "logits/rejected": -2.335073947906494,
+      "logps/chosen": -315.92718505859375,
+      "logps/pi_response": -209.3319091796875,
+      "logps/ref_response": -67.93054962158203,
+      "logps/rejected": -312.83465576171875,
+      "loss": 0.5915,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.7186356782913208,
+      "rewards/margins": 0.3434630036354065,
+      "rewards/rejected": -1.062098741531372,
+      "step": 130
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": -2.2413113117218018,
+      "logits/rejected": -2.211487293243408,
+      "logps/chosen": -371.6416931152344,
+      "logps/pi_response": -265.18096923828125,
+      "logps/ref_response": -74.56938171386719,
+      "logps/rejected": -376.59466552734375,
+      "loss": 0.6044,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1778886318206787,
+      "rewards/margins": 0.32092222571372986,
+      "rewards/rejected": -1.4988110065460205,
+      "step": 140
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1779403380910425e-07,
+      "logits/chosen": -2.240055561065674,
+      "logits/rejected": -2.1979031562805176,
+      "logps/chosen": -307.1637878417969,
+      "logps/pi_response": -187.8131866455078,
+      "logps/ref_response": -69.4150390625,
+      "logps/rejected": -316.88873291015625,
+      "loss": 0.5816,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5614252090454102,
+      "rewards/margins": 0.41102123260498047,
+      "rewards/rejected": -0.9724465608596802,
+      "step": 150
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": -2.2194037437438965,
+      "logits/rejected": -2.2016549110412598,
+      "logps/chosen": -308.1712646484375,
+      "logps/pi_response": -226.4488983154297,
+      "logps/ref_response": -81.69718933105469,
+      "logps/rejected": -343.4349365234375,
+      "loss": 0.5699,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6651563048362732,
+      "rewards/margins": 0.4220353662967682,
+      "rewards/rejected": -1.0871917009353638,
+      "step": 160
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6372383496608186e-07,
+      "logits/chosen": -2.233020305633545,
+      "logits/rejected": -2.1675045490264893,
+      "logps/chosen": -348.2539978027344,
+      "logps/pi_response": -231.01455688476562,
+      "logps/ref_response": -75.20973205566406,
+      "logps/rejected": -334.9202880859375,
+      "loss": 0.6045,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6902152895927429,
+      "rewards/margins": 0.4886610507965088,
+      "rewards/rejected": -1.1788761615753174,
+      "step": 170
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": -2.137673854827881,
+      "logits/rejected": -2.0975818634033203,
+      "logps/chosen": -286.08172607421875,
+      "logps/pi_response": -202.77284240722656,
+      "logps/ref_response": -60.3970947265625,
+      "logps/rejected": -297.22442626953125,
+      "loss": 0.5776,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.6755498051643372,
+      "rewards/margins": 0.44268402457237244,
+      "rewards/rejected": -1.1182337999343872,
+      "step": 180
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.089939221172446e-07,
+      "logits/chosen": -2.1633880138397217,
+      "logits/rejected": -2.1441142559051514,
+      "logps/chosen": -301.385498046875,
+      "logps/pi_response": -249.9862060546875,
+      "logps/ref_response": -72.61909484863281,
+      "logps/rejected": -376.3998718261719,
+      "loss": 0.5583,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.8089518547058105,
+      "rewards/margins": 0.643667995929718,
+      "rewards/rejected": -1.4526197910308838,
+      "step": 190
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": -2.21587872505188,
+      "logits/rejected": -2.1797242164611816,
+      "logps/chosen": -383.5548400878906,
+      "logps/pi_response": -243.202880859375,
+      "logps/ref_response": -76.12086486816406,
+      "logps/rejected": -376.36138916015625,
+      "loss": 0.5584,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7832227349281311,
+      "rewards/margins": 0.4175417423248291,
+      "rewards/rejected": -1.2007642984390259,
+      "step": 200
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.562351990976095e-07,
+      "logits/chosen": -2.150066614151001,
+      "logits/rejected": -2.121753692626953,
+      "logps/chosen": -293.81488037109375,
+      "logps/pi_response": -219.0130157470703,
+      "logps/ref_response": -72.12835693359375,
+      "logps/rejected": -308.10919189453125,
+      "loss": 0.575,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8007175326347351,
+      "rewards/margins": 0.48071402311325073,
+      "rewards/rejected": -1.2814315557479858,
+      "step": 210
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": -2.182744026184082,
+      "logits/rejected": -2.1633565425872803,
+      "logps/chosen": -320.0791931152344,
+      "logps/pi_response": -229.01052856445312,
+      "logps/ref_response": -76.02903747558594,
+      "logps/rejected": -319.9397277832031,
+      "loss": 0.5582,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.7191491723060608,
+      "rewards/margins": 0.45336204767227173,
+      "rewards/rejected": -1.172511339187622,
+      "step": 220
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0798381331721107e-07,
+      "logits/chosen": -2.094500780105591,
+      "logits/rejected": -2.094238758087158,
+      "logps/chosen": -340.3441467285156,
+      "logps/pi_response": -242.52029418945312,
+      "logps/ref_response": -78.22608184814453,
+      "logps/rejected": -359.96136474609375,
+      "loss": 0.5815,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8624930381774902,
+      "rewards/margins": 0.3955257534980774,
+      "rewards/rejected": -1.2580187320709229,
+      "step": 230
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": -2.1416382789611816,
+      "logits/rejected": -2.107877254486084,
+      "logps/chosen": -334.34051513671875,
+      "logps/pi_response": -253.45559692382812,
+      "logps/ref_response": -71.44368743896484,
+      "logps/rejected": -343.25750732421875,
+      "loss": 0.567,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9990061521530151,
+      "rewards/margins": 0.39658617973327637,
+      "rewards/rejected": -1.3955923318862915,
+      "step": 240
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.655924144404906e-08,
+      "logits/chosen": -2.109405279159546,
+      "logits/rejected": -2.070265769958496,
+      "logps/chosen": -366.1980285644531,
+      "logps/pi_response": -274.1239318847656,
+      "logps/ref_response": -80.24897003173828,
+      "logps/rejected": -391.06976318359375,
+      "loss": 0.5633,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0960283279418945,
+      "rewards/margins": 0.4408624768257141,
+      "rewards/rejected": -1.5368907451629639,
+      "step": 250
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": -2.1373915672302246,
+      "logits/rejected": -2.050778865814209,
+      "logps/chosen": -357.5392150878906,
+      "logps/pi_response": -278.73681640625,
+      "logps/ref_response": -74.540283203125,
+      "logps/rejected": -366.91839599609375,
+      "loss": 0.5474,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.0771440267562866,
+      "rewards/margins": 0.7008964419364929,
+      "rewards/rejected": -1.7780405282974243,
+      "step": 260
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3952790595787986e-08,
+      "logits/chosen": -2.031721353530884,
+      "logits/rejected": -2.014037609100342,
+      "logps/chosen": -339.314453125,
+      "logps/pi_response": -293.8365783691406,
+      "logps/ref_response": -78.09291076660156,
+      "logps/rejected": -406.5647888183594,
+      "loss": 0.5568,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1859939098358154,
+      "rewards/margins": 0.4465731084346771,
+      "rewards/rejected": -1.6325668096542358,
+      "step": 270
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": -2.1228859424591064,
+      "logits/rejected": -2.0986340045928955,
+      "logps/chosen": -329.19244384765625,
+      "logps/pi_response": -264.7164306640625,
+      "logps/ref_response": -76.1480712890625,
+      "logps/rejected": -405.26287841796875,
+      "loss": 0.5487,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.001992106437683,
+      "rewards/margins": 0.5500288009643555,
+      "rewards/rejected": -1.552020788192749,
+      "step": 280
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1731874863145142e-08,
+      "logits/chosen": -2.1019539833068848,
+      "logits/rejected": -2.051776885986328,
+      "logps/chosen": -354.9820861816406,
+      "logps/pi_response": -268.553466796875,
+      "logps/ref_response": -72.30916595458984,
+      "logps/rejected": -364.39324951171875,
+      "loss": 0.5676,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.99445641040802,
+      "rewards/margins": 0.649712324142456,
+      "rewards/rejected": -1.6441688537597656,
+      "step": 290
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": -2.1944591999053955,
+      "logits/rejected": -2.160240650177002,
+      "logps/chosen": -352.7145080566406,
+      "logps/pi_response": -258.2195129394531,
+      "logps/ref_response": -73.66411590576172,
+      "logps/rejected": -385.5911865234375,
+      "loss": 0.5459,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9718688726425171,
+      "rewards/margins": 0.5499030351638794,
+      "rewards/rejected": -1.521772027015686,
+      "step": 300
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 9.64668657069706e-10,
+      "logits/chosen": -2.138836622238159,
+      "logits/rejected": -2.0898382663726807,
+      "logps/chosen": -320.83782958984375,
+      "logps/pi_response": -251.64108276367188,
+      "logps/ref_response": -73.73819732666016,
+      "logps/rejected": -360.6897277832031,
+      "loss": 0.5544,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.9648464322090149,
+      "rewards/margins": 0.5148458480834961,
+      "rewards/rejected": -1.4796922206878662,
+      "step": 310
+    },
+    {
+      "epoch": 1.0,
+      "step": 318,
+      "total_flos": 0.0,
+      "train_loss": 0.5994115610542537,
+      "train_runtime": 8245.2497,
+      "train_samples_per_second": 4.943,
+      "train_steps_per_second": 0.039
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 318,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}