Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: mit
+base_model: Katayoon/DPO-Zephyr-7B
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: SELM-Zephyr-7B-iter-1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# SELM-Zephyr-7B-iter-1
+
+This model is a fine-tuned version of [Katayoon/DPO-Zephyr-7B](https://huggingface.co/Katayoon/DPO-Zephyr-7B) on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 16
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 64
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.40.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.5111325620108569,
+    "train_runtime": 3660.7297,
+    "train_samples": 15283,
+    "train_samples_per_second": 4.175,
+    "train_steps_per_second": 0.065
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.40.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c729fb6f9665bc8778e01384e8103794a1d160f86297962f3b6b4ae2f6300da
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66f270924bfba998bfdce687ff475c60b26899c8b90946429c51726df30c01b2
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87a6f48fa806c3699373befe26df073d744e40e067a2a269eb27083493a97906
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.5111325620108569,
+    "train_runtime": 3660.7297,
+    "train_samples": 15283,
+    "train_samples_per_second": 4.175,
+    "train_steps_per_second": 0.065
+}

trainer_state.json

@@ -0,0 +1,438 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 239,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0041841004184100415,
+      "grad_norm": 14.184057660146673,
+      "learning_rate": 2.083333333333333e-08,
+      "logits/chosen": -3.0079779624938965,
+      "logits/rejected": -2.942084312438965,
+      "logps/chosen": -420.3988037109375,
+      "logps/pi_response": -130.643798828125,
+      "logps/ref_response": -130.66250610351562,
+      "logps/rejected": -287.5452575683594,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.00042182920151390135,
+      "rewards/margins": -0.0013115692418068647,
+      "rewards/rejected": 0.0017333984142169356,
+      "step": 1
+    },
+    {
+      "epoch": 0.04184100418410042,
+      "grad_norm": 17.93219296708063,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.8013784885406494,
+      "logits/rejected": -2.748150110244751,
+      "logps/chosen": -206.9554901123047,
+      "logps/pi_response": -156.2655029296875,
+      "logps/ref_response": -156.39459228515625,
+      "logps/rejected": -245.3439178466797,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.3888888955116272,
+      "rewards/chosen": -0.0007119781221263111,
+      "rewards/margins": -0.0006557225133292377,
+      "rewards/rejected": -5.62555760552641e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.08368200836820083,
+      "grad_norm": 17.194106706089425,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.8823390007019043,
+      "logits/rejected": -2.8756155967712402,
+      "logps/chosen": -174.19937133789062,
+      "logps/pi_response": -118.446533203125,
+      "logps/ref_response": -120.41975402832031,
+      "logps/rejected": -269.61676025390625,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.006955099292099476,
+      "rewards/margins": 0.053968124091625214,
+      "rewards/rejected": -0.04701302573084831,
+      "step": 20
+    },
+    {
+      "epoch": 0.12552301255230125,
+      "grad_norm": 18.192768474274775,
+      "learning_rate": 4.990398100856366e-07,
+      "logits/chosen": -2.9674675464630127,
+      "logits/rejected": -2.870899200439453,
+      "logps/chosen": -198.1211700439453,
+      "logps/pi_response": -132.55484008789062,
+      "logps/ref_response": -147.1282501220703,
+      "logps/rejected": -273.9727783203125,
+      "loss": 0.6414,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.011768990196287632,
+      "rewards/margins": 0.12694349884986877,
+      "rewards/rejected": -0.11517448723316193,
+      "step": 30
+    },
+    {
+      "epoch": 0.16736401673640167,
+      "grad_norm": 29.202771667007173,
+      "learning_rate": 4.931986719649298e-07,
+      "logits/chosen": -2.967485189437866,
+      "logits/rejected": -2.9260916709899902,
+      "logps/chosen": -260.45697021484375,
+      "logps/pi_response": -128.0771942138672,
+      "logps/ref_response": -139.22616577148438,
+      "logps/rejected": -309.2800598144531,
+      "loss": 0.5839,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.22019478678703308,
+      "rewards/margins": 0.4916856288909912,
+      "rewards/rejected": -0.7118803858757019,
+      "step": 40
+    },
+    {
+      "epoch": 0.20920502092050208,
+      "grad_norm": 25.274131617507575,
+      "learning_rate": 4.821741763807186e-07,
+      "logits/chosen": -3.052572727203369,
+      "logits/rejected": -3.0400023460388184,
+      "logps/chosen": -234.84664916992188,
+      "logps/pi_response": -131.11582946777344,
+      "logps/ref_response": -131.74740600585938,
+      "logps/rejected": -355.964111328125,
+      "loss": 0.5563,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.368060439825058,
+      "rewards/margins": 0.5468977093696594,
+      "rewards/rejected": -0.9149581789970398,
+      "step": 50
+    },
+    {
+      "epoch": 0.2510460251046025,
+      "grad_norm": 20.870691249572463,
+      "learning_rate": 4.662012913161997e-07,
+      "logits/chosen": -3.0132408142089844,
+      "logits/rejected": -2.9283642768859863,
+      "logps/chosen": -305.3451843261719,
+      "logps/pi_response": -169.0743865966797,
+      "logps/ref_response": -161.09674072265625,
+      "logps/rejected": -397.9371643066406,
+      "loss": 0.566,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.6505581140518188,
+      "rewards/margins": 0.46749958395957947,
+      "rewards/rejected": -1.1180577278137207,
+      "step": 60
+    },
+    {
+      "epoch": 0.2928870292887029,
+      "grad_norm": 21.658467534309377,
+      "learning_rate": 4.456204510851956e-07,
+      "logits/chosen": -2.993476152420044,
+      "logits/rejected": -2.8870837688446045,
+      "logps/chosen": -249.2686004638672,
+      "logps/pi_response": -158.66510009765625,
+      "logps/ref_response": -148.65969848632812,
+      "logps/rejected": -391.9620361328125,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.40117892622947693,
+      "rewards/margins": 0.8544867634773254,
+      "rewards/rejected": -1.2556655406951904,
+      "step": 70
+    },
+    {
+      "epoch": 0.33472803347280333,
+      "grad_norm": 30.641531156406053,
+      "learning_rate": 4.2087030056579986e-07,
+      "logits/chosen": -2.87054443359375,
+      "logits/rejected": -2.7671115398406982,
+      "logps/chosen": -234.6735076904297,
+      "logps/pi_response": -190.28416442871094,
+      "logps/ref_response": -169.210205078125,
+      "logps/rejected": -352.5207824707031,
+      "loss": 0.5218,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.49536633491516113,
+      "rewards/margins": 0.7785670161247253,
+      "rewards/rejected": -1.2739332914352417,
+      "step": 80
+    },
+    {
+      "epoch": 0.37656903765690375,
+      "grad_norm": 25.635102515969344,
+      "learning_rate": 3.9247834624635404e-07,
+      "logits/chosen": -2.6343486309051514,
+      "logits/rejected": -2.6347906589508057,
+      "logps/chosen": -208.4500732421875,
+      "logps/pi_response": -147.683349609375,
+      "logps/ref_response": -130.34739685058594,
+      "logps/rejected": -388.5314025878906,
+      "loss": 0.487,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.4610714018344879,
+      "rewards/margins": 0.9515409469604492,
+      "rewards/rejected": -1.4126123189926147,
+      "step": 90
+    },
+    {
+      "epoch": 0.41841004184100417,
+      "grad_norm": 44.7138776674502,
+      "learning_rate": 3.610497133404795e-07,
+      "logits/chosen": -2.872772693634033,
+      "logits/rejected": -2.794921636581421,
+      "logps/chosen": -346.27325439453125,
+      "logps/pi_response": -213.10733032226562,
+      "logps/ref_response": -154.00936889648438,
+      "logps/rejected": -404.7713928222656,
+      "loss": 0.4942,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0241730213165283,
+      "rewards/margins": 0.9248477816581726,
+      "rewards/rejected": -1.9490207433700562,
+      "step": 100
+    },
+    {
+      "epoch": 0.4602510460251046,
+      "grad_norm": 35.75822161142808,
+      "learning_rate": 3.272542485937368e-07,
+      "logits/chosen": -2.634904623031616,
+      "logits/rejected": -2.599783182144165,
+      "logps/chosen": -300.8379211425781,
+      "logps/pi_response": -184.91305541992188,
+      "logps/ref_response": -139.14535522460938,
+      "logps/rejected": -374.28448486328125,
+      "loss": 0.49,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.842223048210144,
+      "rewards/margins": 0.6618617177009583,
+      "rewards/rejected": -1.504084825515747,
+      "step": 110
+    },
+    {
+      "epoch": 0.502092050209205,
+      "grad_norm": 26.087019733451935,
+      "learning_rate": 2.9181224366319943e-07,
+      "logits/chosen": -2.7194137573242188,
+      "logits/rejected": -2.6171250343322754,
+      "logps/chosen": -368.3160705566406,
+      "logps/pi_response": -202.00479125976562,
+      "logps/ref_response": -160.38339233398438,
+      "logps/rejected": -430.47021484375,
+      "loss": 0.4916,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.8208904266357422,
+      "rewards/margins": 0.858447253704071,
+      "rewards/rejected": -1.679337739944458,
+      "step": 120
+    },
+    {
+      "epoch": 0.5439330543933054,
+      "grad_norm": 29.140061940651456,
+      "learning_rate": 2.55479083351317e-07,
+      "logits/chosen": -2.735330820083618,
+      "logits/rejected": -2.6966159343719482,
+      "logps/chosen": -353.4627990722656,
+      "logps/pi_response": -210.87979125976562,
+      "logps/ref_response": -147.25827026367188,
+      "logps/rejected": -425.4654846191406,
+      "loss": 0.4753,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.119887113571167,
+      "rewards/margins": 0.6405670642852783,
+      "rewards/rejected": -1.7604541778564453,
+      "step": 130
+    },
+    {
+      "epoch": 0.5857740585774058,
+      "grad_norm": 37.19290026362009,
+      "learning_rate": 2.19029145890313e-07,
+      "logits/chosen": -2.5590577125549316,
+      "logits/rejected": -2.5353775024414062,
+      "logps/chosen": -266.3215637207031,
+      "logps/pi_response": -199.64517211914062,
+      "logps/ref_response": -137.48602294921875,
+      "logps/rejected": -411.43792724609375,
+      "loss": 0.4785,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.9264217615127563,
+      "rewards/margins": 0.8948189616203308,
+      "rewards/rejected": -1.8212406635284424,
+      "step": 140
+    },
+    {
+      "epoch": 0.6276150627615062,
+      "grad_norm": 29.759542463318493,
+      "learning_rate": 1.8323929841460178e-07,
+      "logits/chosen": -2.417973279953003,
+      "logits/rejected": -2.5954794883728027,
+      "logps/chosen": -252.88388061523438,
+      "logps/pi_response": -204.92825317382812,
+      "logps/ref_response": -141.11904907226562,
+      "logps/rejected": -497.9703674316406,
+      "loss": 0.4538,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9101165533065796,
+      "rewards/margins": 1.0697840452194214,
+      "rewards/rejected": -1.9799007177352905,
+      "step": 150
+    },
+    {
+      "epoch": 0.6694560669456067,
+      "grad_norm": 39.070777526950835,
+      "learning_rate": 1.488723393865766e-07,
+      "logits/chosen": -2.204148054122925,
+      "logits/rejected": -2.2511379718780518,
+      "logps/chosen": -302.68292236328125,
+      "logps/pi_response": -211.3453369140625,
+      "logps/ref_response": -142.22390747070312,
+      "logps/rejected": -432.41949462890625,
+      "loss": 0.4928,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.000138521194458,
+      "rewards/margins": 0.8643314242362976,
+      "rewards/rejected": -1.8644697666168213,
+      "step": 160
+    },
+    {
+      "epoch": 0.7112970711297071,
+      "grad_norm": 31.540461400821204,
+      "learning_rate": 1.1666074087171627e-07,
+      "logits/chosen": -2.669570207595825,
+      "logits/rejected": -2.576828718185425,
+      "logps/chosen": -353.13861083984375,
+      "logps/pi_response": -229.06674194335938,
+      "logps/ref_response": -147.0096435546875,
+      "logps/rejected": -430.3352966308594,
+      "loss": 0.4414,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1818138360977173,
+      "rewards/margins": 1.0494412183761597,
+      "rewards/rejected": -2.231255054473877,
+      "step": 170
+    },
+    {
+      "epoch": 0.7531380753138075,
+      "grad_norm": 34.811019754957,
+      "learning_rate": 8.729103716819111e-08,
+      "logits/chosen": -2.3912153244018555,
+      "logits/rejected": -2.3354153633117676,
+      "logps/chosen": -316.1119689941406,
+      "logps/pi_response": -212.54348754882812,
+      "logps/ref_response": -119.88712310791016,
+      "logps/rejected": -427.4794921875,
+      "loss": 0.4602,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.295536756515503,
+      "rewards/margins": 1.0199306011199951,
+      "rewards/rejected": -2.315467357635498,
+      "step": 180
+    },
+    {
+      "epoch": 0.7949790794979079,
+      "grad_norm": 38.75219012751155,
+      "learning_rate": 6.138919252022435e-08,
+      "logits/chosen": -2.5271811485290527,
+      "logits/rejected": -2.4973433017730713,
+      "logps/chosen": -340.279052734375,
+      "logps/pi_response": -243.56185913085938,
+      "logps/ref_response": -154.7118377685547,
+      "logps/rejected": -425.8468322753906,
+      "loss": 0.4598,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1167631149291992,
+      "rewards/margins": 0.9512733221054077,
+      "rewards/rejected": -2.0680365562438965,
+      "step": 190
+    },
+    {
+      "epoch": 0.8368200836820083,
+      "grad_norm": 36.04921418642425,
+      "learning_rate": 3.9507259776993954e-08,
+      "logits/chosen": -2.605112075805664,
+      "logits/rejected": -2.642122268676758,
+      "logps/chosen": -282.02960205078125,
+      "logps/pi_response": -225.96115112304688,
+      "logps/ref_response": -147.7294921875,
+      "logps/rejected": -466.0926818847656,
+      "loss": 0.448,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1095210313796997,
+      "rewards/margins": 1.1342695951461792,
+      "rewards/rejected": -2.2437903881073,
+      "step": 200
+    },
+    {
+      "epoch": 0.8786610878661087,
+      "grad_norm": 36.74338685244281,
+      "learning_rate": 2.2111614344599684e-08,
+      "logits/chosen": -2.589224338531494,
+      "logits/rejected": -2.5517868995666504,
+      "logps/chosen": -320.30889892578125,
+      "logps/pi_response": -234.22390747070312,
+      "logps/ref_response": -146.21420288085938,
+      "logps/rejected": -408.4784240722656,
+      "loss": 0.4717,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1880152225494385,
+      "rewards/margins": 0.8048989176750183,
+      "rewards/rejected": -1.9929141998291016,
+      "step": 210
+    },
+    {
+      "epoch": 0.9205020920502092,
+      "grad_norm": 38.40646706464352,
+      "learning_rate": 9.57301420397924e-09,
+      "logits/chosen": -2.4826607704162598,
+      "logits/rejected": -2.428887128829956,
+      "logps/chosen": -263.56396484375,
+      "logps/pi_response": -225.4619903564453,
+      "logps/ref_response": -148.5628662109375,
+      "logps/rejected": -453.89263916015625,
+      "loss": 0.446,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.8476516008377075,
+      "rewards/margins": 1.5934152603149414,
+      "rewards/rejected": -2.4410667419433594,
+      "step": 220
+    },
+    {
+      "epoch": 0.9623430962343096,
+      "grad_norm": 39.635773892484444,
+      "learning_rate": 2.158697848236607e-09,
+      "logits/chosen": -2.5632519721984863,
+      "logits/rejected": -2.492157220840454,
+      "logps/chosen": -320.2151184082031,
+      "logps/pi_response": -234.2839813232422,
+      "logps/ref_response": -144.2044219970703,
+      "logps/rejected": -454.62713623046875,
+      "loss": 0.4716,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.25511634349823,
+      "rewards/margins": 1.1321589946746826,
+      "rewards/rejected": -2.387275218963623,
+      "step": 230
+    },
+    {
+      "epoch": 1.0,
+      "step": 239,
+      "total_flos": 0.0,
+      "train_loss": 0.5111325620108569,
+      "train_runtime": 3660.7297,
+      "train_samples_per_second": 4.175,
+      "train_steps_per_second": 0.065
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 239,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}