Model save

README.md

@@ -0,0 +1,58 @@
+---
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: two_agent_rdpo_iter_3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# two_agent_rdpo_iter_3
+
+This model was trained from scratch on the None dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 1
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 5
+- gradient_accumulation_steps: 26
+- total_train_batch_size: 130
+- total_eval_batch_size: 10
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.38.2
+- Pytorch 2.3.1+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6842105340810469,
+    "train_runtime": 43334.2513,
+    "train_samples": 21135,
+    "train_samples_per_second": 0.488,
+    "train_steps_per_second": 0.004
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.38.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a071a4deca75805ef2950edceb22a3e250c0c844d714a96e74166e7d79491918
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f03c46358fb0e38affe0cec71b74547604821342f9fc6ef8324ac7a7cb2dbfe
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bea42a800ecdc52cb14e1ff5b50bc749b917d985c570b979de653823626c7992
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.6842105340810469,
+    "train_runtime": 43334.2513,
+    "train_samples": 21135,
+    "train_samples_per_second": 0.488,
+    "train_steps_per_second": 0.004
+}

trainer_state.json

@@ -0,0 +1,336 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9964513839602555,
+  "eval_steps": 500,
+  "global_step": 162,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 22.254964893351787,
+      "learning_rate": 2.941176470588235e-08,
+      "logits/chosen": -2.4821925163269043,
+      "logits/rejected": -2.235710382461548,
+      "logps/chosen": -280.2124328613281,
+      "logps/pi_response": -436.6075744628906,
+      "logps/ref_response": -436.6075744628906,
+      "logps/rejected": -273.4680480957031,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.06,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 23.328818767881323,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -2.3237369060516357,
+      "logits/rejected": -2.2622690200805664,
+      "logps/chosen": -239.0484619140625,
+      "logps/pi_response": -371.8123474121094,
+      "logps/ref_response": -370.81756591796875,
+      "logps/rejected": -253.18133544921875,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": -0.009806615300476551,
+      "rewards/margins": 0.001764079206623137,
+      "rewards/rejected": -0.011570693925023079,
+      "step": 10
+    },
+    {
+      "epoch": 0.12,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 31.176954929712068,
+      "learning_rate": 4.99472085783721e-07,
+      "logits/chosen": -2.121244430541992,
+      "logits/rejected": -2.1339948177337646,
+      "logps/chosen": -252.5571746826172,
+      "logps/pi_response": -376.398193359375,
+      "logps/ref_response": -357.7356872558594,
+      "logps/rejected": -262.6872863769531,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.5076923370361328,
+      "rewards/chosen": -0.21289017796516418,
+      "rewards/margins": -0.0055158319883048534,
+      "rewards/rejected": -0.20737436413764954,
+      "step": 20
+    },
+    {
+      "epoch": 0.18,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 29.374574206663485,
+      "learning_rate": 4.901488388458247e-07,
+      "logits/chosen": -2.0692079067230225,
+      "logits/rejected": -2.033362865447998,
+      "logps/chosen": -240.89402770996094,
+      "logps/pi_response": -404.3094787597656,
+      "logps/ref_response": -375.89013671875,
+      "logps/rejected": -253.24954223632812,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.5038461685180664,
+      "rewards/chosen": -0.07824164628982544,
+      "rewards/margins": 0.01707359030842781,
+      "rewards/rejected": -0.09531523287296295,
+      "step": 30
+    },
+    {
+      "epoch": 0.25,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 30.44936672759969,
+      "learning_rate": 4.695964991097616e-07,
+      "logits/chosen": -1.8107295036315918,
+      "logits/rejected": -2.005711555480957,
+      "logps/chosen": -303.3603515625,
+      "logps/pi_response": -455.744873046875,
+      "logps/ref_response": -391.4946594238281,
+      "logps/rejected": -326.20098876953125,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.5307692289352417,
+      "rewards/chosen": -0.6260919570922852,
+      "rewards/margins": 0.034338854253292084,
+      "rewards/rejected": -0.6604308485984802,
+      "step": 40
+    },
+    {
+      "epoch": 0.31,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 23.599398572955327,
+      "learning_rate": 4.3877607113930516e-07,
+      "logits/chosen": -1.907410979270935,
+      "logits/rejected": -1.855884075164795,
+      "logps/chosen": -248.3097686767578,
+      "logps/pi_response": -406.1575012207031,
+      "logps/ref_response": -367.650390625,
+      "logps/rejected": -256.7185974121094,
+      "loss": 0.7017,
+      "rewards/accuracies": 0.48076921701431274,
+      "rewards/chosen": -0.12874366343021393,
+      "rewards/margins": -0.0029275938868522644,
+      "rewards/rejected": -0.12581607699394226,
+      "step": 50
+    },
+    {
+      "epoch": 0.37,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 23.113382825281022,
+      "learning_rate": 3.991286838919086e-07,
+      "logits/chosen": -1.7167693376541138,
+      "logits/rejected": -1.6973148584365845,
+      "logps/chosen": -320.3218688964844,
+      "logps/pi_response": -487.9947204589844,
+      "logps/ref_response": -405.68280029296875,
+      "logps/rejected": -322.4259948730469,
+      "loss": 0.6945,
+      "rewards/accuracies": 0.5615384578704834,
+      "rewards/chosen": -0.5926837921142578,
+      "rewards/margins": 0.038080714643001556,
+      "rewards/rejected": -0.6307645440101624,
+      "step": 60
+    },
+    {
+      "epoch": 0.43,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 31.784636012816986,
+      "learning_rate": 3.52508205130354e-07,
+      "logits/chosen": -1.5467545986175537,
+      "logits/rejected": -1.5477180480957031,
+      "logps/chosen": -306.8106994628906,
+      "logps/pi_response": -480.5911865234375,
+      "logps/ref_response": -377.5798034667969,
+      "logps/rejected": -334.5196533203125,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.787551760673523,
+      "rewards/margins": 0.06417767703533173,
+      "rewards/rejected": -0.8517293930053711,
+      "step": 70
+    },
+    {
+      "epoch": 0.49,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 35.2255241905511,
+      "learning_rate": 3.010945566265912e-07,
+      "logits/chosen": -1.584012508392334,
+      "logits/rejected": -1.7565481662750244,
+      "logps/chosen": -292.3354797363281,
+      "logps/pi_response": -463.38507080078125,
+      "logps/ref_response": -351.9272766113281,
+      "logps/rejected": -315.16680908203125,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.5961538553237915,
+      "rewards/chosen": -0.6284902691841125,
+      "rewards/margins": 0.07952240109443665,
+      "rewards/rejected": -0.7080127596855164,
+      "step": 80
+    },
+    {
+      "epoch": 0.55,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 24.755608967697253,
+      "learning_rate": 2.4729178344249006e-07,
+      "logits/chosen": -1.6239264011383057,
+      "logits/rejected": -1.7038928270339966,
+      "logps/chosen": -333.8011474609375,
+      "logps/pi_response": -518.841064453125,
+      "logps/ref_response": -389.2355651855469,
+      "logps/rejected": -362.0622253417969,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.5538461804389954,
+      "rewards/chosen": -0.9721028804779053,
+      "rewards/margins": 0.06480876356363297,
+      "rewards/rejected": -1.0369116067886353,
+      "step": 90
+    },
+    {
+      "epoch": 0.62,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 28.32503077830092,
+      "learning_rate": 1.9361564345465145e-07,
+      "logits/chosen": -1.6863784790039062,
+      "logits/rejected": -1.576540470123291,
+      "logps/chosen": -324.2234802246094,
+      "logps/pi_response": -477.94110107421875,
+      "logps/ref_response": -352.850830078125,
+      "logps/rejected": -337.5308532714844,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.5692307949066162,
+      "rewards/chosen": -0.95728600025177,
+      "rewards/margins": 0.05953861400485039,
+      "rewards/rejected": -1.0168246030807495,
+      "step": 100
+    },
+    {
+      "epoch": 0.68,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 24.75749478420196,
+      "learning_rate": 1.4257597331216208e-07,
+      "logits/chosen": -1.5470343828201294,
+      "logits/rejected": -1.6905183792114258,
+      "logps/chosen": -339.9206848144531,
+      "logps/pi_response": -517.6537475585938,
+      "logps/ref_response": -383.1593322753906,
+      "logps/rejected": -354.73150634765625,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.4923076927661896,
+      "rewards/chosen": -1.0359457731246948,
+      "rewards/margins": 0.02877073362469673,
+      "rewards/rejected": -1.0647164583206177,
+      "step": 110
+    },
+    {
+      "epoch": 0.74,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 27.374335177920894,
+      "learning_rate": 9.655933126436563e-08,
+      "logits/chosen": -1.7334721088409424,
+      "logits/rejected": -1.6775918006896973,
+      "logps/chosen": -323.307861328125,
+      "logps/pi_response": -500.0389709472656,
+      "logps/ref_response": -373.8336181640625,
+      "logps/rejected": -337.35687255859375,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.5730769038200378,
+      "rewards/chosen": -0.884036660194397,
+      "rewards/margins": 0.05840235576033592,
+      "rewards/rejected": -0.9424390196800232,
+      "step": 120
+    },
+    {
+      "epoch": 0.8,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 29.06627362451846,
+      "learning_rate": 5.771740434959277e-08,
+      "logits/chosen": -1.7559294700622559,
+      "logits/rejected": -1.656353235244751,
+      "logps/chosen": -322.9090270996094,
+      "logps/pi_response": -490.60504150390625,
+      "logps/ref_response": -358.2728271484375,
+      "logps/rejected": -327.773681640625,
+      "loss": 0.677,
+      "rewards/accuracies": 0.5884615182876587,
+      "rewards/chosen": -0.8856447339057922,
+      "rewards/margins": 0.060253795236349106,
+      "rewards/rejected": -0.9458985328674316,
+      "step": 130
+    },
+    {
+      "epoch": 0.86,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 29.276601449078793,
+      "learning_rate": 2.7866397900677185e-08,
+      "logits/chosen": -1.5905346870422363,
+      "logits/rejected": -1.5064502954483032,
+      "logps/chosen": -327.3086242675781,
+      "logps/pi_response": -507.3574523925781,
+      "logps/ref_response": -365.5458984375,
+      "logps/rejected": -337.4426574707031,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.6307692527770996,
+      "rewards/chosen": -0.9561350345611572,
+      "rewards/margins": 0.08353108912706375,
+      "rewards/rejected": -1.0396660566329956,
+      "step": 140
+    },
+    {
+      "epoch": 0.92,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 30.496204004580633,
+      "learning_rate": 8.402111802159412e-09,
+      "logits/chosen": -1.74466872215271,
+      "logits/rejected": -1.6440544128417969,
+      "logps/chosen": -315.1639709472656,
+      "logps/pi_response": -511.9558410644531,
+      "logps/ref_response": -375.8587951660156,
+      "logps/rejected": -326.924560546875,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.5846154093742371,
+      "rewards/chosen": -0.8547829389572144,
+      "rewards/margins": 0.06981422752141953,
+      "rewards/rejected": -0.9245970845222473,
+      "step": 150
+    },
+    {
+      "epoch": 0.98,
+      "eta": 0.0010000000474974513,
+      "grad_norm": 29.5797991936398,
+      "learning_rate": 2.3467443900582197e-10,
+      "logits/chosen": -1.836174726486206,
+      "logits/rejected": -1.7468369007110596,
+      "logps/chosen": -307.39251708984375,
+      "logps/pi_response": -489.5311279296875,
+      "logps/ref_response": -352.3800354003906,
+      "logps/rejected": -338.456787109375,
+      "loss": 0.6738,
+      "rewards/accuracies": 0.5846154093742371,
+      "rewards/chosen": -0.8461953401565552,
+      "rewards/margins": 0.09575623273849487,
+      "rewards/rejected": -0.9419515132904053,
+      "step": 160
+    },
+    {
+      "epoch": 1.0,
+      "step": 162,
+      "total_flos": 0.0,
+      "train_loss": 0.6842105340810469,
+      "train_runtime": 43334.2513,
+      "train_samples_per_second": 0.488,
+      "train_steps_per_second": 0.004
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 162,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}