Model save

README.md

@@ -0,0 +1,73 @@
+---
+license: other
+base_model: lewtun/gemma-7b-sft-full-deita-10k-v0
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: gemma-7b-dpo-full-mix1-beta-0.1-epoch-3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gemma-7b-dpo-full-mix1-beta-0.1-epoch-3
+
+This model is a fine-tuned version of [lewtun/gemma-7b-sft-full-deita-10k-v0](https://huggingface.co/lewtun/gemma-7b-sft-full-deita-10k-v0) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5052
+- Rewards/chosen: -2.9796
+- Rewards/rejected: -5.1227
+- Rewards/accuracies: 0.7708
+- Rewards/margins: 2.1431
+- Logps/rejected: -502.7766
+- Logps/chosen: -483.2003
+- Logits/rejected: 96.3785
+- Logits/chosen: 102.3770
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 3.0
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.1371        | 1.9   | 100  | 0.5052          | -2.9796        | -5.1227          | 0.7708             | 2.1431          | -502.7766      | -483.2003    | 96.3785         | 102.3770      |
+
+
+### Framework versions
+
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.1

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.96,
+    "train_loss": 0.27917395799587935,
+    "train_runtime": 1799.8719,
+    "train_samples": 6750,
+    "train_samples_per_second": 11.251,
+    "train_steps_per_second": 0.087
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.39.0.dev0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b58d2c1a08ad8641421be10c1ee00a78cf6dbe8eaf28ff14c85b6dde37ba062e
+size 4995496656

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b0b6dec1a1035d69ac45ea06f2511af7ca2a48d849cfc8dfbba6cccb5308ea5
+size 4982953168

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab9d1cbcc6e28742508e38fd027303045dfc9bbced8ccf2dc2c638882397f4ee
+size 4982953200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b17b8e29209c5e55d20d4c05b791be91c882ec6de966a04c5351fb74ab24b17a
+size 2113988336

model.safetensors.index.json

@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 17075361792
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.96,
+    "train_loss": 0.27917395799587935,
+    "train_runtime": 1799.8719,
+    "train_samples": 6750,
+    "train_samples_per_second": 11.251,
+    "train_steps_per_second": 0.087
+}

trainer_state.json

@@ -0,0 +1,286 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.957345971563981,
+  "eval_steps": 100,
+  "global_step": 156,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02,
+      "grad_norm": 265.9657020142556,
+      "learning_rate": 3.125e-08,
+      "logits/chosen": 123.11854553222656,
+      "logits/rejected": 97.00198364257812,
+      "logps/chosen": -425.18585205078125,
+      "logps/rejected": -424.1869201660156,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 372.96475604860683,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": 117.39414978027344,
+      "logits/rejected": 136.32305908203125,
+      "logps/chosen": -442.28045654296875,
+      "logps/rejected": -524.1576538085938,
+      "loss": 0.7748,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 0.1107947826385498,
+      "rewards/margins": 0.115071140229702,
+      "rewards/rejected": -0.004276359919458628,
+      "step": 10
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 213.17360776034892,
+      "learning_rate": 4.989935734988097e-07,
+      "logits/chosen": 125.45426177978516,
+      "logits/rejected": 133.0935821533203,
+      "logps/chosen": -426.3124084472656,
+      "logps/rejected": -493.6737365722656,
+      "loss": 0.6502,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.004780772142112255,
+      "rewards/margins": 0.6115384101867676,
+      "rewards/rejected": -0.6163192987442017,
+      "step": 20
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 204.32015271324505,
+      "learning_rate": 4.877641290737883e-07,
+      "logits/chosen": 123.5244369506836,
+      "logits/rejected": 126.99784851074219,
+      "logps/chosen": -466.4947814941406,
+      "logps/rejected": -534.4097900390625,
+      "loss": 0.6377,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.137466549873352,
+      "rewards/margins": 1.5820324420928955,
+      "rewards/rejected": -2.719499111175537,
+      "step": 30
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 200.5277184123756,
+      "learning_rate": 4.646121984004665e-07,
+      "logits/chosen": 126.0523910522461,
+      "logits/rejected": 120.98759460449219,
+      "logps/chosen": -489.78033447265625,
+      "logps/rejected": -514.3892211914062,
+      "loss": 0.6145,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.659891128540039,
+      "rewards/margins": 1.3025611639022827,
+      "rewards/rejected": -2.9624521732330322,
+      "step": 40
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 205.88684610219255,
+      "learning_rate": 4.3069871595684787e-07,
+      "logits/chosen": 134.4938201904297,
+      "logits/rejected": 134.79849243164062,
+      "logps/chosen": -502.4814453125,
+      "logps/rejected": -523.4627685546875,
+      "loss": 0.5901,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9658725261688232,
+      "rewards/margins": 1.2555662393569946,
+      "rewards/rejected": -3.2214386463165283,
+      "step": 50
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 87.33372866728267,
+      "learning_rate": 3.877242453630256e-07,
+      "logits/chosen": 133.0808563232422,
+      "logits/rejected": 136.4200897216797,
+      "logps/chosen": -467.0462951660156,
+      "logps/rejected": -506.5567321777344,
+      "loss": 0.2517,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.0280801057815552,
+      "rewards/margins": 3.1976516246795654,
+      "rewards/rejected": -4.22573184967041,
+      "step": 60
+    },
+    {
+      "epoch": 1.33,
+      "grad_norm": 71.00765502932312,
+      "learning_rate": 3.378437060203357e-07,
+      "logits/chosen": 128.8972625732422,
+      "logits/rejected": 129.42971801757812,
+      "logps/chosen": -440.6957092285156,
+      "logps/rejected": -552.7991943359375,
+      "loss": 0.1231,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.1781926155090332,
+      "rewards/margins": 3.8879973888397217,
+      "rewards/rejected": -5.066189765930176,
+      "step": 70
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 68.54852270837435,
+      "learning_rate": 2.8355831645441387e-07,
+      "logits/chosen": 133.30160522460938,
+      "logits/rejected": 134.1454620361328,
+      "logps/chosen": -499.032958984375,
+      "logps/rejected": -585.234130859375,
+      "loss": 0.1193,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.9932767748832703,
+      "rewards/margins": 4.856534481048584,
+      "rewards/rejected": -5.849810600280762,
+      "step": 80
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 96.90359229935069,
+      "learning_rate": 2.2759017277414164e-07,
+      "logits/chosen": 120.51957702636719,
+      "logits/rejected": 123.01716613769531,
+      "logps/chosen": -477.44366455078125,
+      "logps/rejected": -548.8724365234375,
+      "loss": 0.1182,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -1.5276060104370117,
+      "rewards/margins": 4.922442436218262,
+      "rewards/rejected": -6.450047969818115,
+      "step": 90
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 57.0241100183833,
+      "learning_rate": 1.7274575140626315e-07,
+      "logits/chosen": 134.63583374023438,
+      "logits/rejected": 125.1494140625,
+      "logps/chosen": -492.84930419921875,
+      "logps/rejected": -590.646728515625,
+      "loss": 0.1371,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.4459283351898193,
+      "rewards/margins": 5.340741157531738,
+      "rewards/rejected": -6.7866692543029785,
+      "step": 100
+    },
+    {
+      "epoch": 1.9,
+      "eval_logits/chosen": 102.37699127197266,
+      "eval_logits/rejected": 96.37848663330078,
+      "eval_logps/chosen": -483.2002868652344,
+      "eval_logps/rejected": -502.7766418457031,
+      "eval_loss": 0.5052046775817871,
+      "eval_rewards/accuracies": 0.7708333134651184,
+      "eval_rewards/chosen": -2.9795796871185303,
+      "eval_rewards/margins": 2.143129587173462,
+      "eval_rewards/rejected": -5.122709274291992,
+      "eval_runtime": 48.3488,
+      "eval_samples_per_second": 15.512,
+      "eval_steps_per_second": 0.496,
+      "step": 100
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 49.58918461406247,
+      "learning_rate": 1.2177518064852348e-07,
+      "logits/chosen": 113.58479309082031,
+      "logits/rejected": 127.19537353515625,
+      "logps/chosen": -511.4662170410156,
+      "logps/rejected": -620.75537109375,
+      "loss": 0.0929,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -1.6378005743026733,
+      "rewards/margins": 5.002486228942871,
+      "rewards/rejected": -6.640286922454834,
+      "step": 110
+    },
+    {
+      "epoch": 2.27,
+      "grad_norm": 30.33881666826054,
+      "learning_rate": 7.723433775328384e-08,
+      "logits/chosen": 124.79753112792969,
+      "logits/rejected": 125.82017517089844,
+      "logps/chosen": -493.6553649902344,
+      "logps/rejected": -574.6585693359375,
+      "loss": 0.053,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -1.6378300189971924,
+      "rewards/margins": 5.504368305206299,
+      "rewards/rejected": -7.142198085784912,
+      "step": 120
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 23.06668849787155,
+      "learning_rate": 4.1356686569674335e-08,
+      "logits/chosen": 129.1642608642578,
+      "logits/rejected": 134.2584228515625,
+      "logps/chosen": -508.4425354003906,
+      "logps/rejected": -594.6097412109375,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -1.4683927297592163,
+      "rewards/margins": 5.420845985412598,
+      "rewards/rejected": -6.8892388343811035,
+      "step": 130
+    },
+    {
+      "epoch": 2.65,
+      "grad_norm": 38.07392153670512,
+      "learning_rate": 1.5941282340065697e-08,
+      "logits/chosen": 114.87281799316406,
+      "logits/rejected": 115.1664047241211,
+      "logps/chosen": -465.3794860839844,
+      "logps/rejected": -584.0101318359375,
+      "loss": 0.0538,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9863094091415405,
+      "rewards/margins": 5.705286979675293,
+      "rewards/rejected": -7.691596984863281,
+      "step": 140
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 46.62238024751895,
+      "learning_rate": 2.2625595580163247e-09,
+      "logits/chosen": 121.98884582519531,
+      "logits/rejected": 135.7744598388672,
+      "logps/chosen": -492.438720703125,
+      "logps/rejected": -580.8795166015625,
+      "loss": 0.0589,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -1.7609163522720337,
+      "rewards/margins": 5.512405872344971,
+      "rewards/rejected": -7.273321628570557,
+      "step": 150
+    },
+    {
+      "epoch": 2.96,
+      "step": 156,
+      "total_flos": 0.0,
+      "train_loss": 0.27917395799587935,
+      "train_runtime": 1799.8719,
+      "train_samples_per_second": 11.251,
+      "train_steps_per_second": 0.087
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 156,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}