Model save

README.md

@@ -2,16 +2,9 @@
 license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- alignment-handbook
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 base_model: mistralai/Mistral-7B-v0.1
 model-index:
 - name: zephyr-7b-dpo-qlora-fsdp
@@ -23,7 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-qlora-fsdp
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-qlora](https://huggingface.co/alignment-handbook/zephyr-7b-sft-qlora) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6865
+- Rewards/chosen: 0.0331
+- Rewards/rejected: 0.0188
+- Rewards/accuracies: 0.5935
+- Rewards/margins: 0.0143
+- Logps/rejected: -257.1393
+- Logps/chosen: -276.4896
+- Logits/rejected: -2.3640
+- Logits/chosen: -2.4104
 
 ## Model description
 
@@ -43,16 +46,20 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-06
-- train_batch_size: 14
+- train_batch_size: 5
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 56
+- total_train_batch_size: 20
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 10
+- num_epochs: 0.01
+
+### Training results
+
+
 
 ### Framework versions
 

adapter_config.json

@@ -19,12 +19,12 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "v_proj",
-    "q_proj",
-    "o_proj",
     "gate_proj",
     "down_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "q_proj",
     "k_proj"
   ],
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d11c467ea2ced3ece27c4663979d7c2e938981d71182ed8d30aedc7f981b8b34
+oid sha256:1f7925055af868d3b38756325d43d85883ad8c14753b88f93c708d2886161f80
 size 671150064

all_results.json

@@ -1,21 +1,21 @@
 {
-    "epoch": 4.99,
-    "eval_logits/chosen": -2.4107861518859863,
-    "eval_logits/rejected": -2.3742854595184326,
-    "eval_logps/chosen": -266.8597106933594,
-    "eval_logps/rejected": -242.3632049560547,
-    "eval_loss": 0.8742221593856812,
-    "eval_rewards/accuracies": 0.6726190447807312,
-    "eval_rewards/chosen": 0.008189404383301735,
-    "eval_rewards/margins": 0.007871923968195915,
-    "eval_rewards/rejected": 0.00031748018227517605,
-    "eval_runtime": 123.0411,
+    "epoch": 0.01,
+    "eval_logits/chosen": -2.410409688949585,
+    "eval_logits/rejected": -2.363970994949341,
+    "eval_logps/chosen": -276.4896240234375,
+    "eval_logps/rejected": -257.1393127441406,
+    "eval_loss": 0.6865259408950806,
+    "eval_rewards/accuracies": 0.593500018119812,
+    "eval_rewards/chosen": 0.03312591835856438,
+    "eval_rewards/margins": 0.014345898292958736,
+    "eval_rewards/rejected": 0.018780020996928215,
+    "eval_runtime": 615.2299,
     "eval_samples": 2000,
-    "eval_samples_per_second": 16.255,
-    "eval_steps_per_second": 0.341,
-    "train_loss": 0.164279118501603,
-    "train_runtime": 43545.4617,
+    "eval_samples_per_second": 3.251,
+    "eval_steps_per_second": 0.406,
+    "train_loss": 0.6888245363389293,
+    "train_runtime": 439.9957,
     "train_samples": 61135,
-    "train_samples_per_second": 7.02,
-    "train_steps_per_second": 0.029
+    "train_samples_per_second": 1.389,
+    "train_steps_per_second": 0.07
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 4.99,
-    "eval_logits/chosen": -2.4107861518859863,
-    "eval_logits/rejected": -2.3742854595184326,
-    "eval_logps/chosen": -266.8597106933594,
-    "eval_logps/rejected": -242.3632049560547,
-    "eval_loss": 0.8742221593856812,
-    "eval_rewards/accuracies": 0.6726190447807312,
-    "eval_rewards/chosen": 0.008189404383301735,
-    "eval_rewards/margins": 0.007871923968195915,
-    "eval_rewards/rejected": 0.00031748018227517605,
-    "eval_runtime": 123.0411,
+    "epoch": 0.01,
+    "eval_logits/chosen": -2.410409688949585,
+    "eval_logits/rejected": -2.363970994949341,
+    "eval_logps/chosen": -276.4896240234375,
+    "eval_logps/rejected": -257.1393127441406,
+    "eval_loss": 0.6865259408950806,
+    "eval_rewards/accuracies": 0.593500018119812,
+    "eval_rewards/chosen": 0.03312591835856438,
+    "eval_rewards/margins": 0.014345898292958736,
+    "eval_rewards/rejected": 0.018780020996928215,
+    "eval_runtime": 615.2299,
     "eval_samples": 2000,
-    "eval_samples_per_second": 16.255,
-    "eval_steps_per_second": 0.341
+    "eval_samples_per_second": 3.251,
+    "eval_steps_per_second": 0.406
 }

runs/Mar08_12-18-16_cccxc542/events.out.tfevents.1709918369.cccxc542.189796.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5549be835e11c00b340a35080e1210a2db92e5f9b1b1cde78d382f6a17fbd24
+size 6153

runs/Mar08_12-24-55_cccxc542/events.out.tfevents.1709918735.cccxc542.206115.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39a6f70c0b3c32891fcedfdb147a75fb2a64a7f48f2a10fbed5eb5358af4eb0d
+size 7856

runs/Mar08_12-24-55_cccxc542/events.out.tfevents.1709919790.cccxc542.206115.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:688ec8b9ca686285a75a2c1a6b95f9810f5c451d30dff932e8748ffa77413659
+size 815

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 4.99,
-    "train_loss": 0.164279118501603,
-    "train_runtime": 43545.4617,
+    "epoch": 0.01,
+    "train_loss": 0.6888245363389293,
+    "train_runtime": 439.9957,
     "train_samples": 61135,
-    "train_samples_per_second": 7.02,
-    "train_steps_per_second": 0.029
+    "train_samples_per_second": 1.389,
+    "train_steps_per_second": 0.07
 }

trainer_state.json

@@ -1,20 +1,90 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": null,
-  "eval_steps": 500,
-  "global_step": 0,
+  "epoch": 0.010141490144761593,
+  "eval_steps": 100,
+  "global_step": 31,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
-  "log_history": [],
-  "logging_steps": 500,
-  "max_steps": 0,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.25e-06,
+      "logits/chosen": -2.3689165115356445,
+      "logits/rejected": -2.3419089317321777,
+      "logps/chosen": -304.96429443359375,
+      "logps/rejected": -224.31954956054688,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 1.9296875,
+      "learning_rate": 4.415111107797445e-06,
+      "logits/chosen": -2.3774471282958984,
+      "logits/rejected": -2.358837127685547,
+      "logps/chosen": -267.6408386230469,
+      "logps/rejected": -221.9726104736328,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.5166666507720947,
+      "rewards/chosen": 0.008927525021135807,
+      "rewards/margins": 0.002250629710033536,
+      "rewards/rejected": 0.006676895078271627,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.7829919182222752e-06,
+      "logits/chosen": -2.4560706615448,
+      "logits/rejected": -2.402303695678711,
+      "logps/chosen": -265.12762451171875,
+      "logps/rejected": -272.61566162109375,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.5900000333786011,
+      "rewards/chosen": 0.028245043009519577,
+      "rewards/margins": 0.009032377041876316,
+      "rewards/rejected": 0.019212666898965836,
+      "step": 20
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 2.375,
+      "learning_rate": 1.6904105645142443e-08,
+      "logits/chosen": -2.3814165592193604,
+      "logits/rejected": -2.3470723628997803,
+      "logps/chosen": -304.08697509765625,
+      "logps/rejected": -281.0203552246094,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.6299999952316284,
+      "rewards/chosen": 0.03472686558961868,
+      "rewards/margins": 0.019056813791394234,
+      "rewards/rejected": 0.015670055523514748,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "step": 31,
+      "total_flos": 0.0,
+      "train_loss": 0.6888245363389293,
+      "train_runtime": 439.9957,
+      "train_samples_per_second": 1.389,
+      "train_steps_per_second": 0.07
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 31,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 0,
-  "save_steps": 500,
-  "total_flos": 0,
-  "train_batch_size": null,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 5,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:92fee4626f14c7e9e46522e5b22431404ce3eaa07753dde47b190bdd90f9420d
+oid sha256:f21d618e5d6b3867a78c9fe4f30b09ded8634419a2c3a54b84735333c2487436
 size 5112