End of training

README.md

@@ -3,9 +3,10 @@ base_model: google/gemma-2-9b-it
 library_name: peft
 license: gemma
 tags:
+- llama-factory
+- lora
 - trl
 - dpo
-- llama-factory
 - generated_from_trainer
 model-index:
 - name: Gemma-2-9B-It-SAA-Half
@@ -17,7 +18,19 @@ should probably proofread and complete it, then remove this comment. -->
 
 # Gemma-2-9B-It-SAA-Half
 
-This model is a fine-tuned version of [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) on an unknown dataset.
+This model is a fine-tuned version of [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) on the bct_non_cot_dpo_500 dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.1383
+- Rewards/chosen: -0.1098
+- Rewards/rejected: -0.1934
+- Rewards/accuracies: 0.8400
+- Rewards/margins: 0.0836
+- Logps/rejected: -1.9339
+- Logps/chosen: -1.0982
+- Logits/rejected: 0.3316
+- Logits/chosen: 0.1306
+- Sft Loss: 0.1185
+- Odds Ratio Loss: 10.1978
 
 ## Model description
 

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 2.986666666666667,
+    "eval_logits/chosen": 0.13058120012283325,
+    "eval_logits/rejected": 0.3316400647163391,
+    "eval_logps/chosen": -1.0981853008270264,
+    "eval_logps/rejected": -1.93387770652771,
+    "eval_loss": 1.138285517692566,
+    "eval_odds_ratio_loss": 10.197811126708984,
+    "eval_rewards/accuracies": 0.8399999737739563,
+    "eval_rewards/chosen": -0.10981854796409607,
+    "eval_rewards/margins": 0.0835692435503006,
+    "eval_rewards/rejected": -0.19338779151439667,
+    "eval_runtime": 3.5214,
+    "eval_samples_per_second": 14.199,
+    "eval_sft_loss": 0.11850442737340927,
+    "eval_steps_per_second": 7.099,
+    "total_flos": 1.79183583387648e+16,
+    "train_loss": 1.8150039003008889,
+    "train_runtime": 321.7396,
+    "train_samples_per_second": 4.196,
+    "train_steps_per_second": 0.261
+}

eval_results.json

@@ -0,0 +1,17 @@
+{
+    "epoch": 2.986666666666667,
+    "eval_logits/chosen": 0.13058120012283325,
+    "eval_logits/rejected": 0.3316400647163391,
+    "eval_logps/chosen": -1.0981853008270264,
+    "eval_logps/rejected": -1.93387770652771,
+    "eval_loss": 1.138285517692566,
+    "eval_odds_ratio_loss": 10.197811126708984,
+    "eval_rewards/accuracies": 0.8399999737739563,
+    "eval_rewards/chosen": -0.10981854796409607,
+    "eval_rewards/margins": 0.0835692435503006,
+    "eval_rewards/rejected": -0.19338779151439667,
+    "eval_runtime": 3.5214,
+    "eval_samples_per_second": 14.199,
+    "eval_sft_loss": 0.11850442737340927,
+    "eval_steps_per_second": 7.099
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.986666666666667,
+    "total_flos": 1.79183583387648e+16,
+    "train_loss": 1.8150039003008889,
+    "train_runtime": 321.7396,
+    "train_samples_per_second": 4.196,
+    "train_steps_per_second": 0.261
+}

trainer_state.json

@@ -0,0 +1,178 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.986666666666667,
+  "eval_steps": 500,
+  "global_step": 84,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.35555555555555557,
+      "grad_norm": 8.401800155639648,
+      "learning_rate": 4.997807075247147e-06,
+      "logits/chosen": -0.8945320248603821,
+      "logits/rejected": -0.5788193345069885,
+      "logps/chosen": -2.9349863529205322,
+      "logps/rejected": -3.64762806892395,
+      "loss": 2.984,
+      "odds_ratio_loss": 25.994487762451172,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.2934986352920532,
+      "rewards/margins": 0.07126419246196747,
+      "rewards/rejected": -0.3647628426551819,
+      "sft_loss": 0.38450396060943604,
+      "step": 10
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "grad_norm": 8.160788536071777,
+      "learning_rate": 4.7392794005985324e-06,
+      "logits/chosen": -0.042264025658369064,
+      "logits/rejected": 0.3497065603733063,
+      "logps/chosen": -2.580463171005249,
+      "logps/rejected": -3.380384922027588,
+      "loss": 2.6249,
+      "odds_ratio_loss": 22.97456932067871,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.25804632902145386,
+      "rewards/margins": 0.07999216020107269,
+      "rewards/rejected": -0.33803847432136536,
+      "sft_loss": 0.327446311712265,
+      "step": 20
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 6.696309566497803,
+      "learning_rate": 4.093559974371725e-06,
+      "logits/chosen": 0.13843640685081482,
+      "logits/rejected": 0.34011250734329224,
+      "logps/chosen": -2.284363031387329,
+      "logps/rejected": -3.1281402111053467,
+      "loss": 2.3288,
+      "odds_ratio_loss": 20.365238189697266,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.22843630611896515,
+      "rewards/margins": 0.084377720952034,
+      "rewards/rejected": -0.31281405687332153,
+      "sft_loss": 0.2922500967979431,
+      "step": 30
+    },
+    {
+      "epoch": 1.4222222222222223,
+      "grad_norm": 6.540295600891113,
+      "learning_rate": 3.1722995515381644e-06,
+      "logits/chosen": -0.17821866273880005,
+      "logits/rejected": 0.14577758312225342,
+      "logps/chosen": -1.8487365245819092,
+      "logps/rejected": -2.766937494277954,
+      "loss": 1.8902,
+      "odds_ratio_loss": 16.7292537689209,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.18487365543842316,
+      "rewards/margins": 0.0918201133608818,
+      "rewards/rejected": -0.27669376134872437,
+      "sft_loss": 0.21722586452960968,
+      "step": 40
+    },
+    {
+      "epoch": 1.7777777777777777,
+      "grad_norm": 6.058716297149658,
+      "learning_rate": 2.134792428593971e-06,
+      "logits/chosen": -0.18059103190898895,
+      "logits/rejected": 0.23373714089393616,
+      "logps/chosen": -1.3791837692260742,
+      "logps/rejected": -2.2727575302124023,
+      "loss": 1.4255,
+      "odds_ratio_loss": 12.77796459197998,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.1379183828830719,
+      "rewards/margins": 0.08935736119747162,
+      "rewards/rejected": -0.2272757589817047,
+      "sft_loss": 0.14766769111156464,
+      "step": 50
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 5.3382248878479,
+      "learning_rate": 1.160433012552508e-06,
+      "logits/chosen": 0.018472682684659958,
+      "logits/rejected": 0.20830214023590088,
+      "logps/chosen": -1.2152187824249268,
+      "logps/rejected": -2.189805269241333,
+      "loss": 1.2625,
+      "odds_ratio_loss": 11.279905319213867,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.12152187526226044,
+      "rewards/margins": 0.09745865315198898,
+      "rewards/rejected": -0.21898052096366882,
+      "sft_loss": 0.13448992371559143,
+      "step": 60
+    },
+    {
+      "epoch": 2.488888888888889,
+      "grad_norm": 4.452148914337158,
+      "learning_rate": 4.1769689822475147e-07,
+      "logits/chosen": -0.5339587330818176,
+      "logits/rejected": -0.18176746368408203,
+      "logps/chosen": -1.101285457611084,
+      "logps/rejected": -2.074463367462158,
+      "loss": 1.1431,
+      "odds_ratio_loss": 10.337614059448242,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.11012855917215347,
+      "rewards/margins": 0.09731777757406235,
+      "rewards/rejected": -0.20744633674621582,
+      "sft_loss": 0.10934151709079742,
+      "step": 70
+    },
+    {
+      "epoch": 2.8444444444444446,
+      "grad_norm": 5.778459072113037,
+      "learning_rate": 3.5009907323737826e-08,
+      "logits/chosen": -0.40168875455856323,
+      "logits/rejected": -0.02362009882926941,
+      "logps/chosen": -1.0387601852416992,
+      "logps/rejected": -2.0947299003601074,
+      "loss": 1.0746,
+      "odds_ratio_loss": 9.696004867553711,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.1038760170340538,
+      "rewards/margins": 0.10559698194265366,
+      "rewards/rejected": -0.20947298407554626,
+      "sft_loss": 0.10503064095973969,
+      "step": 80
+    },
+    {
+      "epoch": 2.986666666666667,
+      "step": 84,
+      "total_flos": 1.79183583387648e+16,
+      "train_loss": 1.8150039003008889,
+      "train_runtime": 321.7396,
+      "train_samples_per_second": 4.196,
+      "train_steps_per_second": 0.261
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 84,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.79183583387648e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}