End of training

README.md

@@ -3,9 +3,10 @@ base_model: meta-llama/Llama-3.1-8B-Instruct
 library_name: peft
 license: llama3.1
 tags:
+- llama-factory
+- lora
 - trl
 - dpo
-- llama-factory
 - generated_from_trainer
 model-index:
 - name: Llama-3.1-8B-Instruct-SAA
@@ -17,7 +18,19 @@ should probably proofread and complete it, then remove this comment. -->
 
 # Llama-3.1-8B-Instruct-SAA
 
-This model is a fine-tuned version of [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) on an unknown dataset.
+This model is a fine-tuned version of [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) on the bct_non_cot_dpo_1000 dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.2005
+- Rewards/chosen: -0.0159
+- Rewards/rejected: -0.0611
+- Rewards/accuracies: 0.8300
+- Rewards/margins: 0.0452
+- Logps/rejected: -0.6110
+- Logps/chosen: -0.1590
+- Logits/rejected: -0.4566
+- Logits/chosen: -0.3865
+- Sft Loss: 0.0178
+- Odds Ratio Loss: 1.8274
 
 ## Model description
 

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 2.986666666666667,
+    "eval_logits/chosen": -0.3865179717540741,
+    "eval_logits/rejected": -0.45662835240364075,
+    "eval_logps/chosen": -0.15898314118385315,
+    "eval_logps/rejected": -0.6109744310379028,
+    "eval_loss": 0.20052169263362885,
+    "eval_odds_ratio_loss": 1.8274307250976562,
+    "eval_rewards/accuracies": 0.8299999833106995,
+    "eval_rewards/chosen": -0.015898313373327255,
+    "eval_rewards/margins": 0.045199137181043625,
+    "eval_rewards/rejected": -0.06109744682908058,
+    "eval_runtime": 3.8864,
+    "eval_samples_per_second": 25.73,
+    "eval_sft_loss": 0.01777861826121807,
+    "eval_steps_per_second": 12.865,
+    "total_flos": 2.94774302048256e+16,
+    "train_loss": 0.703327341448693,
+    "train_runtime": 380.828,
+    "train_samples_per_second": 7.09,
+    "train_steps_per_second": 0.441
+}

eval_results.json

@@ -0,0 +1,17 @@
+{
+    "epoch": 2.986666666666667,
+    "eval_logits/chosen": -0.3865179717540741,
+    "eval_logits/rejected": -0.45662835240364075,
+    "eval_logps/chosen": -0.15898314118385315,
+    "eval_logps/rejected": -0.6109744310379028,
+    "eval_loss": 0.20052169263362885,
+    "eval_odds_ratio_loss": 1.8274307250976562,
+    "eval_rewards/accuracies": 0.8299999833106995,
+    "eval_rewards/chosen": -0.015898313373327255,
+    "eval_rewards/margins": 0.045199137181043625,
+    "eval_rewards/rejected": -0.06109744682908058,
+    "eval_runtime": 3.8864,
+    "eval_samples_per_second": 25.73,
+    "eval_sft_loss": 0.01777861826121807,
+    "eval_steps_per_second": 12.865
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.986666666666667,
+    "total_flos": 2.94774302048256e+16,
+    "train_loss": 0.703327341448693,
+    "train_runtime": 380.828,
+    "train_samples_per_second": 7.09,
+    "train_steps_per_second": 0.441
+}

trainer_state.json

@@ -0,0 +1,314 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.986666666666667,
+  "eval_steps": 500,
+  "global_step": 168,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.17777777777777778,
+      "grad_norm": 7.277172565460205,
+      "learning_rate": 2.9411764705882355e-06,
+      "logits/chosen": -0.4413936734199524,
+      "logits/rejected": -0.5268994569778442,
+      "logps/chosen": -1.6909767389297485,
+      "logps/rejected": -2.157106876373291,
+      "loss": 1.742,
+      "odds_ratio_loss": 15.103334426879883,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.1690976917743683,
+      "rewards/margins": 0.046613018959760666,
+      "rewards/rejected": -0.21571068465709686,
+      "sft_loss": 0.2316340208053589,
+      "step": 10
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "grad_norm": 6.993782043457031,
+      "learning_rate": 4.995131923687488e-06,
+      "logits/chosen": -0.4119906425476074,
+      "logits/rejected": -0.48687878251075745,
+      "logps/chosen": -1.74609375,
+      "logps/rejected": -2.23624849319458,
+      "loss": 1.7977,
+      "odds_ratio_loss": 15.667881965637207,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.17460939288139343,
+      "rewards/margins": 0.04901544004678726,
+      "rewards/rejected": -0.22362485527992249,
+      "sft_loss": 0.23094086349010468,
+      "step": 20
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 7.126066207885742,
+      "learning_rate": 4.90911473983908e-06,
+      "logits/chosen": -0.3993254601955414,
+      "logits/rejected": -0.47226476669311523,
+      "logps/chosen": -1.6316206455230713,
+      "logps/rejected": -2.0731987953186035,
+      "loss": 1.685,
+      "odds_ratio_loss": 14.636944770812988,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.16316203773021698,
+      "rewards/margins": 0.044157832860946655,
+      "rewards/rejected": -0.20731988549232483,
+      "sft_loss": 0.22126968204975128,
+      "step": 30
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "grad_norm": 6.876996040344238,
+      "learning_rate": 4.71919261421297e-06,
+      "logits/chosen": -0.4370671212673187,
+      "logits/rejected": -0.5137215852737427,
+      "logps/chosen": -1.3095628023147583,
+      "logps/rejected": -1.8373353481292725,
+      "loss": 1.3576,
+      "odds_ratio_loss": 11.971807479858398,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.13095629215240479,
+      "rewards/margins": 0.052777260541915894,
+      "rewards/rejected": -0.18373355269432068,
+      "sft_loss": 0.16043850779533386,
+      "step": 40
+    },
+    {
+      "epoch": 0.8888888888888888,
+      "grad_norm": 3.4186394214630127,
+      "learning_rate": 4.43355687413747e-06,
+      "logits/chosen": -0.4473581910133362,
+      "logits/rejected": -0.5121863484382629,
+      "logps/chosen": -1.047701120376587,
+      "logps/rejected": -1.479405164718628,
+      "loss": 1.0996,
+      "odds_ratio_loss": 9.729310989379883,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.10477012395858765,
+      "rewards/margins": 0.04317039996385574,
+      "rewards/rejected": -0.1479405164718628,
+      "sft_loss": 0.12663736939430237,
+      "step": 50
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 4.834543704986572,
+      "learning_rate": 4.064526968101844e-06,
+      "logits/chosen": -0.39888542890548706,
+      "logits/rejected": -0.4785118103027344,
+      "logps/chosen": -0.7870318293571472,
+      "logps/rejected": -1.2147241830825806,
+      "loss": 0.8362,
+      "odds_ratio_loss": 7.4531731605529785,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.07870318740606308,
+      "rewards/margins": 0.04276924580335617,
+      "rewards/rejected": -0.12147243320941925,
+      "sft_loss": 0.09090302884578705,
+      "step": 60
+    },
+    {
+      "epoch": 1.2444444444444445,
+      "grad_norm": 2.8773512840270996,
+      "learning_rate": 3.6280191288478437e-06,
+      "logits/chosen": -0.4217619299888611,
+      "logits/rejected": -0.49851852655410767,
+      "logps/chosen": -0.5581613779067993,
+      "logps/rejected": -1.076547384262085,
+      "loss": 0.5998,
+      "odds_ratio_loss": 5.396755695343018,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.05581614375114441,
+      "rewards/margins": 0.051838599145412445,
+      "rewards/rejected": -0.10765473544597626,
+      "sft_loss": 0.06012673303484917,
+      "step": 70
+    },
+    {
+      "epoch": 1.4222222222222223,
+      "grad_norm": 2.8275744915008545,
+      "learning_rate": 3.142859907420615e-06,
+      "logits/chosen": -0.39267396926879883,
+      "logits/rejected": -0.46761494874954224,
+      "logps/chosen": -0.43946489691734314,
+      "logps/rejected": -0.9307826161384583,
+      "loss": 0.4834,
+      "odds_ratio_loss": 4.3501482009887695,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.04394649341702461,
+      "rewards/margins": 0.049131762236356735,
+      "rewards/rejected": -0.09307824075222015,
+      "sft_loss": 0.04834695905447006,
+      "step": 80
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 2.467008590698242,
+      "learning_rate": 2.629974185404951e-06,
+      "logits/chosen": -0.3878730535507202,
+      "logits/rejected": -0.4690762460231781,
+      "logps/chosen": -0.3482344150543213,
+      "logps/rejected": -0.9180346727371216,
+      "loss": 0.3918,
+      "odds_ratio_loss": 3.5334713459014893,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03482344001531601,
+      "rewards/margins": 0.05698002502322197,
+      "rewards/rejected": -0.09180346131324768,
+      "sft_loss": 0.03849966078996658,
+      "step": 90
+    },
+    {
+      "epoch": 1.7777777777777777,
+      "grad_norm": 2.4158785343170166,
+      "learning_rate": 2.1114826863194882e-06,
+      "logits/chosen": -0.3763123154640198,
+      "logits/rejected": -0.47139158844947815,
+      "logps/chosen": -0.27210018038749695,
+      "logps/rejected": -0.8651291728019714,
+      "loss": 0.3089,
+      "odds_ratio_loss": 2.8031647205352783,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.027210021391510963,
+      "rewards/margins": 0.05930290371179581,
+      "rewards/rejected": -0.08651291579008102,
+      "sft_loss": 0.02853994630277157,
+      "step": 100
+    },
+    {
+      "epoch": 1.9555555555555557,
+      "grad_norm": 2.5559449195861816,
+      "learning_rate": 1.6097479104361328e-06,
+      "logits/chosen": -0.4010067880153656,
+      "logits/rejected": -0.4844478666782379,
+      "logps/chosen": -0.2540341019630432,
+      "logps/rejected": -0.6811193823814392,
+      "loss": 0.3004,
+      "odds_ratio_loss": 2.749537706375122,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.02540341392159462,
+      "rewards/margins": 0.04270853474736214,
+      "rewards/rejected": -0.06811194121837616,
+      "sft_loss": 0.025406217202544212,
+      "step": 110
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 2.1673660278320312,
+      "learning_rate": 1.1464096417858821e-06,
+      "logits/chosen": -0.3880887031555176,
+      "logits/rejected": -0.47383826971054077,
+      "logps/chosen": -0.17626990377902985,
+      "logps/rejected": -0.8207994699478149,
+      "loss": 0.2064,
+      "odds_ratio_loss": 1.8402626514434814,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.017626991495490074,
+      "rewards/margins": 0.06445295363664627,
+      "rewards/rejected": -0.0820799320936203,
+      "sft_loss": 0.022340241819620132,
+      "step": 120
+    },
+    {
+      "epoch": 2.311111111111111,
+      "grad_norm": 1.6694434881210327,
+      "learning_rate": 7.414516258630245e-07,
+      "logits/chosen": -0.4004104733467102,
+      "logits/rejected": -0.48559218645095825,
+      "logps/chosen": -0.18084414303302765,
+      "logps/rejected": -0.7277659177780151,
+      "loss": 0.2208,
+      "odds_ratio_loss": 1.993011474609375,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.018084412440657616,
+      "rewards/margins": 0.054692186415195465,
+      "rewards/rejected": -0.07277660071849823,
+      "sft_loss": 0.021496199071407318,
+      "step": 130
+    },
+    {
+      "epoch": 2.488888888888889,
+      "grad_norm": 2.6840009689331055,
+      "learning_rate": 4.123396721497977e-07,
+      "logits/chosen": -0.38309675455093384,
+      "logits/rejected": -0.4632611870765686,
+      "logps/chosen": -0.17778636515140533,
+      "logps/rejected": -0.7036616802215576,
+      "loss": 0.2182,
+      "odds_ratio_loss": 1.9652601480484009,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.017778638750314713,
+      "rewards/margins": 0.052587538957595825,
+      "rewards/rejected": -0.07036618143320084,
+      "sft_loss": 0.021626269444823265,
+      "step": 140
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 2.5022149085998535,
+      "learning_rate": 1.7326835503629542e-07,
+      "logits/chosen": -0.41912952065467834,
+      "logits/rejected": -0.48157674074172974,
+      "logps/chosen": -0.1846485584974289,
+      "logps/rejected": -0.6781209707260132,
+      "loss": 0.2302,
+      "odds_ratio_loss": 2.0842204093933105,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.01846485398709774,
+      "rewards/margins": 0.04934724420309067,
+      "rewards/rejected": -0.06781210005283356,
+      "sft_loss": 0.02174842543900013,
+      "step": 150
+    },
+    {
+      "epoch": 2.8444444444444446,
+      "grad_norm": 2.2835490703582764,
+      "learning_rate": 3.4548802869627806e-08,
+      "logits/chosen": -0.3882659673690796,
+      "logits/rejected": -0.46682611107826233,
+      "logps/chosen": -0.15616735816001892,
+      "logps/rejected": -0.6447629332542419,
+      "loss": 0.193,
+      "odds_ratio_loss": 1.7490888833999634,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.015616737306118011,
+      "rewards/margins": 0.04885955527424812,
+      "rewards/rejected": -0.06447629630565643,
+      "sft_loss": 0.018099095672369003,
+      "step": 160
+    },
+    {
+      "epoch": 2.986666666666667,
+      "step": 168,
+      "total_flos": 2.94774302048256e+16,
+      "train_loss": 0.703327341448693,
+      "train_runtime": 380.828,
+      "train_samples_per_second": 7.09,
+      "train_steps_per_second": 0.441
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 168,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.94774302048256e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}