Model save

README.md

@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6366
-- Rewards/chosen: 0.0331
-- Rewards/rejected: -0.0356
-- Rewards/accuracies: 0.6320
-- Rewards/margins: 0.0687
-- Logps/rejected: -250.3080
-- Logps/chosen: -272.9035
-- Logits/rejected: -2.5200
-- Logits/chosen: -2.5064
-- Use Label: 9174.8564
-- Pred Label: 6857.1440
+- Loss: 0.6354
+- Rewards/chosen: 0.0271
+- Rewards/rejected: -0.0297
+- Rewards/accuracies: 0.6260
+- Rewards/margins: 0.0568
+- Logps/rejected: -253.2359
+- Logps/chosen: -269.2855
+- Logits/rejected: -2.4958
+- Logits/chosen: -2.4939
+- Use Label: 18265.6758
+- Pred Label: 13796.3242
 
 ## Model description
 
@@ -45,14 +45,14 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 4
+- train_batch_size: 2
 - eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 4
-- gradient_accumulation_steps: 4
+- num_devices: 2
+- gradient_accumulation_steps: 16
 - total_train_batch_size: 64
-- total_eval_batch_size: 16
+- total_eval_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.1
@@ -60,9 +60,9 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label | Pred Label |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:---------:|:----------:|
-| 0.6393        | 1.0   | 955  | 0.6366          | 0.0331         | -0.0356          | 0.6320             | 0.0687          | -250.3080      | -272.9035    | -2.5200         | -2.5064       | 8966.8564 | 6565.1440  |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label  | Pred Label |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:----------:|:----------:|
+| 0.6377        | 1.0   | 955  | 0.6354          | 0.0271         | -0.0297          | 0.6260             | 0.0568          | -253.2359      | -269.2855    | -2.4958         | -2.4939       | 17827.6758 | 13234.3242 |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:687f28b2c569851076b1156cac98b07b2e15e82c56032251b902e8d7a7a2eb57
+oid sha256:93eff2aaaece374224573b008ce1ce472e09be6ef9c4a68d282729c77e326912
 size 109086672

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.5063796043395996,
-    "eval_logits/rejected": -2.5199685096740723,
-    "eval_logps/chosen": -272.9034729003906,
-    "eval_logps/rejected": -250.30796813964844,
-    "eval_loss": 0.6366299986839294,
-    "eval_pred_label": 6857.14404296875,
-    "eval_rewards/accuracies": 0.6320000290870667,
-    "eval_rewards/chosen": 0.03307868540287018,
-    "eval_rewards/margins": 0.06870328634977341,
-    "eval_rewards/rejected": -0.03562460467219353,
-    "eval_runtime": 441.6525,
+    "eval_logits/chosen": -2.4939169883728027,
+    "eval_logits/rejected": -2.495774507522583,
+    "eval_logps/chosen": -269.28546142578125,
+    "eval_logps/rejected": -253.23594665527344,
+    "eval_loss": 0.6354129910469055,
+    "eval_pred_label": 13796.32421875,
+    "eval_rewards/accuracies": 0.6259999871253967,
+    "eval_rewards/chosen": 0.027118388563394547,
+    "eval_rewards/margins": 0.056793875992298126,
+    "eval_rewards/rejected": -0.029675481840968132,
+    "eval_runtime": 1013.6049,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.528,
-    "eval_steps_per_second": 0.283,
-    "eval_use_label": 9174.8564453125,
-    "train_loss": 0.6567496789063458,
-    "train_runtime": 24439.922,
+    "eval_samples_per_second": 1.973,
+    "eval_steps_per_second": 0.247,
+    "eval_use_label": 18265.67578125,
+    "train_loss": 0.6554346030919339,
+    "train_runtime": 50166.5495,
     "train_samples": 61135,
-    "train_samples_per_second": 2.501,
-    "train_steps_per_second": 0.039
+    "train_samples_per_second": 1.219,
+    "train_steps_per_second": 0.019
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.5063796043395996,
-    "eval_logits/rejected": -2.5199685096740723,
-    "eval_logps/chosen": -272.9034729003906,
-    "eval_logps/rejected": -250.30796813964844,
-    "eval_loss": 0.6366299986839294,
-    "eval_pred_label": 6857.14404296875,
-    "eval_rewards/accuracies": 0.6320000290870667,
-    "eval_rewards/chosen": 0.03307868540287018,
-    "eval_rewards/margins": 0.06870328634977341,
-    "eval_rewards/rejected": -0.03562460467219353,
-    "eval_runtime": 441.6525,
+    "eval_logits/chosen": -2.4939169883728027,
+    "eval_logits/rejected": -2.495774507522583,
+    "eval_logps/chosen": -269.28546142578125,
+    "eval_logps/rejected": -253.23594665527344,
+    "eval_loss": 0.6354129910469055,
+    "eval_pred_label": 13796.32421875,
+    "eval_rewards/accuracies": 0.6259999871253967,
+    "eval_rewards/chosen": 0.027118388563394547,
+    "eval_rewards/margins": 0.056793875992298126,
+    "eval_rewards/rejected": -0.029675481840968132,
+    "eval_runtime": 1013.6049,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.528,
-    "eval_steps_per_second": 0.283,
-    "eval_use_label": 9174.8564453125
+    "eval_samples_per_second": 1.973,
+    "eval_steps_per_second": 0.247,
+    "eval_use_label": 18265.67578125
 }

runs/Jan17_10-24-46_uclaml03.cs.ucla.edu/events.out.tfevents.1705516166.uclaml03.cs.ucla.edu.3614264.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2cdcea9253154718e626c44be407e4ceee6445a33899a4484e6c8c20846bb33
-size 11768
+oid sha256:2736f15944d801784c31bc0b45a9d5a501f9144d3a985e1feac4fd8716584c23
+size 12969

runs/Jan17_10-24-46_uclaml03.cs.ucla.edu/events.out.tfevents.1705567345.uclaml03.cs.ucla.edu.3614264.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eae34ab475ac4656727c96107ce229d954f9a60d4530ec3a4557a0a514c79326
+size 935

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6567496789063458,
-    "train_runtime": 24439.922,
+    "train_loss": 0.6554346030919339,
+    "train_runtime": 50166.5495,
     "train_samples": 61135,
-    "train_samples_per_second": 2.501,
-    "train_steps_per_second": 0.039
+    "train_samples_per_second": 1.219,
+    "train_steps_per_second": 0.019
 }

trainer_state.json

@@ -11,10 +11,10 @@
     {
       "epoch": 0.0,
       "learning_rate": 5.208333333333333e-09,
-      "logits/chosen": -2.676934003829956,
-      "logits/rejected": -2.509021043777466,
-      "logps/chosen": -304.709228515625,
-      "logps/rejected": -229.49505615234375,
+      "logits/chosen": -2.7525930404663086,
+      "logits/rejected": -2.6732418537139893,
+      "logps/chosen": -297.177001953125,
+      "logps/rejected": -236.72621154785156,
       "loss": 0.6931,
       "pred_label": 0.0,
       "rewards/accuracies": 0.0,
@@ -22,184 +22,184 @@
       "rewards/margins": 0.0,
       "rewards/rejected": 0.0,
       "step": 1,
-      "use_label": 10.0
+      "use_label": 17.0
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.976717112922002e-07,
-      "logits/chosen": -2.4758388996124268,
-      "logits/rejected": -2.4836206436157227,
-      "logps/chosen": -273.62322998046875,
-      "logps/rejected": -258.89813232421875,
-      "loss": 0.6823,
-      "pred_label": 156.05050659179688,
-      "rewards/accuracies": 0.4886363744735718,
-      "rewards/chosen": -0.002133187372237444,
-      "rewards/margins": -0.0010164172854274511,
-      "rewards/rejected": -0.0011167696211487055,
+      "logits/chosen": -2.6616106033325195,
+      "logits/rejected": -2.6597719192504883,
+      "logps/chosen": -270.4000244140625,
+      "logps/rejected": -249.33827209472656,
+      "loss": 0.6829,
+      "pred_label": 333.43182373046875,
+      "rewards/accuracies": 0.4965277910232544,
+      "rewards/chosen": 0.0011782451765611768,
+      "rewards/margins": 0.001073930412530899,
+      "rewards/rejected": 0.00010431456030346453,
       "step": 100,
-      "use_label": 653.9495239257812
+      "use_label": 1283.5682373046875
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.3946449359720607e-07,
-      "logits/chosen": -2.4878945350646973,
-      "logits/rejected": -2.4750781059265137,
-      "logps/chosen": -267.72540283203125,
-      "logps/rejected": -250.30291748046875,
-      "loss": 0.6801,
-      "pred_label": 512.7750244140625,
-      "rewards/accuracies": 0.5575000047683716,
-      "rewards/chosen": 0.003957623615860939,
-      "rewards/margins": 0.008218127302825451,
-      "rewards/rejected": -0.0042605032213032246,
+      "logits/chosen": -2.6945221424102783,
+      "logits/rejected": -2.678621530532837,
+      "logps/chosen": -271.6979064941406,
+      "logps/rejected": -254.37026977539062,
+      "loss": 0.6799,
+      "pred_label": 1038.7462158203125,
+      "rewards/accuracies": 0.5350000262260437,
+      "rewards/chosen": 0.004888341296464205,
+      "rewards/margins": 0.007898561656475067,
+      "rewards/rejected": -0.0030102210585027933,
       "step": 200,
-      "use_label": 1889.2249755859375
+      "use_label": 3762.253662109375
     },
     {
       "epoch": 0.31,
       "learning_rate": 3.812572759022118e-07,
-      "logits/chosen": -2.47322940826416,
-      "logits/rejected": -2.4768893718719482,
-      "logps/chosen": -272.0399169921875,
-      "logps/rejected": -244.90228271484375,
-      "loss": 0.6733,
-      "pred_label": 943.1199951171875,
-      "rewards/accuracies": 0.6075000166893005,
-      "rewards/chosen": 0.010850328952074051,
-      "rewards/margins": 0.01955023780465126,
-      "rewards/rejected": -0.008699909783899784,
+      "logits/chosen": -2.6708526611328125,
+      "logits/rejected": -2.6628105640411377,
+      "logps/chosen": -272.3077392578125,
+      "logps/rejected": -253.75027465820312,
+      "loss": 0.6728,
+      "pred_label": 1884.596923828125,
+      "rewards/accuracies": 0.5653125047683716,
+      "rewards/chosen": 0.010109632275998592,
+      "rewards/margins": 0.016557401046156883,
+      "rewards/rejected": -0.006447767838835716,
       "step": 300,
-      "use_label": 3058.8798828125
+      "use_label": 6116.4033203125
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.230500582072177e-07,
-      "logits/chosen": -2.486250400543213,
-      "logits/rejected": -2.466102361679077,
-      "logps/chosen": -270.7090759277344,
-      "logps/rejected": -252.9911346435547,
-      "loss": 0.6635,
-      "pred_label": 1496.31005859375,
-      "rewards/accuracies": 0.628125011920929,
-      "rewards/chosen": 0.01578013226389885,
-      "rewards/margins": 0.0331539586186409,
-      "rewards/rejected": -0.01737382635474205,
+      "logits/chosen": -2.668009042739868,
+      "logits/rejected": -2.650494337081909,
+      "logps/chosen": -267.6447448730469,
+      "logps/rejected": -253.59107971191406,
+      "loss": 0.6616,
+      "pred_label": 3012.675537109375,
+      "rewards/accuracies": 0.6193749904632568,
+      "rewards/chosen": 0.017754318192601204,
+      "rewards/margins": 0.030351871624588966,
+      "rewards/rejected": -0.012597555294632912,
       "step": 400,
-      "use_label": 4105.68994140625
+      "use_label": 8188.32421875
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.648428405122235e-07,
-      "logits/chosen": -2.46098256111145,
-      "logits/rejected": -2.446549654006958,
-      "logps/chosen": -259.1031799316406,
-      "logps/rejected": -241.98345947265625,
-      "loss": 0.6543,
-      "pred_label": 2180.89990234375,
-      "rewards/accuracies": 0.6175000071525574,
-      "rewards/chosen": 0.019046209752559662,
-      "rewards/margins": 0.03530467674136162,
-      "rewards/rejected": -0.016258466988801956,
+      "logits/chosen": -2.6697680950164795,
+      "logits/rejected": -2.6707708835601807,
+      "logps/chosen": -271.2095642089844,
+      "logps/rejected": -247.21224975585938,
+      "loss": 0.6528,
+      "pred_label": 4377.916015625,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.024391591548919678,
+      "rewards/margins": 0.04303843528032303,
+      "rewards/rejected": -0.01864684373140335,
       "step": 500,
-      "use_label": 5021.10009765625
+      "use_label": 10023.083984375
     },
     {
       "epoch": 0.63,
       "learning_rate": 2.0663562281722933e-07,
-      "logits/chosen": -2.4896275997161865,
-      "logits/rejected": -2.4663710594177246,
-      "logps/chosen": -271.8394470214844,
-      "logps/rejected": -252.83351135253906,
-      "loss": 0.646,
-      "pred_label": 2942.87255859375,
-      "rewards/accuracies": 0.6600000262260437,
-      "rewards/chosen": 0.02863229252398014,
-      "rewards/margins": 0.05473264306783676,
-      "rewards/rejected": -0.02610035613179207,
+      "logits/chosen": -2.659043073654175,
+      "logits/rejected": -2.6555004119873047,
+      "logps/chosen": -272.95050048828125,
+      "logps/rejected": -251.1392364501953,
+      "loss": 0.6442,
+      "pred_label": 5962.0673828125,
+      "rewards/accuracies": 0.6553124785423279,
+      "rewards/chosen": 0.030743848532438278,
+      "rewards/margins": 0.0554736964404583,
+      "rewards/rejected": -0.024729840457439423,
       "step": 600,
-      "use_label": 5859.12744140625
+      "use_label": 11638.9326171875
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.4842840512223514e-07,
-      "logits/chosen": -2.485153913497925,
-      "logits/rejected": -2.472170352935791,
-      "logps/chosen": -269.14117431640625,
-      "logps/rejected": -238.4977569580078,
-      "loss": 0.6421,
-      "pred_label": 3756.925048828125,
-      "rewards/accuracies": 0.6418750286102295,
-      "rewards/chosen": 0.03320219740271568,
-      "rewards/margins": 0.05917687341570854,
-      "rewards/rejected": -0.02597467601299286,
+      "logits/chosen": -2.6591668128967285,
+      "logits/rejected": -2.6622869968414307,
+      "logps/chosen": -269.9889221191406,
+      "logps/rejected": -245.4040985107422,
+      "loss": 0.64,
+      "pred_label": 7640.8505859375,
+      "rewards/accuracies": 0.6478124856948853,
+      "rewards/chosen": 0.03263993561267853,
+      "rewards/margins": 0.061180587857961655,
+      "rewards/rejected": -0.02854064851999283,
       "step": 700,
-      "use_label": 6645.0751953125
+      "use_label": 13160.150390625
     },
     {
       "epoch": 0.84,
       "learning_rate": 9.022118742724097e-08,
-      "logits/chosen": -2.4880199432373047,
-      "logits/rejected": -2.5014524459838867,
-      "logps/chosen": -276.52313232421875,
-      "logps/rejected": -252.34681701660156,
-      "loss": 0.6386,
-      "pred_label": 4625.42236328125,
-      "rewards/accuracies": 0.640625,
-      "rewards/chosen": 0.03754829242825508,
-      "rewards/margins": 0.06682833284139633,
-      "rewards/rejected": -0.02928004413843155,
+      "logits/chosen": -2.650268793106079,
+      "logits/rejected": -2.6555473804473877,
+      "logps/chosen": -272.705322265625,
+      "logps/rejected": -252.30169677734375,
+      "loss": 0.6368,
+      "pred_label": 9366.9609375,
+      "rewards/accuracies": 0.6415625214576721,
+      "rewards/chosen": 0.031398553401231766,
+      "rewards/margins": 0.06083739921450615,
+      "rewards/rejected": -0.029438842087984085,
       "step": 800,
-      "use_label": 7376.57763671875
+      "use_label": 14634.0390625
     },
     {
       "epoch": 0.94,
       "learning_rate": 3.20139697322468e-08,
-      "logits/chosen": -2.4699513912200928,
-      "logits/rejected": -2.485213279724121,
-      "logps/chosen": -266.04229736328125,
-      "logps/rejected": -247.93341064453125,
-      "loss": 0.6393,
-      "pred_label": 5488.7001953125,
-      "rewards/accuracies": 0.6587499976158142,
-      "rewards/chosen": 0.03272656351327896,
-      "rewards/margins": 0.06201673671603203,
-      "rewards/rejected": -0.02929016388952732,
+      "logits/chosen": -2.6563680171966553,
+      "logits/rejected": -2.6590001583099365,
+      "logps/chosen": -269.04559326171875,
+      "logps/rejected": -253.2301025390625,
+      "loss": 0.6377,
+      "pred_label": 11126.677734375,
+      "rewards/accuracies": 0.6418750286102295,
+      "rewards/chosen": 0.02964354306459427,
+      "rewards/margins": 0.05687180534005165,
+      "rewards/rejected": -0.027228260412812233,
       "step": 900,
-      "use_label": 8113.2998046875
+      "use_label": 16074.322265625
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -2.5063796043395996,
-      "eval_logits/rejected": -2.5199685096740723,
-      "eval_logps/chosen": -272.9034729003906,
-      "eval_logps/rejected": -250.30796813964844,
-      "eval_loss": 0.6366299986839294,
-      "eval_pred_label": 6565.14404296875,
-      "eval_rewards/accuracies": 0.6320000290870667,
-      "eval_rewards/chosen": 0.03307868540287018,
-      "eval_rewards/margins": 0.06870328634977341,
-      "eval_rewards/rejected": -0.03562460467219353,
-      "eval_runtime": 443.034,
-      "eval_samples_per_second": 4.514,
-      "eval_steps_per_second": 0.282,
-      "eval_use_label": 8966.8564453125,
+      "eval_logits/chosen": -2.4939169883728027,
+      "eval_logits/rejected": -2.495774507522583,
+      "eval_logps/chosen": -269.28546142578125,
+      "eval_logps/rejected": -253.23594665527344,
+      "eval_loss": 0.6354129910469055,
+      "eval_pred_label": 13234.32421875,
+      "eval_rewards/accuracies": 0.6259999871253967,
+      "eval_rewards/chosen": 0.027118388563394547,
+      "eval_rewards/margins": 0.056793875992298126,
+      "eval_rewards/rejected": -0.029675481840968132,
+      "eval_runtime": 1016.337,
+      "eval_samples_per_second": 1.968,
+      "eval_steps_per_second": 0.246,
+      "eval_use_label": 17827.67578125,
       "step": 955
     },
     {
       "epoch": 1.0,
       "step": 955,
       "total_flos": 0.0,
-      "train_loss": 0.6567496789063458,
-      "train_runtime": 24439.922,
-      "train_samples_per_second": 2.501,
-      "train_steps_per_second": 0.039
+      "train_loss": 0.6554346030919339,
+      "train_runtime": 50166.5495,
+      "train_samples_per_second": 1.219,
+      "train_steps_per_second": 0.019
     }
   ],
   "logging_steps": 100,
   "max_steps": 955,
   "num_train_epochs": 1,
-  "save_steps": 500,
+  "save_steps": 10,
   "total_flos": 0.0,
   "trial_name": null,
   "trial_params": null