Training in progress, step 10

README.md

@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6351
-- Rewards/chosen: 0.0300
-- Rewards/rejected: -0.0335
-- Rewards/accuracies: 0.6200
-- Rewards/margins: 0.0635
-- Logps/rejected: -250.2864
-- Logps/chosen: -272.9344
+- Loss: 0.6366
+- Rewards/chosen: 0.0331
+- Rewards/rejected: -0.0356
+- Rewards/accuracies: 0.6320
+- Rewards/margins: 0.0687
+- Logps/rejected: -250.3080
+- Logps/chosen: -272.9035
 - Logits/rejected: -2.5200
-- Logits/chosen: -2.5063
-- Use Label: 7566.9282
-- Pred Label: 8465.0723
+- Logits/chosen: -2.5064
+- Use Label: 9174.8564
+- Pred Label: 6857.1440
 
 ## Model description
 
@@ -62,7 +62,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label | Pred Label |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:---------:|:----------:|
-| 0.6382        | 1.0   | 955  | 0.6351          | 0.0300         | -0.0335          | 0.6200             | 0.0635          | -250.2864      | -272.9344    | -2.5200         | -2.5063       | 7400.9282 | 8131.0718  |
+| 0.6393        | 1.0   | 955  | 0.6366          | 0.0331         | -0.0356          | 0.6320             | 0.0687          | -250.3080      | -272.9035    | -2.5200         | -2.5064       | 8966.8564 | 6565.1440  |
 
 
 ### Framework versions

adapter_config.json

@@ -19,10 +19,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
-    "v_proj",
     "k_proj",
-    "o_proj"
+    "o_proj",
+    "v_proj",
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee1f96b1ff91748644141001f4b3a08b51dc4fcb31e400b5beb778684bf0c37b
+oid sha256:722f0729e0dd5f38ace74c0bf4bfba907e4ee8f4504edeaa6b00d325eb28a9e5
 size 109086672

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.5062618255615234,
-    "eval_logits/rejected": -2.5199859142303467,
-    "eval_logps/chosen": -272.93438720703125,
-    "eval_logps/rejected": -250.28643798828125,
-    "eval_loss": 0.6350826025009155,
-    "eval_pred_label": 8465.072265625,
-    "eval_rewards/accuracies": 0.6200000047683716,
-    "eval_rewards/chosen": 0.029984984546899796,
-    "eval_rewards/margins": 0.06345725804567337,
-    "eval_rewards/rejected": -0.033472273498773575,
-    "eval_runtime": 439.1916,
+    "eval_logits/chosen": -2.5063796043395996,
+    "eval_logits/rejected": -2.5199685096740723,
+    "eval_logps/chosen": -272.9034729003906,
+    "eval_logps/rejected": -250.30796813964844,
+    "eval_loss": 0.6366299986839294,
+    "eval_pred_label": 6857.14404296875,
+    "eval_rewards/accuracies": 0.6320000290870667,
+    "eval_rewards/chosen": 0.03307868540287018,
+    "eval_rewards/margins": 0.06870328634977341,
+    "eval_rewards/rejected": -0.03562460467219353,
+    "eval_runtime": 441.6525,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.554,
-    "eval_steps_per_second": 0.285,
-    "eval_use_label": 7566.92822265625,
-    "train_loss": 0.6553713429036565,
-    "train_runtime": 24224.386,
+    "eval_samples_per_second": 4.528,
+    "eval_steps_per_second": 0.283,
+    "eval_use_label": 9174.8564453125,
+    "train_loss": 0.6567496789063458,
+    "train_runtime": 24439.922,
     "train_samples": 61135,
-    "train_samples_per_second": 2.524,
+    "train_samples_per_second": 2.501,
     "train_steps_per_second": 0.039
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.5062618255615234,
-    "eval_logits/rejected": -2.5199859142303467,
-    "eval_logps/chosen": -272.93438720703125,
-    "eval_logps/rejected": -250.28643798828125,
-    "eval_loss": 0.6350826025009155,
-    "eval_pred_label": 8465.072265625,
-    "eval_rewards/accuracies": 0.6200000047683716,
-    "eval_rewards/chosen": 0.029984984546899796,
-    "eval_rewards/margins": 0.06345725804567337,
-    "eval_rewards/rejected": -0.033472273498773575,
-    "eval_runtime": 439.1916,
+    "eval_logits/chosen": -2.5063796043395996,
+    "eval_logits/rejected": -2.5199685096740723,
+    "eval_logps/chosen": -272.9034729003906,
+    "eval_logps/rejected": -250.30796813964844,
+    "eval_loss": 0.6366299986839294,
+    "eval_pred_label": 6857.14404296875,
+    "eval_rewards/accuracies": 0.6320000290870667,
+    "eval_rewards/chosen": 0.03307868540287018,
+    "eval_rewards/margins": 0.06870328634977341,
+    "eval_rewards/rejected": -0.03562460467219353,
+    "eval_runtime": 441.6525,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.554,
-    "eval_steps_per_second": 0.285,
-    "eval_use_label": 7566.92822265625
+    "eval_samples_per_second": 4.528,
+    "eval_steps_per_second": 0.283,
+    "eval_use_label": 9174.8564453125
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6553713429036565,
-    "train_runtime": 24224.386,
+    "train_loss": 0.6567496789063458,
+    "train_runtime": 24439.922,
     "train_samples": 61135,
-    "train_samples_per_second": 2.524,
+    "train_samples_per_second": 2.501,
     "train_steps_per_second": 0.039
 }

trainer_state.json

@@ -27,179 +27,179 @@
     {
       "epoch": 0.1,
       "learning_rate": 4.976717112922002e-07,
-      "logits/chosen": -2.475785255432129,
-      "logits/rejected": -2.48360276222229,
-      "logps/chosen": -273.616943359375,
-      "logps/rejected": -258.89666748046875,
-      "loss": 0.6789,
-      "pred_label": 242.72474670410156,
-      "rewards/accuracies": 0.5082070827484131,
-      "rewards/chosen": -0.0015035810647532344,
-      "rewards/margins": -0.000531451019924134,
-      "rewards/rejected": -0.0009721297537907958,
+      "logits/chosen": -2.4758388996124268,
+      "logits/rejected": -2.4836206436157227,
+      "logps/chosen": -273.62322998046875,
+      "logps/rejected": -258.89813232421875,
+      "loss": 0.6823,
+      "pred_label": 156.05050659179688,
+      "rewards/accuracies": 0.4886363744735718,
+      "rewards/chosen": -0.002133187372237444,
+      "rewards/margins": -0.0010164172854274511,
+      "rewards/rejected": -0.0011167696211487055,
       "step": 100,
-      "use_label": 567.2752685546875
+      "use_label": 653.9495239257812
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.3946449359720607e-07,
-      "logits/chosen": -2.4878406524658203,
-      "logits/rejected": -2.475137233734131,
-      "logps/chosen": -267.71783447265625,
-      "logps/rejected": -250.2951202392578,
-      "loss": 0.6774,
-      "pred_label": 765.6174926757812,
-      "rewards/accuracies": 0.5337499976158142,
-      "rewards/chosen": 0.004710530862212181,
-      "rewards/margins": 0.008190509863197803,
-      "rewards/rejected": -0.0034799796994775534,
+      "logits/chosen": -2.4878945350646973,
+      "logits/rejected": -2.4750781059265137,
+      "logps/chosen": -267.72540283203125,
+      "logps/rejected": -250.30291748046875,
+      "loss": 0.6801,
+      "pred_label": 512.7750244140625,
+      "rewards/accuracies": 0.5575000047683716,
+      "rewards/chosen": 0.003957623615860939,
+      "rewards/margins": 0.008218127302825451,
+      "rewards/rejected": -0.0042605032213032246,
       "step": 200,
-      "use_label": 1636.3824462890625
+      "use_label": 1889.2249755859375
     },
     {
       "epoch": 0.31,
       "learning_rate": 3.812572759022118e-07,
-      "logits/chosen": -2.4730782508850098,
-      "logits/rejected": -2.476684808731079,
-      "logps/chosen": -272.0400085449219,
-      "logps/rejected": -244.8907470703125,
-      "loss": 0.6717,
-      "pred_label": 1391.907470703125,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.010839122347533703,
-      "rewards/margins": 0.01838376186788082,
-      "rewards/rejected": -0.007544639054685831,
+      "logits/chosen": -2.47322940826416,
+      "logits/rejected": -2.4768893718719482,
+      "logps/chosen": -272.0399169921875,
+      "logps/rejected": -244.90228271484375,
+      "loss": 0.6733,
+      "pred_label": 943.1199951171875,
+      "rewards/accuracies": 0.6075000166893005,
+      "rewards/chosen": 0.010850328952074051,
+      "rewards/margins": 0.01955023780465126,
+      "rewards/rejected": -0.008699909783899784,
       "step": 300,
-      "use_label": 2610.092529296875
+      "use_label": 3058.8798828125
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.230500582072177e-07,
-      "logits/chosen": -2.4861884117126465,
-      "logits/rejected": -2.465975761413574,
-      "logps/chosen": -270.7182922363281,
-      "logps/rejected": -252.98463439941406,
-      "loss": 0.6633,
-      "pred_label": 2104.0400390625,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.014859138987958431,
-      "rewards/margins": 0.03158174827694893,
-      "rewards/rejected": -0.016722608357667923,
+      "logits/chosen": -2.486250400543213,
+      "logits/rejected": -2.466102361679077,
+      "logps/chosen": -270.7090759277344,
+      "logps/rejected": -252.9911346435547,
+      "loss": 0.6635,
+      "pred_label": 1496.31005859375,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": 0.01578013226389885,
+      "rewards/margins": 0.0331539586186409,
+      "rewards/rejected": -0.01737382635474205,
       "step": 400,
-      "use_label": 3497.9599609375
+      "use_label": 4105.68994140625
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.648428405122235e-07,
-      "logits/chosen": -2.4607770442962646,
-      "logits/rejected": -2.4463818073272705,
-      "logps/chosen": -259.13006591796875,
-      "logps/rejected": -241.9766845703125,
-      "loss": 0.6539,
-      "pred_label": 2935.78759765625,
-      "rewards/accuracies": 0.6106250286102295,
-      "rewards/chosen": 0.01635747402906418,
-      "rewards/margins": 0.031940869987010956,
-      "rewards/rejected": -0.015583401545882225,
+      "logits/chosen": -2.46098256111145,
+      "logits/rejected": -2.446549654006958,
+      "logps/chosen": -259.1031799316406,
+      "logps/rejected": -241.98345947265625,
+      "loss": 0.6543,
+      "pred_label": 2180.89990234375,
+      "rewards/accuracies": 0.6175000071525574,
+      "rewards/chosen": 0.019046209752559662,
+      "rewards/margins": 0.03530467674136162,
+      "rewards/rejected": -0.016258466988801956,
       "step": 500,
-      "use_label": 4266.21240234375
+      "use_label": 5021.10009765625
     },
     {
       "epoch": 0.63,
       "learning_rate": 2.0663562281722933e-07,
-      "logits/chosen": -2.4893369674682617,
-      "logits/rejected": -2.4660918712615967,
-      "logps/chosen": -271.8557434082031,
-      "logps/rejected": -252.8357391357422,
-      "loss": 0.6455,
-      "pred_label": 3848.889892578125,
-      "rewards/accuracies": 0.6412500143051147,
-      "rewards/chosen": 0.02700289897620678,
-      "rewards/margins": 0.05332392081618309,
-      "rewards/rejected": -0.02632101997733116,
+      "logits/chosen": -2.4896275997161865,
+      "logits/rejected": -2.4663710594177246,
+      "logps/chosen": -271.8394470214844,
+      "logps/rejected": -252.83351135253906,
+      "loss": 0.646,
+      "pred_label": 2942.87255859375,
+      "rewards/accuracies": 0.6600000262260437,
+      "rewards/chosen": 0.02863229252398014,
+      "rewards/margins": 0.05473264306783676,
+      "rewards/rejected": -0.02610035613179207,
       "step": 600,
-      "use_label": 4953.10986328125
+      "use_label": 5859.12744140625
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.4842840512223514e-07,
-      "logits/chosen": -2.484968423843384,
-      "logits/rejected": -2.4719583988189697,
-      "logps/chosen": -269.18682861328125,
-      "logps/rejected": -238.5116729736328,
-      "loss": 0.6409,
-      "pred_label": 4827.14501953125,
-      "rewards/accuracies": 0.6518750190734863,
-      "rewards/chosen": 0.02863721363246441,
-      "rewards/margins": 0.056007277220487595,
-      "rewards/rejected": -0.027370067313313484,
+      "logits/chosen": -2.485153913497925,
+      "logits/rejected": -2.472170352935791,
+      "logps/chosen": -269.14117431640625,
+      "logps/rejected": -238.4977569580078,
+      "loss": 0.6421,
+      "pred_label": 3756.925048828125,
+      "rewards/accuracies": 0.6418750286102295,
+      "rewards/chosen": 0.03320219740271568,
+      "rewards/margins": 0.05917687341570854,
+      "rewards/rejected": -0.02597467601299286,
       "step": 700,
-      "use_label": 5574.85498046875
+      "use_label": 6645.0751953125
     },
     {
       "epoch": 0.84,
       "learning_rate": 9.022118742724097e-08,
-      "logits/chosen": -2.4878268241882324,
-      "logits/rejected": -2.501347064971924,
-      "logps/chosen": -276.5606384277344,
-      "logps/rejected": -252.3398895263672,
-      "loss": 0.6369,
-      "pred_label": 5855.009765625,
-      "rewards/accuracies": 0.6299999952316284,
-      "rewards/chosen": 0.03379346430301666,
-      "rewards/margins": 0.062380146235227585,
-      "rewards/rejected": -0.028586676344275475,
+      "logits/chosen": -2.4880199432373047,
+      "logits/rejected": -2.5014524459838867,
+      "logps/chosen": -276.52313232421875,
+      "logps/rejected": -252.34681701660156,
+      "loss": 0.6386,
+      "pred_label": 4625.42236328125,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": 0.03754829242825508,
+      "rewards/margins": 0.06682833284139633,
+      "rewards/rejected": -0.02928004413843155,
       "step": 800,
-      "use_label": 6146.990234375
+      "use_label": 7376.57763671875
     },
     {
       "epoch": 0.94,
       "learning_rate": 3.20139697322468e-08,
-      "logits/chosen": -2.4698288440704346,
-      "logits/rejected": -2.4851109981536865,
-      "logps/chosen": -266.0604553222656,
-      "logps/rejected": -247.9416046142578,
-      "loss": 0.6382,
-      "pred_label": 6887.15771484375,
+      "logits/chosen": -2.4699513912200928,
+      "logits/rejected": -2.485213279724121,
+      "logps/chosen": -266.04229736328125,
+      "logps/rejected": -247.93341064453125,
+      "loss": 0.6393,
+      "pred_label": 5488.7001953125,
       "rewards/accuracies": 0.6587499976158142,
-      "rewards/chosen": 0.030911343172192574,
-      "rewards/margins": 0.061020173132419586,
-      "rewards/rejected": -0.03010883368551731,
+      "rewards/chosen": 0.03272656351327896,
+      "rewards/margins": 0.06201673671603203,
+      "rewards/rejected": -0.02929016388952732,
       "step": 900,
-      "use_label": 6714.84228515625
+      "use_label": 8113.2998046875
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -2.5062618255615234,
-      "eval_logits/rejected": -2.5199859142303467,
-      "eval_logps/chosen": -272.93438720703125,
-      "eval_logps/rejected": -250.28643798828125,
-      "eval_loss": 0.6350826025009155,
-      "eval_pred_label": 8131.07177734375,
-      "eval_rewards/accuracies": 0.6200000047683716,
-      "eval_rewards/chosen": 0.029984984546899796,
-      "eval_rewards/margins": 0.06345725804567337,
-      "eval_rewards/rejected": -0.033472273498773575,
-      "eval_runtime": 439.2922,
-      "eval_samples_per_second": 4.553,
-      "eval_steps_per_second": 0.285,
-      "eval_use_label": 7400.92822265625,
+      "eval_logits/chosen": -2.5063796043395996,
+      "eval_logits/rejected": -2.5199685096740723,
+      "eval_logps/chosen": -272.9034729003906,
+      "eval_logps/rejected": -250.30796813964844,
+      "eval_loss": 0.6366299986839294,
+      "eval_pred_label": 6565.14404296875,
+      "eval_rewards/accuracies": 0.6320000290870667,
+      "eval_rewards/chosen": 0.03307868540287018,
+      "eval_rewards/margins": 0.06870328634977341,
+      "eval_rewards/rejected": -0.03562460467219353,
+      "eval_runtime": 443.034,
+      "eval_samples_per_second": 4.514,
+      "eval_steps_per_second": 0.282,
+      "eval_use_label": 8966.8564453125,
       "step": 955
     },
     {
       "epoch": 1.0,
       "step": 955,
       "total_flos": 0.0,
-      "train_loss": 0.6553713429036565,
-      "train_runtime": 24224.386,
-      "train_samples_per_second": 2.524,
+      "train_loss": 0.6567496789063458,
+      "train_runtime": 24439.922,
+      "train_samples_per_second": 2.501,
       "train_steps_per_second": 0.039
     }
   ],
   "logging_steps": 100,
   "max_steps": 955,
   "num_train_epochs": 1,
-  "save_steps": 10,
+  "save_steps": 500,
   "total_flos": 0.0,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3088848f6347f07a0174d8f855b61982c881fa3825ffbdb936ea9d0fa60ba543
+oid sha256:3939afbabc4baeb60d396786c0ae42b352034e4fb51ab2354f56d3c071536d9f
 size 4792