Model save

README.md

@@ -17,15 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.3352
-- Rewards/chosen: -1.8457
-- Rewards/rejected: -5.5205
-- Rewards/accuracies: 0.8729
-- Rewards/margins: 3.6748
-- Logps/rejected: -1564.3583
-- Logps/chosen: -751.3719
-- Logits/rejected: -2.9460
-- Logits/chosen: -3.1365
+- Logits/chosen: -3.1281
+- Logits/rejected: -2.9646
+- Logps/chosen: -755.8733
+- Logps/rejected: -1595.0167
+- Loss: 0.3332
+- Rewards/accuracies: 0.8686
+- Rewards/chosen: -1.8907
+- Rewards/margins: 3.9363
+- Rewards/rejected: -5.8271
 
 ## Model description
 
@@ -60,9 +60,9 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.165         | 0.6   | 100  | 0.3352          | -1.8457        | -5.5205          | 0.8729             | 3.6748          | -1564.3583     | -751.3719    | -2.9460         | -3.1365       |
+| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
+|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
+| 0.1684        | 0.6   | 100  | -3.1281       | -2.9646         | -755.8733    | -1595.0167     | 0.3332          | 0.8686             | -1.8907        | 3.9363          | -5.8271          |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.2817753102807772,
-    "train_runtime": 2916.9515,
+    "train_loss": 0.05682134947606495,
+    "train_runtime": 921.1998,
     "train_samples": 10744,
-    "train_samples_per_second": 3.683,
-    "train_steps_per_second": 0.058
+    "train_samples_per_second": 11.663,
+    "train_steps_per_second": 0.182
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e54965f7b11a7a21455bc4c5b01631545242d6b7d21a1914d1abac427dc7d53f
+oid sha256:fe0127f15a4461d6a7c28b08e611bd232f2d4bca94dce568ac53e325cd40fea2
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a20a85a91d65ef47d88b7a38be3f15d950615e532666830e610a7e45fa9aa1f4
+oid sha256:4cd4000af7519e59aed8955cdef8a0605c985111576d9adab856d7fab36509a6
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37aaaaca7d689f3c0c042b750c9bde846e812bf6610ca9ce994adc74b2b2c302
+oid sha256:633705aecaeaa274c0c1294d292969ed4084b3468ed9aa8295673918e39a7d31
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.2817753102807772,
-    "train_runtime": 2916.9515,
+    "train_loss": 0.05682134947606495,
+    "train_runtime": 921.1998,
     "train_samples": 10744,
-    "train_samples_per_second": 3.683,
-    "train_steps_per_second": 0.058
+    "train_samples_per_second": 11.663,
+    "train_steps_per_second": 0.182
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.01,
-      "grad_norm": 17.799428866198873,
+      "grad_norm": 17.800336779303485,
       "learning_rate": 2.941176470588235e-08,
       "logits/chosen": -3.0399391651153564,
       "logits/rejected": -2.5624823570251465,
@@ -25,268 +25,268 @@
     },
     {
       "epoch": 0.06,
-      "grad_norm": 18.84470879916231,
+      "grad_norm": 18.838509896307443,
       "learning_rate": 2.941176470588235e-07,
-      "logits/chosen": -2.7761378288269043,
-      "logits/rejected": -2.7907724380493164,
-      "logps/chosen": -503.99615478515625,
+      "logits/chosen": -2.7760655879974365,
+      "logits/rejected": -2.790731906890869,
+      "logps/chosen": -503.9539489746094,
       "logps/rejected": -1057.63232421875,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.4930555522441864,
-      "rewards/chosen": 0.0016675572842359543,
-      "rewards/margins": 0.0015082131139934063,
-      "rewards/rejected": 0.00015934415569063276,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5763888955116272,
+      "rewards/chosen": 0.0020897421054542065,
+      "rewards/margins": 0.001932685961946845,
+      "rewards/rejected": 0.0001570563472341746,
       "step": 10
     },
     {
       "epoch": 0.12,
-      "grad_norm": 20.728431631666236,
+      "grad_norm": 20.774413071968993,
       "learning_rate": 4.995131923687487e-07,
-      "logits/chosen": -2.7420125007629395,
-      "logits/rejected": -2.6966209411621094,
-      "logps/chosen": -548.6405029296875,
-      "logps/rejected": -999.8572998046875,
-      "loss": 0.6742,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.03226347267627716,
-      "rewards/margins": 0.042478956282138824,
-      "rewards/rejected": -0.010215478017926216,
+      "logits/chosen": -2.7422165870666504,
+      "logits/rejected": -2.696645498275757,
+      "logps/chosen": -548.654296875,
+      "logps/rejected": -999.8585815429688,
+      "loss": 0.6741,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.03212609142065048,
+      "rewards/margins": 0.042353663593530655,
+      "rewards/rejected": -0.010227566584944725,
       "step": 20
     },
     {
       "epoch": 0.18,
-      "grad_norm": 24.840501208406245,
+      "grad_norm": 24.957524647561584,
       "learning_rate": 4.909114739839079e-07,
-      "logits/chosen": -2.9764866828918457,
-      "logits/rejected": -2.835902690887451,
-      "logps/chosen": -549.3971557617188,
-      "logps/rejected": -1060.723876953125,
+      "logits/chosen": -2.9761385917663574,
+      "logits/rejected": -2.836198329925537,
+      "logps/chosen": -549.5018310546875,
+      "logps/rejected": -1060.82763671875,
       "loss": 0.6134,
       "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.06945657730102539,
-      "rewards/margins": 0.18815770745277405,
-      "rewards/rejected": -0.11870112270116806,
+      "rewards/chosen": 0.06840862333774567,
+      "rewards/margins": 0.18814805150032043,
+      "rewards/rejected": -0.11973947286605835,
       "step": 30
     },
     {
       "epoch": 0.24,
-      "grad_norm": 28.735895531088158,
+      "grad_norm": 29.2493335087329,
       "learning_rate": 4.719192614212969e-07,
-      "logits/chosen": -3.0226290225982666,
-      "logits/rejected": -2.992037296295166,
-      "logps/chosen": -569.0553588867188,
-      "logps/rejected": -1104.0389404296875,
-      "loss": 0.4727,
+      "logits/chosen": -3.023461103439331,
+      "logits/rejected": -2.9939627647399902,
+      "logps/chosen": -569.5960083007812,
+      "logps/rejected": -1105.576416015625,
+      "loss": 0.4716,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -0.17996565997600555,
-      "rewards/margins": 0.7053974866867065,
-      "rewards/rejected": -0.8853631019592285,
+      "rewards/chosen": -0.18537160754203796,
+      "rewards/margins": 0.7153643369674683,
+      "rewards/rejected": -0.9007358551025391,
       "step": 40
     },
     {
       "epoch": 0.3,
-      "grad_norm": 34.5936973521811,
+      "grad_norm": 34.52922011163671,
       "learning_rate": 4.4335568741374695e-07,
-      "logits/chosen": -3.085718870162964,
-      "logits/rejected": -3.139434814453125,
-      "logps/chosen": -724.1493530273438,
-      "logps/rejected": -1332.0565185546875,
-      "loss": 0.3358,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.0768327713012695,
-      "rewards/margins": 1.9719797372817993,
-      "rewards/rejected": -3.0488123893737793,
+      "logits/chosen": -3.0847573280334473,
+      "logits/rejected": -3.1392886638641357,
+      "logps/chosen": -724.968505859375,
+      "logps/rejected": -1334.053955078125,
+      "loss": 0.3356,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.0850236415863037,
+      "rewards/margins": 1.9837610721588135,
+      "rewards/rejected": -3.068784713745117,
       "step": 50
     },
     {
       "epoch": 0.36,
-      "grad_norm": 75.01680127687075,
+      "grad_norm": 74.1970434298791,
       "learning_rate": 4.0645269681018434e-07,
-      "logits/chosen": -3.012761354446411,
-      "logits/rejected": -3.0863022804260254,
-      "logps/chosen": -767.4573364257812,
-      "logps/rejected": -1472.573486328125,
-      "loss": 0.2652,
+      "logits/chosen": -3.01001238822937,
+      "logits/rejected": -3.0855369567871094,
+      "logps/chosen": -763.7335815429688,
+      "logps/rejected": -1466.1148681640625,
+      "loss": 0.2659,
       "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -1.2531887292861938,
-      "rewards/margins": 3.80780291557312,
-      "rewards/rejected": -5.060992240905762,
+      "rewards/chosen": -1.215951681137085,
+      "rewards/margins": 3.7804553508758545,
+      "rewards/rejected": -4.996407508850098,
       "step": 60
     },
     {
       "epoch": 0.42,
-      "grad_norm": 26.06996725714517,
+      "grad_norm": 24.166083003223434,
       "learning_rate": 3.6280191288478435e-07,
-      "logits/chosen": -2.997857093811035,
-      "logits/rejected": -3.0774106979370117,
-      "logps/chosen": -651.4761962890625,
-      "logps/rejected": -1660.883056640625,
-      "loss": 0.2146,
+      "logits/chosen": -2.9843807220458984,
+      "logits/rejected": -3.0654196739196777,
+      "logps/chosen": -652.9537353515625,
+      "logps/rejected": -1663.2164306640625,
+      "loss": 0.2156,
       "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": -0.9511808156967163,
-      "rewards/margins": 4.4248552322387695,
-      "rewards/rejected": -5.376035690307617,
+      "rewards/chosen": -0.9659550786018372,
+      "rewards/margins": 4.433414459228516,
+      "rewards/rejected": -5.399369716644287,
       "step": 70
     },
     {
       "epoch": 0.48,
-      "grad_norm": 19.983426827432616,
+      "grad_norm": 17.793411473003722,
       "learning_rate": 3.142859907420615e-07,
-      "logits/chosen": -3.0228724479675293,
-      "logits/rejected": -3.0555529594421387,
-      "logps/chosen": -668.3040771484375,
-      "logps/rejected": -1470.601806640625,
-      "loss": 0.1981,
+      "logits/chosen": -3.0192360877990723,
+      "logits/rejected": -3.050020217895508,
+      "logps/chosen": -661.2525634765625,
+      "logps/rejected": -1447.4609375,
+      "loss": 0.1989,
       "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -1.036330223083496,
-      "rewards/margins": 4.10854959487915,
-      "rewards/rejected": -5.1448798179626465,
+      "rewards/chosen": -0.9658153653144836,
+      "rewards/margins": 3.9476540088653564,
+      "rewards/rejected": -4.913470268249512,
       "step": 80
     },
     {
       "epoch": 0.54,
-      "grad_norm": 48.7371474187784,
+      "grad_norm": 55.47746105929639,
       "learning_rate": 2.629974185404951e-07,
-      "logits/chosen": -2.960538387298584,
-      "logits/rejected": -3.0230116844177246,
-      "logps/chosen": -626.430908203125,
-      "logps/rejected": -1550.2908935546875,
-      "loss": 0.1793,
+      "logits/chosen": -2.978877305984497,
+      "logits/rejected": -3.029723644256592,
+      "logps/chosen": -613.0646362304688,
+      "logps/rejected": -1531.10546875,
+      "loss": 0.1844,
       "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -1.4039939641952515,
-      "rewards/margins": 4.859676361083984,
-      "rewards/rejected": -6.263670921325684,
+      "rewards/chosen": -1.270330786705017,
+      "rewards/margins": 4.801483154296875,
+      "rewards/rejected": -6.071813583374023,
       "step": 90
     },
     {
       "epoch": 0.6,
-      "grad_norm": 35.72914831932015,
+      "grad_norm": 33.71679055058231,
       "learning_rate": 2.1114826863194878e-07,
-      "logits/chosen": -2.9680638313293457,
-      "logits/rejected": -3.0059874057769775,
-      "logps/chosen": -673.1527099609375,
-      "logps/rejected": -1737.0718994140625,
-      "loss": 0.165,
+      "logits/chosen": -2.983506441116333,
+      "logits/rejected": -3.02791690826416,
+      "logps/chosen": -676.8287353515625,
+      "logps/rejected": -1784.007568359375,
+      "loss": 0.1684,
       "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": -1.3123265504837036,
-      "rewards/margins": 5.412586212158203,
-      "rewards/rejected": -6.724913120269775,
+      "rewards/chosen": -1.3490874767303467,
+      "rewards/margins": 5.845183849334717,
+      "rewards/rejected": -7.194271087646484,
       "step": 100
     },
     {
       "epoch": 0.6,
-      "eval_logits/chosen": -3.136507034301758,
-      "eval_logits/rejected": -2.9459588527679443,
-      "eval_logps/chosen": -751.3718872070312,
-      "eval_logps/rejected": -1564.3582763671875,
-      "eval_loss": 0.33522769808769226,
-      "eval_rewards/accuracies": 0.8728813529014587,
-      "eval_rewards/chosen": -1.8457263708114624,
-      "eval_rewards/margins": 3.6747524738311768,
-      "eval_rewards/rejected": -5.520478248596191,
-      "eval_runtime": 229.3003,
-      "eval_samples_per_second": 8.147,
-      "eval_steps_per_second": 0.257,
+      "eval_logits/chosen": -3.1281025409698486,
+      "eval_logits/rejected": -2.964585304260254,
+      "eval_logps/chosen": -755.873291015625,
+      "eval_logps/rejected": -1595.0167236328125,
+      "eval_loss": 0.3332486152648926,
+      "eval_rewards/accuracies": 0.8686440587043762,
+      "eval_rewards/chosen": -1.8907400369644165,
+      "eval_rewards/margins": 3.9363222122192383,
+      "eval_rewards/rejected": -5.827062606811523,
+      "eval_runtime": 195.0133,
+      "eval_samples_per_second": 9.579,
+      "eval_steps_per_second": 0.303,
       "step": 100
     },
     {
       "epoch": 0.65,
-      "grad_norm": 35.289047373056896,
+      "grad_norm": 36.49070034902309,
       "learning_rate": 1.6097479104361326e-07,
-      "logits/chosen": -2.9342598915100098,
-      "logits/rejected": -2.9889469146728516,
-      "logps/chosen": -731.7562255859375,
-      "logps/rejected": -1630.530517578125,
-      "loss": 0.1931,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -1.4041537046432495,
-      "rewards/margins": 5.037538051605225,
-      "rewards/rejected": -6.4416913986206055,
+      "logits/chosen": -2.9343550205230713,
+      "logits/rejected": -3.0102057456970215,
+      "logps/chosen": -727.3587646484375,
+      "logps/rejected": -1638.8382568359375,
+      "loss": 0.1954,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.3601789474487305,
+      "rewards/margins": 5.164590835571289,
+      "rewards/rejected": -6.524770259857178,
       "step": 110
     },
     {
       "epoch": 0.71,
-      "grad_norm": 22.970974992307827,
+      "grad_norm": 23.81218846817336,
       "learning_rate": 1.146409641785882e-07,
-      "logits/chosen": -2.8369693756103516,
-      "logits/rejected": -2.999558925628662,
-      "logps/chosen": -675.0198364257812,
-      "logps/rejected": -1771.7913818359375,
-      "loss": 0.1462,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -1.2913109064102173,
-      "rewards/margins": 5.878513336181641,
-      "rewards/rejected": -7.16982364654541,
+      "logits/chosen": -2.8368687629699707,
+      "logits/rejected": -3.0225136280059814,
+      "logps/chosen": -666.7093505859375,
+      "logps/rejected": -1738.1021728515625,
+      "loss": 0.1519,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -1.2082051038742065,
+      "rewards/margins": 5.624727249145508,
+      "rewards/rejected": -6.832932472229004,
       "step": 120
     },
     {
       "epoch": 0.77,
-      "grad_norm": 19.730658509992367,
+      "grad_norm": 20.21152127941066,
       "learning_rate": 7.414516258630244e-08,
-      "logits/chosen": -2.9381890296936035,
-      "logits/rejected": -2.9967472553253174,
-      "logps/chosen": -715.4029541015625,
-      "logps/rejected": -1808.675048828125,
-      "loss": 0.1393,
+      "logits/chosen": -2.9129068851470947,
+      "logits/rejected": -3.0287842750549316,
+      "logps/chosen": -708.4816284179688,
+      "logps/rejected": -1789.521728515625,
+      "loss": 0.1424,
       "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -1.4525569677352905,
-      "rewards/margins": 5.484087944030762,
-      "rewards/rejected": -6.936644554138184,
+      "rewards/chosen": -1.3833436965942383,
+      "rewards/margins": 5.361766338348389,
+      "rewards/rejected": -6.745110511779785,
       "step": 130
     },
     {
       "epoch": 0.83,
-      "grad_norm": 18.529137261894174,
+      "grad_norm": 19.81204521774453,
       "learning_rate": 4.1233967214979764e-08,
-      "logits/chosen": -2.980013370513916,
-      "logits/rejected": -2.982477903366089,
-      "logps/chosen": -672.929443359375,
-      "logps/rejected": -1697.8675537109375,
-      "loss": 0.1436,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -1.4015110731124878,
-      "rewards/margins": 5.538142681121826,
-      "rewards/rejected": -6.9396538734436035,
+      "logits/chosen": -2.979017734527588,
+      "logits/rejected": -3.0025482177734375,
+      "logps/chosen": -664.9674072265625,
+      "logps/rejected": -1679.622802734375,
+      "loss": 0.1512,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -1.3218904733657837,
+      "rewards/margins": 5.435315132141113,
+      "rewards/rejected": -6.757205963134766,
       "step": 140
     },
     {
       "epoch": 0.89,
-      "grad_norm": 31.45814992703821,
+      "grad_norm": 33.74454128402628,
       "learning_rate": 1.732683550362954e-08,
-      "logits/chosen": -2.9663195610046387,
-      "logits/rejected": -3.070772409439087,
-      "logps/chosen": -677.1956787109375,
-      "logps/rejected": -2003.745361328125,
-      "loss": 0.0987,
+      "logits/chosen": -2.971137523651123,
+      "logits/rejected": -3.0957703590393066,
+      "logps/chosen": -662.1959228515625,
+      "logps/rejected": -1967.736083984375,
+      "loss": 0.1032,
       "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": -1.4282945394515991,
-      "rewards/margins": 7.419705867767334,
-      "rewards/rejected": -8.848000526428223,
+      "rewards/chosen": -1.2782970666885376,
+      "rewards/margins": 7.209610939025879,
+      "rewards/rejected": -8.487907409667969,
       "step": 150
     },
     {
       "epoch": 0.95,
-      "grad_norm": 35.76052098122492,
+      "grad_norm": 35.57488951709773,
       "learning_rate": 3.4548802869627804e-09,
-      "logits/chosen": -2.954775810241699,
-      "logits/rejected": -2.984683036804199,
-      "logps/chosen": -716.851806640625,
-      "logps/rejected": -1857.295654296875,
-      "loss": 0.1224,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": -1.7806593179702759,
-      "rewards/margins": 6.663559913635254,
-      "rewards/rejected": -8.444218635559082,
+      "logits/chosen": -2.9537863731384277,
+      "logits/rejected": -3.0035552978515625,
+      "logps/chosen": -699.1692504882812,
+      "logps/rejected": -1824.681884765625,
+      "loss": 0.1233,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -1.6038345098495483,
+      "rewards/margins": 6.514244079589844,
+      "rewards/rejected": -8.118078231811523,
       "step": 160
     },
     {
       "epoch": 1.0,
       "step": 168,
       "total_flos": 0.0,
-      "train_loss": 0.2817753102807772,
-      "train_runtime": 2916.9515,
-      "train_samples_per_second": 3.683,
-      "train_steps_per_second": 0.058
+      "train_loss": 0.05682134947606495,
+      "train_runtime": 921.1998,
+      "train_samples_per_second": 11.663,
+      "train_steps_per_second": 0.182
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b09216a460e233950cd867da55d3c38f67a050a1cae634495d856cb2637f3be1
+oid sha256:9aeb8f297f211597416ec319893fcc5d1625d3b2d5ea38c7e95ed7d3b16cbf25
 size 6264