Model save

README.md

@@ -13,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on the None dataset.
+This model was trained from scratch on an unknown dataset.
 
 ## Model description
 
@@ -44,7 +44,7 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 1
+- num_epochs: 2
 
 ### Training results
 
@@ -54,5 +54,5 @@ The following hyperparameters were used during training:
 
 - Transformers 4.39.3
 - Pytorch 2.1.2+cu118
-- Datasets 2.16.1
+- Datasets 2.19.1
 - Tokenizers 0.15.2

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.7017018035803427,
-    "train_runtime": 5276.0496,
+    "epoch": 1.99,
+    "train_loss": 0.6178853964672408,
+    "train_runtime": 10771.3374,
     "train_samples": 46008,
-    "train_samples_per_second": 8.72,
-    "train_steps_per_second": 0.034
+    "train_samples_per_second": 8.543,
+    "train_steps_per_second": 0.033
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78023790d283daa5ac216bca5118f7f16d3456d271b829d8d4c420bb806d2361
+oid sha256:2e2280e1e450848ff611a22e91fa55b332aa9726ecc5d70924cd3609b00fb212
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:46c6d2f9dc71af031bc8877acf06f617c43d4c9a595373f147771924b45f71a4
+oid sha256:cf8335c8343439de60d7780dfff5b3af0920ec66940d7c67e6667917b0311519
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d162b61250d806211784d7670553be059e9804860cd77fea33f82fe78db8eca4
+oid sha256:52b7afca158c67eebe0c04a03e046b31ea0cd95308e37ffe3f73753ba89be2b5
 size 4540516344

runs/Jun19_01-26-02_n136-112-146/events.out.tfevents.1718732243.n136-112-146.1799798.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89c783b8734ffc11cc5123ccba305ba1bfbc03953668d7a3eec4922d2ed9615f
-size 26094
+oid sha256:c3f45fa1feffcc08ec9979e8592dfd6c0f3f8c7ebb292fc28d15ebf03c3d18ed
+size 29888

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.7017018035803427,
-    "train_runtime": 5276.0496,
+    "epoch": 1.99,
+    "train_loss": 0.6178853964672408,
+    "train_runtime": 10771.3374,
     "train_samples": 46008,
-    "train_samples_per_second": 8.72,
-    "train_steps_per_second": 0.034
+    "train_samples_per_second": 8.543,
+    "train_steps_per_second": 0.033
 }

trainer_state.json

@@ -1,17 +1,17 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9958275382475661,
+  "epoch": 1.9916550764951322,
   "eval_steps": 500,
-  "global_step": 179,
+  "global_step": 358,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.01,
-      "grad_norm": 265.8884735732537,
-      "learning_rate": 5.555555555555555e-10,
+      "grad_norm": 664.5507726524561,
+      "learning_rate": 2.7777777777777777e-10,
       "logits/chosen": -1.145136833190918,
       "logits/rejected": -0.7638764977455139,
       "logps/chosen": -369.05181884765625,
@@ -25,273 +25,543 @@
     },
     {
       "epoch": 0.06,
-      "grad_norm": 336.072912015809,
-      "learning_rate": 5.555555555555556e-09,
-      "logits/chosen": -1.0928783416748047,
-      "logits/rejected": -0.9423567652702332,
-      "logps/chosen": -372.36029052734375,
-      "logps/rejected": -339.4759826660156,
-      "loss": 0.7082,
-      "rewards/accuracies": 0.4270833432674408,
-      "rewards/chosen": -0.016076795756816864,
-      "rewards/margins": -0.023205766454339027,
-      "rewards/rejected": 0.007128970697522163,
+      "grad_norm": 784.2820602567579,
+      "learning_rate": 2.777777777777778e-09,
+      "logits/chosen": -1.0929811000823975,
+      "logits/rejected": -0.9427244663238525,
+      "logps/chosen": -372.3557434082031,
+      "logps/rejected": -339.4920959472656,
+      "loss": 0.7141,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.013798276893794537,
+      "rewards/margins": -0.012861602939665318,
+      "rewards/rejected": -0.0009366737212985754,
       "step": 10
     },
     {
       "epoch": 0.11,
-      "grad_norm": 322.5517219745342,
-      "learning_rate": 9.99619291237835e-09,
-      "logits/chosen": -0.8586678504943848,
-      "logits/rejected": -0.733450710773468,
-      "logps/chosen": -365.1810302734375,
-      "logps/rejected": -340.0671081542969,
-      "loss": 0.7124,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -0.01668541505932808,
-      "rewards/margins": -0.02395111322402954,
-      "rewards/rejected": 0.007265699096024036,
+      "grad_norm": 708.7597502584047,
+      "learning_rate": 5.555555555555556e-09,
+      "logits/chosen": -0.8585885167121887,
+      "logits/rejected": -0.7338771820068359,
+      "logps/chosen": -365.14849853515625,
+      "logps/rejected": -340.0997619628906,
+      "loss": 0.7105,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.00040760039701126516,
+      "rewards/margins": 0.008629530668258667,
+      "rewards/rejected": -0.009037131443619728,
       "step": 20
     },
     {
       "epoch": 0.17,
-      "grad_norm": 286.6512881766183,
-      "learning_rate": 9.863552602006434e-09,
-      "logits/chosen": -1.1635475158691406,
-      "logits/rejected": -0.8620313405990601,
-      "logps/chosen": -386.59136962890625,
-      "logps/rejected": -348.4241027832031,
-      "loss": 0.7091,
-      "rewards/accuracies": 0.5531250238418579,
-      "rewards/chosen": 0.008167916908860207,
-      "rewards/margins": 0.05863186717033386,
-      "rewards/rejected": -0.050463952124118805,
+      "grad_norm": 671.6766126687812,
+      "learning_rate": 8.333333333333334e-09,
+      "logits/chosen": -1.1642935276031494,
+      "logits/rejected": -0.8622403144836426,
+      "logps/chosen": -386.5867614746094,
+      "logps/rejected": -348.39959716796875,
+      "loss": 0.7069,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.01048402488231659,
+      "rewards/margins": 0.048695411533117294,
+      "rewards/rejected": -0.038211386650800705,
       "step": 30
     },
     {
       "epoch": 0.22,
-      "grad_norm": 251.72489049021175,
-      "learning_rate": 9.546315917055362e-09,
-      "logits/chosen": -1.0636483430862427,
-      "logits/rejected": -0.8499566912651062,
-      "logps/chosen": -362.8810119628906,
-      "logps/rejected": -332.20843505859375,
-      "loss": 0.7101,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -0.03652422875165939,
-      "rewards/margins": -0.019151169806718826,
-      "rewards/rejected": -0.017373060807585716,
+      "grad_norm": 642.0282612566415,
+      "learning_rate": 9.99619291237835e-09,
+      "logits/chosen": -1.0643326044082642,
+      "logits/rejected": -0.8505460619926453,
+      "logps/chosen": -362.8887023925781,
+      "logps/rejected": -332.2599792480469,
+      "loss": 0.7045,
+      "rewards/accuracies": 0.503125011920929,
+      "rewards/chosen": -0.04036983847618103,
+      "rewards/margins": 0.002776807639747858,
+      "rewards/rejected": -0.04314664751291275,
       "step": 40
     },
     {
       "epoch": 0.28,
-      "grad_norm": 296.71671546416985,
-      "learning_rate": 9.056523591268063e-09,
-      "logits/chosen": -1.0690479278564453,
-      "logits/rejected": -0.8786741495132446,
-      "logps/chosen": -377.68011474609375,
-      "logps/rejected": -340.79656982421875,
-      "loss": 0.7089,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.00048580169095657766,
-      "rewards/margins": 0.035611577332019806,
-      "rewards/rejected": -0.03609737753868103,
+      "grad_norm": 708.8205943616936,
+      "learning_rate": 9.953429730181653e-09,
+      "logits/chosen": -1.069666862487793,
+      "logits/rejected": -0.8791624903678894,
+      "logps/chosen": -377.6866760253906,
+      "logps/rejected": -340.76434326171875,
+      "loss": 0.7045,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.0037647963035851717,
+      "rewards/margins": 0.01622203178703785,
+      "rewards/rejected": -0.019986826926469803,
       "step": 50
     },
     {
       "epoch": 0.33,
-      "grad_norm": 244.469289636431,
-      "learning_rate": 8.41276571609327e-09,
-      "logits/chosen": -1.0523345470428467,
-      "logits/rejected": -0.8627673983573914,
-      "logps/chosen": -368.83758544921875,
-      "logps/rejected": -334.1421203613281,
-      "loss": 0.7039,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.014057522639632225,
-      "rewards/margins": 0.05172234773635864,
-      "rewards/rejected": -0.03766482323408127,
+      "grad_norm": 615.8031557981225,
+      "learning_rate": 9.863552602006434e-09,
+      "logits/chosen": -1.0530660152435303,
+      "logits/rejected": -0.8627260327339172,
+      "logps/chosen": -368.82427978515625,
+      "logps/rejected": -334.1483154296875,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.559374988079071,
+      "rewards/chosen": 0.020714253187179565,
+      "rewards/margins": 0.061485253274440765,
+      "rewards/rejected": -0.0407710075378418,
       "step": 60
     },
     {
       "epoch": 0.39,
-      "grad_norm": 218.81492341857816,
-      "learning_rate": 7.639476152864163e-09,
-      "logits/chosen": -1.0656211376190186,
-      "logits/rejected": -0.9337635040283203,
-      "logps/chosen": -361.3856506347656,
-      "logps/rejected": -338.86590576171875,
-      "loss": 0.7048,
-      "rewards/accuracies": 0.559374988079071,
-      "rewards/chosen": 0.010572463274002075,
-      "rewards/margins": 0.05579069256782532,
-      "rewards/rejected": -0.04521822929382324,
+      "grad_norm": 603.388531475607,
+      "learning_rate": 9.72741638325434e-09,
+      "logits/chosen": -1.0651648044586182,
+      "logits/rejected": -0.9337506294250488,
+      "logps/chosen": -361.4559631347656,
+      "logps/rejected": -338.84661865234375,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.4906249940395355,
+      "rewards/chosen": -0.02459469996392727,
+      "rewards/margins": 0.010977035388350487,
+      "rewards/rejected": -0.035571735352277756,
       "step": 70
     },
     {
       "epoch": 0.45,
-      "grad_norm": 233.90515893411515,
-      "learning_rate": 6.766005144407989e-09,
-      "logits/chosen": -0.9778022766113281,
-      "logits/rejected": -0.7526016235351562,
-      "logps/chosen": -369.4658508300781,
-      "logps/rejected": -332.49652099609375,
-      "loss": 0.6986,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.050749681890010834,
-      "rewards/margins": 0.07469334453344345,
-      "rewards/rejected": -0.023943662643432617,
+      "grad_norm": 596.781435384469,
+      "learning_rate": 9.546315917055362e-09,
+      "logits/chosen": -0.9770561456680298,
+      "logits/rejected": -0.7514477372169495,
+      "logps/chosen": -369.4873046875,
+      "logps/rejected": -332.56134033203125,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.04002801328897476,
+      "rewards/margins": 0.09637657552957535,
+      "rewards/rejected": -0.056348562240600586,
       "step": 80
     },
     {
       "epoch": 0.5,
-      "grad_norm": 236.56099599203856,
-      "learning_rate": 5.825505325157961e-09,
-      "logits/chosen": -0.9711936712265015,
-      "logits/rejected": -0.7899622917175293,
-      "logps/chosen": -360.23504638671875,
-      "logps/rejected": -336.3778076171875,
-      "loss": 0.7004,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.045762062072753906,
-      "rewards/margins": 0.08318910002708435,
-      "rewards/rejected": -0.037427037954330444,
+      "grad_norm": 587.0211902729591,
+      "learning_rate": 9.321973718524472e-09,
+      "logits/chosen": -0.9680253863334656,
+      "logits/rejected": -0.7858943343162537,
+      "logps/chosen": -360.33001708984375,
+      "logps/rejected": -336.4738464355469,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.001699769520200789,
+      "rewards/margins": 0.08374340832233429,
+      "rewards/rejected": -0.08544318377971649,
       "step": 90
     },
     {
       "epoch": 0.56,
-      "grad_norm": 243.70225224613213,
-      "learning_rate": 4.853673411307564e-09,
-      "logits/chosen": -1.0938254594802856,
-      "logits/rejected": -0.9187259674072266,
-      "logps/chosen": -381.3570861816406,
-      "logps/rejected": -355.38067626953125,
-      "loss": 0.6995,
-      "rewards/accuracies": 0.590624988079071,
-      "rewards/chosen": 0.049023449420928955,
-      "rewards/margins": 0.06866957992315292,
-      "rewards/rejected": -0.019646134227514267,
+      "grad_norm": 589.9118067767693,
+      "learning_rate": 9.056523591268063e-09,
+      "logits/chosen": -1.0888255834579468,
+      "logits/rejected": -0.9123393893241882,
+      "logps/chosen": -381.3514099121094,
+      "logps/rejected": -355.5039978027344,
+      "loss": 0.6515,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.051872074604034424,
+      "rewards/margins": 0.133183091878891,
+      "rewards/rejected": -0.08131101727485657,
       "step": 100
     },
     {
       "epoch": 0.61,
-      "grad_norm": 260.62196642098917,
-      "learning_rate": 3.887395330218429e-09,
-      "logits/chosen": -1.0548704862594604,
-      "logits/rejected": -0.8432992696762085,
-      "logps/chosen": -379.90887451171875,
-      "logps/rejected": -349.34161376953125,
-      "loss": 0.6973,
-      "rewards/accuracies": 0.6343749761581421,
-      "rewards/chosen": 0.11532249301671982,
-      "rewards/margins": 0.1085214838385582,
-      "rewards/rejected": 0.006801015231758356,
+      "grad_norm": 589.5691633116944,
+      "learning_rate": 8.752490331969806e-09,
+      "logits/chosen": -1.048461675643921,
+      "logits/rejected": -0.8349950909614563,
+      "logps/chosen": -379.940185546875,
+      "logps/rejected": -349.4921569824219,
+      "loss": 0.6436,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.09967100620269775,
+      "rewards/margins": 0.1681671440601349,
+      "rewards/rejected": -0.06849612295627594,
       "step": 110
     },
     {
       "epoch": 0.67,
-      "grad_norm": 253.16395063942525,
-      "learning_rate": 2.963346213260737e-09,
-      "logits/chosen": -1.1116909980773926,
-      "logits/rejected": -0.8191507458686829,
-      "logps/chosen": -378.15948486328125,
-      "logps/rejected": -347.91815185546875,
-      "loss": 0.7002,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.11069490760564804,
-      "rewards/margins": 0.1103142723441124,
-      "rewards/rejected": 0.0003806292952504009,
+      "grad_norm": 556.6131343001705,
+      "learning_rate": 8.41276571609327e-09,
+      "logits/chosen": -1.1043516397476196,
+      "logits/rejected": -0.8103276491165161,
+      "logps/chosen": -378.1798095703125,
+      "logps/rejected": -348.0905456542969,
+      "loss": 0.6435,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": 0.10052381455898285,
+      "rewards/margins": 0.18633946776390076,
+      "rewards/rejected": -0.08581562340259552,
       "step": 120
     },
     {
       "epoch": 0.72,
-      "grad_norm": 231.95660361604925,
-      "learning_rate": 2.1165983894256645e-09,
-      "logits/chosen": -1.1268573999404907,
-      "logits/rejected": -0.9338471293449402,
-      "logps/chosen": -374.75958251953125,
-      "logps/rejected": -351.5363464355469,
-      "loss": 0.6982,
-      "rewards/accuracies": 0.578125,
-      "rewards/chosen": 0.12768486142158508,
-      "rewards/margins": 0.12136401981115341,
-      "rewards/rejected": 0.006320851854979992,
+      "grad_norm": 585.5616649485556,
+      "learning_rate": 8.040580993110404e-09,
+      "logits/chosen": -1.119443416595459,
+      "logits/rejected": -0.9243392944335938,
+      "logps/chosen": -374.62371826171875,
+      "logps/rejected": -351.63323974609375,
+      "loss": 0.6349,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.19561262428760529,
+      "rewards/margins": 0.23775608837604523,
+      "rewards/rejected": -0.04214347153902054,
       "step": 130
     },
     {
       "epoch": 0.78,
-      "grad_norm": 253.82374639748912,
-      "learning_rate": 1.3792902133797692e-09,
-      "logits/chosen": -1.1068191528320312,
-      "logits/rejected": -0.99742591381073,
-      "logps/chosen": -356.70904541015625,
-      "logps/rejected": -332.5885009765625,
-      "loss": 0.6989,
-      "rewards/accuracies": 0.590624988079071,
-      "rewards/chosen": 0.13411644101142883,
-      "rewards/margins": 0.12641935050487518,
-      "rewards/rejected": 0.007697069551795721,
+      "grad_norm": 568.4650526780266,
+      "learning_rate": 7.639476152864163e-09,
+      "logits/chosen": -1.0983831882476807,
+      "logits/rejected": -0.987181544303894,
+      "logps/chosen": -356.64117431640625,
+      "logps/rejected": -332.7178039550781,
+      "loss": 0.6295,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.1680435836315155,
+      "rewards/margins": 0.22499410808086395,
+      "rewards/rejected": -0.05695053189992905,
       "step": 140
     },
     {
       "epoch": 0.83,
-      "grad_norm": 240.7477679446011,
-      "learning_rate": 7.794062526569734e-10,
-      "logits/chosen": -1.04066801071167,
-      "logits/rejected": -0.7631799578666687,
-      "logps/chosen": -373.73486328125,
-      "logps/rejected": -343.91748046875,
-      "loss": 0.6939,
-      "rewards/accuracies": 0.609375,
-      "rewards/chosen": 0.14227721095085144,
-      "rewards/margins": 0.10592744499444962,
-      "rewards/rejected": 0.03634977340698242,
+      "grad_norm": 565.1483875486134,
+      "learning_rate": 7.21326625538456e-09,
+      "logits/chosen": -1.0306283235549927,
+      "logits/rejected": -0.7497280836105347,
+      "logps/chosen": -373.5386047363281,
+      "logps/rejected": -344.0720520019531,
+      "loss": 0.6098,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": 0.2403784692287445,
+      "rewards/margins": 0.28130704164505005,
+      "rewards/rejected": -0.04092860221862793,
       "step": 150
     },
     {
       "epoch": 0.89,
-      "grad_norm": 231.3816310913791,
-      "learning_rate": 3.397151320423647e-10,
-      "logits/chosen": -1.139892578125,
-      "logits/rejected": -0.8710765838623047,
-      "logps/chosen": -368.18963623046875,
-      "logps/rejected": -335.36260986328125,
-      "loss": 0.6985,
-      "rewards/accuracies": 0.6156250238418579,
-      "rewards/chosen": 0.1571863293647766,
-      "rewards/margins": 0.1425299346446991,
-      "rewards/rejected": 0.014656400308012962,
+      "grad_norm": 532.1675415448219,
+      "learning_rate": 6.766005144407989e-09,
+      "logits/chosen": -1.1308726072311401,
+      "logits/rejected": -0.8597946166992188,
+      "logps/chosen": -368.006591796875,
+      "logps/rejected": -335.4107360839844,
+      "loss": 0.6206,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": 0.24870140850543976,
+      "rewards/margins": 0.258108526468277,
+      "rewards/rejected": -0.00940710585564375,
       "step": 160
     },
     {
       "epoch": 0.95,
-      "grad_norm": 265.6241152762033,
-      "learning_rate": 7.690534931565518e-11,
-      "logits/chosen": -0.9450071454048157,
-      "logits/rejected": -0.8331435918807983,
-      "logps/chosen": -363.50225830078125,
-      "logps/rejected": -337.5380554199219,
-      "loss": 0.6982,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.144387885928154,
-      "rewards/margins": 0.17387160658836365,
-      "rewards/rejected": -0.029483716934919357,
+      "grad_norm": 740.3690117109163,
+      "learning_rate": 6.301946889734301e-09,
+      "logits/chosen": -0.9349819421768188,
+      "logits/rejected": -0.8212488293647766,
+      "logps/chosen": -363.19781494140625,
+      "logps/rejected": -337.5945739746094,
+      "loss": 0.6152,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": 0.2966246008872986,
+      "rewards/margins": 0.3543737530708313,
+      "rewards/rejected": -0.05774913355708122,
       "step": 170
     },
     {
       "epoch": 1.0,
-      "step": 179,
+      "grad_norm": 511.7643196913959,
+      "learning_rate": 5.825505325157961e-09,
+      "logits/chosen": -1.017749547958374,
+      "logits/rejected": -0.809060275554657,
+      "logps/chosen": -368.81463623046875,
+      "logps/rejected": -349.66790771484375,
+      "loss": 0.6065,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": 0.3305915594100952,
+      "rewards/margins": 0.28414201736450195,
+      "rewards/rejected": 0.04644955322146416,
+      "step": 180
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 540.5118356148005,
+      "learning_rate": 5.341212066823356e-09,
+      "logits/chosen": -1.1874797344207764,
+      "logits/rejected": -0.9048231244087219,
+      "logps/chosen": -382.1863708496094,
+      "logps/rejected": -338.0008239746094,
+      "loss": 0.5876,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": 0.40876778960227966,
+      "rewards/margins": 0.41164499521255493,
+      "rewards/rejected": -0.0028771907091140747,
+      "step": 190
+    },
+    {
+      "epoch": 1.11,
+      "grad_norm": 574.532818545015,
+      "learning_rate": 4.853673411307564e-09,
+      "logits/chosen": -1.0419094562530518,
+      "logits/rejected": -0.8011609315872192,
+      "logps/chosen": -383.00933837890625,
+      "logps/rejected": -344.86761474609375,
+      "loss": 0.5989,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": 0.4167671799659729,
+      "rewards/margins": 0.3280712068080902,
+      "rewards/rejected": 0.08869597315788269,
+      "step": 200
+    },
+    {
+      "epoch": 1.17,
+      "grad_norm": 567.697203170915,
+      "learning_rate": 4.367526523389253e-09,
+      "logits/chosen": -1.1011512279510498,
+      "logits/rejected": -0.8666337132453918,
+      "logps/chosen": -370.8396911621094,
+      "logps/rejected": -342.78558349609375,
+      "loss": 0.5881,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": 0.4040229916572571,
+      "rewards/margins": 0.35025185346603394,
+      "rewards/rejected": 0.05377109721302986,
+      "step": 210
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 541.6922856310176,
+      "learning_rate": 3.887395330218429e-09,
+      "logits/chosen": -1.077133297920227,
+      "logits/rejected": -0.8909082412719727,
+      "logps/chosen": -376.4487609863281,
+      "logps/rejected": -351.6236877441406,
+      "loss": 0.5866,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": 0.4883548319339752,
+      "rewards/margins": 0.3798820674419403,
+      "rewards/rejected": 0.10847274214029312,
+      "step": 220
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 527.0093020370833,
+      "learning_rate": 3.4178465413942626e-09,
+      "logits/chosen": -1.108720302581787,
+      "logits/rejected": -0.8026013374328613,
+      "logps/chosen": -378.4242248535156,
+      "logps/rejected": -342.5303649902344,
+      "loss": 0.5738,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": 0.5319411158561707,
+      "rewards/margins": 0.44675689935684204,
+      "rewards/rejected": 0.08518422394990921,
+      "step": 230
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 486.03500376882494,
+      "learning_rate": 2.963346213260737e-09,
+      "logits/chosen": -0.9698505401611328,
+      "logits/rejected": -0.8225222826004028,
+      "logps/chosen": -367.175048828125,
+      "logps/rejected": -342.53631591796875,
+      "loss": 0.5707,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": 0.5555700063705444,
+      "rewards/margins": 0.44443902373313904,
+      "rewards/rejected": 0.11113103479146957,
+      "step": 240
+    },
+    {
+      "epoch": 1.39,
+      "grad_norm": 534.3215854444298,
+      "learning_rate": 2.528217270553501e-09,
+      "logits/chosen": -0.9723398089408875,
+      "logits/rejected": -0.8604954481124878,
+      "logps/chosen": -359.05828857421875,
+      "logps/rejected": -328.1695556640625,
+      "loss": 0.5785,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": 0.47152179479599,
+      "rewards/margins": 0.378196656703949,
+      "rewards/rejected": 0.09332513064146042,
+      "step": 250
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 511.2341258250477,
+      "learning_rate": 2.1165983894256645e-09,
+      "logits/chosen": -0.9806705713272095,
+      "logits/rejected": -0.8469393849372864,
+      "logps/chosen": -367.417724609375,
+      "logps/rejected": -341.4565124511719,
+      "loss": 0.5765,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.540740430355072,
+      "rewards/margins": 0.34819895029067993,
+      "rewards/rejected": 0.19254149496555328,
+      "step": 260
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 481.01667031921556,
+      "learning_rate": 1.7324046329316252e-09,
+      "logits/chosen": -1.0429353713989258,
+      "logits/rejected": -0.745051920413971,
+      "logps/chosen": -371.4905090332031,
+      "logps/rejected": -336.188720703125,
+      "loss": 0.5641,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": 0.5642050504684448,
+      "rewards/margins": 0.4432891011238098,
+      "rewards/rejected": 0.12091599404811859,
+      "step": 270
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 476.885773993008,
+      "learning_rate": 1.3792902133797692e-09,
+      "logits/chosen": -1.0373092889785767,
+      "logits/rejected": -0.8002932667732239,
+      "logps/chosen": -375.8576354980469,
+      "logps/rejected": -339.48870849609375,
+      "loss": 0.5606,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": 0.6401156187057495,
+      "rewards/margins": 0.4620915353298187,
+      "rewards/rejected": 0.178024023771286,
+      "step": 280
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 571.353263468623,
+      "learning_rate": 1.060613735735384e-09,
+      "logits/chosen": -0.9055244326591492,
+      "logits/rejected": -0.6923736333847046,
+      "logps/chosen": -358.05694580078125,
+      "logps/rejected": -331.3091125488281,
+      "loss": 0.5778,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": 0.5616304278373718,
+      "rewards/margins": 0.4886326193809509,
+      "rewards/rejected": 0.0729978084564209,
+      "step": 290
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 521.7732616506195,
+      "learning_rate": 7.794062526569734e-10,
+      "logits/chosen": -1.1269298791885376,
+      "logits/rejected": -0.7582497596740723,
+      "logps/chosen": -392.01593017578125,
+      "logps/rejected": -350.402587890625,
+      "loss": 0.5693,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.6667734980583191,
+      "rewards/margins": 0.451296329498291,
+      "rewards/rejected": 0.2154771387577057,
+      "step": 300
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 590.1584228378923,
+      "learning_rate": 5.383424350065824e-10,
+      "logits/chosen": -1.1271040439605713,
+      "logits/rejected": -0.9299942255020142,
+      "logps/chosen": -392.9803161621094,
+      "logps/rejected": -365.2398376464844,
+      "loss": 0.5713,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": 0.710626482963562,
+      "rewards/margins": 0.4400349259376526,
+      "rewards/rejected": 0.27059149742126465,
+      "step": 310
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 547.631467225418,
+      "learning_rate": 3.397151320423647e-10,
+      "logits/chosen": -1.1201112270355225,
+      "logits/rejected": -0.8372467756271362,
+      "logps/chosen": -377.406982421875,
+      "logps/rejected": -341.136474609375,
+      "loss": 0.5715,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": 0.6100808382034302,
+      "rewards/margins": 0.44269323348999023,
+      "rewards/rejected": 0.16738756000995636,
+      "step": 320
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 532.2336174316262,
+      "learning_rate": 1.8541356326100434e-10,
+      "logits/chosen": -1.0350492000579834,
+      "logits/rejected": -0.8257778882980347,
+      "logps/chosen": -369.24896240234375,
+      "logps/rejected": -344.9882507324219,
+      "loss": 0.5662,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": 0.6600748896598816,
+      "rewards/margins": 0.4713415205478668,
+      "rewards/rejected": 0.18873335421085358,
+      "step": 330
+    },
+    {
+      "epoch": 1.89,
+      "grad_norm": 513.0198840165194,
+      "learning_rate": 7.690534931565518e-11,
+      "logits/chosen": -1.0103225708007812,
+      "logits/rejected": -0.7807376384735107,
+      "logps/chosen": -372.0942687988281,
+      "logps/rejected": -339.8793029785156,
+      "loss": 0.5724,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.6672757863998413,
+      "rewards/margins": 0.49458056688308716,
+      "rewards/rejected": 0.1726953089237213,
+      "step": 340
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 524.4542591483576,
+      "learning_rate": 1.5222552920138853e-11,
+      "logits/chosen": -1.0104167461395264,
+      "logits/rejected": -0.8294402360916138,
+      "logps/chosen": -391.20452880859375,
+      "logps/rejected": -356.1845703125,
+      "loss": 0.5659,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.7216753959655762,
+      "rewards/margins": 0.5309287309646606,
+      "rewards/rejected": 0.19074663519859314,
+      "step": 350
+    },
+    {
+      "epoch": 1.99,
+      "step": 358,
       "total_flos": 0.0,
-      "train_loss": 0.7017018035803427,
-      "train_runtime": 5276.0496,
-      "train_samples_per_second": 8.72,
-      "train_steps_per_second": 0.034
+      "train_loss": 0.6178853964672408,
+      "train_runtime": 10771.3374,
+      "train_samples_per_second": 8.543,
+      "train_steps_per_second": 0.033
     }
   ],
   "logging_steps": 10,
-  "max_steps": 179,
+  "max_steps": 358,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 2,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 8,