Model save

README.md

@@ -37,7 +37,7 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 4
+- seed: 2
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.3777129889306127,
-    "train_runtime": 2383.9738,
-    "train_samples": 39494,
-    "train_samples_per_second": 16.566,
-    "train_steps_per_second": 0.13
+    "train_loss": 0.08395375216871263,
+    "train_runtime": 6990.4881,
+    "train_samples": 113028,
+    "train_samples_per_second": 16.169,
+    "train_steps_per_second": 0.126
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e3cd7fb95d1f8e7496d67ae6da91127ac1b68d984262c173d0dbc9c4fdefec2
+oid sha256:b44ac21cf9cdfda1a74fca2e9fd5cf72676aa2a876bfbde6ce2b6be5cd738446
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:51c8d3d5862def84d35eb99192c611bae08c9b3b157b6e6571ab9f2773bfc47d
+oid sha256:833f242819a8426b1cff793b3130f0a72468fcea63b2a2626da5b05c7d7c7bf1
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2533672c62e1b1d85a9920d93d7222ed657bc44d2ceea09665a7bb19e029dc5d
+oid sha256:2af6c791fd1e731817d3206d95755c8d6dcfdc8d1654a6f08896b5156fe0547a
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.3777129889306127,
-    "train_runtime": 2383.9738,
-    "train_samples": 39494,
-    "train_samples_per_second": 16.566,
-    "train_steps_per_second": 0.13
+    "train_loss": 0.08395375216871263,
+    "train_runtime": 6990.4881,
+    "train_samples": 113028,
+    "train_samples_per_second": 16.169,
+    "train_steps_per_second": 0.126
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.9994340690435767,
   "eval_steps": 1000,
-  "global_step": 309,
+  "global_step": 883,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 1.6129032258064514e-08,
-      "logits/chosen": -2.861602783203125,
-      "logits/rejected": -2.8706541061401367,
-      "logps/chosen": -108.31307983398438,
-      "logps/rejected": -122.1865234375,
-      "loss": 0.5468,
+      "learning_rate": 5.617977528089887e-09,
+      "logits/chosen": -2.7943434715270996,
+      "logits/rejected": -2.817823886871338,
+      "logps/chosen": -334.107666015625,
+      "logps/rejected": -197.05621337890625,
+      "loss": 0.353,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -23,437 +23,1249 @@
       "step": 1
     },
     {
-      "epoch": 0.03,
-      "learning_rate": 1.6129032258064515e-07,
-      "logits/chosen": -2.7833333015441895,
-      "logits/rejected": -2.801340103149414,
-      "logps/chosen": -103.0541000366211,
-      "logps/rejected": -108.88811492919922,
-      "loss": 0.5605,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0004288229683879763,
-      "rewards/margins": 0.00026609853375703096,
-      "rewards/rejected": 0.0001627243764232844,
+      "epoch": 0.01,
+      "learning_rate": 5.617977528089887e-08,
+      "logits/chosen": -2.8335936069488525,
+      "logits/rejected": -2.782947540283203,
+      "logps/chosen": -323.8160400390625,
+      "logps/rejected": -189.45599365234375,
+      "loss": 0.3374,
+      "rewards/accuracies": 0.4652777910232544,
+      "rewards/chosen": 0.0004737268900498748,
+      "rewards/margins": 0.000818206463009119,
+      "rewards/rejected": -0.00034447951475158334,
       "step": 10
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 3.225806451612903e-07,
-      "logits/chosen": -2.798549175262451,
-      "logits/rejected": -2.7971768379211426,
-      "logps/chosen": -99.93330383300781,
-      "logps/rejected": -100.25025177001953,
-      "loss": 0.5601,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0004881693166680634,
-      "rewards/margins": 0.0005250017857179046,
-      "rewards/rejected": -3.683247996377759e-05,
+      "epoch": 0.02,
+      "learning_rate": 1.1235955056179774e-07,
+      "logits/chosen": -2.778644561767578,
+      "logits/rejected": -2.7627484798431396,
+      "logps/chosen": -323.32391357421875,
+      "logps/rejected": -168.40980529785156,
+      "loss": 0.3385,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.0011631561210379004,
+      "rewards/margins": 0.0020370460115373135,
+      "rewards/rejected": -0.0008738901233300567,
       "step": 20
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 4.838709677419355e-07,
-      "logits/chosen": -2.816439390182495,
-      "logits/rejected": -2.815070867538452,
-      "logps/chosen": -100.30064392089844,
-      "logps/rejected": -93.25160217285156,
-      "loss": 0.5575,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0009209408308379352,
-      "rewards/margins": 0.001128857722505927,
-      "rewards/rejected": -0.00020791687711607665,
+      "epoch": 0.03,
+      "learning_rate": 1.6853932584269663e-07,
+      "logits/chosen": -2.7863869667053223,
+      "logits/rejected": -2.7318475246429443,
+      "logps/chosen": -305.9979248046875,
+      "logps/rejected": -180.11251831054688,
+      "loss": 0.3398,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.009158268570899963,
+      "rewards/margins": 0.01635834574699402,
+      "rewards/rejected": -0.00720007810741663,
       "step": 30
     },
     {
-      "epoch": 0.13,
-      "learning_rate": 4.987080943856886e-07,
-      "logits/chosen": -2.8334474563598633,
-      "logits/rejected": -2.8349099159240723,
-      "logps/chosen": -99.36518859863281,
-      "logps/rejected": -105.44466400146484,
-      "loss": 0.5683,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.01019326876848936,
-      "rewards/margins": 0.00500866025686264,
-      "rewards/rejected": 0.005184608977288008,
+      "epoch": 0.05,
+      "learning_rate": 2.2471910112359549e-07,
+      "logits/chosen": -2.7181248664855957,
+      "logits/rejected": -2.71014142036438,
+      "logps/chosen": -314.97698974609375,
+      "logps/rejected": -178.74220275878906,
+      "loss": 0.3414,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.033507008105516434,
+      "rewards/margins": 0.06880000978708267,
+      "rewards/rejected": -0.03529299795627594,
       "step": 40
     },
     {
-      "epoch": 0.16,
-      "learning_rate": 4.942593872763566e-07,
-      "logits/chosen": -2.802186965942383,
-      "logits/rejected": -2.7889404296875,
-      "logps/chosen": -88.57516479492188,
-      "logps/rejected": -89.25424194335938,
-      "loss": 0.5728,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.017298558726906776,
-      "rewards/margins": 0.008996127173304558,
-      "rewards/rejected": 0.008302430622279644,
+      "epoch": 0.06,
+      "learning_rate": 2.8089887640449437e-07,
+      "logits/chosen": -2.6565139293670654,
+      "logits/rejected": -2.6516811847686768,
+      "logps/chosen": -341.38330078125,
+      "logps/rejected": -193.6222381591797,
+      "loss": 0.352,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.03725530207157135,
+      "rewards/margins": 0.15392780303955078,
+      "rewards/rejected": -0.11667251586914062,
       "step": 50
     },
     {
-      "epoch": 0.19,
-      "learning_rate": 4.866946677079314e-07,
-      "logits/chosen": -2.828808307647705,
-      "logits/rejected": -2.8317344188690186,
-      "logps/chosen": -85.52920532226562,
-      "logps/rejected": -90.01313781738281,
-      "loss": 0.6048,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.05936245992779732,
-      "rewards/margins": 0.013923106715083122,
-      "rewards/rejected": 0.045439351350069046,
+      "epoch": 0.07,
+      "learning_rate": 3.3707865168539325e-07,
+      "logits/chosen": -2.6464648246765137,
+      "logits/rejected": -2.6180057525634766,
+      "logps/chosen": -292.76690673828125,
+      "logps/rejected": -203.3881378173828,
+      "loss": 0.3395,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06494523584842682,
+      "rewards/margins": 0.14356324076652527,
+      "rewards/rejected": -0.20850849151611328,
       "step": 60
     },
     {
-      "epoch": 0.23,
-      "learning_rate": 4.7611043866720737e-07,
-      "logits/chosen": -2.7826099395751953,
-      "logits/rejected": -2.7798409461975098,
-      "logps/chosen": -91.76524353027344,
-      "logps/rejected": -97.0423583984375,
-      "loss": 0.5521,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.02179296873509884,
-      "rewards/margins": 0.02971211075782776,
-      "rewards/rejected": -0.007919139228761196,
+      "epoch": 0.08,
+      "learning_rate": 3.9325842696629214e-07,
+      "logits/chosen": -2.5645086765289307,
+      "logits/rejected": -2.5475826263427734,
+      "logps/chosen": -325.9661865234375,
+      "logps/rejected": -226.6337890625,
+      "loss": 0.289,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1385149508714676,
+      "rewards/margins": 0.243437722325325,
+      "rewards/rejected": -0.3819526731967926,
       "step": 70
     },
     {
-      "epoch": 0.26,
-      "learning_rate": 4.6264172296714e-07,
-      "logits/chosen": -2.759918689727783,
-      "logits/rejected": -2.766831874847412,
-      "logps/chosen": -101.35821533203125,
-      "logps/rejected": -112.35334777832031,
-      "loss": 0.4972,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.044755686074495316,
-      "rewards/margins": 0.022299829870462418,
-      "rewards/rejected": -0.06705550849437714,
+      "epoch": 0.09,
+      "learning_rate": 4.4943820224719097e-07,
+      "logits/chosen": -2.5285604000091553,
+      "logits/rejected": -2.5033059120178223,
+      "logps/chosen": -408.848388671875,
+      "logps/rejected": -271.00811767578125,
+      "loss": 0.2518,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.13917537033557892,
+      "rewards/margins": 0.5459555387496948,
+      "rewards/rejected": -0.6851309537887573,
       "step": 80
     },
     {
-      "epoch": 0.29,
-      "learning_rate": 4.4646034076333254e-07,
-      "logits/chosen": -2.7790584564208984,
-      "logits/rejected": -2.76528263092041,
-      "logps/chosen": -117.88861083984375,
-      "logps/rejected": -124.70438385009766,
-      "loss": 0.4598,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.11965729296207428,
-      "rewards/margins": 0.023207509890198708,
-      "rewards/rejected": -0.14286477863788605,
+      "epoch": 0.1,
+      "learning_rate": 4.999980431020109e-07,
+      "logits/chosen": -2.5432324409484863,
+      "logits/rejected": -2.517310380935669,
+      "logps/chosen": -390.67340087890625,
+      "logps/rejected": -274.8931579589844,
+      "loss": 0.1964,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.34996914863586426,
+      "rewards/margins": 0.5996032953262329,
+      "rewards/rejected": -0.9495723843574524,
       "step": 90
     },
     {
-      "epoch": 0.32,
-      "learning_rate": 4.27772717647508e-07,
-      "logits/chosen": -2.737382650375366,
-      "logits/rejected": -2.748401641845703,
-      "logps/chosen": -119.01786804199219,
-      "logps/rejected": -131.80020141601562,
-      "loss": 0.4473,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.135704904794693,
-      "rewards/margins": 0.061350900679826736,
-      "rewards/rejected": -0.19705583155155182,
+      "epoch": 0.11,
+      "learning_rate": 4.997632524101301e-07,
+      "logits/chosen": -2.5553176403045654,
+      "logits/rejected": -2.534501552581787,
+      "logps/chosen": -390.3453063964844,
+      "logps/rejected": -304.5262451171875,
+      "loss": 0.1877,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.45027050375938416,
+      "rewards/margins": 0.5258683562278748,
+      "rewards/rejected": -0.9761388897895813,
       "step": 100
     },
     {
-      "epoch": 0.36,
-      "learning_rate": 4.068172512800759e-07,
-      "logits/chosen": -2.703746795654297,
-      "logits/rejected": -2.698978900909424,
-      "logps/chosen": -125.7964096069336,
-      "logps/rejected": -135.111328125,
-      "loss": 0.4126,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.18998780846595764,
-      "rewards/margins": 0.04047433286905289,
-      "rewards/rejected": -0.23046214878559113,
+      "epoch": 0.12,
+      "learning_rate": 4.991375032514749e-07,
+      "logits/chosen": -2.485586643218994,
+      "logits/rejected": -2.4563088417053223,
+      "logps/chosen": -409.3569030761719,
+      "logps/rejected": -326.25390625,
+      "loss": 0.1404,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.779589056968689,
+      "rewards/margins": 0.6373748779296875,
+      "rewards/rejected": -1.4169639348983765,
       "step": 110
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 3.8386127015561377e-07,
-      "logits/chosen": -2.728740930557251,
-      "logits/rejected": -2.703545331954956,
-      "logps/chosen": -120.62841796875,
-      "logps/rejected": -128.31187438964844,
-      "loss": 0.4431,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.15412963926792145,
-      "rewards/margins": 0.07443422079086304,
-      "rewards/rejected": -0.22856386005878448,
+      "epoch": 0.14,
+      "learning_rate": 4.98121775121344e-07,
+      "logits/chosen": -2.5598855018615723,
+      "logits/rejected": -2.524275541305542,
+      "logps/chosen": -468.26318359375,
+      "logps/rejected": -383.34808349609375,
+      "loss": 0.1197,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.8500933647155762,
+      "rewards/margins": 0.8599642515182495,
+      "rewards/rejected": -1.7100576162338257,
       "step": 120
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 3.591976232982355e-07,
-      "logits/chosen": -2.649157762527466,
-      "logits/rejected": -2.644134044647217,
-      "logps/chosen": -100.65953826904297,
-      "logps/rejected": -113.25675964355469,
-      "loss": 0.4739,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.10916206985712051,
-      "rewards/margins": 0.07735568284988403,
-      "rewards/rejected": -0.18651774525642395,
+      "epoch": 0.15,
+      "learning_rate": 4.96717657955441e-07,
+      "logits/chosen": -2.5483665466308594,
+      "logits/rejected": -2.4886937141418457,
+      "logps/chosen": -449.65167236328125,
+      "logps/rejected": -355.64581298828125,
+      "loss": 0.1298,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.7365049719810486,
+      "rewards/margins": 0.864643394947052,
+      "rewards/rejected": -1.6011483669281006,
       "step": 130
     },
     {
-      "epoch": 0.45,
-      "learning_rate": 3.33140944392039e-07,
-      "logits/chosen": -2.664492607116699,
-      "logits/rejected": -2.631776809692383,
-      "logps/chosen": -142.42654418945312,
-      "logps/rejected": -146.1369171142578,
-      "loss": 0.4342,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.24194404482841492,
-      "rewards/margins": 0.08811615407466888,
-      "rewards/rejected": -0.3300601840019226,
+      "epoch": 0.16,
+      "learning_rate": 4.949273496411216e-07,
+      "logits/chosen": -2.4851553440093994,
+      "logits/rejected": -2.45190167427063,
+      "logps/chosen": -422.0953063964844,
+      "logps/rejected": -381.73773193359375,
+      "loss": 0.123,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.8808937072753906,
+      "rewards/margins": 0.8638134002685547,
+      "rewards/rejected": -1.7447071075439453,
       "step": 140
     },
     {
-      "epoch": 0.49,
-      "learning_rate": 3.060236380050519e-07,
-      "logits/chosen": -2.662078380584717,
-      "logits/rejected": -2.667717456817627,
-      "logps/chosen": -139.26730346679688,
-      "logps/rejected": -157.76010131835938,
-      "loss": 0.3688,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.3599378764629364,
-      "rewards/margins": 0.12340853363275528,
-      "rewards/rejected": -0.48334646224975586,
+      "epoch": 0.17,
+      "learning_rate": 4.927536525770046e-07,
+      "logits/chosen": -2.4234509468078613,
+      "logits/rejected": -2.379763126373291,
+      "logps/chosen": -488.06671142578125,
+      "logps/rejected": -402.50836181640625,
+      "loss": 0.0874,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1678707599639893,
+      "rewards/margins": 0.8644298315048218,
+      "rewards/rejected": -2.0323004722595215,
       "step": 150
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 2.781916391103417e-07,
-      "logits/chosen": -2.6194424629211426,
-      "logits/rejected": -2.588695526123047,
-      "logps/chosen": -162.62075805664062,
-      "logps/rejected": -163.50631713867188,
-      "loss": 0.302,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.5319477319717407,
-      "rewards/margins": 0.09751905500888824,
-      "rewards/rejected": -0.6294667720794678,
+      "epoch": 0.18,
+      "learning_rate": 4.901999692863326e-07,
+      "logits/chosen": -2.438655138015747,
+      "logits/rejected": -2.369347095489502,
+      "logps/chosen": -527.1495361328125,
+      "logps/rejected": -413.07891845703125,
+      "loss": 0.0927,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.8715424537658691,
+      "rewards/margins": 1.1340700387954712,
+      "rewards/rejected": -2.0056121349334717,
       "step": 160
     },
     {
-      "epoch": 0.55,
-      "learning_rate": 2.5e-07,
-      "logits/chosen": -2.631842851638794,
-      "logits/rejected": -2.6272921562194824,
-      "logps/chosen": -160.49258422851562,
-      "logps/rejected": -172.91624450683594,
-      "loss": 0.3113,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.5451255440711975,
-      "rewards/margins": 0.0745776817202568,
-      "rewards/rejected": -0.6197031736373901,
+      "epoch": 0.19,
+      "learning_rate": 4.872702970909464e-07,
+      "logits/chosen": -2.32255220413208,
+      "logits/rejected": -2.2418487071990967,
+      "logps/chosen": -489.12957763671875,
+      "logps/rejected": -404.12091064453125,
+      "loss": 0.0753,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1407396793365479,
+      "rewards/margins": 1.0352163314819336,
+      "rewards/rejected": -2.1759560108184814,
       "step": 170
     },
     {
-      "epoch": 0.58,
-      "learning_rate": 2.218083608896583e-07,
-      "logits/chosen": -2.5954980850219727,
-      "logits/rejected": -2.6070828437805176,
-      "logps/chosen": -163.6999969482422,
-      "logps/rejected": -186.43154907226562,
-      "loss": 0.2944,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.5556995868682861,
-      "rewards/margins": 0.18364927172660828,
-      "rewards/rejected": -0.7393488883972168,
+      "epoch": 0.2,
+      "learning_rate": 4.839692218542131e-07,
+      "logits/chosen": -2.1711273193359375,
+      "logits/rejected": -2.1464552879333496,
+      "logps/chosen": -474.91253662109375,
+      "logps/rejected": -437.1839294433594,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8761800527572632,
+      "rewards/margins": 0.8142536282539368,
+      "rewards/rejected": -2.690433979034424,
       "step": 180
     },
     {
-      "epoch": 0.61,
-      "learning_rate": 1.9397636199494806e-07,
-      "logits/chosen": -2.557199716567993,
-      "logits/rejected": -2.5558159351348877,
-      "logps/chosen": -162.37643432617188,
-      "logps/rejected": -181.76522827148438,
-      "loss": 0.2797,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.5982151627540588,
-      "rewards/margins": 0.14243794977664948,
-      "rewards/rejected": -0.7406530976295471,
+      "epoch": 0.22,
+      "learning_rate": 4.803019108026997e-07,
+      "logits/chosen": -2.144486904144287,
+      "logits/rejected": -2.093785285949707,
+      "logps/chosen": -481.8885192871094,
+      "logps/rejected": -427.78875732421875,
+      "loss": 0.0661,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5085182189941406,
+      "rewards/margins": 0.933529257774353,
+      "rewards/rejected": -2.442047595977783,
       "step": 190
     },
     {
-      "epoch": 0.65,
-      "learning_rate": 1.6685905560796098e-07,
-      "logits/chosen": -2.5707640647888184,
-      "logits/rejected": -2.5763020515441895,
-      "logps/chosen": -155.4540252685547,
-      "logps/rejected": -169.99282836914062,
-      "loss": 0.2629,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.6057673692703247,
-      "rewards/margins": 0.10265880823135376,
-      "rewards/rejected": -0.7084261178970337,
+      "epoch": 0.23,
+      "learning_rate": 4.7627410443782887e-07,
+      "logits/chosen": -2.084139347076416,
+      "logits/rejected": -2.0504238605499268,
+      "logps/chosen": -442.94427490234375,
+      "logps/rejected": -425.2293395996094,
+      "loss": 0.0772,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.3089433908462524,
+      "rewards/margins": 1.0439088344573975,
+      "rewards/rejected": -2.3528523445129395,
       "step": 200
     },
     {
-      "epoch": 0.68,
-      "learning_rate": 1.4080237670176453e-07,
-      "logits/chosen": -2.573857545852661,
-      "logits/rejected": -2.563162088394165,
-      "logps/chosen": -159.9458465576172,
-      "logps/rejected": -175.49703979492188,
-      "loss": 0.2737,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.6361742615699768,
-      "rewards/margins": 0.15395954251289368,
-      "rewards/rejected": -0.7901338338851929,
+      "epoch": 0.24,
+      "learning_rate": 4.7189210755018034e-07,
+      "logits/chosen": -2.0335006713867188,
+      "logits/rejected": -1.9534251689910889,
+      "logps/chosen": -519.8855590820312,
+      "logps/rejected": -478.9068298339844,
+      "loss": 0.0603,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.4866015911102295,
+      "rewards/margins": 1.284881353378296,
+      "rewards/rejected": -2.7714829444885254,
       "step": 210
     },
     {
-      "epoch": 0.71,
-      "learning_rate": 1.1613872984438628e-07,
-      "logits/chosen": -2.5723612308502197,
-      "logits/rejected": -2.554568290710449,
-      "logps/chosen": -171.87423706054688,
-      "logps/rejected": -192.27273559570312,
-      "loss": 0.2636,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.7052888870239258,
-      "rewards/margins": 0.1558634340763092,
-      "rewards/rejected": -0.8611523509025574,
+      "epoch": 0.25,
+      "learning_rate": 4.671627793504988e-07,
+      "logits/chosen": -2.10530424118042,
+      "logits/rejected": -2.0339417457580566,
+      "logps/chosen": -527.8480834960938,
+      "logps/rejected": -493.0773010253906,
+      "loss": 0.0601,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.485095500946045,
+      "rewards/margins": 1.381305456161499,
+      "rewards/rejected": -2.866400957107544,
       "step": 220
     },
     {
-      "epoch": 0.74,
-      "learning_rate": 9.318274871992407e-08,
-      "logits/chosen": -2.5391600131988525,
-      "logits/rejected": -2.533364772796631,
-      "logps/chosen": -184.72389221191406,
-      "logps/rejected": -205.41574096679688,
-      "loss": 0.2406,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.7657197713851929,
-      "rewards/margins": 0.16964995861053467,
-      "rewards/rejected": -0.9353697896003723,
+      "epoch": 0.26,
+      "learning_rate": 4.6209352273286095e-07,
+      "logits/chosen": -2.0227694511413574,
+      "logits/rejected": -1.9594017267227173,
+      "logps/chosen": -497.4979553222656,
+      "logps/rejected": -502.72174072265625,
+      "loss": 0.0694,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6086342334747314,
+      "rewards/margins": 1.0697346925735474,
+      "rewards/rejected": -2.6783690452575684,
       "step": 230
     },
     {
-      "epoch": 0.78,
-      "learning_rate": 7.222728235249195e-08,
-      "logits/chosen": -2.4788975715637207,
-      "logits/rejected": -2.4929776191711426,
-      "logps/chosen": -185.28515625,
-      "logps/rejected": -199.28814697265625,
-      "loss": 0.2322,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.7988831996917725,
-      "rewards/margins": 0.1449696272611618,
-      "rewards/rejected": -0.9438527822494507,
+      "epoch": 0.27,
+      "learning_rate": 4.56692272686805e-07,
+      "logits/chosen": -2.0572688579559326,
+      "logits/rejected": -2.007657289505005,
+      "logps/chosen": -469.2767028808594,
+      "logps/rejected": -439.0755920410156,
+      "loss": 0.0704,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.490736961364746,
+      "rewards/margins": 1.089231252670288,
+      "rewards/rejected": -2.579967975616455,
       "step": 240
     },
     {
-      "epoch": 0.81,
-      "learning_rate": 5.353965923666742e-08,
-      "logits/chosen": -2.5501561164855957,
-      "logits/rejected": -2.537123441696167,
-      "logps/chosen": -177.23464965820312,
-      "logps/rejected": -201.24827575683594,
-      "loss": 0.222,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.8073725700378418,
-      "rewards/margins": 0.1908084899187088,
-      "rewards/rejected": -0.9981809854507446,
+      "epoch": 0.28,
+      "learning_rate": 4.5096748387656326e-07,
+      "logits/chosen": -1.9701417684555054,
+      "logits/rejected": -1.8864467144012451,
+      "logps/chosen": -502.487060546875,
+      "logps/rejected": -456.74957275390625,
+      "loss": 0.0586,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.8166824579238892,
+      "rewards/margins": 0.9140488505363464,
+      "rewards/rejected": -2.730731248855591,
       "step": 250
     },
     {
-      "epoch": 0.84,
-      "learning_rate": 3.7358277032860016e-08,
-      "logits/chosen": -2.6011037826538086,
-      "logits/rejected": -2.5914735794067383,
-      "logps/chosen": -193.97116088867188,
-      "logps/rejected": -208.1275177001953,
-      "loss": 0.2522,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.7856981158256531,
-      "rewards/margins": 0.17516431212425232,
-      "rewards/rejected": -0.9608623385429382,
+      "epoch": 0.29,
+      "learning_rate": 4.4492811740683877e-07,
+      "logits/chosen": -1.9581239223480225,
+      "logits/rejected": -1.8596645593643188,
+      "logps/chosen": -496.4566345214844,
+      "logps/rejected": -464.6380310058594,
+      "loss": 0.0574,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9547615051269531,
+      "rewards/margins": 0.8960165977478027,
+      "rewards/rejected": -2.850778102874756,
       "step": 260
     },
     {
-      "epoch": 0.87,
-      "learning_rate": 2.3889561332792657e-08,
-      "logits/chosen": -2.5599663257598877,
-      "logits/rejected": -2.547809362411499,
-      "logps/chosen": -166.72702026367188,
-      "logps/rejected": -184.97462463378906,
-      "loss": 0.2529,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.6648862361907959,
-      "rewards/margins": 0.1466326266527176,
-      "rewards/rejected": -0.8115188479423523,
+      "epoch": 0.31,
+      "learning_rate": 4.3858362679584354e-07,
+      "logits/chosen": -2.0671660900115967,
+      "logits/rejected": -1.9386441707611084,
+      "logps/chosen": -496.42669677734375,
+      "logps/rejected": -445.92218017578125,
+      "loss": 0.0564,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.5112464427947998,
+      "rewards/margins": 1.217174768447876,
+      "rewards/rejected": -2.728421688079834,
       "step": 270
     },
     {
-      "epoch": 0.91,
-      "learning_rate": 1.3305332292068705e-08,
-      "logits/chosen": -2.482130527496338,
-      "logits/rejected": -2.4812235832214355,
-      "logps/chosen": -186.01287841796875,
-      "logps/rejected": -200.09637451171875,
-      "loss": 0.2508,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.7596431970596313,
-      "rewards/margins": 0.09221551567316055,
-      "rewards/rejected": -0.8518587350845337,
+      "epoch": 0.32,
+      "learning_rate": 4.3194394317755245e-07,
+      "logits/chosen": -2.0471348762512207,
+      "logits/rejected": -1.9359214305877686,
+      "logps/chosen": -517.4642333984375,
+      "logps/rejected": -449.72607421875,
+      "loss": 0.059,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.718505859375,
+      "rewards/margins": 1.0684489011764526,
+      "rewards/rejected": -2.786954641342163,
       "step": 280
     },
     {
-      "epoch": 0.94,
-      "learning_rate": 5.740612723643401e-09,
-      "logits/chosen": -2.5973570346832275,
-      "logits/rejected": -2.5847537517547607,
-      "logps/chosen": -185.65423583984375,
-      "logps/rejected": -196.46435546875,
-      "loss": 0.244,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.770811915397644,
-      "rewards/margins": 0.1569802314043045,
-      "rewards/rejected": -0.9277920722961426,
+      "epoch": 0.33,
+      "learning_rate": 4.2501945975633914e-07,
+      "logits/chosen": -2.1385433673858643,
+      "logits/rejected": -2.0321764945983887,
+      "logps/chosen": -524.6473388671875,
+      "logps/rejected": -449.234375,
+      "loss": 0.0705,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.6018638610839844,
+      "rewards/margins": 1.1273199319839478,
+      "rewards/rejected": -2.7291836738586426,
       "step": 290
     },
     {
-      "epoch": 0.97,
-      "learning_rate": 1.2919056143113061e-09,
-      "logits/chosen": -2.5530037879943848,
-      "logits/rejected": -2.544957399368286,
-      "logps/chosen": -184.2729034423828,
-      "logps/rejected": -192.07211303710938,
-      "loss": 0.2475,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.7763879895210266,
-      "rewards/margins": 0.143040731549263,
-      "rewards/rejected": -0.919428825378418,
+      "epoch": 0.34,
+      "learning_rate": 4.1782101553832405e-07,
+      "logits/chosen": -2.0962741374969482,
+      "logits/rejected": -2.0118086338043213,
+      "logps/chosen": -513.7835693359375,
+      "logps/rejected": -460.288330078125,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.9815715551376343,
+      "rewards/margins": 0.8931058645248413,
+      "rewards/rejected": -2.8746774196624756,
       "step": 300
     },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.103598783649029e-07,
+      "logits/chosen": -2.0219855308532715,
+      "logits/rejected": -1.9064286947250366,
+      "logps/chosen": -591.9143676757812,
+      "logps/rejected": -519.5982666015625,
+      "loss": 0.0463,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.2725110054016113,
+      "rewards/margins": 1.1850159168243408,
+      "rewards/rejected": -3.457526683807373,
+      "step": 310
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.026477272750119e-07,
+      "logits/chosen": -1.9796558618545532,
+      "logits/rejected": -1.875626802444458,
+      "logps/chosen": -566.9561157226562,
+      "logps/rejected": -512.3988037109375,
+      "loss": 0.0417,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.3572134971618652,
+      "rewards/margins": 0.9829473495483398,
+      "rewards/rejected": -3.340160846710205,
+      "step": 320
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9469663422373864e-07,
+      "logits/chosen": -2.0174968242645264,
+      "logits/rejected": -1.9458332061767578,
+      "logps/chosen": -552.6922607421875,
+      "logps/rejected": -521.9205322265625,
+      "loss": 0.0479,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.164801836013794,
+      "rewards/margins": 1.167511224746704,
+      "rewards/rejected": -3.332312822341919,
+      "step": 330
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.865190451858954e-07,
+      "logits/chosen": -2.0744147300720215,
+      "logits/rejected": -1.9883348941802979,
+      "logps/chosen": -588.4783935546875,
+      "logps/rejected": -547.9783325195312,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0789949893951416,
+      "rewards/margins": 1.3585710525512695,
+      "rewards/rejected": -3.437565565109253,
+      "step": 340
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.781277606741327e-07,
+      "logits/chosen": -2.0338993072509766,
+      "logits/rejected": -1.9896167516708374,
+      "logps/chosen": -492.18743896484375,
+      "logps/rejected": -475.5653381347656,
+      "loss": 0.0634,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.845071792602539,
+      "rewards/margins": 1.0096709728240967,
+      "rewards/rejected": -2.854743003845215,
+      "step": 350
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6953591570208996e-07,
+      "logits/chosen": -2.158510684967041,
+      "logits/rejected": -2.0470757484436035,
+      "logps/chosen": -531.6489868164062,
+      "logps/rejected": -506.73388671875,
+      "loss": 0.0646,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.6158654689788818,
+      "rewards/margins": 1.420132040977478,
+      "rewards/rejected": -3.0359978675842285,
+      "step": 360
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.607569592239452e-07,
+      "logits/chosen": -2.0580546855926514,
+      "logits/rejected": -1.9891626834869385,
+      "logps/chosen": -569.1202392578125,
+      "logps/rejected": -509.3130798339844,
+      "loss": 0.0481,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.8223598003387451,
+      "rewards/margins": 1.3710863590240479,
+      "rewards/rejected": -3.193446636199951,
+      "step": 370
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.518046330825494e-07,
+      "logits/chosen": -2.0791687965393066,
+      "logits/rejected": -1.981591820716858,
+      "logps/chosen": -581.3887329101562,
+      "logps/rejected": -514.6013793945312,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.0335278511047363,
+      "rewards/margins": 1.2213140726089478,
+      "rewards/rejected": -3.2548420429229736,
+      "step": 380
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4269295049909713e-07,
+      "logits/chosen": -2.034996747970581,
+      "logits/rejected": -1.9570029973983765,
+      "logps/chosen": -493.1822204589844,
+      "logps/rejected": -479.6783752441406,
+      "loss": 0.06,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.9078280925750732,
+      "rewards/margins": 1.182108759880066,
+      "rewards/rejected": -3.0899367332458496,
+      "step": 390
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3343617413800453e-07,
+      "logits/chosen": -2.1511921882629395,
+      "logits/rejected": -2.0434978008270264,
+      "logps/chosen": -550.9173583984375,
+      "logps/rejected": -495.8582458496094,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.865997552871704,
+      "rewards/margins": 1.4241960048675537,
+      "rewards/rejected": -3.2901930809020996,
+      "step": 400
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2404879378132893e-07,
+      "logits/chosen": -2.0529592037200928,
+      "logits/rejected": -1.988389253616333,
+      "logps/chosen": -507.41534423828125,
+      "logps/rejected": -502.86517333984375,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.0051426887512207,
+      "rewards/margins": 1.3514044284820557,
+      "rewards/rejected": -3.3565471172332764,
+      "step": 410
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1454550364767894e-07,
+      "logits/chosen": -2.0849246978759766,
+      "logits/rejected": -1.9924323558807373,
+      "logps/chosen": -562.3327026367188,
+      "logps/rejected": -560.0533447265625,
+      "loss": 0.0558,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.299450397491455,
+      "rewards/margins": 1.2988755702972412,
+      "rewards/rejected": -3.598325729370117,
+      "step": 420
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.049411793911154e-07,
+      "logits/chosen": -2.082733631134033,
+      "logits/rejected": -2.014369487762451,
+      "logps/chosen": -515.5167236328125,
+      "logps/rejected": -504.70538330078125,
+      "loss": 0.0609,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.9115734100341797,
+      "rewards/margins": 1.2012805938720703,
+      "rewards/rejected": -3.112853765487671,
+      "step": 430
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9525085481604914e-07,
+      "logits/chosen": -2.0253331661224365,
+      "logits/rejected": -1.9336645603179932,
+      "logps/chosen": -499.57501220703125,
+      "logps/rejected": -480.38873291015625,
+      "loss": 0.0659,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.7531890869140625,
+      "rewards/margins": 1.1956068277359009,
+      "rewards/rejected": -2.948795795440674,
+      "step": 440
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.854896983445833e-07,
+      "logits/chosen": -1.9761031866073608,
+      "logits/rejected": -1.9000879526138306,
+      "logps/chosen": -585.724853515625,
+      "logps/rejected": -522.663818359375,
+      "loss": 0.0585,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.1158130168914795,
+      "rewards/margins": 1.2392785549163818,
+      "rewards/rejected": -3.3550915718078613,
+      "step": 450
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7567298927313654e-07,
+      "logits/chosen": -2.024075984954834,
+      "logits/rejected": -1.994297742843628,
+      "logps/chosen": -490.19671630859375,
+      "logps/rejected": -488.1590881347656,
+      "loss": 0.0698,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.692265510559082,
+      "rewards/margins": 1.1264708042144775,
+      "rewards/rejected": -2.8187363147735596,
+      "step": 460
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.658160938555123e-07,
+      "logits/chosen": -2.0123002529144287,
+      "logits/rejected": -1.9370654821395874,
+      "logps/chosen": -550.9243774414062,
+      "logps/rejected": -521.9213256835938,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.8550266027450562,
+      "rewards/margins": 1.2366045713424683,
+      "rewards/rejected": -3.0916314125061035,
+      "step": 470
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.559344412498532e-07,
+      "logits/chosen": -1.8818950653076172,
+      "logits/rejected": -1.8040504455566406,
+      "logps/chosen": -594.8109130859375,
+      "logps/rejected": -539.5858154296875,
+      "loss": 0.0436,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.505542755126953,
+      "rewards/margins": 0.9630386233329773,
+      "rewards/rejected": -3.468581438064575,
+      "step": 480
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.460434993671294e-07,
+      "logits/chosen": -1.8864244222640991,
+      "logits/rejected": -1.827561378479004,
+      "logps/chosen": -592.8574829101562,
+      "logps/rejected": -558.6998291015625,
+      "loss": 0.0362,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.819347858428955,
+      "rewards/margins": 1.06614089012146,
+      "rewards/rejected": -3.885488510131836,
+      "step": 490
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.361587506589672e-07,
+      "logits/chosen": -1.9717912673950195,
+      "logits/rejected": -1.8653414249420166,
+      "logps/chosen": -641.2883911132812,
+      "logps/rejected": -575.3276977539062,
+      "loss": 0.0414,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.6623339653015137,
+      "rewards/margins": 1.177682638168335,
+      "rewards/rejected": -3.8400166034698486,
+      "step": 500
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2629566788271613e-07,
+      "logits/chosen": -1.9816091060638428,
+      "logits/rejected": -1.8679695129394531,
+      "logps/chosen": -567.7015991210938,
+      "logps/rejected": -527.6968383789062,
+      "loss": 0.0435,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.42421555519104,
+      "rewards/margins": 1.159529447555542,
+      "rewards/rejected": -3.583745241165161,
+      "step": 510
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1646968988169135e-07,
+      "logits/chosen": -2.013672113418579,
+      "logits/rejected": -1.9170252084732056,
+      "logps/chosen": -607.8392333984375,
+      "logps/rejected": -590.5955810546875,
+      "loss": 0.039,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.4571421146392822,
+      "rewards/margins": 1.2694391012191772,
+      "rewards/rejected": -3.72658109664917,
+      "step": 520
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0669619741850232e-07,
+      "logits/chosen": -2.058300733566284,
+      "logits/rejected": -1.970088005065918,
+      "logps/chosen": -582.6038818359375,
+      "logps/rejected": -512.1895141601562,
+      "loss": 0.044,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.335970401763916,
+      "rewards/margins": 1.0607550144195557,
+      "rewards/rejected": -3.3967254161834717,
+      "step": 530
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9699048909929518e-07,
+      "logits/chosen": -2.0398476123809814,
+      "logits/rejected": -1.943453073501587,
+      "logps/chosen": -545.3948974609375,
+      "logps/rejected": -499.24444580078125,
+      "loss": 0.0497,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.121716022491455,
+      "rewards/margins": 0.9951685070991516,
+      "rewards/rejected": -3.116884231567383,
+      "step": 540
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8736775742659732e-07,
+      "logits/chosen": -2.01509952545166,
+      "logits/rejected": -1.9400924444198608,
+      "logps/chosen": -526.3953247070312,
+      "logps/rejected": -530.0167236328125,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.0396695137023926,
+      "rewards/margins": 1.3093147277832031,
+      "rewards/rejected": -3.3489837646484375,
+      "step": 550
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7784306501824616e-07,
+      "logits/chosen": -2.055781126022339,
+      "logits/rejected": -1.9646365642547607,
+      "logps/chosen": -597.9435424804688,
+      "logps/rejected": -532.0874633789062,
+      "loss": 0.0454,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.338007688522339,
+      "rewards/margins": 1.085502028465271,
+      "rewards/rejected": -3.4235095977783203,
+      "step": 560
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6843132102963025e-07,
+      "logits/chosen": -1.9799835681915283,
+      "logits/rejected": -1.9040464162826538,
+      "logps/chosen": -601.7384643554688,
+      "logps/rejected": -558.1231689453125,
+      "loss": 0.0397,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.291151523590088,
+      "rewards/margins": 1.366019368171692,
+      "rewards/rejected": -3.657170534133911,
+      "step": 570
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.591472578161458e-07,
+      "logits/chosen": -2.005979299545288,
+      "logits/rejected": -1.9207054376602173,
+      "logps/chosen": -568.8519287109375,
+      "logps/rejected": -540.73876953125,
+      "loss": 0.0424,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.294837474822998,
+      "rewards/margins": 1.2771284580230713,
+      "rewards/rejected": -3.5719656944274902,
+      "step": 580
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5000540787240274e-07,
+      "logits/chosen": -2.0050690174102783,
+      "logits/rejected": -1.9278829097747803,
+      "logps/chosen": -566.6500244140625,
+      "logps/rejected": -545.9078369140625,
+      "loss": 0.045,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.4012954235076904,
+      "rewards/margins": 1.2149646282196045,
+      "rewards/rejected": -3.616260051727295,
+      "step": 590
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.410200810842749e-07,
+      "logits/chosen": -2.0124361515045166,
+      "logits/rejected": -1.9107856750488281,
+      "logps/chosen": -535.6881713867188,
+      "logps/rejected": -506.49676513671875,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.0316760540008545,
+      "rewards/margins": 1.1934245824813843,
+      "rewards/rejected": -3.2251009941101074,
+      "step": 600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.322053423294041e-07,
+      "logits/chosen": -2.0051183700561523,
+      "logits/rejected": -1.9159631729125977,
+      "logps/chosen": -529.7169799804688,
+      "logps/rejected": -532.310546875,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.024324655532837,
+      "rewards/margins": 1.4423547983169556,
+      "rewards/rejected": -3.466679334640503,
+      "step": 610
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2357498946121905e-07,
+      "logits/chosen": -2.0541529655456543,
+      "logits/rejected": -1.964544653892517,
+      "logps/chosen": -566.3477783203125,
+      "logps/rejected": -529.6947021484375,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.169382095336914,
+      "rewards/margins": 1.2838075160980225,
+      "rewards/rejected": -3.4531898498535156,
+      "step": 620
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1514253171093161e-07,
+      "logits/chosen": -2.0569446086883545,
+      "logits/rejected": -1.9314994812011719,
+      "logps/chosen": -529.5950927734375,
+      "logps/rejected": -520.3991088867188,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.9739675521850586,
+      "rewards/margins": 1.362341284751892,
+      "rewards/rejected": -3.3363089561462402,
+      "step": 630
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0692116854131883e-07,
+      "logits/chosen": -2.0125861167907715,
+      "logits/rejected": -1.9420621395111084,
+      "logps/chosen": -524.7649536132812,
+      "logps/rejected": -528.4769287109375,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.092562675476074,
+      "rewards/margins": 1.1686738729476929,
+      "rewards/rejected": -3.2612366676330566,
+      "step": 640
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.89237689853889e-08,
+      "logits/chosen": -2.0095937252044678,
+      "logits/rejected": -1.942928671836853,
+      "logps/chosen": -512.7869873046875,
+      "logps/rejected": -487.248046875,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.9598802328109741,
+      "rewards/margins": 1.2647455930709839,
+      "rewards/rejected": -3.2246253490448,
+      "step": 650
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.11628515022765e-08,
+      "logits/chosen": -2.037865400314331,
+      "logits/rejected": -1.9250373840332031,
+      "logps/chosen": -533.2716674804688,
+      "logps/rejected": -528.3024291992188,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.9517757892608643,
+      "rewards/margins": 1.4488084316253662,
+      "rewards/rejected": -3.4005839824676514,
+      "step": 660
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.365056438189486e-08,
+      "logits/chosen": -2.0413284301757812,
+      "logits/rejected": -1.9342155456542969,
+      "logps/chosen": -561.7138671875,
+      "logps/rejected": -549.7591552734375,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.183668851852417,
+      "rewards/margins": 1.2829086780548096,
+      "rewards/rejected": -3.4665775299072266,
+      "step": 670
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.639866672902101e-08,
+      "logits/chosen": -2.0364794731140137,
+      "logits/rejected": -1.9118995666503906,
+      "logps/chosen": -569.7603149414062,
+      "logps/rejected": -554.3267822265625,
+      "loss": 0.0559,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.052215099334717,
+      "rewards/margins": 1.4867579936981201,
+      "rewards/rejected": -3.538973569869995,
+      "step": 680
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.941851005657851e-08,
+      "logits/chosen": -2.0689234733581543,
+      "logits/rejected": -1.9631000757217407,
+      "logps/chosen": -537.5634765625,
+      "logps/rejected": -517.525146484375,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.1864445209503174,
+      "rewards/margins": 1.1380784511566162,
+      "rewards/rejected": -3.3245227336883545,
+      "step": 690
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.272102051693051e-08,
+      "logits/chosen": -2.0574021339416504,
+      "logits/rejected": -1.998944640159607,
+      "logps/chosen": -598.1875,
+      "logps/rejected": -531.4752807617188,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.1735377311706543,
+      "rewards/margins": 1.160771369934082,
+      "rewards/rejected": -3.3343091011047363,
+      "step": 700
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.6316681798995844e-08,
+      "logits/chosen": -2.01491379737854,
+      "logits/rejected": -1.9562265872955322,
+      "logps/chosen": -530.241455078125,
+      "logps/rejected": -535.0760498046875,
+      "loss": 0.0428,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.1387624740600586,
+      "rewards/margins": 1.4437265396118164,
+      "rewards/rejected": -3.582489013671875,
+      "step": 710
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.0215518717961256e-08,
+      "logits/chosen": -1.996011734008789,
+      "logits/rejected": -1.9084312915802002,
+      "logps/chosen": -559.4378662109375,
+      "logps/rejected": -546.0494995117188,
+      "loss": 0.0449,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.1538968086242676,
+      "rewards/margins": 1.5510085821151733,
+      "rewards/rejected": -3.7049052715301514,
+      "step": 720
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.4427081523275925e-08,
+      "logits/chosen": -1.9798328876495361,
+      "logits/rejected": -1.915225625038147,
+      "logps/chosen": -537.6295166015625,
+      "logps/rejected": -551.8641357421875,
+      "loss": 0.0414,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.211373805999756,
+      "rewards/margins": 1.3579763174057007,
+      "rewards/rejected": -3.569350004196167,
+      "step": 730
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.896043094949061e-08,
+      "logits/chosen": -2.0534861087799072,
+      "logits/rejected": -1.947218894958496,
+      "logps/chosen": -572.7033081054688,
+      "logps/rejected": -552.8514404296875,
+      "loss": 0.0491,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.2802767753601074,
+      "rewards/margins": 1.3190699815750122,
+      "rewards/rejected": -3.599346876144409,
+      "step": 740
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3824124033343557e-08,
+      "logits/chosen": -2.0076358318328857,
+      "logits/rejected": -1.946637511253357,
+      "logps/chosen": -583.5534057617188,
+      "logps/rejected": -575.9400024414062,
+      "loss": 0.0441,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.3145105838775635,
+      "rewards/margins": 1.4341070652008057,
+      "rewards/rejected": -3.748617649078369,
+      "step": 750
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9026200719291904e-08,
+      "logits/chosen": -2.0077717304229736,
+      "logits/rejected": -1.9336059093475342,
+      "logps/chosen": -549.6090698242188,
+      "logps/rejected": -544.0575561523438,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.3540844917297363,
+      "rewards/margins": 1.253685712814331,
+      "rewards/rejected": -3.6077704429626465,
+      "step": 760
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4574171274456433e-08,
+      "logits/chosen": -2.040750503540039,
+      "logits/rejected": -1.956707239151001,
+      "logps/chosen": -548.0321655273438,
+      "logps/rejected": -525.2894287109375,
+      "loss": 0.043,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.166091203689575,
+      "rewards/margins": 1.3392621278762817,
+      "rewards/rejected": -3.5053532123565674,
+      "step": 770
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.047500453267881e-08,
+      "logits/chosen": -2.015049457550049,
+      "logits/rejected": -1.9243285655975342,
+      "logps/chosen": -562.6207275390625,
+      "logps/rejected": -560.2086181640625,
+      "loss": 0.0461,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.27638578414917,
+      "rewards/margins": 1.389906644821167,
+      "rewards/rejected": -3.666292190551758,
+      "step": 780
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.673511698609292e-08,
+      "logits/chosen": -1.9878380298614502,
+      "logits/rejected": -1.9071290493011475,
+      "logps/chosen": -591.1964721679688,
+      "logps/rejected": -575.9303588867188,
+      "loss": 0.0436,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.2814955711364746,
+      "rewards/margins": 1.4738110303878784,
+      "rewards/rejected": -3.7553062438964844,
+      "step": 790
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.3360362741285769e-08,
+      "logits/chosen": -1.998380422592163,
+      "logits/rejected": -1.910592794418335,
+      "logps/chosen": -541.6677856445312,
+      "logps/rejected": -533.7774047851562,
+      "loss": 0.0425,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.2717394828796387,
+      "rewards/margins": 1.2483633756637573,
+      "rewards/rejected": -3.5201034545898438,
+      "step": 800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0356024355769433e-08,
+      "logits/chosen": -2.0231940746307373,
+      "logits/rejected": -1.9754924774169922,
+      "logps/chosen": -576.0597534179688,
+      "logps/rejected": -548.6602172851562,
+      "loss": 0.0447,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.2617714405059814,
+      "rewards/margins": 1.3260424137115479,
+      "rewards/rejected": -3.58781361579895,
+      "step": 810
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.726804569108597e-09,
+      "logits/chosen": -2.033327102661133,
+      "logits/rejected": -1.9357963800430298,
+      "logps/chosen": -587.4612426757812,
+      "logps/rejected": -580.3048095703125,
+      "loss": 0.0476,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.3224897384643555,
+      "rewards/margins": 1.3850138187408447,
+      "rewards/rejected": -3.7075035572052,
+      "step": 820
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.476818941645561e-09,
+      "logits/chosen": -2.0493969917297363,
+      "logits/rejected": -1.9231374263763428,
+      "logps/chosen": -621.8226318359375,
+      "logps/rejected": -563.610595703125,
+      "loss": 0.0437,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.3366944789886475,
+      "rewards/margins": 1.3244359493255615,
+      "rewards/rejected": -3.661130428314209,
+      "step": 830
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.609589412347347e-09,
+      "logits/chosen": -1.9982448816299438,
+      "logits/rejected": -1.920509696006775,
+      "logps/chosen": -563.5140380859375,
+      "logps/rejected": -574.1109619140625,
+      "loss": 0.0384,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.203871488571167,
+      "rewards/margins": 1.5941288471221924,
+      "rewards/rejected": -3.7980003356933594,
+      "step": 840
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1280387858572667e-09,
+      "logits/chosen": -2.0206615924835205,
+      "logits/rejected": -1.9191792011260986,
+      "logps/chosen": -529.5126342773438,
+      "logps/rejected": -513.9107666015625,
+      "loss": 0.0425,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.1540675163269043,
+      "rewards/margins": 1.2586270570755005,
+      "rewards/rejected": -3.4126949310302734,
+      "step": 850
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.03448615738172e-09,
+      "logits/chosen": -1.9819049835205078,
+      "logits/rejected": -1.8794002532958984,
+      "logps/chosen": -554.608642578125,
+      "logps/rejected": -551.7212524414062,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.191262722015381,
+      "rewards/margins": 1.487153172492981,
+      "rewards/rejected": -3.6784160137176514,
+      "step": 860
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.3064328257259575e-10,
+      "logits/chosen": -2.011251211166382,
+      "logits/rejected": -1.9388446807861328,
+      "logps/chosen": -545.1489868164062,
+      "logps/rejected": -542.4119873046875,
+      "loss": 0.0461,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.230153799057007,
+      "rewards/margins": 1.3988468647003174,
+      "rewards/rejected": -3.6290009021759033,
+      "step": 870
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.7611898088715216e-11,
+      "logits/chosen": -1.9472382068634033,
+      "logits/rejected": -1.8980674743652344,
+      "logps/chosen": -579.8809814453125,
+      "logps/rejected": -560.6386108398438,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.2138962745666504,
+      "rewards/margins": 1.272986650466919,
+      "rewards/rejected": -3.4868831634521484,
+      "step": 880
+    },
     {
       "epoch": 1.0,
-      "step": 309,
+      "step": 883,
       "total_flos": 0.0,
-      "train_loss": 0.3777129889306127,
-      "train_runtime": 2383.9738,
-      "train_samples_per_second": 16.566,
-      "train_steps_per_second": 0.13
+      "train_loss": 0.08395375216871263,
+      "train_runtime": 6990.4881,
+      "train_samples_per_second": 16.169,
+      "train_steps_per_second": 0.126
     }
   ],
   "logging_steps": 10,
-  "max_steps": 309,
+  "max_steps": 883,
   "num_train_epochs": 1,
   "save_steps": 1000,
   "total_flos": 0.0,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69f452db3f4d6246f461dfea40f35365fa560c9e1d85d96dab21f005c15bec92
+oid sha256:8842f3333f3a8ac784afd1829b17d717e3c9fd1d2f4fe455962584f6e4e1556d
 size 5944