Model save

README.md

@@ -32,10 +32,10 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 2e-06
+- learning_rate: 1e-06
 - train_batch_size: 2
 - eval_batch_size: 8
-- seed: 5
+- seed: 2
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 8

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.21128464219435839,
-    "train_runtime": 10587.5728,
+    "train_loss": 0.30386436640376774,
+    "train_runtime": 10428.2681,
     "train_samples": 45548,
-    "train_samples_per_second": 8.604,
-    "train_steps_per_second": 0.067
+    "train_samples_per_second": 8.735,
+    "train_steps_per_second": 0.068
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8edcf20d9c2d026fabec9dd725ddcb99eb9468d119764f360eb50eb30cbd68da
+oid sha256:fad56123ab40af80b65ee5b63fa2134010a944b41f9ae923d9221170956f7649
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9bf81822232bc24fae74426ebfdabfed75c1be44be206a3ef5b25a37f3bff5e3
+oid sha256:e97d5c4777c113244429fccde8dc290fc576953853257deac3d98bb33f36025d
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1597fd9f46a0099cfde0de1c18d44e1b23c4e2dfa75bb389ea71848dbbbb6af
+oid sha256:94e1fd4e76c84221a4e3f2fffd9f145b436a33225719831f05393709b2c48b8e
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5e65c84cef0798a2d14db45a99b308a303d9843d5be5de1f10c03f8530dfe2d
+oid sha256:4bd1298e2d78e75607e9514d3b106ce709ec2d57ea16b46c0f79bf9fc4a696d7
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.21128464219435839,
-    "train_runtime": 10587.5728,
+    "train_loss": 0.30386436640376774,
+    "train_runtime": 10428.2681,
     "train_samples": 45548,
-    "train_samples_per_second": 8.604,
-    "train_steps_per_second": 0.067
+    "train_samples_per_second": 8.735,
+    "train_steps_per_second": 0.068
 }

trainer_state.json

@@ -10,1006 +10,1006 @@
   "log_history": [
     {
       "epoch": 0.03,
-      "learning_rate": 2.8169014084507043e-07,
-      "logits/chosen": -0.01849319413304329,
-      "logits/rejected": 0.04447399824857712,
-      "logps/chosen": -322.30413818359375,
-      "logps/rejected": -218.52719116210938,
-      "loss": 0.5192,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": 0.0008185860933735967,
-      "rewards/margins": 0.0013399553718045354,
-      "rewards/rejected": -0.0005213693948462605,
+      "learning_rate": 1.4084507042253522e-07,
+      "logits/chosen": -0.023548124358057976,
+      "logits/rejected": 0.04590621590614319,
+      "logps/chosen": -317.1582336425781,
+      "logps/rejected": -207.426513671875,
+      "loss": 0.5133,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.000816057319752872,
+      "rewards/margins": -0.0010325554758310318,
+      "rewards/rejected": 0.00021649803966283798,
       "step": 10
     },
     {
       "epoch": 0.06,
-      "learning_rate": 5.633802816901409e-07,
-      "logits/chosen": -0.04029911756515503,
-      "logits/rejected": -0.037409596145153046,
-      "logps/chosen": -334.2061767578125,
-      "logps/rejected": -200.04428100585938,
-      "loss": 0.5024,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": 0.001267000799998641,
-      "rewards/margins": 0.0049448576755821705,
-      "rewards/rejected": -0.0036778573412448168,
+      "learning_rate": 2.8169014084507043e-07,
+      "logits/chosen": -0.04698050394654274,
+      "logits/rejected": -0.011301965452730656,
+      "logps/chosen": -294.093994140625,
+      "logps/rejected": -201.62612915039062,
+      "loss": 0.5062,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.00026954649365507066,
+      "rewards/margins": 0.002362610073760152,
+      "rewards/rejected": -0.002632156480103731,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "learning_rate": 8.450704225352112e-07,
-      "logits/chosen": -0.09712442010641098,
-      "logits/rejected": -0.016413463279604912,
-      "logps/chosen": -428.060302734375,
-      "logps/rejected": -255.72323608398438,
-      "loss": 0.5169,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.025075193494558334,
-      "rewards/margins": 0.06105799600481987,
-      "rewards/rejected": -0.035982806235551834,
+      "learning_rate": 4.225352112676056e-07,
+      "logits/chosen": -0.04627276584506035,
+      "logits/rejected": 0.00557746272534132,
+      "logps/chosen": -361.0042724609375,
+      "logps/rejected": -249.05715942382812,
+      "loss": 0.5083,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.00011544860899448395,
+      "rewards/margins": 0.010346856899559498,
+      "rewards/rejected": -0.01023140735924244,
       "step": 30
     },
     {
       "epoch": 0.11,
-      "learning_rate": 1.1267605633802817e-06,
-      "logits/chosen": 0.01622619666159153,
-      "logits/rejected": 0.05923817679286003,
-      "logps/chosen": -367.3197326660156,
-      "logps/rejected": -274.74176025390625,
-      "loss": 0.531,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.02706316113471985,
-      "rewards/margins": 0.09234372526407242,
-      "rewards/rejected": -0.11940689384937286,
+      "learning_rate": 5.633802816901409e-07,
+      "logits/chosen": -0.052781809121370316,
+      "logits/rejected": -0.003899569856002927,
+      "logps/chosen": -290.2967834472656,
+      "logps/rejected": -194.12432861328125,
+      "loss": 0.5245,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.004079835955053568,
+      "rewards/margins": 0.03794458881020546,
+      "rewards/rejected": -0.042024414986371994,
       "step": 40
     },
     {
       "epoch": 0.14,
-      "learning_rate": 1.408450704225352e-06,
-      "logits/chosen": 0.048659004271030426,
-      "logits/rejected": 0.06660661846399307,
-      "logps/chosen": -283.6951599121094,
-      "logps/rejected": -219.31021118164062,
-      "loss": 0.5435,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.0874737948179245,
-      "rewards/margins": 0.12040810286998749,
-      "rewards/rejected": -0.207881897687912,
+      "learning_rate": 7.04225352112676e-07,
+      "logits/chosen": 0.019576847553253174,
+      "logits/rejected": 0.06315603107213974,
+      "logps/chosen": -370.23211669921875,
+      "logps/rejected": -221.93215942382812,
+      "loss": 0.5437,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.02125033549964428,
+      "rewards/margins": 0.08855441212654114,
+      "rewards/rejected": -0.10980476438999176,
       "step": 50
     },
     {
       "epoch": 0.17,
-      "learning_rate": 1.6901408450704225e-06,
-      "logits/chosen": 0.08613400161266327,
-      "logits/rejected": 0.1546694040298462,
-      "logps/chosen": -362.0628662109375,
-      "logps/rejected": -212.6298828125,
-      "loss": 0.5299,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.0036375909112393856,
-      "rewards/margins": 0.32788988947868347,
-      "rewards/rejected": -0.32425227761268616,
+      "learning_rate": 8.450704225352112e-07,
+      "logits/chosen": -0.042748045176267624,
+      "logits/rejected": 0.01742837205529213,
+      "logps/chosen": -321.1835632324219,
+      "logps/rejected": -227.04336547851562,
+      "loss": 0.5284,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.013542826287448406,
+      "rewards/margins": 0.09336896240711212,
+      "rewards/rejected": -0.10691177845001221,
       "step": 60
     },
     {
       "epoch": 0.2,
-      "learning_rate": 1.971830985915493e-06,
-      "logits/chosen": 0.010642724111676216,
-      "logits/rejected": 0.03363212198019028,
-      "logps/chosen": -288.052978515625,
-      "logps/rejected": -233.4365997314453,
-      "loss": 0.5253,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.11775505542755127,
-      "rewards/margins": 0.16322237253189087,
-      "rewards/rejected": -0.28097742795944214,
+      "learning_rate": 9.859154929577465e-07,
+      "logits/chosen": -0.0619073323905468,
+      "logits/rejected": -0.0089653879404068,
+      "logps/chosen": -356.95880126953125,
+      "logps/rejected": -257.0391845703125,
+      "loss": 0.5376,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.05503483861684799,
+      "rewards/margins": 0.16907523572444916,
+      "rewards/rejected": -0.22411008179187775,
       "step": 70
     },
     {
       "epoch": 0.22,
-      "learning_rate": 1.9990212265199736e-06,
-      "logits/chosen": -0.18143758177757263,
-      "logits/rejected": -0.11922919750213623,
-      "logps/chosen": -372.3688049316406,
-      "logps/rejected": -212.8850555419922,
-      "loss": 0.4977,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.05031196400523186,
-      "rewards/margins": 0.2945536971092224,
-      "rewards/rejected": -0.24424175918102264,
+      "learning_rate": 9.995106132599868e-07,
+      "logits/chosen": 0.04335876554250717,
+      "logits/rejected": 0.09913833439350128,
+      "logps/chosen": -323.42095947265625,
+      "logps/rejected": -238.894775390625,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.10817662626504898,
+      "rewards/margins": 0.1278904229402542,
+      "rewards/rejected": -0.2360670566558838,
       "step": 80
     },
     {
       "epoch": 0.25,
-      "learning_rate": 1.995640271796129e-06,
-      "logits/chosen": -0.10464553534984589,
-      "logits/rejected": -0.10386872291564941,
-      "logps/chosen": -271.26727294921875,
-      "logps/rejected": -232.1707763671875,
-      "loss": 0.4754,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.046878136694431305,
-      "rewards/margins": 0.11948816478252411,
-      "rewards/rejected": -0.166366308927536,
+      "learning_rate": 9.978201358980644e-07,
+      "logits/chosen": -0.012843596749007702,
+      "logits/rejected": 0.04404326155781746,
+      "logps/chosen": -363.57000732421875,
+      "logps/rejected": -281.42022705078125,
+      "loss": 0.5228,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.005774746648967266,
+      "rewards/margins": 0.13524329662322998,
+      "rewards/rejected": -0.12946854531764984,
       "step": 90
     },
     {
       "epoch": 0.28,
-      "learning_rate": 1.9898532207817787e-06,
-      "logits/chosen": -0.10582619905471802,
-      "logits/rejected": -0.06019941717386246,
-      "logps/chosen": -338.7351989746094,
-      "logps/rejected": -235.93240356445312,
-      "loss": 0.4609,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.09618537873029709,
-      "rewards/margins": 0.24425363540649414,
-      "rewards/rejected": -0.34043899178504944,
+      "learning_rate": 9.949266103908894e-07,
+      "logits/chosen": -0.006999261677265167,
+      "logits/rejected": 0.056761473417282104,
+      "logps/chosen": -276.5388488769531,
+      "logps/rejected": -212.882568359375,
+      "loss": 0.5063,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.031200706958770752,
+      "rewards/margins": 0.1093803197145462,
+      "rewards/rejected": -0.07817960530519485,
       "step": 100
     },
     {
       "epoch": 0.31,
-      "learning_rate": 1.9816740586504575e-06,
-      "logits/chosen": -0.15991339087486267,
-      "logits/rejected": -0.08021946251392365,
-      "logps/chosen": -347.1170959472656,
-      "logps/rejected": -260.5694580078125,
-      "loss": 0.4339,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.1928514689207077,
-      "rewards/margins": 0.27897369861602783,
-      "rewards/rejected": -0.4718252122402191,
+      "learning_rate": 9.908370293252287e-07,
+      "logits/chosen": -0.0054739052429795265,
+      "logits/rejected": 0.018744127824902534,
+      "logps/chosen": -324.8765869140625,
+      "logps/rejected": -246.21212768554688,
+      "loss": 0.5016,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.05711355805397034,
+      "rewards/margins": 0.14952385425567627,
+      "rewards/rejected": -0.09241029620170593,
       "step": 110
     },
     {
       "epoch": 0.34,
-      "learning_rate": 1.971122551428331e-06,
-      "logits/chosen": -0.173538938164711,
-      "logits/rejected": -0.14118380844593048,
-      "logps/chosen": -322.6651306152344,
-      "logps/rejected": -249.2017822265625,
-      "loss": 0.4292,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.25639423727989197,
-      "rewards/margins": 0.15481036901474,
-      "rewards/rejected": -0.41120463609695435,
+      "learning_rate": 9.855612757141654e-07,
+      "logits/chosen": -0.06719125807285309,
+      "logits/rejected": -0.008927728049457073,
+      "logps/chosen": -338.69720458984375,
+      "logps/rejected": -210.80349731445312,
+      "loss": 0.5033,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.007685990538448095,
+      "rewards/margins": 0.2579048275947571,
+      "rewards/rejected": -0.25021880865097046,
       "step": 120
     },
     {
       "epoch": 0.37,
-      "learning_rate": 1.9582241982269803e-06,
-      "logits/chosen": -0.171969935297966,
-      "logits/rejected": -0.11264105141162872,
-      "logps/chosen": -367.5609130859375,
-      "logps/rejected": -256.74334716796875,
-      "loss": 0.4099,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.23346960544586182,
-      "rewards/margins": 0.23451845347881317,
-      "rewards/rejected": -0.4679880142211914,
+      "learning_rate": 9.791120991134902e-07,
+      "logits/chosen": -0.10314974933862686,
+      "logits/rejected": -0.011623701080679893,
+      "logps/chosen": -380.0775451660156,
+      "logps/rejected": -255.47286987304688,
+      "loss": 0.5122,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.06153721362352371,
+      "rewards/margins": 0.2917006313800812,
+      "rewards/rejected": -0.3532378077507019,
       "step": 130
     },
     {
       "epoch": 0.39,
-      "learning_rate": 1.9430101696214336e-06,
-      "logits/chosen": -0.2663022577762604,
-      "logits/rejected": -0.17533616721630096,
-      "logps/chosen": -351.01544189453125,
-      "logps/rejected": -236.74423217773438,
-      "loss": 0.4005,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.3526380658149719,
-      "rewards/margins": 0.29506126046180725,
-      "rewards/rejected": -0.6476993560791016,
+      "learning_rate": 9.715050848107168e-07,
+      "logits/chosen": -0.17015981674194336,
+      "logits/rejected": -0.08929729461669922,
+      "logps/chosen": -344.5144958496094,
+      "logps/rejected": -232.58786010742188,
+      "loss": 0.4817,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.038360703736543655,
+      "rewards/margins": 0.26252132654190063,
+      "rewards/rejected": -0.3008820414543152,
       "step": 140
     },
     {
       "epoch": 0.42,
-      "learning_rate": 1.9255172323223463e-06,
-      "logits/chosen": -0.2499375343322754,
-      "logits/rejected": -0.2070433646440506,
-      "logps/chosen": -352.9437561035156,
-      "logps/rejected": -292.28277587890625,
-      "loss": 0.3763,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.4557357430458069,
-      "rewards/margins": 0.24104443192481995,
-      "rewards/rejected": -0.6967801451683044,
+      "learning_rate": 9.627586161611731e-07,
+      "logits/chosen": -0.09996357560157776,
+      "logits/rejected": -0.0434463731944561,
+      "logps/chosen": -285.2746887207031,
+      "logps/rejected": -217.32177734375,
+      "loss": 0.479,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06696902960538864,
+      "rewards/margins": 0.18583206832408905,
+      "rewards/rejected": -0.2528010904788971,
       "step": 150
     },
     {
       "epoch": 0.45,
-      "learning_rate": 1.905787660324391e-06,
-      "logits/chosen": -0.29798978567123413,
-      "logits/rejected": -0.23274393379688263,
-      "logps/chosen": -361.73760986328125,
-      "logps/rejected": -273.33648681640625,
-      "loss": 0.358,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.4945393204689026,
-      "rewards/margins": 0.26065993309020996,
-      "rewards/rejected": -0.7551992535591125,
+      "learning_rate": 9.528938301621955e-07,
+      "logits/chosen": -0.17547622323036194,
+      "logits/rejected": -0.11714746057987213,
+      "logps/chosen": -346.12451171875,
+      "logps/rejected": -225.3998565673828,
+      "loss": 0.4459,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.02809586003422737,
+      "rewards/margins": 0.17596405744552612,
+      "rewards/rejected": -0.2040599286556244,
       "step": 160
     },
     {
       "epoch": 0.48,
-      "learning_rate": 1.8838691327455609e-06,
-      "logits/chosen": -0.3041500151157379,
-      "logits/rejected": -0.251315176486969,
-      "logps/chosen": -417.890625,
-      "logps/rejected": -283.72235107421875,
-      "loss": 0.3654,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.44542360305786133,
-      "rewards/margins": 0.34727293252944946,
-      "rewards/rejected": -0.792696475982666,
+      "learning_rate": 9.419345663727804e-07,
+      "logits/chosen": -0.14170362055301666,
+      "logits/rejected": -0.08402098715305328,
+      "logps/chosen": -348.41986083984375,
+      "logps/rejected": -234.4894561767578,
+      "loss": 0.4488,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.03395242244005203,
+      "rewards/margins": 0.22700819373130798,
+      "rewards/rejected": -0.2609606385231018,
       "step": 170
     },
     {
       "epoch": 0.51,
-      "learning_rate": 1.8598146186042808e-06,
-      "logits/chosen": -0.3496856689453125,
-      "logits/rejected": -0.3132048547267914,
-      "logps/chosen": -389.0525817871094,
-      "logps/rejected": -277.38348388671875,
-      "loss": 0.3465,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.5126581788063049,
-      "rewards/margins": 0.2432871311903,
-      "rewards/rejected": -0.7559453248977661,
+      "learning_rate": 9.299073093021404e-07,
+      "logits/chosen": -0.07009784877300262,
+      "logits/rejected": -0.014887778088450432,
+      "logps/chosen": -359.72296142578125,
+      "logps/rejected": -259.1075439453125,
+      "loss": 0.472,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.18398258090019226,
+      "rewards/margins": 0.25430697202682495,
+      "rewards/rejected": -0.4382895827293396,
       "step": 180
     },
     {
       "epoch": 0.53,
-      "learning_rate": 1.8336822488127723e-06,
-      "logits/chosen": -0.31627652049064636,
-      "logits/rejected": -0.2888433635234833,
-      "logps/chosen": -325.4978942871094,
-      "logps/rejected": -290.9458923339844,
-      "loss": 0.3386,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.706427276134491,
-      "rewards/margins": 0.11679848283529282,
-      "rewards/rejected": -0.8232257962226868,
+      "learning_rate": 9.168411244063861e-07,
+      "logits/chosen": -0.15516254305839539,
+      "logits/rejected": -0.09202875196933746,
+      "logps/chosen": -376.22027587890625,
+      "logps/rejected": -267.2409362792969,
+      "loss": 0.4593,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.2544471025466919,
+      "rewards/margins": 0.2352083921432495,
+      "rewards/rejected": -0.4896554946899414,
       "step": 190
     },
     {
       "epoch": 0.56,
-      "learning_rate": 1.805535175696026e-06,
-      "logits/chosen": -0.28905901312828064,
-      "logits/rejected": -0.2691965699195862,
-      "logps/chosen": -340.1316833496094,
-      "logps/rejected": -311.392822265625,
-      "loss": 0.3529,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.6956604719161987,
-      "rewards/margins": 0.2750135064125061,
-      "rewards/rejected": -0.9706739187240601,
+      "learning_rate": 9.02767587848013e-07,
+      "logits/chosen": -0.16307282447814941,
+      "logits/rejected": -0.11575134098529816,
+      "logps/chosen": -327.9281311035156,
+      "logps/rejected": -276.331787109375,
+      "loss": 0.448,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.21112385392189026,
+      "rewards/margins": 0.1783614158630371,
+      "rewards/rejected": -0.38948526978492737,
       "step": 200
     },
     {
       "epoch": 0.59,
-      "learning_rate": 1.7754414203758602e-06,
-      "logits/chosen": -0.3484232425689697,
-      "logits/rejected": -0.29674482345581055,
-      "logps/chosen": -359.6042785644531,
-      "logps/rejected": -294.1127014160156,
-      "loss": 0.3339,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.667249858379364,
-      "rewards/margins": 0.21498624980449677,
-      "rewards/rejected": -0.882236123085022,
+      "learning_rate": 8.877207101879301e-07,
+      "logits/chosen": -0.11954480409622192,
+      "logits/rejected": -0.06289126724004745,
+      "logps/chosen": -306.99871826171875,
+      "logps/rejected": -228.0065460205078,
+      "loss": 0.4238,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.18940094113349915,
+      "rewards/margins": 0.19614948332309723,
+      "rewards/rejected": -0.3855504095554352,
       "step": 210
     },
     {
       "epoch": 0.62,
-      "learning_rate": 1.7434737083888904e-06,
-      "logits/chosen": -0.3094923198223114,
-      "logits/rejected": -0.27373385429382324,
-      "logps/chosen": -407.7154846191406,
-      "logps/rejected": -308.6554870605469,
-      "loss": 0.3407,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.6901127099990845,
-      "rewards/margins": 0.24910616874694824,
-      "rewards/rejected": -0.9392188787460327,
+      "learning_rate": 8.717368541944452e-07,
+      "logits/chosen": -0.14218950271606445,
+      "logits/rejected": -0.12026125192642212,
+      "logps/chosen": -317.53515625,
+      "logps/rejected": -235.4711456298828,
+      "loss": 0.4305,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.2156684845685959,
+      "rewards/margins": 0.20358431339263916,
+      "rewards/rejected": -0.41925281286239624,
       "step": 220
     },
     {
       "epoch": 0.65,
-      "learning_rate": 1.709709293935662e-06,
-      "logits/chosen": -0.3457449972629547,
-      "logits/rejected": -0.296016126871109,
-      "logps/chosen": -364.3536682128906,
-      "logps/rejected": -335.47021484375,
-      "loss": 0.3523,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.7736875414848328,
-      "rewards/margins": 0.3423411250114441,
-      "rewards/rejected": -1.1160286664962769,
+      "learning_rate": 8.54854646967831e-07,
+      "logits/chosen": -0.2029605209827423,
+      "logits/rejected": -0.1495995819568634,
+      "logps/chosen": -356.8483581542969,
+      "logps/rejected": -289.8570251464844,
+      "loss": 0.4198,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.2576315402984619,
+      "rewards/margins": 0.24145260453224182,
+      "rewards/rejected": -0.49908414483070374,
       "step": 230
     },
     {
       "epoch": 0.67,
-      "learning_rate": 1.6742297731856636e-06,
-      "logits/chosen": -0.3264350891113281,
-      "logits/rejected": -0.3357524275779724,
-      "logps/chosen": -403.2978515625,
-      "logps/rejected": -324.7033996582031,
-      "loss": 0.3164,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.7830929756164551,
-      "rewards/margins": 0.2251376211643219,
-      "rewards/rejected": -1.0082306861877441,
+      "learning_rate": 8.371148865928318e-07,
+      "logits/chosen": -0.14190950989723206,
+      "logits/rejected": -0.10926549136638641,
+      "logps/chosen": -341.60406494140625,
+      "logps/rejected": -277.4010009765625,
+      "loss": 0.3943,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.26126688718795776,
+      "rewards/margins": 0.23097483813762665,
+      "rewards/rejected": -0.492241770029068,
       "step": 240
     },
     {
       "epoch": 0.7,
-      "learning_rate": 1.6371208870894001e-06,
-      "logits/chosen": -0.39360159635543823,
-      "logits/rejected": -0.3303549587726593,
-      "logps/chosen": -440.0787658691406,
-      "logps/rejected": -334.0160217285156,
-      "loss": 0.3344,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.8457928895950317,
-      "rewards/margins": 0.20620958507061005,
-      "rewards/rejected": -1.0520025491714478,
+      "learning_rate": 8.185604435447001e-07,
+      "logits/chosen": -0.11840251833200455,
+      "logits/rejected": -0.07302571833133698,
+      "logps/chosen": -352.67669677734375,
+      "logps/rejected": -278.2833557128906,
+      "loss": 0.4072,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.27463454008102417,
+      "rewards/margins": 0.2554408013820648,
+      "rewards/rejected": -0.5300754308700562,
       "step": 250
     },
     {
       "epoch": 0.73,
-      "learning_rate": 1.5984723141740574e-06,
-      "logits/chosen": -0.3018794655799866,
-      "logits/rejected": -0.24929973483085632,
-      "logps/chosen": -420.9669494628906,
-      "logps/rejected": -344.3638000488281,
-      "loss": 0.3565,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.7072012424468994,
-      "rewards/margins": 0.3442048728466034,
-      "rewards/rejected": -1.0514062643051147,
+      "learning_rate": 7.992361570870287e-07,
+      "logits/chosen": -0.20119109749794006,
+      "logits/rejected": -0.1608552783727646,
+      "logps/chosen": -312.8297424316406,
+      "logps/rejected": -253.52206420898438,
+      "loss": 0.4189,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.3588634431362152,
+      "rewards/margins": 0.18340489268302917,
+      "rewards/rejected": -0.5422683358192444,
       "step": 260
     },
     {
       "epoch": 0.76,
-      "learning_rate": 1.5583774538234882e-06,
-      "logits/chosen": -0.26736167073249817,
-      "logits/rejected": -0.21991169452667236,
-      "logps/chosen": -406.1842346191406,
-      "logps/rejected": -344.8274841308594,
-      "loss": 0.3229,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.8611620664596558,
-      "rewards/margins": 0.3229338824748993,
-      "rewards/rejected": -1.1840959787368774,
+      "learning_rate": 7.791887269117441e-07,
+      "logits/chosen": -0.130401611328125,
+      "logits/rejected": -0.061286091804504395,
+      "logps/chosen": -357.2626953125,
+      "logps/rejected": -254.2506866455078,
+      "loss": 0.4325,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.39726734161376953,
+      "rewards/margins": 0.31203722953796387,
+      "rewards/rejected": -0.7093045711517334,
       "step": 270
     },
     {
       "epoch": 0.79,
-      "learning_rate": 1.5169332005662589e-06,
-      "logits/chosen": -0.33921122550964355,
-      "logits/rejected": -0.2950724959373474,
-      "logps/chosen": -423.4300231933594,
-      "logps/rejected": -334.3074645996094,
-      "loss": 0.3151,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.9226363897323608,
-      "rewards/margins": 0.36767420172691345,
-      "rewards/rejected": -1.2903106212615967,
+      "learning_rate": 7.584666002831294e-07,
+      "logits/chosen": -0.21146509051322937,
+      "logits/rejected": -0.15420952439308167,
+      "logps/chosen": -403.4113464355469,
+      "logps/rejected": -285.39385986328125,
+      "loss": 0.411,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.3059775233268738,
+      "rewards/margins": 0.321936696767807,
+      "rewards/rejected": -0.6279141902923584,
       "step": 280
     },
     {
       "epoch": 0.81,
-      "learning_rate": 1.474239709917218e-06,
-      "logits/chosen": -0.3765650689601898,
-      "logits/rejected": -0.34539324045181274,
-      "logps/chosen": -417.9461364746094,
-      "logps/rejected": -340.34716796875,
-      "loss": 0.2955,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.843783974647522,
-      "rewards/margins": 0.4681572914123535,
-      "rewards/rejected": -1.311941146850586,
+      "learning_rate": 7.37119854958609e-07,
+      "logits/chosen": -0.20524680614471436,
+      "logits/rejected": -0.16661730408668518,
+      "logps/chosen": -373.66961669921875,
+      "logps/rejected": -300.1796569824219,
+      "loss": 0.3785,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.35920846462249756,
+      "rewards/margins": 0.23060360550880432,
+      "rewards/rejected": -0.5898120403289795,
       "step": 290
     },
     {
       "epoch": 0.84,
-      "learning_rate": 1.430400156338457e-06,
-      "logits/chosen": -0.45963913202285767,
-      "logits/rejected": -0.3955768346786499,
-      "logps/chosen": -441.5418395996094,
-      "logps/rejected": -356.2896423339844,
-      "loss": 0.2834,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.962236225605011,
-      "rewards/margins": 0.41569775342941284,
-      "rewards/rejected": -1.3779337406158447,
+      "learning_rate": 7.152000781692285e-07,
+      "logits/chosen": -0.2269669473171234,
+      "logits/rejected": -0.20886960625648499,
+      "logps/chosen": -376.932373046875,
+      "logps/rejected": -305.97137451171875,
+      "loss": 0.353,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.34051477909088135,
+      "rewards/margins": 0.1766359806060791,
+      "rewards/rejected": -0.5171507596969604,
       "step": 300
     },
     {
       "epoch": 0.87,
-      "learning_rate": 1.3855204839045892e-06,
-      "logits/chosen": -0.48744335770606995,
-      "logits/rejected": -0.43016400933265686,
-      "logps/chosen": -453.83502197265625,
-      "logps/rejected": -341.6754455566406,
-      "loss": 0.2911,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.0828750133514404,
-      "rewards/margins": 0.3926982283592224,
-      "rewards/rejected": -1.4755733013153076,
+      "learning_rate": 6.927602419522946e-07,
+      "logits/chosen": -0.1920723021030426,
+      "logits/rejected": -0.19167286157608032,
+      "logps/chosen": -328.34930419921875,
+      "logps/rejected": -271.737548828125,
+      "loss": 0.3549,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.4537748396396637,
+      "rewards/margins": 0.15218928456306458,
+      "rewards/rejected": -0.6059640645980835,
       "step": 310
     },
     {
       "epoch": 0.9,
-      "learning_rate": 1.3397091502748927e-06,
-      "logits/chosen": -0.4381836950778961,
-      "logits/rejected": -0.38170838356018066,
-      "logps/chosen": -443.5067443847656,
-      "logps/rejected": -344.43914794921875,
-      "loss": 0.299,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.0263779163360596,
-      "rewards/margins": 0.3381286859512329,
-      "rewards/rejected": -1.3645066022872925,
+      "learning_rate": 6.698545751374463e-07,
+      "logits/chosen": -0.2686254382133484,
+      "logits/rejected": -0.18076984584331512,
+      "logps/chosen": -424.98052978515625,
+      "logps/rejected": -309.3459777832031,
+      "loss": 0.38,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4298086166381836,
+      "rewards/margins": 0.33044153451919556,
+      "rewards/rejected": -0.7602501511573792,
       "step": 320
     },
     {
       "epoch": 0.93,
-      "learning_rate": 1.2930768645910449e-06,
-      "logits/chosen": -0.40465015172958374,
-      "logits/rejected": -0.35638627409935,
-      "logps/chosen": -396.4947204589844,
-      "logps/rejected": -333.97650146484375,
-      "loss": 0.3027,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.8788374662399292,
-      "rewards/margins": 0.32027482986450195,
-      "rewards/rejected": -1.1991122961044312,
+      "learning_rate": 6.465384322955224e-07,
+      "logits/chosen": -0.21811941266059875,
+      "logits/rejected": -0.1839132010936737,
+      "logps/chosen": -372.3172607421875,
+      "logps/rejected": -272.31597900390625,
+      "loss": 0.3957,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.42889633774757385,
+      "rewards/margins": 0.3255406320095062,
+      "rewards/rejected": -0.7544369697570801,
       "step": 330
     },
     {
       "epoch": 0.96,
-      "learning_rate": 1.2457363199338495e-06,
-      "logits/chosen": -0.46247321367263794,
-      "logits/rejected": -0.3894230127334595,
-      "logps/chosen": -367.4734802246094,
-      "logps/rejected": -282.1875,
-      "loss": 0.3023,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.8268691301345825,
-      "rewards/margins": 0.23292645812034607,
-      "rewards/rejected": -1.059795618057251,
+      "learning_rate": 6.228681599669248e-07,
+      "logits/chosen": -0.19736522436141968,
+      "logits/rejected": -0.13541147112846375,
+      "logps/chosen": -430.74932861328125,
+      "logps/rejected": -293.2500915527344,
+      "loss": 0.3951,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.36328303813934326,
+      "rewards/margins": 0.36417144536972046,
+      "rewards/rejected": -0.7274545431137085,
       "step": 340
     },
     {
       "epoch": 0.98,
-      "learning_rate": 1.1978019209855173e-06,
-      "logits/chosen": -0.46343177556991577,
-      "logits/rejected": -0.4581407904624939,
-      "logps/chosen": -406.019775390625,
-      "logps/rejected": -358.7392883300781,
-      "loss": 0.3244,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.9359294772148132,
-      "rewards/margins": 0.3501017987728119,
-      "rewards/rejected": -1.2860312461853027,
+      "learning_rate": 5.989009604927586e-07,
+      "logits/chosen": -0.18117669224739075,
+      "logits/rejected": -0.08865699172019958,
+      "logps/chosen": -387.6150207519531,
+      "logps/rejected": -295.8270263671875,
+      "loss": 0.3683,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2865923345088959,
+      "rewards/margins": 0.37007588148117065,
+      "rewards/rejected": -0.6566681861877441,
       "step": 350
     },
     {
       "epoch": 1.01,
-      "learning_rate": 1.14938950755563e-06,
-      "logits/chosen": -0.5021234154701233,
-      "logits/rejected": -0.4483606219291687,
-      "logps/chosen": -400.4501037597656,
-      "logps/rejected": -345.84906005859375,
-      "loss": 0.2429,
+      "learning_rate": 5.74694753777815e-07,
+      "logits/chosen": -0.18364325165748596,
+      "logits/rejected": -0.16212408244609833,
+      "logps/chosen": -325.68133544921875,
+      "logps/rejected": -274.33734130859375,
+      "loss": 0.3395,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.8362014889717102,
-      "rewards/margins": 0.6666104197502136,
-      "rewards/rejected": -1.5028117895126343,
+      "rewards/chosen": -0.40283432602882385,
+      "rewards/margins": 0.3764493465423584,
+      "rewards/rejected": -0.7792836427688599,
       "step": 360
     },
     {
       "epoch": 1.04,
-      "learning_rate": 1.1006160746389332e-06,
-      "logits/chosen": -0.48441916704177856,
-      "logits/rejected": -0.4754874110221863,
-      "logps/chosen": -414.5755310058594,
-      "logps/rejected": -454.7470703125,
-      "loss": 0.1045,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.2475640773773193,
-      "rewards/margins": 1.0972559452056885,
-      "rewards/rejected": -2.344820261001587,
+      "learning_rate": 5.503080373194666e-07,
+      "logits/chosen": -0.28244373202323914,
+      "logits/rejected": -0.2197529822587967,
+      "logps/chosen": -375.904541015625,
+      "logps/rejected": -331.39263916015625,
+      "loss": 0.2837,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.45602947473526,
+      "rewards/margins": 0.6367942094802856,
+      "rewards/rejected": -1.0928236246109009,
       "step": 370
     },
     {
       "epoch": 1.07,
-      "learning_rate": 1.0515994896814731e-06,
-      "logits/chosen": -0.535969614982605,
-      "logits/rejected": -0.5428508520126343,
-      "logps/chosen": -626.1917724609375,
-      "logps/rejected": -632.7891845703125,
-      "loss": 0.0454,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.6970226764678955,
-      "rewards/margins": 1.3931865692138672,
-      "rewards/rejected": -4.090209007263184,
+      "learning_rate": 5.257997448407366e-07,
+      "logits/chosen": -0.24875327944755554,
+      "logits/rejected": -0.1415044665336609,
+      "logps/chosen": -409.04010009765625,
+      "logps/rejected": -306.00079345703125,
+      "loss": 0.2459,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.48139676451683044,
+      "rewards/margins": 0.6319422721862793,
+      "rewards/rejected": -1.1133390665054321,
       "step": 380
     },
     {
       "epoch": 1.1,
-      "learning_rate": 1.002458207738333e-06,
-      "logits/chosen": -0.3777693808078766,
-      "logits/rejected": -0.3439493179321289,
-      "logps/chosen": -622.327880859375,
-      "logps/rejected": -571.1327514648438,
-      "loss": 0.0574,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -2.0124690532684326,
-      "rewards/margins": 1.4971076250076294,
-      "rewards/rejected": -3.5095767974853516,
+      "learning_rate": 5.012291038691665e-07,
+      "logits/chosen": -0.2663780748844147,
+      "logits/rejected": -0.18359437584877014,
+      "logps/chosen": -421.8829650878906,
+      "logps/rejected": -379.4888916015625,
+      "loss": 0.2079,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.7042829394340515,
+      "rewards/margins": 0.6811956167221069,
+      "rewards/rejected": -1.3854784965515137,
       "step": 390
     },
     {
       "epoch": 1.12,
-      "learning_rate": 9.533109852113413e-07,
-      "logits/chosen": -0.40328603982925415,
-      "logits/rejected": -0.34989938139915466,
-      "logps/chosen": -453.33563232421875,
-      "logps/rejected": -487.29742431640625,
-      "loss": 0.0638,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.598747968673706,
-      "rewards/margins": 1.2498724460601807,
-      "rewards/rejected": -2.8486204147338867,
+      "learning_rate": 4.7665549260567063e-07,
+      "logits/chosen": -0.30431362986564636,
+      "logits/rejected": -0.24137239158153534,
+      "logps/chosen": -431.7137145996094,
+      "logps/rejected": -361.01959228515625,
+      "loss": 0.1728,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.903692364692688,
+      "rewards/margins": 0.7888330221176147,
+      "rewards/rejected": -1.6925252676010132,
       "step": 400
     },
     {
       "epoch": 1.15,
-      "learning_rate": 9.042765928585326e-07,
-      "logits/chosen": -0.3043842315673828,
-      "logits/rejected": -0.31053781509399414,
-      "logps/chosen": -483.6888122558594,
-      "logps/rejected": -555.2555541992188,
-      "loss": 0.0511,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.9158967733383179,
-      "rewards/margins": 1.495742917060852,
-      "rewards/rejected": -3.41163969039917,
+      "learning_rate": 4.521382964292663e-07,
+      "logits/chosen": -0.23668956756591797,
+      "logits/rejected": -0.19462040066719055,
+      "logps/chosen": -423.92254638671875,
+      "logps/rejected": -366.35723876953125,
+      "loss": 0.1768,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.9347349405288696,
+      "rewards/margins": 0.7054546475410461,
+      "rewards/rejected": -1.640189528465271,
       "step": 410
     },
     {
       "epoch": 1.18,
-      "learning_rate": 8.554735287689148e-07,
-      "logits/chosen": -0.2635629177093506,
-      "logits/rejected": -0.22284087538719177,
-      "logps/chosen": -522.8436279296875,
-      "logps/rejected": -567.716552734375,
-      "loss": 0.0467,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -1.8879692554473877,
-      "rewards/margins": 1.4974491596221924,
-      "rewards/rejected": -3.385418653488159,
+      "learning_rate": 4.277367643844574e-07,
+      "logits/chosen": -0.2629498541355133,
+      "logits/rejected": -0.19311530888080597,
+      "logps/chosen": -465.82305908203125,
+      "logps/rejected": -373.01806640625,
+      "loss": 0.1767,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0273025035858154,
+      "rewards/margins": 0.6468140482902527,
+      "rewards/rejected": -1.6741164922714233,
       "step": 420
     },
     {
       "epoch": 1.21,
-      "learning_rate": 8.070197319961782e-07,
-      "logits/chosen": -0.14604279398918152,
-      "logits/rejected": -0.1439618021249771,
-      "logps/chosen": -484.59967041015625,
-      "logps/rejected": -506.89483642578125,
-      "loss": 0.0408,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.8626312017440796,
-      "rewards/margins": 1.5483224391937256,
-      "rewards/rejected": -3.4109535217285156,
+      "learning_rate": 4.035098659980891e-07,
+      "logits/chosen": -0.24669210612773895,
+      "logits/rejected": -0.1399877965450287,
+      "logps/chosen": -415.6297912597656,
+      "logps/rejected": -384.90948486328125,
+      "loss": 0.1839,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.9914595484733582,
+      "rewards/margins": 0.6535197496414185,
+      "rewards/rejected": -1.6449792385101318,
       "step": 430
     },
     {
       "epoch": 1.24,
-      "learning_rate": 7.590322975433856e-07,
-      "logits/chosen": -0.22616323828697205,
-      "logits/rejected": -0.14803537726402283,
-      "logps/chosen": -720.8937377929688,
-      "logps/rejected": -669.2205200195312,
-      "loss": 0.0367,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -2.2607662677764893,
-      "rewards/margins": 1.8761478662490845,
-      "rewards/rejected": -4.136914253234863,
+      "learning_rate": 3.795161487716928e-07,
+      "logits/chosen": -0.22286108136177063,
+      "logits/rejected": -0.14977149665355682,
+      "logps/chosen": -483.51092529296875,
+      "logps/rejected": -409.53411865234375,
+      "loss": 0.1723,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.9548113942146301,
+      "rewards/margins": 0.8746234178543091,
+      "rewards/rejected": -1.8294346332550049,
       "step": 440
     },
     {
       "epoch": 1.26,
-      "learning_rate": 7.116271933874245e-07,
-      "logits/chosen": -0.16615112125873566,
-      "logits/rejected": -0.06526105105876923,
-      "logps/chosen": -621.0664672851562,
-      "logps/rejected": -645.3672485351562,
-      "loss": 0.03,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -2.6638753414154053,
-      "rewards/margins": 1.425157904624939,
-      "rewards/rejected": -4.089033603668213,
+      "learning_rate": 3.5581359669371223e-07,
+      "logits/chosen": -0.17988570034503937,
+      "logits/rejected": -0.17582079768180847,
+      "logps/chosen": -420.56048583984375,
+      "logps/rejected": -363.8465270996094,
+      "loss": 0.1609,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.0016785860061646,
+      "rewards/margins": 0.5783125758171082,
+      "rewards/rejected": -1.5799912214279175,
       "step": 450
     },
     {
       "epoch": 1.29,
-      "learning_rate": 6.649189802270652e-07,
-      "logits/chosen": -0.12236519157886505,
-      "logits/rejected": -0.05993504449725151,
-      "logps/chosen": -556.0875244140625,
-      "logps/rejected": -621.5553588867188,
-      "loss": 0.0311,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -2.585132122039795,
-      "rewards/margins": 1.5961207151412964,
-      "rewards/rejected": -4.181252479553223,
+      "learning_rate": 3.324594901135326e-07,
+      "logits/chosen": -0.22410225868225098,
+      "logits/rejected": -0.16186970472335815,
+      "logps/chosen": -437.3802185058594,
+      "logps/rejected": -374.47247314453125,
+      "loss": 0.1541,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.0312308073043823,
+      "rewards/margins": 0.6275131702423096,
+      "rewards/rejected": -1.6587440967559814,
       "step": 460
     },
     {
       "epoch": 1.32,
-      "learning_rate": 6.190205346318926e-07,
-      "logits/chosen": -0.07673145830631256,
-      "logits/rejected": -0.08171101659536362,
-      "logps/chosen": -561.4466552734375,
-      "logps/rejected": -633.3453369140625,
-      "loss": 0.0306,
+      "learning_rate": 3.095102673159463e-07,
+      "logits/chosen": -0.18208977580070496,
+      "logits/rejected": -0.15096168220043182,
+      "logps/chosen": -473.5821228027344,
+      "logps/rejected": -409.6917419433594,
+      "loss": 0.1473,
       "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -2.4036738872528076,
-      "rewards/margins": 1.636997938156128,
-      "rewards/rejected": -4.040672302246094,
+      "rewards/chosen": -1.1605236530303955,
+      "rewards/margins": 0.7620750069618225,
+      "rewards/rejected": -1.9225986003875732,
       "step": 470
     },
     {
       "epoch": 1.35,
-      "learning_rate": 5.740427762611604e-07,
-      "logits/chosen": -0.09908205270767212,
-      "logits/rejected": -0.03128683939576149,
-      "logps/chosen": -573.7949829101562,
-      "logps/rejected": -645.1226806640625,
-      "loss": 0.0314,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.471958637237549,
-      "rewards/margins": 1.6115013360977173,
-      "rewards/rejected": -4.083459377288818,
+      "learning_rate": 2.870213881305802e-07,
+      "logits/chosen": -0.14899012446403503,
+      "logits/rejected": -0.05964149162173271,
+      "logps/chosen": -452.955078125,
+      "logps/rejected": -390.2185974121094,
+      "loss": 0.1482,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.1373943090438843,
+      "rewards/margins": 0.7015290856361389,
+      "rewards/rejected": -1.8389232158660889,
       "step": 480
     },
     {
       "epoch": 1.38,
-      "learning_rate": 5.300943998117749e-07,
-      "logits/chosen": 0.031324755400419235,
-      "logits/rejected": 0.16325363516807556,
-      "logps/chosen": -535.8555908203125,
-      "logps/rejected": -618.23193359375,
-      "loss": 0.0266,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.483048915863037,
-      "rewards/margins": 1.7440725564956665,
-      "rewards/rejected": -4.227121829986572,
+      "learning_rate": 2.6504719990588745e-07,
+      "logits/chosen": -0.14091524481773376,
+      "logits/rejected": -0.09124572575092316,
+      "logps/chosen": -451.630126953125,
+      "logps/rejected": -381.64276123046875,
+      "loss": 0.1474,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.1757558584213257,
+      "rewards/margins": 0.7031680345535278,
+      "rewards/rejected": -1.878924012184143,
       "step": 490
     },
     {
       "epoch": 1.4,
-      "learning_rate": 4.872816123431976e-07,
-      "logits/chosen": -0.025389358401298523,
-      "logits/rejected": 0.06323707848787308,
-      "logps/chosen": -554.9032592773438,
-      "logps/rejected": -576.2711181640625,
-      "loss": 0.029,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -2.5237717628479004,
-      "rewards/margins": 1.4061567783355713,
-      "rewards/rejected": -3.9299285411834717,
+      "learning_rate": 2.436408061715988e-07,
+      "logits/chosen": -0.10461604595184326,
+      "logits/rejected": -0.0906451866030693,
+      "logps/chosen": -386.41705322265625,
+      "logps/rejected": -403.9093933105469,
+      "loss": 0.1412,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.1071223020553589,
+      "rewards/margins": 0.6409605145454407,
+      "rewards/rejected": -1.7480828762054443,
       "step": 500
     },
     {
       "epoch": 1.43,
-      "learning_rate": 4.4570787661405e-07,
-      "logits/chosen": 0.053199104964733124,
-      "logits/rejected": 0.13226789236068726,
-      "logps/chosen": -601.7017822265625,
-      "logps/rejected": -650.9783935546875,
-      "loss": 0.0294,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -2.425492763519287,
-      "rewards/margins": 1.747205138206482,
-      "rewards/rejected": -4.172698020935059,
+      "learning_rate": 2.22853938307025e-07,
+      "logits/chosen": -0.053080081939697266,
+      "logits/rejected": -0.0005340933566913009,
+      "logps/chosen": -396.68548583984375,
+      "logps/rejected": -341.5348815917969,
+      "loss": 0.1348,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1873186826705933,
+      "rewards/margins": 0.5248215794563293,
+      "rewards/rejected": -1.7121403217315674,
       "step": 510
     },
     {
       "epoch": 1.46,
-      "learning_rate": 4.0547366105068347e-07,
-      "logits/chosen": 0.018769674003124237,
-      "logits/rejected": 0.1149587631225586,
-      "logps/chosen": -578.7161865234375,
-      "logps/rejected": -623.023193359375,
-      "loss": 0.0286,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -2.5578720569610596,
-      "rewards/margins": 1.705985426902771,
-      "rewards/rejected": -4.263857841491699,
+      "learning_rate": 2.0273683052534173e-07,
+      "logits/chosen": 0.018692368641495705,
+      "logits/rejected": 0.09040405601263046,
+      "logps/chosen": -455.23944091796875,
+      "logps/rejected": -418.3058166503906,
+      "loss": 0.14,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.1594905853271484,
+      "rewards/margins": 0.8578866720199585,
+      "rewards/rejected": -2.0173771381378174,
       "step": 520
     },
     {
       "epoch": 1.49,
-      "learning_rate": 3.666761969519528e-07,
-      "logits/chosen": 0.045023586601018906,
-      "logits/rejected": 0.06367478519678116,
-      "logps/chosen": -585.3010864257812,
-      "logps/rejected": -640.3782958984375,
-      "loss": 0.0262,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -2.601722240447998,
-      "rewards/margins": 1.804703950881958,
-      "rewards/rejected": -4.406426429748535,
+      "learning_rate": 1.833380984759764e-07,
+      "logits/chosen": -0.05254416540265083,
+      "logits/rejected": 0.048647552728652954,
+      "logps/chosen": -407.8287048339844,
+      "logps/rejected": -425.9695739746094,
+      "loss": 0.1465,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.2748953104019165,
+      "rewards/margins": 0.7285683155059814,
+      "rewards/rejected": -2.0034632682800293,
       "step": 530
     },
     {
       "epoch": 1.52,
-      "learning_rate": 3.2940924351693213e-07,
-      "logits/chosen": 0.11350098997354507,
-      "logits/rejected": 0.18628571927547455,
-      "logps/chosen": -568.986328125,
-      "logps/rejected": -641.9813232421875,
-      "loss": 0.0238,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -2.5491175651550293,
-      "rewards/margins": 1.643070936203003,
-      "rewards/rejected": -4.192188262939453,
+      "learning_rate": 1.6470462175846606e-07,
+      "logits/chosen": -0.017660032957792282,
+      "logits/rejected": 0.017109563574194908,
+      "logps/chosen": -465.2491149902344,
+      "logps/rejected": -433.9029846191406,
+      "loss": 0.1469,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.0953867435455322,
+      "rewards/margins": 0.8007495999336243,
+      "rewards/rejected": -1.8961362838745117,
       "step": 540
     },
     {
       "epoch": 1.55,
-      "learning_rate": 2.937628612634184e-07,
-      "logits/chosen": 0.013865552842617035,
-      "logits/rejected": 0.027750706300139427,
-      "logps/chosen": -620.2090454101562,
-      "logps/rejected": -650.0667724609375,
-      "loss": 0.0239,
+      "learning_rate": 1.468814306317092e-07,
+      "logits/chosen": 0.009503689594566822,
+      "logits/rejected": 0.04763117805123329,
+      "logps/chosen": -430.349609375,
+      "logps/rejected": -371.52532958984375,
+      "loss": 0.1389,
       "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -2.666903018951416,
-      "rewards/margins": 1.803750991821289,
-      "rewards/rejected": -4.470653533935547,
+      "rewards/chosen": -1.115271806716919,
+      "rewards/margins": 0.7877473831176758,
+      "rewards/rejected": -1.9030193090438843,
       "step": 550
     },
     {
       "epoch": 1.57,
-      "learning_rate": 2.598231943847916e-07,
-      "logits/chosen": 0.07076794654130936,
-      "logits/rejected": 0.15127721428871155,
-      "logps/chosen": -636.1734008789062,
-      "logps/rejected": -626.8192138671875,
-      "loss": 0.0229,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.7836403846740723,
-      "rewards/margins": 1.6109874248504639,
-      "rewards/rejected": -4.394627571105957,
+      "learning_rate": 1.299115971923958e-07,
+      "logits/chosen": -0.010196239687502384,
+      "logits/rejected": 0.055187441408634186,
+      "logps/chosen": -443.1453552246094,
+      "logps/rejected": -429.2059020996094,
+      "loss": 0.1395,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.179884910583496,
+      "rewards/margins": 0.8822008967399597,
+      "rewards/rejected": -2.0620856285095215,
       "step": 560
     },
     {
       "epoch": 1.6,
-      "learning_rate": 2.276722625711861e-07,
-      "logits/chosen": 0.008342927321791649,
-      "logits/rejected": 0.10422797501087189,
-      "logps/chosen": -604.8076782226562,
-      "logps/rejected": -674.343505859375,
-      "loss": 0.0205,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.9131510257720947,
-      "rewards/margins": 1.7855117321014404,
-      "rewards/rejected": -4.698662757873535,
+      "learning_rate": 1.1383613128559305e-07,
+      "logits/chosen": -0.03149424493312836,
+      "logits/rejected": 0.04653029888868332,
+      "logps/chosen": -498.27069091796875,
+      "logps/rejected": -488.0616760253906,
+      "loss": 0.137,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.2219698429107666,
+      "rewards/margins": 0.9878193140029907,
+      "rewards/rejected": -2.209789276123047,
       "step": 570
     },
     {
       "epoch": 1.63,
-      "learning_rate": 1.973877627980699e-07,
-      "logits/chosen": 0.03685791790485382,
-      "logits/rejected": 0.02010912261903286,
-      "logps/chosen": -591.6558227539062,
-      "logps/rejected": -699.0025634765625,
-      "loss": 0.0194,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -3.0175013542175293,
-      "rewards/margins": 1.4967237710952759,
-      "rewards/rejected": -4.514225482940674,
+      "learning_rate": 9.869388139903495e-08,
+      "logits/chosen": -0.0809461921453476,
+      "logits/rejected": -0.015049537643790245,
+      "logps/chosen": -539.2957763671875,
+      "logps/rejected": -481.3929748535156,
+      "loss": 0.132,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.3534362316131592,
+      "rewards/margins": 0.8015187978744507,
+      "rewards/rejected": -2.1549549102783203,
       "step": 580
     },
     {
       "epoch": 1.66,
-      "learning_rate": 1.6904288156123636e-07,
-      "logits/chosen": 0.039042066782712936,
-      "logits/rejected": 0.11644144356250763,
-      "logps/chosen": -615.1423950195312,
-      "logps/rejected": -660.23486328125,
-      "loss": 0.0198,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -2.8786990642547607,
-      "rewards/margins": 1.8589880466461182,
-      "rewards/rejected": -4.737687110900879,
+      "learning_rate": 8.452144078061818e-08,
+      "logits/chosen": 0.025172684341669083,
+      "logits/rejected": 0.056724805384874344,
+      "logps/chosen": -418.3292541503906,
+      "logps/rejected": -396.272216796875,
+      "loss": 0.1244,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.2570348978042603,
+      "rewards/margins": 0.773349940776825,
+      "rewards/rejected": -2.0303850173950195,
       "step": 590
     },
     {
       "epoch": 1.69,
-      "learning_rate": 1.4270611801196642e-07,
-      "logits/chosen": 0.1002052053809166,
-      "logits/rejected": 0.1534217894077301,
-      "logps/chosen": -646.0867309570312,
-      "logps/rejected": -681.2635498046875,
-      "loss": 0.0202,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.978548526763916,
-      "rewards/margins": 1.8796141147613525,
-      "rewards/rejected": -4.858162879943848,
+      "learning_rate": 7.135305900598321e-08,
+      "logits/chosen": -0.09145348519086838,
+      "logits/rejected": -0.024112572893500328,
+      "logps/chosen": -490.0101623535156,
+      "logps/rejected": -435.8873596191406,
+      "loss": 0.1211,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.354788899421692,
+      "rewards/margins": 0.9178797006607056,
+      "rewards/rejected": -2.2726683616638184,
       "step": 600
     },
     {
       "epoch": 1.71,
-      "learning_rate": 1.1844111841977633e-07,
-      "logits/chosen": 0.22774501144886017,
-      "logits/rejected": 0.21065323054790497,
-      "logps/chosen": -563.5538330078125,
-      "logps/rejected": -615.5939331054688,
-      "loss": 0.0205,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -2.7306342124938965,
-      "rewards/margins": 1.688939094543457,
-      "rewards/rejected": -4.4195733070373535,
+      "learning_rate": 5.9220559209888166e-08,
+      "logits/chosen": 0.052403099834918976,
+      "logits/rejected": 0.09970308840274811,
+      "logps/chosen": -431.0038146972656,
+      "logps/rejected": -452.14459228515625,
+      "loss": 0.1293,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.2271394729614258,
+      "rewards/margins": 0.8939367532730103,
+      "rewards/rejected": -2.1210761070251465,
       "step": 610
     },
     {
       "epoch": 1.74,
-      "learning_rate": 9.630652236279625e-08,
-      "logits/chosen": 0.14569208025932312,
-      "logits/rejected": 0.17595478892326355,
-      "logps/chosen": -612.38671875,
-      "logps/rejected": -682.3800048828125,
-      "loss": 0.0196,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -2.906216859817505,
-      "rewards/margins": 1.8156111240386963,
-      "rewards/rejected": -4.721828460693359,
+      "learning_rate": 4.815326118139812e-08,
+      "logits/chosen": 0.004505271557718515,
+      "logits/rejected": 0.142390176653862,
+      "logps/chosen": -400.72796630859375,
+      "logps/rejected": -388.5762634277344,
+      "loss": 0.135,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.3616560697555542,
+      "rewards/margins": 0.6720155477523804,
+      "rewards/rejected": -2.0336716175079346,
       "step": 620
     },
     {
       "epoch": 1.77,
-      "learning_rate": 7.63558210174814e-08,
-      "logits/chosen": 0.10530801862478256,
-      "logits/rejected": 0.17252102494239807,
-      "logps/chosen": -622.7254638671875,
-      "logps/rejected": -673.253173828125,
-      "loss": 0.0194,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -2.9717555046081543,
-      "rewards/margins": 1.84465754032135,
-      "rewards/rejected": -4.816412925720215,
+      "learning_rate": 3.81779105087407e-08,
+      "logits/chosen": -0.03825841844081879,
+      "logits/rejected": 0.05763017386198044,
+      "logps/chosen": -464.5328063964844,
+      "logps/rejected": -442.33404541015625,
+      "loss": 0.1335,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4398839473724365,
+      "rewards/margins": 0.7831242680549622,
+      "rewards/rejected": -2.223008394241333,
       "step": 630
     },
     {
       "epoch": 1.8,
-      "learning_rate": 5.8637227890115273e-08,
-      "logits/chosen": 0.05660830810666084,
-      "logits/rejected": 0.15187661349773407,
-      "logps/chosen": -602.2574462890625,
-      "logps/rejected": -671.6920776367188,
-      "loss": 0.0204,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -2.8500285148620605,
-      "rewards/margins": 1.8192436695098877,
-      "rewards/rejected": -4.669272422790527,
+      "learning_rate": 2.9318613945057637e-08,
+      "logits/chosen": 0.01526588760316372,
+      "logits/rejected": 0.07733525335788727,
+      "logps/chosen": -488.5757751464844,
+      "logps/rejected": -456.2373962402344,
+      "loss": 0.1244,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.3201992511749268,
+      "rewards/margins": 0.9031252861022949,
+      "rewards/rejected": -2.2233245372772217,
       "step": 640
     },
     {
       "epoch": 1.83,
-      "learning_rate": 4.3193562302499046e-08,
-      "logits/chosen": 0.00826293509453535,
-      "logits/rejected": -0.00432767765596509,
-      "logps/chosen": -674.0474853515625,
-      "logps/rejected": -713.0694580078125,
-      "loss": 0.0175,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -3.099088668823242,
-      "rewards/margins": 1.819131851196289,
-      "rewards/rejected": -4.918220520019531,
+      "learning_rate": 2.1596781151249523e-08,
+      "logits/chosen": -0.0026476040948182344,
+      "logits/rejected": 0.11588595062494278,
+      "logps/chosen": -447.08331298828125,
+      "logps/rejected": -423.4645080566406,
+      "loss": 0.1269,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3190840482711792,
+      "rewards/margins": 0.6764702796936035,
+      "rewards/rejected": -1.9955543279647827,
       "step": 650
     },
     {
       "epoch": 1.85,
-      "learning_rate": 3.006214591340339e-08,
-      "logits/chosen": 0.14368341863155365,
-      "logits/rejected": 0.2905717194080353,
-      "logps/chosen": -601.8983764648438,
-      "logps/rejected": -663.1688232421875,
-      "loss": 0.019,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.9037740230560303,
-      "rewards/margins": 1.60433828830719,
-      "rewards/rejected": -4.50811243057251,
+      "learning_rate": 1.5031072956701695e-08,
+      "logits/chosen": -0.013560554012656212,
+      "logits/rejected": 0.0541529655456543,
+      "logps/chosen": -485.8223571777344,
+      "logps/rejected": -464.66143798828125,
+      "loss": 0.1228,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.3514947891235352,
+      "rewards/margins": 0.9442659616470337,
+      "rewards/rejected": -2.2957608699798584,
       "step": 660
     },
     {
       "epoch": 1.88,
-      "learning_rate": 1.9274712525847447e-08,
-      "logits/chosen": 0.0023043565452098846,
-      "logits/rejected": 0.013824631460011005,
-      "logps/chosen": -618.7401123046875,
-      "logps/rejected": -675.178466796875,
-      "loss": 0.0212,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -2.900660276412964,
-      "rewards/margins": 1.8997567892074585,
-      "rewards/rejected": -4.800417423248291,
+      "learning_rate": 9.637356262923723e-09,
+      "logits/chosen": 0.08311934769153595,
+      "logits/rejected": 0.10251543670892715,
+      "logps/chosen": -446.80938720703125,
+      "logps/rejected": -408.7090759277344,
+      "loss": 0.1266,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.3170959949493408,
+      "rewards/margins": 0.7009096145629883,
+      "rewards/rejected": -2.018005847930908,
       "step": 670
     },
     {
       "epoch": 1.91,
-      "learning_rate": 1.0857331398169577e-08,
-      "logits/chosen": 0.062491677701473236,
-      "logits/rejected": 0.17225809395313263,
-      "logps/chosen": -628.8497314453125,
-      "logps/rejected": -684.4832763671875,
-      "loss": 0.019,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.9637694358825684,
-      "rewards/margins": 1.725229024887085,
-      "rewards/rejected": -4.688998699188232,
+      "learning_rate": 5.428665699084789e-09,
+      "logits/chosen": -0.018842682242393494,
+      "logits/rejected": -0.013545280322432518,
+      "logps/chosen": -475.67669677734375,
+      "logps/rejected": -463.5384826660156,
+      "loss": 0.1214,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.2991926670074463,
+      "rewards/margins": 0.9007900357246399,
+      "rewards/rejected": -2.1999831199645996,
       "step": 680
     },
     {
       "epoch": 1.94,
-      "learning_rate": 4.830344244220686e-09,
-      "logits/chosen": 0.13519100844860077,
-      "logits/rejected": 0.12914660573005676,
-      "logps/chosen": -621.2411499023438,
-      "logps/rejected": -750.7539672851562,
-      "loss": 0.0209,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -3.056195020675659,
-      "rewards/margins": 1.9564968347549438,
-      "rewards/rejected": -5.012692451477051,
+      "learning_rate": 2.415172122110343e-09,
+      "logits/chosen": 0.025611836463212967,
+      "logits/rejected": 0.10699748992919922,
+      "logps/chosen": -474.1285095214844,
+      "logps/rejected": -452.504150390625,
+      "loss": 0.125,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.3146600723266602,
+      "rewards/margins": 0.7926696538925171,
+      "rewards/rejected": -2.107329845428467,
       "step": 690
     },
     {
       "epoch": 1.97,
-      "learning_rate": 1.2083160749236653e-09,
-      "logits/chosen": 0.21589338779449463,
-      "logits/rejected": 0.24260704219341278,
-      "logps/chosen": -601.9577026367188,
-      "logps/rejected": -672.21875,
-      "loss": 0.0185,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -3.022331476211548,
-      "rewards/margins": 1.748324990272522,
-      "rewards/rejected": -4.770656108856201,
+      "learning_rate": 6.041580374618327e-10,
+      "logits/chosen": -0.046818483620882034,
+      "logits/rejected": 0.026954257860779762,
+      "logps/chosen": -467.6886291503906,
+      "logps/rejected": -427.25994873046875,
+      "loss": 0.1285,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.3291774988174438,
+      "rewards/margins": 0.7668731212615967,
+      "rewards/rejected": -2.096050500869751,
       "step": 700
     },
     {
       "epoch": 2.0,
       "learning_rate": 0.0,
-      "logits/chosen": 0.1099398136138916,
-      "logits/rejected": 0.22721083462238312,
-      "logps/chosen": -618.4613037109375,
-      "logps/rejected": -655.667236328125,
-      "loss": 0.0185,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.7860031127929688,
-      "rewards/margins": 1.7313239574432373,
-      "rewards/rejected": -4.517327308654785,
+      "logits/chosen": 0.03512246161699295,
+      "logits/rejected": 0.0718744620680809,
+      "logps/chosen": -398.6717224121094,
+      "logps/rejected": -408.44000244140625,
+      "loss": 0.1214,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.3393633365631104,
+      "rewards/margins": 0.6785662174224854,
+      "rewards/rejected": -2.0179295539855957,
       "step": 710
     },
     {
       "epoch": 2.0,
       "step": 710,
       "total_flos": 0.0,
-      "train_loss": 0.21128464219435839,
-      "train_runtime": 10587.5728,
-      "train_samples_per_second": 8.604,
-      "train_steps_per_second": 0.067
+      "train_loss": 0.30386436640376774,
+      "train_runtime": 10428.2681,
+      "train_samples_per_second": 8.735,
+      "train_steps_per_second": 0.068
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9da800f5e96dd69f198232ea91e3c7ca9805289c25d3d61903ec9fddff6d182
+oid sha256:70fd862b96091c63a464a5db443d907f10b5d3ebcf6fc2771bb9babd8af279fa
 size 6648