Model save

README.md

@@ -35,7 +35,7 @@ The following hyperparameters were used during training:
 - learning_rate: 2e-06
 - train_batch_size: 2
 - eval_batch_size: 8
-- seed: 1
+- seed: 2
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 8
@@ -44,7 +44,7 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 1
+- num_epochs: 2
 
 ### Training results
 

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.30111024614790793,
-    "train_runtime": 5278.2802,
+    "epoch": 2.0,
+    "train_loss": 0.21065937945960272,
+    "train_runtime": 10560.1161,
     "train_samples": 45548,
-    "train_samples_per_second": 8.629,
+    "train_samples_per_second": 8.626,
     "train_steps_per_second": 0.067
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f99564dd0b61c7960459a4d1dfdd645c1a78dc3e9fb3833889cd3c356f999c7
+oid sha256:017ec8abde50ce1610f6890d47e40e50eff062df000d1fa889ec708d377a3118
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8800f44937f05d718368505c0913363a01793524ed590c7a4de9fbaf4903cda1
+oid sha256:b58c87e610451d748c1d531a107e90bb750f3b5e94c11d0a293007ceaa82ec11
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e8602e8be4a1b909cf92a9245cbaff8682a9966b1ca54cf1713d1447aee48d0
+oid sha256:1a95089f9da264ce34eb8ac1c56338154e6e293ee5fb51ab15ee7ba91df63e46
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4caf53fe4f749740b40900d01c4e026065a5cd96a5dc8400797a1228b7dd7149
+oid sha256:bac8791b6da670c1b08dbffd9c1fb56e46109f5846709c905830aa4d5a751715
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.30111024614790793,
-    "train_runtime": 5278.2802,
+    "epoch": 2.0,
+    "train_loss": 0.21065937945960272,
+    "train_runtime": 10560.1161,
     "train_samples": 45548,
-    "train_samples_per_second": 8.629,
+    "train_samples_per_second": 8.626,
     "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -1,516 +1,1020 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9975412715138743,
+  "epoch": 1.9950825430277486,
   "eval_steps": 10000,
-  "global_step": 355,
+  "global_step": 710,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.03,
-      "learning_rate": 5.555555555555555e-07,
-      "logits/chosen": -0.09875188767910004,
-      "logits/rejected": 0.0006220974028110504,
-      "logps/chosen": -327.19207763671875,
-      "logps/rejected": -192.9109344482422,
-      "loss": 0.5078,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.000275815516943112,
-      "rewards/margins": 0.0012074653059244156,
-      "rewards/rejected": -0.0009316497598774731,
+      "learning_rate": 2.8169014084507043e-07,
+      "logits/chosen": -0.023287910968065262,
+      "logits/rejected": 0.045911647379398346,
+      "logps/chosen": -317.10211181640625,
+      "logps/rejected": -207.3465118408203,
+      "loss": 0.5132,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.0002548714110162109,
+      "rewards/margins": -0.0012714166659861803,
+      "rewards/rejected": 0.0010165453422814608,
       "step": 10
     },
     {
       "epoch": 0.06,
-      "learning_rate": 1.111111111111111e-06,
-      "logits/chosen": -0.061609845608472824,
-      "logits/rejected": -0.0017540112603455782,
-      "logps/chosen": -295.9917297363281,
-      "logps/rejected": -183.76776123046875,
-      "loss": 0.5151,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.007897479459643364,
-      "rewards/margins": 0.025164177641272545,
-      "rewards/rejected": -0.01726669631898403,
+      "learning_rate": 5.633802816901409e-07,
+      "logits/chosen": -0.046765245497226715,
+      "logits/rejected": -0.011256822384893894,
+      "logps/chosen": -293.9908752441406,
+      "logps/rejected": -202.18402099609375,
+      "loss": 0.5075,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.000761970819439739,
+      "rewards/margins": 0.008972947485744953,
+      "rewards/rejected": -0.008210976608097553,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "learning_rate": 1.6666666666666667e-06,
-      "logits/chosen": -0.027067899703979492,
-      "logits/rejected": 0.09714551270008087,
-      "logps/chosen": -340.88226318359375,
-      "logps/rejected": -180.27139282226562,
-      "loss": 0.5184,
+      "learning_rate": 8.450704225352112e-07,
+      "logits/chosen": -0.03957567363977432,
+      "logits/rejected": 0.012707856483757496,
+      "logps/chosen": -362.0414123535156,
+      "logps/rejected": -252.41909790039062,
+      "loss": 0.5164,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.03142847493290901,
-      "rewards/margins": 0.1530241072177887,
-      "rewards/rejected": -0.12159563601016998,
+      "rewards/chosen": -0.010256086476147175,
+      "rewards/margins": 0.03359478712081909,
+      "rewards/rejected": -0.04385087639093399,
       "step": 30
     },
     {
       "epoch": 0.11,
-      "learning_rate": 1.999224195661986e-06,
-      "logits/chosen": -0.08356816321611404,
-      "logits/rejected": 0.00294572114944458,
-      "logps/chosen": -316.5859680175781,
-      "logps/rejected": -197.39263916015625,
-      "loss": 0.5192,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.1317664086818695,
-      "rewards/margins": 0.20280234515666962,
-      "rewards/rejected": -0.3345687687397003,
+      "learning_rate": 1.1267605633802817e-06,
+      "logits/chosen": -0.041558656841516495,
+      "logits/rejected": 0.009781199507415295,
+      "logps/chosen": -295.82379150390625,
+      "logps/rejected": -205.8635711669922,
+      "loss": 0.5435,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.059349894523620605,
+      "rewards/margins": 0.10006687790155411,
+      "rewards/rejected": -0.1594167947769165,
       "step": 40
     },
     {
       "epoch": 0.14,
-      "learning_rate": 1.9905102152171726e-06,
-      "logits/chosen": -0.12071399390697479,
-      "logits/rejected": -0.04788393899798393,
-      "logps/chosen": -332.5281066894531,
-      "logps/rejected": -256.2742004394531,
-      "loss": 0.4751,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.09092732518911362,
-      "rewards/margins": 0.18247266113758087,
-      "rewards/rejected": -0.2734000086784363,
+      "learning_rate": 1.408450704225352e-06,
+      "logits/chosen": 0.018238263204693794,
+      "logits/rejected": 0.06442906707525253,
+      "logps/chosen": -377.36151123046875,
+      "logps/rejected": -239.4322052001953,
+      "loss": 0.5573,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.0925443023443222,
+      "rewards/margins": 0.19226112961769104,
+      "rewards/rejected": -0.28480541706085205,
       "step": 50
     },
     {
       "epoch": 0.17,
-      "learning_rate": 1.9721972279988477e-06,
-      "logits/chosen": -0.22579865157604218,
-      "logits/rejected": -0.1116095557808876,
-      "logps/chosen": -396.06378173828125,
-      "logps/rejected": -215.12844848632812,
-      "loss": 0.4328,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.09131719172000885,
-      "rewards/margins": 0.4695609211921692,
-      "rewards/rejected": -0.5608780384063721,
+      "learning_rate": 1.6901408450704225e-06,
+      "logits/chosen": -0.07315438240766525,
+      "logits/rejected": -0.011159000918269157,
+      "logps/chosen": -319.10455322265625,
+      "logps/rejected": -229.79824829101562,
+      "loss": 0.5171,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.007246834225952625,
+      "rewards/margins": 0.14170756936073303,
+      "rewards/rejected": -0.13446073234081268,
       "step": 60
     },
     {
       "epoch": 0.2,
-      "learning_rate": 1.9444627046536053e-06,
-      "logits/chosen": -0.21056826412677765,
-      "logits/rejected": -0.09725789725780487,
-      "logps/chosen": -391.73223876953125,
-      "logps/rejected": -242.24807739257812,
-      "loss": 0.426,
+      "learning_rate": 1.971830985915493e-06,
+      "logits/chosen": -0.11702132225036621,
+      "logits/rejected": -0.06476567685604095,
+      "logps/chosen": -346.78509521484375,
+      "logps/rejected": -251.37466430664062,
+      "loss": 0.5065,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.25291183590888977,
-      "rewards/margins": 0.42755264043807983,
-      "rewards/rejected": -0.6804644465446472,
+      "rewards/chosen": 0.046701572835445404,
+      "rewards/margins": 0.21416659653186798,
+      "rewards/rejected": -0.16746501624584198,
       "step": 70
     },
     {
       "epoch": 0.22,
-      "learning_rate": 1.907575419670957e-06,
-      "logits/chosen": -0.15251095592975616,
-      "logits/rejected": -0.10242275148630142,
-      "logps/chosen": -286.48260498046875,
-      "logps/rejected": -209.681884765625,
-      "loss": 0.3996,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.2705255448818207,
-      "rewards/margins": 0.31923907995224,
-      "rewards/rejected": -0.5897646546363831,
+      "learning_rate": 1.9990212265199736e-06,
+      "logits/chosen": -0.011112675070762634,
+      "logits/rejected": 0.0456564836204052,
+      "logps/chosen": -316.1672668457031,
+      "logps/rejected": -235.3020782470703,
+      "loss": 0.5047,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.035639651119709015,
+      "rewards/margins": 0.16450051963329315,
+      "rewards/rejected": -0.20014019310474396,
       "step": 80
     },
     {
       "epoch": 0.25,
-      "learning_rate": 1.861892846697277e-06,
-      "logits/chosen": -0.2654028832912445,
-      "logits/rejected": -0.21472935378551483,
-      "logps/chosen": -388.46490478515625,
-      "logps/rejected": -240.60079956054688,
-      "loss": 0.3895,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.2387591302394867,
-      "rewards/margins": 0.526225209236145,
-      "rewards/rejected": -0.7649842500686646,
+      "learning_rate": 1.995640271796129e-06,
+      "logits/chosen": -0.06501901894807816,
+      "logits/rejected": -0.005411559250205755,
+      "logps/chosen": -363.4024963378906,
+      "logps/rejected": -284.32769775390625,
+      "loss": 0.4601,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.007450317498296499,
+      "rewards/margins": 0.165993332862854,
+      "rewards/rejected": -0.15854302048683167,
       "step": 90
     },
     {
       "epoch": 0.28,
-      "learning_rate": 1.8078576942687008e-06,
-      "logits/chosen": -0.28679296374320984,
-      "logits/rejected": -0.2434661090373993,
-      "logps/chosen": -391.65081787109375,
-      "logps/rejected": -258.97772216796875,
-      "loss": 0.3588,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.48451095819473267,
-      "rewards/margins": 0.38693904876708984,
-      "rewards/rejected": -0.8714500665664673,
+      "learning_rate": 1.9898532207817787e-06,
+      "logits/chosen": -0.06524594128131866,
+      "logits/rejected": 0.0007957996567711234,
+      "logps/chosen": -287.18389892578125,
+      "logps/rejected": -225.53396606445312,
+      "loss": 0.4422,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.07524963468313217,
+      "rewards/margins": 0.12944354116916656,
+      "rewards/rejected": -0.20469316840171814,
       "step": 100
     },
     {
       "epoch": 0.31,
-      "learning_rate": 1.7459936155350907e-06,
-      "logits/chosen": -0.35570335388183594,
-      "logits/rejected": -0.3073640465736389,
-      "logps/chosen": -344.8442687988281,
-      "logps/rejected": -265.22894287109375,
-      "loss": 0.3365,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.5254753232002258,
-      "rewards/margins": 0.33020466566085815,
-      "rewards/rejected": -0.855679988861084,
+      "learning_rate": 1.9816740586504575e-06,
+      "logits/chosen": -0.05093986541032791,
+      "logits/rejected": -0.023259857669472694,
+      "logps/chosen": -346.22552490234375,
+      "logps/rejected": -273.53070068359375,
+      "loss": 0.4475,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.15637585520744324,
+      "rewards/margins": 0.209220290184021,
+      "rewards/rejected": -0.36559611558914185,
       "step": 110
     },
     {
       "epoch": 0.34,
-      "learning_rate": 1.6769001335520179e-06,
-      "logits/chosen": -0.4098650813102722,
-      "logits/rejected": -0.3523326814174652,
-      "logps/chosen": -385.4635314941406,
-      "logps/rejected": -273.0743408203125,
-      "loss": 0.3171,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.6758801937103271,
-      "rewards/margins": 0.47352728247642517,
-      "rewards/rejected": -1.1494075059890747,
+      "learning_rate": 1.971122551428331e-06,
+      "logits/chosen": -0.1468496024608612,
+      "logits/rejected": -0.08133789896965027,
+      "logps/chosen": -362.974609375,
+      "logps/rejected": -242.92843627929688,
+      "loss": 0.4502,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.23508784174919128,
+      "rewards/margins": 0.3363807797431946,
+      "rewards/rejected": -0.571468710899353,
       "step": 120
     },
     {
       "epoch": 0.37,
-      "learning_rate": 1.6012468313196084e-06,
-      "logits/chosen": -0.46511369943618774,
-      "logits/rejected": -0.414213091135025,
-      "logps/chosen": -450.9639587402344,
-      "logps/rejected": -316.42022705078125,
-      "loss": 0.2859,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -1.0311425924301147,
-      "rewards/margins": 0.6003210544586182,
-      "rewards/rejected": -1.631463646888733,
+      "learning_rate": 1.9582241982269803e-06,
+      "logits/chosen": -0.18417930603027344,
+      "logits/rejected": -0.09607286751270294,
+      "logps/chosen": -402.79461669921875,
+      "logps/rejected": -280.8565979003906,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2887083888053894,
+      "rewards/margins": 0.3183668255805969,
+      "rewards/rejected": -0.6070752143859863,
       "step": 130
     },
     {
       "epoch": 0.39,
-      "learning_rate": 1.51976686287243e-06,
-      "logits/chosen": -0.4082161486148834,
-      "logits/rejected": -0.3946232795715332,
-      "logps/chosen": -385.8199768066406,
-      "logps/rejected": -287.4693603515625,
-      "loss": 0.2727,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.8687955737113953,
-      "rewards/margins": 0.3416849970817566,
-      "rewards/rejected": -1.2104805707931519,
+      "learning_rate": 1.9430101696214336e-06,
+      "logits/chosen": -0.24745997786521912,
+      "logits/rejected": -0.15786592662334442,
+      "logps/chosen": -370.55670166015625,
+      "logps/rejected": -257.5110168457031,
+      "loss": 0.3939,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.29878249764442444,
+      "rewards/margins": 0.2513308823108673,
+      "rewards/rejected": -0.5501133799552917,
       "step": 140
     },
     {
       "epoch": 0.42,
-      "learning_rate": 1.4332498483042636e-06,
-      "logits/chosen": -0.32807427644729614,
-      "logits/rejected": -0.34388530254364014,
-      "logps/chosen": -380.92181396484375,
-      "logps/rejected": -347.18682861328125,
-      "loss": 0.2712,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.110830545425415,
-      "rewards/margins": 0.5045827627182007,
-      "rewards/rejected": -1.6154134273529053,
+      "learning_rate": 1.9255172323223463e-06,
+      "logits/chosen": -0.16580908000469208,
+      "logits/rejected": -0.11968035995960236,
+      "logps/chosen": -318.001708984375,
+      "logps/rejected": -258.27215576171875,
+      "loss": 0.4097,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3942390978336334,
+      "rewards/margins": 0.2680663764476776,
+      "rewards/rejected": -0.662305474281311,
       "step": 150
     },
     {
       "epoch": 0.45,
-      "learning_rate": 1.3425342215818716e-06,
-      "logits/chosen": -0.4168078899383545,
-      "logits/rejected": -0.39402490854263306,
-      "logps/chosen": -463.38140869140625,
-      "logps/rejected": -365.8263854980469,
-      "loss": 0.2538,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.1672489643096924,
-      "rewards/margins": 0.6873351335525513,
-      "rewards/rejected": -1.854583978652954,
+      "learning_rate": 1.905787660324391e-06,
+      "logits/chosen": -0.249535471200943,
+      "logits/rejected": -0.19402232766151428,
+      "logps/chosen": -375.15972900390625,
+      "logps/rejected": -257.4052429199219,
+      "loss": 0.3704,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.31844764947891235,
+      "rewards/margins": 0.20566639304161072,
+      "rewards/rejected": -0.5241140127182007,
       "step": 160
     },
     {
       "epoch": 0.48,
-      "learning_rate": 1.248499105304894e-06,
-      "logits/chosen": -0.4047406315803528,
-      "logits/rejected": -0.4051821827888489,
-      "logps/chosen": -429.53961181640625,
-      "logps/rejected": -394.43450927734375,
-      "loss": 0.2281,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.3412402868270874,
-      "rewards/margins": 0.7463828921318054,
-      "rewards/rejected": -2.087623119354248,
+      "learning_rate": 1.8838691327455609e-06,
+      "logits/chosen": -0.2230146825313568,
+      "logits/rejected": -0.17071188986301422,
+      "logps/chosen": -381.15631103515625,
+      "logps/rejected": -274.1378173828125,
+      "loss": 0.405,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3613168001174927,
+      "rewards/margins": 0.29612740874290466,
+      "rewards/rejected": -0.6574442386627197,
       "step": 170
     },
     {
       "epoch": 0.51,
-      "learning_rate": 1.1520557911533388e-06,
-      "logits/chosen": -0.5203784704208374,
-      "logits/rejected": -0.4828321933746338,
-      "logps/chosen": -459.611572265625,
-      "logps/rejected": -371.26312255859375,
-      "loss": 0.2364,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -1.5552146434783936,
-      "rewards/margins": 0.40483903884887695,
-      "rewards/rejected": -1.96005380153656,
+      "learning_rate": 1.8598146186042808e-06,
+      "logits/chosen": -0.16511419415473938,
+      "logits/rejected": -0.11704270541667938,
+      "logps/chosen": -397.783203125,
+      "logps/rejected": -304.19879150390625,
+      "loss": 0.437,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5645850300788879,
+      "rewards/margins": 0.3246172070503235,
+      "rewards/rejected": -0.8892022967338562,
       "step": 180
     },
     {
       "epoch": 0.53,
-      "learning_rate": 1.0541389085854176e-06,
-      "logits/chosen": -0.4102029800415039,
-      "logits/rejected": -0.36990243196487427,
-      "logps/chosen": -395.46234130859375,
-      "logps/rejected": -275.96533203125,
-      "loss": 0.2695,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.7516598701477051,
-      "rewards/margins": 0.5604479908943176,
-      "rewards/rejected": -1.312107801437378,
+      "learning_rate": 1.8336822488127723e-06,
+      "logits/chosen": -0.27783218026161194,
+      "logits/rejected": -0.2260020524263382,
+      "logps/chosen": -405.33746337890625,
+      "logps/rejected": -301.2474365234375,
+      "loss": 0.3796,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.5456187725067139,
+      "rewards/margins": 0.2841017246246338,
+      "rewards/rejected": -0.8297204971313477,
       "step": 190
     },
     {
       "epoch": 0.56,
-      "learning_rate": 9.556973673696213e-07,
-      "logits/chosen": -0.44092226028442383,
-      "logits/rejected": -0.3557354211807251,
-      "logps/chosen": -399.0208435058594,
-      "logps/rejected": -312.0525817871094,
-      "loss": 0.272,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.0618005990982056,
-      "rewards/margins": 0.38875648379325867,
-      "rewards/rejected": -1.450556993484497,
+      "learning_rate": 1.805535175696026e-06,
+      "logits/chosen": -0.30949804186820984,
+      "logits/rejected": -0.28316643834114075,
+      "logps/chosen": -362.0370178222656,
+      "logps/rejected": -315.319580078125,
+      "loss": 0.3464,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.5522123575210571,
+      "rewards/margins": 0.22715091705322266,
+      "rewards/rejected": -0.7793632745742798,
       "step": 200
     },
     {
       "epoch": 0.59,
-      "learning_rate": 8.576851617267149e-07,
-      "logits/chosen": -0.4460463523864746,
-      "logits/rejected": -0.4336074888706207,
-      "logps/chosen": -500.722412109375,
-      "logps/rejected": -389.33233642578125,
-      "loss": 0.245,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -1.3941071033477783,
-      "rewards/margins": 0.843894362449646,
-      "rewards/rejected": -2.2380013465881348,
+      "learning_rate": 1.7754414203758602e-06,
+      "logits/chosen": -0.28907471895217896,
+      "logits/rejected": -0.2575899660587311,
+      "logps/chosen": -353.92681884765625,
+      "logps/rejected": -277.845458984375,
+      "loss": 0.3405,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6586817502975464,
+      "rewards/margins": 0.22525759041309357,
+      "rewards/rejected": -0.883939266204834,
       "step": 210
     },
     {
       "epoch": 0.62,
-      "learning_rate": 7.610521251984419e-07,
-      "logits/chosen": -0.4878757894039154,
-      "logits/rejected": -0.4867759346961975,
-      "logps/chosen": -469.9151306152344,
-      "logps/rejected": -378.65228271484375,
-      "loss": 0.2402,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.3634860515594482,
-      "rewards/margins": 0.7561300992965698,
-      "rewards/rejected": -2.1196160316467285,
+      "learning_rate": 1.7434737083888904e-06,
+      "logits/chosen": -0.31758958101272583,
+      "logits/rejected": -0.3126702904701233,
+      "logps/chosen": -361.3816833496094,
+      "logps/rejected": -286.2841796875,
+      "loss": 0.3756,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.6541340351104736,
+      "rewards/margins": 0.27324938774108887,
+      "rewards/rejected": -0.9273834228515625,
       "step": 220
     },
     {
       "epoch": 0.65,
-      "learning_rate": 6.667347258372558e-07,
-      "logits/chosen": -0.44877737760543823,
-      "logits/rejected": -0.4299391210079193,
-      "logps/chosen": -442.70233154296875,
-      "logps/rejected": -345.2084655761719,
-      "loss": 0.215,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.3881723880767822,
-      "rewards/margins": 0.553728461265564,
-      "rewards/rejected": -1.9419008493423462,
+      "learning_rate": 1.709709293935662e-06,
+      "logits/chosen": -0.37347474694252014,
+      "logits/rejected": -0.3521464467048645,
+      "logps/chosen": -406.6834411621094,
+      "logps/rejected": -342.6564025878906,
+      "loss": 0.3197,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.7559819221496582,
+      "rewards/margins": 0.2710956037044525,
+      "rewards/rejected": -1.0270774364471436,
       "step": 230
     },
     {
       "epoch": 0.67,
-      "learning_rate": 5.756469909206334e-07,
-      "logits/chosen": -0.46246570348739624,
-      "logits/rejected": -0.3849286139011383,
-      "logps/chosen": -504.61883544921875,
-      "logps/rejected": -392.6059875488281,
-      "loss": 0.1812,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.5720837116241455,
-      "rewards/margins": 0.6570929288864136,
-      "rewards/rejected": -2.2291767597198486,
+      "learning_rate": 1.6742297731856636e-06,
+      "logits/chosen": -0.3079659342765808,
+      "logits/rejected": -0.2845328748226166,
+      "logps/chosen": -397.04522705078125,
+      "logps/rejected": -340.85748291015625,
+      "loss": 0.2793,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.8156784176826477,
+      "rewards/margins": 0.31112828850746155,
+      "rewards/rejected": -1.1268064975738525,
       "step": 240
     },
     {
       "epoch": 0.7,
-      "learning_rate": 4.88671649138311e-07,
-      "logits/chosen": -0.46375352144241333,
-      "logits/rejected": -0.41425347328186035,
-      "logps/chosen": -511.947265625,
-      "logps/rejected": -397.8071594238281,
-      "loss": 0.1967,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.5960875749588013,
-      "rewards/margins": 0.8036051988601685,
-      "rewards/rejected": -2.3996925354003906,
+      "learning_rate": 1.6371208870894001e-06,
+      "logits/chosen": -0.2892235517501831,
+      "logits/rejected": -0.2505740821361542,
+      "logps/chosen": -432.062255859375,
+      "logps/rejected": -362.1026306152344,
+      "loss": 0.2851,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.0684902667999268,
+      "rewards/margins": 0.2997783124446869,
+      "rewards/rejected": -1.368268609046936,
       "step": 250
     },
     {
       "epoch": 0.73,
-      "learning_rate": 4.0665157609325563e-07,
-      "logits/chosen": -0.5020807981491089,
-      "logits/rejected": -0.4912947714328766,
-      "logps/chosen": -487.6549377441406,
-      "logps/rejected": -400.78643798828125,
-      "loss": 0.2117,
+      "learning_rate": 1.5984723141740574e-06,
+      "logits/chosen": -0.36091741919517517,
+      "logits/rejected": -0.3296750485897064,
+      "logps/chosen": -361.447021484375,
+      "logps/rejected": -308.85284423828125,
+      "loss": 0.3182,
       "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.5476312637329102,
-      "rewards/margins": 0.7439574003219604,
-      "rewards/rejected": -2.291588544845581,
+      "rewards/chosen": -0.8450363874435425,
+      "rewards/margins": 0.2505396008491516,
+      "rewards/rejected": -1.0955758094787598,
       "step": 260
     },
     {
       "epoch": 0.76,
-      "learning_rate": 3.303816260177894e-07,
-      "logits/chosen": -0.4641796052455902,
-      "logits/rejected": -0.49887222051620483,
-      "logps/chosen": -459.90106201171875,
-      "logps/rejected": -416.5171813964844,
-      "loss": 0.2198,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.617098093032837,
-      "rewards/margins": 0.6207329034805298,
-      "rewards/rejected": -2.237830877304077,
+      "learning_rate": 1.5583774538234882e-06,
+      "logits/chosen": -0.26542288064956665,
+      "logits/rejected": -0.2120533287525177,
+      "logps/chosen": -402.22021484375,
+      "logps/rejected": -305.82586669921875,
+      "loss": 0.3267,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.8468425869941711,
+      "rewards/margins": 0.3782137632369995,
+      "rewards/rejected": -1.2250562906265259,
       "step": 270
     },
     {
       "epoch": 0.79,
-      "learning_rate": 2.6060092886346885e-07,
-      "logits/chosen": -0.39373284578323364,
-      "logits/rejected": -0.38700538873672485,
-      "logps/chosen": -497.85321044921875,
-      "logps/rejected": -401.1759338378906,
-      "loss": 0.2288,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.4469443559646606,
-      "rewards/margins": 0.7936018705368042,
-      "rewards/rejected": -2.240546226501465,
+      "learning_rate": 1.5169332005662589e-06,
+      "logits/chosen": -0.3561258912086487,
+      "logits/rejected": -0.3128196597099304,
+      "logps/chosen": -451.0619201660156,
+      "logps/rejected": -341.95355224609375,
+      "loss": 0.3046,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.7824829816818237,
+      "rewards/margins": 0.41102856397628784,
+      "rewards/rejected": -1.1935116052627563,
       "step": 280
     },
     {
       "epoch": 0.81,
-      "learning_rate": 1.9798572741341148e-07,
-      "logits/chosen": -0.3860500752925873,
-      "logits/rejected": -0.3293386399745941,
-      "logps/chosen": -446.89111328125,
-      "logps/rejected": -367.6509094238281,
-      "loss": 0.226,
+      "learning_rate": 1.474239709917218e-06,
+      "logits/chosen": -0.32344120740890503,
+      "logits/rejected": -0.30548325181007385,
+      "logps/chosen": -421.78668212890625,
+      "logps/rejected": -358.4979553222656,
+      "loss": 0.2834,
       "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.3792494535446167,
-      "rewards/margins": 0.6391651034355164,
-      "rewards/rejected": -2.0184144973754883,
+      "rewards/chosen": -0.8403790593147278,
+      "rewards/margins": 0.33261531591415405,
+      "rewards/rejected": -1.1729944944381714,
       "step": 290
     },
     {
       "epoch": 0.84,
-      "learning_rate": 1.4314282383241095e-07,
-      "logits/chosen": -0.43343037366867065,
-      "logits/rejected": -0.4359092116355896,
-      "logps/chosen": -452.3334045410156,
-      "logps/rejected": -351.99755859375,
-      "loss": 0.2277,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.0607109069824219,
-      "rewards/margins": 0.8778518438339233,
-      "rewards/rejected": -1.9385627508163452,
+      "learning_rate": 1.430400156338457e-06,
+      "logits/chosen": -0.3465970456600189,
+      "logits/rejected": -0.3216686546802521,
+      "logps/chosen": -420.833740234375,
+      "logps/rejected": -353.2898864746094,
+      "loss": 0.2659,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.7795278429985046,
+      "rewards/margins": 0.21080787479877472,
+      "rewards/rejected": -0.9903356432914734,
       "step": 300
     },
     {
       "epoch": 0.87,
-      "learning_rate": 9.660369916414013e-08,
-      "logits/chosen": -0.4283617436885834,
-      "logits/rejected": -0.43107232451438904,
-      "logps/chosen": -437.2748107910156,
-      "logps/rejected": -371.6737976074219,
-      "loss": 0.2184,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.2600263357162476,
-      "rewards/margins": 0.5959557294845581,
-      "rewards/rejected": -1.8559820652008057,
+      "learning_rate": 1.3855204839045892e-06,
+      "logits/chosen": -0.3354475796222687,
+      "logits/rejected": -0.35353055596351624,
+      "logps/chosen": -389.02545166015625,
+      "logps/rejected": -333.0558776855469,
+      "loss": 0.2704,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.0605363845825195,
+      "rewards/margins": 0.15861138701438904,
+      "rewards/rejected": -1.219147801399231,
       "step": 310
     },
     {
       "epoch": 0.9,
-      "learning_rate": 5.881936276323462e-08,
-      "logits/chosen": -0.3460317552089691,
-      "logits/rejected": -0.34460192918777466,
-      "logps/chosen": -416.1355895996094,
-      "logps/rejected": -334.4111328125,
-      "loss": 0.2229,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -1.2378016710281372,
-      "rewards/margins": 0.5023729205131531,
-      "rewards/rejected": -1.7401745319366455,
+      "learning_rate": 1.3397091502748927e-06,
+      "logits/chosen": -0.39062121510505676,
+      "logits/rejected": -0.32264286279678345,
+      "logps/chosen": -479.73822021484375,
+      "logps/rejected": -373.89080810546875,
+      "loss": 0.3049,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.9773856997489929,
+      "rewards/margins": 0.4283124506473541,
+      "rewards/rejected": -1.4056981801986694,
       "step": 320
     },
     {
       "epoch": 0.93,
-      "learning_rate": 3.015598157625598e-08,
-      "logits/chosen": -0.46672359108924866,
-      "logits/rejected": -0.43555861711502075,
-      "logps/chosen": -527.4755249023438,
-      "logps/rejected": -394.07135009765625,
-      "loss": 0.2113,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.4215977191925049,
-      "rewards/margins": 0.7233748435974121,
-      "rewards/rejected": -2.144972324371338,
+      "learning_rate": 1.2930768645910449e-06,
+      "logits/chosen": -0.302212119102478,
+      "logits/rejected": -0.2810123860836029,
+      "logps/chosen": -395.77923583984375,
+      "logps/rejected": -299.944091796875,
+      "loss": 0.3347,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.663515567779541,
+      "rewards/margins": 0.3672025799751282,
+      "rewards/rejected": -1.0307180881500244,
       "step": 330
     },
     {
       "epoch": 0.96,
-      "learning_rate": 1.0891331628063882e-08,
-      "logits/chosen": -0.43030333518981934,
-      "logits/rejected": -0.39171096682548523,
-      "logps/chosen": -477.61883544921875,
-      "logps/rejected": -401.7641906738281,
-      "loss": 0.2215,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.4620156288146973,
-      "rewards/margins": 0.7660819888114929,
-      "rewards/rejected": -2.228097677230835,
+      "learning_rate": 1.2457363199338495e-06,
+      "logits/chosen": -0.2840663194656372,
+      "logits/rejected": -0.2467239648103714,
+      "logps/chosen": -466.4864196777344,
+      "logps/rejected": -333.0160217285156,
+      "loss": 0.3193,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7206543684005737,
+      "rewards/margins": 0.4044593274593353,
+      "rewards/rejected": -1.1251137256622314,
       "step": 340
     },
     {
       "epoch": 0.98,
-      "learning_rate": 1.212106102131849e-09,
-      "logits/chosen": -0.45886820554733276,
-      "logits/rejected": -0.4877847731113434,
-      "logps/chosen": -446.5228576660156,
-      "logps/rejected": -395.54852294921875,
-      "loss": 0.2179,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.3368722200393677,
-      "rewards/margins": 0.8557122945785522,
-      "rewards/rejected": -2.19258451461792,
+      "learning_rate": 1.1978019209855173e-06,
+      "logits/chosen": -0.32623833417892456,
+      "logits/rejected": -0.26132825016975403,
+      "logps/chosen": -438.72607421875,
+      "logps/rejected": -354.7231140136719,
+      "loss": 0.2759,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7977033853530884,
+      "rewards/margins": 0.4479256272315979,
+      "rewards/rejected": -1.245629072189331,
       "step": 350
     },
     {
-      "epoch": 1.0,
-      "step": 355,
+      "epoch": 1.01,
+      "learning_rate": 1.14938950755563e-06,
+      "logits/chosen": -0.31017881631851196,
+      "logits/rejected": -0.296619713306427,
+      "logps/chosen": -379.47064208984375,
+      "logps/rejected": -352.1556091308594,
+      "loss": 0.2171,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.9407274127006531,
+      "rewards/margins": 0.6167389154434204,
+      "rewards/rejected": -1.5574663877487183,
+      "step": 360
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 1.1006160746389332e-06,
+      "logits/chosen": -0.45612698793411255,
+      "logits/rejected": -0.4245428442955017,
+      "logps/chosen": -480.1871032714844,
+      "logps/rejected": -501.958740234375,
+      "loss": 0.0979,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.4988553524017334,
+      "rewards/margins": 1.2996289730072021,
+      "rewards/rejected": -2.7984843254089355,
+      "step": 370
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 1.0515994896814731e-06,
+      "logits/chosen": -0.32600560784339905,
+      "logits/rejected": -0.2724960744380951,
+      "logps/chosen": -578.824951171875,
+      "logps/rejected": -540.9967651367188,
+      "loss": 0.0553,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.1792449951171875,
+      "rewards/margins": 1.2840534448623657,
+      "rewards/rejected": -3.463298797607422,
+      "step": 380
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 1.002458207738333e-06,
+      "logits/chosen": -0.25963398814201355,
+      "logits/rejected": -0.22093424201011658,
+      "logps/chosen": -530.6353759765625,
+      "logps/rejected": -565.1375122070312,
+      "loss": 0.0574,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.7918068170547485,
+      "rewards/margins": 1.450157880783081,
+      "rewards/rejected": -3.241964817047119,
+      "step": 390
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 9.533109852113413e-07,
+      "logits/chosen": -0.21417060494422913,
+      "logits/rejected": -0.18584421277046204,
+      "logps/chosen": -536.5563354492188,
+      "logps/rejected": -544.37890625,
+      "loss": 0.047,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -1.9521188735961914,
+      "rewards/margins": 1.5740001201629639,
+      "rewards/rejected": -3.526118755340576,
+      "step": 400
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 9.042765928585326e-07,
+      "logits/chosen": -0.12677066028118134,
+      "logits/rejected": -0.11945654451847076,
+      "logps/chosen": -547.3245849609375,
+      "logps/rejected": -567.9544067382812,
+      "loss": 0.0401,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -2.1687557697296143,
+      "rewards/margins": 1.4874060153961182,
+      "rewards/rejected": -3.6561615467071533,
+      "step": 410
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 8.554735287689148e-07,
+      "logits/chosen": -0.14333295822143555,
+      "logits/rejected": -0.09979396313428879,
+      "logps/chosen": -634.6866455078125,
+      "logps/rejected": -613.4561767578125,
+      "loss": 0.0348,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -2.715937852859497,
+      "rewards/margins": 1.36255943775177,
+      "rewards/rejected": -4.078497409820557,
+      "step": 420
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 8.070197319961782e-07,
+      "logits/chosen": -0.10927991569042206,
+      "logits/rejected": -0.020826727151870728,
+      "logps/chosen": -580.0132446289062,
+      "logps/rejected": -619.2887573242188,
+      "loss": 0.0341,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.63529372215271,
+      "rewards/margins": 1.3534786701202393,
+      "rewards/rejected": -3.988771915435791,
+      "step": 430
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 7.590322975433856e-07,
+      "logits/chosen": -0.1185801774263382,
+      "logits/rejected": -0.03847898915410042,
+      "logps/chosen": -641.1858520507812,
+      "logps/rejected": -655.4496459960938,
+      "loss": 0.0317,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -2.531560182571411,
+      "rewards/margins": 1.7570297718048096,
+      "rewards/rejected": -4.288589954376221,
+      "step": 440
+    },
+    {
+      "epoch": 1.26,
+      "learning_rate": 7.116271933874245e-07,
+      "logits/chosen": -0.001342842006124556,
+      "logits/rejected": -0.04314468055963516,
+      "logps/chosen": -561.9754638671875,
+      "logps/rejected": -599.1416625976562,
+      "loss": 0.0298,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -2.415828227996826,
+      "rewards/margins": 1.5171138048171997,
+      "rewards/rejected": -3.9329421520233154,
+      "step": 450
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 6.649189802270652e-07,
+      "logits/chosen": -0.042498912662267685,
+      "logits/rejected": -0.012727165594696999,
+      "logps/chosen": -590.1478881835938,
+      "logps/rejected": -614.4903564453125,
+      "loss": 0.0297,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.558908224105835,
+      "rewards/margins": 1.5000147819519043,
+      "rewards/rejected": -4.05892276763916,
+      "step": 460
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 6.190205346318926e-07,
+      "logits/chosen": -0.005924136843532324,
+      "logits/rejected": 0.028465991839766502,
+      "logps/chosen": -631.77685546875,
+      "logps/rejected": -660.1925659179688,
+      "loss": 0.0263,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.74247145652771,
+      "rewards/margins": 1.6851346492767334,
+      "rewards/rejected": -4.427606105804443,
+      "step": 470
+    },
+    {
+      "epoch": 1.35,
+      "learning_rate": 5.740427762611604e-07,
+      "logits/chosen": 0.06077251955866814,
+      "logits/rejected": 0.14472587406635284,
+      "logps/chosen": -601.584716796875,
+      "logps/rejected": -623.94287109375,
+      "loss": 0.0282,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -2.6236908435821533,
+      "rewards/margins": 1.5524755716323853,
+      "rewards/rejected": -4.176166534423828,
+      "step": 480
+    },
+    {
+      "epoch": 1.38,
+      "learning_rate": 5.300943998117749e-07,
+      "logits/chosen": 0.058325447142124176,
+      "logits/rejected": 0.08030878007411957,
+      "logps/chosen": -585.04052734375,
+      "logps/rejected": -604.9239501953125,
+      "loss": 0.0261,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.509859561920166,
+      "rewards/margins": 1.6018766164779663,
+      "rewards/rejected": -4.111736297607422,
+      "step": 490
+    },
+    {
+      "epoch": 1.4,
+      "learning_rate": 4.872816123431976e-07,
+      "logits/chosen": 0.1183939203619957,
+      "logits/rejected": 0.08584292232990265,
+      "logps/chosen": -526.6102294921875,
+      "logps/rejected": -640.744873046875,
+      "loss": 0.0244,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -2.509054660797119,
+      "rewards/margins": 1.6073827743530273,
+      "rewards/rejected": -4.1164374351501465,
+      "step": 500
+    },
+    {
+      "epoch": 1.43,
+      "learning_rate": 4.4570787661405e-07,
+      "logits/chosen": 0.16773128509521484,
+      "logits/rejected": 0.2288293093442917,
+      "logps/chosen": -553.6146850585938,
+      "logps/rejected": -577.7164306640625,
+      "loss": 0.0246,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.75661039352417,
+      "rewards/margins": 1.317345380783081,
+      "rewards/rejected": -4.073955535888672,
+      "step": 510
+    },
+    {
+      "epoch": 1.46,
+      "learning_rate": 4.0547366105068347e-07,
+      "logits/chosen": 0.2202252447605133,
+      "logits/rejected": 0.26597005128860474,
+      "logps/chosen": -630.9942626953125,
+      "logps/rejected": -691.7838134765625,
+      "loss": 0.0242,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -2.9170384407043457,
+      "rewards/margins": 1.8351190090179443,
+      "rewards/rejected": -4.752157688140869,
+      "step": 520
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 3.666761969519528e-07,
+      "logits/chosen": 0.17129948735237122,
+      "logits/rejected": 0.2111537903547287,
+      "logps/chosen": -559.1640625,
+      "logps/rejected": -672.8851928710938,
+      "loss": 0.0224,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.7882485389709473,
+      "rewards/margins": 1.6843712329864502,
+      "rewards/rejected": -4.472619533538818,
+      "step": 530
+    },
+    {
+      "epoch": 1.52,
+      "learning_rate": 3.2940924351693213e-07,
+      "logits/chosen": 0.19039176404476166,
+      "logits/rejected": 0.21573393046855927,
+      "logps/chosen": -628.3604736328125,
+      "logps/rejected": -705.090087890625,
+      "loss": 0.0211,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.726500988006592,
+      "rewards/margins": 1.8815057277679443,
+      "rewards/rejected": -4.608006954193115,
+      "step": 540
+    },
+    {
+      "epoch": 1.55,
+      "learning_rate": 2.937628612634184e-07,
+      "logits/chosen": 0.23304316401481628,
+      "logits/rejected": 0.2361479252576828,
+      "logps/chosen": -604.546630859375,
+      "logps/rejected": -636.0623779296875,
+      "loss": 0.0197,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.8572418689727783,
+      "rewards/margins": 1.691147804260254,
+      "rewards/rejected": -4.548389911651611,
+      "step": 550
+    },
+    {
+      "epoch": 1.57,
+      "learning_rate": 2.598231943847916e-07,
+      "logits/chosen": 0.19534823298454285,
+      "logits/rejected": 0.2395998239517212,
+      "logps/chosen": -624.6097412109375,
+      "logps/rejected": -710.4221801757812,
+      "loss": 0.0194,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.994528293609619,
+      "rewards/margins": 1.8797197341918945,
+      "rewards/rejected": -4.8742475509643555,
+      "step": 560
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 2.276722625711861e-07,
+      "logits/chosen": 0.11042364686727524,
+      "logits/rejected": 0.18681105971336365,
+      "logps/chosen": -684.1854248046875,
+      "logps/rejected": -786.4215087890625,
+      "loss": 0.0194,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -3.0811171531677246,
+      "rewards/margins": 2.1122710704803467,
+      "rewards/rejected": -5.19338846206665,
+      "step": 570
+    },
+    {
+      "epoch": 1.63,
+      "learning_rate": 1.973877627980699e-07,
+      "logits/chosen": 0.11730021238327026,
+      "logits/rejected": 0.1679680496454239,
+      "logps/chosen": -716.9166259765625,
+      "logps/rejected": -759.8760375976562,
+      "loss": 0.0171,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -3.1296448707580566,
+      "rewards/margins": 1.810140609741211,
+      "rewards/rejected": -4.939785480499268,
+      "step": 580
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 1.6904288156123636e-07,
+      "logits/chosen": 0.25147971510887146,
+      "logits/rejected": 0.24109426140785217,
+      "logps/chosen": -578.8203735351562,
+      "logps/rejected": -653.1304321289062,
+      "loss": 0.0179,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.8619461059570312,
+      "rewards/margins": 1.7370202541351318,
+      "rewards/rejected": -4.598966598510742,
+      "step": 590
+    },
+    {
+      "epoch": 1.69,
+      "learning_rate": 1.4270611801196642e-07,
+      "logits/chosen": 0.10723473876714706,
+      "logits/rejected": 0.1447157859802246,
+      "logps/chosen": -672.2132568359375,
+      "logps/rejected": -709.0341796875,
+      "loss": 0.0178,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -3.1768195629119873,
+      "rewards/margins": 1.8273181915283203,
+      "rewards/rejected": -5.0041375160217285,
+      "step": 600
+    },
+    {
+      "epoch": 1.71,
+      "learning_rate": 1.1844111841977633e-07,
+      "logits/chosen": 0.2875896990299225,
+      "logits/rejected": 0.30075111985206604,
+      "logps/chosen": -604.6449584960938,
+      "logps/rejected": -719.7764892578125,
+      "loss": 0.0192,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -2.9635517597198486,
+      "rewards/margins": 1.8338435888290405,
+      "rewards/rejected": -4.797394752502441,
+      "step": 610
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 9.630652236279625e-08,
+      "logits/chosen": 0.221513032913208,
+      "logits/rejected": 0.3305627405643463,
+      "logps/chosen": -552.0277099609375,
+      "logps/rejected": -626.5653076171875,
+      "loss": 0.0202,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.8746533393859863,
+      "rewards/margins": 1.538907766342163,
+      "rewards/rejected": -4.4135613441467285,
+      "step": 620
+    },
+    {
+      "epoch": 1.77,
+      "learning_rate": 7.63558210174814e-08,
+      "logits/chosen": 0.1648671180009842,
+      "logits/rejected": 0.23953859508037567,
+      "logps/chosen": -628.3140869140625,
+      "logps/rejected": -684.0084228515625,
+      "loss": 0.0207,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -3.077697277069092,
+      "rewards/margins": 1.5620543956756592,
+      "rewards/rejected": -4.639751434326172,
+      "step": 630
+    },
+    {
+      "epoch": 1.8,
+      "learning_rate": 5.8637227890115273e-08,
+      "logits/chosen": 0.23085036873817444,
+      "logits/rejected": 0.26288902759552,
+      "logps/chosen": -644.9296875,
+      "logps/rejected": -711.1282348632812,
+      "loss": 0.0211,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.8837387561798096,
+      "rewards/margins": 1.888494849205017,
+      "rewards/rejected": -4.772233963012695,
+      "step": 640
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 4.3193562302499046e-08,
+      "logits/chosen": 0.21587777137756348,
+      "logits/rejected": 0.3251447081565857,
+      "logps/chosen": -587.2296752929688,
+      "logps/rejected": -655.6583251953125,
+      "loss": 0.0202,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.720548629760742,
+      "rewards/margins": 1.5969436168670654,
+      "rewards/rejected": -4.317492485046387,
+      "step": 650
+    },
+    {
+      "epoch": 1.85,
+      "learning_rate": 3.006214591340339e-08,
+      "logits/chosen": 0.20460787415504456,
+      "logits/rejected": 0.23312047123908997,
+      "logps/chosen": -648.768798828125,
+      "logps/rejected": -724.6717529296875,
+      "loss": 0.0191,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.980959177017212,
+      "rewards/margins": 1.9149051904678345,
+      "rewards/rejected": -4.895864009857178,
+      "step": 660
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 1.9274712525847447e-08,
+      "logits/chosen": 0.28748980164527893,
+      "logits/rejected": 0.27773481607437134,
+      "logps/chosen": -597.6702880859375,
+      "logps/rejected": -656.3522338867188,
+      "loss": 0.0206,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.8257060050964355,
+      "rewards/margins": 1.6687313318252563,
+      "rewards/rejected": -4.494436740875244,
+      "step": 670
+    },
+    {
+      "epoch": 1.91,
+      "learning_rate": 1.0857331398169577e-08,
+      "logits/chosen": 0.19040969014167786,
+      "logits/rejected": 0.18281084299087524,
+      "logps/chosen": -629.0142822265625,
+      "logps/rejected": -732.8785400390625,
+      "loss": 0.0189,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -2.8325679302215576,
+      "rewards/margins": 2.0608155727386475,
+      "rewards/rejected": -4.893383979797363,
+      "step": 680
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 4.830344244220686e-09,
+      "logits/chosen": 0.2010643184185028,
+      "logits/rejected": 0.30202826857566833,
+      "logps/chosen": -644.9835815429688,
+      "logps/rejected": -706.9172973632812,
+      "loss": 0.0224,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -3.023210287094116,
+      "rewards/margins": 1.628251314163208,
+      "rewards/rejected": -4.651461601257324,
+      "step": 690
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 1.2083160749236653e-09,
+      "logits/chosen": 0.16012658178806305,
+      "logits/rejected": 0.22160223126411438,
+      "logps/chosen": -615.0055541992188,
+      "logps/rejected": -672.9305419921875,
+      "loss": 0.0211,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -2.802346706390381,
+      "rewards/margins": 1.7504093647003174,
+      "rewards/rejected": -4.552755832672119,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "learning_rate": 0.0,
+      "logits/chosen": 0.26305443048477173,
+      "logits/rejected": 0.27064579725265503,
+      "logps/chosen": -540.3432006835938,
+      "logps/rejected": -643.52099609375,
+      "loss": 0.0194,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -2.756078004837036,
+      "rewards/margins": 1.6126619577407837,
+      "rewards/rejected": -4.368740081787109,
+      "step": 710
+    },
+    {
+      "epoch": 2.0,
+      "step": 710,
       "total_flos": 0.0,
-      "train_loss": 0.30111024614790793,
-      "train_runtime": 5278.2802,
-      "train_samples_per_second": 8.629,
+      "train_loss": 0.21065937945960272,
+      "train_runtime": 10560.1161,
+      "train_samples_per_second": 8.626,
       "train_steps_per_second": 0.067
     }
   ],
   "logging_steps": 10,
-  "max_steps": 355,
-  "num_train_epochs": 1,
+  "max_steps": 710,
+  "num_train_epochs": 2,
   "save_steps": 10000,
   "total_flos": 0.0,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d0b6200b167da15766e0f1c1654349573916e6e47dd7b4ffd0acb38743edddd
+oid sha256:896a531bb106e3c3ac0c19175569fde1dd12c8fcc0ef3098c749a00ff2d0b2f1
 size 6648