Model save

README.md

@@ -16,14 +16,14 @@ should probably proofread and complete it, then remove this comment. -->
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
 - Loss: 0.6931
-- Rewards/chosen: -2.1301
-- Rewards/rejected: -2.1301
+- Rewards/chosen: -1.7904
+- Rewards/rejected: -1.7904
 - Rewards/accuracies: 0.0
 - Rewards/margins: 0.0
-- Logps/rejected: -159.8372
-- Logps/chosen: -159.8372
-- Logits/rejected: -3.1995
-- Logits/chosen: -3.1995
+- Logps/rejected: -157.3674
+- Logps/chosen: -157.3674
+- Logits/rejected: -3.2202
+- Logits/chosen: -3.2202
 
 ## Model description
 
@@ -42,7 +42,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 2e-07
+- learning_rate: 5e-08
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -53,16 +53,15 @@ The following hyperparameters were used during training:
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
-- lr_scheduler_warmup_ratio: 0.1
 - num_epochs: 1
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.3114        | 0.29  | 100  | 0.6931          | -2.8588        | -2.8588          | 0.0                | 0.0             | -161.2947      | -161.2947    | -3.2262         | -3.2262       |
-| 0.2741        | 0.57  | 200  | 0.6931          | -2.6760        | -2.6760          | 0.0                | 0.0             | -160.9289      | -160.9289    | -3.2033         | -3.2033       |
-| 0.2695        | 0.86  | 300  | 0.6931          | -2.1301        | -2.1301          | 0.0                | 0.0             | -159.8372      | -159.8372    | -3.1995         | -3.1995       |
+| 0.3573        | 0.29  | 100  | 0.6931          | -2.2646        | -2.2646          | 0.0                | 0.0             | -157.8416      | -157.8416    | -3.2259         | -3.2259       |
+| 0.3184        | 0.57  | 200  | 0.6931          | -1.8023        | -1.8023          | 0.0                | 0.0             | -157.3793      | -157.3793    | -3.2195         | -3.2195       |
+| 0.3594        | 0.86  | 300  | 0.6931          | -1.7904        | -1.7904          | 0.0                | 0.0             | -157.3674      | -157.3674    | -3.2202         | -3.2202       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.30806411576476,
-    "train_runtime": 5299.0292,
-    "train_samples": 44682,
-    "train_samples_per_second": 8.432,
+    "train_loss": 0.36299856867109026,
+    "train_runtime": 5294.123,
+    "train_samples": 44755,
+    "train_samples_per_second": 8.454,
     "train_steps_per_second": 0.066
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.41.1",
+  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 32002
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff3d0ac00c3304811f4f56db0c3a49ea7bf825902fea45c8a9791a2e4f7431ab
+oid sha256:3d36163549571106621ebac26e38ec0b50cd9e0d81a1c0f2736d167eeed2d940
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1122a41f096dacbe12e70bff78c3951321a0a6f46db2c7995a726fd8f909bf84
+oid sha256:60f039e2e1484a47443bf956b71679bc1af07034027486d7035fb8ea530cd16f
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c18a2acaf553956d0cddde18a5d52d992dfb5723fc496a27bcde59c9bb62529
+oid sha256:552efc8fed558e148b25650586e753393bf9251331d7541a0c3aa3f226b236d6
 size 4540532728

runs/May30_23-06-15_n136-129-074/events.out.tfevents.1717081710.n136-129-074.2680508.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:900e43821138ab2f4858e5dd2280d3934b7a82a306dcb7660db2a9221551344a
-size 28313
+oid sha256:2eb9c3ef9c137948b60c969d7d5ff4259f0fd72becfb1520727015878e012f26
+size 32107

tokenizer.json

@@ -152,7 +152,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.30806411576476,
-    "train_runtime": 5299.0292,
-    "train_samples": 44682,
-    "train_samples_per_second": 8.432,
+    "train_loss": 0.36299856867109026,
+    "train_runtime": 5294.123,
+    "train_samples": 44755,
+    "train_samples_per_second": 8.454,
     "train_steps_per_second": 0.066
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9985693848354793,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 349,
+  "global_step": 350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 1483.079147156977,
-      "learning_rate": 5.7142857142857136e-09,
-      "logits/chosen": -4.490396976470947,
-      "logits/rejected": -4.787891387939453,
-      "logps/chosen": -300.56573486328125,
-      "logps/rejected": -263.39849853515625,
+      "grad_norm": 3067.8628948133914,
+      "learning_rate": 4.9998992904271775e-08,
+      "logits/chosen": -4.185730934143066,
+      "logits/rejected": -4.509836196899414,
+      "logps/chosen": -274.000732421875,
+      "logps/rejected": -205.8054962158203,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,574 +25,589 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 1678.9009350663757,
-      "learning_rate": 5.714285714285714e-08,
-      "logits/chosen": -4.292892932891846,
-      "logits/rejected": -4.4704790115356445,
-      "logps/chosen": -285.7367858886719,
-      "logps/rejected": -241.40879821777344,
-      "loss": 0.7232,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.03021715022623539,
-      "rewards/margins": 0.018014244735240936,
-      "rewards/rejected": 0.012202905490994453,
+      "grad_norm": 3330.3974170986107,
+      "learning_rate": 4.9899357349880975e-08,
+      "logits/chosen": -4.211880207061768,
+      "logits/rejected": -4.48573637008667,
+      "logps/chosen": -318.31072998046875,
+      "logps/rejected": -257.18267822265625,
+      "loss": 0.7459,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.200405091047287,
+      "rewards/margins": 0.10155472159385681,
+      "rewards/rejected": 0.09885036945343018,
       "step": 10
     },
     {
       "epoch": 0.06,
-      "grad_norm": 1307.5912774162387,
-      "learning_rate": 1.1428571428571427e-07,
-      "logits/chosen": -4.212913990020752,
-      "logits/rejected": -4.504936695098877,
-      "logps/chosen": -312.25714111328125,
-      "logps/rejected": -247.5605926513672,
-      "loss": 0.6348,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.25936827063560486,
-      "rewards/margins": 0.15551437437534332,
-      "rewards/rejected": 0.10385391861200333,
+      "grad_norm": 2932.727170813642,
+      "learning_rate": 4.959823971496574e-08,
+      "logits/chosen": -4.2464704513549805,
+      "logits/rejected": -4.50115966796875,
+      "logps/chosen": -304.53350830078125,
+      "logps/rejected": -244.1282501220703,
+      "loss": 0.6293,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.7030802965164185,
+      "rewards/margins": 0.6052380800247192,
+      "rewards/rejected": 0.09784229844808578,
       "step": 20
     },
     {
       "epoch": 0.09,
-      "grad_norm": 877.5830541670848,
-      "learning_rate": 1.714285714285714e-07,
-      "logits/chosen": -4.158999919891357,
-      "logits/rejected": -4.378296852111816,
-      "logps/chosen": -320.1895446777344,
-      "logps/rejected": -273.6168518066406,
-      "loss": 0.4394,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 1.6762861013412476,
-      "rewards/margins": 1.1370527744293213,
-      "rewards/rejected": 0.5392333269119263,
+      "grad_norm": 2159.097276891197,
+      "learning_rate": 4.9099071517396326e-08,
+      "logits/chosen": -4.3018364906311035,
+      "logits/rejected": -4.5636820793151855,
+      "logps/chosen": -305.11822509765625,
+      "logps/rejected": -258.89215087890625,
+      "loss": 0.5093,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 1.3964869976043701,
+      "rewards/margins": 0.9537334442138672,
+      "rewards/rejected": 0.44275355339050293,
       "step": 30
     },
     {
       "epoch": 0.11,
-      "grad_norm": 1094.7268353859254,
-      "learning_rate": 1.9987489925657699e-07,
-      "logits/chosen": -4.327412128448486,
-      "logits/rejected": -4.590438365936279,
-      "logps/chosen": -272.52813720703125,
-      "logps/rejected": -226.4479522705078,
-      "loss": 0.3519,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.0370898246765137,
-      "rewards/margins": 2.1227898597717285,
-      "rewards/rejected": 0.9143003225326538,
+      "grad_norm": 2233.10446662558,
+      "learning_rate": 4.8405871765993426e-08,
+      "logits/chosen": -4.304145812988281,
+      "logits/rejected": -4.571420192718506,
+      "logps/chosen": -293.4151916503906,
+      "logps/rejected": -234.4054412841797,
+      "loss": 0.4371,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 2.119215488433838,
+      "rewards/margins": 1.3193193674087524,
+      "rewards/rejected": 0.7998961806297302,
       "step": 40
     },
     {
       "epoch": 0.14,
-      "grad_norm": 1142.545986046605,
-      "learning_rate": 1.98875970549573e-07,
-      "logits/chosen": -4.374251365661621,
-      "logits/rejected": -4.627996921539307,
-      "logps/chosen": -264.49615478515625,
-      "logps/rejected": -226.27197265625,
-      "loss": 0.3689,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 4.1793413162231445,
-      "rewards/margins": 2.736725091934204,
-      "rewards/rejected": 1.4426158666610718,
+      "grad_norm": 1863.9092640792912,
+      "learning_rate": 4.7524221697560474e-08,
+      "logits/chosen": -4.298985481262207,
+      "logits/rejected": -4.545313835144043,
+      "logps/chosen": -299.71026611328125,
+      "logps/rejected": -252.57339477539062,
+      "loss": 0.4054,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 3.015381336212158,
+      "rewards/margins": 1.8283360004425049,
+      "rewards/rejected": 1.1870452165603638,
       "step": 50
     },
     {
       "epoch": 0.17,
-      "grad_norm": 635.1242404757879,
-      "learning_rate": 1.968881042201029e-07,
-      "logits/chosen": -4.35978889465332,
-      "logits/rejected": -4.594507694244385,
-      "logps/chosen": -296.86907958984375,
-      "logps/rejected": -251.1016845703125,
-      "loss": 0.3246,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 4.9698944091796875,
-      "rewards/margins": 3.8340022563934326,
-      "rewards/rejected": 1.1358922719955444,
+      "grad_norm": 1861.0742759245438,
+      "learning_rate": 4.646121984004665e-08,
+      "logits/chosen": -4.3018717765808105,
+      "logits/rejected": -4.5299859046936035,
+      "logps/chosen": -308.25457763671875,
+      "logps/rejected": -261.1996154785156,
+      "loss": 0.3815,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 3.097055673599243,
+      "rewards/margins": 1.6846046447753906,
+      "rewards/rejected": 1.412451148033142,
       "step": 60
     },
     {
       "epoch": 0.2,
-      "grad_norm": 871.5895225513464,
-      "learning_rate": 1.9393118250841894e-07,
-      "logits/chosen": -4.349625587463379,
-      "logits/rejected": -4.586968421936035,
-      "logps/chosen": -285.0860595703125,
-      "logps/rejected": -241.4550018310547,
-      "loss": 0.3259,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 4.21413516998291,
-      "rewards/margins": 3.7849984169006348,
-      "rewards/rejected": 0.42913690209388733,
+      "grad_norm": 2083.1341477087894,
+      "learning_rate": 4.522542485937369e-08,
+      "logits/chosen": -4.417206764221191,
+      "logits/rejected": -4.548245429992676,
+      "logps/chosen": -285.4747009277344,
+      "logps/rejected": -236.24136352539062,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 3.4294419288635254,
+      "rewards/margins": 2.4485509395599365,
+      "rewards/rejected": 0.9808910489082336,
       "step": 70
     },
     {
       "epoch": 0.23,
-      "grad_norm": 869.2800598030351,
-      "learning_rate": 1.900347799523094e-07,
-      "logits/chosen": -4.277868747711182,
-      "logits/rejected": -4.535857677459717,
-      "logps/chosen": -318.2683410644531,
-      "logps/rejected": -272.18206787109375,
-      "loss": 0.2501,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 4.3253655433654785,
-      "rewards/margins": 4.170114040374756,
-      "rewards/rejected": 0.15525080263614655,
+      "grad_norm": 1999.1118673285923,
+      "learning_rate": 4.3826786650090273e-08,
+      "logits/chosen": -4.271725177764893,
+      "logits/rejected": -4.525103569030762,
+      "logps/chosen": -292.2157897949219,
+      "logps/rejected": -239.5623321533203,
+      "loss": 0.3663,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.471898317337036,
+      "rewards/margins": 2.5827386379241943,
+      "rewards/rejected": 0.8891592025756836,
       "step": 80
     },
     {
       "epoch": 0.26,
-      "grad_norm": 616.9171514391553,
-      "learning_rate": 1.8523786758850436e-07,
-      "logits/chosen": -4.401219844818115,
-      "logits/rejected": -4.6477251052856445,
-      "logps/chosen": -280.7421569824219,
-      "logps/rejected": -236.2926025390625,
-      "loss": 0.2859,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.923023223876953,
-      "rewards/margins": 4.055254936218262,
-      "rewards/rejected": -0.13223108649253845,
+      "grad_norm": 1543.0151245523064,
+      "learning_rate": 4.2276566224671614e-08,
+      "logits/chosen": -4.196888446807861,
+      "logits/rejected": -4.430451393127441,
+      "logps/chosen": -303.9364929199219,
+      "logps/rejected": -258.19708251953125,
+      "loss": 0.37,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 3.6267776489257812,
+      "rewards/margins": 2.5005435943603516,
+      "rewards/rejected": 1.1262344121932983,
       "step": 90
     },
     {
       "epoch": 0.29,
-      "grad_norm": 735.4670744748103,
-      "learning_rate": 1.795884231721841e-07,
-      "logits/chosen": -4.318561553955078,
-      "logits/rejected": -4.602439880371094,
-      "logps/chosen": -267.45794677734375,
-      "logps/rejected": -232.0417938232422,
-      "loss": 0.3114,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 4.1319708824157715,
-      "rewards/margins": 4.702456951141357,
-      "rewards/rejected": -0.5704857110977173,
+      "grad_norm": 2558.2358091969077,
+      "learning_rate": 4.058724504646834e-08,
+      "logits/chosen": -4.298203468322754,
+      "logits/rejected": -4.51765251159668,
+      "logps/chosen": -291.99151611328125,
+      "logps/rejected": -240.97909545898438,
+      "loss": 0.3573,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 3.8364264965057373,
+      "rewards/margins": 2.6143250465393066,
+      "rewards/rejected": 1.2221016883850098,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "eval_logits/chosen": -3.226163148880005,
-      "eval_logits/rejected": -3.226163148880005,
-      "eval_logps/chosen": -161.294677734375,
-      "eval_logps/rejected": -161.294677734375,
+      "eval_logits/chosen": -3.2259409427642822,
+      "eval_logits/rejected": -3.2259409427642822,
+      "eval_logps/chosen": -157.8415985107422,
+      "eval_logps/rejected": -157.8415985107422,
       "eval_loss": 0.6931471824645996,
       "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": -2.8588294982910156,
+      "eval_rewards/chosen": -2.2645912170410156,
       "eval_rewards/margins": 0.0,
-      "eval_rewards/rejected": -2.8588294982910156,
-      "eval_runtime": 1.5114,
-      "eval_samples_per_second": 0.662,
-      "eval_steps_per_second": 0.662,
+      "eval_rewards/rejected": -2.2645912170410156,
+      "eval_runtime": 1.5044,
+      "eval_samples_per_second": 0.665,
+      "eval_steps_per_second": 0.665,
       "step": 100
     },
     {
       "epoch": 0.31,
-      "grad_norm": 817.8641381018862,
-      "learning_rate": 1.7314295131309637e-07,
-      "logits/chosen": -4.324714660644531,
-      "logits/rejected": -4.614729404449463,
-      "logps/chosen": -293.42364501953125,
-      "logps/rejected": -245.01620483398438,
-      "loss": 0.3079,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 4.0577192306518555,
-      "rewards/margins": 4.518521308898926,
-      "rewards/rejected": -0.4608024060726166,
+      "grad_norm": 2075.8470964199623,
+      "learning_rate": 3.8772424536302564e-08,
+      "logits/chosen": -4.3160247802734375,
+      "logits/rejected": -4.557186126708984,
+      "logps/chosen": -299.556640625,
+      "logps/rejected": -250.2120361328125,
+      "loss": 0.3653,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.8744053840637207,
+      "rewards/margins": 2.781764268875122,
+      "rewards/rejected": 1.0926413536071777,
       "step": 110
     },
     {
       "epoch": 0.34,
-      "grad_norm": 926.642960527248,
-      "learning_rate": 1.6596591832778466e-07,
-      "logits/chosen": -4.334780216217041,
-      "logits/rejected": -4.567566394805908,
-      "logps/chosen": -298.56976318359375,
-      "logps/rejected": -266.8869323730469,
-      "loss": 0.2848,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 3.83539080619812,
-      "rewards/margins": 4.040234088897705,
-      "rewards/rejected": -0.20484237372875214,
+      "grad_norm": 2129.2578794603846,
+      "learning_rate": 3.6846716561824964e-08,
+      "logits/chosen": -4.358242988586426,
+      "logits/rejected": -4.6036834716796875,
+      "logps/chosen": -288.9602966308594,
+      "logps/rejected": -237.98257446289062,
+      "loss": 0.346,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.973881959915161,
+      "rewards/margins": 2.8389506340026855,
+      "rewards/rejected": 1.1349313259124756,
       "step": 120
     },
     {
       "epoch": 0.37,
-      "grad_norm": 675.9811999128359,
-      "learning_rate": 1.5812910746042256e-07,
-      "logits/chosen": -4.3064866065979,
-      "logits/rejected": -4.553826332092285,
-      "logps/chosen": -286.5328063964844,
-      "logps/rejected": -242.8630828857422,
-      "loss": 0.2782,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 3.7822985649108887,
-      "rewards/margins": 4.371214389801025,
-      "rewards/rejected": -0.5889158248901367,
+      "grad_norm": 1374.3088736284383,
+      "learning_rate": 3.482562579134809e-08,
+      "logits/chosen": -4.360684871673584,
+      "logits/rejected": -4.608490467071533,
+      "logps/chosen": -278.861572265625,
+      "logps/rejected": -218.7367706298828,
+      "loss": 0.3426,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.8384926319122314,
+      "rewards/margins": 2.634833812713623,
+      "rewards/rejected": 1.2036586999893188,
       "step": 130
     },
     {
       "epoch": 0.4,
-      "grad_norm": 1048.4484509272795,
-      "learning_rate": 1.4971090092120542e-07,
-      "logits/chosen": -4.304565906524658,
-      "logits/rejected": -4.545400142669678,
-      "logps/chosen": -294.17095947265625,
-      "logps/rejected": -243.83935546875,
-      "loss": 0.269,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.8339390754699707,
-      "rewards/margins": 4.558647155761719,
-      "rewards/rejected": -0.7247086763381958,
+      "grad_norm": 1741.7465783603645,
+      "learning_rate": 3.272542485937369e-08,
+      "logits/chosen": -4.276978969573975,
+      "logits/rejected": -4.593733787536621,
+      "logps/chosen": -296.0984191894531,
+      "logps/rejected": -240.01248168945312,
+      "loss": 0.3729,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.785256862640381,
+      "rewards/margins": 2.9941701889038086,
+      "rewards/rejected": 0.79108726978302,
       "step": 140
     },
     {
       "epoch": 0.43,
-      "grad_norm": 844.0558457976633,
-      "learning_rate": 1.4079549592320782e-07,
-      "logits/chosen": -4.313709735870361,
-      "logits/rejected": -4.621119499206543,
-      "logps/chosen": -292.96661376953125,
-      "logps/rejected": -248.0418701171875,
-      "loss": 0.2727,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.8533847332000732,
-      "rewards/margins": 4.262558937072754,
-      "rewards/rejected": -0.4091736674308777,
+      "grad_norm": 1837.7137132104272,
+      "learning_rate": 3.056302334890786e-08,
+      "logits/chosen": -4.245262622833252,
+      "logits/rejected": -4.510401725769043,
+      "logps/chosen": -295.3984680175781,
+      "logps/rejected": -250.73580932617188,
+      "loss": 0.3235,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 4.011710166931152,
+      "rewards/margins": 3.0462794303894043,
+      "rewards/rejected": 0.9654304385185242,
       "step": 150
     },
     {
       "epoch": 0.46,
-      "grad_norm": 684.2161612155076,
-      "learning_rate": 1.3147206255874882e-07,
-      "logits/chosen": -4.248518943786621,
-      "logits/rejected": -4.44361686706543,
-      "logps/chosen": -309.7388610839844,
-      "logps/rejected": -272.90252685546875,
-      "loss": 0.2961,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 4.352166652679443,
-      "rewards/margins": 4.611546516418457,
-      "rewards/rejected": -0.2593800127506256,
+      "grad_norm": 1744.335126050233,
+      "learning_rate": 2.8355831645441387e-08,
+      "logits/chosen": -4.277425765991211,
+      "logits/rejected": -4.570274829864502,
+      "logps/chosen": -296.66839599609375,
+      "logps/rejected": -235.6475372314453,
+      "loss": 0.36,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 4.259499549865723,
+      "rewards/margins": 3.127065658569336,
+      "rewards/rejected": 1.1324341297149658,
       "step": 160
     },
     {
       "epoch": 0.49,
-      "grad_norm": 795.319436926025,
-      "learning_rate": 1.2183385193801653e-07,
-      "logits/chosen": -4.289888381958008,
-      "logits/rejected": -4.489100933074951,
-      "logps/chosen": -272.3153991699219,
-      "logps/rejected": -237.64114379882812,
-      "loss": 0.2547,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 4.135977745056152,
-      "rewards/margins": 4.544943809509277,
-      "rewards/rejected": -0.40896692872047424,
+      "grad_norm": 1875.319827037545,
+      "learning_rate": 2.6121620758762875e-08,
+      "logits/chosen": -4.229983329772949,
+      "logits/rejected": -4.467092990875244,
+      "logps/chosen": -296.31683349609375,
+      "logps/rejected": -241.3401336669922,
+      "loss": 0.3474,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.343829154968262,
+      "rewards/margins": 3.233609437942505,
+      "rewards/rejected": 1.1102204322814941,
       "step": 170
     },
     {
-      "epoch": 0.52,
-      "grad_norm": 917.2913745554154,
-      "learning_rate": 1.1197726351017051e-07,
-      "logits/chosen": -4.37355375289917,
-      "logits/rejected": -4.5346221923828125,
-      "logps/chosen": -261.6694641113281,
-      "logps/rejected": -230.89599609375,
-      "loss": 0.3104,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 3.4763107299804688,
-      "rewards/margins": 3.877528429031372,
-      "rewards/rejected": -0.4012181758880615,
+      "epoch": 0.51,
+      "grad_norm": 2082.5003671787076,
+      "learning_rate": 2.3878379241237133e-08,
+      "logits/chosen": -4.364750862121582,
+      "logits/rejected": -4.597868919372559,
+      "logps/chosen": -285.72869873046875,
+      "logps/rejected": -241.40652465820312,
+      "loss": 0.3417,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 4.1484293937683105,
+      "rewards/margins": 3.0738511085510254,
+      "rewards/rejected": 1.074578046798706,
       "step": 180
     },
     {
       "epoch": 0.54,
-      "grad_norm": 880.6071936953809,
-      "learning_rate": 1.0200088089538943e-07,
-      "logits/chosen": -4.33815860748291,
-      "logits/rejected": -4.5543646812438965,
-      "logps/chosen": -288.26873779296875,
-      "logps/rejected": -253.7147216796875,
-      "loss": 0.3129,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 4.4660868644714355,
-      "rewards/margins": 4.566544055938721,
-      "rewards/rejected": -0.10045762360095978,
+      "grad_norm": 1597.9774938638957,
+      "learning_rate": 2.164416835455862e-08,
+      "logits/chosen": -4.3281121253967285,
+      "logits/rejected": -4.498069763183594,
+      "logps/chosen": -308.14776611328125,
+      "logps/rejected": -257.7415466308594,
+      "loss": 0.2852,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 4.146700859069824,
+      "rewards/margins": 3.202249526977539,
+      "rewards/rejected": 0.9444509744644165,
       "step": 190
     },
     {
       "epoch": 0.57,
-      "grad_norm": 781.3066688232569,
-      "learning_rate": 9.20044858712785e-08,
-      "logits/chosen": -4.292983055114746,
-      "logits/rejected": -4.566291332244873,
-      "logps/chosen": -315.58251953125,
-      "logps/rejected": -270.0286560058594,
-      "loss": 0.2741,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 4.502341270446777,
-      "rewards/margins": 4.4218339920043945,
-      "rewards/rejected": 0.08050797879695892,
+      "grad_norm": 1601.8580723204816,
+      "learning_rate": 1.943697665109214e-08,
+      "logits/chosen": -4.358348846435547,
+      "logits/rejected": -4.601215839385986,
+      "logps/chosen": -292.93658447265625,
+      "logps/rejected": -249.59469604492188,
+      "loss": 0.3184,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 4.194998741149902,
+      "rewards/margins": 2.974621534347534,
+      "rewards/rejected": 1.2203772068023682,
       "step": 200
     },
     {
       "epoch": 0.57,
-      "eval_logits/chosen": -3.2033309936523438,
-      "eval_logits/rejected": -3.2033309936523438,
-      "eval_logps/chosen": -160.92892456054688,
-      "eval_logps/rejected": -160.92892456054688,
+      "eval_logits/chosen": -3.2195205688476562,
+      "eval_logits/rejected": -3.2195205688476562,
+      "eval_logps/chosen": -157.37933349609375,
+      "eval_logps/rejected": -157.37933349609375,
       "eval_loss": 0.6931471824645996,
       "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": -2.675952911376953,
+      "eval_rewards/chosen": -1.8023262023925781,
       "eval_rewards/margins": 0.0,
-      "eval_rewards/rejected": -2.675952911376953,
-      "eval_runtime": 1.48,
-      "eval_samples_per_second": 0.676,
-      "eval_steps_per_second": 0.676,
+      "eval_rewards/rejected": -1.8023262023925781,
+      "eval_runtime": 1.4741,
+      "eval_samples_per_second": 0.678,
+      "eval_steps_per_second": 0.678,
       "step": 200
     },
     {
       "epoch": 0.6,
-      "grad_norm": 1021.5696401893487,
-      "learning_rate": 8.208806037554645e-08,
-      "logits/chosen": -4.327147483825684,
-      "logits/rejected": -4.6312479972839355,
-      "logps/chosen": -292.85919189453125,
-      "logps/rejected": -236.7397918701172,
-      "loss": 0.2345,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 4.086556434631348,
-      "rewards/margins": 4.589372634887695,
-      "rewards/rejected": -0.5028160810470581,
+      "grad_norm": 1818.1510653253358,
+      "learning_rate": 1.7274575140626317e-08,
+      "logits/chosen": -4.293700218200684,
+      "logits/rejected": -4.587708473205566,
+      "logps/chosen": -306.94647216796875,
+      "logps/rejected": -254.83981323242188,
+      "loss": 0.3169,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 4.274092674255371,
+      "rewards/margins": 3.556690216064453,
+      "rewards/rejected": 0.7174022793769836,
       "step": 210
     },
     {
       "epoch": 0.63,
-      "grad_norm": 720.6830161967047,
-      "learning_rate": 7.23507865067214e-08,
-      "logits/chosen": -4.357525825500488,
-      "logits/rejected": -4.7082133293151855,
-      "logps/chosen": -298.2110900878906,
-      "logps/rejected": -237.28903198242188,
-      "loss": 0.241,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.84515118598938,
-      "rewards/margins": 4.655531406402588,
-      "rewards/rejected": -0.8103805780410767,
+      "grad_norm": 2084.9707047014217,
+      "learning_rate": 1.517437420865191e-08,
+      "logits/chosen": -4.2438554763793945,
+      "logits/rejected": -4.590119361877441,
+      "logps/chosen": -297.3277587890625,
+      "logps/rejected": -225.09414672851562,
+      "loss": 0.3117,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 4.186089515686035,
+      "rewards/margins": 3.6873459815979004,
+      "rewards/rejected": 0.4987434446811676,
       "step": 220
     },
     {
       "epoch": 0.66,
-      "grad_norm": 784.9312094778309,
-      "learning_rate": 6.289005452469778e-08,
-      "logits/chosen": -4.388636112213135,
-      "logits/rejected": -4.748753070831299,
-      "logps/chosen": -288.0061340332031,
-      "logps/rejected": -236.6077880859375,
-      "loss": 0.268,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.5908846855163574,
-      "rewards/margins": 4.1128339767456055,
-      "rewards/rejected": -0.5219489932060242,
+      "grad_norm": 1793.5243127965375,
+      "learning_rate": 1.3153283438175034e-08,
+      "logits/chosen": -4.3719801902771,
+      "logits/rejected": -4.563234806060791,
+      "logps/chosen": -281.373779296875,
+      "logps/rejected": -226.25576782226562,
+      "loss": 0.2879,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 3.584429979324341,
+      "rewards/margins": 2.8807406425476074,
+      "rewards/rejected": 0.7036892771720886,
       "step": 230
     },
     {
       "epoch": 0.69,
-      "grad_norm": 765.61870702253,
-      "learning_rate": 5.38004887728938e-08,
-      "logits/chosen": -4.234927177429199,
-      "logits/rejected": -4.550938606262207,
-      "logps/chosen": -283.8558654785156,
-      "logps/rejected": -242.86538696289062,
-      "loss": 0.2908,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.562377452850342,
-      "rewards/margins": 4.074954032897949,
-      "rewards/rejected": -0.5125768780708313,
+      "grad_norm": 1621.528952660571,
+      "learning_rate": 1.1227575463697438e-08,
+      "logits/chosen": -4.3936567306518555,
+      "logits/rejected": -4.714280128479004,
+      "logps/chosen": -258.6517639160156,
+      "logps/rejected": -215.28759765625,
+      "loss": 0.3042,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 3.862626552581787,
+      "rewards/margins": 3.0625431537628174,
+      "rewards/rejected": 0.8000835180282593,
       "step": 240
     },
     {
-      "epoch": 0.72,
-      "grad_norm": 787.5249084258048,
-      "learning_rate": 4.517300126455066e-08,
-      "logits/chosen": -4.290322303771973,
-      "logits/rejected": -4.522026538848877,
-      "logps/chosen": -292.47442626953125,
-      "logps/rejected": -257.0675354003906,
-      "loss": 0.2635,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 3.95862078666687,
-      "rewards/margins": 4.515480995178223,
-      "rewards/rejected": -0.5568601489067078,
+      "epoch": 0.71,
+      "grad_norm": 2231.5682374793205,
+      "learning_rate": 9.412754953531663e-09,
+      "logits/chosen": -4.34213924407959,
+      "logits/rejected": -4.6162428855896,
+      "logps/chosen": -278.9085388183594,
+      "logps/rejected": -232.6056365966797,
+      "loss": 0.3109,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 3.6670002937316895,
+      "rewards/margins": 2.8657121658325195,
+      "rewards/rejected": 0.8012881278991699,
       "step": 250
     },
     {
       "epoch": 0.74,
-      "grad_norm": 894.7487355536125,
-      "learning_rate": 3.70938823990135e-08,
-      "logits/chosen": -4.367636680603027,
-      "logits/rejected": -4.524864196777344,
-      "logps/chosen": -293.8681640625,
-      "logps/rejected": -266.19622802734375,
-      "loss": 0.2999,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.311814069747925,
-      "rewards/margins": 3.883411407470703,
-      "rewards/rejected": -0.5715969800949097,
+      "grad_norm": 1668.5476234310504,
+      "learning_rate": 7.723433775328384e-09,
+      "logits/chosen": -4.386145114898682,
+      "logits/rejected": -4.632050037384033,
+      "logps/chosen": -271.8704833984375,
+      "logps/rejected": -240.48257446289062,
+      "loss": 0.3039,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.5878806114196777,
+      "rewards/margins": 2.9320101737976074,
+      "rewards/rejected": 0.6558703184127808,
       "step": 260
     },
     {
       "epoch": 0.77,
-      "grad_norm": 582.5138549963486,
-      "learning_rate": 2.9643937902467277e-08,
-      "logits/chosen": -4.23429012298584,
-      "logits/rejected": -4.521827697753906,
-      "logps/chosen": -280.5379333496094,
-      "logps/rejected": -241.7247314453125,
-      "loss": 0.2388,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 3.911522626876831,
-      "rewards/margins": 4.728894233703613,
-      "rewards/rejected": -0.8173705339431763,
+      "grad_norm": 1726.7631750123023,
+      "learning_rate": 6.173213349909728e-09,
+      "logits/chosen": -4.517698764801025,
+      "logits/rejected": -4.687317848205566,
+      "logps/chosen": -273.4754943847656,
+      "logps/rejected": -228.2833709716797,
+      "loss": 0.3356,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 3.6022744178771973,
+      "rewards/margins": 2.6589503288269043,
+      "rewards/rejected": 0.9433239698410034,
       "step": 270
     },
     {
       "epoch": 0.8,
-      "grad_norm": 810.3954826895692,
-      "learning_rate": 2.289768062527362e-08,
-      "logits/chosen": -4.273584842681885,
-      "logits/rejected": -4.6050519943237305,
-      "logps/chosen": -283.619384765625,
-      "logps/rejected": -233.9824981689453,
-      "loss": 0.2895,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 3.576643466949463,
-      "rewards/margins": 3.991179943084717,
-      "rewards/rejected": -0.41453617811203003,
+      "grad_norm": 1197.1122441391342,
+      "learning_rate": 4.7745751406263165e-09,
+      "logits/chosen": -4.299304008483887,
+      "logits/rejected": -4.589285850524902,
+      "logps/chosen": -274.9901123046875,
+      "logps/rejected": -229.76449584960938,
+      "loss": 0.2791,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 4.094995021820068,
+      "rewards/margins": 3.0975327491760254,
+      "rewards/rejected": 0.997462272644043,
       "step": 280
     },
     {
       "epoch": 0.83,
-      "grad_norm": 730.6997584190797,
-      "learning_rate": 1.6922585279389035e-08,
-      "logits/chosen": -4.406073093414307,
-      "logits/rejected": -4.6901140213012695,
-      "logps/chosen": -294.35699462890625,
-      "logps/rejected": -233.74594116210938,
-      "loss": 0.2669,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 3.295900344848633,
-      "rewards/margins": 3.856853485107422,
-      "rewards/rejected": -0.5609533786773682,
+      "grad_norm": 2356.4193384705377,
+      "learning_rate": 3.5387801599533474e-09,
+      "logits/chosen": -4.320891857147217,
+      "logits/rejected": -4.508334636688232,
+      "logps/chosen": -282.45013427734375,
+      "logps/rejected": -236.50424194335938,
+      "loss": 0.3316,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 4.115664958953857,
+      "rewards/margins": 3.2731566429138184,
+      "rewards/rejected": 0.8425084948539734,
       "step": 290
     },
     {
       "epoch": 0.86,
-      "grad_norm": 1254.3257141110294,
-      "learning_rate": 1.1778413569831725e-08,
-      "logits/chosen": -4.323153495788574,
-      "logits/rejected": -4.481599807739258,
-      "logps/chosen": -291.29742431640625,
-      "logps/rejected": -254.1814422607422,
-      "loss": 0.2695,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.8485710620880127,
-      "rewards/margins": 3.8490569591522217,
-      "rewards/rejected": -0.0004864335060119629,
+      "grad_norm": 1485.14332328563,
+      "learning_rate": 2.475778302439524e-09,
+      "logits/chosen": -4.295617580413818,
+      "logits/rejected": -4.5400543212890625,
+      "logps/chosen": -298.4153137207031,
+      "logps/rejected": -240.1478271484375,
+      "loss": 0.3594,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 4.416214942932129,
+      "rewards/margins": 3.1984994411468506,
+      "rewards/rejected": 1.2177152633666992,
       "step": 300
     },
     {
       "epoch": 0.86,
-      "eval_logits/chosen": -3.199521064758301,
-      "eval_logits/rejected": -3.199521064758301,
-      "eval_logps/chosen": -159.83721923828125,
-      "eval_logps/rejected": -159.83721923828125,
+      "eval_logits/chosen": -3.220174551010132,
+      "eval_logits/rejected": -3.220174551010132,
+      "eval_logps/chosen": -157.367431640625,
+      "eval_logps/rejected": -157.367431640625,
       "eval_loss": 0.6931471824645996,
       "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": -2.1301021575927734,
+      "eval_rewards/chosen": -1.7904319763183594,
       "eval_rewards/margins": 0.0,
-      "eval_rewards/rejected": -2.1301021575927734,
-      "eval_runtime": 1.4681,
-      "eval_samples_per_second": 0.681,
-      "eval_steps_per_second": 0.681,
+      "eval_rewards/rejected": -1.7904319763183594,
+      "eval_runtime": 1.47,
+      "eval_samples_per_second": 0.68,
+      "eval_steps_per_second": 0.68,
       "step": 300
     },
     {
       "epoch": 0.89,
-      "grad_norm": 1302.095010227008,
-      "learning_rate": 7.516616470096315e-09,
-      "logits/chosen": -4.3807525634765625,
-      "logits/rejected": -4.600883483886719,
-      "logps/chosen": -299.8600158691406,
-      "logps/rejected": -248.4993438720703,
-      "loss": 0.2545,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 4.215323448181152,
-      "rewards/margins": 4.587640762329102,
-      "rewards/rejected": -0.3723169267177582,
+      "grad_norm": 2625.0873445651387,
+      "learning_rate": 1.5941282340065698e-09,
+      "logits/chosen": -4.43851900100708,
+      "logits/rejected": -4.580752372741699,
+      "logps/chosen": -262.37445068359375,
+      "logps/rejected": -226.46572875976562,
+      "loss": 0.3007,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.741738796234131,
+      "rewards/margins": 2.9144444465637207,
+      "rewards/rejected": 0.8272944688796997,
       "step": 310
     },
     {
-      "epoch": 0.92,
-      "grad_norm": 875.4939401121867,
-      "learning_rate": 4.179819619838454e-09,
-      "logits/chosen": -4.2180280685424805,
-      "logits/rejected": -4.570239067077637,
-      "logps/chosen": -286.0614013671875,
-      "logps/rejected": -234.3032684326172,
-      "loss": 0.269,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.9898929595947266,
-      "rewards/margins": 4.13907527923584,
-      "rewards/rejected": -0.1491830050945282,
+      "epoch": 0.91,
+      "grad_norm": 1589.6112135444553,
+      "learning_rate": 9.009284826036689e-10,
+      "logits/chosen": -4.277141094207764,
+      "logits/rejected": -4.5314412117004395,
+      "logps/chosen": -292.65875244140625,
+      "logps/rejected": -243.8509063720703,
+      "loss": 0.3277,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.2250542640686035,
+      "rewards/margins": 3.1320207118988037,
+      "rewards/rejected": 1.093034029006958,
       "step": 320
     },
     {
       "epoch": 0.94,
-      "grad_norm": 1010.1848642033616,
-      "learning_rate": 1.8013969917777483e-09,
-      "logits/chosen": -4.380679130554199,
-      "logits/rejected": -4.608093738555908,
-      "logps/chosen": -278.8153381347656,
-      "logps/rejected": -235.4047088623047,
-      "loss": 0.247,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.914651870727539,
-      "rewards/margins": 4.33533239364624,
-      "rewards/rejected": -0.42068085074424744,
+      "grad_norm": 2192.855370501752,
+      "learning_rate": 4.017602850342583e-10,
+      "logits/chosen": -4.330888271331787,
+      "logits/rejected": -4.536975383758545,
+      "logps/chosen": -305.5764465332031,
+      "logps/rejected": -252.0467529296875,
+      "loss": 0.3203,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.009243488311768,
+      "rewards/margins": 3.056270122528076,
+      "rewards/rejected": 0.9529730677604675,
       "step": 330
     },
     {
       "epoch": 0.97,
-      "grad_norm": 702.698454631269,
-      "learning_rate": 4.051370919176289e-10,
-      "logits/chosen": -4.314477920532227,
-      "logits/rejected": -4.502932548522949,
-      "logps/chosen": -282.89666748046875,
-      "logps/rejected": -246.3631134033203,
-      "loss": 0.243,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 3.8934428691864014,
-      "rewards/margins": 4.308337688446045,
-      "rewards/rejected": -0.41489481925964355,
+      "grad_norm": 2158.7231383937637,
+      "learning_rate": 1.0064265011902328e-10,
+      "logits/chosen": -4.319821357727051,
+      "logits/rejected": -4.614516735076904,
+      "logps/chosen": -285.194091796875,
+      "logps/rejected": -227.5124053955078,
+      "loss": 0.3239,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.751185178756714,
+      "rewards/margins": 2.758882522583008,
+      "rewards/rejected": 0.9923027753829956,
       "step": 340
     },
     {
       "epoch": 1.0,
-      "step": 349,
+      "grad_norm": 1350.3403367664616,
+      "learning_rate": 0.0,
+      "logits/chosen": -4.290497779846191,
+      "logits/rejected": -4.4949870109558105,
+      "logps/chosen": -291.93768310546875,
+      "logps/rejected": -244.3520965576172,
+      "loss": 0.3142,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 4.083470344543457,
+      "rewards/margins": 3.2036800384521484,
+      "rewards/rejected": 0.8797903060913086,
+      "step": 350
+    },
+    {
+      "epoch": 1.0,
+      "step": 350,
       "total_flos": 0.0,
-      "train_loss": 0.30806411576476,
-      "train_runtime": 5299.0292,
-      "train_samples_per_second": 8.432,
+      "train_loss": 0.36299856867109026,
+      "train_runtime": 5294.123,
+      "train_samples_per_second": 8.454,
       "train_steps_per_second": 0.066
     }
   ],
   "logging_steps": 10,
-  "max_steps": 349,
+  "max_steps": 350,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64b9ce726680b0e48b6ed9b9cf02bd9886d9c4dc43b9488efca75e67b2fe6cd7
-size 6520
+oid sha256:22e6e8fd2db42b5bcee0f6513108dbd03df91bb18e83b0bf1c0a9a1f8f523058
+size 6264