Model save

README.md

@@ -32,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-08
+- learning_rate: 1e-07
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -52,7 +52,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.39.3
+- Transformers 4.41.1
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.15.2
+- Tokenizers 0.19.1

all_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.5129612217778745,
-    "train_runtime": 5390.6556,
-    "train_samples": 47205,
-    "train_samples_per_second": 8.757,
-    "train_steps_per_second": 0.034
+    "epoch": 0.9972602739726028,
+    "total_flos": 0.0,
+    "train_loss": 0.41821049595927146,
+    "train_runtime": 5820.0845,
+    "train_samples": 46672,
+    "train_samples_per_second": 8.019,
+    "train_steps_per_second": 0.031
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.39.3",
+  "transformers_version": "4.41.1",
   "use_cache": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.39.3"
+  "transformers_version": "4.41.1"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8cf429f2365d057d537eb6eb684ec0e28181ee38fa55cd21735e7e6f691bdff9
+oid sha256:7290f1e2c26695e1d8c538d9fe41285a5e52f43708406dcae0878541a4f6d105
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9331d412f5bc9c44c9792f8a2afaa5a28095041cc6f7c254ae83a5ed634d33d8
+oid sha256:55ca4a716c63ab1b6533bd8ba1f04ef1b977d004fe97544fdad8b7fcb3020a04
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43b920dd703fb81aa8790bea0c995d0448371190e6f8d23f5e71d685ce1bda1e
+oid sha256:c07c8434b14ba8d74fb0863d0c4cafada3c494cf0444a8a71edfb894b7e87540
 size 4540516344

runs/Jun18_02-32-34_n136-082-130/events.out.tfevents.1718649430.n136-082-130.4153555.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c381b50203c3d1d9fd503e67ae99e380cc5e921417f821c0da1d88de5759371
-size 12552
+oid sha256:5836659d6ac9640c5792995c106c1fd759200b2ff12676c12d8bd4326a2db255
+size 18386

tokenizer.json

@@ -134,6 +134,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.5129612217778745,
-    "train_runtime": 5390.6556,
-    "train_samples": 47205,
-    "train_samples_per_second": 8.757,
-    "train_steps_per_second": 0.034
+    "epoch": 0.9972602739726028,
+    "total_flos": 0.0,
+    "train_loss": 0.41821049595927146,
+    "train_runtime": 5820.0845,
+    "train_samples": 46672,
+    "train_samples_per_second": 8.019,
+    "train_steps_per_second": 0.031
 }

trainer_state.json

@@ -1,22 +1,22 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.997289972899729,
+  "epoch": 0.9972602739726028,
   "eval_steps": 500,
-  "global_step": 184,
+  "global_step": 182,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.01,
-      "grad_norm": 699.419624450993,
-      "learning_rate": 2.6315789473684206e-09,
-      "logits/chosen": -0.8214728236198425,
-      "logits/rejected": -0.3221539855003357,
-      "logps/chosen": -281.03485107421875,
-      "logps/rejected": -258.44012451171875,
-      "loss": 0.6984,
+      "epoch": 0.005479452054794521,
+      "grad_norm": 749.8344019634822,
+      "learning_rate": 5.263157894736841e-09,
+      "logits/chosen": -0.7030794620513916,
+      "logits/rejected": -0.3951629400253296,
+      "logps/chosen": -341.73382568359375,
+      "logps/rejected": -292.9862060546875,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,290 +24,302 @@
       "step": 1
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 688.9664821722622,
-      "learning_rate": 2.6315789473684208e-08,
-      "logits/chosen": -0.8804507255554199,
-      "logits/rejected": -0.6175813674926758,
-      "logps/chosen": -333.9942932128906,
-      "logps/rejected": -301.1158142089844,
-      "loss": 0.7112,
-      "rewards/accuracies": 0.4583333432674408,
-      "rewards/chosen": 0.02290305122733116,
-      "rewards/margins": 0.000978956581093371,
-      "rewards/rejected": 0.0219240952283144,
+      "epoch": 0.0547945205479452,
+      "grad_norm": 712.5612854291265,
+      "learning_rate": 5.2631578947368416e-08,
+      "logits/chosen": -0.8543994426727295,
+      "logits/rejected": -0.5004442930221558,
+      "logps/chosen": -343.0475158691406,
+      "logps/rejected": -297.69866943359375,
+      "loss": 0.7047,
+      "rewards/accuracies": 0.4652777910232544,
+      "rewards/chosen": 0.028890050947666168,
+      "rewards/margins": 0.015952428802847862,
+      "rewards/rejected": 0.012937622144818306,
       "step": 10
     },
     {
-      "epoch": 0.11,
-      "grad_norm": 708.2483637617346,
-      "learning_rate": 4.9995468638088146e-08,
-      "logits/chosen": -0.8514911532402039,
-      "logits/rejected": -0.6387242078781128,
-      "logps/chosen": -341.757080078125,
-      "logps/rejected": -314.51751708984375,
-      "loss": 0.7023,
-      "rewards/accuracies": 0.4906249940395355,
-      "rewards/chosen": 0.009864812716841698,
-      "rewards/margins": 0.0014371216529980302,
-      "rewards/rejected": 0.008427691645920277,
+      "epoch": 0.1095890410958904,
+      "grad_norm": 633.132107951183,
+      "learning_rate": 9.999071352056673e-08,
+      "logits/chosen": -0.8784568905830383,
+      "logits/rejected": -0.5061537027359009,
+      "logps/chosen": -333.73223876953125,
+      "logps/rejected": -288.03704833984375,
+      "loss": 0.6532,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.058163970708847046,
+      "rewards/margins": 0.1349145770072937,
+      "rewards/rejected": -0.19307854771614075,
       "step": 20
     },
     {
-      "epoch": 0.16,
-      "grad_norm": 577.1248066355766,
-      "learning_rate": 4.945369001834514e-08,
-      "logits/chosen": -0.9569602012634277,
-      "logits/rejected": -0.7534269690513611,
-      "logps/chosen": -346.7052001953125,
-      "logps/rejected": -312.9559631347656,
-      "loss": 0.6348,
-      "rewards/accuracies": 0.596875011920929,
-      "rewards/chosen": 0.04675150662660599,
-      "rewards/margins": 0.11421873420476913,
-      "rewards/rejected": -0.06746722757816315,
+      "epoch": 0.1643835616438356,
+      "grad_norm": 498.73085461545014,
+      "learning_rate": 9.888050389939171e-08,
+      "logits/chosen": -0.9048949480056763,
+      "logits/rejected": -0.5001607537269592,
+      "logps/chosen": -331.8179626464844,
+      "logps/rejected": -290.59527587890625,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": 0.2456372082233429,
+      "rewards/margins": 0.7688120603561401,
+      "rewards/rejected": -0.5231748819351196,
       "step": 30
     },
     {
-      "epoch": 0.22,
-      "grad_norm": 579.099443117778,
-      "learning_rate": 4.802809132787125e-08,
-      "logits/chosen": -0.9495086669921875,
-      "logits/rejected": -0.69568932056427,
-      "logps/chosen": -342.0091247558594,
-      "logps/rejected": -313.8953552246094,
-      "loss": 0.5992,
-      "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": 0.18988165259361267,
-      "rewards/margins": 0.39908966422080994,
-      "rewards/rejected": -0.20920801162719727,
+      "epoch": 0.2191780821917808,
+      "grad_norm": 468.38328488591065,
+      "learning_rate": 9.596014080776422e-08,
+      "logits/chosen": -0.8753167986869812,
+      "logits/rejected": -0.5111785531044006,
+      "logps/chosen": -335.57745361328125,
+      "logps/rejected": -306.9455261230469,
+      "loss": 0.4522,
+      "rewards/accuracies": 0.8218749761581421,
+      "rewards/chosen": 1.0787924528121948,
+      "rewards/margins": 1.3148835897445679,
+      "rewards/rejected": -0.23609113693237305,
       "step": 40
     },
     {
-      "epoch": 0.27,
-      "grad_norm": 522.7924733724386,
-      "learning_rate": 4.577019741770136e-08,
-      "logits/chosen": -0.9217802882194519,
-      "logits/rejected": -0.5782603025436401,
-      "logps/chosen": -324.60467529296875,
-      "logps/rejected": -291.2610778808594,
-      "loss": 0.5445,
-      "rewards/accuracies": 0.734375,
-      "rewards/chosen": 0.451026976108551,
-      "rewards/margins": 0.6606618762016296,
-      "rewards/rejected": -0.209634929895401,
+      "epoch": 0.273972602739726,
+      "grad_norm": 405.49075569087563,
+      "learning_rate": 9.133777176798013e-08,
+      "logits/chosen": -0.9362503886222839,
+      "logits/rejected": -0.493760883808136,
+      "logps/chosen": -329.0995178222656,
+      "logps/rejected": -294.7619323730469,
+      "loss": 0.4065,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 2.0607025623321533,
+      "rewards/margins": 1.7504669427871704,
+      "rewards/rejected": 0.3102358281612396,
       "step": 50
     },
     {
-      "epoch": 0.33,
-      "grad_norm": 441.70974291440746,
-      "learning_rate": 4.276161445663422e-08,
-      "logits/chosen": -0.9271470308303833,
-      "logits/rejected": -0.5116554498672485,
-      "logps/chosen": -351.0398864746094,
-      "logps/rejected": -313.02984619140625,
-      "loss": 0.5198,
-      "rewards/accuracies": 0.7718750238418579,
-      "rewards/chosen": 0.6475666761398315,
-      "rewards/margins": 0.7143425941467285,
-      "rewards/rejected": -0.06677587330341339,
+      "epoch": 0.3287671232876712,
+      "grad_norm": 531.0849294440985,
+      "learning_rate": 8.518457335743925e-08,
+      "logits/chosen": -0.8332949876785278,
+      "logits/rejected": -0.44049328565597534,
+      "logps/chosen": -328.05059814453125,
+      "logps/rejected": -295.0623474121094,
+      "loss": 0.397,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 1.8471581935882568,
+      "rewards/margins": 1.8996388912200928,
+      "rewards/rejected": -0.05248071998357773,
       "step": 60
     },
     {
-      "epoch": 0.38,
-      "grad_norm": 542.2213130459819,
-      "learning_rate": 3.9111080471669236e-08,
-      "logits/chosen": -0.8212447166442871,
-      "logits/rejected": -0.4982244372367859,
-      "logps/chosen": -323.27349853515625,
-      "logps/rejected": -298.29718017578125,
-      "loss": 0.4963,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.8154565095901489,
-      "rewards/margins": 0.7821646928787231,
-      "rewards/rejected": 0.033291809260845184,
+      "epoch": 0.3835616438356164,
+      "grad_norm": 410.01119928479034,
+      "learning_rate": 7.772841216033532e-08,
+      "logits/chosen": -1.053856611251831,
+      "logits/rejected": -0.6151344776153564,
+      "logps/chosen": -347.8240966796875,
+      "logps/rejected": -304.7304382324219,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 1.6472972631454468,
+      "rewards/margins": 1.7940982580184937,
+      "rewards/rejected": -0.14680106937885284,
       "step": 70
     },
     {
-      "epoch": 0.43,
-      "grad_norm": 474.1471440060756,
-      "learning_rate": 3.495053527239656e-08,
-      "logits/chosen": -0.853967547416687,
-      "logits/rejected": -0.506403923034668,
-      "logps/chosen": -326.6103820800781,
-      "logps/rejected": -298.58380126953125,
-      "loss": 0.4837,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.8894384503364563,
-      "rewards/margins": 0.8635719418525696,
-      "rewards/rejected": 0.025866415351629257,
+      "epoch": 0.4383561643835616,
+      "grad_norm": 422.6505304779972,
+      "learning_rate": 6.924540636266271e-08,
+      "logits/chosen": -0.9396562576293945,
+      "logits/rejected": -0.3743361532688141,
+      "logps/chosen": -341.07342529296875,
+      "logps/rejected": -294.79632568359375,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 1.4693505764007568,
+      "rewards/margins": 1.931475043296814,
+      "rewards/rejected": -0.46212440729141235,
       "step": 80
     },
     {
-      "epoch": 0.49,
-      "grad_norm": 425.2259909957891,
-      "learning_rate": 3.0430351802512695e-08,
-      "logits/chosen": -0.9206312298774719,
-      "logits/rejected": -0.5899518728256226,
-      "logps/chosen": -332.1669921875,
-      "logps/rejected": -301.76629638671875,
-      "loss": 0.4805,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 1.0642414093017578,
-      "rewards/margins": 0.9260485768318176,
-      "rewards/rejected": 0.13819284737110138,
+      "epoch": 0.4931506849315068,
+      "grad_norm": 435.4943837818754,
+      "learning_rate": 6.004970048339225e-08,
+      "logits/chosen": -1.0359487533569336,
+      "logits/rejected": -0.6516568064689636,
+      "logps/chosen": -337.1309509277344,
+      "logps/rejected": -307.5055847167969,
+      "loss": 0.3792,
+      "rewards/accuracies": 0.8218749761581421,
+      "rewards/chosen": 1.8926198482513428,
+      "rewards/margins": 1.9282394647598267,
+      "rewards/rejected": -0.03561948612332344,
       "step": 90
     },
     {
-      "epoch": 0.54,
-      "grad_norm": 522.4112789139864,
-      "learning_rate": 2.5713901269842402e-08,
-      "logits/chosen": -0.9058634042739868,
-      "logits/rejected": -0.5568256974220276,
-      "logps/chosen": -339.79473876953125,
-      "logps/rejected": -303.04571533203125,
-      "loss": 0.4627,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 1.063265085220337,
-      "rewards/margins": 1.0086718797683716,
-      "rewards/rejected": 0.05459331348538399,
+      "epoch": 0.547945205479452,
+      "grad_norm": 424.3608116994891,
+      "learning_rate": 5.0481831906199036e-08,
+      "logits/chosen": -0.8561469316482544,
+      "logits/rejected": -0.5456556081771851,
+      "logps/chosen": -315.7440185546875,
+      "logps/rejected": -284.4901428222656,
+      "loss": 0.3607,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 1.5119119882583618,
+      "rewards/margins": 1.8389956951141357,
+      "rewards/rejected": -0.3270837962627411,
       "step": 100
     },
     {
-      "epoch": 0.6,
-      "grad_norm": 559.2014619266797,
-      "learning_rate": 2.09716484852284e-08,
-      "logits/chosen": -0.8277750015258789,
-      "logits/rejected": -0.4014664590358734,
-      "logps/chosen": -327.2757568359375,
-      "logps/rejected": -290.67767333984375,
-      "loss": 0.4654,
-      "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": 1.061645269393921,
-      "rewards/margins": 1.108440637588501,
-      "rewards/rejected": -0.04679515212774277,
+      "epoch": 0.6027397260273972,
+      "grad_norm": 376.9480565248481,
+      "learning_rate": 4.0896120024874283e-08,
+      "logits/chosen": -0.7713192105293274,
+      "logits/rejected": -0.4743874967098236,
+      "logps/chosen": -332.10638427734375,
+      "logps/rejected": -302.2787780761719,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 1.8271310329437256,
+      "rewards/margins": 1.9777942895889282,
+      "rewards/rejected": -0.15066327154636383,
       "step": 110
     },
     {
-      "epoch": 0.65,
-      "grad_norm": 422.80636655795246,
-      "learning_rate": 1.637499082012574e-08,
-      "logits/chosen": -0.8766531944274902,
-      "logits/rejected": -0.6933692097663879,
-      "logps/chosen": -318.14697265625,
-      "logps/rejected": -298.24462890625,
-      "loss": 0.4535,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 1.206791639328003,
-      "rewards/margins": 1.0834404230117798,
-      "rewards/rejected": 0.12335111945867538,
+      "epoch": 0.6575342465753424,
+      "grad_norm": 452.4360162863762,
+      "learning_rate": 3.1647545010335394e-08,
+      "logits/chosen": -0.8479810953140259,
+      "logits/rejected": -0.47955965995788574,
+      "logps/chosen": -345.54071044921875,
+      "logps/rejected": -308.65179443359375,
+      "loss": 0.3527,
+      "rewards/accuracies": 0.840624988079071,
+      "rewards/chosen": 1.8864860534667969,
+      "rewards/margins": 2.1840381622314453,
+      "rewards/rejected": -0.2975522577762604,
       "step": 120
     },
     {
-      "epoch": 0.7,
-      "grad_norm": 446.84426011615477,
-      "learning_rate": 1.2090063459025954e-08,
-      "logits/chosen": -0.9841381907463074,
-      "logits/rejected": -0.7362494468688965,
-      "logps/chosen": -345.0978088378906,
-      "logps/rejected": -313.47198486328125,
-      "loss": 0.4637,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 1.106555700302124,
-      "rewards/margins": 1.0541633367538452,
-      "rewards/rejected": 0.05239236354827881,
+      "epoch": 0.7123287671232876,
+      "grad_norm": 315.5021095438557,
+      "learning_rate": 2.3078602107593897e-08,
+      "logits/chosen": -0.8250066041946411,
+      "logits/rejected": -0.49061331152915955,
+      "logps/chosen": -325.8185119628906,
+      "logps/rejected": -298.6694030761719,
+      "loss": 0.3638,
+      "rewards/accuracies": 0.7906249761581421,
+      "rewards/chosen": 1.3761605024337769,
+      "rewards/margins": 1.7671406269073486,
+      "rewards/rejected": -0.3909802734851837,
       "step": 130
     },
     {
-      "epoch": 0.76,
-      "grad_norm": 437.6664172133176,
-      "learning_rate": 8.271734841028552e-09,
-      "logits/chosen": -0.9155263900756836,
-      "logits/rejected": -0.6956790685653687,
-      "logps/chosen": -343.3347473144531,
-      "logps/rejected": -319.8084411621094,
-      "loss": 0.4456,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": 1.1650675535202026,
-      "rewards/margins": 1.0745846033096313,
-      "rewards/rejected": 0.09048305451869965,
+      "epoch": 0.7671232876712328,
+      "grad_norm": 420.91275678465917,
+      "learning_rate": 1.5506618277219406e-08,
+      "logits/chosen": -1.0089247226715088,
+      "logits/rejected": -0.5239464044570923,
+      "logps/chosen": -334.70111083984375,
+      "logps/rejected": -293.3878479003906,
+      "loss": 0.3607,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": 1.8048830032348633,
+      "rewards/margins": 2.049487829208374,
+      "rewards/rejected": -0.2446049153804779,
       "step": 140
     },
     {
-      "epoch": 0.81,
-      "grad_norm": 547.6078246967138,
-      "learning_rate": 5.0580093109461184e-09,
-      "logits/chosen": -0.8691400289535522,
-      "logits/rejected": -0.42544013261795044,
-      "logps/chosen": -331.86328125,
-      "logps/rejected": -290.8024597167969,
-      "loss": 0.4532,
-      "rewards/accuracies": 0.7906249761581421,
-      "rewards/chosen": 1.2534332275390625,
-      "rewards/margins": 1.3035576343536377,
-      "rewards/rejected": -0.05012448504567146,
+      "epoch": 0.821917808219178,
+      "grad_norm": 450.70186106616944,
+      "learning_rate": 9.212000874196951e-09,
+      "logits/chosen": -0.9020761251449585,
+      "logits/rejected": -0.5833565592765808,
+      "logps/chosen": -328.1164245605469,
+      "logps/rejected": -298.4231872558594,
+      "loss": 0.349,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 1.6165488958358765,
+      "rewards/margins": 1.8227107524871826,
+      "rewards/rejected": -0.20616202056407928,
       "step": 150
     },
     {
-      "epoch": 0.87,
-      "grad_norm": 491.3660284508518,
-      "learning_rate": 2.5650392827160444e-09,
-      "logits/chosen": -0.8275870084762573,
-      "logits/rejected": -0.5543009042739868,
-      "logps/chosen": -331.1053466796875,
-      "logps/rejected": -297.59521484375,
-      "loss": 0.4545,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": 1.089099645614624,
-      "rewards/margins": 1.1679441928863525,
-      "rewards/rejected": -0.07884454727172852,
+      "epoch": 0.8767123287671232,
+      "grad_norm": 399.4981855057766,
+      "learning_rate": 4.427853541662091e-09,
+      "logits/chosen": -0.9409669041633606,
+      "logits/rejected": -0.5335644483566284,
+      "logps/chosen": -337.31488037109375,
+      "logps/rejected": -302.06365966796875,
+      "loss": 0.3548,
+      "rewards/accuracies": 0.846875011920929,
+      "rewards/chosen": 1.994319200515747,
+      "rewards/margins": 2.289970636367798,
+      "rewards/rejected": -0.29565146565437317,
       "step": 160
     },
     {
-      "epoch": 0.92,
-      "grad_norm": 411.0060600793946,
-      "learning_rate": 8.829271885286093e-10,
-      "logits/chosen": -0.9505836367607117,
-      "logits/rejected": -0.7405703067779541,
-      "logps/chosen": -334.4549865722656,
-      "logps/rejected": -305.47021484375,
-      "loss": 0.4601,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": 1.0208919048309326,
-      "rewards/margins": 1.136797308921814,
-      "rewards/rejected": -0.11590544134378433,
+      "epoch": 0.9315068493150684,
+      "grad_norm": 381.8223301695622,
+      "learning_rate": 1.3313438659999398e-09,
+      "logits/chosen": -0.6689538955688477,
+      "logits/rejected": -0.37201908230781555,
+      "logps/chosen": -323.4323425292969,
+      "logps/rejected": -292.47857666015625,
+      "loss": 0.3629,
+      "rewards/accuracies": 0.840624988079071,
+      "rewards/chosen": 2.0121166706085205,
+      "rewards/margins": 2.282526731491089,
+      "rewards/rejected": -0.2704099416732788,
       "step": 170
     },
     {
-      "epoch": 0.98,
-      "grad_norm": 459.96112806668134,
-      "learning_rate": 7.246894216806354e-11,
-      "logits/chosen": -0.7150970697402954,
-      "logits/rejected": -0.2916146218776703,
-      "logps/chosen": -316.0776062011719,
-      "logps/rejected": -275.6624450683594,
-      "loss": 0.4537,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 1.0451656579971313,
-      "rewards/margins": 1.1178200244903564,
-      "rewards/rejected": -0.07265423238277435,
+      "epoch": 0.9863013698630136,
+      "grad_norm": 380.7644210750603,
+      "learning_rate": 3.7142468185014095e-11,
+      "logits/chosen": -0.8619197010993958,
+      "logits/rejected": -0.5339924097061157,
+      "logps/chosen": -319.94158935546875,
+      "logps/rejected": -290.5792236328125,
+      "loss": 0.367,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 1.9236304759979248,
+      "rewards/margins": 2.2062630653381348,
+      "rewards/rejected": -0.2826324701309204,
       "step": 180
     },
     {
-      "epoch": 1.0,
-      "step": 184,
+      "epoch": 0.9972602739726028,
+      "step": 182,
       "total_flos": 0.0,
-      "train_loss": 0.5129612217778745,
-      "train_runtime": 5390.6556,
-      "train_samples_per_second": 8.757,
-      "train_steps_per_second": 0.034
+      "train_loss": 0.41821049595927146,
+      "train_runtime": 5820.0845,
+      "train_samples_per_second": 8.019,
+      "train_steps_per_second": 0.031
     }
   ],
   "logging_steps": 10,
-  "max_steps": 184,
+  "max_steps": 182,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
   "total_flos": 0.0,
   "train_batch_size": 8,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ca21fcd79c3354846e41641aa75c59cd98e420df7941b233d35b073affbb313
-size 6264
+oid sha256:91a455adcc1dd46693a6fa3188bc76c135951e0b0806108b876f1cac5c69def0
+size 6520