End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 83390522990.19534,
-    "train_runtime": 166.1732,
+    "train_loss": 47.32144293799065,
+    "train_runtime": 162.7274,
     "train_samples": 28,
-    "train_samples_per_second": 1.926,
-    "train_steps_per_second": 0.12
+    "train_samples_per_second": 1.966,
+    "train_steps_per_second": 0.123
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bdf77f1ce1cc5144631a03dc9d89e3fdaf01dfcdfa8d10dab8280fffd6ae3c04
+oid sha256:522e8e5a75bfa7fe97e6ae3bc8bec9eed01b090c0544351cf179a16afba71a1a
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 83390522990.19534,
-    "train_runtime": 166.1732,
+    "train_loss": 47.32144293799065,
+    "train_runtime": 162.7274,
     "train_samples": 28,
-    "train_samples_per_second": 1.926,
-    "train_steps_per_second": 0.12
+    "train_samples_per_second": 1.966,
+    "train_steps_per_second": 0.123
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 268.03125,
+      "completion_length": 220.8125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 32.7193603515625,
+      "grad_norm": 31.785186767578125,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 2.738486973568797,
-      "reward_std": 1.045973977074027,
-      "rewards/concensus_correctness_reward_func": 0.5634375028312206,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 2.792306764051318,
+      "reward_std": 0.5164533269235108,
+      "rewards/concensus_correctness_reward_func": 0.4905625022947788,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.665549517609179,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.7258380672428757,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7594999810680747,
+      "rewards/xmlcount_reward_func": 0.8259062492288649,
       "step": 2
     },
     {
-      "completion_length": 197.625,
+      "completion_length": 206.875,
       "epoch": 1.0,
-      "grad_norm": 4494813.5,
-      "kl": 77244.30558661713,
+      "grad_norm": 122.33345794677734,
+      "kl": 2.1089118261976787,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 57.9332,
-      "reward": 5.624423126379649,
-      "reward_std": 0.9885286254187425,
-      "rewards/concensus_correctness_reward_func": 1.8576666861772537,
+      "loss": 0.0016,
+      "reward": 5.921782890955607,
+      "reward_std": 0.693140882688264,
+      "rewards/concensus_correctness_reward_func": 2.2554166664679847,
       "rewards/consensus_reward_func": 1.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.8387147883574168,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.6714495979249477,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1780416369438171,
+      "rewards/xmlcount_reward_func": 1.16158331433932,
       "step": 4
     },
     {
-      "completion_length": 163.46875,
+      "completion_length": 194.9375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 8508.677734375,
-      "kl": 489.67826106213033,
+      "grad_norm": 95.088623046875,
+      "kl": 23.749349858495407,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.4897,
-      "reward": 4.677710190415382,
-      "reward_std": 1.1291005668463185,
-      "rewards/concensus_correctness_reward_func": 1.2935625053942204,
-      "rewards/consensus_reward_func": 1.0625,
+      "loss": 0.0237,
+      "reward": 5.464593216776848,
+      "reward_std": 1.1199153698980808,
+      "rewards/concensus_correctness_reward_func": 1.8176249973475933,
+      "rewards/consensus_reward_func": 1.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.7832414712756872,
+      "rewards/question_recreation_reward_func": 0.7988119628280401,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1634062230587006,
+      "rewards/xmlcount_reward_func": 1.1606562212109566,
       "step": 6
     },
     {
-      "completion_length": 192.33333333333334,
+      "completion_length": 184.70833333333334,
       "epoch": 2.0,
-      "grad_norm": 158.87815856933594,
-      "kl": 69.09905876715978,
+      "grad_norm": 776.6046142578125,
+      "kl": 622373.1227876631,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0518,
-      "reward": 4.211712867021561,
-      "reward_std": 0.8359420706207553,
-      "rewards/concensus_correctness_reward_func": 1.0596666783094406,
+      "loss": 466.7799,
+      "reward": 4.530350744724274,
+      "reward_std": 1.4232008809728238,
+      "rewards/concensus_correctness_reward_func": 1.124416669209798,
       "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.841879591345787,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.8376424163579941,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0601666470368702,
+      "rewards/xmlcount_reward_func": 1.1516249626874924,
       "step": 8
     },
     {
-      "completion_length": 192.0625,
+      "completion_length": 193.875,
       "epoch": 2.571428571428571,
-      "grad_norm": 725865.1875,
-      "kl": 18468.439875485376,
+      "grad_norm": 7172.392578125,
+      "kl": 285.61230134323705,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 18.4684,
-      "reward": 4.7563991993665695,
-      "reward_std": 0.9598100583534688,
-      "rewards/concensus_correctness_reward_func": 1.3004374988377094,
-      "rewards/consensus_reward_func": 1.1875,
+      "loss": 0.2856,
+      "reward": 4.841844812035561,
+      "reward_std": 1.6193378757598111,
+      "rewards/concensus_correctness_reward_func": 1.285937488079071,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8741804650053382,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.8501260448247194,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1442812159657478,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 1.1276562362909317,
       "step": 10
     },
     {
-      "completion_length": 175.20833333333334,
+      "completion_length": 186.5,
       "epoch": 3.0,
-      "grad_norm": 95072.6015625,
-      "kl": 1111873527925294.6,
+      "grad_norm": 35.74233627319336,
+      "kl": 208.1803734153509,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 833905229824.0,
-      "reward": 4.924509227275848,
-      "reward_std": 0.6583995566082498,
-      "rewards/concensus_correctness_reward_func": 1.387500022848447,
-      "rewards/consensus_reward_func": 1.3333333333333333,
+      "loss": 0.1561,
+      "reward": 5.420761525630951,
+      "reward_std": 0.8128547375478471,
+      "rewards/concensus_correctness_reward_func": 1.6499166786670685,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.904009203116099,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8841781715552012,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1329999715089798,
+      "rewards/xmlcount_reward_func": 1.1366666704416275,
       "step": 12
     },
     {
-      "completion_length": 185.59375,
+      "completion_length": 195.5625,
       "epoch": 3.571428571428571,
-      "grad_norm": 106898.2109375,
-      "kl": 939.9219960086048,
+      "grad_norm": 87303.8671875,
+      "kl": 2730.737289617886,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.9399,
-      "reward": 5.27150709182024,
-      "reward_std": 0.7421304500458064,
-      "rewards/concensus_correctness_reward_func": 1.5866250172257423,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 2.7307,
+      "reward": 5.099680621176958,
+      "reward_std": 0.8015707243248471,
+      "rewards/concensus_correctness_reward_func": 1.429125003516674,
+      "rewards/consensus_reward_func": 1.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8275696504861116,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.795399272814393,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1698124669492245,
+      "rewards/xmlcount_reward_func": 1.1251562237739563,
       "step": 14
     },
     {
-      "completion_length": 170.75,
+      "completion_length": 189.66666666666666,
       "epoch": 4.0,
-      "grad_norm": 177.8111114501953,
-      "kl": 26.596267540939152,
+      "grad_norm": 123.0419692993164,
+      "kl": 9.184569627046585,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0199,
-      "reward": 4.877232213815053,
-      "reward_std": 0.6873337930689255,
-      "rewards/concensus_correctness_reward_func": 1.5078333367904027,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0069,
+      "reward": 4.068873226642609,
+      "reward_std": 1.8017581754053633,
+      "rewards/concensus_correctness_reward_func": 1.0760833323001862,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.8671071976423264,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.852789948383967,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.168958306312561,
+      "rewards/xmlcount_reward_func": 1.0566666399439175,
       "step": 16
     },
     {
-      "completion_length": 181.5625,
+      "completion_length": 178.6875,
       "epoch": 4.571428571428571,
-      "grad_norm": 85.26807403564453,
-      "kl": 23.618711924995296,
+      "grad_norm": 156.2207794189453,
+      "kl": 3191.5056470287964,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0236,
-      "reward": 4.853519447147846,
-      "reward_std": 1.444929251272697,
-      "rewards/concensus_correctness_reward_func": 1.3307500071823597,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 3.1915,
+      "reward": 5.771281331777573,
+      "reward_std": 1.3582927275274415,
+      "rewards/concensus_correctness_reward_func": 1.7130624987185001,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7981132548302412,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.8282500533387065,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1621562354266644,
+      "rewards/xmlcount_reward_func": 1.1674687415361404,
       "step": 18
     },
     {
-      "completion_length": 214.75,
+      "completion_length": 194.08333333333334,
       "epoch": 5.0,
-      "grad_norm": 567.6781005859375,
-      "kl": 35.83177708213528,
+      "grad_norm": 1508.404296875,
+      "kl": 51.10552144042837,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0269,
-      "reward": 4.769765257835388,
-      "reward_std": 0.5614059701717148,
-      "rewards/concensus_correctness_reward_func": 1.456250011920929,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "loss": 0.0383,
+      "reward": 4.8991721868515015,
+      "reward_std": 0.9636430108221248,
+      "rewards/concensus_correctness_reward_func": 1.5873333364725113,
+      "rewards/consensus_reward_func": 1.0833333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.8139318724473318,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8430471594134966,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1662499606609344,
+      "rewards/xmlcount_reward_func": 1.1354583154122035,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 83390522990.19534,
-      "train_runtime": 166.1732,
-      "train_samples_per_second": 1.926,
-      "train_steps_per_second": 0.12
+      "train_loss": 47.32144293799065,
+      "train_runtime": 162.7274,
+      "train_samples_per_second": 1.966,
+      "train_steps_per_second": 0.123
     }
   ],
   "logging_steps": 2,