End of training

README.md

@@ -2,11 +2,23 @@
 library_name: transformers
 tags:
 - generated_from_trainer
+datasets:
+- kanishka/babylm2-clean
 metrics:
 - accuracy
 model-index:
 - name: opt-babylm2-clean-20-epochs-earlystop_seed-42_1e-3
-  results: []
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: kanishka/babylm2-clean
+      type: kanishka/babylm2-clean
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.46725332517112805
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -14,7 +26,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # opt-babylm2-clean-20-epochs-earlystop_seed-42_1e-3
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the kanishka/babylm2-clean dataset.
 It achieves the following results on the evaluation set:
 - Loss: 2.7611
 - Accuracy: 0.4673

all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 19.995451444166477,
+    "eval_accuracy": 0.46725332517112805,
+    "eval_loss": 2.7611162662506104,
+    "eval_runtime": 117.7988,
+    "eval_samples": 59192,
+    "eval_samples_per_second": 502.484,
+    "eval_steps_per_second": 7.852,
+    "perplexity": 15.817489627072094,
+    "total_flos": 1.470253944471552e+18,
+    "train_loss": 2.893489753646781,
+    "train_runtime": 41720.7377,
+    "train_samples": 562814,
+    "train_samples_per_second": 269.801,
+    "train_steps_per_second": 1.054
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 19.995451444166477,
+    "eval_accuracy": 0.46725332517112805,
+    "eval_loss": 2.7611162662506104,
+    "eval_runtime": 117.7988,
+    "eval_samples": 59192,
+    "eval_samples_per_second": 502.484,
+    "eval_steps_per_second": 7.852,
+    "perplexity": 15.817489627072094
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 19.995451444166477,
+    "total_flos": 1.470253944471552e+18,
+    "train_loss": 2.893489753646781,
+    "train_runtime": 41720.7377,
+    "train_samples": 562814,
+    "train_samples_per_second": 269.801,
+    "train_steps_per_second": 1.054
+}

trainer_state.json

@@ -0,0 +1,532 @@
+{
+  "best_metric": 2.7611162662506104,
+  "best_model_checkpoint": "models/opt-babylm2-clean-20-epochs-earlystop_seed-42_1e-3/checkpoint-43960",
+  "epoch": 19.995451444166477,
+  "eval_steps": 500,
+  "global_step": 43960,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.45485558335228565,
+      "grad_norm": 0.429352343082428,
+      "learning_rate": 3.125e-05,
+      "loss": 5.6789,
+      "step": 1000
+    },
+    {
+      "epoch": 0.9097111667045713,
+      "grad_norm": 0.6156922578811646,
+      "learning_rate": 6.25e-05,
+      "loss": 4.24,
+      "step": 2000
+    },
+    {
+      "epoch": 0.9997725722083238,
+      "eval_accuracy": 0.3447452490930147,
+      "eval_loss": 3.9528262615203857,
+      "eval_runtime": 118.313,
+      "eval_samples_per_second": 500.3,
+      "eval_steps_per_second": 7.818,
+      "step": 2198
+    },
+    {
+      "epoch": 1.364566750056857,
+      "grad_norm": 0.531734824180603,
+      "learning_rate": 9.375e-05,
+      "loss": 3.8229,
+      "step": 3000
+    },
+    {
+      "epoch": 1.8194223334091426,
+      "grad_norm": 0.5823336839675903,
+      "learning_rate": 0.000125,
+      "loss": 3.5581,
+      "step": 4000
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.39479023397438445,
+      "eval_loss": 3.411952018737793,
+      "eval_runtime": 118.0185,
+      "eval_samples_per_second": 501.549,
+      "eval_steps_per_second": 7.838,
+      "step": 4397
+    },
+    {
+      "epoch": 2.274277916761428,
+      "grad_norm": 0.533871591091156,
+      "learning_rate": 0.00015625,
+      "loss": 3.3475,
+      "step": 5000
+    },
+    {
+      "epoch": 2.729133500113714,
+      "grad_norm": 0.48165056109428406,
+      "learning_rate": 0.0001875,
+      "loss": 3.2191,
+      "step": 6000
+    },
+    {
+      "epoch": 2.999772572208324,
+      "eval_accuracy": 0.4179818284929866,
+      "eval_loss": 3.180582046508789,
+      "eval_runtime": 118.1479,
+      "eval_samples_per_second": 500.999,
+      "eval_steps_per_second": 7.829,
+      "step": 6595
+    },
+    {
+      "epoch": 3.1839890834659994,
+      "grad_norm": 0.4450700879096985,
+      "learning_rate": 0.00021875,
+      "loss": 3.1207,
+      "step": 7000
+    },
+    {
+      "epoch": 3.6388446668182852,
+      "grad_norm": 0.3985724449157715,
+      "learning_rate": 0.00025,
+      "loss": 3.0495,
+      "step": 8000
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.42872771625206374,
+      "eval_loss": 3.0720248222351074,
+      "eval_runtime": 118.2962,
+      "eval_samples_per_second": 500.371,
+      "eval_steps_per_second": 7.819,
+      "step": 8794
+    },
+    {
+      "epoch": 4.093700250170571,
+      "grad_norm": 0.3782784938812256,
+      "learning_rate": 0.00028125000000000003,
+      "loss": 2.9985,
+      "step": 9000
+    },
+    {
+      "epoch": 4.548555833522856,
+      "grad_norm": 0.3812876343727112,
+      "learning_rate": 0.0003125,
+      "loss": 2.9458,
+      "step": 10000
+    },
+    {
+      "epoch": 4.9997725722083235,
+      "eval_accuracy": 0.4353868037281138,
+      "eval_loss": 3.005370616912842,
+      "eval_runtime": 118.0467,
+      "eval_samples_per_second": 501.428,
+      "eval_steps_per_second": 7.836,
+      "step": 10992
+    },
+    {
+      "epoch": 5.003411416875142,
+      "grad_norm": 0.34280985593795776,
+      "learning_rate": 0.00034375,
+      "loss": 2.9247,
+      "step": 11000
+    },
+    {
+      "epoch": 5.458267000227428,
+      "grad_norm": 0.33273234963417053,
+      "learning_rate": 0.000375,
+      "loss": 2.8732,
+      "step": 12000
+    },
+    {
+      "epoch": 5.913122583579714,
+      "grad_norm": 0.3201596736907959,
+      "learning_rate": 0.00040621875,
+      "loss": 2.8669,
+      "step": 13000
+    },
+    {
+      "epoch": 6.0,
+      "eval_accuracy": 0.43990927496826543,
+      "eval_loss": 2.966292381286621,
+      "eval_runtime": 117.0736,
+      "eval_samples_per_second": 505.597,
+      "eval_steps_per_second": 7.901,
+      "step": 13191
+    },
+    {
+      "epoch": 6.367978166931999,
+      "grad_norm": 0.3063608705997467,
+      "learning_rate": 0.00043746875000000003,
+      "loss": 2.8263,
+      "step": 14000
+    },
+    {
+      "epoch": 6.822833750284285,
+      "grad_norm": 0.2880852520465851,
+      "learning_rate": 0.0004686875,
+      "loss": 2.8256,
+      "step": 15000
+    },
+    {
+      "epoch": 6.9997725722083235,
+      "eval_accuracy": 0.4429723545047158,
+      "eval_loss": 2.938157081604004,
+      "eval_runtime": 117.2958,
+      "eval_samples_per_second": 504.639,
+      "eval_steps_per_second": 7.886,
+      "step": 15389
+    },
+    {
+      "epoch": 7.2776893336365704,
+      "grad_norm": 0.26923319697380066,
+      "learning_rate": 0.0004999375,
+      "loss": 2.7938,
+      "step": 16000
+    },
+    {
+      "epoch": 7.732544916988856,
+      "grad_norm": 0.25492221117019653,
+      "learning_rate": 0.00053115625,
+      "loss": 2.79,
+      "step": 17000
+    },
+    {
+      "epoch": 8.0,
+      "eval_accuracy": 0.4452082157366258,
+      "eval_loss": 2.919893741607666,
+      "eval_runtime": 117.0581,
+      "eval_samples_per_second": 505.664,
+      "eval_steps_per_second": 7.902,
+      "step": 17588
+    },
+    {
+      "epoch": 8.187400500341141,
+      "grad_norm": 0.251396507024765,
+      "learning_rate": 0.0005623749999999999,
+      "loss": 2.7715,
+      "step": 18000
+    },
+    {
+      "epoch": 8.642256083693427,
+      "grad_norm": 0.2381746470928192,
+      "learning_rate": 0.000593625,
+      "loss": 2.7624,
+      "step": 19000
+    },
+    {
+      "epoch": 8.999772572208323,
+      "eval_accuracy": 0.4468282677309334,
+      "eval_loss": 2.9052329063415527,
+      "eval_runtime": 117.1996,
+      "eval_samples_per_second": 505.053,
+      "eval_steps_per_second": 7.893,
+      "step": 19786
+    },
+    {
+      "epoch": 9.097111667045713,
+      "grad_norm": 0.2604175806045532,
+      "learning_rate": 0.000624875,
+      "loss": 2.7571,
+      "step": 20000
+    },
+    {
+      "epoch": 9.551967250397999,
+      "grad_norm": 0.22635479271411896,
+      "learning_rate": 0.000656125,
+      "loss": 2.7361,
+      "step": 21000
+    },
+    {
+      "epoch": 10.0,
+      "eval_accuracy": 0.44817423658205,
+      "eval_loss": 2.891494035720825,
+      "eval_runtime": 117.1018,
+      "eval_samples_per_second": 505.475,
+      "eval_steps_per_second": 7.899,
+      "step": 21985
+    },
+    {
+      "epoch": 10.006822833750284,
+      "grad_norm": 0.23204845190048218,
+      "learning_rate": 0.0006873125,
+      "loss": 2.7513,
+      "step": 22000
+    },
+    {
+      "epoch": 10.46167841710257,
+      "grad_norm": 0.22187474370002747,
+      "learning_rate": 0.0007185625,
+      "loss": 2.714,
+      "step": 23000
+    },
+    {
+      "epoch": 10.916534000454856,
+      "grad_norm": 0.20946632325649261,
+      "learning_rate": 0.0007498125,
+      "loss": 2.7354,
+      "step": 24000
+    },
+    {
+      "epoch": 10.999772572208323,
+      "eval_accuracy": 0.44908744954935614,
+      "eval_loss": 2.8843252658843994,
+      "eval_runtime": 117.0503,
+      "eval_samples_per_second": 505.697,
+      "eval_steps_per_second": 7.903,
+      "step": 24183
+    },
+    {
+      "epoch": 11.371389583807142,
+      "grad_norm": 0.2115669995546341,
+      "learning_rate": 0.0007810625,
+      "loss": 2.7054,
+      "step": 25000
+    },
+    {
+      "epoch": 11.826245167159428,
+      "grad_norm": 0.19731654226779938,
+      "learning_rate": 0.00081228125,
+      "loss": 2.7225,
+      "step": 26000
+    },
+    {
+      "epoch": 12.0,
+      "eval_accuracy": 0.4499743606051692,
+      "eval_loss": 2.8777432441711426,
+      "eval_runtime": 117.2326,
+      "eval_samples_per_second": 504.911,
+      "eval_steps_per_second": 7.89,
+      "step": 26382
+    },
+    {
+      "epoch": 12.281100750511712,
+      "grad_norm": 0.1893465369939804,
+      "learning_rate": 0.0008435000000000001,
+      "loss": 2.6994,
+      "step": 27000
+    },
+    {
+      "epoch": 12.735956333863998,
+      "grad_norm": 0.2169094681739807,
+      "learning_rate": 0.0008747500000000001,
+      "loss": 2.7092,
+      "step": 28000
+    },
+    {
+      "epoch": 12.999772572208323,
+      "eval_accuracy": 0.4505424023914201,
+      "eval_loss": 2.870769739151001,
+      "eval_runtime": 117.0845,
+      "eval_samples_per_second": 505.549,
+      "eval_steps_per_second": 7.9,
+      "step": 28580
+    },
+    {
+      "epoch": 13.190811917216283,
+      "grad_norm": 0.1946067363023758,
+      "learning_rate": 0.000906,
+      "loss": 2.6995,
+      "step": 29000
+    },
+    {
+      "epoch": 13.64566750056857,
+      "grad_norm": 0.18853700160980225,
+      "learning_rate": 0.00093725,
+      "loss": 2.6987,
+      "step": 30000
+    },
+    {
+      "epoch": 14.0,
+      "eval_accuracy": 0.4508735944709009,
+      "eval_loss": 2.8687899112701416,
+      "eval_runtime": 117.3614,
+      "eval_samples_per_second": 504.356,
+      "eval_steps_per_second": 7.882,
+      "step": 30779
+    },
+    {
+      "epoch": 14.100523083920855,
+      "grad_norm": 0.19028180837631226,
+      "learning_rate": 0.0009685000000000001,
+      "loss": 2.7009,
+      "step": 31000
+    },
+    {
+      "epoch": 14.555378667273141,
+      "grad_norm": 0.1784777194261551,
+      "learning_rate": 0.00099971875,
+      "loss": 2.6894,
+      "step": 32000
+    },
+    {
+      "epoch": 14.999772572208323,
+      "eval_accuracy": 0.4526506629141723,
+      "eval_loss": 2.854236364364624,
+      "eval_runtime": 117.0791,
+      "eval_samples_per_second": 505.573,
+      "eval_steps_per_second": 7.901,
+      "step": 32977
+    },
+    {
+      "epoch": 15.010234250625427,
+      "grad_norm": 0.1875610202550888,
+      "learning_rate": 0.0009171404682274248,
+      "loss": 2.6972,
+      "step": 33000
+    },
+    {
+      "epoch": 15.465089833977713,
+      "grad_norm": 0.18635216355323792,
+      "learning_rate": 0.0008336120401337793,
+      "loss": 2.647,
+      "step": 34000
+    },
+    {
+      "epoch": 15.919945417329998,
+      "grad_norm": 0.19111545383930206,
+      "learning_rate": 0.00075,
+      "loss": 2.6561,
+      "step": 35000
+    },
+    {
+      "epoch": 16.0,
+      "eval_accuracy": 0.4563634062896682,
+      "eval_loss": 2.825805187225342,
+      "eval_runtime": 117.2142,
+      "eval_samples_per_second": 504.99,
+      "eval_steps_per_second": 7.892,
+      "step": 35176
+    },
+    {
+      "epoch": 16.374801000682282,
+      "grad_norm": 0.1833820343017578,
+      "learning_rate": 0.0006663879598662207,
+      "loss": 2.6017,
+      "step": 36000
+    },
+    {
+      "epoch": 16.82965658403457,
+      "grad_norm": 0.18850316107273102,
+      "learning_rate": 0.0005828595317725752,
+      "loss": 2.6055,
+      "step": 37000
+    },
+    {
+      "epoch": 16.999772572208325,
+      "eval_accuracy": 0.45953467479707116,
+      "eval_loss": 2.800510883331299,
+      "eval_runtime": 118.167,
+      "eval_samples_per_second": 500.918,
+      "eval_steps_per_second": 7.828,
+      "step": 37374
+    },
+    {
+      "epoch": 17.284512167386854,
+      "grad_norm": 0.19247354567050934,
+      "learning_rate": 0.000499247491638796,
+      "loss": 2.557,
+      "step": 38000
+    },
+    {
+      "epoch": 17.73936775073914,
+      "grad_norm": 0.18034116923809052,
+      "learning_rate": 0.0004156354515050167,
+      "loss": 2.5464,
+      "step": 39000
+    },
+    {
+      "epoch": 18.0,
+      "eval_accuracy": 0.4627209824327082,
+      "eval_loss": 2.7813873291015625,
+      "eval_runtime": 118.2688,
+      "eval_samples_per_second": 500.487,
+      "eval_steps_per_second": 7.821,
+      "step": 39573
+    },
+    {
+      "epoch": 18.194223334091426,
+      "grad_norm": 0.19630157947540283,
+      "learning_rate": 0.0003320234113712375,
+      "loss": 2.5107,
+      "step": 40000
+    },
+    {
+      "epoch": 18.649078917443713,
+      "grad_norm": 0.20289246737957,
+      "learning_rate": 0.000248494983277592,
+      "loss": 2.4778,
+      "step": 41000
+    },
+    {
+      "epoch": 18.999772572208325,
+      "eval_accuracy": 0.4658446822437584,
+      "eval_loss": 2.7629892826080322,
+      "eval_runtime": 118.2031,
+      "eval_samples_per_second": 500.765,
+      "eval_steps_per_second": 7.826,
+      "step": 41771
+    },
+    {
+      "epoch": 19.103934500795997,
+      "grad_norm": 0.19990454614162445,
+      "learning_rate": 0.00016496655518394648,
+      "loss": 2.458,
+      "step": 42000
+    },
+    {
+      "epoch": 19.55879008414828,
+      "grad_norm": 0.2090277075767517,
+      "learning_rate": 8.135451505016722e-05,
+      "loss": 2.4036,
+      "step": 43000
+    },
+    {
+      "epoch": 19.995451444166477,
+      "eval_accuracy": 0.46725332517112805,
+      "eval_loss": 2.7611162662506104,
+      "eval_runtime": 118.1837,
+      "eval_samples_per_second": 500.847,
+      "eval_steps_per_second": 7.827,
+      "step": 43960
+    },
+    {
+      "epoch": 19.995451444166477,
+      "step": 43960,
+      "total_flos": 1.470253944471552e+18,
+      "train_loss": 2.893489753646781,
+      "train_runtime": 41720.7377,
+      "train_samples_per_second": 269.801,
+      "train_steps_per_second": 1.054
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 43960,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 3,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.470253944471552e+18,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}