End of training

README.md

@@ -18,11 +18,11 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [riotu-lab/ArabianGPT-01B](https://huggingface.co/riotu-lab/ArabianGPT-01B) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6658
-- Bleu: 0.2770
-- Rouge1: 0.6173
-- Rouge2: 0.3729
-- Rougel: 0.6153
+- Loss: 0.6266
+- Bleu: 0.2679
+- Rouge1: 0.5977
+- Rouge2: 0.3443
+- Rougel: 0.5959
 
 ## Model description
 

all_results.json

@@ -1,19 +1,19 @@
 {
-    "epoch": 3.0,
-    "eval_bleu": 0.2518224574721449,
-    "eval_loss": 0.6498554348945618,
-    "eval_rouge1": 0.57015304568835,
-    "eval_rouge2": 0.3118394078609683,
-    "eval_rougeL": 0.5676740384593599,
-    "eval_runtime": 36.3335,
+    "epoch": 10.0,
+    "eval_bleu": 0.2678870499231116,
+    "eval_loss": 0.6265950798988342,
+    "eval_rouge1": 0.5977012354572853,
+    "eval_rouge2": 0.34430833134800065,
+    "eval_rougeL": 0.5958973349618409,
+    "eval_runtime": 58.6194,
     "eval_samples": 5405,
-    "eval_samples_per_second": 148.761,
-    "eval_steps_per_second": 18.605,
-    "perplexity": 1.9152639286675246,
-    "total_flos": 4237242236928000.0,
-    "train_loss": 0.0,
-    "train_runtime": 0.8029,
+    "eval_samples_per_second": 92.205,
+    "eval_steps_per_second": 11.532,
+    "perplexity": 1.8712283369394682,
+    "total_flos": 1.412414078976e+16,
+    "train_loss": 0.26264210826945755,
+    "train_runtime": 7304.9287,
     "train_samples": 21622,
-    "train_samples_per_second": 80792.461,
-    "train_steps_per_second": 10099.992
+    "train_samples_per_second": 59.198,
+    "train_steps_per_second": 7.4
 }

egy_training_log.txt

@@ -459,3 +459,5 @@ INFO:root:Epoch 9.0: Train Loss = 0.3108, Eval Loss = 0.646577000617981
 INFO:absl:Using default tokenizer.
 INFO:root:Epoch 10.0: Train Loss = 0.2719, Eval Loss = 0.6550981998443604
 INFO:absl:Using default tokenizer.
+INFO:__main__:*** Evaluate ***
+INFO:absl:Using default tokenizer.

eval_results.json

@@ -1,13 +1,13 @@
 {
-    "epoch": 3.0,
-    "eval_bleu": 0.2518224574721449,
-    "eval_loss": 0.6498554348945618,
-    "eval_rouge1": 0.57015304568835,
-    "eval_rouge2": 0.3118394078609683,
-    "eval_rougeL": 0.5676740384593599,
-    "eval_runtime": 36.3335,
+    "epoch": 10.0,
+    "eval_bleu": 0.2678870499231116,
+    "eval_loss": 0.6265950798988342,
+    "eval_rouge1": 0.5977012354572853,
+    "eval_rouge2": 0.34430833134800065,
+    "eval_rougeL": 0.5958973349618409,
+    "eval_runtime": 58.6194,
     "eval_samples": 5405,
-    "eval_samples_per_second": 148.761,
-    "eval_steps_per_second": 18.605,
-    "perplexity": 1.9152639286675246
+    "eval_samples_per_second": 92.205,
+    "eval_steps_per_second": 11.532,
+    "perplexity": 1.8712283369394682
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 3.0,
-    "total_flos": 4237242236928000.0,
-    "train_loss": 0.0,
-    "train_runtime": 0.8029,
+    "epoch": 10.0,
+    "total_flos": 1.412414078976e+16,
+    "train_loss": 0.26264210826945755,
+    "train_runtime": 7304.9287,
     "train_samples": 21622,
-    "train_samples_per_second": 80792.461,
-    "train_steps_per_second": 10099.992
+    "train_samples_per_second": 59.198,
+    "train_steps_per_second": 7.4
 }

trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.6498554348945618,
-  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/res_nw_dj/checkpoint-8109",
-  "epoch": 3.0,
+  "best_metric": 0.6265950798988342,
+  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/res_nw_dj/checkpoint-13515",
+  "epoch": 10.0,
   "eval_steps": 500,
-  "global_step": 8109,
+  "global_step": 27030,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -66,19 +66,152 @@
       "step": 8109
     },
     {
-      "epoch": 3.0,
-      "step": 8109,
-      "total_flos": 4237242236928000.0,
-      "train_loss": 0.0,
-      "train_runtime": 0.8029,
-      "train_samples_per_second": 80792.461,
-      "train_steps_per_second": 10099.992
+      "epoch": 4.0,
+      "grad_norm": 1.4183156490325928,
+      "learning_rate": 4.037341299477222e-05,
+      "loss": 0.556,
+      "step": 10812
+    },
+    {
+      "epoch": 4.0,
+      "eval_bleu": 0.25983253831381714,
+      "eval_loss": 0.63295978307724,
+      "eval_rouge1": 0.5829310998998971,
+      "eval_rouge2": 0.326193735265751,
+      "eval_rougeL": 0.5807349807527458,
+      "eval_runtime": 40.5509,
+      "eval_samples_per_second": 133.289,
+      "eval_steps_per_second": 16.67,
+      "step": 10812
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 1.186664342880249,
+      "learning_rate": 3.785007468259896e-05,
+      "loss": 0.4786,
+      "step": 13515
+    },
+    {
+      "epoch": 5.0,
+      "eval_bleu": 0.2678870499231116,
+      "eval_loss": 0.6265950798988342,
+      "eval_rouge1": 0.5977012354572853,
+      "eval_rouge2": 0.34430833134800065,
+      "eval_rougeL": 0.5958973349618409,
+      "eval_runtime": 36.0053,
+      "eval_samples_per_second": 150.117,
+      "eval_steps_per_second": 18.775,
+      "step": 13515
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 1.1597645282745361,
+      "learning_rate": 3.5326736370425696e-05,
+      "loss": 0.4123,
+      "step": 16218
+    },
+    {
+      "epoch": 6.0,
+      "eval_bleu": 0.2707063976336222,
+      "eval_loss": 0.630312979221344,
+      "eval_rouge1": 0.604436172840783,
+      "eval_rouge2": 0.35482389915380186,
+      "eval_rougeL": 0.6026792061487433,
+      "eval_runtime": 161.1219,
+      "eval_samples_per_second": 33.546,
+      "eval_steps_per_second": 4.196,
+      "step": 16218
+    },
+    {
+      "epoch": 7.0,
+      "grad_norm": 1.27555251121521,
+      "learning_rate": 3.280339805825243e-05,
+      "loss": 0.3573,
+      "step": 18921
+    },
+    {
+      "epoch": 7.0,
+      "eval_bleu": 0.27393786087649125,
+      "eval_loss": 0.6372247338294983,
+      "eval_rouge1": 0.6108088318784769,
+      "eval_rouge2": 0.36307215008060617,
+      "eval_rougeL": 0.6088126794948809,
+      "eval_runtime": 159.7397,
+      "eval_samples_per_second": 33.836,
+      "eval_steps_per_second": 4.232,
+      "step": 18921
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 1.6310491561889648,
+      "learning_rate": 3.0280059746079166e-05,
+      "loss": 0.3108,
+      "step": 21624
+    },
+    {
+      "epoch": 8.0,
+      "eval_bleu": 0.2759919170184585,
+      "eval_loss": 0.646577000617981,
+      "eval_rouge1": 0.6130373826214599,
+      "eval_rouge2": 0.36776670773707465,
+      "eval_rougeL": 0.6111699095881062,
+      "eval_runtime": 150.1086,
+      "eval_samples_per_second": 36.007,
+      "eval_steps_per_second": 4.503,
+      "step": 21624
+    },
+    {
+      "epoch": 9.0,
+      "grad_norm": 1.6629250049591064,
+      "learning_rate": 2.77567214339059e-05,
+      "loss": 0.2719,
+      "step": 24327
+    },
+    {
+      "epoch": 9.0,
+      "eval_bleu": 0.2801810856751608,
+      "eval_loss": 0.6550981998443604,
+      "eval_rouge1": 0.6177880665801061,
+      "eval_rouge2": 0.3727206564119486,
+      "eval_rougeL": 0.6159303776582042,
+      "eval_runtime": 159.1145,
+      "eval_samples_per_second": 33.969,
+      "eval_steps_per_second": 4.249,
+      "step": 24327
+    },
+    {
+      "epoch": 10.0,
+      "grad_norm": 1.5184797048568726,
+      "learning_rate": 2.523338312173264e-05,
+      "loss": 0.2396,
+      "step": 27030
+    },
+    {
+      "epoch": 10.0,
+      "eval_bleu": 0.27697504925957206,
+      "eval_loss": 0.665830671787262,
+      "eval_rouge1": 0.6173182845826588,
+      "eval_rouge2": 0.37288754745117264,
+      "eval_rougeL": 0.6152835100872982,
+      "eval_runtime": 110.2964,
+      "eval_samples_per_second": 49.004,
+      "eval_steps_per_second": 6.129,
+      "step": 27030
+    },
+    {
+      "epoch": 10.0,
+      "step": 27030,
+      "total_flos": 1.412414078976e+16,
+      "train_loss": 0.26264210826945755,
+      "train_runtime": 7304.9287,
+      "train_samples_per_second": 59.198,
+      "train_steps_per_second": 7.4
     }
   ],
   "logging_steps": 500,
-  "max_steps": 8109,
+  "max_steps": 54060,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 20,
   "save_steps": 500,
   "stateful_callbacks": {
     "EarlyStoppingCallback": {
@@ -101,7 +234,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4237242236928000.0,
+  "total_flos": 1.412414078976e+16,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null