Add batch=128 model

README.md

@@ -37,11 +37,12 @@ The following hyperparameters were used during training:
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 4
-- total_train_batch_size: 4
+- gradient_accumulation_steps: 64
+- total_train_batch_size: 256
 - total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- num_epochs: 1.0
+- num_epochs: 2.0
 
 ### Training results
 

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.0,
-    "train_loss": 1.1057900991015273,
-    "train_runtime": 20175.6269,
+    "epoch": 1.99,
+    "train_loss": 0.7372576573065349,
+    "train_runtime": 28706.0932,
     "train_samples": 57593,
-    "train_samples_per_second": 2.855,
-    "train_steps_per_second": 0.714
+    "train_samples_per_second": 4.013,
+    "train_steps_per_second": 0.016
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62609e6df7ed82684c17b082b397f393363c8bcd6d70e058e77552379acd90ce
+oid sha256:4b06c217dc641008ad56030c95d59940d9fd021d87a9c1321e6a5bc721bcc6ed
 size 25953607721

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.0,
-    "train_loss": 1.1057900991015273,
-    "train_runtime": 20175.6269,
+    "epoch": 1.99,
+    "train_loss": 0.7372576573065349,
+    "train_runtime": 28706.0932,
     "train_samples": 57593,
-    "train_samples_per_second": 2.855,
-    "train_steps_per_second": 0.714
+    "train_samples_per_second": 4.013,
+    "train_steps_per_second": 0.016
 }

trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
-  "global_step": 14399,
+  "epoch": 1.991249392318911,
+  "global_step": 448,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -10,190 +10,22 @@
     {
       "epoch": 0.0,
       "learning_rate": 0,
-      "loss": 2.4746,
+      "loss": 2.4259,
       "step": 1
     },
     {
-      "epoch": 0.03,
-      "learning_rate": 5e-05,
-      "loss": 1.9874,
-      "step": 500
-    },
-    {
-      "epoch": 0.07,
-      "learning_rate": 5e-05,
-      "loss": 1.9445,
-      "step": 1000
-    },
-    {
-      "epoch": 0.1,
-      "learning_rate": 5e-05,
-      "loss": 1.8596,
-      "step": 1500
-    },
-    {
-      "epoch": 0.14,
-      "learning_rate": 5e-05,
-      "loss": 1.7807,
-      "step": 2000
-    },
-    {
-      "epoch": 0.17,
-      "learning_rate": 5e-05,
-      "loss": 1.7298,
-      "step": 2500
-    },
-    {
-      "epoch": 0.21,
-      "learning_rate": 5e-05,
-      "loss": 1.6565,
-      "step": 3000
-    },
-    {
-      "epoch": 0.24,
-      "learning_rate": 5e-05,
-      "loss": 1.5484,
-      "step": 3500
-    },
-    {
-      "epoch": 0.28,
-      "learning_rate": 5e-05,
-      "loss": 1.5049,
-      "step": 4000
-    },
-    {
-      "epoch": 0.31,
-      "learning_rate": 5e-05,
-      "loss": 1.4478,
-      "step": 4500
-    },
-    {
-      "epoch": 0.35,
-      "learning_rate": 5e-05,
-      "loss": 1.3567,
-      "step": 5000
-    },
-    {
-      "epoch": 0.38,
-      "learning_rate": 5e-05,
-      "loss": 1.2812,
-      "step": 5500
-    },
-    {
-      "epoch": 0.42,
-      "learning_rate": 5e-05,
-      "loss": 1.1882,
-      "step": 6000
-    },
-    {
-      "epoch": 0.45,
-      "learning_rate": 5e-05,
-      "loss": 1.1447,
-      "step": 6500
-    },
-    {
-      "epoch": 0.49,
-      "learning_rate": 5e-05,
-      "loss": 1.0854,
-      "step": 7000
-    },
-    {
-      "epoch": 0.52,
-      "learning_rate": 5e-05,
-      "loss": 1.0182,
-      "step": 7500
-    },
-    {
-      "epoch": 0.56,
-      "learning_rate": 5e-05,
-      "loss": 0.9675,
-      "step": 8000
-    },
-    {
-      "epoch": 0.59,
-      "learning_rate": 5e-05,
-      "loss": 0.9076,
-      "step": 8500
-    },
-    {
-      "epoch": 0.63,
-      "learning_rate": 5e-05,
-      "loss": 0.8415,
-      "step": 9000
-    },
-    {
-      "epoch": 0.66,
-      "learning_rate": 5e-05,
-      "loss": 0.8228,
-      "step": 9500
-    },
-    {
-      "epoch": 0.69,
-      "learning_rate": 5e-05,
-      "loss": 0.7656,
-      "step": 10000
-    },
-    {
-      "epoch": 0.73,
-      "learning_rate": 5e-05,
-      "loss": 0.7029,
-      "step": 10500
-    },
-    {
-      "epoch": 0.76,
-      "learning_rate": 5e-05,
-      "loss": 0.6793,
-      "step": 11000
-    },
-    {
-      "epoch": 0.8,
-      "learning_rate": 5e-05,
-      "loss": 0.6369,
-      "step": 11500
-    },
-    {
-      "epoch": 0.83,
-      "learning_rate": 5e-05,
-      "loss": 0.5936,
-      "step": 12000
-    },
-    {
-      "epoch": 0.87,
-      "learning_rate": 5e-05,
-      "loss": 0.5489,
-      "step": 12500
-    },
-    {
-      "epoch": 0.9,
-      "learning_rate": 5e-05,
-      "loss": 0.531,
-      "step": 13000
-    },
-    {
-      "epoch": 0.94,
-      "learning_rate": 5e-05,
-      "loss": 0.4948,
-      "step": 13500
-    },
-    {
-      "epoch": 0.97,
-      "learning_rate": 5e-05,
-      "loss": 0.4555,
-      "step": 14000
-    },
-    {
-      "epoch": 1.0,
-      "step": 14399,
-      "total_flos": 945700963614720.0,
-      "train_loss": 1.1057900991015273,
-      "train_runtime": 20175.6269,
-      "train_samples_per_second": 2.855,
-      "train_steps_per_second": 0.714
+      "epoch": 1.99,
+      "step": 448,
+      "total_flos": 1883191577149440.0,
+      "train_loss": 0.7372576573065349,
+      "train_runtime": 28706.0932,
+      "train_samples_per_second": 4.013,
+      "train_steps_per_second": 0.016
     }
   ],
-  "max_steps": 14399,
-  "num_train_epochs": 1,
-  "total_flos": 945700963614720.0,
+  "max_steps": 448,
+  "num_train_epochs": 2,
+  "total_flos": 1883191577149440.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6056591c3ad528ed37e907ac6ffb1883ba10604b406d19e94a3ddbe33e4c9d02
+oid sha256:d16517ebe6740e1b5ae9b137c22d1caae3fb8e1663fe042699314cf61bc64831
 size 4859