Saving weights and logs of step 10000

events.out.tfevents.1630324517.t1v-n-1ae8dadb-w-0.551349.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1203aed8447582f392a2f352627ab346fc40d4eab7f73dce0c7c7662f7336091
+size 40

events.out.tfevents.1630325064.t1v-n-1ae8dadb-w-0.554071.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea235530adbca24b3e9dc2315a3fa56a6022c7e39fc733baf5a4aafee9fad5dd
+size 1470757

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f87e2ed3eebcebe5f871845c9acd8a737b4575431f9ac5e651d96a6bd77826e2
+oid sha256:c54f3c2c1b748cf9a071b21240bd58e11b66c7f53878208fcf8521d6d3346e0a
 size 1421662309

run_mlm_flax.py

@@ -494,6 +494,14 @@ if __name__ == "__main__":
     
     # save the tokenized dataset for future runs
     if data_args.save_tokenized_dataset_filepath is not None:
+        if data_args.dataset_filepath is not None:
+            try:
+                os.system(f"sudo rm {data_args.dataset_filepath}/train/cache*")
+                os.system(f"sudo rm {data_args.dataset_filepath}/validation/cache*")
+                os.system(f"sudo rm {data_args.dataset_filepath}/train/tmp*")
+                os.system(f"sudo rm {data_args.dataset_filepath}/validation/tmp*")
+            except:
+                pass
         tokenized_datasets.save_to_disk(data_args.save_tokenized_dataset_filepath)
 
     # Enable tensorboard only on the master node

start_train.sh

@@ -6,12 +6,12 @@ python3 run_mlm_flax.py \
     --config_name="./" \
     --tokenizer_name="./" \
     --dataset_filepath="/researchdisk1/data/training_data_full" \
-    --tokenized_dataset_filepath="/researchdisk1/data/training_data_full_tokenized_128" \
-    --max_seq_length="128" \
+    --tokenized_dataset_filepath="/researchdisk1/data/training_data_full_tokenized_512" \
+    --max_seq_length="512" \
     --pad_to_max_length \
     --preprocessing_num_workers="96" \
-    --per_device_train_batch_size="32" \
-    --per_device_eval_batch_size="32" \
+    --per_device_train_batch_size="16" \
+    --per_device_eval_batch_size="16" \
     --adam_beta1="0.9" \
     --adam_beta2="0.98" \
     --adam_epsilon="1e-6" \