Saving weights and logs of step 10000

flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:551fdc978da02b8bb2d4c532488e94af3ed1bb471671edc02611cd5ef07f1e00
+size 711588089

run_mlm_flax.py

@@ -31,7 +31,7 @@ from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
 import numpy as np
-from datasets import load_dataset
+from datasets import load_dataset, load_from_disk
 from tqdm import tqdm
 
 import flax

start_train.sh

@@ -6,7 +6,7 @@ python3 run_mlm_flax.py \
     --config_name="./" \
     --tokenizer_name="./" \
     --dataset_filepath="/researchdisk1/data/training_data_full" \
-    --save_tokenized_dataset_filepath="/researchdisk1/data/training_data_full_tokenized_128" \
+    --tokenized_dataset_filepath="/researchdisk1/data/training_data_full_tokenized_128" \
     --max_seq_length="128" \
     --pad_to_max_length \
     --preprocessing_num_workers="96" \
@@ -22,4 +22,5 @@ python3 run_mlm_flax.py \
     --eval_steps="10000" \
     --logging_steps="1000" \
     --dtype="bfloat16" \
+    --adafactor \
     --push_to_hub