Saving weights and logs of step 10000

config.json

@@ -20,8 +20,8 @@
   "num_hidden_layers": 12,
   "pad_token_id": 1,
   "position_embedding_type": "absolute",
-  "torch_dtype": "float32",
-  "transformers_version": "4.14.0.dev0",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.15.0.dev0",
   "type_vocab_size": 1,
   "use_cache": true,
   "vocab_size": 50265

events.out.tfevents.1640267893.t1v-n-ccbf3e94-w-0.1815882.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2402ba2035cc5336fb1a525316962406a94006a1949d4f3a9abc23687f18b6d7
+size 40

events.out.tfevents.1640306668.t1v-n-ccbf3e94-w-0.1858138.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5042739f45e5ff277b7816530227712dbc44d14f1ecb4e087857508c6aec056
+size 40

events.out.tfevents.1640342709.t1v-n-ccbf3e94-w-0.1891938.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4706a8941689a1d8e657aaa7e746f5cf087039377bcdf6727b0cdbd0bb99afb0
+size 40

events.out.tfevents.1640584052.t1v-n-ccbf3e94-w-0.2048063.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f36c3f39db8a456e90edbc8efa0b2ca1aac1fdf9b1c2952289d87c703ff98b7b
+size 40

events.out.tfevents.1640604030.t1v-n-ccbf3e94-w-0.2117285.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcf63d7a4b4b2bf4dcb10bd3f7a292a719a6060663d1fd1454cd2a6fdf42d1e6
+size 40

events.out.tfevents.1640614846.t1v-n-ccbf3e94-w-0.2129895.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdf83876709cfc72a855f8adb08a79225eb1a8fa3f3dd0545e9596948a94a2cc
+size 40

events.out.tfevents.1640973870.t1v-n-ccbf3e94-w-0.135363.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e35f3e489425792028defb716d72f7d8744224524fa57d8dd819b8b6247fd9f3
+size 1470136

flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f8a01e29c7428d66b11693f80574203b31ab8f28bb1bd23313d222eb521c3e6
+size 498796983

run_mlm_flax.py

@@ -508,7 +508,7 @@ if __name__ == "__main__":
         init_value=0.0, end_value=training_args.learning_rate, transition_steps=training_args.warmup_steps
     )
 
-    if data_argsdata_argtatic_learning_rate:
+    if data_args.static_learning_rate:
         end_lr_value = training_args.learning_rate
     else:
         end_lr_value = 0

run_step1.sh

@@ -3,16 +3,17 @@
     --model_type="roberta" \
     --config_name="./" \
     --tokenizer_name="./" \
-    --dataset_name="NbAiLab/nbailab_extended" \
+    --train_file /mnt/disks/flaxdisk/corpus/train_1_4.json \
+    --validation_file /mnt/disks/flaxdisk/corpus/validation.json \
     --cache_dir="/mnt/disks/flaxdisk/cache/" \
     --max_seq_length="512" \
     --weight_decay="0.01" \
-    --per_device_train_batch_size="48" \
-    --per_device_eval_batch_size="48" \
-    --learning_rate="4e-4" \
+    --per_device_train_batch_size="40" \
+    --per_device_eval_batch_size="40" \
+    --learning_rate="2e-4" \
     --warmup_steps="10000" \
     --overwrite_output_dir \
-    --num_train_epochs="1000" \
+    --num_train_epochs="2" \
     --adam_beta1="0.9" \
     --adam_beta2="0.98" \
     --adam_epsilon="1e-6" \
@@ -22,4 +23,5 @@
     --preprocessing_num_workers="64" \
     --auth_token="True" \
     --static_learning_rate="True" \
+    --dtype="bfloat16" \
     --push_to_hub