Update model

README.md

@@ -1 +1 @@
-Logs at https://wandb.ai/yepster/long-t5-tglobal-small/runs/1s5jeq5q?workspace=user-yepster
+Logs at https://wandb.ai/yepster/long-t5-tglobal-small/runs/2wiy76y6?workspace=user-yepster

eval_results.json

@@ -1,4 +1,4 @@
 {
-    "eval_accuracy": 0.6216245889663696,
-    "eval_loss": 1.904692530632019
+    "eval_accuracy": 0.6504417657852173,
+    "eval_loss": 1.9302031993865967
 }

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95364c438eca009d45f05cf04a79d78d949c17c6855db8545983940388f205aa
+oid sha256:2d22d88f2f0bee62ab83ce809dd14295a22da03baab455e4ae5797bc693f9ec8
 size 307750439

run_longt5-tglobal-small-mc4.sh

@@ -26,32 +26,33 @@ python ../train/run_t5_mlm_flax_pmap.py \
     --dataset_name="${DATASET}" \
     --dataset_config_name="${DATASET_CONFIG}" \
     --max_seq_length="1024" \
-    --per_device_train_batch_size="32" \
-    --per_device_eval_batch_size="32" \
-    --gradient_accumulation_steps="4" \
+    --per_device_train_batch_size="64" \
+    --per_device_eval_batch_size="64" \
+    --gradient_accumulation_steps="1" \
     --mean_noise_span_length="3" \
-    --dtype="float32" \
+    --gradient_checkpointing="false" \
+    --dtype="bfloat16" \
+    --z_loss="1e-4" \
     --optim="adafactor" \
     --learning_rate="0.005" \
-    --lr_decay="linear" \
+    --lr_scheduler_type="linear" \
     --overwrite_output_dir \
-    --num_train_epochs="4" \
-    --logging_steps="20" \
-    --save_steps="1000" \
-    --eval_steps="1000" \
-    --warmup_steps="300" \
-    --validation_split_count="15000" \
+    --num_train_epochs="6" \
+    --logging_steps="80" \
+    --save_steps="4000" \
+    --eval_steps="4000" \
+    --warmup_steps="3000" \
     --wandb_project="long-t5-tglobal-small" \
     --wandb_job_type="pmap"
 
 
 #    --max_train_samples="160000" \
 #    --max_eval_samples="1000"
+#    --resume_from_checkpoint="${MODEL_PATH}" \
 
 #    --model_name_or_path="${MODEL_PATH}" \
 
 #     \
-#    --resume_from_checkpoint="${MODEL_PATH}"
 
 #    --lr_decay="exponential" \
 #    --lr_transition_steps="400000" \

training_state.json

@@ -1 +1 @@
-{"step": 112001}
+{"step": 84001}