Update model

README.md

@@ -1 +1 @@
-See logs at https://wandb.ai/yepster/long-t5-local-small/runs/3c5gvwod?workspace=user-yepster
+See logs at https://wandb.ai/yepster/long-t5-local-small/runs/37o59uad?workspace=user-yepster

eval_results.json

@@ -1,4 +1,4 @@
 {
-    "eval_accuracy": 0.6216245889663696,
-    "eval_loss": 1.904692530632019
+    "eval_accuracy": 0.6356837749481201,
+    "eval_loss": 1.9501113891601562
 }

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95364c438eca009d45f05cf04a79d78d949c17c6855db8545983940388f205aa
+oid sha256:00bb5b987a1bba701e3d03578ceed23e976c4b9e22db5d095c12495abd5e3a09
 size 307750439

run_longt5-local-small-mc4.sh

@@ -30,17 +30,17 @@ python ../train/run_t5_mlm_flax_pmap.py \
     --per_device_eval_batch_size="32" \
     --gradient_accumulation_steps="4" \
     --mean_noise_span_length="3" \
-    --dtype="float32" \
+    --dtype="bfloat16" \
+    --z_loss="1e-4" \
     --optim="adafactor" \
     --learning_rate="0.005" \
     --lr_decay="linear" \
     --overwrite_output_dir \
-    --num_train_epochs="4" \
+    --num_train_epochs="6" \
     --logging_steps="20" \
     --save_steps="1000" \
     --eval_steps="1000" \
     --warmup_steps="300" \
-    --validation_split_count="15000" \
     --wandb_project="long-t5-local-small" \
     --wandb_job_type="pmap"
 

opt_state.msgpack → runs/Aug14_21-50-45_t1v-n-40c2b24e-w-0/events.out.tfevents.1660513900.t1v-n-40c2b24e-w-0.1025593.0.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d175742112303b753a95793344147207e107480407df90ecb849e89cc28ace2
-size 308759699
+oid sha256:a1863a6bfac6afc7708d3bb6e53d4f1b1eaa7d05a3adeabb256ed2e600622e52
+size 1086215

training_state.json

@@ -1 +1 @@
-{"step": 112001}
+{"step": 168001}