Saving weights and logs of step 300

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:017c34e041bd61f90e1e78c986113b87116dbefe5ed2db7c8343058b9b025a6e
+oid sha256:f1d8c763d288f266b49c1a11c5f341c28fd493185af0e577eb4f972c9de4afd8
 size 891548548

run_t5.sh

@@ -17,10 +17,10 @@ mkdir -p "${MODEL_DIR}/runs"
     --do_train --do_eval \
     --adafactor \
     --max_seq_length="512" \
-    --gradient_accumulation_steps="16" \
     --per_device_train_batch_size="32" \
     --per_device_eval_batch_size="32" \
-    --learning_rate="5e-3" \
+    --learning_rate="1e-2" \
+    --dtype="bfloat16" \
     --overwrite_output_dir \
     --num_train_epochs="1" \
     --logging_steps="15" \
@@ -32,6 +32,7 @@ mkdir -p "${MODEL_DIR}/runs"
 #git commit -m "Update pytorch model after training"
 #git push origin main
 
+#    --learning_rate="5e-3" \
+#    --gradient_accumulation_steps="2" \
 
-#    --dtype="bfloat16" \
-#    --resume_from_checkpoint="${MODEL_DIR}/ckpt-3300" \
+#    --resume_from_checkpoint="${MODEL_DIR}/ckpt-3300" \

runs/Jul10_07-45-49_t1v-n-0e7426e8-w-0/events.out.tfevents.1625903173.t1v-n-0e7426e8-w-0.20563.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc901d2e6b8e7e9bb6834c37930f08ed3dd4704877c26d80a07afe26132d6168
-size 136359
+oid sha256:07af3cc1b9177077621a9f9cd2b3d36aa028237973caf686c9e324e89cc25de9
+size 147729

runs/Jul10_08-38-10_t1v-n-0e7426e8-w-0/events.out.tfevents.1625906314.t1v-n-0e7426e8-w-0.25839.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f87e0b93569f9ebb013314589bc05f05421f4cad6c6903192a28805a6a75ca26
+size 45399