Add run script

run_long-byt5-tglobal-small-mc4.sh

@@ -0,0 +1,55 @@
+export CORES=`grep -c ^processor /proc/cpuinfo`
+export XLA_PYTHON_CLIENT_PREALLOCATE=false
+
+export HF_PROJECT="long-byt5-tglobal-small-dutch-english"
+export DATASET="yhavinga/mc4_nl_cleaned"
+export DATASET_CONFIG="tiny_en_nl" # Config of the dataset in the Huggingface Hub
+export DATASET_SPLIT="train" # Split to use for training tokenizer and model
+export CONFIG_NAME="../config/config-long-byt5-tglobal-small.json"
+export TOKENIZER_NAME="google/byt5-small"
+export MODEL_PATH="${HOME}/data/${HF_PROJECT}" # Path to the model
+
+mkdir -p "${MODEL_PATH}"
+
+python ../train/run_t5_mlm_flax_pmap.py \
+    --output_dir="${MODEL_PATH}" \
+    --resume_from_checkpoint="${MODEL_PATH}" \
+    --model_type="longt5" \
+    --config_name="${CONFIG_NAME}" \
+    --tokenizer_name="${TOKENIZER_NAME}" \
+    --preprocessing_num_workers="${CORES}" \
+    --do_train --do_eval \
+    --dataset_name="${DATASET}" \
+    --dataset_config_name="${DATASET_CONFIG}" \
+    --max_seq_length="1024" \
+    --per_device_train_batch_size="16" \
+    --per_device_eval_batch_size="16" \
+    --gradient_accumulation_steps="8" \
+    --mean_noise_span_length="20" \
+    --dtype="float32" \
+    --optim="adafactor" \
+    --learning_rate="0.0034" \
+    --lr_decay="linear" \
+    --overwrite_output_dir \
+    --num_train_epochs="2" \
+    --logging_steps="20" \
+    --save_steps="1000" \
+    --eval_steps="1000" \
+    --warmup_steps="300" \
+    --validation_split_count="15000" \
+    --wandb_project="long-byt5-tglobal-small" \
+    --wandb_job_type="pmap"
+
+#     \
+#    --max_train_samples="160000" \
+#    --max_eval_samples="1000" \
+
+#    --model_name_or_path="${MODEL_PATH}" \
+
+#     \
+
+#    --lr_decay="exponential" \
+#    --lr_transition_steps="400000" \
+#    --lr_decay_rate="0.7" \
+#    --lr_staircase="false" \
+#    --auth_token="$(cat ~/.huggingface/token)" \