yhavinga
/

t5-v1.1-base-dutch-uncased

Text2Text Generation Transformers JAX TensorBoard

Dutch t5 seq2seq text-generation-inference

Model card Files Files and versions Metrics Training metrics Community

t5-v1.1-base-dutch-uncased / run_t5.sh

yhavinga's picture

Add train script

f89796f over 2 years ago

raw history blame contribute delete

No virus

1.34 kB

	#!/bin/bash

	export HF_PROJECT="t5-v1.1-base-dutch-uncased"
	export VOCAB_SIZE="32000"
	export N_INPUT_SENTENCES="1000000" # Num of sentences to train the tokenizer
	export DATASET="yhavinga/mc4_nl_cleaned" # Name of the dataset in the Huggingface Hub
	export DATASET_CONFIG="full" # Config of the dataset in the Huggingface Hub
	export DATASET_SPLIT="train" # Split to use for training tokenizer and model
	export TEXT_FIELD="text" # Field containing the text to be used for training
	export CONFIG_TYPE="google/t5-v1_1-base" # Config that our model will use
	export MODEL_PATH="${HOME}/data/${HF_PROJECT}" # Path to the model

	python run_t5_mlm_flax.py \
	--output_dir="${MODEL_PATH}" \
	--model_type="t5" \
	--config_name="${MODEL_PATH}" \
	--tokenizer_name="${MODEL_PATH}" \
	--preprocessing_num_workers="96" \
	--do_train --do_eval \
	--dataset_name="${DATASET}" \
	--dataset_config_name="${DATASET_CONFIG}" \
	--max_seq_length="1024" \
	--per_device_train_batch_size="8" \
	--per_device_eval_batch_size="8" \
	--adafactor \
	--learning_rate="0.005" \
	--overwrite_output_dir \
	--num_train_epochs="2" \
	--logging_steps="500" \
	--save_steps="80000" \
	--eval_steps="2500" \
	--weight_decay="0.001" \
	--warmup_steps="10000" \
	--validation_split_count="15000" \
	--push_to_hub