Add tokenizer and model training script

t5-train.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+HF_DATASETS_IN_MEMORY_MAX_SIZE=200000000
+MODEL_DIR=bengali-t5-on-mc4
+run_t5_mlm_flax.py \
+    --output_dir="${MODEL_DIR}" \
+    --model_type="t5" \
+    --config_name="${MODEL_DIR}" \
+    --tokenizer_name="${MODEL_DIR}" \
+    --dataset_name="mc4" \
+    --dataset_config_name="bn" \
+    --max_seq_length="512" \
+    --per_device_train_batch_size="128" \
+    --per_device_eval_batch_size="128" \
+    --learning_rate="1e-3" \
+    --weight_decay="0.001" \
+    --warmup_steps="500" \
+    --overwrite_output_dir \
+    --num_train_epochs="10" \
+    --logging_steps="500" \
+    --save_steps="500" \
+    --eval_steps="500" \
+    #--push_to_hub

train_unigram_tokenizer_for_t5.py

@@ -0,0 +1,20 @@
+from tokenizers import Tokenizer
+from tokenizers.models import Unigram
+from tokenizers.trainers import UnigramTrainer
+from tokenizers.normalizers import NFKC
+from tokenizers.pre_tokenizers import UnicodeScripts, Digits, Sequence
+from datasets import load_dataset
+
+mc4 = load_dataset('mc4', 'bn', split='train')
+
+def batch_iterator(dataset, batch_size=1000):
+    for i in range(0, len(dataset), batch_size):
+        yield dataset[i: i + batch_size]["text"]
+
+tokenizer = Tokenizer(Unigram())
+tokenizer.normalizer = NFKC()
+tokenizer.pre_tokenizer = Sequence([UnicodeScripts(), Digits(individual_digits=True)]) 
+trainer = UnigramTrainer(vocab_size=32000, special_tokens=['<s>', '<pad>', '</s>', '<unk>', '<mask>'])
+tokenizer.train_from_iterator(batch_iterator(mc4), trainer=trainer, length=len(mc4))
+tokenizer.save('tokenizer-mc4-unigram.json')
+