File size: 1,076 Bytes
6a876f9 398ec26 139e10d 398ec26 139e10d 6a876f9 139e10d 6a876f9 398ec26 139e10d 6a876f9 398ec26 6a876f9 139e10d 6a876f9 139e10d 6a876f9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
#!/usr/bin/env python3
from datasets import load_dataset
from datasets import load_from_disk
from tokenizers import ByteLevelBPETokenizer, SentencePieceBPETokenizer
from tqdm import tqdm
from utils import keep_devnagri
# load dataset
dataset = load_dataset("mc4", "hi", split="train", streaming=True)
# Instantiate tokenizer
tokenizer = SentencePieceBPETokenizer(add_prefix_space=True)
def batch_iterator(batch_size=100_000):
# total docs: 1,85,07,273
text_ls = []
for example in dataset:
devnagari_text, is_just_punctuation = keep_devnagri(example['text'])
if not is_just_punctuation:
text_ls.append(devnagari_text)
if len(text_ls) == batch_size:
yield text_ls
text_ls = []
if len(text_ls) > 0:
yield text_ls
# Customized training
tokenizer.train_from_iterator(batch_iterator(), vocab_size=50265, min_frequency=50, special_tokens=[
"<s>",
"<pad>",
"</s>",
"<unk>",
"<mask>",
], )
# Save files to disk
tokenizer.save("/home/khandelia1000/tokenizer.json")
|