File size: 1,371 Bytes
f5e3080 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
from datasets import load_dataset, concatenate_datasets
from tokenizers import ByteLevelBPETokenizer
from pathlib import Path
dataset_language = "su"
validation_split_percentage = 10
# load dataset
# only the train subset for tokenizing purposes
oscar = load_dataset(
"oscar", f"unshuffled_deduplicated_{dataset_language}", split="train",
)
cc100 = load_dataset("cc100", lang=dataset_language, split="train")
mc4 = load_dataset("mc4", dataset_language, split="train")
wiki_files = [str(x) for x in Path("../docs").glob("*.txt")]
wiki = load_dataset("text", data_files=wiki_files)
# want: text column only!
oscar = oscar.remove_columns("id")
mc4 = mc4.remove_columns(["url", "timestamp"])
cc100 = cc100.remove_columns("id")
dataset = concatenate_datasets([oscar, mc4, cc100, wiki["train"]])
dataset = dataset.train_test_split(test_size=validation_split_percentage / 100, seed=42)
# Instantiate tokenizer
tokenizer = ByteLevelBPETokenizer()
def batch_iterator(batch_size=10000):
for i in range(0, len(dataset), batch_size):
yield dataset["train"][i : i + batch_size]["text"]
# Customized training
tokenizer.train_from_iterator(
batch_iterator(),
vocab_size=50265,
min_frequency=2,
special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>",],
)
# Save files to disk
model_dir = "."
tokenizer.save(f"{model_dir}/tokenizer.json")
|