|
from datasets import load_dataset |
|
from tokenizers import ByteLevelBPETokenizer |
|
from pythainlp.tokenize import word_tokenize |
|
|
|
|
|
dataset = load_dataset("oscar", "unshuffled_deduplicated_th", split="train") |
|
|
|
|
|
tokenizer = ByteLevelBPETokenizer() |
|
|
|
|
|
def th_tokenize(text): |
|
result = " ".join(word_tokenize(text, engine="newmm", keep_whitespace=False)) |
|
return result |
|
|
|
|
|
def batch_iterator(batch_size=1000): |
|
for i in range(0, len(dataset), batch_size): |
|
yield [th_tokenize(text) for text in dataset[i : i + batch_size]["text"]] |
|
|
|
|
|
|
|
tokenizer.train_from_iterator( |
|
batch_iterator(), |
|
vocab_size=50265, |
|
min_frequency=2, |
|
special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>",], |
|
) |
|
|
|
|
|
tokenizer.save(f"./tokenizer.json") |
|
|
|
|
