change config and fix training tokenizer script

config.json

@@ -21,5 +21,5 @@
   "transformers_version": "4.9.0.dev0",
   "type_vocab_size": 1,
   "use_cache": true,
-  "vocab_size": 21153
+  "vocab_size": 50265
 }

events.out.tfevents.1626329391.t1v-n-bf8aeee7-w-0.4713.3.v2

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0306cd10b128d1cf1aa7395239854fb0fea8ac8263aebc218c879dc141dbece3
-size 73496

train_tokenizer.py

@@ -1,11 +1,8 @@
-#pip install -U pythainlp
-
 from datasets import load_dataset, concatenate_datasets
 from tokenizers import ByteLevelBPETokenizer
 from transformers import AutoConfig
 from pythainlp.tokenize import word_tokenize
 
-
 language = "th"
 model_config = "roberta-base"
 model_dir = model_config + f"-pretrained-{language}"
@@ -14,21 +11,22 @@ config.save_pretrained(f"{model_dir}")
 
 # load dataset
 # only the train subset for tokenizing purposes
-raw_dataset = load_dataset("oscar", f"unshuffled_deduplicated_{language}")
-raw_dataset = load_dataset("oscar", f"unshuffled_deduplicated_th")
+raw_dataset = load_dataset(
+    "oscar", f"unshuffled_deduplicated_{language}", split="train"
+)
 
 # Instantiate tokenizer
 tokenizer = ByteLevelBPETokenizer()
 
-
 ## For Thai NLP Library, please feel free to check https://pythainlp.github.io/docs/2.3/api/tokenize.html
 def th_tokenize(text):
     result = " ".join(word_tokenize(text, engine="newmm", keep_whitespace=False))
     return result
 
+
 def batch_iterator(batch_size=1000):
     for i in range(0, len(raw_dataset), batch_size):
-        yield [th_tokenize(text) for text in raw_dataset["train"][i: i + batch_size]["text"]]
+        yield [th_tokenize(text) for text in raw_dataset[i : i + batch_size]["text"]]
 
 
 # Customized training
@@ -36,7 +34,7 @@ tokenizer.train_from_iterator(
     batch_iterator(),
     vocab_size=50265,
     min_frequency=2,
-    special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>", ],
+    special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>",],
 )
 
 # Save files to disk