train with 10K batch_size

train_tokenizer.py

@@ -24,7 +24,7 @@ def th_tokenize(text):
     return result
 
 
-def batch_iterator(batch_size=1000):
+def batch_iterator(batch_size=10000):
     for i in range(0, len(raw_dataset), batch_size):
         yield [th_tokenize(text) for text in raw_dataset[i : i + batch_size]["text"]]