add python file

tokenizer.py

@@ -1,8 +1,5 @@
 from datasets import load_dataset
 from transformers import AutoTokenizer
-from huggingface_hub import Repository
-
-repo = Repository(".", clone_from="ybelkada/japanese-dummy-tokenizer")
 
 def get_training_corpus(dataset):
     """
@@ -22,7 +19,4 @@ print("Old Tokenizer:", old_tokenizer.tokenize("誰が一番に着くか私に
 new_tokenizer = old_tokenizer.train_new_from_iterator(get_training_corpus(train_dataset), 52000)
 
 print("New Tokenizer:",new_tokenizer.tokenize("誰が一番に着くか私には分かりません。"))
-new_tokenizer.save_pretrained("japanese-dummy-tokenizer")
-repo.git_add()
-repo.git_commit("Add tokenizer file")
-repo.git_push()
+new_tokenizer.save_pretrained("japanese-dummy-tokenizer")