Spaces:

nroggendorff
/

train-llama

Runtime error

nroggendorff commited on Jul 24, 2024

Commit

db0f387

verified ·

1 Parent(s): c2f601d

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -30,7 +30,7 @@ def create_tokenizer(training_corpus):
         special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>", "<|user|>", "<|bot|>", "<|end|>"]
     )
-    fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)
     return fast_tokenizer
 def get_training_corpus(dataset):

         special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>", "<|user|>", "<|bot|>", "<|end|>"]
     )
+    fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer._tokenizer)
     return fast_tokenizer
 def get_training_corpus(dataset):