Update app.py

app.py

@@ -20,7 +20,7 @@ import jellyfish
 from gensim.models import Word2Vec
 from gensim.models.fasttext import FastText
 from collections import Counter
-from tokenizers import Tokenizer, models
+from tokenizers import Tokenizer, models, trainers
 from tokenizers.models import WordLevel
 from tokenizers.trainers import WordLevelTrainer
 from tokenizers.pre_tokenizers import Whitespace
@@ -357,7 +357,7 @@ def optimize_vocabulary(texts, vocab_size=10000, min_frequency=2):
     
     # Train BPE tokenizer
 #    tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
-    trainer = models.BpeTrainer(vocab_size=vocab_size, special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
+    trainer = trainers.BpeTrainer(vocab_size=vocab_size, special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
     tokenizer.train_from_iterator(optimized_texts, trainer)
     
     return tokenizer, optimized_texts