Switch to BertTokenizer

app.py

@@ -11,7 +11,7 @@ import numpy as np
 import tensorflow as tf
 import streamlit as st
 from gazpacho import Soup, get
-from transformers import AutoTokenizer, TFAutoModelForMaskedLM
+from transformers import BertTokenizer, TFAutoModelForMaskedLM
 
 
 DEFAULT_QUERY = "Machines will take over the world soon"
@@ -87,7 +87,7 @@ def start_rhyming(query, rhyme_words_options):
 def load_model(model_path):
     return (
         TFAutoModelForMaskedLM.from_pretrained(model_path),
-        AutoTokenizer.from_pretrained(model_path),
+        BertTokenizer.from_pretrained(model_path),
     )
 
 
@@ -121,7 +121,7 @@ class RhymeGenerator:
     def __init__(
         self,
         model: TFAutoModelForMaskedLM,
-        tokenizer: AutoTokenizer,
+        tokenizer: BertTokenizer,
         token_weighter: TokenWeighter = None,
     ):
         """Generate rhymes.