vocab dict.txt added

app.py

@@ -16,7 +16,7 @@ DEFAULT_QUERY = "Machines will take over the world soon"
 N_RHYMES = 10
 
 
-LANGUAGE = st.sidebar.radio("Language", ["english", "dutch", "french"],0)
+"""LANGUAGE = st.sidebar.radio("Language", ["english", "dutch", "french"],0)
 if LANGUAGE == "english":
     MODEL_PATH = "bert-large-cased-whole-word-masking"
     ITER_FACTOR = 5
@@ -24,10 +24,14 @@ elif LANGUAGE == "dutch":
     MODEL_PATH = "GroNLP/bert-base-dutch-cased"
     ITER_FACTOR = 10  # Faster model
 elif LANGUAGE == "french":
-    MODEL_PATH = "camembert-base"
-    ITER_FACTOR = 5
+
 else:
     raise NotImplementedError(f"Unsupported language ({LANGUAGE}) expected 'english','dutch' or 'french.")
+"""
+
+LANGUAGE = "french"
+MODEL_PATH = "camembert-base"
+ITER_FACTOR = 5
 
 def main():
     st.markdown(
@@ -93,6 +97,7 @@ def load_model(model_path, language):
             BertTokenizer.from_pretrained(model_path),
         )
     else :
+        tokenizer = CamembertTokenizer(vocab_file='rhyme_with_ai/dict.txt')
         return (
         CamembertModel.from_pretrained(model_path),
         CamembertTokenizer.from_pretrained(model_path),

rhyme_with_ai/token_weighter.py

@@ -7,7 +7,7 @@ class TokenWeighter:
         self.proba = self.get_token_proba()
 
     def get_token_proba(self):
-        valid_token_mask = self._filter_short_partial(self.tokenizer_.vocab)
+        valid_token_mask = self._filter_short_partial(self.tokenizer_.vocabulaire)
         return valid_token_mask
 
     def _filter_short_partial(self, vocab):