Update app.py

app.py

@@ -14,20 +14,20 @@ def load_tokenizer(tokenizer_choice):
         "aranizer_sp64k": aranizer_sp64k.get_tokenizer(),
         "aranizer_sp86k": aranizer_sp86k.get_tokenizer(),
     }
-    
+
     return tokenizer_map.get(tokenizer_choice, None)
 
 def tokenize_and_encode_and_embed(text, tokenizer_choice):
-    tokenizer = get_tokenizer(tokenizer_choice)
+    tokenizer = load_tokenizer(tokenizer_choice)  # Use load_tokenizer here
     if tokenizer:
         # Example methods. Replace with actual methods from your tokenizer
         tokens = tokenizer.tokenize(text)
         encoded_output = tokenizer.encode(text, add_special_tokens=True)
         decoded_text = tokenizer.decode(encoded_output)
-        
+
         # Example embedding (replace with actual embedding generation from your model)
         embeddings = [random() for _ in range(10)]  # Example 10-dimensional embedding vector
-        
+
         return " ".join(tokens), str(encoded_output), decoded_text, embeddings
     else:
         return "Tokenizer not loaded correctly", "", "", []
@@ -48,4 +48,4 @@ demo = gr.Interface(
     description="حدد نوع المحلل اللفظي وأدخل نصًا لرؤية النتائج ومتجه التضمين.",
 )
 
-demo.launch()
+demo.launch()