Update app.py

app.py

@@ -1,14 +1,20 @@
 import gradio as gr
 from random import random
+from aranizer import aranizer_bpe32k, aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
 
-# Assuming `aranizer` is a library that provides these tokenizer classes or functions (pseudo-code for demonstration only)
-# Please implement actual imports and tokenizer initialization logic
-def get_tokenizer(tokenizer_choice):
-    # Placeholder - Replace with actual tokenizer loading logic
+def load_tokenizer(tokenizer_choice):
+    # Dictionary mapping tokenizer choice to actual tokenizer initializer
     tokenizer_map = {
-        "aranizer_bpe32k": None,  # Replace None with actual tokenizer, e.g., aranizer_bpe32k.get_tokenizer()
-        # Add other tokenizers here
+        "aranizer_bpe32k": aranizer_bpe32k.get_tokenizer(),
+        "aranizer_bpe50k": aranizer_bpe50k.get_tokenizer(),
+        "aranizer_bpe64k": aranizer_bpe64k.get_tokenizer(),
+        "aranizer_bpe86k": aranizer_bpe86k.get_tokenizer(),
+        "aranizer_sp32k": aranizer_sp32k.get_tokenizer(),
+        "aranizer_sp50k": aranizer_sp50k.get_tokenizer(),
+        "aranizer_sp64k": aranizer_sp64k.get_tokenizer(),
+        "aranizer_sp86k": aranizer_sp86k.get_tokenizer(),
     }
+    
     return tokenizer_map.get(tokenizer_choice, None)
 
 def tokenize_and_encode_and_embed(text, tokenizer_choice):