Update app.py

app.py

@@ -1,14 +1,16 @@
-from gradio.inputs import Textbox
-from gradio.outputs import Textbox, Table
-from gradio import Interface
+from gradio import inputs, outputs, Interface
 import aranizer
-
-# Load your tokenizers
+from aranizer import aranizer_bpe32k, aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k,aranizer_sp50k,aranizer_sp64k, aranizer_sp86k
+# Load all available tokenizers
 tokenizers = {
+    "aranizer_bpe32k": aranizer.aranizer_bpe32k.get_tokenizer(),
     "aranizer_bpe50k": aranizer.aranizer_bpe50k.get_tokenizer(),
     "aranizer_bpe64k": aranizer.aranizer_bpe64k.get_tokenizer(),
+    "aranizer_bpe86k": aranizer.aranizer_bpe86k.get_tokenizer(),
     "aranizer_sp32k": aranizer.aranizer_sp32k.get_tokenizer(),
-    # Add more tokenizers as needed
+    "aranizer_sp50k": aranizer.aranizer_sp50k.get_tokenizer(),
+    "aranizer_sp64k": aranizer.aranizer_sp64k.get_tokenizer(),
+    "aranizer_sp86k": aranizer.aranizer_sp86k.get_tokenizer(),
 }
 
 def compare_tokenizers(text):
@@ -20,8 +22,8 @@ def compare_tokenizers(text):
         results.append((name, tokens, encoded_output, decoded_text))
     return results
 
-inputs = Textbox(label="Enter Arabic text")
-outputs = Table(label="Results", columns=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"])
+inputs = inputs.Textbox(label="Enter Arabic text")
+outputs = outputs.Table(label="Results", columns=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"])
 
 iface = Interface(fn=compare_tokenizers, inputs=inputs, outputs=outputs)