Update app.py

app.py

@@ -1,18 +1,17 @@
 from gradio import Interface
 import gradio as gr
 import aranizer
-from aranizer import aranizer_bpe32k, aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
+from aranizer import aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
 
-# Load all available tokenizers
+# Correct way to load all available tokenizers as per the provided usage information
 tokenizers = {
-    "aranizer_bpe32k": aranizer.aranizer_bpe32k.get_tokenizer(),
-    "aranizer_bpe50k": aranizer.aranizer_bpe50k.get_tokenizer(),
-    "aranizer_bpe64k": aranizer.aranizer_bpe64k.get_tokenizer(),
-    "aranizer_bpe86k": aranizer.aranizer_bpe86k.get_tokenizer(),
-    "aranizer_sp32k": aranizer.aranizer_sp32k.get_tokenizer(),
-    "aranizer_sp50k": aranizer.aranizer_sp50k.get_tokenizer(),
-    "aranizer_sp64k": aranizer.aranizer_sp64k.get_tokenizer(),
-    "aranizer_sp86k": aranizer.aranizer_sp86k.get_tokenizer(),
+    "aranizer_bpe50k": aranizer_bpe50k.get_tokenizer(),
+    "aranizer_bpe64k": aranizer_bpe64k.get_tokenizer(),
+    "aranizer_bpe86k": aranizer_bpe86k.get_tokenizer(),
+    "aranizer_sp32k": aranizer_sp32k.get_tokenizer(),
+    "aranizer_sp50k": aranizer_sp50k.get_tokenizer(),
+    "aranizer_sp64k": aranizer_sp64k.get_tokenizer(),
+    "aranizer_sp86k": aranizer_sp86k.get_tokenizer(),
 }
 
 def compare_tokenizers(text):
@@ -21,13 +20,16 @@ def compare_tokenizers(text):
         tokens = tokenizer.tokenize(text)
         encoded_output = tokenizer.encode(text, add_special_tokens=True)
         decoded_text = tokenizer.decode(encoded_output)
+        # Collect each tokenizer's results
         results.append((name, tokens, encoded_output, decoded_text))
     return results
 
-# Define the Gradio interface components properly based on the Gradio API
+# Correctly use Gradio's components for inputs and outputs
 inputs_component = gr.components.Textbox(lines=2, placeholder="Enter Arabic text here...", label="Input Text")
 outputs_component = gr.components.Table(label="Results", headers=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"])
 
-iface = Interface(fn=compare_tokenizers, inputs=inputs_component, outputs=outputs_component)
+# Setting up the interface
+iface = Interface(fn=compare_tokenizers, inputs=inputs_component, outputs=outputs_component, title="AraNizer Tokenizer Comparison")
 
+# Launching the Gradio app
 iface.launch()