Update app.py

app.py

@@ -1,40 +1,33 @@
+from gradio import Interface
 import gradio as gr
 import aranizer
+from aranizer import aranizer_bpe32k, aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
 
-# Function to dynamically load tokenizers
-def load_tokenizers():
-    tokenizer_methods = [
-        "aranizer_bpe32k", "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k",
-        "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k"
-    ]
-    tokenizers = {}
-    for method in tokenizer_methods:
-        module, function = method.rsplit("_", 1)
-        try:
-            tokenizer = getattr(getattr(aranizer, module), f'get_tokenizer_{function}')()
-            tokenizers[method] = tokenizer
-        except AttributeError as e:
-            print(f"Error loading tokenizer {method}: {e}")
-    return tokenizers
-
-# Now, let's load all available tokenizers
-tokenizers = load_tokenizers()
+# Load all available tokenizers
+tokenizers = {
+    "aranizer_bpe32k": aranizer.aranizer_bpe32k.get_tokenizer(),
+    "aranizer_bpe50k": aranizer.aranizer_bpe50k.get_tokenizer(),
+    "aranizer_bpe64k": aranizer.aranizer_bpe64k.get_tokenizer(),
+    "aranizer_bpe86k": aranizer.aranizer_bpe86k.get_tokenizer(),
+    "aranizer_sp32k": aranizer.aranizer_sp32k.get_tokenizer(),
+    "aranizer_sp50k": aranizer.aranizer_sp50k.get_tokenizer(),
+    "aranizer_sp64k": aranizer.aranizer_sp64k.get_tokenizer(),
+    "aranizer_sp86k": aranizer.aranizer_sp86k.get_tokenizer(),
+}
 
 def compare_tokenizers(text):
     results = []
     for name, tokenizer in tokenizers.items():
-        try:
-            tokens = tokenizer.tokenize(text)
-            encoded_output = tokenizer.encode(text, add_special_tokens=True)
-            decoded_text = tokenizer.decode(encoded_output)
-            results.append((name, tokens, encoded_output, decoded_text))
-        except Exception as e:
-            results.append((name, f"Error: {e}", "", ""))
+        tokens = tokenizer.tokenize(text)
+        encoded_output = tokenizer.encode(text, add_special_tokens=True)
+        decoded_text = tokenizer.decode(encoded_output)
+        results.append((name, tokens, encoded_output, decoded_text))
     return results
 
-inputs_component = gr.inputs.Textbox(lines=2, placeholder="Enter Arabic text here...", label="Input Text")
-outputs_component = gr.outputs.Table(label="Results", headers=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"])
+# Define the Gradio interface components properly based on the Gradio API
+inputs_component = gr.components.Textbox(lines=2, placeholder="Enter Arabic text here...", label="Input Text")
+outputs_component = gr.components.Table(label="Results", headers=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"])
 
-iface = gr.Interface(fn=compare_tokenizers, inputs=inputs_component, outputs=outputs_component, title="Tokenizer Comparison Tool")
+iface = Interface(fn=compare_tokenizers, inputs=inputs_component, outputs=outputs_component)
 
 iface.launch()