Update app.py

app.py

@@ -1,51 +1,28 @@
-import gradio as gr
-from random import random
-from aranizer import aranizer_bpe32k, aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
+from gradio.inputs import Textbox
+from gradio.outputs import Textbox, Table
+from gradio import Interface
+import aranizer
 
-def load_tokenizer(tokenizer_choice):
-    # Dictionary mapping tokenizer choice to actual tokenizer initializer
-    tokenizer_map = {
-        "aranizer_bpe32k": aranizer_bpe32k.get_tokenizer(),
-        "aranizer_bpe50k": aranizer_bpe50k.get_tokenizer(),
-        "aranizer_bpe64k": aranizer_bpe64k.get_tokenizer(),
-        "aranizer_bpe86k": aranizer_bpe86k.get_tokenizer(),
-        "aranizer_sp32k": aranizer_sp32k.get_tokenizer(),
-        "aranizer_sp50k": aranizer_sp50k.get_tokenizer(),
-        "aranizer_sp64k": aranizer_sp64k.get_tokenizer(),
-        "aranizer_sp86k": aranizer_sp86k.get_tokenizer(),
-    }
+# Load your tokenizers
+tokenizers = {
+    "aranizer_bpe50k": aranizer.aranizer_bpe50k.get_tokenizer(),
+    "aranizer_bpe64k": aranizer.aranizer_bpe64k.get_tokenizer(),
+    "aranizer_sp32k": aranizer.aranizer_sp32k.get_tokenizer(),
+    # Add more tokenizers as needed
+}
 
-    return tokenizer_map.get(tokenizer_choice, None)
-
-def tokenize_and_encode_and_embed(text, tokenizer_choice):
-    tokenizer = load_tokenizer(tokenizer_choice)  # Use load_tokenizer here
-    if tokenizer:
-        # Example methods. Replace with actual methods from your tokenizer
+def compare_tokenizers(text):
+    results = []
+    for name, tokenizer in tokenizers.items():
         tokens = tokenizer.tokenize(text)
         encoded_output = tokenizer.encode(text, add_special_tokens=True)
         decoded_text = tokenizer.decode(encoded_output)
+        results.append((name, tokens, encoded_output, decoded_text))
+    return results
 
-        # Example embedding (replace with actual embedding generation from your model)
-        embeddings = [random() for _ in range(10)]  # Example 10-dimensional embedding vector
-
-        return " ".join(tokens), str(encoded_output), decoded_text, embeddings
-    else:
-        return "Tokenizer not loaded correctly", "", "", []
+inputs = Textbox(label="Enter Arabic text")
+outputs = Table(label="Results", columns=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"])
 
-demo = gr.Interface(
-    fn=tokenize_and_encode_and_embed,
-    inputs=[
-        gr.Textbox(lines=5, label="النص العربي"),
-        gr.Dropdown(choices=["aranizer_bpe32k", "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k", "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k"], label="اختر المحلل اللفظي")
-    ],
-    outputs=[
-        gr.Text(label="Tokens"),
-        gr.Text(label="Encoded Output"),
-        gr.Text(label="Decoded Text"),
-        gr.Text(label="Embeddings (Example Vector)")
-    ],
-    title="مقارنة المحللات اللفظية وعمليات التضمين للنص العربي",
-    description="حدد نوع المحلل اللفظي وأدخل نصًا لرؤية النتائج ومتجه التضمين.",
-)
+iface = Interface(fn=compare_tokenizers, inputs=inputs, outputs=outputs)
 
-demo.launch()
+iface.launch()