Update app.py

app.py

@@ -3,18 +3,12 @@ import gradio as gr
 import aranizer
 from aranizer import aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
 
-# List of available tokenizers for the Radio buttons
-tokenizer_options = {
-    "aranizer_bpe50k": "BPE 50k",
-    "aranizer_bpe64k": "BPE 64k",
-    "aranizer_bpe86k": "BPE 86k",
-    "aranizer_sp32k": "SP 32k",
-    "aranizer_sp50k": "SP 50k",
-    "aranizer_sp64k": "SP 64k",
-    "aranizer_sp86k": "SP 86k",
-}
+# List of available tokenizers and a dictionary to load them
+tokenizer_options = [
+    "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k",
+    "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k"
+]
 
-# Mapping from names to tokenizer getters
 tokenizers = {
     "aranizer_bpe50k": aranizer_bpe50k.get_tokenizer,
     "aranizer_bpe64k": aranizer_bpe64k.get_tokenizer,
@@ -26,21 +20,22 @@ tokenizers = {
 }
 
 def compare_tokenizers(tokenizer_name, text):
-    tokenizer = tokenizers[tokenizer_name]()  # Loading the selected tokenizer
+    # Load the selected tokenizer
+    tokenizer = tokenizers[tokenizer_name]()
     tokens = tokenizer.tokenize(text)
     encoded_output = tokenizer.encode(text, add_special_tokens=True)
+    decoded_text = tokenizer.decode(encoded_output)
     
-    # Prepare the results to be displayed without Decoded Text
-    results = [(tokenizer_name, tokens, encoded_output)]
+    # Prepare the results to be displayed
+    results = [(tokenizer_name, tokens, encoded_output, decoded_text)]
     return results
 
-# Define Gradio interface components with a Radio for model selection (pretend grid with inline display)
+# Define the Gradio interface components with a dropdown for model selection
 inputs_component = [
-    gr.Radio(choices=list(tokenizer_options.values()), label="Select Tokenizer", inline=True),
+    gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer"),
     gr.Textbox(lines=2, placeholder="Enter Arabic text here...", label="Input Text")
 ]
-# Adjusted outputs to exclude the Decoded Text
-outputs_component = gr.Dataframe(headers=["Tokenizer", "Tokens", "Encoded Output"], label="Results")
+outputs_component = gr.Dataframe(headers=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"], label="Results", elem_height=500, elem_width='100%')
 
 # Setting up the interface
 iface = Interface(fn=compare_tokenizers, inputs=inputs_component, outputs=outputs_component, title="AraNizer Tokenizer Comparison")