Update app.py

app.py

@@ -1,63 +1,74 @@
 from gradio import Interface
 import gradio as gr
 import aranizer
+from aranizer import aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
 from transformers import AutoTokenizer
-import codecs
 
-# Loading tokenizer instances from Transformers
+# Load additional tokenizers from transformers
 gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
 gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
 jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
 
-# Assuming the existence of get_tokenizer() method for aranizer models in your setup
+# List of available tokenizers and a dictionary to load them
+tokenizer_options = [
+    "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k",
+    "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k",
+    "FreedomIntelligence/AceGPT-13B",  
+    "FreedomIntelligence/AceGPT-7B",   
+    "inception-mbzuai/jais-13b"        
+]
+
 tokenizers = {
-    "aranizer_bpe50k": lambda: aranizer.aranizer_bpe50k.get_tokenizer(),
-    "aranizer_bpe64k": lambda: aranizer.aranizer_bpe64k.get_tokenizer(),
-    "aranizer_bpe86k": lambda: aranizer.aranizer_bpe86k.get_tokenizer(),
-    "aranizer_sp32k": lambda: aranizer.aranizer_sp32k.get_tokenizer(),
-    "aranizer_sp50k": lambda: aranizer.aranizer_sp50k.get_tokenizer(),
-    "aranizer_sp64k": lambda: aranizer.aranizer_sp64k.get_tokenizer(),
-    "aranizer_sp86k": lambda: aranizer.aranizer_sp86k.get_tokenizer(),
+    "aranizer_bpe50k": aranizer_bpe50k.get_tokenizer,
+    "aranizer_bpe64k": aranizer_bpe64k.get_tokenizer,
+    "aranizer_bpe86k": aranizer_bpe86k.get_tokenizer,
+    "aranizer_sp32k": aranizer_sp32k.get_tokenizer,
+    "aranizer_sp50k": aranizer_sp50k.get_tokenizer,
+    "aranizer_sp64k": aranizer_sp64k.get_tokenizer,
+    "aranizer_sp86k": aranizer_sp86k.get_tokenizer,
     "FreedomIntelligence/AceGPT-13B": lambda: gpt_13b_tokenizer,
     "FreedomIntelligence/AceGPT-7B": lambda: gpt_7b_tokenizer,
-    "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
+    "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer  
 }
 
-# Define tokenizer options for dropdown menu
-tokenizer_options = list(tokenizers.keys())
-
 def compare_tokenizers(tokenizer_name, text):
-    # UTF-8 encoding assertion for the input text
-    text = codecs.decode(text.encode('utf-8'), 'utf-8')
+    # Handle the transformer tokenizers separately due to API differences
+    if tokenizer_name in ["FreedomIntelligence/AceGPT-13B", "FreedomIntelligence/AceGPT-7B", "inception-mbzuai/jais-13b"]:
+        tokenizer = tokenizers[tokenizer_name]()
+        tokens = tokenizer.tokenize(text)
+        encoded_output = tokenizer.encode(text, add_special_tokens=True, return_tensors="pt")
+        decoded_text = tokenizer.decode(encoded_output[0], skip_special_tokens=True)
+    else:
+        # AraNizer tokenizers
+        tokenizer = tokenizers[tokenizer_name]()
+        tokens = tokenizer.tokenize(text)
+        encoded_output = tokenizer.encode(text, add_special_tokens=True)
+        decoded_text = tokenizer.decode(encoded_output)
     
-    tokenizer = tokenizers[tokenizer_name]()
-    tokens = tokenizer.tokenize(text)
-    encoded_output = tokenizer.encode(text, add_special_tokens=True, return_tensors="pt")
-    decoded_text = tokenizer.decode(encoded_output[0], skip_special_tokens=True)
-    
-    # Ensuring the tokens are iterated and converted correctly
-    tokens_utf8 = [codecs.decode(token.encode('utf-8'), 'utf-8', errors='ignore') for token in tokens]
-
-    # Preparing and returning results in UTF-8
-    results = [(tokenizer_name, tokens_utf8, encoded_output.tolist(), decoded_text)]
+    # Prepare the results to be displayed
+    results = [(tokenizer_name, tokens, encoded_output, decoded_text)]
     return results
 
+# Define the Gradio interface components with a dropdown for model selection
 inputs_component = [
-    gr.Dropdown(choices=tokenizer_options, label="اختر Tokenizer"),
-    gr.Textbox(lines=2, placeholder="اكتب النص الخاص بك هنا...", label="النص المدخل")  
+    gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer"),
+    gr.Textbox(lines=2, placeholder="اكتب النص هنا...", label="Input Text")
 ]
 
 outputs_component = gr.Dataframe(
     headers=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"], 
-    label="النتائج",
+    label="Results",
+    type="pandas"
 )
 
+# Setting up the interface
 iface = Interface(
     fn=compare_tokenizers, 
     inputs=inputs_component, 
     outputs=outputs_component, 
     title="Arabic Tokenizer Arena",
-    live=True,
+    live=True
 )
 
+# Launching the Gradio app
 iface.launch()