Update app.py

app.py

@@ -8,6 +8,7 @@ from transformers import AutoTokenizer
 gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
 gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
 jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
+arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv2")
 
 # List of available tokenizers and a dictionary to load them
 tokenizer_options = [
@@ -15,7 +16,8 @@ tokenizer_options = [
     "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k",
     "FreedomIntelligence/AceGPT-13B",  
     "FreedomIntelligence/AceGPT-7B",   
-    "inception-mbzuai/jais-13b"        
+    "inception-mbzuai/jais-13b",
+    "aubmindlab/bert-base-arabertv2"   
 ]
 
 tokenizers = {
@@ -28,12 +30,13 @@ tokenizers = {
     "aranizer_sp86k": aranizer_sp86k.get_tokenizer,
     "FreedomIntelligence/AceGPT-13B": lambda: gpt_13b_tokenizer,
     "FreedomIntelligence/AceGPT-7B": lambda: gpt_7b_tokenizer,
-    "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer  
+    "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
+    "aubmindlab/bert-base-arabertv2": lambda: arabert_tokenizer
 }
 
 def compare_tokenizers(tokenizer_name, text):
     # Handle the transformer tokenizers separately due to API differences
-    if tokenizer_name in ["FreedomIntelligence/AceGPT-13B", "FreedomIntelligence/AceGPT-7B", "inception-mbzuai/jais-13b"]:
+    if tokenizer_name in ["FreedomIntelligence/AceGPT-13B", "FreedomIntelligence/AceGPT-7B", "inception-mbzuai/jais-13b", "aubmindlab/bert-base-arabertv2"]:
         tokenizer = tokenizers[tokenizer_name]()
         tokens = tokenizer.tokenize(text)
         tokens_arabic = [token.encode('utf-8').decode('utf-8') for token in tokens]
@@ -48,7 +51,7 @@ def compare_tokenizers(tokenizer_name, text):
         tokens_arabic = [token.encode('utf-8').decode('utf-8') for token in tokens]
     
     # Prepare the results to be displayed
-    results = [(tokenizer_name, tokens_arabic, encoded_output, decoded_text)]
+    results = [(tokenizer_name, tokens_arabic, encoded_output.tolist(), decoded_text)]
     return results
 
 # Define the Gradio interface components with a dropdown for model selection
@@ -73,4 +76,4 @@ iface = Interface(
 )
 
 # Launching the Gradio app
-iface.launch()
+iface.launch()