Update app.py

app.py

@@ -9,6 +9,9 @@ gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13
 gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
 jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
 arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv2")
+meta_llama_tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
+cohere_command_r_v01_tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-v01")
+cohere_command_r_plus_tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-plus")
 
 # List of available tokenizers and a dictionary to load them
 tokenizer_options = [
@@ -17,7 +20,10 @@ tokenizer_options = [
     "FreedomIntelligence/AceGPT-13B",  
     "FreedomIntelligence/AceGPT-7B",   
     "inception-mbzuai/jais-13b",
-    "aubmindlab/bert-base-arabertv2"   
+    "aubmindlab/bert-base-arabertv2",
+    "meta-llama/Meta-Llama-3-8B",
+    "CohereForAI/c4ai-command-r-v01",
+    "CohereForAI/c4ai-command-r-plus"   
 ]
 
 tokenizers = {
@@ -31,15 +37,21 @@ tokenizers = {
     "FreedomIntelligence/AceGPT-13B": lambda: gpt_13b_tokenizer,
     "FreedomIntelligence/AceGPT-7B": lambda: gpt_7b_tokenizer,
     "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
-    "aubmindlab/bert-base-arabertv2": lambda: arabert_tokenizer
+    "aubmindlab/bert-base-arabertv2": lambda: arabert_tokenizer,
+    "meta-llama/Meta-Llama-3-8B": lambda: meta_llama_tokenizer,
+    "CohereForAI/c4ai-command-r-v01": lambda: cohere_command_r_v01_tokenizer,
+    "CohereForAI/c4ai-command-r-plus": lambda: cohere_command_r_plus_tokenizer
 }
 
 def compare_tokenizers(tokenizer_name, text):
     # Handle the transformer tokenizers separately due to API differences
-    if tokenizer_name in ["FreedomIntelligence/AceGPT-13B", "FreedomIntelligence/AceGPT-7B", "inception-mbzuai/jais-13b", "aubmindlab/bert-base-arabertv2"]:
+    if tokenizer_name in [
+        "FreedomIntelligence/AceGPT-13B", "FreedomIntelligence/AceGPT-7B", 
+        "inception-mbzuai/jais-13b", "aubmindlab/bert-base-arabertv2",
+        "meta-llama/Meta-Llama-3-8B", "CohereForAI/c4ai-command-r-v01", "CohereForAI/c4ai-command-r-plus"
+    ]:
         tokenizer = tokenizers[tokenizer_name]()
         tokens = tokenizer.tokenize(text)
-        tokens_arabic = [token.encode('utf-8').decode('utf-8') for token in tokens]
         encoded_output = tokenizer.encode(text, add_special_tokens=True)
         decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
     else:
@@ -48,9 +60,9 @@ def compare_tokenizers(tokenizer_name, text):
         tokens = tokenizer.tokenize(text)
         encoded_output = tokenizer.encode(text, add_special_tokens=True)
         decoded_text = tokenizer.decode(encoded_output)
-        tokens_arabic = [token.encode('utf-8').decode('utf-8') for token in tokens]
     
     # Prepare the results to be displayed in HTML format
+    tokens_arabic = [token.encode('utf-8').decode('utf-8') if isinstance(token, bytes) else token for token in tokens]
     results_html = f"""
     <div>
         <h3>Tokenizer: {tokenizer_name}</h3>