HeshamHaroon's picture
Update app.py
2b73d7a verified
from gradio import Interface
import gradio as gr
import aranizer
from aranizer import (
aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k,
aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
)
from transformers import AutoTokenizer, logging
from huggingface_hub import login
import os
# Retrieve your Hugging Face token from the environment variable
HF_TOKEN = os.getenv('HF_TOKEN')
if HF_TOKEN:
HF_TOKEN = HF_TOKEN.strip() # Remove any leading or trailing whitespace/newlines
login(token=HF_TOKEN)
# Load additional tokenizers from transformers
gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv2")
# Try to load the gated tokenizer
try:
meta_llama_tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
except Exception as e:
meta_llama_tokenizer = None
logging.warning(f"Could not load meta-llama/Meta-Llama-3-8B tokenizer: {e}")
# List of available tokenizers and a dictionary to load them
tokenizer_options = [
"aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k",
"aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k",
"FreedomIntelligence/AceGPT-13B",
"FreedomIntelligence/AceGPT-7B",
"inception-mbzuai/jais-13b",
"aubmindlab/bert-base-arabertv2"
]
if meta_llama_tokenizer:
tokenizer_options.append("meta-llama/Meta-Llama-3-8B")
tokenizers = {
"aranizer_bpe50k": aranizer_bpe50k.get_tokenizer,
"aranizer_bpe64k": aranizer_bpe64k.get_tokenizer,
"aranizer_bpe86k": aranizer_bpe86k.get_tokenizer,
"aranizer_sp32k": aranizer_sp32k.get_tokenizer,
"aranizer_sp50k": aranizer_sp50k.get_tokenizer,
"aranizer_sp64k": aranizer_sp64k.get_tokenizer,
"aranizer_sp86k": aranizer_sp86k.get_tokenizer,
"FreedomIntelligence/AceGPT-13B": lambda: gpt_13b_tokenizer,
"FreedomIntelligence/AceGPT-7B": lambda: gpt_7b_tokenizer,
"inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
"aubmindlab/bert-base-arabertv2": lambda: arabert_tokenizer
}
if meta_llama_tokenizer:
tokenizers["meta-llama/Meta-Llama-3-8B"] = lambda: meta_llama_tokenizer
def compare_tokenizers(tokenizer_index, text):
tokenizer_name = tokenizer_options[tokenizer_index]
tokenizer = tokenizers[tokenizer_name]()
tokens = tokenizer.tokenize(text)
encoded_output = tokenizer.encode(text, add_special_tokens=True)
decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
# Ensure the tokens are properly decoded
tokens_display = [token.encode('utf-8').decode('utf-8') if isinstance(token, bytes) else token for token in tokens]
# Prepare the results to be displayed in HTML format
tokens_html = "".join([
f"<span style='background-color:#eeeeee; color: #333333; padding:4px; margin:2px; border-radius:3px; border:1px solid #cccccc;'>{token}</span>"
for token in tokens_display
])
encoded_html = "".join([
f"<span style='background-color:#e0e0e0; color: #000000; padding:4px; margin:2px; border-radius:3px; border:1px solid #aaaaaa;'>{token}</span>"
for token in encoded_output
])
decoded_html = f"<div style='background-color:#f5f5f5; color: #444444; padding:10px; border-radius:3px; border:1px solid #999999;'>{decoded_text}</div>"
results_html = f"""
<div style='font-family: Arial, sans-serif;'>
<h3 style='color: #2e7d32;'>Tokenizer: {tokenizer_name}</h3>
<p><strong>Tokens:</strong> {tokens_html}</p>
<p><strong>Encoded:</strong> {encoded_html}</p>
<p><strong>Decoded:</strong> {decoded_html}</p>
</div>
"""
return results_html
# Define the Gradio interface components with a dropdown for model selection
inputs_component = [
gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer", type="index"),
gr.Textbox(lines=2, placeholder="اكتب النص هنا...", label="Input Text")
]
outputs_component = gr.HTML(label="Results")
# Setting up the interface
iface = Interface(
fn=compare_tokenizers,
inputs=inputs_component,
outputs=outputs_component,
title="Arabic Tokenizer Arena",
live=True
)
# Launching the Gradio app
iface.launch()