Spaces:
Sleeping
Sleeping
File size: 2,309 Bytes
98985f3 bbc0512 f3369dd a17b6c0 72a73bd a17b6c0 9f66134 a17b6c0 72a73bd a17b6c0 9f66134 a17b6c0 98985f3 2789d18 b084026 72a73bd e363f01 9f66134 e363f01 72a73bd e363f01 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
from transformers import AutoTokenizer
import gradio as gr
def tokenize(input_text):
llama_tokens = len(
llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
)
llama3_tokens = len(
llama3_tokenizer(input_text, add_special_tokens=True)["input_ids"]
)
mistral_tokens = len(
mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"]
)
gpt2_tokens = len(gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
gpt_neox_tokens = len(
gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]
)
falcon_tokens = len(
falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]
)
phi2_tokens = len(phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
t5_tokens = len(t5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
gemma_tokens = len(gemma_tokenizer(input_text, add_special_tokens=True)["input_ids"])
results = {
"LLaMa": llama_tokens,
"LLaMa-3": llama_tokens,
"Mistral": mistral_tokens,
"GPT-2/GPT-J": gpt2_tokens,
"GPT-NeoX": gpt_neox_tokens,
"Falcon": falcon_tokens,
"Phi-2": phi2_tokens,
"T5": t5_tokens,
"Gemma": gemma_tokens
}
# Sort the results in descending order based on token length
sorted_results = sorted(results.items(), key=lambda x: x[1], reverse=True)
return "\n".join([f"{model}: {tokens}" for model, tokens in sorted_results])
if __name__ == "__main__":
llama_tokenizer = AutoTokenizer.from_pretrained("TheBloke/Llama-2-7B-fp16")
llama3_tokenizer = AutoTokenizer.from_pretraines("unsloth/llama-3-8b")
mistral_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
t5_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl")
gemma_tokenizer = AutoTokenizer.from_pretrained("alpindale/gemma-2b")
iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=9), outputs="text")
iface.launch()
|