from typing import List import gradio as gr from transformers import AutoTokenizer import tiktoken hf_tokenizer_list = [ ("tugstugi/bert-large-mongolian-cased", False), ("tugstugi/bert-large-mongolian-uncased", False), ("bayartsogt/mongolian-roberta-large", True), ("meta-llama/Llama-2-13b-hf", True), ("bigscience/bloom", True), ] openai_tokenizer_list = [ "text-davinci-003", "gpt-4" ] # load tokenizers hf_tokenizers = [ AutoTokenizer.from_pretrained(model_name_or_id, use_fast=use_fast, trust_remote_code=True) for model_name_or_id, use_fast in hf_tokenizer_list ] openai_tokenizers = [ tiktoken.encoding_for_model(name) for name in openai_tokenizer_list ] def do_tokenize(tokenizer: AutoTokenizer, text: str) -> List[str]: return [(tokenizer.decode([token_id]), str(i)) for i, token_id in enumerate(tokenizer.encode(text))] def do_simple_split(text: str): return [(x, str(i)) for i, x in enumerate(text.split())] def do_function(text: str): return ( text, len(text), do_simple_split(text), *[do_tokenize(tokenizer, text) for tokenizer in hf_tokenizers], *[do_tokenize(tokenizer, text) for tokenizer in openai_tokenizers], ) demo = gr.Interface( do_function, [ gr.Text("", placeholder="Мөнгөө тушаачихсаныхаа дараа мэдэгдээрэй") ], [ gr.Text("", label="input"), gr.Number(0, label="Character Count"), gr.HighlightedText("", label="Simple Split"), *[gr.HighlightedText("", label=tokenizer_name) for tokenizer_name, _ in hf_tokenizer_list], *[gr.HighlightedText("", label="openai/" + tokenizer_name) for tokenizer_name in openai_tokenizer_list], ], live=True, allow_flagging="never", title="Real-Time Tokenizer", description=( "**Tokenizers:**\n" + "\n".join( [ f"🤗 [{x}](https://huggingface.co/{x})" for x, _ in hf_tokenizer_list ] + [ f"⏳ [{x}](https://github.com/openai/tiktoken)" for x in openai_tokenizer_list ]) ), ) if __name__ == "__main__": demo.launch()