Spaces:
Running
Running
| # coding=utf-8 | |
| # author: xusong | |
| # time: 2022/8/23 16:06 | |
| """ | |
| plots | |
| table | |
| ## related demo | |
| - [](http://text-processing.com/demo/tokenize/) | |
| - [gpt-tokenizer](https://gpt-tokenizer.dev/) | |
| - [llama-tokenizer-js](https://belladoreai.github.io/llama-tokenizer-js/example-demo/build/) | |
| - [](https://huggingface.co/spaces/Xenova/the-tokenizer-playground) | |
| ## 可视化 | |
| [ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ] | |
| """ | |
| import json | |
| import pandas as pd | |
| import gradio as gr | |
| from vocab import all_tokenizers, load_tokener | |
| # 显示空格：https://blog.csdn.net/liuxiao723846/article/details/118994673 | |
| # 隐藏legend： | |
| css = """ | |
| .space-show {white-space: pre-wrap;} | |
| .cell-wrap {white-space: pre-wrap;} | |
| .category-legend {display: none !important} | |
| .statistics textarea {min-width: min(50px,100%) !important; font-size: 20px !important; font-weight: 600 !important; text-align: center !important; border: none !important;} | |
| .statistics label {text-align: center !important;} | |
| """ | |
| example_text = """Replace this text in the input field to see how tokenization works | |
| 华为智能音箱发布：华为Sound X""" | |
| # llama chatglm_6b gpt_nexo_20b baichuan baichuan_7b | |
| examples = [ | |
| # ["空格测试： 2个空格 8个空格", "llama", "chatglm_6b"], # chatglm 有blank_n, | |
| ["标点测试：，。！？；", "baichuan_7b", "llama"], | |
| ["符号测试：🦙", "baichuan_7b", "llama"], | |
| ["中文测试：🦙", "baichuan_7b", "llama"], | |
| ["数字测试：(10086 + 98) = 100184", "baichuan_7b", "llama"], | |
| ] | |
| def tokenize(text, tokenizer_type, color_num=5): | |
| """ | |
| TODO: cache tokenizer | |
| """ | |
| print(text, tokenizer_type) | |
| pos_tokens = [] | |
| tokenizer = load_tokener(tokenizer_type) | |
| encoding = tokenizer.encode(text) | |
| table = [] | |
| for idx, token_id in enumerate(encoding): | |
| decode_text = tokenizer.decode([token_id]) # 特殊字符解码后会统一变成 �，对应 "\ufffd" | |
| pos_tokens.extend([(decode_text, str(idx % color_num))]) | |
| # token "Byte": # 这是 utf-8编码吧？ | |
| token = tokenizer.convert_ids_to_tokens([token_id])[0] | |
| if isinstance(token, bytes): | |
| try: | |
| token_str = token.decode("utf-8") | |
| except: | |
| token_str = token.decode("utf-8", errors="ignore") | |
| print("decode_error", token, token_str) | |
| token_bytes = token | |
| json_dumps = json.dumps(token_str) | |
| elif isinstance(token, str): | |
| token_str = token | |
| token_bytes = bytes(token_str, "utf-8") | |
| json_dumps = json.dumps(token_str) | |
| else: | |
| return | |
| # ⭐ | |
| table.append( | |
| {"TokenID": token_id, | |
| "Token": token_str, # utf-8解码后的字符串，为什么有些是 <0xE7>，表示什么？比如llama | |
| "Text": decode_text, # | |
| # "Bytes": token_bytes, # bytes类型在gradio前端页面被解码成字符串，比如 b'\xe4\xb8\xad' 仍然显示成 "中"。因此 str(token_bytes) | |
| "Bytes": str(token_bytes), | |
| # "Unicode": json_dumps # unicode, 如果是ascii码，就直接显示。如果不是ascii码，就显示unicode | |
| } | |
| ) | |
| table_df = pd.DataFrame(table) | |
| print(table) | |
| # print(table_df) | |
| return pos_tokens, table_df, len(encoding) | |
| def tokenize_pair(text, tokenizer_type_1, tokenizer_type_2): | |
| pos_tokens_1, table_df_1, token_size_1 = tokenize(text, tokenizer_type_1) | |
| pos_tokens_2, table_df_2, token_size_2 = tokenize(text, tokenizer_type_2) | |
| return pos_tokens_1, table_df_1, token_size_1, pos_tokens_2, table_df_2, token_size_2 | |
| def get_vocab_size(tokenizer_type): | |
| tokenizer = load_tokener(tokenizer_type) | |
| return tokenizer.vocab_size | |
| def test_coding(): | |
| bytes1 = b'\xe4\xb8\xad' | |
| print(bytes1) # b'\xe4\xb8\xad' | |
| with gr.Blocks(css=css) as demo: | |
| gr.HTML("""<h1 align="center">Tokenizer Arena ⚔️</h1>""") | |
| # links: https://www.coderstool.com/utf8-encoding-decoding | |
| # 功能：输入文本，进行分词 | |
| # 分词器：常见的分词器有集中， | |
| # 背景：方便分词、看词粒度、对比 | |
| # | |
| # Byte: 表示分词 | |
| gr.Markdown("## Input Text") | |
| user_input = gr.Textbox( | |
| value=example_text, | |
| label="Input Text", | |
| lines=5, | |
| show_label=False, | |
| ) # placeholder="Enter sentence here..." | |
| # submitBtn = gr.Button("生成回复", variant="primary") | |
| gr.Markdown("## Tokenization") | |
| with gr.Row(): | |
| with gr.Column(scale=6): | |
| with gr.Group(): | |
| tokenizer_type_1 = gr.Dropdown( | |
| all_tokenizers, | |
| value="llama", | |
| label="Tokenizer 1", | |
| ) | |
| with gr.Group(): | |
| """ | |
| <div class="stat"><div class="stat-value">69</div><div class="stat-label">Characters</div></div> | |
| """ | |
| with gr.Row(): | |
| stats_vocab_size_1 = gr.TextArea( | |
| label="VocabSize", | |
| lines=1, | |
| elem_classes="statistics" | |
| ) | |
| stats_token_size_1 = gr.TextArea( | |
| label="Tokens", | |
| lines=1, | |
| elem_classes="statistics" | |
| ) | |
| stats_3 = gr.TextArea( | |
| label="Compress Rate", | |
| lines=1, | |
| elem_classes="statistics" | |
| ) | |
| # https://www.onlinewebfonts.com/icon/418591 | |
| gr.Image("images/VS.svg", scale=1, show_label=False, show_download_button=False, container=False) # height=10, | |
| with gr.Column(scale=6): | |
| with gr.Group(): | |
| tokenizer_type_2 = gr.Dropdown( | |
| all_tokenizers, | |
| value="baichuan_7b", | |
| label="Tokenizer 2", | |
| ) | |
| with gr.Group(): | |
| with gr.Row(): | |
| stats_vocab_size_2 = gr.TextArea( | |
| label="VocabSize", | |
| lines=1, | |
| elem_classes="statistics" | |
| ) | |
| stats_token_size_2 = gr.TextArea( | |
| label="Tokens", | |
| lines=1, | |
| elem_classes="statistics" | |
| ) | |
| stats_6 = gr.TextArea( | |
| label="Compress Rate", | |
| lines=1, | |
| elem_classes="statistics" | |
| ) | |
| # TODO: 图 表 压缩率 | |
| with gr.Row(): | |
| with gr.Column(): | |
| output_text_1 = gr.Highlightedtext( | |
| label="Tokens 1", | |
| show_legend=True, | |
| elem_classes="space-show" | |
| ) | |
| with gr.Column(): | |
| output_text_2 = gr.Highlightedtext( | |
| label="Tokens 2", | |
| show_legend=True, | |
| elem_classes="space-show" | |
| ) | |
| with gr.Row(): | |
| output_table_1 = gr.Dataframe( | |
| # headers=["TokenID", "Byte", "Text"], | |
| # datatype=["str", "str", "str"], | |
| # elem_classes="space-show", # 给整个Dataframe加这个css不起作用，因此直接修改cell-wrap | |
| ) | |
| output_table_2 = gr.Dataframe( | |
| # headers=["TokenID", "Token", "Text"], | |
| # datatype=["str", "str", "str"], | |
| ) | |
| tokenizer_type_1.change(tokenize, [user_input, tokenizer_type_1], [output_text_1, output_table_1, stats_token_size_1]) | |
| tokenizer_type_1.change(get_vocab_size, [tokenizer_type_1], [stats_vocab_size_1]) | |
| user_input.change(tokenize_pair, | |
| [user_input, tokenizer_type_1, tokenizer_type_2], | |
| [output_text_1, output_table_1, stats_token_size_1, output_text_2, output_table_2, stats_token_size_2]) | |
| tokenizer_type_2.change(tokenize, [user_input, tokenizer_type_2], [output_text_2, output_table_2, stats_token_size_2]) | |
| tokenizer_type_2.change(get_vocab_size, [tokenizer_type_2], [stats_vocab_size_2]) | |
| gr.Examples( | |
| examples, | |
| [user_input, tokenizer_type_1, tokenizer_type_2], | |
| [output_text_1, output_table_1, stats_token_size_1, output_text_2, output_table_2, stats_token_size_2], | |
| tokenize_pair, | |
| cache_examples=True, | |
| ) | |
| # submitBtn.click(tokenize, [user_input, tokenizer_type], outputs, | |
| # show_progress=True) | |
| # examples=[ | |
| # ["What a beautiful morning for a walk!"], | |
| # ["It was the best of times, it was the worst of times."], | |
| # ["多个空格 It ss was the best of times, it was the worst of times."], | |
| # ] | |
| if __name__ == "__main__": | |
| demo.launch() | |