Spaces:
Running
Running
| # coding=utf-8 | |
| # author: xusong | |
| # time: 2022/8/23 16:06 | |
| """ | |
| ## TODO: | |
| - i18 国际化 https://blog.csdn.net/qq_26212731/article/details/78457198 request.header中也有language | |
| - iter_vocab 的 warmup | |
| - 开关 | |
| - add_special_token 开关 | |
| - theme 开关 light/dark | |
| - token_id/tokens/bytes 开关 | |
| - 中文字词统计，是否要包括 _ G 等字符 | |
| - 评测 | |
| - OOV评测 | |
| - 通过 javascript 添加 hover_text | |
| - 英文 utf-8编码 | |
| - 词典支持下载，借用image下载的标签， | |
| - baichuan的单字数量怎么两万多个？ | |
| - qwen: ValueError: Unclosed image token | |
| - 路径修改为全path meta-llama/Llama-2-13b-hf | |
| plots | |
| table | |
| ## related demo | |
| - [](http://text-processing.com/demo/tokenize/) | |
| - [gpt-tokenizer](https://gpt-tokenizer.dev/) | |
| - [llama-tokenizer-js](https://belladoreai.github.io/llama-tokenizer-js/example-demo/build/) | |
| - [](https://huggingface.co/spaces/Xenova/the-tokenizer-playground) | |
| ## 可视化 | |
| [ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ] | |
| """ | |
| import gradio as gr | |
| from vocab import all_tokenizers | |
| from util import * | |
| from examples import example_fn, example_types | |
| get_window_url_params = """ | |
| function(url_params) { | |
| const params = new URLSearchParams(window.location.search); | |
| url_params = JSON.stringify(Object.fromEntries(params)); | |
| return url_params; | |
| } | |
| """ | |
| with gr.Blocks() as demo: | |
| # links: https://www.coderstool.com/utf8-encoding-decoding | |
| # 功能：输入文本，进行分词 | |
| # 分词器：常见的分词器有集中， | |
| # 背景：方便分词、看词粒度、对比 | |
| with gr.Row(): | |
| gr.Markdown("## Input Text") | |
| dropdown_examples = gr.Dropdown( | |
| example_types, | |
| value="Examples", | |
| type="index", | |
| show_label=False, | |
| container=False, | |
| scale=0, | |
| elem_classes="example-style" | |
| ) | |
| user_input = gr.Textbox( | |
| # value=default_user_input, | |
| label="Input Text", | |
| lines=5, | |
| show_label=False, | |
| ) | |
| gr.Markdown("## Tokenization") | |
| # compress rate setting TODO: 将 这个模块调整到下面 | |
| # with gr.Accordion("Compress Rate Setting", open=True): | |
| # gr.Markdown( | |
| # "Please select corpus and unit of compress rate, get more details at [github](https://github.com/xu-song/tokenizer-arena/). ") | |
| # with gr.Row(): | |
| # compress_rate_corpus = gr.CheckboxGroup( | |
| # common_corpuses, # , "code" | |
| # value=["cc100-en", "cc100-zh-Hans"], | |
| # label="corpus", | |
| # # info="" | |
| # ) | |
| # compress_rate_unit = gr.Radio( | |
| # common_units, | |
| # value="b_tokens/g_bytes", | |
| # label="unit", | |
| # ) | |
| # TODO: Token Setting | |
| # with gr.Accordion("Token Filter Setting", open=False): | |
| # gr.Markdown( | |
| # "Get total number of tokens which contain the following character)") | |
| # gr.Radio( | |
| # ["zh-Hans", "", "number", "space"], | |
| # value="zh", | |
| # ) | |
| with gr.Row(): | |
| with gr.Column(scale=6): | |
| with gr.Group(): | |
| tokenizer_name_1 = gr.Dropdown( | |
| all_tokenizers, | |
| label="Tokenizer 1", | |
| ) | |
| with gr.Group(): | |
| with gr.Row(): | |
| stats_vocab_size_1 = gr.TextArea( | |
| label="Vocab Size", | |
| lines=1, | |
| elem_classes="statistics" | |
| ) | |
| stats_zh_token_size_1 = gr.TextArea( | |
| label="ZH char/word", | |
| lines=1, | |
| elem_classes="statistics", | |
| ) | |
| # stats_compress_rate_1 = gr.TextArea( | |
| # label="Compress Rate", | |
| # lines=1, | |
| # elem_classes="statistics", | |
| # ) | |
| stats_overlap_token_size_1 = gr.TextArea( | |
| # value=default_stats_overlap_token_size, | |
| label="Overlap Tokens", | |
| lines=1, | |
| elem_classes="statistics" | |
| ) | |
| # stats_3 = gr.TextArea( | |
| # label="Compress Rate", | |
| # lines=1, | |
| # elem_classes="statistics" | |
| # ) | |
| # https://www.onlinewebfonts.com/icon/418591 | |
| gr.Image("images/VS.svg", scale=1, show_label=False, | |
| show_download_button=False, container=False, | |
| show_share_button=False) | |
| with gr.Column(scale=6): | |
| with gr.Group(): | |
| tokenizer_name_2 = gr.Dropdown( | |
| all_tokenizers, | |
| label="Tokenizer 2", | |
| ) | |
| with gr.Group(): | |
| with gr.Row(): | |
| stats_vocab_size_2 = gr.TextArea( | |
| label="VocabSize", | |
| lines=1, | |
| elem_classes="statistics" | |
| ) | |
| stats_zh_token_size_2 = gr.TextArea( | |
| label="ZH char/word", # 中文字/词 | |
| lines=1, | |
| elem_classes="statistics", | |
| ) | |
| # stats_compress_rate_2 = gr.TextArea( | |
| # label="Compress Rate", | |
| # lines=1, | |
| # elem_classes="statistics" | |
| # ) | |
| stats_filtered_token_2 = gr.TextArea( | |
| label="filtered tokens", | |
| lines=1, | |
| elem_classes="statistics", | |
| visible=False | |
| ) | |
| stats_overlap_token_size_2 = gr.TextArea( | |
| label="Overlap Tokens", | |
| lines=1, | |
| elem_classes="statistics" | |
| ) | |
| # TODO: 图 表 压缩率 | |
| with gr.Row(): | |
| # dynamic change label | |
| with gr.Column(): | |
| output_text_1 = gr.Highlightedtext( | |
| show_legend=True, | |
| elem_classes="space-show" | |
| ) | |
| with gr.Column(): | |
| output_text_2 = gr.Highlightedtext( | |
| show_legend=True, | |
| elem_classes="space-show" | |
| ) | |
| with gr.Row(): | |
| output_table_1 = gr.Dataframe() | |
| output_table_2 = gr.Dataframe() | |
| # setting | |
| # compress_rate_unit.change(compress_rate_unit_change, [compress_rate_unit], | |
| # [stats_compress_rate_1, stats_compress_rate_2]) | |
| tokenizer_name_1.change(tokenize, [user_input, tokenizer_name_1], | |
| [output_text_1, output_table_1]) | |
| tokenizer_name_1.change(basic_count, [tokenizer_name_1], [stats_vocab_size_1, stats_zh_token_size_1]) | |
| tokenizer_name_1.change(get_overlap_token_size, [tokenizer_name_1, tokenizer_name_2], | |
| [stats_overlap_token_size_1, stats_overlap_token_size_2]) | |
| # tokenizer_type_1.change(get_compress_rate, [tokenizer_type_1, compress_rate_corpus, compress_rate_unit], | |
| # [stats_compress_rate_1]) | |
| # TODO: every=3 | |
| user_input.change(tokenize_pair, | |
| [user_input, tokenizer_name_1, tokenizer_name_2], | |
| [output_text_1, output_table_1, output_text_2, output_table_2]) # , pass_request=1 | |
| tokenizer_name_2.change(tokenize, [user_input, tokenizer_name_2], | |
| [output_text_2, output_table_2]) | |
| tokenizer_name_2.change(basic_count, [tokenizer_name_2], [stats_vocab_size_2, stats_zh_token_size_2]) | |
| tokenizer_name_2.change(get_overlap_token_size, [tokenizer_name_1, tokenizer_name_2], | |
| [stats_overlap_token_size_1, stats_overlap_token_size_2]) | |
| # tokenizer_type_2.change(get_compress_rate, | |
| # [tokenizer_type_2, compress_rate_corpus, compress_rate_unit], | |
| # [stats_compress_rate_2]) | |
| # | |
| # compress_rate_unit.change(get_compress_rate, | |
| # [tokenizer_type_1, compress_rate_corpus, compress_rate_unit], | |
| # [stats_compress_rate_1]) | |
| # compress_rate_unit.change(get_compress_rate, | |
| # [tokenizer_type_2, compress_rate_corpus, compress_rate_unit], | |
| # [stats_compress_rate_2]) | |
| # compress_rate_corpus.change(get_compress_rate, | |
| # [tokenizer_type_1, compress_rate_corpus, compress_rate_unit], | |
| # [stats_compress_rate_1]) | |
| # compress_rate_corpus.change(get_compress_rate, | |
| # [tokenizer_type_2, compress_rate_corpus, compress_rate_unit], | |
| # [stats_compress_rate_2]) | |
| dropdown_examples.change( | |
| example_fn, | |
| dropdown_examples, | |
| [user_input, tokenizer_name_1, tokenizer_name_2] | |
| ) | |
| demo.load(js=open("js/onload.js", "r", encoding="utf-8").read()) | |
| demo.load( | |
| fn=on_load, | |
| inputs=[user_input], # 这里只需要传个空object即可。 | |
| outputs=[user_input, tokenizer_name_1, tokenizer_name_2], | |
| js=get_window_url_params | |
| ) | |
| if __name__ == "__main__": | |
| # demo.queue(max_size=20).launch() | |
| demo.launch() | |
| # demo.launch(share=True) | |