app.py

# coding=utf-8
# author: xusong
# time: 2022/8/23 16:06

"""
## TODO:
- http get方式获取参数，(高优先级)
- i18 国际化  https://blog.csdn.net/qq_26212731/article/details/78457198   request.header中也有language
- iter_vocab 的 warmup
- add_special_token 开关
- theme 开关 light/dark
- token_id/tokens/bytes 开关
- 通过 javascript 添加 hover_text
- 给方法 + 缓存，避免重复调用
- 英文 utf-8编码
- 词典支持下载
- 中文字词统计，是否要包括 _ G 等字符
- baichuan的单字数量怎么两万多个？
- OOV
- feedback位置
- gpt4, gpt3.5 的overlap tokens 有问题。
- qwen:  ValueError: Unclosed image token

plots

table

## related demo
- [](http://text-processing.com/demo/tokenize/)
- [gpt-tokenizer](https://gpt-tokenizer.dev/)
- [llama-tokenizer-js](https://belladoreai.github.io/llama-tokenizer-js/example-demo/build/)
- [](https://huggingface.co/spaces/Xenova/the-tokenizer-playground)

## 可视化

[ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ]
"""

import gradio as gr
from vocab import all_tokenizers
from util import *

# llama chatglm_6b gpt_nexo_20b baichuan  baichuan_7b
examples_zh = [
    ["空格测试：  2个空格        8个空格", "llama", "chatglm_6b"],  # chatglm 有blank_n,
    ["标点测试：，。！？；", "baichuan_7b", "llama"],
    ["符号测试：🦙❤❥웃유♋☮✊☏☢☚✔☑♚▢♪✈✞÷↑↓▤▥⊙■□▣▽¿─│♥❣▬▫☿Ⓐ ✋✉☣☤", "baichuan_7b", "llama"],
    ["数字测试：(10086 + 98) = 100184", "baichuan_7b", "llama"],
    ["中文简体：宽带，繁体：樂來", "baichuan_7b", "llama"],
]

examples = [
    ["spaces:  2spaces        8spaces", "llama", "chatglm_6b"],  # chatglm 有blank_n,
    ["punctuations: ,./?\"，。！？；", "baichuan_7b", "llama"],
    ["symbols: 🦙❤❥웃유♋☮✊☏☢☚✔☑♚▢♪✈✞÷↑↓▤▥⊙■□▣▽¿─│♥❣▬▫☿Ⓐ ✋✉☣☤", "baichuan_7b", "llama"],
    ["digits: (10086 + 98) = 100184", "baichuan_7b", "llama"],
]


# jieba.enable_parallel()  # flask中没办法parallel




def example_fn(example_idx):
    return examples[example_idx]


"""Replace this text in the input field to see how tokenization works


"""

default_user_input = """Replace this text in the input field to see how tokenization works
华为发布Mate60手机
ラグビーワールドカップ2023フランス"""
default_tokenizer_type_1 = "llama"
default_tokenizer_type_2 = "internlm_chat_7b"
default_stats_vocab_size_1, default_stats_zh_token_size_1 = basic_count(default_tokenizer_type_1)
default_stats_vocab_size_2, default_stats_zh_token_size_2 = basic_count(default_tokenizer_type_2)
default_stats_overlap_token_size = get_overlap_token_size(default_tokenizer_type_1, default_tokenizer_type_2)[0]
default_output_text_1, default_output_table_1, default_output_len_1 = tokenize(default_user_input, default_tokenizer_type_1, update=False)
default_output_text_2, default_output_table_2, default_output_len_2 = tokenize(default_user_input, default_tokenizer_type_2, update=False)

with gr.Blocks(css="style.css") as demo:
    gr.HTML("""<h1 align="center">Tokenizer Arena ⚔️</h1>""")
    # links: https://www.coderstool.com/utf8-encoding-decoding
    # 功能：输入文本，进行分词
    # 分词器：常见的分词器有集中，
    # 背景：方便分词、看词粒度、对比
    #
    # Byte: 表示分词

    with gr.Row():
        gr.Markdown("## Input Text")
        dropdown_examples = gr.Dropdown(
            # ["空格测试", "标点测试", "符号测试", "数字测试"],
            ["spaces", "punctuations", "symbols", "digits"],
            value="Examples",
            type="index",
            show_label=False,
            container=False,
            scale=0,
            elem_classes="example-style"
        )

    user_input = gr.Textbox(
        value=default_user_input,
        label="Input Text",
        lines=5,
        show_label=False,
    )  # placeholder="Enter sentence here..."
    # gr.Examples(
    #     examples,
    #     None,
    # )

    gr.Markdown("## Tokenization")

    with gr.Row():
        with gr.Column(scale=6):
            with gr.Group():
                tokenizer_type_1 = gr.Dropdown(
                    all_tokenizers,
                    value=default_tokenizer_type_1,
                    label="Tokenizer 1",
                )
                with gr.Group():
                    """
                    <div class="stat"><div class="stat-value">69</div><div class="stat-label">Characters</div></div>
                    """
                    with gr.Row():
                        stats_vocab_size_1 = gr.TextArea(
                            value=default_stats_vocab_size_1,
                            label="VocabSize",
                            lines=1,
                            elem_classes="statistics"
                        )
                        stats_zh_token_size_1 = gr.TextArea(
                            value=default_stats_zh_token_size_1,
                            label="ZH char/word",
                            lines=1,
                            elem_classes="statistics"
                        )
                        stats_overlap_token_size_1 = gr.TextArea(
                            value=default_stats_overlap_token_size,
                            label="Overlap Tokens",
                            lines=1,
                            elem_classes="statistics"
                        )
                        # stats_3 = gr.TextArea(
                        #     label="Compress Rate",
                        #     lines=1,
                        #     elem_classes="statistics"
                        # )
        # https://www.onlinewebfonts.com/icon/418591
        gr.Image("images/VS.svg", scale=1, show_label=False,
                 show_download_button=False, container=False,
                 show_share_button=False)
        with gr.Column(scale=6):
            with gr.Group():
                tokenizer_type_2 = gr.Dropdown(
                    all_tokenizers,
                    value=default_tokenizer_type_2,
                    label="Tokenizer 2",
                )
                with gr.Group():
                    with gr.Row():
                        stats_vocab_size_2 = gr.TextArea(
                            value=default_stats_vocab_size_2,
                            label="VocabSize",
                            lines=1,
                            elem_classes="statistics"
                        )
                        stats_zh_token_size_2 = gr.TextArea(
                            value=default_stats_zh_token_size_2,
                            label="ZH char/word",  # 中文字/词
                            lines=1,
                            elem_classes="statistics"
                        )
                        # stats_6 = gr.TextArea(
                        #     label="Compress Rate",
                        #     lines=1,
                        #     elem_classes="statistics"
                        # )
                        stats_overlap_token_size_2 = gr.TextArea(
                            value=default_stats_overlap_token_size,
                            label="Overlap Tokens",
                            lines=1,
                            elem_classes="statistics"
                        )

    # TODO: 图 表 压缩率
    with gr.Row():
        with gr.Column():
            output_text_1 = gr.Highlightedtext(
                value=default_output_text_1,
                label=f"Tokens: {default_output_len_1}",
                show_legend=True,
                elem_classes="space-show"
            )
        with gr.Column():
            output_text_2 = gr.Highlightedtext(
                value=default_output_text_2,
                label=f"Tokens: {default_output_len_2}",
                show_legend=True,
                elem_classes="space-show"
            )

    with gr.Row():
        output_table_1 = gr.Dataframe(
            value=default_output_table_1,
            headers=["TokenID", "Byte", "Text"],
            datatype=["str", "str", "str"],
            # elem_classes="space-show",   # 给整个Dataframe加这个css不起作用，因此直接修改cell-wrap
        )
        output_table_2 = gr.Dataframe(
            value=default_output_table_2,
            headers=["TokenID", "Token", "Text"],
            datatype=["str", "str", "str"],
        )

    tokenizer_type_1.change(tokenize, [user_input, tokenizer_type_1],
                            [output_text_1, output_table_1])
    # 下面两个好像可以合并
    tokenizer_type_1.change(basic_count, [tokenizer_type_1], [stats_vocab_size_1, stats_zh_token_size_1])
    tokenizer_type_1.change(get_overlap_token_size, [tokenizer_type_1, tokenizer_type_2],
                            [stats_overlap_token_size_1, stats_overlap_token_size_2])

    user_input.change(tokenize_pair,
                      [user_input, tokenizer_type_1, tokenizer_type_2],
                      [output_text_1, output_table_1, output_text_2, output_table_2]) # , pass_request=1

    tokenizer_type_2.change(tokenize, [user_input, tokenizer_type_2],
                            [output_text_2, output_table_2])
    tokenizer_type_2.change(basic_count, [tokenizer_type_2], [stats_vocab_size_2, stats_zh_token_size_2])
    tokenizer_type_2.change(get_overlap_token_size, [tokenizer_type_1, tokenizer_type_2],
                            [stats_overlap_token_size_1, stats_overlap_token_size_2])

    dropdown_examples.change(
        example_fn,
        dropdown_examples,
        [user_input, tokenizer_type_1, tokenizer_type_2]
    )

    # start up 初始化
    # user_input.update(user_input.value + "___")

if __name__ == "__main__":
    demo.queue(max_size=20).launch()
    # demo.launch()