File size: 5,032 Bytes
751936e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
# coding=utf-8
# author: xusong
# time: 2022/8/23 16:06

"""

plots

table

## related demo
http://text-processing.com/demo/tokenize/

## 可视化

[ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ]
"""

import json
import pandas as pd
import gradio as gr

from vocab import all_tokenizers, load_tokener

# 显示空格:https://blog.csdn.net/liuxiao723846/article/details/118994673
# 隐藏legend:
css = """
.space-show {white-space: pre-wrap;}
.cell-wrap {white-space: pre-wrap;}
.category-legend {display: none !important}
"""

example_text = """中文测试:华为智能音箱发布:华为Sound X。維基百科由非營利組織──維基媒體基金會負責維持
标点测试:,。!?;
空格测试:  2个空格        8个空格
数字测试:(10086 + 98) = 100184"""


def tokenize(text, tokenizer_type):
    print(text, tokenizer_type)
    pos_tokens = []
    tokenizer = load_tokener(tokenizer_type)
    encoding = tokenizer.encode(text)

    table = []

    for idx, token_id in enumerate(encoding):
        decode_text = tokenizer.decode([token_id])  # 特殊字符解码后会统一变成 �,对应 "\ufffd"
        pos_tokens.extend([(decode_text, str(idx % 3))])

        # token  "Byte":  # 这是 utf-8编码吧?
        token = tokenizer.convert_ids_to_tokens([token_id])[0]
        if isinstance(token, bytes):
            token_str = token.decode("utf-8")
            token_bytes = token
            json_dumps = json.dumps(token_str)
        elif isinstance(token, str):
            token_str = token
            token_bytes = bytes(token_str, "utf-8")
            json_dumps = json.dumps(token_str)
        else:
            return

        table.append(
            {"TokenID": token_id,
             "Token": token_str,  # utf-8解码后的字符串,为什么有些是 <0xE7>,表示什么?比如llama
             "Text": decode_text,  #
             # "Bytes": token_bytes,  # bytes类型在gradio前端页面被解码成字符串,比如   b'\xe4\xb8\xad' 仍然显示成 "中"。因此 str(token_bytes)
             "Bytes": str(token_bytes),
             # "Unicode": json_dumps  # unicode, 如果是ascii码,就直接显示。如果不是ascii码,就显示unicode
             }
        )

    table_df = pd.DataFrame(table)
    print(table)
    print(table_df)

    return pos_tokens, table_df


def test_coding():
    bytes1 = b'\xe4\xb8\xad'
    print(bytes1)  # b'\xe4\xb8\xad'


with gr.Blocks(css=css) as demo:
    gr.HTML("""<h1 align="center">Tokenizer Arena</h1>""")
    # links: https://www.coderstool.com/utf8-encoding-decoding
    #


    user_input = gr.Textbox(
        value=example_text,
        lines=5
    )  # placeholder="Enter sentence here..."

    # submitBtn = gr.Button("生成回复", variant="primary")

    # TODO: 图 表 压缩率
    # llama chatglm gpt_nexo_20b baichuan  baichuan_7b
    with gr.Row():
        with gr.Column():
            tokenizer_type_1 = gr.Dropdown(
                all_tokenizers, value="llama", label="tokenizer"
            )
            token_counter_1 = None  # 计数器
            output_text_1 = gr.Highlightedtext(
                label="Tokenization",
                show_legend=True,
                elem_classes="space-show"
            )

            output_table_1 = gr.Dataframe(
                headers=["TokenID", "Byte", "Text"],
                datatype=["str", "str", "str"],
                #elem_classes="space-show",   # 给整个Dataframe加这个css不起作用,因此直接修改cell-wrap
            )

        with gr.Column():
            tokenizer_type_2 = gr.Dropdown(
                all_tokenizers, value="baichuan_7b", label="tokenizer"
            )
            token_counter_2 = None  # 计数器
            output_text_2 = gr.Highlightedtext(
                label="Tokenization",
                show_legend=True,
                elem_classes="space-show"
            )

            output_table_2 = gr.Dataframe(
                headers=["TokenID", "Token", "Text"],
                datatype=["str", "str", "str"],
            )

        user_input.change(tokenize,
                          [user_input, tokenizer_type_1],
                          [output_text_1, output_table_1])
        tokenizer_type_1.change(tokenize, [user_input, tokenizer_type_1], [output_text_1, output_table_1])

        user_input.change(tokenize,
                          [user_input, tokenizer_type_2],
                          [output_text_2, output_table_2])
        tokenizer_type_2.change(tokenize, [user_input, tokenizer_type_2], [output_text_2, output_table_2])

    # submitBtn.click(tokenize, [user_input, tokenizer_type], outputs,
    #                 show_progress=True)

    # examples=[
    #     ["What a beautiful morning for a walk!"],
    #     ["It was the best of times, it was the worst of times."],
    #     ["多个空格    It  ss  was the best of times, it was the worst of times."],
    # ]

if __name__ == "__main__":
    demo.launch()