update compress rate

app.py

@@ -39,6 +39,7 @@ import gradio as gr
 from vocab import all_tokenizers
 from util import *
 from examples import example_fn, example_types
+from utils.compress_rate_util import common_units, common_corpuses
 
 get_window_url_params = """
     function(url_params) {
@@ -75,16 +76,17 @@ with gr.Blocks(css="css/style.css", title="Tokenizer Arena") as demo:
 
     # compress rate setting
     with gr.Accordion("Compress Rate Setting", open=True):
-        gr.Markdown("Please select corpus and unit of compress rate, get more details at [github](https://github.com/xu-song/tokenizer-arena/). ")
+        gr.Markdown(
+            "Please select corpus and unit of compress rate, get more details at [github](https://github.com/xu-song/tokenizer-arena/). ")
         with gr.Row():
             compress_rate_corpus = gr.CheckboxGroup(
-                ["cc100-en", "cc100-zh-Hans", "cc100-es"],  # , "code"
+                common_corpuses,  # , "code"
                 value=["cc100-en", "cc100-zh-Hans"],
                 label="corpus",
                 # info=""
             )
             compress_rate_unit = gr.Radio(
-                ["b_tokens/g_bytes", "g_bytes/b_tokens", "t_tokens/t_bytes", "t_bytes/t_tokens", "n_chars/n_tokens"],
+                common_units,
                 value="b_tokens/g_bytes",
                 label="unit",
             )
@@ -194,12 +196,10 @@ with gr.Blocks(css="css/style.css", title="Tokenizer Arena") as demo:
         output_table_1 = gr.Dataframe()
         output_table_2 = gr.Dataframe()
 
-
     # setting
     # compress_rate_unit.change(compress_rate_unit_change, [compress_rate_unit],
     #                             [stats_compress_rate_1, stats_compress_rate_2])
 
-
     tokenizer_type_1.change(tokenize, [user_input, tokenizer_type_1],
                             [output_text_1, output_table_1])
     tokenizer_type_1.change(basic_count, [tokenizer_type_1], [stats_vocab_size_1, stats_zh_token_size_1])
@@ -218,15 +218,22 @@ with gr.Blocks(css="css/style.css", title="Tokenizer Arena") as demo:
     tokenizer_type_2.change(basic_count, [tokenizer_type_2], [stats_vocab_size_2, stats_zh_token_size_2])
     tokenizer_type_2.change(get_overlap_token_size, [tokenizer_type_1, tokenizer_type_2],
                             [stats_overlap_token_size_1, stats_overlap_token_size_2])
-    tokenizer_type_2.change(get_compress_rate, [tokenizer_type_2, compress_rate_corpus, compress_rate_unit],
-                            [stats_compress_rate_2])
-
-
-    compress_rate_unit.change(get_compress_rate, [tokenizer_type_1, compress_rate_corpus, compress_rate_unit],
-                            [stats_compress_rate_1])
-    compress_rate_unit.change(get_compress_rate, [tokenizer_type_2, compress_rate_corpus, compress_rate_unit],
+    tokenizer_type_2.change(get_compress_rate,
+                            [tokenizer_type_2, compress_rate_corpus, compress_rate_unit],
                             [stats_compress_rate_2])
 
+    compress_rate_unit.change(get_compress_rate,
+                              [tokenizer_type_1, compress_rate_corpus, compress_rate_unit],
+                              [stats_compress_rate_1])
+    compress_rate_unit.change(get_compress_rate,
+                              [tokenizer_type_2, compress_rate_corpus, compress_rate_unit],
+                              [stats_compress_rate_2])
+    compress_rate_corpus.change(get_compress_rate,
+                                [tokenizer_type_1, compress_rate_corpus, compress_rate_unit],
+                                [stats_compress_rate_1])
+    compress_rate_corpus.change(get_compress_rate,
+                                [tokenizer_type_2, compress_rate_corpus, compress_rate_unit],
+                                [stats_compress_rate_2])
 
     dropdown_examples.change(
         example_fn,

utils/compress_rate_util.py

@@ -18,6 +18,10 @@ from typing import List
 
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 
+common_units = ["g_bytes/b_tokens", "b_tokens/g_bytes", "t_bytes/t_tokens", "t_tokens/t_bytes", "n_chars/n_tokens", ]
+common_corpuses = ["cc100-en", "cc100-zh-Hans", "cc100-es"]
+# code: https://huggingface.co/datasets/codeparrot/github-code-clean  python java c sql html
+# math:
 
 def get_n_bytes_of_string(string_text):
     n_bytes = len(string_text.encode("utf-8"))
@@ -55,14 +59,12 @@ def unit_convertor(stat, unit):
     return round(value, 2)
 
 
-all_units = ["g_bytes/b_tokens", "b_tokens/g_bytes", "t_bytes/t_tokens", "t_tokens/t_bytes", "n_chars/n_tokens", ]
-
 
 def pprint(stats):
     table = []
     for tokenizer_name, stat in stats.items():
         columns = {"tokenizer": tokenizer_name, "vocab_size": stat["vocab_size"]}
-        for unit in all_units:
+        for unit in common_units:
             if unit not in stat:
                 columns[unit] = unit_convertor(stat, unit)
             else:
@@ -146,7 +148,7 @@ def main():
         corpuses = [sys.argv[2]]
     else:
         tokenizers = all_tokenizers
-        corpuses = ["en", "zh-Hans"]
+        corpuses = common_corpuses
 
     stats = {}
     for lang in corpuses: