update

app.py

@@ -16,7 +16,7 @@
 - 词典支持下载
 - 中文字词统计，是否要包括 _ G 等字符
 - baichuan的单字数量怎么两万多个？
-- gpt4
+- OOV
 
 
 plots
@@ -35,7 +35,6 @@ table
 """
 
 import gradio as gr
-
 from vocab import all_tokenizers
 from util import *
 
@@ -48,6 +47,10 @@ examples = [
     ["数字测试：(10086 + 98) = 100184", "baichuan_7b", "llama"],
 ]
 
+# jieba.enable_parallel()  # flask中没办法parallel
+
+
+
 
 def example_fn(example_idx):
     return examples[example_idx]
@@ -138,7 +141,7 @@ with gr.Blocks(css="style.css") as demo:
                         # )
         # https://www.onlinewebfonts.com/icon/418591
         gr.Image("images/VS.svg", scale=1, show_label=False,
-                 show_download_button=True, container=False,
+                 show_download_button=False, container=False,
                  show_share_button=False)
         with gr.Column(scale=6):
             with gr.Group():

util.py

@@ -1,19 +1,16 @@
-
-
 import gradio as gr
 import json
 import pandas as pd
 from vocab import load_tokener
 from utils.zh_util import iter_vocab
-
-
+from utils.log_util import logger
 
 
 def tokenize(text, tokenizer_type, color_num=5, update=True):
     """
     TODO: cache tokenizer
     """
-    print(f"入参：tokenize, {text}, {tokenizer_type}")
+    logger.info("[param]:" + json.dumps({"text": text, "tokenizer_type": tokenizer_type}, ensure_ascii=False))
     pos_tokens = []
     tokenizer = load_tokener(tokenizer_type)
     encoding = tokenizer.encode(text)
@@ -31,7 +28,9 @@ def tokenize(text, tokenizer_type, color_num=5, update=True):
                 token_str = token.decode("utf-8")
             except:
                 token_str = token.decode("utf-8", errors="ignore")
-                print("decode_error", tokenizer_type, token, token_str)
+                logger.info("[decode_error]: " + json.dumps(
+                    {"tokenizer_type": tokenizer_type, "token": str(token), "token_str": token_str},
+                    ensure_ascii=False))
 
             token_bytes = token
             json_dumps = json.dumps(token_str)
@@ -54,7 +53,7 @@ def tokenize(text, tokenizer_type, color_num=5, update=True):
         )
 
     table_df = pd.DataFrame(table)
-    print(f"Tokenization[{tokenizer_type}]: {table}")
+    logger.info(f"[Tokens {tokenizer_type}]: {table[:2]}")
     # print(table_df)
 
     if update:
@@ -82,16 +81,14 @@ def get_overlap_token_size(tokenizer_type_1, tokenizer_type_2):
     vocab2 = tokenizer2.get_vocab()
     overlap_tokens = vocab1.keys() & vocab2.keys()
     overlap_token_size = len(overlap_tokens)
-    print(f"OverlapTokens: {tokenizer_type_1}, {tokenizer_type_2} {list(overlap_tokens)[:10]}")
+    logger.info(f"[OverlapTokens of {tokenizer_type_1} {tokenizer_type_2}]: {list(overlap_tokens)[:10]}")
     return overlap_token_size, overlap_token_size
 
 
-
-
 def test_coding():
     bytes1 = b'\xe4\xb8\xad'
     print(bytes1)  # b'\xe4\xb8\xad'
 
 
 if __name__ == "__main__":
-    print(basic_count("internlm_chat_7b"))
+    print(basic_count("internlm_chat_7b"))

utils/log_util.py

@@ -0,0 +1,10 @@
+
+import logging
+
+logging.basicConfig(
+    format='%(asctime)s - %(filename)s - %(levelname)s - %(process)d - %(thread)d - %(message)s',
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)