launch with queue

app.py

@@ -21,4 +21,4 @@ demo = TabbedInterface(
 demo.load(js=open("js/onload.js", "r", encoding="utf-8").read())
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.queue(max_size=1024, default_concurrency_limit=80).launch()

compression_util.py

@@ -133,15 +133,14 @@ cache = {}
 
 
 def tokenize_corpus(
-        tokenizer_name: str,  # 可以免加载tokenizer直接出结果
+        tokenizer_name: str,
         corpuses: List[str],
         cache_dir: str = "stats"
 ) -> dict:
     """
-    这个要独立的cache，因为速度慢。
-    :param tokenizer_config: 可以不加载就
+    :param tokenizer_name:
     :param corpuses:
-    :param cache_path:
+    :param cache_dir:
     :return:
     """
 
@@ -157,7 +156,6 @@ def tokenize_corpus(
 
     def _tokenize(tokenizer, datasets, detail_path=None):
         """
-        export_diff: true | false
         :param tokenizer:
         :param datasets:
         :param detail_path:

vocab.py

@@ -182,8 +182,8 @@ class TokenizerConfig:
         return hash(self.name_or_path)
 
 
-# format: , description, hf_path, tokenizer_class/type, comments, Organization
 # TODO: append link and description to the end of dropdown button.
+# Add tokenizer_class/type, comments
 _all_tokenizer_config = [
     ##### bert 系列
     TokenizerConfig("google-bert/bert-base-cased", impl=TokenizerImpl.BertTokenizer, org="Google",
@@ -229,7 +229,9 @@ _all_tokenizer_config = [
     TokenizerConfig("EleutherAI/gpt-neox-20b", impl=TokenizerImpl.SentencePiece, org="EleutherAI"),  # 5万
     TokenizerConfig("cyberagent/open-calm-7b", impl=TokenizerImpl.SentencePiece, org="CyberAgent"),  # GPTNeoXTokenizer
     TokenizerConfig("abeja/gpt-neox-japanese-2.7b", impl=TokenizerImpl.SentencePiece, org="ABEJA"),
-    TokenizerConfig("Qwen/Qwen1.5-14B-Chat", name_display="Qwen/Qwen1.5", impl=TokenizerImpl.SentencePiece, org="Alibaba"),  # 15万，速度有点慢
+    TokenizerConfig("Qwen/Qwen1.5-14B", impl=TokenizerImpl.SentencePiece, org="Alibaba"),  # 15万，速度有点慢
+    TokenizerConfig("Qwen/Qwen1.5-110B ", impl=TokenizerImpl.SentencePiece, org="Alibaba"),
+    TokenizerConfig("Qwen/Qwen1.5-1.8B ", impl=TokenizerImpl.SentencePiece, org="Alibaba"),
     TokenizerConfig("HuggingFaceH4/starchat-alpha", impl=TokenizerImpl.SentencePiece, org="-"),
 
     ####### google/sentencepiece tokenizer:
@@ -385,7 +387,7 @@ class TokenizerFactory:
 
     def get_tokenizer(self, tokenizer_name: str):
         """
-        :param tokenizer_config:
+        :param tokenizer_name:
         :return:
         """
         tokenizer_config = self.get_tokenizer_config(tokenizer_name)
@@ -407,7 +409,7 @@ class TokenizerFactory:
         self.tokenizer_cache[tokenizer_config] = tokenizer
         return tokenizer
 
-    def get_name_with_hyperlink(self, tokenizer_name):
+    def get_name_with_hyperlink(self, tokenizer_name: str):
         def model_hyperlink(link, model_name):
             model_name = model_name
             return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'