Spaces:
Running
Running
""" | |
TODO: 繁体、简体 | |
""" | |
import os | |
import json | |
from collections import Counter | |
from utils.text_util import is_chinese, has_chinese | |
from zhon.hanzi import punctuation as zh_punc | |
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
zh_tokens = [line.strip() for line in open(os.path.join(CURRENT_DIR, "vocab.jd.txt.v2"), "r", encoding="utf-8") if is_chinese(line.strip())] | |
def zh_iterator(): | |
for idx in range(ord(u'\u4e00'), ord(u'\u9fa5')): | |
yield (chr(idx)) | |
def get_coding_length(tokenizer, vocab, filter=None): | |
all_length = [] | |
for word in vocab: | |
if len(word) > 1: | |
continue | |
if filter is not None and filter(word): | |
continue | |
tokens = tokenizer.encode(word) | |
all_length.append(len(tokens)) | |
# if len(tokens.ids) > 1: | |
# if len(tokens) > 3: | |
# print(word, tokens) | |
dist_length = Counter(all_length) | |
mean_length = round(sum(all_length) / len(all_length), 2) | |
return dist_length, mean_length | |
def has_zh_char(text): | |
return any(ch in zh_punc for ch in text) | |
cache = {} | |
def iter_vocab(tokenizer, name="", from_cache=True): | |
if from_cache and name in cache: | |
return cache[name] | |
f_out = open(name + "_vocab.zh.jsonl", "w", encoding="utf-8") | |
zh_token_count = {"total": 0, "中文单字": 0, "中文多字": 0} | |
all_single_zh_tokens = set() | |
zh_symbol_count = 0 | |
for idx in range(tokenizer.vocab_size): | |
decode_str = tokenizer.decode([idx]) | |
if has_chinese(decode_str): | |
# bert词典有 ##开头的 | |
# byteBPE词典有带空格的 | |
decode_str = decode_str.strip().replace("#", "") # TODO, 按类型 | |
zh_token_count["total"] += 1 | |
if len(decode_str) > 1: | |
zh_token_count["中文多字"] += 1 | |
f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文多字"}, | |
ensure_ascii=False) + "\n") | |
else: | |
all_single_zh_tokens.add(decode_str) | |
zh_token_count["中文单字"] += 1 | |
f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文单字"}, | |
ensure_ascii=False) + "\n") | |
elif has_zh_char(decode_str): | |
zh_symbol_count += 1 | |
f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文标点"}, | |
ensure_ascii=False) + "\n") | |
# | |
dist_length, mean_length = get_coding_length(tokenizer, zh_tokens, filter=lambda k: not is_chinese(k)) | |
# TODO: 繁体字,简体字 | |
zh_token_count["中文单字-去重后"] = len(all_single_zh_tokens) | |
result = { | |
"name": name, | |
"impl": str(tokenizer.__class__), | |
"vocab_size": tokenizer.vocab_size, | |
"中文汉字数": zh_token_count, | |
"中文标点数": zh_symbol_count, | |
"中文汉字编码长度均值": mean_length, | |
"中文汉字编码长度分布": json.dumps(dist_length), | |
} | |
cache[name] = result | |
return result | |
if __name__ == "__main__": | |
# test_coding_length(jd_vocab_tokens, filter=lambda k: not is_chinese(k)) | |
# test_coding_length(zh_punc) | |
# test_coding_length(zh_iterator()) | |
from vocab.gpt_35_turbo import tokenizer | |
iter_vocab(tokenizer) | |