utils/zh_util.py

"""
TODO: 繁体、简体
"""
import os
import json
from collections import Counter
from utils.text_util import is_chinese, has_chinese
from zhon.hanzi import punctuation as zh_punc

CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))

zh_tokens = [line.strip() for line in open(os.path.join(CURRENT_DIR, "vocab.jd.txt.v2"), "r", encoding="utf-8") if is_chinese(line.strip())]

def zh_iterator():
    for idx in range(ord(u'\u4e00'), ord(u'\u9fa5')):
        yield (chr(idx))


def get_coding_length(tokenizer, vocab, filter=None):
    all_length = []
    for word in vocab:
        if len(word) > 1:
            continue
        if filter is not None and filter(word):
            continue
        tokens = tokenizer.encode(word)
        all_length.append(len(tokens))
        # if len(tokens.ids) > 1:
        # if len(tokens) > 3:
        #     print(word, tokens)

    dist_length = Counter(all_length)
    mean_length = round(sum(all_length) / len(all_length), 2)
    return dist_length, mean_length


def has_zh_char(text):
    return any(ch in zh_punc for ch in text)

cache = {}

def iter_vocab(tokenizer, name="", from_cache=True):
    if from_cache and name in cache:
        return cache[name]

    f_out = open(name + "_vocab.zh.jsonl", "w", encoding="utf-8")
    zh_token_count = {"total": 0, "中文单字": 0, "中文多字": 0}
    all_single_zh_tokens = set()
    zh_symbol_count = 0
    for idx in range(tokenizer.vocab_size):
        decode_str = tokenizer.decode([idx])
        if has_chinese(decode_str):
            # bert词典有 ##开头的
            # byteBPE词典有带空格的
            decode_str = decode_str.strip().replace("#", "")
            zh_token_count["total"] += 1
            if len(decode_str) > 1:
                zh_token_count["中文多字"] += 1
                f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文多字"},
                                       ensure_ascii=False) + "\n")
            else:
                all_single_zh_tokens.add(decode_str)
                zh_token_count["中文单字"] += 1
                f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文单字"},
                                       ensure_ascii=False) + "\n")

        elif has_zh_char(decode_str):
            zh_symbol_count += 1
            f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文标点"},
                                   ensure_ascii=False) + "\n")

    #

    dist_length, mean_length = get_coding_length(tokenizer, zh_tokens, filter=lambda k: not is_chinese(k))

    # TODO: 繁体字，简体字
    zh_token_count["中文单字-去重后"] = len(all_single_zh_tokens)

    result = {
        "name": name,
        "impl": str(tokenizer.__class__),
        "vocab_size": tokenizer.vocab_size,
        "中文汉字数": zh_token_count,
        "中文标点数": zh_symbol_count,
        "中文汉字编码长度均值": mean_length,
        "中文汉字编码长度分布": json.dumps(dist_length),
    }
    cache[name] = result
    return result


if __name__ == "__main__":
    # test_coding_length(jd_vocab_tokens, filter=lambda k: not is_chinese(k))
    # test_coding_length(zh_punc)
    # test_coding_length(zh_iterator())
    iter_vocab()