Spaces:
Running
Running
| """ | |
| TODO: 繁体、简体 | |
| """ | |
| import os | |
| import json | |
| from collections import Counter | |
| from utils.text_util import is_chinese, has_chinese | |
| from zhon.hanzi import punctuation as zh_punc | |
| CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| zh_tokens = [line.strip() for line in open(os.path.join(CURRENT_DIR, "vocab.jd.txt.v2"), "r", encoding="utf-8") if is_chinese(line.strip())] | |
| def zh_iterator(): | |
| for idx in range(ord(u'\u4e00'), ord(u'\u9fa5')): | |
| yield (chr(idx)) | |
| def get_coding_length(tokenizer, vocab, filter=None): | |
| all_length = [] | |
| for word in vocab: | |
| if len(word) > 1: | |
| continue | |
| if filter is not None and filter(word): | |
| continue | |
| tokens = tokenizer.encode(word) | |
| all_length.append(len(tokens)) | |
| # if len(tokens.ids) > 1: | |
| # if len(tokens) > 3: | |
| # print(word, tokens) | |
| dist_length = Counter(all_length) | |
| mean_length = round(sum(all_length) / len(all_length), 2) | |
| return dist_length, mean_length | |
| def has_zh_char(text): | |
| return any(ch in zh_punc for ch in text) | |
| cache = {} | |
| def iter_vocab(tokenizer, name="", from_cache=True): | |
| if from_cache and name in cache: | |
| return cache[name] | |
| f_out = open(name + "_vocab.zh.jsonl", "w", encoding="utf-8") | |
| zh_token_count = {"total": 0, "中文单字": 0, "中文多字": 0} | |
| all_single_zh_tokens = set() | |
| zh_symbol_count = 0 | |
| for idx in range(tokenizer.vocab_size): | |
| decode_str = tokenizer.decode([idx]) | |
| if has_chinese(decode_str): | |
| # bert词典有 ##开头的 | |
| # byteBPE词典有带空格的 | |
| decode_str = decode_str.strip().replace("#", "") | |
| zh_token_count["total"] += 1 | |
| if len(decode_str) > 1: | |
| zh_token_count["中文多字"] += 1 | |
| f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文多字"}, | |
| ensure_ascii=False) + "\n") | |
| else: | |
| all_single_zh_tokens.add(decode_str) | |
| zh_token_count["中文单字"] += 1 | |
| f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文单字"}, | |
| ensure_ascii=False) + "\n") | |
| elif has_zh_char(decode_str): | |
| zh_symbol_count += 1 | |
| f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文标点"}, | |
| ensure_ascii=False) + "\n") | |
| # | |
| dist_length, mean_length = get_coding_length(tokenizer, zh_tokens, filter=lambda k: not is_chinese(k)) | |
| # TODO: 繁体字，简体字 | |
| zh_token_count["中文单字-去重后"] = len(all_single_zh_tokens) | |
| result = { | |
| "name": name, | |
| "impl": str(tokenizer.__class__), | |
| "vocab_size": tokenizer.vocab_size, | |
| "中文汉字数": zh_token_count, | |
| "中文标点数": zh_symbol_count, | |
| "中文汉字编码长度均值": mean_length, | |
| "中文汉字编码长度分布": json.dumps(dist_length), | |
| } | |
| cache[name] = result | |
| return result | |
| if __name__ == "__main__": | |
| # test_coding_length(jd_vocab_tokens, filter=lambda k: not is_chinese(k)) | |
| # test_coding_length(zh_punc) | |
| # test_coding_length(zh_iterator()) | |
| iter_vocab() | |