tokenizer-arena / vocab /gpt_neox_chinese_v2 /build_tokenizer_chinese.py
xu-song's picture
update
751936e
raw
history blame
1.53 kB
"""
merge 是干嘛的?
## 结果
共merge 4357 个 token
"""
import json
from tokenizers import Tokenizer
oov_tokens = [line.strip().split("\t")[0] for line in open("../gpt_neox_chinese_v1/oov.txt", "r", encoding="utf-8")]
def load_base_tokenizer():
old_vocab_path = "../gpt_neox_chinese_v1/20B_tokenizer_chinese.json"
data = json.load(open(old_vocab_path, "r", encoding="utf-8"))
tokenizer = Tokenizer.from_file(old_vocab_path)
print("vocab_size with added_tokens:", )
return data, tokenizer
data, base_tokenizer = load_base_tokenizer()
vocab = data["model"]["vocab"]
merges = data["model"]["merges"]
vocab_size = base_tokenizer.get_vocab_size(with_added_tokens=True)
"""
方式一:原有的added_tokens保持id不变。方式二:原有的added_tokens进行id移位。
以下采用方式一。
"""
new_added_tokens = set()
for word in oov_tokens:
if len(word) > 1 or word in new_added_tokens:
continue
encoding = base_tokenizer.encode(word)
# if len(encoding.ids) > 1:
if len(encoding.ids) == 2: # 3个的,怎么处理?
tokens = [base_tokenizer.id_to_token(token_id) for token_id in encoding.ids]
print("merging", word, json.dumps(tokens))
vocab["".join(tokens)] = vocab_size
vocab_size += 1
merges.append(" ".join(tokens))
new_added_tokens.add(word)
print("共merge %d 个 token" % (len(new_added_tokens)))
f_out = open("20B_tokenizer_chinese.v2.json", "w", encoding="utf-8")
json.dump(data, f_out, indent=2)