vocab/glm_chinese/test_glm.py

"""

num_text_tokens: 50000
num_tokens: 50009
command_tokens: ['<|endoftext|>', '[SEP]', '[CLS]', '[MASK]', '[UNK]', '<|startofpiece|>', '<|endofpiece|>', '[gMASK]', '[sMASK]']

{'pad': 50000, 'eos': 50000, 'sep': 50001, 'ENC': 50002, 'MASK': 50003, 'unk': 50004, 'sop': 50006, 'eop': 50007, 'gMASK': 50007, 'sMASK': 50008}

padded vocab (size: 50009) with 39 dummy tokens (new size: 50048)

## 注意区分
12758 ▁凯
45012 凯

这两个语义一样，id却完全不同，是bug吧。

## ss
"""

from tokenization import make_tokenizer

add_sentinel_token = 0
tokenizer = make_tokenizer("ChineseSPTokenizer", None, "tokenizer.model", "50048",
                           None, add_block_symbols=True, cache_dir="cache",
                           add_sentinel_token=add_sentinel_token, add_task_mask=True,
                           add_decoder_mask=False,
                           fix_command_token=False)

text = "凯凯凯旋门位于意大利米兰市古城堡旁。1807年为纪念[MASK]而建，门高25米，顶上矗立两武士青铜古兵车铸像。"
text = " 凯凯凯旋门位于意大利米兰市古城堡旁。1807年为纪念[MASK]而建，门高25米，顶上矗立两武士青铜古兵车铸像。"


def bad_casse():
    tokens = tokenizer.text_tokenizer.tokenize(text)
    tokens_id = tokenizer.text_tokenizer.encode(text)
    decode_text = tokenizer.text_tokenizer.decode(tokens_id)


def good_case():
    tokens_id = tokenizer.EncodeAsIds(text).tokenization
    # tokens_id = [12758, 44916, 43696, 801, 1435, 5140, 1876, 2669, 50003, 13702, 43359, 44095, 48151, 44321, 44537,
    #              6687, 34043, 44291, 43361, 43362, 26124, 43377, 2366, 50003, 1934, 7004, 44771, 44362, 43795, 5837,
    #              2790, 43359, 43437, 180, 2426, 43388, 14682, 7866, 12845, 43361]
    #
    # tokens_id = [50002, 27318, 43359,    17, 43785, 44326,   277, 14821, 44460, 44773,
    #      43389,    23,  1344, 43383, 43398,   348,   128,   403,   513,  4258,
    #      44460, 44773,  8471, 43689, 43389,   320,   439, 43383,   682,  2222,
    #      14676, 43383, 50007]
    tokens_id = [9737, 50007]  # 圣[gMASK]
    tokens_id = [14643, 50007]  # ▁这款[gMASK]
    tokens_id = [12014, 50007, 44435, 6703, 33751, 992, 16791, 43359, 5101, 44612]



    decode_text = tokenizer.DecodeIds(tokens_id)
    print(decode_text)
    for idx in tokens_id:
        print(idx, tokenizer.IdToToken(idx))
    # decode_text = tokenizer.DecodeIds(tokens_id)


# bad_casse()
good_case()