from transformers import BertTokenizer

tokenizer = BertTokenizer.from_pretrained(
    pretrained_model_name_or_path='bert-base-chinese',
    cache_dir=None,
    force_download=False
)

sents = ['选择珠江花园的原因就是方便。',
'笔记本的键盘确实爽。',
'房间太小。其他的都一般。',
'今天才知道这书还有第6卷，真有点郁闷。',
'机器背面似平被撕了张什么标签，残胶还在。']

# simple clause enc & dec
enc_out = tokenizer.encode(text=sents[0], text_pair=sents[1], truncation=True, padding='max_length', add_special_tokens=True, max_length=30, return_tensors=None)
print(enc_out)
dec_out = tokenizer.decode(enc_out)
print(dec_out)


# add new token into dict
all_dict = tokenizer.get_vocab()
print('dict length:', len(all_dict))
print('月光' in all_dict)