Package: bert-pretty
bert_pretty is a text encoder and result decoder# -*- coding:utf-8 -*-'''bert input_instance encode and result decodehttps://github.com/ssbuild/bert_pretty.git'''importnumpyasnp#FullTokenizer is official and you can use your tokenization .frombert_prettyimportFullTokenizer,\text_feature,\text_feature_char_level,\text_feature_word_level,\text_feature_char_level_input_ids_mask,\text_feature_word_level_input_ids_mask,\text_feature_char_level_input_ids_segment,\text_feature_word_level_input_ids_segment,\seqs_padding,rematchfrombert_pretty.nerimportload_label_bioes,load_label_bio,load_labelsasner_load_labelsfrombert_pretty.nerimportner_crf_decoding,\ner_pointer_decoding,\ner_pointer_decoding_with_mapping,\ner_pointer_double_decoding,ner_pointer_double_decoding_with_mappingfrombert_pretty.clsimportcls_softmax_decoding,cls_sigmoid_decoding,load_labelsascls_load_labelstokenizer=FullTokenizer(vocab_file=r'F:\pretrain\chinese_L-12_H-768_A-12\vocab.txt',do_lower_case=True)text_list=["你是谁123aa\ta嘂a","嘂adasd"]deftest():maxlen=512do_lower_case=tokenizer.basic_tokenizer.do_lower_caseinputs=[['[CLS]']+tokenizer.tokenize(text)[:maxlen-2]+['[SEP]']fortextintext_list]mapping=[rematch(text,tokens,do_lower_case)fortext,tokensinzip(text_list,inputs)]inputs=[tokenizer.convert_tokens_to_ids(input)forinputininputs]input_mask=[[1]*len(input)forinputininputs]input_segment=[[0]*len(input)forinputininputs]input_ids=seqs_padding(inputs)input_mask=seqs_padding(input_mask)input_segment=seqs_padding(input_segment)input_ids=np.asarray(input_ids,dtype=np.int32)input_mask=np.asarray(input_mask,dtype=np.int32)input_segment=np.asarray(input_segment,dtype=np.int32)print('input_ids\n',input_ids)print('mapping\n',mapping)print('input_mask\n',input_mask)print('input_segment\n',input_segment)print('\n\n')deftest_charlevel():do_lower_case=tokenizer.basic_tokenizer.do_lower_casemaxlen=512ifdo_lower_case:inputs=[['[CLS]']+tokenizer.tokenize(text.lower())[:maxlen-2]+['[SEP]']fortextintext_list]else:inputs=[['[CLS]']+tokenizer.tokenize(text)[:maxlen-2]+['[SEP]']fortextintext_list]inputs=[tokenizer.convert_tokens_to_ids(input)forinputininputs]input_mask=[[1]*len(input)forinputininputs]input_segment=[[0]*len(input)forinputininputs]input_ids=seqs_padding(inputs)input_mask=seqs_padding(input_mask)input_segment=seqs_padding(input_segment)input_ids=np.asarray(input_ids,dtype=np.int32)input_mask=np.asarray(input_mask,dtype=np.int32)input_segment=np.asarray(input_segment,dtype=np.int32)print('input_ids\n',input_ids)print('input_mask\n',input_mask)print('input_segment\n',input_segment)print('\n\n')# labels = ['标签1','标签2']# print(cls.load_labels(labels))## print(ner.load_label_bio(labels))'''# def ner_crf_decoding(batch_text, id2label, batch_logits, trans=None,batch_mapping=None,with_dict=True):ner crf decode 解析crf序列  or 解析 已经解析过的crf序列batch_text input_instance list ,id2label 标签 list or dictbatch_logits 为bert 预测结果 logits_all (batch,seq_len,num_tags) or (batch,seq_len)trans 是否启用trans预测 , 2Dbatch_mapping 映射序列''''''def ner_pointer_decoding(batch_text, id2label, batch_logits, threshold=1e-8,coordinates_minus=False,with_dict=True)batch_text text list ,id2label 标签 list or dictbatch_logits (batch,num_labels,seq_len,seq_len)threshold 阈值coordinates_minus''''''def ner_pointer_decoding_with_mapping(batch_text, id2label, batch_logits, batch_mapping,threshold=1e-8,coordinates_minus=False,with_dict=True)batch_text text list ,id2label 标签 list or dictbatch_logits (batch,num_labels,seq_len,seq_len)threshold 阈值coordinates_minus''''''cls_softmax_decoding(batch_text, id2label, batch_logits,threshold=None)batch_text 文本list ,id2label 标签 list or dictbatch_logits (batch,num_classes)threshold 阈值''''''cls_sigmoid_decoding(batch_text, id2label, batch_logits,threshold=0.5)batch_text 文本list ,id2label 标签 list or dictbatch_logits (batch,num_classes)threshold 阈值'''deftest_cls_decode():num_label=3np.random.seed(123)batch_logits=np.random.rand(2,num_label)result=cls_softmax_decoding(text_list,['标签1','标签2','标签3'],batch_logits,threshold=None)print(result)batch_logits=np.random.rand(2,num_label)print(batch_logits)result=cls_sigmoid_decoding(text_list,['标签1','标签2','标签3'],batch_logits,threshold=0.5)print(result)if__name__=='__main__':test()test_charlevel()test_cls_decode()