Upload 23 files

app.py

@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Nov 22 09:54:41 2022
+
+@author: luol2
+"""
+
+
+
+import streamlit as st
+import argparse
+from src.nn_model import bioTag_CNN,bioTag_BERT,bioTag_Bioformer
+from src.dic_ner import dic_ont
+from src.tagging_text import bioTag
+import os
+import time
+import json
+import sys
+
+
+st.set_page_config(
+    page_title="PhenoTagger",
+    page_icon=":shark:",
+    #  layout="wide",
+    initial_sidebar_state="expanded",
+    menu_items={
+        'Get Help': 'https://www.extremelycoolapp.com/help',
+        'Report a bug': "https://www.extremelycoolapp.com/bug",
+        'About': "# This is a header. This is an *extremely* cool app!"
+    }
+)
+st.title('PhenoTagger Demo')
+
+
+
+
+    
+
+
+# with st.spinner('Model is being loaded..'):
+  
+#   print('load model done!')
+ 
+
+
+    
+with st.form(key="my_form"):
+        
+    @st.cache(allow_output_mutation=True)
+    def load_model():
+        ontfiles={'dic_file':'./dict_new/noabb_lemma.dic',
+                  'word_hpo_file':'./dict_new/word_id_map.json',
+                  'hpo_word_file':'./dict_new/id_word_map.json'}
+        
+        # if para_set['model_type']=='cnn':
+        #     vocabfiles={'w2vfile':'../vocab/bio_embedding_intrinsic.d200',   
+        #                 'charfile':'../vocab/char.vocab',
+        #                 'labelfile':'../dict_new/lable.vocab',
+        #                 'posfile':'../vocab/pos.vocab'}
+        #     modelfile='../models/cnn_p5n5_b128_95_hponew1.h5'
+        
+        # elif para_set['model_type']=='bioformer':
+        vocabfiles={'labelfile':'./dict_new/lable.vocab',
+                    'config_path':'./vocab/bioformer-cased-v1.0/bert_config.json',
+                    'checkpoint_path':'./vocab/bioformer-cased-v1.0/bioformer-cased-v1.0-model.ckpt-2000000',
+                    'vocab_path':'./vocab/bioformer-cased-v1.0/vocab.txt'}
+        modelfile='./vocab/bioformer_p5n5_b64_1e-5_95_hponew3.h5'
+        # else:
+        #     print('Model type is wrong, please select cnn or bioformer.')
+        #     sys.exit()
+        
+    
+        biotag_dic=dic_ont(ontfiles)    
+    
+        # if para_set['model_type']=='cnn':
+        #     nn_model=bioTag_CNN(vocabfiles)
+        #     nn_model.load_model(modelfile)
+        # elif para_set['model_type']=='bioformer':
+        nn_model=bioTag_Bioformer(vocabfiles)
+        session=nn_model.load_model(modelfile)
+        test_tag='1232'
+        return nn_model,biotag_dic,test_tag,session
+
+
+    #hyper-parameter
+    st.sidebar.header("Hyperparameter Settings")
+    sbform = st.sidebar.form("Hyper-paramiters")
+    # para_model=sbform.selectbox('Model', ['cnn', 'bioformer'])
+    para_overlap=sbform.selectbox('Return overlapping concepts', ['True', 'False'])
+    para_abbr=sbform.selectbox('Identify abbreviations', ['True', 'False'])
+    para_threshold = sbform.slider('Threshold:', min_value=0.5, max_value=0.95, value=0.95, step=0.05)
+    sbform.form_submit_button("Setting")
+    
+    st.write('parameters:', para_overlap,para_abbr,para_threshold)
+    nn_model,biotag_dic,test_tag,session=load_model()
+    
+    
+    input_text = st.text_area(
+        "Paste your text below (max 500 words)",
+        height=510,
+    )
+
+    MAX_WORDS = 500
+    import re
+    res = len(re.findall(r"\w+", input_text))
+    if res > MAX_WORDS:
+        st.warning(
+            "⚠️ Your text contains "
+            + str(res)
+            + " words."
+            + " Only the first 500 words will be reviewed. Stay tuned as increased allowance is coming! 😊"
+        )
+
+        input_text = input_text[:MAX_WORDS]
+
+    submit_button = st.form_submit_button(label="✨ Get me the data!")
+    
+    if para_overlap=='True':
+        para_overlap=True
+    else:
+        para_overlap=False
+    if para_abbr=='True':
+        para_abbr=True
+    else:
+        para_abbr=False
+    para_set={
+              #model_type':para_model, # cnn or bioformer
+              'onlyLongest':para_overlap, # False: return overlap concepts, True only longgest
+              'abbrRecog':para_abbr,# False: don't identify abbr, True: identify abbr
+              'ML_Threshold':para_threshold,# the Threshold of deep learning model
+              }
+    
+    
+
+if not submit_button:
+    st.stop()
+        
+
+st.markdown(f"""**Results:**\n""")
+# print('dic...........:',biotag_dic.keys())
+print('........:',test_tag)
+print('........!!!!!!:',input_text)
+print('...input:',input_text)
+tag_result=bioTag(session,input_text,biotag_dic,nn_model,onlyLongest=para_set['onlyLongest'], abbrRecog=para_set['abbrRecog'],Threshold=para_set['ML_Threshold'])
+for ele in tag_result:
+    start = ele[0]
+    last = ele[1]
+    mention = input_text[int(ele[0]):int(ele[1])]
+    type='Phenotype'
+    id=ele[2]
+    score=ele[3]
+    output=start+"\t"+last+"\t"+mention+"\t"+id+'\t'+score+"\n"
+    st.info(output)
+    

requirements.txt

@@ -0,0 +1,6 @@
+TensorFlow==2.3.0
+Keras==2.4.3
+nltk==3.5
+keras-bert==0.86.0
+bioc==1.3.4
+streamlit==1.13.0

src/abbre_resolution.py

@@ -0,0 +1,434 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Aug 11 16:52:40 2020
+
+@author: luol2
+"""
+
+import logging
+import regex
+import sys
+import io
+
+"""
+A Python 3 refactoring of Vincent Van Asch's Python 2 code at
+
+http://www.cnts.ua.ac.be/~vincent/scripts/abbreviations.py
+
+Based on
+
+A Simple Algorithm for Identifying Abbreviations Definitions in Biomedical Text
+A. Schwartz and M. Hearst
+Biocomputing, 2003, pp 451-462.
+
+"""
+
+logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
+log = logging.getLogger('Abbre')
+
+
+class Candidate(str):
+    def __init__(self, value):
+        super().__init__()
+        self.start = 0
+        self.stop = 0
+
+    def set_position(self, start, stop):
+        self.start = start
+        self.stop = stop
+
+
+def yield_lines_from_file(file_path):
+    with open(file_path, 'rb') as f:
+        for line in f:
+            try:
+                line = line.decode('utf-8')
+            except UnicodeDecodeError:
+                line = line.decode('latin-1').encode('utf-8').decode('utf-8')
+            line = line.strip()
+            yield line
+        f.close()
+
+
+def yield_lines_from_doc(doc_text):
+    for line in doc_text.split("\n"):
+        yield line.strip()
+
+
+def best_candidates(sentence):
+    """
+    :param sentence: line read from input file
+    :return: a Candidate iterator
+    """
+
+    if '(' in sentence:
+        # Check some things first
+        if sentence.count('(') != sentence.count(')'):
+            raise ValueError("Unbalanced parentheses: {}".format(sentence))
+
+        if sentence.find('(') > sentence.find(')'):
+            raise ValueError("First parentheses is right: {}".format(sentence))
+
+        closeindex = -1
+        while 1:
+            # Look for open parenthesis
+            openindex = sentence.find('(', closeindex + 1)
+
+            if openindex == -1: break
+
+            # Look for closing parentheses
+            closeindex = openindex + 1
+            open = 1
+            skip = False
+            while open:
+                try:
+                    char = sentence[closeindex]
+                except IndexError:
+                    # We found an opening bracket but no associated closing bracket
+                    # Skip the opening bracket
+                    skip = True
+                    break
+                if char == '(':
+                    open += 1
+                elif char in [')', ';', ':']:
+                    open -= 1
+                closeindex += 1
+
+            if skip:
+                closeindex = openindex + 1
+                continue
+
+            # Output if conditions are met
+            start = openindex + 1
+            stop = closeindex - 1
+            candidate = sentence[start:stop]
+
+            # Take into account whitespace that should be removed
+            start = start + len(candidate) - len(candidate.lstrip())
+            stop = stop - len(candidate) + len(candidate.rstrip())
+            candidate = sentence[start:stop]
+
+            if conditions(candidate):
+                new_candidate = Candidate(candidate)
+                new_candidate.set_position(start, stop)
+                yield new_candidate
+
+
+def conditions(candidate):
+    """
+    Based on Schwartz&Hearst
+
+    2 <= len(str) <= 10
+    len(tokens) <= 2
+    re.search('\p{L}', str)
+    str[0].isalnum()
+
+    and extra:
+    if it matches (\p{L}\.?\s?){2,}
+    it is a good candidate.
+
+    :param candidate: candidate abbreviation
+    :return: True if this is a good candidate
+    """
+    viable = True
+    if regex.match('(\p{L}\.?\s?){2,}', candidate.lstrip()):
+        viable = True
+    if len(candidate) < 2 or len(candidate) > 10:
+        viable = False
+    if len(candidate.split()) > 2:
+        viable = False
+    if not regex.search('\p{L}', candidate):
+        viable = False
+    if not candidate[0].isalnum():
+        viable = False
+
+    return viable
+
+
+def get_definition(candidate, sentence):
+    """
+    Takes a candidate and a sentence and returns the definition candidate.
+
+    The definintion candidate is the set of tokens (in front of the candidate)
+    that starts with a token starting with the first character of the candidate
+
+    :param candidate: candidate abbreviation
+    :param sentence: current sentence (single line from input file)
+    :return: candidate definition for this abbreviation
+    """
+    # Take the tokens in front of the candidate
+    tokens = regex.split(r'[\s\-]+', sentence[:candidate.start - 2].lower())
+    #print(tokens)
+    # the char that we are looking for
+    key = candidate[0].lower()
+
+    # Count the number of tokens that start with the same character as the candidate
+#     print(tokens)
+    firstchars = [t[0] for t in tokens]
+#     print(firstchars)
+    definition_freq = firstchars.count(key)
+    candidate_freq = candidate.lower().count(key)
+
+    # Look for the list of tokens in front of candidate that
+    # have a sufficient number of tokens starting with key
+    if candidate_freq <= definition_freq:
+        # we should at least have a good number of starts
+        count = 0
+        start = 0
+        startindex = len(firstchars) - 1
+        
+        while count < candidate_freq:
+            if abs(start) > len(firstchars):
+                raise ValueError("candiate {} not found".format(candidate))
+            start -= 1
+            # Look up key in the definition
+            try:
+                startindex = firstchars.index(key, len(firstchars) + start)
+            except ValueError:
+                pass
+
+            # Count the number of keys in definition
+            count = firstchars[startindex:].count(key)
+        
+        # We found enough keys in the definition so return the definition as a definition candidate
+        start = len(' '.join(tokens[:startindex]))
+        stop = candidate.start - 1
+        candidate = sentence[start:stop]
+
+        # Remove whitespace
+        start = start + len(candidate) - len(candidate.lstrip())
+        stop = stop - len(candidate) + len(candidate.rstrip())
+        candidate = sentence[start:stop]
+
+        new_candidate = Candidate(candidate)
+        new_candidate.set_position(start, stop)
+        #print('new_candidate:')
+        #print(new_candidate,start,stop)
+        return new_candidate
+
+    else:
+        raise ValueError('There are less keys in the tokens in front of candidate than there are in the candidate')
+
+
+def select_definition(definition, abbrev):
+    """
+    Takes a definition candidate and an abbreviation candidate
+    and returns True if the chars in the abbreviation occur in the definition
+
+    Based on
+    A simple algorithm for identifying abbreviation definitions in biomedical texts, Schwartz & Hearst
+    :param definition: candidate definition
+    :param abbrev: candidate abbreviation
+    :return:
+    """
+
+
+    if len(definition) < len(abbrev):
+        raise ValueError('Abbreviation is longer than definition')
+
+    if abbrev in definition.split():
+        raise ValueError('Abbreviation is full word of definition')
+
+    sindex = -1
+    lindex = -1
+
+    while 1:
+        try:
+            longchar = definition[lindex].lower()
+        except IndexError:
+            raise
+
+        shortchar = abbrev[sindex].lower()
+
+        if not shortchar.isalnum():
+            sindex -= 1
+
+        if sindex == -1 * len(abbrev):
+            if shortchar == longchar:
+                if lindex == -1 * len(definition) or not definition[lindex - 1].isalnum():
+                    break
+                else:
+                    lindex -= 1
+            else:
+                lindex -= 1
+                if lindex == -1 * (len(definition) + 1):
+                    raise ValueError("definition {} was not found in {}".format(abbrev, definition))
+
+        else:
+            if shortchar == longchar:
+                sindex -= 1
+                lindex -= 1
+            else:
+                lindex -= 1
+#     print('lindex:',lindex,len(definition),definition[lindex:len(definition)])
+    new_candidate = Candidate(definition[lindex:len(definition)])
+    new_candidate.set_position(definition.start+lindex+len(definition), definition.stop)
+    definition = new_candidate
+
+    tokens = len(definition.split())
+    length = len(abbrev)
+
+    if tokens > min([length + 5, length * 2]):
+        raise ValueError("did not meet min(|A|+5, |A|*2) constraint")
+
+    # Do not return definitions that contain unbalanced parentheses
+    if definition.count('(') != definition.count(')'):
+        raise ValueError("Unbalanced parentheses not allowed in a definition")
+#     print('select:')
+#     print(definition,definition.start, definition.stop)
+    new_definition_dict={'definition':definition,'start':definition.start,'stop':definition.stop}
+    return new_definition_dict
+
+
+def extract_abbreviation_definition_pairs(file_path=None, doc_text=None):
+    abbrev_map = []
+    omit = 0
+    written = 0
+    if file_path:
+        sentence_iterator = enumerate(yield_lines_from_file(file_path))
+    elif doc_text:
+        sentence_iterator = enumerate(yield_lines_from_doc(doc_text))
+    else:
+        return abbrev_map
+
+    for i, sentence in sentence_iterator:
+        #print(sentence)
+        try:
+            for candidate in best_candidates(sentence):
+                #print(candidate)
+                try:
+                    #print('begin get definition')
+                    definition = get_definition(candidate, sentence)
+                    #print('get_definition:')
+                    #print(definition)
+                    
+                except (ValueError, IndexError) as e:
+                    #log.debug("{} Omitting candidate {}. Reason: {}".format(i, candidate, e.args[0]))
+                    omit += 1
+                else:
+                    try:
+                        definition_dict = select_definition(definition, candidate)
+                    except (ValueError, IndexError) as e:
+                        #log.debug("{} Omitting definition {} for candidate {}. Reason: {}".format(i, definition_dict, candidate, e.args[0]))
+                        omit += 1
+                    else:
+                        definition_dict['abbre']=candidate
+                        abbrev_map.append(definition_dict)
+                        written += 1
+        except (ValueError, IndexError) as e:
+            log.debug("{} Error processing sentence {}: {}".format(i, sentence, e.args[0]))
+    log.debug("{} abbreviations detected and kept ({} omitted)".format(written, omit))
+    return abbrev_map
+
+def postprocess_abbr(ner_result,ori_text):
+    
+    final_result={}
+    if len(ner_result)==0:
+        return []
+    # abbr recognition
+    abbr_result=extract_abbreviation_definition_pairs(doc_text=ori_text)
+    
+    # read ner results
+    nor_loc_list={} #{entity_name_location:entity_information}
+
+    for ele in ner_result:
+        nor_loc_list[str(ele[0])+' '+str(ele[1])]=ele
+        final_result['\t'.join(ele)]=[int(ele[0]),int(ele[1])]
+    
+    #abbr matching
+    for abbr in abbr_result:
+        abbr_index=str(abbr['start'])+' '+str(abbr['stop'])
+        if abbr_index in nor_loc_list.keys():
+
+            line=ori_text
+            abbr_text=abbr['abbre']
+            abbr_eid=0
+            while line.find(abbr_text)>=0:
+                abbr_sid=line.find(abbr_text)+abbr_eid
+                abbr_eid=abbr_sid+len(abbr_text)
+                # print(abbr_sid,abbr_eid)
+                if abbr_sid>0 and abbr_eid<len(ori_text):
+                    if ori_text[abbr_sid-1].isalnum()==False and ori_text[abbr_eid].isalnum()==False:
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+nor_loc_list[abbr_index][2]+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                elif abbr_sid==0 and abbr_eid<len(ori_text):
+                    if ori_text[abbr_eid].isalnum()==False:
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+nor_loc_list[abbr_index][2]+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                elif abbr_sid>0 and abbr_eid==len(ori_text):
+                    if ori_text[abbr_sid-1].isalnum()==False :
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+nor_loc_list[abbr_index][2]+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                line=ori_text[abbr_eid:]
+    # print(final_result)
+    sorted_final_result=sorted(final_result.items(), key=lambda kv:(kv[1]), reverse=False)
+    final_result=[]
+    for ele in sorted_final_result:
+        final_result.append(ele[0].split('\t'))
+    return final_result
+
+def ner_abbr(ner_result,abbr_result,ori_text):
+    # read ner results
+    nor_name_list={} #{entity_name:entity_information}
+    nor_loc_list={} #{entity_name_location:entity_information}
+    final_result={} #{entity_information:location}  use to sort
+    for ele in ner_result:
+        temp_seg=ele.split('\t')
+        nor_loc_list[temp_seg[0]+' '+temp_seg[1]]=temp_seg
+        nor_name_list[temp_seg[2].lower()]=temp_seg
+        final_result['\t'.join(temp_seg[0:4])]=[int(temp_seg[0]),int(temp_seg[1])]
+    
+    #abbr matching
+    for abbr in abbr_result:
+        abbr_index=str(abbr['start'])+' '+str(abbr['stop'])
+        if abbr_index in nor_loc_list.keys():
+
+            line=ori_text
+            abbr_text=abbr['abbre']
+            abbr_eid=0
+            while line.find(abbr_text)>=0:
+                abbr_sid=line.find(abbr_text)+abbr_eid
+                abbr_eid=abbr_sid+len(abbr_text)
+                # print(abbr_sid,abbr_eid)
+                if abbr_sid>0 and abbr_eid<len(ori_text):
+                    if ori_text[abbr_sid-1].isalnum()==False and ori_text[abbr_eid].isalnum()==False:
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+abbr_text+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                elif abbr_sid==0 and abbr_eid<len(ori_text):
+                    if ori_text[abbr_eid].isalnum()==False:
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+abbr_text+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                elif abbr_sid>0 and abbr_eid==len(ori_text):
+                    if ori_text[abbr_sid-1].isalnum()==False :
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+abbr_text+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                line=ori_text[abbr_eid:]
+    # print(final_result)
+    final_result=sorted(final_result.items(), key=lambda kv:(kv[1]), reverse=False)
+    
+    return final_result
+
+
+            
+            
+if __name__ == '__main__':
+    path='//panfs/pan1/bionlp/lulab/luoling/HPO_project/diseaseTag/data/test/results/'
+    fin=open(path+'NCBI_test_phecr_95.tsv','r',encoding='utf-8')
+    context=fin.read().strip().split('\n\n')
+    fin.close()
+    fout=open(path+'NCBI_test_phecr_abbre_95.tsv','w',encoding='utf-8')
+    for doc in context:
+        lines=doc.split('\n')
+        ori_text=lines[1]
+        # print(ori_text)
+        fout.write(lines[0]+'\n'+lines[1]+'\n')
+        if len(lines)>2:
+            abbr_result=extract_abbreviation_definition_pairs(doc_text=ori_text)
+            print(abbr_result)
+            abbr_out=ner_abbr(lines[2:],abbr_result,ori_text)
+        else:
+            abbr_out=[]
+        # print('final:',abbr_out)
+        for ele in abbr_out:
+            fout.write(ele[0]+'\n')
+        fout.write('\n')
+        # sys.exit()
+    fout.close()
+    #last_out=combine_ml_dict_fn(abbr_out,infile)
+    #print(last_out)
+    
+    

src/combine_result.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Jun 15 11:24:45 2020
+
+@author: luol2
+"""
+
+import io
+def nest_overlap_entity(nest_list):
+    temp_result_list={}
+    for i in range(0, len(nest_list)):
+        hpoid=nest_list[i][3]
+        if hpoid not in temp_result_list.keys():
+            temp_result_list[hpoid]=nest_list[i]
+        else:
+            score=float(nest_list[i][4])
+            old_score=float(temp_result_list[hpoid][4])
+            if score>old_score: # retain higer score concept
+                temp_result_list[hpoid]=nest_list[i]
+    new_list=[]
+    for hpoid in temp_result_list.keys():
+        new_list.append(temp_result_list[hpoid])
+    
+    temp_result_list={} #same index, different ids
+    for i in range(0, len(new_list)):
+        ids=new_list[i][0]+' '+new_list[i][1]
+        if ids not in temp_result_list.keys():
+            temp_result_list[ids]=new_list[i]
+        else:
+            score=float(nest_list[i][4])
+            old_score=float(temp_result_list[ids][4])
+            if score>old_score: 
+                temp_result_list[ids]=new_list[i]
+    final_list=[]
+    for ids in temp_result_list.keys():
+        final_list.append(temp_result_list[ids]) 
+    return final_list
+def combine_ml_dict(dict_tsv,ml_tsv,nest=True):
+    fin_dic=io.StringIO(dict_tsv)
+    fin_ml=io.StringIO(ml_tsv)
+    fout=io.StringIO()
+    all_dic=fin_dic.read().strip().split('\n\n')
+    all_ml=fin_ml.read().strip().split('\n\n')
+    fin_dic.close()
+    fin_ml.close()
+    
+    for i in range(0,len(all_dic)):
+        lines_dic=all_dic[i].split('\n')
+        lines_ml=all_ml[i].split('\n')
+        entity_list={}
+        for j in range(1,len(lines_dic)):
+            seg=lines_dic[j].split('\t')
+            entity_list[lines_dic[j]]=[int(seg[0]),int(seg[1])] #dict results score 1.00
+        for j in range(1,len(lines_ml)):
+            seg=lines_ml[j].split('\t')
+            entity_list[lines_ml[j]]=[int(seg[0]),int(seg[1])]
+                
+        entity_list=sorted(entity_list.items(), key=lambda kv:(kv[1]), reverse=False)
+        entity_list_sort=[]
+        for ele in entity_list:
+            entity_list_sort.append(ele[0])
+        
+        final_entity=[]
+        if len(entity_list_sort)!=0:
+            first_entity=entity_list_sort[0].split('\t')
+            nest_list=[first_entity]
+            max_eid=int(first_entity[1])
+
+            for i in range(1,len(entity_list_sort)):
+                segs=entity_list_sort[i].split('\t')
+                if int(segs[0])> max_eid:
+                    if len(nest_list)==1:
+                        final_entity.append(nest_list[0])
+                        nest_list=[]
+                        nest_list.append(segs)
+                        if int(segs[1])>max_eid:
+                            max_eid=int(segs[1])
+                    else:
+                        tem=nest_overlap_entity(nest_list)
+                        final_entity.extend(tem)
+                        nest_list=[]
+                        nest_list.append(segs)
+                        if int(segs[1])>max_eid:
+                            max_eid=int(segs[1])
+                else:
+                    nest_list.append(segs)
+                    if int(segs[1])>max_eid:
+                        max_eid=int(segs[1])
+            if nest_list!=[]:
+                if len(nest_list)==1:
+                    final_entity.append(nest_list[0])
+
+                else:
+                    tem=nest_overlap_entity(nest_list)#find max entity
+                    final_entity.extend(tem)
+        
+        fout.write(lines_ml[0]+'\n')
+        for ele in final_entity:
+            fout.write('\t'.join(ele)+'\n')
+        fout.write('\n')
+    return fout.getvalue()  
+                

src/dic_ner.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 12 15:05:00 2020
+
+@author: luol2
+"""
+import sys
+import json
+import io
+from src.ssplit_tokenzier import ssplit_token_pos_lemma
+class Trie(object):
+    class Node(object):
+        def __init__(self):
+            self.term = None
+            self.next = {}
+    
+    def __init__(self, terms=[]):
+        self.root = Trie.Node()
+        for term in terms:
+            self.add(term)
+    
+    def add(self, term):
+        node = self.root
+        for char in term:
+            if not char in node.next:
+                node.next[char] = Trie.Node()
+            node = node.next[char]
+        node.term = term
+    
+    def match(self, query):
+        results = []
+        for i in range(len(query)):
+            node = self.root
+            for j in range(i, len(query)):
+                node = node.next.get(query[j])
+                if not node:
+                    break
+                if node.term:
+                    results.append((i, len(node.term)))
+        return results
+    
+    def __repr__(self):
+        output = []
+        def _debug(output, char, node, depth=0):
+            output.append('%s[%s][%s]' % (' '*depth, char, node.term))
+            for (key, n) in node.next.items():
+                _debug(output, key, n, depth+1)
+        _debug(output, '', self.root)
+        return '\n'.join(output)
+    
+class dic_ont():
+
+    def __init__(self, ont_files):
+        
+        dicin=open(ont_files['dic_file'],'r',encoding='utf-8')
+        win_size=50000
+        Dic=[]
+        print("loading dict!") 
+        for line in dicin:
+            line=line.strip()
+            if len(line.split())<=win_size:
+                words=line.split()
+                for i in range(len(words)):
+                    if len(words[i])>3 and (not words[i].isupper()):
+                        words[i]=words[i].lower()
+                line=' '.join(words[0:])
+                Dic.append(line.strip())
+        print("Dic_len:",len(Dic))
+        dicin.close()
+        
+        self.dic_trie = Trie(Dic)
+        print("load dic done!")
+        
+        #load word hpo mapping
+        fin_map=open(ont_files['word_hpo_file'],'r',encoding='utf-8')
+        self.word_hpo=json.load(fin_map)
+        fin_map.close()
+        
+        #load hpo word mapping
+        fin_map=open(ont_files['hpo_word_file'],'r',encoding='utf-8')
+        self.hpo_word=json.load(fin_map)
+        fin_map.close()
+    
+    def matching(self, source):
+    
+        fin=io.StringIO(source)
+        fout=io.StringIO()
+            
+        sent_list=[]
+        sent = []
+        sent_ori_list=[]
+        sent_ori=[]
+        
+        for line in fin:
+            line=line.strip()
+            if line=="":
+                sent_list.append(sent)
+                sent_ori_list.append(sent_ori)
+                sent=[]
+                sent_ori=[]         
+            else:
+                words=line.split('\t')
+                words[1]=words[1].lower()
+                sent.append(words[1])   # word lemma
+                sent_ori.append(words[0])
+        sent=[]       
+        fin.close()
+        
+        for k in range(len(sent_list)):
+            sent = sent_list[k]
+            sentence=' '.join(sent[0:])+" "
+            sentence_ori=' '.join(sent_ori_list[k])
+#            print('sentence:',sentence)
+            result=self.dic_trie.match(sentence) 
+#            print('result:',result)
+            new_result=[]
+            for i in range(0,len(result)):               
+                if result[i][0]==0 and sentence[result[i][1]]==" ":
+                    new_result.append([result[i][0],result[i][0]+result[i][1]])
+                elif result[i][0]>0 and sentence[result[i][0]-1]==' ' and sentence[result[i][0]+result[i][1]]==' ':
+                    new_result.append([result[i][0],result[i][0]+result[i][1]])
+#            print('new result:',new_result)
+            
+           
+                 
+            if len(new_result)==0:
+                fout.write(sentence_ori+'\n\n')
+                 
+            else:
+                fout.write(sentence_ori+'\n')
+                for ele in new_result:
+                    entity_text=sentence[ele[0]:ele[1]]
+                    if entity_text in self.word_hpo.keys():
+                        hpoid=self.word_hpo[entity_text]
+                    else:
+                        print('no id:', entity_text)
+                        hpoid=['None']
+                    if ele[0]==0:
+                        sid="0"
+                    else:
+                        temp_sent=sentence[0:ele[0]]
+                        sid=str(len(temp_sent.rstrip().split(' ')))
+                    temp_sent=sentence[0:ele[1]]
+                    eid=str(len(temp_sent.rstrip().split(' '))-1)
+#                    print(sid,eid,entity_text,hpoid[0])
+                    fout.write(sid+'\t'+eid+'\t'+entity_text+'\t'+";".join(hpoid)+'\t1.00\n')        
+                fout.write('\n')
+        
+        return fout.getvalue()
+     
+
+if __name__=='__main__':
+    
+    ontfiles={'dic_file':'//panfs/pan1/bionlp/lulab/luoling/HPO_project/bioTag/dict/hpo_noabb_lemma.dic',
+              'word_hpo_file':'//panfs/pan1/bionlp/lulab/luoling/HPO_project/bioTag/dict/word_hpoid_map.json',
+              'hpo_word_file':'//panfs/pan1/bionlp/lulab/luoling/HPO_project/bioTag/dict/hpoid_word_map.json'}
+    biotag_dic=dic_ont(ontfiles)
+    text='Nevoid basal cell carcinoma syndrome (NBCCS) is a hereditary condition transmitted as an autosomal dominant trait with complete penetrance and variable expressivity. The syndrome is characterised by numerous basal cell carcinomas (BCCs), odontogenic keratocysts of the jaws, palmar and/or plantar pits, skeletal abnormalities and intracranial calcifications. In this paper, the clinical features of 37 Italian patients are reviewed. Jaw cysts and calcification of falx cerebri were the most frequently observed anomalies, followed by BCCs and palmar/plantar pits. Similar to the case of African Americans, the relatively low frequency of BCCs in the Italian population is probably due to protective skin pigmentation. A future search based on mutation screening might establish a possible genotype phenotype correlation in Italian patients.'
+    ssplit_token=ssplit_token_pos_lemma(text)
+#    print(ssplit_token)
+    dic_result=biotag_dic.matching(ssplit_token)
+    print(dic_result)
+
+    

src/ml_ner.py

@@ -0,0 +1,587 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 12 16:41:54 2020
+
+@author: luol2
+"""
+
+import io
+import time
+import numpy as np
+from keras import backend as K
+def ml_intext(infile):
+    fin=open(infile,'r',encoding='utf-8')
+    alltexts=fin.read().strip().split('\n\n')
+    fin.close()
+    data_list=[]
+    label_list=[]
+    for sents in alltexts:
+        lines=sents.split('\n')
+        temp_sentece=[]
+        label=lines[0].split('\t')[0]
+        label_list.append(label)
+        for i in range(1,len(lines)):
+            seg=lines[i].split('\t')
+            temp_sentece.append(seg)
+        data_list.append(temp_sentece)
+    return data_list,label_list
+def ml_intext_fn(ml_input):
+    fin=io.StringIO(ml_input)
+    alltexts=fin.read().strip().split('\n\n')
+    fin.close()
+    data_list=[]
+    label_list=[]
+    for sents in alltexts:
+        lines=sents.split('\n')
+        temp_sentece=[]
+        label=lines[0].split('\t')[0]
+        label_list.append(label)
+        for i in range(1,len(lines)):
+            seg=lines[i].split('\t')
+            temp_sentece.append(seg)
+        data_list.append(temp_sentece)
+    return data_list,label_list
+def pun_filter(temp_entity):
+    pun_list=[',','.','!',';',':','?','(',')','[',']','{','}']
+    filter_flag=0
+    for ele in temp_entity:
+        if ele in pun_list:
+            filter_flag=1
+            break
+    return filter_flag
+def pos_filter(temp_pos,temp_entity):
+    pos_list_l=['PRP']
+    pos_list=['IN','DT','CC','O','MD','EX','POS','WDT','WP','WP$','WRB','TO','PRP$']
+    verb_word=['is','are','was','were','had','have','has','be','been','also']
+    filter_flag=0
+
+    if (temp_entity[0] in verb_word) or (temp_entity[-1] in verb_word):
+        filter_flag=1
+    if (temp_pos[0] in pos_list) or (temp_pos[-1] in pos_list) or (temp_pos[0] in pos_list_l):
+        filter_flag=1
+    return filter_flag 
+    
+def build_ngram_testset_filted(conll_input,Ngram=8):
+
+    fin_genia=io.StringIO(conll_input)
+    fout_context=io.StringIO()
+    fout_txt=io.StringIO()
+   
+    index_dict={}
+    allentity=[]
+    alltext=fin_genia.read().strip().split('\n\n')
+    fin_genia.close()
+    num_total=0
+    for i in range(0,len(alltext)):
+        
+        lines=alltext[i].split('\n') 
+        ori_txt=[]
+        for ele in lines:
+            seg=ele.split('\t')
+            ori_txt.append(seg[0])
+        fout_txt.write(' '.join(ori_txt)+'\n')
+
+        if Ngram>len(lines):
+            Ngram=len(lines)
+        
+        fout_context_list=[]
+        temp_entity=[]
+        temp_pos=[]
+        for ngram in range(2,Ngram+1):
+            if ngram==1:
+                for j in range(0, len(lines)):                 
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            sid=m
+                            eid=m
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])                            
+                        else:
+                            pass
+    #                        print(sentence[m])
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write('HP:None\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+            elif ngram==2:
+                for j in range(0, len(lines)-1):                  
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            sid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        elif m==j+1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            eid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        else:
+                            pass
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write('HP:None\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+            else :
+                for j in range(0, len(lines)-ngram+1):               
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            sid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        elif m>j and m<j+ngram-1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[2])
+                        elif m==j+ngram-1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            eid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        else:
+                            pass
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write('HP:None\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+
+    return fout_context.getvalue(),fout_txt.getvalue(),index_dict
+
+def build_all_ngram_testset_filted(conll_input,Ngram=8):
+
+    fin_genia=io.StringIO(conll_input)
+    fout_context=io.StringIO()
+    fout_txt=io.StringIO()
+   
+    index_dict={}
+    allentity=[]
+    alltext=fin_genia.read().strip().split('\n\n')
+    fin_genia.close()
+    num_total=0
+    for i in range(0,len(alltext)):
+        
+        lines=alltext[i].split('\n') 
+        ori_txt=[]
+        for ele in lines:
+            seg=ele.split('\t')
+            ori_txt.append(seg[0])
+        fout_txt.write(' '.join(ori_txt)+'\n')
+
+        if Ngram>len(lines):
+            Ngram=len(lines)
+        
+        fout_context_list=[]
+        temp_entity=[]
+        temp_pos=[]
+        for ngram in range(1,Ngram+1):
+            if ngram==1:
+                for j in range(0, len(lines)):                 
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            sid=m
+                            eid=m
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])                            
+                        else:
+                            pass
+    #                        print(sentence[m])
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write('HP:None\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+            elif ngram==2:
+                for j in range(0, len(lines)-1):                  
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            sid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        elif m==j+1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            eid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        else:
+                            pass
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write('HP:None\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+            else :
+                for j in range(0, len(lines)-ngram+1):               
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            sid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        elif m>j and m<j+ngram-1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[2])
+                        elif m==j+ngram-1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            eid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        else:
+                            pass
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write('HP:None\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+
+    return fout_context.getvalue(),fout_txt.getvalue(),index_dict
+
+def output_result(result,label_2_index,Top_N=5):
+
+    fout=io.StringIO()
+    hpo_label={}
+
+    for key in label_2_index.keys():
+        hpo_label[label_2_index[key]]=key
+
+
+    for line in result:
+        #Top_index=line.argsort()[-1*Top_N:][::-1]
+        index_top_unsort=np.argpartition(line,-Top_N)[-Top_N:]
+        values_top=line[index_top_unsort]
+        Top_index=index_top_unsort[np.argsort(-values_top)]
+        temp_list=[]
+        for max_index in Top_index:
+            hpo_id=hpo_label[max_index]
+            hpo_id_value=round(line[max_index],5)
+            temp_list.append(str(hpo_id)+'|'+str(hpo_id_value))
+        fout.write('\t'.join(temp_list)+'\n')
+        
+    return fout.getvalue()
+
+def decode_tsv(test_score, ml_input_index, ml_input_txt, T=0.8):
+    
+    fin_predict=io.StringIO(test_score)
+    fin_text=io.StringIO(ml_input_txt)
+    fout=io.StringIO()
+    
+    test_txt=fin_text.read().strip().split('\n')
+    test_index=ml_input_index
+    test_pre=fin_predict.read().strip().split('\n')
+    
+    fin_text.close()
+    fin_predict.close()
+    
+    sent_result={}
+    for i in range(0,len(test_pre)):
+        seg_pre=test_pre[i].split('\t')[0].split('|')
+        #print(seg_pre,T)
+        if float(seg_pre[1])>T and seg_pre[0]!='HP:None':
+            term_id=str(i+1)
+            pre_result=[test_index[term_id][1],test_index[term_id][2],seg_pre[0],seg_pre[1]]
+            sent_id=str(test_index[term_id][0])
+            if sent_id not in sent_result.keys():
+                sent_result[sent_id]=[pre_result]
+            else:
+                sent_result[sent_id].append(pre_result)
+        
+    for i in range(0,len(test_txt)):
+        fout.write(test_txt[i]+'\n')
+        if str(i) in sent_result.keys():
+            temp_result={}
+            for ele in sent_result[str(i)]:
+                temp_line=str(ele[0])+'\t'+str(ele[1])+'\t'+' '.join(test_txt[i].split()[ele[0]:ele[1]+1])+'\t'+ele[2]+'\t'+ele[3]
+                temp_result[temp_line]=[ele[0],ele[1]]
+            if len(temp_result)>=1:
+                temp_result=sorted(temp_result.items(), key=lambda d: (d[1][0],d[1][1]), reverse=False)
+                for ent in temp_result:
+                    fout.write(ent[0]+'\n')
+        fout.write('\n')
+
+    return fout.getvalue()
+    
+def score_filter(temp_entity,  T=0.1):
+
+    result_list=[]
+    for i in range(0,len(temp_entity)):
+        if float (temp_entity[i][-1])>T:
+            result_list.append(temp_entity[i])
+    return(result_list)
+def find_max_entity_nest(nest_list):
+    temp_result_list={}
+    for i in range(0, len(nest_list)):
+        hpoid=nest_list[i][-2]
+        score=float(nest_list[i][-1])
+        if hpoid not in temp_result_list.keys():
+            temp_result_list[hpoid]=nest_list[i]
+        else:
+            if score>float(temp_result_list[hpoid][-1]):
+                temp_result_list[hpoid]=nest_list[i]
+    new_list=[]
+    for hpoid in temp_result_list.keys():
+        new_list.append(temp_result_list[hpoid])
+    return new_list
+def duplicate_filter(temp_entity):
+    result_list=[]
+    if len(temp_entity)>1:                   
+        first_entity=temp_entity[0]
+        nest_list=[first_entity]
+        max_eid=int(first_entity[1])
+    
+        for i in range(1,len(temp_entity)):
+            segs=temp_entity[i]
+            if int(segs[0])> max_eid:
+                if len(nest_list)==1:
+                    result_list.append(nest_list[0])
+                    nest_list=[segs]
+                    if int(segs[1])>max_eid:
+                        max_eid=int(segs[1])
+                else:
+                    result_list.extend(find_max_entity_nest(nest_list))
+                    nest_list=[segs]
+
+                    if int(segs[1])>max_eid:
+                        max_eid=int(segs[1])
+                    
+            else:
+                nest_list.append(segs)
+                if int(segs[1])>max_eid:
+                    max_eid=int(segs[1])
+        if nest_list!=[]:
+            if len(nest_list)==1:
+                result_list.append(nest_list[0])
+
+            else:
+                result_list.extend(find_max_entity_nest(nest_list))
+    else:
+        result_list=temp_entity
+    return result_list
+def combine_strategy(test_decode_temp, T=0.8):
+    fin=io.StringIO(test_decode_temp)
+    fout=io.StringIO()
+    
+    documents=fin.read().strip().split('\n\n')
+    fin.close()
+    
+    for doc in documents:
+        lines=doc.split('\n')
+        context=lines[0]
+        final_entity_list=[]
+        if len(lines)>1:
+            # all entity candidates
+            temp_entity=[]
+            for i in range(1,len(lines)):
+                temp_entity.append(lines[i].split('\t'))
+            #print('all entity condidates: ',len(temp_entity))
+            
+            # 将阈值低于T的候选过滤
+            filter1=score_filter(temp_entity,T)
+#            print('filter1:', len(filter1))
+            filter2=duplicate_filter(filter1)  
+            #print('filter2:', filter2)
+            final_entity_list=filter2
+            
+        fout.write(context+'\n')
+        for ele in final_entity_list:
+            fout.write('\t'.join(ele)+'\n')
+        fout.write('\n')
+    
+    return fout.getvalue() 
+
+
+def model_predict(session,ml_input,nn_model,ml_input_txt,ml_input_index,Threshold):
+    if nn_model.model_type=='cnn':
+        #startTime=time.time()
+        test_set,test_label = ml_intext_fn(ml_input)
+        test_x, test_y = nn_model.rep.represent_instances_all_feas(test_set,test_label,word_max_len=nn_model.hyper['sen_max'],char_max_len=nn_model.hyper['word_max'])
+        input_test = []
+    
+        if nn_model.fea_dict['word'] == 1:
+            input_test.append(test_x[0])
+    
+        if nn_model.fea_dict['char'] == 1:
+            input_test.append(test_x[1])
+    
+        if nn_model.fea_dict['lemma'] == 1:
+            input_test.append(test_x[2])
+    
+        if nn_model.fea_dict['pos'] == 1:
+            input_test.append(test_x[3])
+        # print('ml-model-represent:',time.time()-startTime)
+        # startTime=time.time()
+        K.set_session(session)
+        test_pre = nn_model.model.predict(input_test)
+        # print('ml-model-predict:',time.time()-startTime)
+    
+    elif nn_model.model_type=='bert' or nn_model.model_type=='bioformer':
+        #startTime=time.time()
+        test_set,test_label = ml_intext_fn(ml_input)
+        test_x,test_y=nn_model.rep.load_data(test_set,test_label,word_max_len=nn_model.maxlen)
+        #print('ml-model-represent:',time.time()-startTime)
+        #startTime=time.time()
+        #K.set_session(session)
+        #with session.as_default():
+            #with session.graph.as_default():
+                #print('......session')
+        test_pre = nn_model.model.predict(test_x)
+        #print('ml-model-modedpred:',time.time()-startTime)
+    # startTime=time.time()
+    test_score=output_result(test_pre, nn_model.rep.label_2_index,Top_N=3)
+    # print('ml-model-output:',time.time()-startTime)
+    #print('test_score:',test_score)
+    # startTime=time.time()
+    test_decode_temp=decode_tsv(test_score, ml_input_index, ml_input_txt,  T=Threshold)
+    # print('ml-model-decode:',time.time()-startTime)
+    #print('decode_temp:\n',test_decode_temp)
+    # test_pre_tsv=combine_strategy(test_decode_temp,T=Threshold) 
+    return test_decode_temp
+
+def model_predict_old(ml_input,nn_model,ml_input_txt,ml_input_index,Threshold):
+    if nn_model.model_type=='cnn':
+        
+        test_set,test_label = ml_intext_fn(ml_input)
+        test_x, test_y = nn_model.rep.represent_instances_all_feas(test_set,test_label,word_max_len=nn_model.hyper['sen_max'],char_max_len=nn_model.hyper['word_max'])
+        input_test = []
+
+        if nn_model.fea_dict['word'] == 1:
+            input_test.append(test_x[0])
+
+        if nn_model.fea_dict['char'] == 1:
+            input_test.append(test_x[1])
+
+        if nn_model.fea_dict['lemma'] == 1:
+            input_test.append(test_x[2])
+
+        if nn_model.fea_dict['pos'] == 1:
+            input_test.append(test_x[3])
+        K.set_session(nn_model.session)
+        with nn_model.session.as_default():
+            with nn_model.session.graph.as_default():
+                test_pre = nn_model.model.predict(input_test,batch_size=256)
+
+    elif nn_model.model_type=='bert' or nn_model.model_type=='bioformer':
+
+        test_set,test_label = ml_intext_fn(ml_input)
+        test_x,test_y=nn_model.rep.load_data(test_set,test_label,word_max_len=nn_model.maxlen)
+        K.set_session(nn_model.session)
+        with nn_model.session.as_default():
+            with nn_model.session.graph.as_default():
+                test_pre = nn_model.model.predict(test_x,batch_size=128)
+
+    test_score=output_result(test_pre, nn_model.rep.label_2_index,Top_N=3)
+    #print('test_score:',test_score)
+    test_decode_temp=decode_tsv(test_score, ml_input_index, ml_input_txt,  T=0.0)
+    #print('decode_temp:\n',test_decode_temp)
+    test_pre_tsv=combine_strategy(test_decode_temp,T=Threshold) 
+    return test_pre_tsv
+
+def output_txt(ml_input_txt):
+    fin_text=io.StringIO(ml_input_txt)
+    fout=io.StringIO()
+
+    test_txt=fin_text.read().strip().split('\n')
+
+    fin_text.close()
+
+    for i in range(0,len(test_txt)):
+        fout.write(test_txt[i]+'\n')
+        fout.write('\n')
+
+    return fout.getvalue()
+
+def ml_tagging(session,ssplit_token,ml_model,Threshold):
+    # startTime=time.time()
+    ml_input, ml_input_txt,ml_input_index=build_ngram_testset_filted(ssplit_token)
+    # print('ml-ngrambuild:',time.time()-startTime)
+    #print('ml_input:')
+    #print(ml_input)
+    # startTime=time.time()
+    if len(ml_input_index)>0:
+        ml_pre_tsv=model_predict(session,ml_input,ml_model,ml_input_txt,ml_input_index,Threshold)
+    else:
+        ml_pre_tsv=output_txt(ml_input_txt)
+    # print('ml-modelpred:',time.time()-startTime)
+    return ml_pre_tsv
+    
+def ml_tagging_allngram(ssplit_token,ml_model,Threshold):
+    ml_input, ml_input_txt,ml_input_index=build_all_ngram_testset_filted(ssplit_token)
+    #print('ml_input:')
+    #print(ml_input)
+    if len(ml_input_index)>0:
+        ml_pre_tsv=model_predict_old(ml_input,ml_model,ml_input_txt,ml_input_index,Threshold)
+    else:
+        ml_pre_tsv=output_txt(ml_input_txt)
+    return ml_pre_tsv

src/nn_model.py

@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Mar 26 09:04:13 2020
+
+@author: luol2
+"""
+
+import time
+import sys
+import numpy as np
+import tensorflow as tf
+import keras
+from src.nn_represent import CNN_RepresentationLayer,BERT_RepresentationLayer
+from keras.layers import *
+from keras.models import Model
+from keras import backend as K
+from keras_bert import load_trained_model_from_checkpoint
+
+
+
+
+class bioTag_CNN():
+    def __init__(self, model_files):
+        self.model_type='cnn'
+        model_test_type='cnn'
+        self.fea_dict = {'word': 1,
+                         'char': 1,
+                         'lemma':0,
+                         'pos':0}
+    
+        self.hyper = {'sen_max'      :20,
+                      'word_max'     :40,
+                      'charvec_size' :50,
+                      'pos_size'     :50}
+             
+        self.w2vfile=model_files['w2vfile']      
+        self.charfile=model_files['charfile']
+        self.labelfile=model_files['labelfile']
+        self.posfile=model_files['posfile']
+        self.session = K.get_session()
+        vocab={'char':self.charfile,'label':self.labelfile,'pos':self.posfile}
+        print('loading w2v model.....') 
+        self.rep = CNN_RepresentationLayer(self.w2vfile,vocab_file=vocab, frequency=400000)
+         
+        print('building  model......')
+        all_fea = []
+        fea_list = []
+        
+        if self.fea_dict['word'] == 1:
+            word_input = Input(shape=(self.hyper['sen_max'],), dtype='int32', name='word_input')  
+            all_fea.append(word_input)
+            word_fea = Embedding(self.rep.vec_table.shape[0], self.rep.vec_table.shape[1], weights=[self.rep.vec_table], trainable=True,mask_zero=False, input_length=self.hyper['sen_max'], name='word_emd')(word_input)
+            fea_list.append(word_fea)
+    
+        if self.fea_dict['char'] == 1:
+            char_input = Input(shape=(self.hyper['sen_max'],self.hyper['word_max']), dtype='int32', name='char_input')
+            all_fea.append(char_input)
+            char_fea = TimeDistributed(Embedding(self.rep.char_table_size, self.hyper['charvec_size'], trainable=True,mask_zero=False),  name='char_emd')(char_input)
+            char_fea = TimeDistributed(Conv1D(self.hyper['charvec_size']*2, 3, padding='same',activation='relu'), name="char_cnn")(char_fea)
+            char_fea_max = TimeDistributed(GlobalMaxPooling1D(), name="char_pooling_max")(char_fea)
+            fea_list.append(char_fea_max)
+            
+        if self.fea_dict['lemma'] == 1:
+            lemma_input = Input(shape=(self.hyper['sen_max'],), dtype='int32', name='lemma_input')  
+            all_fea.append(lemma_input)
+            lemma_fea = Embedding(self.rep.vec_table.shape[0], self.rep.vec_table.shape[1], weights=[self.rep.vec_table], trainable=True,mask_zero=False, input_length=self.hyper['sen_max'], name='lemma_emd')(lemma_input)
+            fea_list.append(lemma_fea)
+            
+        if self.fea_dict['pos'] == 1:
+            pos_input = Input(shape=(self.hyper['sen_max'],), dtype='int32', name='pos_input')
+            all_fea.append(pos_input)
+            pos_fea = Embedding(self.rep.pos_table_size, self.hyper['pos_size'], trainable=True,mask_zero=False, input_length=self.hyper['sen_max'], name='pos_emd')(pos_input)
+            fea_list.append(pos_fea)    
+    
+        if len(fea_list) == 1:
+            concate_vec = fea_list[0]
+        else:
+            concate_vec = Concatenate()(fea_list)
+    
+        concate_vec = Dropout(0.4)(concate_vec)
+    
+        # model
+        if model_test_type=='cnn':    
+            cnn = Conv1D(1024, 1, padding='valid', activation='relu',name='cnn1')(concate_vec)
+            cnn = GlobalMaxPooling1D()(cnn)
+        elif model_test_type=='lstm':
+            bilstm = Bidirectional(LSTM(200, return_sequences=True, implementation=2, dropout=0.4, recurrent_dropout=0.4), name='bilstm1')(concate_vec)
+            cnn = GlobalMaxPooling1D()(bilstm)
+
+    
+        dense = Dense(1024, activation='relu')(cnn)
+        dense= Dropout(0.4)(dense)
+        output = Dense(self.rep.label_table_size, activation='softmax')(dense)
+        self.model = Model(inputs=all_fea, outputs=output)
+    def load_model(self,model_file):
+        self.model.load_weights(model_file)
+        self.session = K.get_session()
+        print(self.session)
+        #self.model.summary()        
+        print('load cnn model done!')
+
+class bioTag_BERT():
+    def __init__(self, model_files):
+        self.model_type='bert'
+        self.maxlen = 64
+        config_path = model_files['config_path']
+        checkpoint_path = model_files['checkpoint_path']
+        vocab_path = model_files['vocab_path']
+        self.label_file=model_files['labelfile']
+        self.session = tf.Session()
+    
+        self.rep = BERT_RepresentationLayer( vocab_path, self.label_file)
+        
+        
+        bert_model = load_trained_model_from_checkpoint(config_path, checkpoint_path, training=False, trainable=True,seq_len=self.maxlen)
+    
+        x1_in = Input(shape=(None,))
+        x2_in = Input(shape=(None,))
+        x = bert_model([x1_in, x2_in])
+        x = Lambda(lambda x: x[:, 0])(x)
+        outputs = Dense(self.rep.label_table_size, activation='softmax')(x)
+    
+        self.model = Model(inputs=[x1_in,x2_in], outputs=outputs)
+
+    def load_model(self,model_file):
+        self.model.load_weights(model_file)
+        self.session = K.get_session()
+        print(self.session)
+        #self.model.summary()        
+
+class bioTag_Bioformer():
+    def __init__(self, model_files):
+        self.model_type='bioformer'
+        self.maxlen = 32
+        config_path = model_files['config_path']
+        checkpoint_path = model_files['checkpoint_path']
+        vocab_path = model_files['vocab_path']
+        self.label_file=model_files['labelfile']
+    
+        self.rep = BERT_RepresentationLayer( vocab_path, self.label_file)
+        
+        
+        bert_model = load_trained_model_from_checkpoint(config_path, checkpoint_path, training=False, trainable=True,seq_len=self.maxlen)
+    
+        x1_in = Input(shape=(None,))
+        x2_in = Input(shape=(None,))
+        x = bert_model([x1_in, x2_in])
+        x = Lambda(lambda x: x[:, 0])(x)
+        outputs = Dense(self.rep.label_table_size, activation='softmax')(x)
+    
+        self.model = Model(inputs=[x1_in,x2_in], outputs=outputs)
+
+    def load_model(self,model_file):
+        self.model.load_weights(model_file)
+        #self.model._make_predict_function()
+        #session = K.get_session()
+        #print(session)
+        #self.model.summary()
+        session=''   
+        return session
+        print('load bioformer model done!')
+      

src/nn_represent.py

@@ -0,0 +1,338 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 12 10:02:20 2020
+
+@author: luol2
+"""
+import time
+import os, sys
+import numpy as np
+from keras.preprocessing.sequence import pad_sequences
+from keras_bert import Tokenizer
+
+
+class CNN_RepresentationLayer(object):
+    
+    
+    def __init__(self, wordvec_file,  vocab_file=[],\
+                 vec_size=50, word_size=10000, frequency=10000):
+        
+        '''
+        wordvec_file    ：    the file path of word embedding
+        vec_size        :    the dimension size of word vector 
+                             learned by word2vec tool
+        
+        word_size       :    the size of word vocabulary
+  
+        frequency       :    the threshold for the words left according to
+                             their frequency appeared in the text
+                             for example, when frequency is 10000, the most
+                             frequent appeared 10000 words are considered
+        
+        '''
+        #load word embedding
+        file = open(wordvec_file)
+        first_line = file.readline().strip()
+        file.close()
+        self.word_size = int(first_line.split()[0])
+        self.vec_size = int(first_line.split()[1])
+        self.frequency = frequency
+        
+        if self.frequency>self.word_size:
+            self.vec_table = np.zeros((self.word_size + 2, self.vec_size))
+        else:
+            self.vec_table = np.zeros((self.frequency + 2, self.vec_size))
+        self.word_2_index = {}
+        self.load_wordvecs(wordvec_file)
+        
+        #other fea
+        self.char_2_index={}
+        self.char_table_size=0
+        if 'char' in vocab_file.keys():
+            self.load_fea_vocab(vocab_file['char'],self.char_2_index)
+            self.char_table_size=len(self.char_2_index)
+            #print(self.char_table_size)
+            #print(self.char_2_index)
+        
+        self.label_2_index={}
+        self.label_table_size=0
+        if 'label' in vocab_file.keys():
+            self.load_label_vocab(vocab_file['label'],self.label_2_index)
+            self.label_table_size=len(self.label_2_index)
+            #print(self.label_table_size)
+            #print(self.char_2_index) 
+                        
+        self.pos_2_index={}
+        self.pos_table_size=0
+        if 'pos' in vocab_file.keys():
+            self.load_fea_vocab(vocab_file['pos'],self.pos_2_index)
+            self.pos_table_size=len(self.pos_2_index)
+            #print(self.pos_table_size)
+            
+
+    
+    def load_wordvecs(self, wordvec_file):
+        
+        file = open(wordvec_file,'r',encoding='utf-8')
+        file.readline()
+        #print(self.word_size)
+        #print(self.vec_size)
+        row = 0
+        self.word_2_index['padding_0'] = row #oov-zero vector
+        row+=1
+        for line in file:
+            if row <= self.word_size and row <= self.frequency:
+                line_split = line.strip().split(' ')
+                self.word_2_index[line_split[0]] = row
+                for col in range(self.vec_size):
+                    self.vec_table[row][col] = float(line_split[col + 1])
+                row += 1
+            else:
+                break
+        
+        self.word_2_index['sparse_vectors'] = row #oov-zero vector      
+        file.close()        
+
+    def load_fea_vocab(self,fea_file,fea_index):
+        fin=open(fea_file,'r',encoding='utf-8')
+        i=0
+        fea_index['padding_0']=i
+        i+=1
+        fea_index['oov_padding']=i
+        i+=1
+        for line in fin:
+            fea_index[line.strip()]=i
+            i+=1
+        fin.close()
+        
+    def load_label_vocab(self,fea_file,fea_index):
+        fin=open(fea_file,'r',encoding='utf-8')
+        i=0
+        for line in fin:
+            fea_index[line.strip()]=i
+            i+=1
+        fin.close()
+     
+    def generate_label_list(self,labels):
+        label_list=[]
+        
+        for label in labels:
+            temp_label=[0]*self.label_table_size
+            temp_label[self.label_2_index[label]]=1
+            label_list.append(temp_label)
+        return label_list
+            
+    def represent_instances_all_feas(self, instances, labels, word_max_len=100, char_max_len=50):
+
+        x_text_list=[]
+        x_word_list=[]
+        x_char_list=[]
+        
+        x_lemma_np=[]
+        x_pos_np=[]
+        y_np=[]
+        startTime=time.time()
+        for sentence in instances:           
+            sentence_list=[]
+            sentence_word_list=[]
+            sentence_lemma_list=[]
+            sentence_pos_list=[]
+            sentence_text=[]   
+            for j in range(0,len(sentence)):
+                word=sentence[j]
+                #char fea               
+                char_list=[0]*char_max_len
+                for i in range(len(word[0])):
+                    if i<char_max_len:
+                        if word[0][i] in self.char_2_index.keys():
+                            char_list[i]=self.char_2_index[word[0][i]]
+                        else:
+                            char_list[i]=self.char_2_index['oov_padding']
+                sentence_word_list.append(char_list)
+                
+                #word fea
+                sentence_text.append(word[0].lower())
+                if word[0].lower() in self.word_2_index.keys():
+                    sentence_list.append(self.word_2_index[word[0].lower()])                        
+                else:
+                    sentence_list.append(self.word_2_index['sparse_vectors'])
+                """
+                #lemma fea
+                if word[1].lower() in self.word_2_index.keys():
+                    sentence_lemma_list.append(self.word_2_index[word[1].lower()])
+                else:
+                    sentence_lemma_list.append(self.word_2_index['sparse_vectors'])
+             
+                #pos fea
+                if word[3] in self.pos_2_index.keys():
+                    sentence_pos_list.append(self.pos_2_index[word[3]])
+                else:
+                    sentence_pos_list.append(self.pos_2_index['oov_padding'])
+                """
+            x_text_list.append(sentence_text)
+            x_word_list.append(sentence_list)
+            x_char_list.append(sentence_word_list)
+            # x_lemma_list.append(sentence_lemma_list)
+            # x_pos_list.append(sentence_pos_list)
+
+       
+        #print('\nword:',x_word_list)
+        #print('\nchar:',x_char_list)
+        #print('\nlemma:',x_lemma_list)
+        #print('\npos:',x_pos_list)
+        #y_list=self.generate_label_list(labels)
+        #print('\ny_list:',y_list)
+       
+        x_word_np = pad_sequences(x_word_list, word_max_len, value=0, padding='post',truncating='post')  # right padding
+        x_char_np = pad_sequences(x_char_list, word_max_len, value=0, padding='post',truncating='post')
+        #x_lemma_np = pad_sequences(x_lemma_list, word_max_len, value=0, padding='post',truncating='post')                
+        #x_pos_np = pad_sequences(x_pos_list, word_max_len, value=0, padding='post',truncating='post')
+        #y_np = np.array(y_list)
+        return [x_word_np, x_char_np, x_lemma_np,  x_pos_np, x_text_list], y_np  
+    
+    def represent_instances_all_feas_myself(self, instances, labels, word_max_len=100, char_max_len=50):
+
+        x_text_list=[]
+        x_word_list=[]
+        x_char_list=[]
+        x_lemma_list=[]
+        x_pos_list=[]
+
+        y_list=[]
+        startTime=time.time()
+        for sentence in instances:           
+            sentence_list=[0]*word_max_len
+            sentence_word_list=[[0]*char_max_len for i in range(word_max_len)]
+            sentence_lemma_list=[0]*word_max_len
+            sentence_pos_list=[0]*word_max_len
+            sentence_text=[]   
+            for j in range(0,len(sentence)):
+                word=sentence[j]
+
+                sentence_text.append(word[0].lower())
+                
+                if j<word_max_len:
+                     #char fea               
+                    for i in range(len(word[0])):
+                        if i<char_max_len:
+                            if word[0][i] in self.char_2_index.keys():
+                                sentence_word_list[j][i]=self.char_2_index[word[0][i]]
+                            else:
+                                sentence_word_list[j][i]=self.char_2_index['oov_padding']
+    
+                    #word fea
+                    if word[0].lower() in self.word_2_index.keys():
+                        sentence_list[j]=self.word_2_index[word[0].lower()]                        
+                    else:
+                        sentence_list[j]=self.word_2_index['sparse_vectors']
+                    
+                    #lemma fea
+                    if word[1].lower() in self.word_2_index.keys():
+                        sentence_lemma_list[j]=self.word_2_index[word[1].lower()]
+                    else:
+                        sentence_lemma_list[j]=self.word_2_index['sparse_vectors']
+                 
+                    #pos fea
+                    if word[3] in self.pos_2_index.keys():
+                        sentence_pos_list[j]=self.pos_2_index[word[3]]
+                    else:
+                        sentence_pos_list[j]=self.pos_2_index['oov_padding']
+                
+            x_text_list.append(sentence_text)
+            x_word_list.append(sentence_list)
+            x_char_list.append(sentence_word_list)
+            x_lemma_list.append(sentence_lemma_list)
+            x_pos_list.append(sentence_pos_list)
+
+        print('ml-model-represent-list:',time.time()-startTime)
+        startTime=time.time()
+        #print('\nword:',x_word_list)
+        #print('\nchar:',x_char_list)
+        #print('\nlemma:',x_lemma_list)
+        #print('\npos:',x_pos_list)
+        y_list=self.generate_label_list(labels)
+        #print('\ny_list:',y_list)
+        # x_word_np = pad_sequences2(x_word_list, word_max_len, value=0, padding='post',truncating='post')  # right padding
+        # x_char_np = pad_sequences2(x_char_list, word_max_len, value=0, padding='post',truncating='post')
+        # x_lemma_np = pad_sequences2(x_lemma_list, word_max_len, value=0, padding='post',truncating='post')                
+        # x_pos_np = pad_sequences2(x_pos_list, word_max_len, value=0, padding='post',truncating='post')
+       
+        x_word_np = np.array(x_word_list)  # right padding
+        x_char_np = pad_sequences2(x_char_list)
+        x_lemma_np = np.array(x_lemma_list)                
+        x_pos_np = np.array(x_pos_list)
+        y_np = np.array(y_list)
+        print('ml-model-represent-pad:',time.time()-startTime)
+        return [x_word_np, x_char_np, x_lemma_np,  x_pos_np, x_text_list], y_np        
+
+
+
+class BERT_RepresentationLayer(object):
+    
+    
+    def __init__(self, vocab_path, label_file):
+        
+
+        #load vocab
+        self.bert_vocab_dict = {}
+        self.load_bert_vocab(vocab_path,self.bert_vocab_dict)
+        self.tokenizer = Tokenizer(self.bert_vocab_dict)
+
+        #load label
+        self.label_2_index={}
+        self.label_table_size=0
+        self.load_label_vocab(label_file,self.label_2_index)
+        self.label_table_size=len(self.label_2_index)
+
+    def load_label_vocab(self,fea_file,fea_index):
+        fin=open(fea_file,'r',encoding='utf-8')
+        i=0
+        for line in fin:
+            fea_index[line.strip()]=i
+            i+=1
+        fin.close()
+    def load_bert_vocab(self,vocab_file,vocab_dict):
+        fin=open(vocab_file,'r',encoding='utf-8')
+        i=0
+        for line in fin:
+            vocab_dict[line.strip()]=i
+            i+=1
+        fin.close()
+            
+    def generate_label_list(self,labels):
+        label_list=[]
+        
+        for label in labels:
+            temp_label=[0]*self.label_table_size
+            temp_label[self.label_2_index[label]]=1
+            label_list.append(temp_label)
+        return label_list
+    
+    def load_data(self,instances, labels,  word_max_len=100):
+    
+        x_index=[]
+        x_seg=[]
+        y_np=[]
+        
+        for sentence in instances:                           
+            sentence_text_list=[]
+            for j in range(0,len(sentence)):
+                sentence_text_list.append(sentence[j][0])
+            sentence_text=' '.join(sentence_text_list)
+            #print(self.tokenizer.tokenize(first=sentence_text))
+            x1, x2 = self.tokenizer.encode(first=sentence_text)
+            x_index.append(x1)
+            x_seg.append(x2)                
+        
+        # y_list=self.generate_label_list(labels)
+        
+        x1_np = pad_sequences(x_index, word_max_len, value=0, padding='post',truncating='post')  # right padding
+        x2_np = pad_sequences(x_seg, word_max_len, value=0, padding='post',truncating='post')
+        # y_np = np.array(y_list)
+
+        return [x1_np, x2_np], y_np  
+
+if __name__ == '__main__':
+    pass
+ 
+            

src/post_processing.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Jun 18 20:08:30 2020
+
+@author: luol2
+"""
+
+def combine_overlap(mention_list):
+   
+    entity_list=[]
+    if len(mention_list)>2:
+        
+        first_entity=mention_list[0]
+        nest_list=[first_entity]
+        max_eid=int(first_entity[1])
+        for i in range(1,len(mention_list)):
+            segs=mention_list[i]
+            if int(segs[0])> max_eid:
+                if len(nest_list)==1:
+                    entity_list.append(nest_list[0])
+                    nest_list=[]
+                    nest_list.append(segs)
+                    if int(segs[1])>max_eid:
+                        max_eid=int(segs[1])
+                else:
+                    tem=find_max_entity(nest_list)#find max entity
+                    entity_list.append(tem)
+                    nest_list=[]
+                    nest_list.append(segs)
+                    if int(segs[1])>max_eid:
+                        max_eid=int(segs[1])
+                    
+            else:
+                nest_list.append(segs)
+                if int(segs[1])>max_eid:
+                    max_eid=int(segs[1])
+        if nest_list!=[]:
+            if len(nest_list)==1:
+                entity_list.append(nest_list[0])
+
+            else:
+                tem=find_max_entity(nest_list)#find max entity
+                entity_list.append(tem)
+    else:
+        entity_list=mention_list
+        
+    return entity_list
+
+def find_max_entity(nest_list):
+    max_len=0
+    max_entity=[]
+    for i in range(0, len(nest_list)):
+        length=int(nest_list[i][1])-int(nest_list[i][0])
+        if length>max_len:
+                max_len=length
+                max_entity=nest_list[i]
+    
+    return max_entity

src/restore_index.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Jun 14 17:19:02 2020
+
+@author: luol2
+"""
+
+import io
+import sys
+        
+def restore_index_nest_fn(ori_text,file_pre):
+
+
+    fin_pre=io.StringIO(file_pre)
+    #print(file_pre)
+    all_pre=fin_pre.read().strip().split('\n\n')
+    fin_pre.close()
+    #print(len(all_pre))
+    
+    new_sentence=''
+    restore_result=[]
+    
+    sentence_ori=ori_text.lower().replace('``','" ')
+    sentence_ori=sentence_ori.replace("''",'" ')
+    for pre_i in range(0,len(all_pre)):
+        pre_lines=all_pre[pre_i].split('\n')
+        #print(pre_lines)
+#        print(sentence_ori)
+        if len(pre_lines)>1:
+            #print(pre_lines)
+            sentence_pre=pre_lines[0].lower().replace('``','"')
+            sentence_pre=sentence_pre.replace("''",'"')
+            sentence_pre=sentence_pre.split()
+            pre_result=[]
+            for i in range(1,len(pre_lines)):
+                pre_result.append(pre_lines[i].split('\t'))
+            
+            restore_sid=0
+            restore_eid=0
+            each_word_id=[]
+            
+            for i in range(0,len(sentence_pre)):
+
+                temp_id=sentence_ori.find(sentence_pre[i])
+                if temp_id<0:
+                    if sentence_pre[i].find('"')>=0:
+                        temp_id = sentence_ori.find(sentence_pre[i].replace('"','" '))
+                    else:
+                        #print('ori:',sentence_ori)
+                        print('resotr index error:',sentence_pre[i])
+                new_sentence+=sentence_ori[0:temp_id]
+                
+                restore_sid=len(new_sentence)
+                restore_eid=len(new_sentence)+len(sentence_pre[i])
+                each_word_id.append([str(restore_sid),str(restore_eid)])
+                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
+                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
+#            print('each_word:',each_word_id)    
+            for pre_ele in pre_result:
+                # if len(pre_ele)>4:
+                #     temp_pre_result=[each_word_id[int(pre_ele[0])][0],each_word_id[int(pre_ele[1])][1],pre_ele[3].split('|')[0],pre_ele[4]]
+                # else:
+                #     temp_pre_result=[each_word_id[int(pre_ele[0])][0],each_word_id[int(pre_ele[1])][1],pre_ele[3].split('|')[0],'1.00']
+                temp_pre_result=[each_word_id[int(pre_ele[0])][0],each_word_id[int(pre_ele[1])][1],pre_ele[3].split('|')[0],pre_ele[4]]
+                if temp_pre_result not in restore_result:
+                    restore_result.append(temp_pre_result)
+        else:
+            sentence_pre=pre_lines[0].lower().replace('``','"')
+            sentence_pre=sentence_pre.replace("''",'"')
+            sentence_pre=sentence_pre.split()
+           
+            for i in range(0,len(sentence_pre)):
+
+                temp_id=sentence_ori.find(sentence_pre[i])
+                if temp_id<0:
+                    if sentence_pre[i].find('"')>=0:
+                        temp_id = sentence_ori.find(sentence_pre[i].replace('"','" '))
+                    else:
+                        print('resotr index error:',sentence_pre[i])
+                new_sentence+=sentence_ori[0:temp_id]
+                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
+                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
+#    print('resotre:',restore_result)
+    return restore_result
+
+if __name__=='__main__':
+    path='//panfs/pan1/bionlp/lulab/luoling/HPO_project/bioTag/data/test/gsc/result/'
+    fin=open(path+'GSCplus_Nest_biobert.tsv','r',encoding='utf-8')
+    fout=open(path+'GSCplus_Nest_restore_biobert.tsv','w',encoding='utf-8')
+    all_context=fin.read().strip().split('\n\n\n\n')
+    fin.close()
+    file_num=0
+    for doc in all_context:
+        file_num+=1
+        print('file_num:',file_num)
+        doc_ele=doc.split('\n\n')
+        first_line = doc_ele[0].split('\n')
+        pmid=first_line[0]
+        ori_text=first_line[1]
+        pre_result='\n\n'.join(doc_ele[1:])
+#        print('pmid:',pmid)
+#        print('ori:',ori_text)
+#        print('pre:',pre_result)
+        final_result=restore_index_nest_fn(ori_text,pre_result)
+        fout.write(pmid+'\n'+ori_text+'\n')
+        for ele in final_result:
+            fout.write('\t'.join(ele)+'\t'+ori_text[int(ele[0]):int(ele[1])]+'\n')
+        fout.write('\n')
+    fout.close()

src/ssplit_tokenzier.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 12 15:26:44 2020
+
+@author: luol2
+"""
+
+import nltk
+from nltk.stem import WordNetLemmatizer
+from nltk.corpus import wordnet
+from nltk.stem.porter import PorterStemmer
+lemmatizer = WordNetLemmatizer() 
+stemmer = PorterStemmer()
+import io
+    
+def get_wordnet_pos(treebank_tag):
+    if treebank_tag.startswith('J'):
+        return wordnet.ADJ
+    elif treebank_tag.startswith('V'):
+        return wordnet.VERB
+    elif treebank_tag.startswith('N'):
+        return wordnet.NOUN
+    elif treebank_tag.startswith('R') or treebank_tag=='IN':
+        return wordnet.ADV
+    else:
+        return wordnet.NOUN
+    
+def ssplit_token_pos_lemma(in_text):
+    
+    fout=io.StringIO()
+
+    line=in_text.strip()
+    line=line.replace('-',' - ').replace('/',' / ')
+    sentences = nltk.sent_tokenize(line)
+    sentences = [nltk.word_tokenize(sent) for sent in sentences] 
+#    print(sentences)
+    for sent in sentences:
+        token_pos = nltk.pos_tag(sent)
+        for token in token_pos:
+            lemma = lemmatizer.lemmatize(token[0].lower(), get_wordnet_pos(token[1]))
+            stem = stemmer.stem(token[0].lower())
+            fout.write(token[0]+'\t'+lemma+'\t'+stem+'\t'+token[1]+'\n')
+        fout.write('\n')
+           
+    return fout.getvalue()

src/tagging_text.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Aug 24 16:21:23 2020
+
+@author: luol2
+"""
+
+import argparse
+from src.ssplit_tokenzier import ssplit_token_pos_lemma
+from src.ml_ner import ml_tagging,ml_tagging_allngram
+from src.combine_result import combine_ml_dict
+from src.restore_index import restore_index_nest_fn
+from src.dic_ner import dic_ont
+from src.post_processing import combine_overlap
+from src.abbre_resolution import postprocess_abbr
+import os
+import time
+import json
+
+#hybrid method
+def bioTag(session,text,biotag_dic,ml_model,onlyLongest=False, abbrRecog=False, Threshold=0.95):
+
+    # startTime=time.time()
+    ssplit_token=ssplit_token_pos_lemma(text)
+#    print(ssplit_token) 
+    # print('ssplit token:',time.time()-startTime)
+    
+    # startTime=time.time()
+    dict_tsv=biotag_dic.matching(ssplit_token)
+#    print('dict tsv:\n',dict_tsv)
+    # print('dict ner:',time.time()-startTime)
+  
+    # startTime=time.time()
+    ml_tsv=ml_tagging(session,ssplit_token,ml_model,Threshold)
+    #print('ml_tsv:\n',ml_tsv)
+    # print('ml ner:',time.time()-startTime)
+    
+    # startTime=time.time()
+    combine_tsv=combine_ml_dict(dict_tsv,ml_tsv)
+    #combine_tsv=combine_ml_dict_fn(ml_tsv,dict_tsv)
+    #print('combine:\n',combine_tsv)
+    # print('combine:',time.time()-startTime)
+    
+    # startTime=time.time()
+    final_result=  restore_index_nest_fn(text,combine_tsv)
+#    print('final ner:',time.time()-startTime)
+    if onlyLongest==True:
+        final_result=combine_overlap(final_result)
+    if abbrRecog==True:
+        final_result=postprocess_abbr(final_result,text)
+#    print('final result:') 
+#    print(final_result)
+    # print('final ner:',time.time()-startTime)
+    
+    return final_result
+
+# only machine learning-based method
+def bioTag_ml(text,ml_model,onlyLongest=False,abbrRecog=False, Threshold=0.95):
+
+#    startTime=time.time()
+    ssplit_token=ssplit_token_pos_lemma(text)
+#    print(ssplit_token) 
+#    print('ssplit token:',time.time()-startTime)
+    
+#    startTime=time.time()
+    ml_tsv=ml_tagging_allngram(ssplit_token,ml_model,Threshold)
+#    print('ml_tsv:\n',ml_tsv)
+#    print('ml ner:',time.time()-startTime)
+       
+    final_result=  restore_index_nest_fn(text,ml_tsv)
+#    print('final ner:',time.time()-startTime)
+    if onlyLongest==True:
+        final_result=combine_overlap(final_result)
+    
+    if abbrRecog==True:
+        final_result=postprocess_abbr(final_result,text)
+    
+    return final_result
+
+# only dict method
+def bioTag_dic(text,biotag_dic,onlyLongest=False, abbrRecog=False):
+
+#    startTime=time.time()
+    ssplit_token=ssplit_token_pos_lemma(text)
+#    print(ssplit_token) 
+#    print('ssplit token:',time.time()-startTime)
+    
+#    startTime=time.time()
+    dict_tsv=biotag_dic.matching(ssplit_token)
+#    print('dict tsv:\n',dict_tsv)
+#    print('dict ner:',time.time()-startTime)
+    
+    final_result=  restore_index_nest_fn(text,dict_tsv)
+#    print('final ner:',time.time()-startTime)
+    if onlyLongest==True:
+        final_result=combine_overlap(final_result)
+    
+    if abbrRecog==True:
+        final_result=postprocess_abbr(final_result,text)
+    
+    return final_result
+