import streamlit as st import pandas as pd # import utils.pharmap_utils.layout as lt from batutils import * # import stanza import requests # import os.path import io # import PyPDF2 from pypdf.pdf import PdfFileReader from urllib.request import Request, urlopen from bs4 import BeautifulSoup from bs4.element import Comment # from utils.pharmap_utils.dtxutils import * # from utils.pharmap_utils.dictutils import * from utils.pharmap_utils.stanzautils import * # @st.cache(show_spinner=True) def get_ner(contents): print('inside get ner') content_list = [] st.write('Reading the page...') nlp = call_nlp_pipeline() doc = nlp(contents.strip()) st.write('Getting disease names...') for ent in doc.entities: if ent.type == 'DISEASE': content_list.append(ent.text.replace('\n', '')) content_list = list(set(content_list)) print('got the disease names', content_list) st.write('Got the disease names...') return content_list def get_ta_mapped_url(content_list): print('inside get_ta_mapped') st.write(content_list) # content_list = content_list st.write('Trying to get Mesh Name..') print('Trying to get Mesh Name..') ta_list = [] ta = [] for condition_text in content_list: # print("printing inside the for loop",condition_text) ta = non_url_flow(condition_text) # print(ta) ta_list.append(ta) # print(ta_list) flat_list = [item for sublist in ta_list for item in sublist] ta = list(set(flat_list)) print("Outside the loop", ta) return ta def check_pdf_html(url): r = requests.get(url) content_type = r.headers.get('content-type') print(content_type) if 'application/pdf' in content_type: ext = 'pdf' elif 'text/html' in content_type: ext = 'html' else: ext = '' print('Unknown type: {}'.format(content_type)) print(ext) return ext # @st.cache def get_disease_html(u): print('inside get disease html') # u="https://www.exelixis.com/pipeline/" # "https://www.roche.com/dam/jcr:22160102-e04d-4484-ae3b-0f474105647e/en/diaq321.pdf" url = Request(u, headers={'User-Agent': 'Mozilla/5.0'}) html = urlopen(url).read() soup = BeautifulSoup(html, features="html.parser") for script in soup(["script", "style"]): script.extract() for footer in soup.findAll('header'): footer.decompose() for footer in soup.findAll('footer'): footer.decompose() text = soup.get_text() lines = (line.strip() for line in text.splitlines()) chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) text = '\n'.join(chunk for chunk in chunks if chunk) # st.write(text) result = get_ner(text) return result # @st.cache(persist=True,show_spinner=True) def get_disease_pdf(url): st.write('get pdf disease') r = requests.get(url) f = io.BytesIO(r.content) reader = PdfFileReader(f) # pnum = reader.getNumPages() # p_num = [] data = [] df = pd.DataFrame() content_list = [] pnum = 2 for p in range(pnum): contents = reader.getPage(p).extractText() content_list = get_ner(contents) # doc = nlp(contents.strip()) # for ent in doc.entities: # if ent.type=='DISEASE': # content_list.append(ent.text.replace('\n','')) # content_list = list(set(content_list)) # print(content_list) # p_num = [p+1] # print('pagenum',p_num) # print('values',content_list) a_dictionary = {'pno:': [p + 1], 'conditions': content_list } content_list = [] # print('a_dictionary',a_dictionary) data.append(a_dictionary) f.close() df = df.append(data, True) return df def get_link_mapped(url): # st.write(url) # url = 'https://www.gene.com/medical-professionals/pipeline' try: get = check_pdf_html(url) # st.write(get) except: get = 'invalid URL' if get == 'pdf': # st.write('inside pdf') pdf_mapped_df = get_disease_pdf(url) st.dataframe(pdf_mapped_df) elif get == 'html': # st.write('inside html') # st.write(url) # print('html') content_list = get_disease_html(url) ta = get_ta_mapped_url(content_list) st.write(ta) elif get == 'invalid URL': print('invalid')