Create streamlit_app.py

streamlit_app.py

@@ -0,0 +1,563 @@
+import streamlit as st
+
+from transformers import pipeline, AutoModel, AutoTokenizer
+
+import time
+
+from time import time as t
+
+from gazpacho import Soup, get
+
+import tokenizers
+
+import json
+
+import requests
+
+#############
+
+# FUNCTIONS #
+
+#############
+
+ex = []
+
+# Query the HuggingFace Inference engine.  
+
+def query(payload):
+
+    data = json.dumps(payload)
+
+    response = requests.request("POST", API_URL, headers=headers, data=data)
+
+    return json.loads(response.content.decode("utf-8"))
+
+    
+
+def ner_query(payload):
+
+    data = json.dumps(payload)
+
+    response = requests.request("POST", NER_API_URL, headers=headers, data=data)
+
+    return json.loads(response.content.decode("utf-8"))
+
+# gets links and identifies if they're cnn or npr
+
+def get_articles(user_choices, cnn_dict, npr_dict):
+
+    clustLinks = []
+
+    heds = {}
+
+    
+
+    # Get all headlines from each cluster -- add to dict and record number of clusters of interest the headline appeared in.
+
+    for each in user_choices:
+
+        for beach in clusters[each.lower()]:
+
+            if beach not in heds:
+
+                heds[beach] = 1
+
+            else:
+
+                heds[beach] += 1
+
+                
+
+    # Convert keys (headlines) to list then sort in descending order of prevalence
+
+    sorted_heds = list(heds.keys())
+
+    sorted_heds.sort(key=lambda b: heds[b], reverse=True)
+
+    
+
+    for each in sorted_heds:
+
+        try:
+
+            # look up the headline in cnn
+
+            clustLinks.append(('cnn',cnn_dict[each]))
+
+            # if exception KeyError then lookup in npr
+
+        except KeyError:
+
+            clustLinks.append(('npr',npr_dict[each]))   
+
+    return clustLinks
+
+# gets articles from source via scraping
+
+def retrieve(input_reslist):
+
+    cnn = 'https://lite.cnn.com'
+
+    npr = 'https://text.npr.org'
+
+    articles = []
+
+    
+
+    # Scrapes from npr or cnn.  Should modularize this and use a dict as a switch-case
+
+    for each in input_reslist:
+
+        if each[0] == 'npr':
+
+            container = Soup(get(npr+each[1])).find('div', {'class': "paragraphs-container"}).find('p')
+
+            articles.append(container)
+
+        if each[0] == 'cnn':
+
+            container = Soup(get(cnn+each[1])).find('div', {'class': 'afe4286c'})
+
+            # Extract all text from paragraph tags, each extracted from container
+
+            #story = '\n'.join([x.text for x in container.find('p') if x.text != ''])
+
+            story = container.find('p')
+
+            articles.append(story[4:])
+
+        time.sleep(1)
+
+    return articles
+
+# Returns a list of articles
+
+# Takes list of articles and assigns each articles' text to an int for some reason....
+
+#
+
+## *** Dictionary might shuffle articles?
+
+#
+
+def art_prep(retrieved):
+
+    a = []
+
+    for each in retrieved:
+
+        if type(each) is not list:
+
+            a.append(each.strip())
+
+        else:
+
+            a.append(''.join([art.strip() for art in each]))
+
+    return a
+
+# User choices is the list of user-chosen entities.
+
+def seek_and_sum(user_choices, cnn_dict, npr_dict):
+
+    # If no topics are selected return nothing
+
+    if len(user_choices) == 0:
+
+        return []
+
+    digs = []
+
+    prepped=art_prep(retrieve(get_articles(user_choices, cnn_dict, npr_dict)))
+
+    # Final is the output...the digest.
+
+    for piece in prepped:
+
+        digs.append(create_summaries(piece, 'sshleifer/distilbart-cnn-12-6'))
+
+    # Opportunity for processing here
+
+    
+
+    return digs
+
+# Chunks
+
+def chunk_piece(piece, limit):
+
+    words = len(piece.split(' ')) # rough estimate of words.  # words <= number tokens generally.
+
+    perchunk = words//limit
+
+    base_range = [i*limit for i in range(perchunk+1)]
+
+    range_list = [i for i in zip(base_range,base_range[1:])]
+
+    #range_list.append((range_list[-1][1],words))   try leaving off the end (or pad it?)
+
+    chunked_pieces = [' '.join(piece.split(' ')[i:j]).replace('\n','').replace('.','. ') for i,j in range_list]
+
+    return chunked_pieces
+
+# Summarizes
+
+def create_summaries(piece, chkpnt, lim=400):
+
+    tokenizer = AutoTokenizer.from_pretrained(chkpnt)
+
+    limit = lim
+
+    count = -1
+
+    summary = []
+
+    words = len(piece.split(' '))
+
+    if words >= limit:
+
+        # chunk the piece
+
+        #print(f'Chunking....')
+
+        proceed = False
+
+        while not proceed:
+
+            try: 
+
+                chunked_pieces = chunk_piece(piece, limit)
+
+                for chunk in chunked_pieces:
+
+                    token_length = len(tokenizer(chunk))
+
+                    
+
+                    # Perform summarization
+
+                    if token_length <= 512:
+
+                        data = query({ "inputs": str(chunk), "parameters": {"do_sample": False} }) # The way I'm passing the chunk could be the problem?  In a loop by ref?
+
+                        summary.append(data[0]['summary_text'])
+
+                        proceed = True 
+
+                    else:
+
+                        proceed = False
+
+                        limit -= 2  # Try to back off as little as possible.
+
+                        summary = []  # empty summary we're starting again.
+
+            except IndexError: # Caused when 400 words get tokenized to > 512 tokens.  Rare.
+
+                proceed = False
+
+                # lower the limit
+
+                limit -= 2  # Try to back off as little as possible.
+
+                summary = []  # empty summary we're starting again.
+
+        days_summary = ' '.join(summary) # Concatenate partial summaries
+
+    else: 
+
+        #print(f'Summarizing whole piece')
+
+        proceed = False
+
+        while not proceed:
+
+            try:
+
+                # Perform summarization
+
+                data = query({ "inputs": str(piece), "parameters": {"do_sample": False} })
+
+                days_summary = data[0]['summary_text']
+
+                proceed= True
+
+            except IndexError:
+
+                proceed = False
+
+                piece = piece[:-4]
+
+                days_summary = ''     
+
+    return days_summary
+
+# This function creates a nice output from the dictionary the NER pipeline returns.
+
+# It works for grouped_entities = True or False.
+
+def ner_results(ner_object, indent=False, groups=True, NER_THRESHOLD=0.5):
+
+    # empty lists to collect our entities
+
+    people, places, orgs, misc = [], [], [], []
+
+    # 'ent' and 'designation' handle the difference between dictionary keys 
+
+    # for aggregation strategy grouped vs ungrouped
+
+    ent = 'entity' if not groups else 'entity_group'
+
+    designation = 'I-' if not groups else ''
+
+    # Define actions -- this is a switch-case dictionary.
+
+    actions = {designation+'PER':people.append,
+
+             designation+'LOC':places.append, 
+
+             designation+'ORG':orgs.append,
+
+             designation+'MISC':misc.append}
+
+    # For each dictionary in the ner result list, if it doesn't contain a '#' 
+
+    #   and the confidence is > 90%, add the entity name to the list for its type.
+
+    readable = [ actions[d[ent]](d['word']) for d in ner_object if '#' not in d['word'] and d['score'] > NER_THRESHOLD ]
+
+    # create list of all entities to return
+
+    ner_list = [i for i in set(people) if len(i) > 2] + [i for i in set(places) if len(i) > 2] + [i for i in set(orgs) if len(i) > 2] + [i for i in set(misc) if len(i) > 2]
+
+    return ner_list
+
+    
+
+@st.cache(hash_funcs={tokenizers.Tokenizer: id})
+
+def create_ner_dicts(state=True):
+
+    # Changing this will run the method again, refreshing the topics
+
+    status = state
+
+    
+
+    url1 = 'https://lite.cnn.com/en'
+
+    soup_cnn = Soup(get(url1))
+
+    # extract each headline from the div containing the links.
+
+    cnn_text = [i.text for i in soup_cnn.find('div', {'class': 'afe4286c'}).find('a')]
+
+    cnn_links = [i.attrs['href'] for i in soup_cnn.find('div', {'class': 'afe4286c'}).find('a')]
+
+    cnn = [i for i in cnn_text if 'Analysis:' not in i and 'Opinion:' not in i]
+
+    
+
+    
+
+    # Get current links...in the future you'll have to check for overlaps.
+
+    url2 = 'https://text.npr.org/1001'
+
+    soup = Soup(get(url2))
+
+    # extract each headline
+
+    npr_text = [i.text for i in soup.find('div', {'class': 'topic-container'}).find('ul').find('a')]
+
+    npr_links = [i.attrs['href'] for i in soup.find('div', {'class': 'topic-container'}).find('ul').find('a')]
+
+    npr = [i for i in npr_text if 'Opinion:' not in i]
+
+    
+
+    cnn_dict = {k[0]:k[1] for k in zip(cnn_text,cnn_links)}
+
+    npr_dict = {k[0]:k[1] for k in zip(npr_text,npr_links)}
+
+   
+
+    # START Perform NER
+
+    cnn_ner = {x:ner_results(ner_query(x)) for x in cnn} ###################################################################################################
+
+    npr_ner = {x:ner_results(ner_query(x)) for x in npr} #################################   #################################    #################################
+
+    
+
+    return cnn_dict, npr_dict, cnn_ner, npr_ner
+
+## A function to change a state variable in create_dicts() above
+
+##      that then runs it and creates updated clusters.
+
+def get_news_topics(cnn_ner, npr_ner):
+
+    
+
+    ## END Perform NER 
+
+    
+
+    # Select from articles.
+
+    
+
+    ## Select from articles that are clusterable only. (Entities were recognized.)
+
+    cnn_final = {x:npr_ner[x] for x in npr_ner.keys() if len(npr_ner[x]) != 0} 
+
+    npr_final =  {y:cnn_ner[y] for y in cnn_ner.keys() if len(cnn_ner[y]) != 0 }
+
+    
+
+    # What's in the news?
+
+    # Get entities named in the pool of articles we're drawing from
+
+    e_list = []
+
+    for i in [i for i in cnn_final.values()]:
+
+        for j in i:
+
+            e_list.append(j)
+
+    for k in [k for k in npr_final.values()]:
+
+        for j in k:
+
+            e_list.append(j)
+
+        
+
+    # This is a dictionary with keys: the list items....
+
+    clusters = {k.lower():[] for k in e_list}
+
+    
+
+    ## Perform Clustering
+
+    for hed in cnn_final.keys():
+
+        for item in cnn_final[hed]:
+
+            clusters[item.lower()].append(hed) # placing the headline in the list corresponding to the dictionary key for each entity.
+
+    for hed in npr_final.keys():
+
+        for item in npr_final[hed]:
+
+            clusters[item.lower()].append(hed) 
+
+    
+
+    return clusters
+
+    
+
+    
+
+def update_topics():
+
+    st.legacy_caching.clear_cache()
+
+    dicts = [i for i in create_ner_dicts()]
+
+    clusters = get_news_topics(cnn_ner, npr_ner)
+
+    return clusters, dicts
+
+    
+
+    
+
+#############
+
+#   SETUP   #
+
+#############
+
+# Auth for HF Inference API and URL to the model we're using -- distilbart-cnn-12-6
+
+headers = {"Authorization": f"""Bearer {st.secrets["ato"]}"""}
+
+API_URL = "https://api-inference.huggingface.co/models/sshleifer/distilbart-cnn-12-6"
+
+NER_API_URL =  "https://api-inference.huggingface.co/models/dbmdz/bert-large-cased-finetuned-conll03-english"
+
+#############
+
+#PROCESSING #
+
+#############
+
+st.write(f"""**Welcome!**\nThis app lets you generate digests of topics currently in the news.  Select up to three current news topics and the digest lets you know what the latest news on those topics is!""") # Can I make this disappear?
+
+cnn_dict, npr_dict, cnn_ner, npr_ner = create_ner_dicts()
+
+clusters = get_news_topics(cnn_ner, npr_ner)
+
+selections = []
+
+choices = [None]
+
+for i in list(clusters.keys()):
+
+    choices.append(i)
+
+# button to refresh topics
+
+if st.button("Refresh topics!"):
+
+    new_data = update_topics()
+
+    clusters = new_data[0]
+
+    cnn_dict, npr_dict, cnn_ner, npr_ner = new_data[1][0], new_data[1][1], new_data[1][2], new_data[1][3]
+
+    
+
+# Form used to take 3 menu inputs
+
+with st.form(key='columns_in_form'):
+
+    cols = st.columns(3)
+
+    for i, col in enumerate(cols):
+
+        selections.append(col.selectbox(f'Make a Selection', choices, key=i))
+
+    submitted = st.form_submit_button('Submit')
+
+    if submitted:
+
+        selections = [i for i in selections if i is not None]
+
+        with st.spinner(text="Digesting...please wait, this may take up to 20 seconds..."):
+
+            digest = seek_and_sum(selections, cnn_dict, npr_dict)
+
+        if len(digest) == 0:
+
+            st.write("You didn't select a topic!")    
+
+        else:
+
+            st.write("Your digest is ready:\n")
+
+      
+
+            count = 0
+
+            for each in digest:
+
+                count += 1
+
+                st.write(each)