File size: 8,203 Bytes
ea5c8f0 5e6dc55 ea5c8f0 d58795c ea5c8f0 56ecfa3 ea5c8f0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 |
from re import split
import streamlit as st
import more_itertools
import spacy
from sentence_transformers import SentenceTransformer, util
st.set_page_config(
page_title="Semantic_Search/Summarization")
choclate_example = """You’d be hard-pressed to find anyone who doesn’t like chocolate. While it's mostly known for its taste (and the associated cravings), it’s also a good source of nutrients when in its pure form and eaten in moderation.
This well-loved food, once called the "drink of the gods" by the Maya people, has a rich history as well. Chocolate comes from the seeds of the cacao pod, which grows on the cacao tree. Theobroma cacao is native to the tropical rainforests of Central America, where it has grown for thousands of years.
"""
nlp = spacy.load('en_core_web_sm')
wit_text = """
. .
. .
.
. .. . .*/, . . .
.*&#&@@@&@@@@&%&&&@&(. .
*#&&@&&@@@@@@@@@%@@@@@@&,... . . . . .
#@@@@@@@@@@@@@@@@@@@@@@@@##... .. .. .
,&@&&@@&&&@@@@@%*........,&@@@&(,,...... .
.&&&@@&/,...... . ......(@@@@@&#...... . . .
.&@@&,... ... .. . ......#@@@@@&....... . .
.&@@&,... .. .............,,*@@@@@(....... . .
.@@@@/*..****,,*,*(&@@@@%(,...&@@@/....... . ....
,@@&,,/&(&@@@@(..(%(,&%,((,...&@&......... . . .. .
(@(..(%**,,(/....,,.............%/........ .. . .
../...........................(#.......... . . . .
.(.(...,....,/@@&&&#......,,,............ .. .....
..,.,,,,.....,,,*,........,..,.......... .. .. . .
../,**,,,.../@@@@@%,.,.,,,,............ . . . .
. .,**,,*#,**#%%/,,,,,,,,,,. ...... . .. .
. ..**********,......,,,**,. . ...... . .
. ....,/(/***,,,**,,,,*/((**,...... .. .
.. ....,/#&@@@@@@@@@@&%%#/**,........ .
....,/*,.,,**/#%&@@@@%#(%#/*,%..(*#(*..
.,(/*,(/#/(..,,,*///(////(((***,..*###%/%/(/,,. .
....,,*(((**/**/#(...,.,,*/(##/,,***/.@..%&#%%///(##/,*,,,. .
.....,,,*%(,&%(/////#*....,.,,***,,,*/(.@..&&%&#%/***#%(,(((*,*,,. ..
..*...,,,*,#%,*%(/*(/*#(##..&,..,,.,**,,*/,.#.&&&&&###((//%%/*****,,,,*.,,
...,,(*,,.,.*&&&%((**#((/(/&,.&..../(////,,..@@@@&&&&%(//,*(##(**/*,*,,/****,
,,/**/#(//*(%%/%@@@%(*/(/%##/,@.....,,*,...(@@@&%%(##(*/@@&((/(#/*/((%/*//***
,,/&/(#(###*/#&**#/%#///*//#//&@(... ....&@&&&###(%#####/%((*(#%%(#&&#%,((%&/
.,(,,*&%*/%&#(&((//#/,*/*/(/*//#&&/,.&.. ./&(((%%%#((%/(#(##,%*#/((%@%(##%&&*/(
.*,,**@#**%%&&#%%#(#/*/***/*/**(//&... , .%####%%&%(#/(%/(/(%&%#(#%@%%(&@(,*%%(
***(@%,,&@&##&%(@(/(//,,///(//*/**%.. ....#((###%&##(/%(&,(&@&#%###&&%@/@&(&%#&#
**/#&(((&&#@@@#%%&*/(**///(/,/,**/*.. ,*##/(((&%(%%#((%%&@@@#####&&@@&(%&@&#/(%%
(%#(*,%@@@(@@@&%%@&//(//(////**,/(#(,*(&%(%#%%&(%%#&(%(#@@@@&&%%%&@@@&/(&&&/*%&%
,*#&@@@@@&/(%@@%&&&%#(#//#/#/****##%*/*/*%&&%&&%%#%%(%@@@&@#%%&&@@@&&@@*#%&#(***
#(/***//%(((((&@&&&@&(/#(**(*((//**#/,/%&%&%&&%&##%%&&&@@@&#%&&&@@%(#*(*((%(%%&%
*,#%&&&@%&&&%(&@@&&&%&/%//(***/(///%(@&#&&&%&&&%&%@&&&##@@@&&&@@@&&((((###%(#/(*
/#**..,,*,/*#&@&@@&%&@%#(((#**(/(//%@@#&@#&&%#%%@@&&@@@@@@&@@@@@&#(###%##%%(%&#/
#%&%(,*,##&&&&&@@@&&@&&#*/*/(/*((//#@&(@&&%&&%&@&&&&@&@@@@@@&&@&&%#%(%(%#&%%#/(%
***,/((&%(/(**//&@@&@@@@(*/(,((#/((&@%%&#%&&&@&&#&&@@@@@@&@&@@@&((#&%#%%////(###
*,/*/#&&@@&@&*,%@@@@@@@&&(**#*(*#%#&@%%&%#&@@@@&@&&&@@@@@@@@@@@%%%%#//(*/#/#(%%%
#%#%&#&%%*,/##&@@@@@@&&&&&#(*#//#%&&&%&(/&@&@@@@@@&&&&@@@@@@@@@&%(%%(%(##%*//(%&
&&%(**/%&&&%&&@&@&@@@%&&@@@#####%&&@##//&%%%&@&&@&@@&@&&@@@@@@&&&%#/##(%##%&&&&#
**(#&%&%%&&%&&@@@@@@@@@@@@@@&%&%@@&&#%(@&&&&%&&&&@@@&&&@@@@@@@@/*%%%&&&&&%%%%##/
%&&%&%/***%&@@@&&@@@@@@&&@&&&@&&@@@%(%&&&&%%&&%&&&%@@@@@@@@@@@@&&&@&&%/,***,***/
The limits of my language means the limits of my world
- Ludwig Wittgenstein
"""
st.title("Unsupervised Extractive Text Summarization and Semantic Search - by Allen Roush!")
st.caption("Under active development! Please contact me or drop an issue on https://github.com/Hellisotherpeople/CX_DB8")
with st.expander("Dedicated to the late..."):
st.text(wit_text)
model_name = st.text_area("Enter the name of the pre-trained model from sentence transformers that we are using for summarization", value = "paraphrase-MiniLM-L3-v2")
st.caption("This will download a new model, so it may take awhile or even break if the model is too large")
st.caption("See the list of pre-trained models that are available here! https://www.sbert.net/docs/pretrained_models.html")
embedder = SentenceTransformer(model_name)
granularity = st.radio("What level of granularity do you want to summarize at?", ('Sentence', 'Word', 'Paragraph'))
y = st.text_area('enter a query', value = "I love chocolate")
x = st.text_area('Enter a document', value = choclate_example)
percentage = st.number_input("Enter the percentage of the text you want highlighted", max_value = 0.99, min_value = 0.01, value = 0.3)
query_embedding = embedder.encode(y, convert_to_tensor=True)
if granularity == "Sentence":
doc = nlp(x)
doc_sents = [str(sent) for sent in doc.sents]
corpus_embeddings = embedder.encode(doc_sents, convert_to_tensor=True)
len_doc = len(doc_sents)
elif granularity == "Word":
split_words = x.split()
ww_size = st.number_input("What size do you want the word window to be?", value = 3, min_value = 1, max_value=int(len(split_words)/4))
window_words = list(more_itertools.windowed(split_words, ww_size)) ###Doesn't seem to be impacting anything??? Must be related to implementation of semantic search by sentence transformers...
len_doc = len(window_words)
corpus_embeddings = embedder.encode(window_words, convert_to_tensor=True)
elif granularity == "Paragraph":
split_lines = x.splitlines()
corpus_embeddings = embedder.encode(split_lines, convert_to_tensor=True)
len_doc = len(split_lines)
semantic_search_results = util.semantic_search(query_embedding, corpus_embeddings, top_k= int(percentage * len_doc))
list_of_selected_extracts = []
for extract in semantic_search_results[0]:
list_of_selected_extracts.append(extract["corpus_id"])
if granularity == "Sentence":
original_granularity = doc_sents
elif granularity == "Word":
original_granularity = split_words
elif granularity == "Paragraph":
original_granularity = split_lines
string_to_print = ""
for count, chunk in enumerate(original_granularity):
if count in list_of_selected_extracts:
string_to_add = "\u0332".join(" " + chunk)
string_to_print += string_to_add
else:
string_to_print += " "
string_to_print += chunk
st.subheader("Output summary")
st.write(string_to_print)
st.subheader("Raw results")
st.caption("corpus_id is the number of the word, sentence, or paragraph. Score is the raw cosine similarty score between the document and the query")
st.write(semantic_search_results[0])
st.subheader("Results of segmentation/tokenization")
st.caption("This shows the representation that the webapp gets of the input document. Useful for debugging if you get strange output")
if granularity == "Sentence":
st.write(doc_sents)
elif granularity == "Word":
st.write(window_words)
elif granularity == "Paragraph":
st.write(split_lines) |