final basic app

.gitignore

@@ -0,0 +1,5 @@
+falgged/
+venv/
+
+test2.py/
+.idea/

.streamlit/config.toml

@@ -0,0 +1,34 @@
+[theme]
+#theme primary
+base="light"
+# Primary accent color for interactive elements.
+#primaryColor =
+
+# Background color for the main content area.
+#backgroundColor =
+
+# Background color used for the sidebar and most interactive widgets.
+#secondaryBackgroundColor ='grey'
+
+# Color used for almost all text.
+#textColor ='blue'
+
+# Font family for all text in the app, except code blocks. One of "sans serif", "serif", or "monospace".
+# Default: "sans serif"
+font = "sans serif"
+
+# [logger]
+# level='info'
+# messageFormat = "%(message)s"
+#messageFormat="%(asctime)s %(message)s"
+
+[global]
+
+# By default, Streamlit checks if the Python watchdog module is available and, if not, prints a warning asking for you to install it. The watchdog module is not required, but highly recommended. It improves Streamlit's ability to detect changes to files in your filesystem.
+# If you'd like to turn off this warning, set this to True.
+# Default: false
+disableWatchdogWarning = false
+
+# If True, will show a warning when you run a Streamlit-enabled script via "python my_script.py".
+# Default: true
+showWarningOnDirectExecution = false

app.py

@@ -0,0 +1,164 @@
+import logging
+import gradio as gr
+import streamlit as st
+import onnxruntime as ort
+import pke
+import nltk
+nltk.download('stopwords')
+nltk.download('wordnet')
+nltk.download('punkt')
+from nltk.corpus import stopwords,wordnet
+from nltk.tokenize import sent_tokenize
+from flashtext import KeywordProcessor
+import regex as re
+import string
+import subprocess
+from PIL import Image
+import multiprocessing
+total_threads=multiprocessing.cpu_count()
+
+st.set_page_config(  # Alternate names: setup_page, page, layout
+    layout="wide",  # Can be "centered" or "wide". In the future also "dashboard", etc.
+    initial_sidebar_state="auto",  # Can be "auto", "expanded", "collapsed"
+    page_title='None',  # String or None. Strings get appended with "• Streamlit".
+)
+
+def set_page_title(title):
+    st.sidebar.markdown(unsafe_allow_html=True, body=f"""
+        <iframe height=0 srcdoc="<script>
+            const title = window.parent.document.querySelector('title') \
+
+            const oldObserver = window.parent.titleObserver
+            if (oldObserver) {{
+                oldObserver.disconnect()
+            }} \
+
+            const newObserver = new MutationObserver(function(mutations) {{
+                const target = mutations[0].target
+                if (target.text !== '{title}') {{
+                    target.text = '{title}'
+                }}
+            }}) \
+
+            newObserver.observe(title, {{ childList: true }})
+            window.parent.titleObserver = newObserver \
+
+            title.text = '{title}'
+        </script>" />
+    """)
+
+
+set_page_title('Fill Blanks')
+
+def tokenize_sentence(text):
+    sentences=sent_tokenize(text)
+    sentences=[s.strip().lstrip().rstrip() for s in sentences if len(s) > 20]
+    return sentences
+
+
+def get_noun_adj_verb(text):
+    output = []
+    try:
+        extractor = pke.unsupervised.MultipartiteRank()
+        extractor.load_document(input=text, language='en',normalization=None)
+
+        # keyphrase candidate selection #'ADJ' 'ADP' 'ADV' 'AUX' 'DET' 'NOUN' 'NUM' 'PART' 'PROPN' 'PUNCT' 'VERB'
+        extractor.candidate_selection(pos={'NOUN', 'VERB', 'ADJ'})
+
+        # candidate weighting,
+        extractor.candidate_weighting(threshold=0.74,method='average',alpha=1.1)
+
+        #extract top n
+        keyphrases = extractor.get_n_best(n=5)
+
+        for val in keyphrases:
+            output.append(val[0])
+    except Exception as e:
+        pass
+    return output
+
+
+
+def get_keywords_sentence(keywords,tokenized_sent):
+    keyword_sent_dict = {}
+
+    for k in keywords:
+        keyword_sent_dict[k]=[]
+        for s in tokenized_sent:
+            if k in s:
+                keyword_sent_dict[k].append(s)
+
+    #sort sentnece in decreasing order of length
+    for k in keyword_sent_dict.keys():
+        values=keyword_sent_dict[k]
+        values=sorted(values,key=len,reverse=True)
+        keyword_sent_dict[k]=values
+
+    return keyword_sent_dict
+
+
+
+def create_blanks(keyword_sentence_dict):
+    answer=[]
+    fib=[]
+    for k in keyword_sentence_dict.keys():
+        sent=keyword_sentence_dict[k][0]#take 1st sentence
+        sent=re.sub(k,'____________',sent)
+        answer.append(k)
+        fib.append(sent)
+    return answer,fib
+
+#title using markdown
+st.markdown("<h1 style='text-align: center; color: #3366ff;'>Create Fill The Blanks Questions</h1>", unsafe_allow_html=True)
+st.markdown("---")
+with st.sidebar:
+    # title using markdown
+    st.markdown("<h1 style='text-align: left; color: ;'>NLP Tasks</h1>", unsafe_allow_html=True)
+    select_task=st.selectbox(label="Select task from drop down menu",
+                 options=['README',
+                          'Basic Fill Blanks'])
+
+default_paratext = """On May 4, the Red Planet was rocked by a roughly magnitude 5 temblor, the largest Marsquake detected to date, NASA’s Jet Propulsion Laboratory in Pasadena, Calif., reports. The shaking lasted for more than six hours and released more than 10 times the energy of the previous record-holding quake.The U.S. space agency’s InSight lander, which has been studying Mars’ deep interior since touching down on the planet in 2018 (SN: 11/26/18), recorded the event. The quake probably originated near the Cerberus Fossae region, which is more than 1,000 kilometers from the lander.Cerberus Fossae is known for its fractured surface and frequent rockfalls. It makes sense that the ground would be shifting there, says geophysicist Philippe Lognonné, principal investigator of the Seismic Experiment for Interior Structure, InSight’s seismometer. “It’s an ancient volcanic bulge.Just like earthquakes reveal information about our planet’s interior structure, Marsquakes can be used to probe what lies beneath Mars’ surface (SN: 7/22/21). And a lot can be learned from studying this whopper of a quake, says Lognonné, of the Institut de Physique du Globe de Paris. “The signal is so good, we’ll be able to work on the details."""
+
+
+
+img = Image.open("hf_space1.png")
+if select_task=='README':
+    st.header("Summary")
+    st.write(f"The App gives you ability to create *Fill in the blanks* Capability just like Ed-Techs.Currently.It has {total_threads} CPU cores but only 1 is available per user so "
+             f"inference time will be on the higher side.")
+    st.markdown("---")
+    st.image(img)
+
+if select_task=='Basic Fill Blanks':
+    input_text = st.text_area(label='Input paragraph', height=500, max_chars=2000, value=default_paratext)
+    create_fib=st.button("Create Questions")
+
+    tokenized_sent = tokenize_sentence(input_text)
+    keywords_noun_adj_verb=get_noun_adj_verb(input_text)
+    keyword_sent_noun_verb_adj=get_keywords_sentence(keywords=keywords_noun_adj_verb,tokenized_sent=tokenized_sent)
+
+    answer,fib=create_blanks(keyword_sentence_dict=keyword_sent_noun_verb_adj)
+
+    if create_fib:
+        for i,(answer,fib) in enumerate(zip(answer,fib)):
+            st.markdown(f"* {fib} | **Answer is *{answer}* ** ",unsafe_allow_html=True)
+
+
+
+# demo = gr.Interface(fn=get_noun_adj_verb,
+#                     inputs=gr.inputs.Textbox(lines=10,default=default_paratext),
+#                     outputs=gr.outputs.Textbox(),
+#                     allow_flagging='never',
+#                     layout='vertical',
+#                     title="Make Fill in the Blanks using your text",
+#                     )
+#
+# if __name__ == "__main__":
+#     demo.launch()
+
+
+
+
+
+

flagged/log.csv

@@ -0,0 +1,3 @@
+'name','output','timestamp'
+'','','2022-05-16 01:13:57.520111'
+'On May 4, the Red Planet was rocked by a roughly magnitude 5 temblor, the largest Marsquake detected to date, NASA’s Jet Propulsion Laboratory in Pasadena, Calif., reports. The shaking lasted for more than six hours and released more than 10 times the energy of the previous record-holding quake.The U.S. space agency’s InSight lander, which has been studying Mars’ deep interior since touching down on the planet in 2018 (SN: 11/26/18), recorded the event. The quake probably originated near the Cerberus Fossae region, which is more than 1,000 kilometers from the lander.Cerberus Fossae is known for its fractured surface and frequent rockfalls. It makes sense that the ground would be shifting there, says geophysicist Philippe Lognonné, principal investigator of the Seismic Experiment for Interior Structure, InSight’s seismometer. “It’s an ancient volcanic bulge.Just like earthquakes reveal information about our planet’s interior structure, Marsquakes can be used to probe what lies beneath Mars’ surface (SN: 7/22/21). And a lot can be learned from studying this whopper of a quake, says Lognonné, of the Institut de Physique du Globe de Paris. “The signal is so good, we’ll be able to work on the details.','','2022-05-16 01:28:38.173283'

requirements.txt

@@ -0,0 +1,4 @@
+streamlit~=1.9.0
+onnxruntime~=1.11.1
+Pillow~=9.1.0
+git+https://github.com/boudinfl/pke.git

test.py

@@ -0,0 +1,98 @@
+from collections import defaultdict
+
+from pke.data_structures import Candidate
+from pke.readers import RawTextReader, SpacyDocReader, PreprocessedReader
+
+from nltk import RegexpParser
+from nltk.stem.snowball import SnowballStemmer
+import pke
+from pke.lang import stopwords, langcodes
+
+from string import punctuation
+import os
+import logging
+import spacy
+
+nlp = spacy.load("en_core_web_sm")
+
+from pke.base import LoadFile
+class LoadFileNew(LoadFile):
+
+    def load_document(self, input, language=None, stoplist=None,
+                      normalization='stemming', spacy_model=nlp):
+        # Reset object for new document
+        self.__init__()
+
+        # get the language parameter
+        if language is None:
+            language = 'en'
+
+        # set the language of the document
+        self.language = language
+
+        # word normalization (filling self.sentences[].stems)
+        self.normalization = normalization
+
+        # initialize the stoplist
+        if stoplist:
+            self.stoplist = stoplist
+        else:
+            self.stoplist = stopwords.get(self.language)
+
+        # check whether input is a spacy doc object instance
+        if isinstance(input, spacy.tokens.doc.Doc):
+            parser = SpacyDocReader()
+            sents = parser.read(spacy_doc=input)
+            logging.error('check whether input is a spacy doc object instance')
+        # check whether input is a string
+        elif isinstance(input, str):
+            parser = RawTextReader(language=self.language)
+            sents = parser.read(text=input, spacy_model=spacy_model)
+            logging.error('check whether input is a string')
+        # check whether input is processed text
+        elif isinstance(input, list) and all(isinstance(item, list) for item in input):
+            parser = PreprocessedReader()
+            sents = parser.read(list_of_sentence_tuples=input)
+            logging.error('check whether input is processed text')
+        else:
+            logging.error('Cannot process input. It is neither a spacy doc or a string: {}'.format(type(input)))
+            # TODO raise TypeError('Cannot process input. It is neither a spacy doc, a string or a list of tuple: {}'.format(type(input)))) ?
+            return
+
+        # populate the sentences
+        self.sentences = sents
+
+        # TODO: this code could go into Reader.normalize ? Hum, not sure
+        if self.normalization == 'stemming':
+            # fall back to porter if english language (or unavailable languages) is used
+            try:
+                langcode = langcodes.get(self.language)
+                if langcode == "english":
+                    langcode = 'porter'
+                stemmer = SnowballStemmer(langcode)
+            except ValueError:
+                logging.error('No stemmer available for \'{}\' language -> fall back to porter.'.format(self.language))
+                stemmer = SnowballStemmer("porter")
+
+            # populate Sentence.stems
+            for i, sentence in enumerate(self.sentences):
+                self.sentences[i].stems = [stemmer.stem(w).lower() for w in sentence.words]
+
+        else:
+            for i, sentence in enumerate(self.sentences):
+                self.sentences[i].stems = [w.lower() for w in sentence.words]
+
+        return self.sentences
+
+#
+test = LoadFileNew()
+text="On May 4, the Red Planet was rocked by a roughly magnitude 5 temblor, the largest Marsquake detected to date, NASA’s Jet Propulsion Laboratory in Pasadena, Calif., reports. The shaking lasted for more than six hours and released more than 10 times the energy of the previous record-holding quake.The U.S. space agency’s InSight lander, which has been studying Mars’ deep interior since touching down on the planet in 2018 (SN: 11/26/18), recorded the event. The quake probably originated near the Cerberus Fossae region, which is more than 1,000 kilometers from the lander.Cerberus Fossae is known for its fractured surface and frequent rockfalls. It makes sense that the ground would be shifting there, says geophysicist Philippe Lognonné, principal investigator of the Seismic Experiment for Interior Structure, InSight’s seismometer. “It’s an ancient volcanic bulge.Just like earthquakes reveal information about our planet’s interior structure, Marsquakes can be used to probe what lies beneath Mars’ surface (SN: 7/22/21). And a lot can be learned from studying this whopper of a quake, says Lognonné, of the Institut de Physique du Globe de Paris. “The signal is so good, we’ll be able to work on the details."
+out=test.load_document(input=text)
+
+
+
+
+
+
+
+

test2.py

@@ -0,0 +1,38 @@
+import pke
+import nltk
+nltk.download('stopwords')
+nltk.download('wordnet')
+nltk.download('punkt')
+from nltk.corpus import stopwords,wordnet
+import string
+
+
+def get_noun_adj_verb(text):
+    output = []
+    try:
+        # initialize keyphrase extraction model, MultipartiteRank is the most recent model
+        extractor = pke.unsupervised.MultipartiteRank()
+        extractor.load_document(input=text, language='en',normalization=None)
+
+        stoplist = list(string.punctuation)
+        stoplist += ['-lrb-', '-rrb-', '-lcb-', '-rcb-', '-lsb-', '-rsb-']
+        stoplist += stopwords.words('english')
+
+        # keyphrase candidate selection #'ADJ' 'ADP' 'ADV' 'AUX' 'DET' 'NOUN' 'NUM' 'PART' 'PROPN' 'PUNCT' 'VERB' e.g POS={'VERB}
+        extractor.candidate_selection(pos={'NOUN', 'PROPN', 'ADJ'})
+        # candidate weighting,
+        extractor.candidate_weighting(threshold=0.74,alpha=1.1,method='average')
+
+        keyphrases = extractor.get_n_best(n=5)
+
+        for val in keyphrases:
+            output.append(val[0])
+
+    except Exception as e:
+        return e
+
+    return output
+
+default_paratext = """On May 4, the Red Planet was rocked by a roughly magnitude 5 temblor, the largest Marsquake detected to date, NASA’s Jet Propulsion Laboratory in Pasadena, Calif., reports. The shaking lasted for more than six hours and released more than 10 times the energy of the previous record-holding quake.The U.S. space agency’s InSight lander, which has been studying Mars’ deep interior since touching down on the planet in 2018 (SN: 11/26/18), recorded the event. The quake probably originated near the Cerberus Fossae region, which is more than 1,000 kilometers from the lander.Cerberus Fossae is known for its fractured surface and frequent rockfalls. It makes sense that the ground would be shifting there, says geophysicist Philippe Lognonné, principal investigator of the Seismic Experiment for Interior Structure, InSight’s seismometer. “It’s an ancient volcanic bulge.Just like earthquakes reveal information about our planet’s interior structure, Marsquakes can be used to probe what lies beneath Mars’ surface (SN: 7/22/21). And a lot can be learned from studying this whopper of a quake, says Lognonné, of the Institut de Physique du Globe de Paris. “The signal is so good, we’ll be able to work on the details."""
+
+print(type(get_noun_adj_verb(default_paratext)))