Upload 8 files

Dockerfile

@@ -0,0 +1,76 @@
+#Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+ENV PYSPARK_PYTHON=python3
+ENV PYSPARK_DRIVER_PYTHON=python3
+
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz
+
+RUN adduser --disabled-password \
+    --gecos "Default user" \
+    --uid ${NB_UID} \
+    ${NB_USER}
+
+# Install OpenJDK-8
+RUN apt-get update && \
+    apt-get install -y openjdk-8-jdk && \
+    apt-get install -y ant && \
+    apt-get clean;
+
+# Fix certificate issues
+RUN apt-get update && \
+    apt-get install ca-certificates-java && \
+    apt-get clean && \
+    update-ca-certificates -f;
+# Setup JAVA_HOME -- useful for docker commandline
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN export JAVA_HOME
+
+RUN echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ~/.bashrc
+
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN apt-get update
+RUN apt-get install -y software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get install -y python3.8 python3-pip
+
+ENV PYSPARK_PYTHON=python3.8
+ENV PYSPARK_DRIVER_PYTHON=python3.8
+
+COPY . .
+
+RUN python3.8 -m pip install --upgrade pip
+RUN python3.8 -m pip install -r requirements.txt
+
+USER root
+RUN chown -R ${NB_UID} ${HOME}
+USER ${NB_USER}
+
+WORKDIR ${HOME}
+
+COPY . .
+
+EXPOSE 7860
+
+ENTRYPOINT ["streamlit", "run", "Home.py", "--server.port=7860", "--server.address=0.0.0.0"]

Home.py

@@ -0,0 +1,104 @@
+import streamlit as st
+
+# Configure Streamlit page
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #333333;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Home page content
+st.markdown('<div class="main-title">Spark NLP: State-of-the-Art Natural Language Processing</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>Spark NLP is a state-of-the-art Natural Language Processing library built on top of Apache Spark. It provides performant & accurate NLP annotations for machine learning pipelines that scale in a distributed environment. With 36,000+ pretrained models and pipelines in over 200 languages, Spark NLP supports a wide range of NLP tasks including:</p>
+    <ul>
+        <li>Tokenization</li>
+        <li>Named Entity Recognition</li>
+        <li>Sentiment Analysis</li>
+        <li>Text Classification</li>
+        <li>Machine Translation</li>
+        <li>Summarization</li>
+        <li>Question Answering</li>
+        <li>Text Generation</li>
+        <li>Image Classification</li>
+        <li>Automatic Speech Recognition</li>
+        <li>And many more</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Key Features</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li>Integration with popular transformers like BERT, RoBERTa, ALBERT, and more</li>
+        <li>Support for Python, R, and JVM (Java, Scala, Kotlin)</li>
+        <li>GPU support for accelerated processing</li>
+        <li>Easy integration with Spark ML functions</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Quick Links</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/docs/en/quickstart" target="_blank">Getting Started</a></li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/models" target="_blank">Pretrained Models</a></li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp/tree/master/examples/python/annotation/text/english" target="_blank">Example Notebooks</a></li>
+        <li><a class="link" href="https://sparknlp.org/docs/en/install" target="_blank">Installation Guide</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

pages/Named Entity Recognition.py

@@ -0,0 +1,272 @@
+import streamlit as st
+import os
+import json
+import pandas as pd
+import numpy as np
+import random
+import base64
+import ast
+import sparknlp
+import pyspark.sql.functions as F
+from pyspark.ml import Pipeline
+from pyspark.sql import SparkSession
+from sparknlp.annotator import *
+from sparknlp.base import *
+from sparknlp.pretrained import PretrainedPipeline
+from pyspark.sql.types import StringType, IntegerType
+
+@st.cache_resource
+def init_spark():
+    spark = sparknlp.start()
+    return spark
+
+@st.cache_resource
+def create_pipeline(model):
+    documentAssembler = DocumentAssembler() \
+        .setInputCol('text') \
+        .setOutputCol('document')
+
+    tokenizer = Tokenizer() \
+        .setInputCols(['document']) \
+        .setOutputCol('token')
+
+    embeddings = WordEmbeddingsModel.pretrained('glove_100d') \
+        .setInputCols(["document", 'token']) \
+        .setOutputCol("embeddings")
+
+    ner_model = NerDLModel.pretrained(model, 'en') \
+        .setInputCols(['document', 'token', 'embeddings']) \
+        .setOutputCol('ner')
+
+    ner_converter = NerConverter() \
+        .setInputCols(['document', 'token', 'ner']) \
+        .setOutputCol('ner_chunk')
+
+    nlp_pipeline = Pipeline(stages=[
+        documentAssembler, 
+        tokenizer,
+        embeddings,
+        ner_model,
+        ner_converter
+    ])
+
+    return nlp_pipeline
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    result = model.fullAnnotate(data)
+    return result
+
+def get_color(l):
+    if str(l).lower() in LABEL_COLORS.keys():
+        return LABEL_COLORS[l.lower()]
+    else:
+        r = lambda: random.randint(0,200)
+        return '#%02X%02X%02X' % (r(), r(), r())
+
+def simplified_display_annotations(text, annotations, labels):
+    def get_html_for_span(span_text, background_color=None, entity_name=None):
+        if background_color:
+            style = f'style="background-color: {background_color};"'
+        else:
+            style = ""
+        if entity_name:
+            return f'<span {style}>{span_text} <span>({entity_name})</span></span>'
+        return f'<span {style}>{span_text}</span>'
+
+    label_color = {label: get_color(label) for label in labels}
+    html_output, pos = "", 0
+
+    for chunk in annotations:
+        begin, end = chunk[1], chunk[2]
+        entity_type = chunk[4]['entity']
+
+        if pos < begin:
+            html_output += get_html_for_span(text[pos:begin])
+        html_output += get_html_for_span(chunk[3], label_color.get(entity_type), entity_type if entity_type in label_color else None)
+        pos = end + 1
+
+    if pos < len(text):
+        html_output += get_html_for_span(text[pos:])
+
+    return html_output
+
+def parse_text_to_complex_list(text):
+    text = text.strip()[11:-1]
+    def parse_inner_structure(s):
+        stack = []
+        result = ''
+        for char in s:
+            if char in ['{', '[']:
+                stack.append(char)
+            elif char in ['}', ']']:
+                stack.pop()
+            if not stack:
+                return result + char
+            result += char
+        return result
+
+    elements = []
+    temp = ''
+    stack = []
+    for char in text:
+        if char in ['{', '[']:
+            stack.append(char)
+        elif char in [']', '}']:
+            stack.pop()
+        elif char == ',' and not stack:
+            elements.append(temp.strip())
+            temp = ''
+            continue
+        temp += char
+    elements.append(temp.strip())
+
+    # Convert elements to the appropriate type
+    parsed_elements = []
+    for element in elements:
+        element = element.strip()
+        if element.isdigit():
+            parsed_elements.append(int(element))
+        elif element.startswith(('\'', '"')):
+            parsed_elements.append(element.strip('\'"'))
+        elif element.startswith(('{', '[')):
+            parsed_elements.append(ast.literal_eval(element))
+        else:
+            parsed_elements.append(element)
+
+    return parsed_elements
+
+############ SETTING UP THE PAGE LAYOUT ############
+### SIDEBAR CONTENT ###
+
+language_info = {
+    "EN": {
+        "title": "Recognize entities in text",
+        "description": "Recognize Persons, Locations, Organizations and Misc entities using out of the box pretrained Deep Learning models based on BERT (ner_dl_bert) word embeddings.",
+    }
+}
+
+model = st.sidebar.selectbox("Choose the pretrained model",  ["ner_dl_bert", "ner_dl"], help="For more info about the models visit: https://sparknlp.org/models")
+
+st.title(language_info["EN"]["title"])
+
+link = """<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb">
+          <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/></a>"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+### MAIN CONTENT ###
+
+examples = [
+        "William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000.[9] He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.",
+        "The Mona Lisa is a 16th century oil painting created by Leonardo. It's held at the Louvre in Paris.",
+        "When Sebastian Thrun started working on self-driving cars at Google in 2007, few people outside of the company took him seriously. “I can tell you very senior CEOs of major American car companies would shake my hand and turn away because I wasn’t worth talking to,” said Thrun, now the co-founder and CEO of online higher education startup Udacity, in an interview with Recode earlier this week.",
+        "Facebook is a social networking service launched as TheFacebook on February 4, 2004. It was founded by Mark Zuckerberg with his college roommates and fellow Harvard University students Eduardo Saverin, Andrew McCollum, Dustin Moskovitz and Chris Hughes. The website's membership was initially limited by the founders to Harvard students, but was expanded to other colleges in the Boston area, the Ivy League, and gradually most universities in the United States and Canada.",
+        "The history of natural language processing generally started in the 1950s, although work can be found from earlier periods. In 1950, Alan Turing published an article titled 'Computing Machinery and Intelligence' which proposed what is now called the Turing test as a criterion of intelligence",
+        "Geoffrey Everest Hinton is an English Canadian cognitive psychologist and computer scientist, most noted for his work on artificial neural networks. Since 2013 he divides his time working for Google and the University of Toronto. In 2017, he cofounded and became the Chief Scientific Advisor of the Vector Institute in Toronto.",
+        "When I told John that I wanted to move to Alaska, he warned me that I'd have trouble finding a Starbucks there.",
+        "Steven Paul Jobs was an American business magnate, industrial designer, investor, and media proprietor. He was the chairman, chief executive officer (CEO), and co-founder of Apple Inc., the chairman and majority shareholder of Pixar, a member of The Walt Disney Company's board of directors following its acquisition of Pixar, and the founder, chairman, and CEO of NeXT. Jobs is widely recognized as a pioneer of the personal computer revolution of the 1970s and 1980s, along with Apple co-founder Steve Wozniak. Jobs was born in San Francisco, California, and put up for adoption. He was raised in the San Francisco Bay Area. He attended Reed College in 1972 before dropping out that same year, and traveled through India in 1974 seeking enlightenment and studying Zen Buddhism.",
+        "Titanic is a 1997 American epic romance and disaster film directed, written, co-produced, and co-edited by James Cameron. Incorporating both historical and fictionalized aspects, it is based on accounts of the sinking of the RMS Titanic, and stars Leonardo DiCaprio and Kate Winslet as members of different social classes who fall in love aboard the ship during its ill-fated maiden voyage.",
+        "Other than being the king of the north, John Snow is a an english physician and a leader in the development of anaesthesia and medical hygiene. He is considered for being the first one using data to cure cholera outbreak in 1834."
+    ]
+
+st.subheader(language_info["EN"]["description"])
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it for yourself!")
+
+if custom_input:
+    selected_text = custom_input
+
+st.subheader('Selected Text')
+st.write(selected_text)
+
+spark = init_spark()
+Pipeline = create_pipeline(model)
+output = fit_data(Pipeline, selected_text)
+
+ner_mapping = {
+    'PERSON': 'People, including fictional.',
+    'PER': 'People, including fictional.',
+    'HUM': 'Humans',
+    'IND': 'Individuals',
+    'NORP': 'Nationalities or religious or political groups.',
+    'FAC': 'Buildings, airports, highways, bridges, etc.',
+    'FACILITY': 'Buildings, airports, highways, bridges, etc.',
+    'STRUCTURE': 'Structures like buildings, bridges, etc.',
+    'ORG': 'Companies, agencies, institutions, etc.',
+    'ORGANIZATION': 'Companies, agencies, institutions, etc.',
+    'INSTITUTION': 'Educational, governmental, and other organizations.',
+    'LOC': 'Countries, cities, states, mountain ranges, bodies of water.',
+    'LOCATION': 'Countries, cities, states, mountain ranges, bodies of water.',
+    'PLACE': 'Specific locations.',
+    'GPE': 'Geopolitical entities, such as countries, cities, states.',
+    'PRODUCT': 'Objects, vehicles, foods, etc. (Not services.)',
+    'PROD': 'Product',
+    'GOOD': 'Goods and products.',
+    'EVENT': 'Named hurricanes, battles, wars, sports events, etc.',
+    'OCCURRENCE': 'Occurrences and events.',
+    'WORK_OF_ART': 'Titles of books, songs, etc.',
+    'ART': 'Works of art, including books, paintings, songs, etc.',
+    'LAW': 'Named documents made into laws.',
+    'LEGISLATION': 'Laws and legal documents.',
+    'LANGUAGE': 'Any named language.',
+    'DATE': 'Absolute or relative dates or periods.',
+    'TIME': 'Times smaller than a day.',
+    'PERCENT': 'Percentage, including ”%“.',
+    'MONEY': 'Monetary values, including unit.',
+    'CURRENCY': 'Monetary values, including unit.',
+    'QUANTITY': 'Measurements, as of weight or distance.',
+    'MEASURE': 'Measurements and quantities.',
+    'ORDINAL': '“first”, “second”, etc.',
+    'CARDINAL': 'Numerals that do not fall under another type.',
+    'NUMBER': 'Numbers and numerals.',
+    'MISC': 'Miscellaneous entities, e.g. events, nationalities, products or works of art.',
+    'MISCELLANEOUS': 'Miscellaneous entities.',
+    'ENT': 'Entity (generic label).',
+    'GPE_LOC': 'Geopolitical Entity',
+    'ANIMAL': 'Animals, including fictional.',
+    'PLANT': 'Plants, including fictional.',
+    'SUBSTANCE': 'Substances and materials.',
+    'DISEASE': 'Diseases and medical conditions.',
+    'SYMPTOM': 'Symptoms and medical signs.',
+    'MEDICAL': 'Medical terms and conditions.',
+    'FOOD': 'Food items.',
+    'DRINK': 'Drinks and beverages.',
+    'VEHICLE': 'Types of vehicles.',
+    'WEAPON': 'Weapons and armaments.',
+    'TECHNOLOGY': 'Technological terms and devices.',
+    'GAME': 'Games and sports.',
+    'HOBBY': 'Hobbies and recreational activities.',
+    'RELIGION': 'Religious terms and entities.',
+    'MYTH': 'Mythological entities.',
+    'ASTRONOMICAL': 'Astronomical entities (e.g., planets, stars).',
+    'NATURAL_PHENOMENON': 'Natural phenomena (e.g., earthquakes, storms).',
+    'CELESTIAL_BODY': 'Celestial bodies (e.g., stars, planets).',
+    'DRV': 'Driver'
+}
+
+entities = [{'text': ent.result, 'start': ent.begin, 'end': ent.end, 'label': ent.metadata['entity'], 'Explain NER Labels': ner_mapping[ent.metadata['entity']]} for ent in output[0]['ner_chunk']]
+
+LABEL_COLORS = {
+    'per': '#0C8888', 'pers': '#0C8888', 'person': '#0C8888',
+    'org': '#FF33C1',
+    'misc': '#3196D4', 'mis': '#3196D4',
+    'loc': '#5B00A3', 'location': '#5B00A3'
+}
+
+HTML_WRAPPER = """<div class="scroll entities" style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem; white-space:pre-wrap">{}</div>"""
+
+chunk_list = []
+
+for n in output[0]['ner_chunk']:
+    parsed_list = parse_text_to_complex_list(str(n))
+    chunk_list.append(parsed_list)
+
+st.subheader("Text annotated with identified Named Entities")
+
+labels = [n.metadata['entity'] for n in output[0]['ner_chunk']]
+ner = simplified_display_annotations(selected_text, chunk_list, labels)
+st.markdown(HTML_WRAPPER.format(ner), unsafe_allow_html=True)
+st.write(pd.DataFrame(entities))

pages/Sentiment Analysis.py

@@ -0,0 +1,117 @@
+import streamlit as st
+import sparknlp
+import os
+from sparknlp.base import *
+from sparknlp.common import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+
+@st.cache_resource
+def init_spark():
+    spark = sparknlp.start()
+    return spark
+
+@st.cache_resource
+def create_pipeline(model):
+    documentAssembler = DocumentAssembler()\
+        .setInputCol("text")\
+        .setOutputCol("document")
+
+    use = UniversalSentenceEncoder.pretrained("tfhub_use", "en")\
+        .setInputCols(["document"])\
+        .setOutputCol("sentence_embeddings")
+
+    sentimentdl = SentimentDLModel.pretrained(model, "en")\
+        .setInputCols(["sentence_embeddings"])\
+        .setOutputCol("sentiment")
+
+    nlpPipeline = Pipeline(stages=[documentAssembler, use, sentimentdl])
+
+    return nlpPipeline
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    results = model.fullAnnotate(data)[0]
+
+    return results['sentiment'][0].result
+
+############ SETTING UP THE PAGE LAYOUT ############
+
+st.title("Analyze sentiment in movie reviews and tweets")
+
+### SIDEBAR CONTENT ###
+
+model_list = ["sentimentdl_use_imdb", "sentimentdl_use_twitter"]
+model = st.sidebar.selectbox(
+    "Choose the pretrained model", 
+    model_list, 
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Let's add the colab link for the notebook.
+
+link = """<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SENTIMENT_EN.ipynb">
+<img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/></a>"""
+st.sidebar.title('')
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+### MAIN CONTENT ###
+
+examples = {
+    "IMDB": [
+        """Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music was rad! Horror and sword fight freaks,buy this movie now!""",
+        """Back when Alec Baldwin and Kim Basinger were a mercurial, hot-tempered, high-powered Hollywood couple they filmed this (nearly) scene-for-scene remake of the 1972 Steve McQueen-Ali MacGraw action-thriller about a fugitive twosome. It almost worked the first time because McQueen was such a vital presence on the screen--even stone silent and weary, you could sense his clock ticking, his cagey magnetism. Baldwin is not in Steve McQueen's league, but he has his charms and is probably a more versatile actor--if so, this is not a showcase for his attributes. Basinger does well and certainly looks good, but James Woods is artificially hammy in a silly mob-magnet role. A sub-plot involving another couple taken hostage by Baldwin's ex-partner was unbearable in the '72 film and plays even worse here. As for the action scenes, they're pretty old hat, which causes one to wonder: why even remake the original?""",
+        """Despite a tight narrative, Johnnie To's Election feels at times like it was once a longer picture, with many characters and plot strands abandoned or ultimately unresolved. Some of these are dealt with in the truly excellent and far superior sequel, Election 2: Harmony is a Virtue, but it's still a dependably enthralling thriller about a contested Triad election that bypasses the usual shootouts and explosions (though not the violence) in favour of constantly shifting alliances that can turn in the time it takes to make a phone call. It's also a film where the most ruthless character isn't always the most threatening one, as the chilling ending makes only too clear: one can imagine a lifetime of psychological counselling being necessary for all the trauma that one inflicts on one unfortunate bystander. Simon Yam, all too often a variable actor but always at his best under To's direction, has possibly never been better in the lead, not least because Tony Leung's much more extrovert performance makes his stillness more the powerful.""",
+        """This movie has successfully proved what we all already know, that professional basket-ball players suck at everything besides playing basket-ball. Especially rapping and acting. I can not even begin to describe how bad this movie truly is. First of all, is it just me, or is that the ugliest kid you have ever seen? I mean, his teeth could be used as a can-opener. Secondly, why would a genie want to pursue a career in the music industry when, even though he has magical powers, he sucks horribly at making music? Third, I have read the Bible. In no way shape or form did it say that Jesus made genies. Fourth, what was the deal with all the crappy special effects? I assure you that any acne-addled nerdy teenager with a computer could make better effects than that. Fifth, why did the ending suck so badly? And what the hell is a djin? And finally, whoever created the nightmare known as Kazaam needs to be thrown off of a plane and onto the Eiffel Tower, because this movie take the word "suck" to an entirely new level.""",
+        """The fluttering of butterfly wings in the Atlantic can unleash a hurricane in the Pacific. According to this theory (somehow related to the Chaos Theory, I'm not sure exactly how), every action, no matter how small or insignificant, will start a chain reaction that can lead to big events. This small jewel of a film shows us a series of seemingly-unrelated characters, most of them in Paris, whose actions will affect each others' lives. (The six-degrees-of-separation theory can be applied as well.) Each story is a facet of the jewel that is this film. The acting is finely-tuned and nuanced (Audrey Tautou is luminous), the stories mesh plausibly, the humor is just right, and the viewer leaves the theatre nodding in agreement.""",
+        """There have been very few films I have not been able to sit through. I made it through Battle Field Earth no problem. But this, This is one of the single worst films EVER to be made. I understand Whoopi Goldberg tried to get of acting in it. I do not blame her. I would feel ashamed to have this on a resume. I belive it is a rare occasion when almost every gag in a film falls flat on it's face. Well it happens here. Not to mention the SFX, look for the dino with the control cables hanging out of it rear end!!!!!! Halfway through the film I was still looking for a plot. I never found one. Save yourself the trouble of renting this and save 90 minutes of your life.""",
+        """After a long hard week behind the desk making all those dam serious decisions this movie is a great way to relax. Like Wells and the original radio broadcast this movie will take you away to a land of alien humor and sci-fi paraday. 'Captain Zippo died in the great charge of the Buick. He was a brave man.' The Jack Nicholson impressions shine right through that alien face with the dark sun glasses and leather jacket. And always remember to beware of the 'doughnut of death!' Keep in mind the number one rule of this movie - suspension of disbelief - sit back and relax - and 'Prepare to die Earth Scum!' You just have to see it for yourself.""",
+        """When Ritchie first burst on to movie scene his films were hailed as funny, witty, well directed and original. If one could compare the hype he had generated with his first two attempts and the almost universal loathing his last two outings have created one should consider - has Ritchie been found out? Is he really that talented? Does he really have any genuine original ideas? Or is he simply a pretentious and egotistical director who really wants to be Fincher, Tarantino and Leone all rolled into one colossal and disorganised heap? After watching Revolver one could be excused for thinking were did it all go wrong? What happened to his great sense of humour? Where did he get all these mixed and convoluted ideas from? Revolver tries to be clever, philosophical and succinct, it tries to be an intelligent psychoanalysis, it tries to be an intricate and complicated thriller. Ritchie does make a gargantuan effort to fulfil all these many objectives and invests great chunks of a script into existential musings and numerous plot twists. However, in the end all it serves is to construct a severely disjointed, unstructured and ultimately unfriendly film to the audience. Its plagiarism is so sinful and blatant that although Ritchie does at least attempt to give his own spin he should be punished for even trying to pass it off as his own work. So what the audience gets ultimately is a terrible screenplay intertwined with many pretentious oneliners and clumsy setpieces.<br /><br />Revolver is ultimately an unoriginal and bland movie that has stolen countless themes from masterpieces like Fight Club, Usual Suspects and Pulp Fiction. It aims high, but inevitably shots blanks aplenty.<br /><br />Revolver deserves to be lambasted, it is a truly poor film masquerading as a wannabe masterpiece from a wannabe auteur. However, it falls flat on its farcical face and just fails at everything it wants to be and achieve.""",
+        """I always thought this would be a long and boring Talking-Heads flick full of static interior takes, dude, I was wrong. "Election" is a highly fascinating and thoroughly captivating thriller-drama, taking a deep and realistic view behind the origins of Triads-Rituals. Characters are constantly on the move, and although as a viewer you kinda always remain an outsider, it's still possible to feel the suspense coming from certain decisions and ambitions of the characters. Furthermore Johnnie To succeeds in creating some truly opulent images due to meticulously composed lighting and atmospheric light-shadow contrasts. Although there's hardly any action, the ending is still shocking in it's ruthless depicting of brutality. Cool movie that deserves more attention, and I came to like the minimalistic acoustic guitar score quite a bit.""",
+        """This is to the Zatoichi movies as the "Star Trek" movies were to "Star Trek"--except that in this case every one of the originals was more entertaining and interesting than this big, shiny re-do, and also better made, if substance is more important than surface. Had I never seen them, I would have thought this good-looking but empty; since I had, I thought its style inappropriate and its content insufficient. The idea of reviving the character in a bigger, slicker production must have sounded good, but there was no point in it, other than the hope of making money; it's just a show, which mostly fails to capture the atmosphere of the character's world and wholly fails to take the character anywhere he hasn't been already (also, the actor wasn't at his best). I'd been hoping to see Ichi at a late stage of life, in a story that would see him out gracefully and draw some conclusion from his experience overall; this just rehashes bits and pieces from the other movies, seasoned with more sex and sfx violence. Not the same experience at all."""
+    ],
+
+    "Twitter": [
+        """@Mbjthegreat i really dont want AT&amp;T phone service..they suck when it comes to having a signal""",
+        """holy crap. I take a nap for 4 hours and Pitchfork blows up my twitter dashboard. I wish I was at Coachella.""",
+        """@Susy412 he is working today  ive tried that still not working..... hmmmm!! im rubbish with computers haha!""",
+        """Brand New Canon EOS 50D 15MP DSLR Camera Canon 17-85mm IS Lens ...: Web Technology Thread, Brand New Canon EOS 5.. http://u.mavrev.com/5a3t""",
+        """Watching a programme about the life of Hitler, its only enhancing my geekiness of history.""",
+        """GM says expects announcment on sale of Hummer soon - Reuters: WDSUGM says expects announcment on sale of Hummer .. http://bit.ly/4E1Fv""",
+        """@accannis @edog1203 Great Stanford course. Thanks for making it available to the public! Really helpful and informative for starting off!""",
+        """@the_real_usher LeBron is cool.  I like his personality...he has good character.""",
+        """@sketchbug Lebron is a hometown hero to me, lol I love the Lakers but let's go Cavs, lol""",
+        """@PDubyaD right!!! LOL we'll get there!! I have high expectations, Warren Buffet style."""
+    ]
+}
+
+st.subheader("Detect the general sentiment expressed in a movie review or tweet by using our pretrained Spark NLP DL classifier.")
+
+selected_text = None
+result_type = 'tweet'
+if 'imdb' in model.lower():
+    selected_text = st.selectbox("Select a sample IMDB review", examples['IMDB'])
+    result_type = 'review'
+else:
+    selected_text = st.selectbox("Select a sample Tweet", examples['Twitter'])
+
+custom_input = st.text_input("Try it for yourself!")
+
+if custom_input:
+    selected_text = custom_input
+
+st.subheader('Selected Text')
+st.write(selected_text)
+
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, selected_text)
+
+if output in ['pos', 'positive', 'POSITIVE']:
+    st.markdown(f"""<h3>This seems like a <span style="color: green">positive</span> {result_type}. <span style="font-size:35px;">&#128515;</span></h3>""", unsafe_allow_html=True)
+elif output in ['neg', 'negative', 'NEGATIVE']:
+    st.markdown(f"""<h3>This seems like a <span style="color: red">negative</span> {result_type}. <span style="font-size:35px;">&#128544;</span></h3>""", unsafe_allow_html=True)

pages/Summarization.py

@@ -0,0 +1,89 @@
+import streamlit as st
+import sparknlp
+import os
+from sparknlp.base import *
+from sparknlp.common import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+
+@st.cache_resource
+def init_spark():
+    spark = sparknlp.start()
+    return spark
+
+@st.cache_resource
+def create_pipeline(model):
+    document_assembler = DocumentAssembler()\
+        .setInputCol("text")\
+        .setOutputCol("documents")
+
+    t5 = T5Transformer() \
+        .pretrained(model, 'en') \
+        .setTask("summarize:")\
+        .setMaxOutputLength(200)\
+        .setInputCols(["documents"]) \
+        .setOutputCol("summaries")
+    
+    pipeline = Pipeline().setStages([document_assembler, t5])
+    return pipeline
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    results = model.fullAnnotate(data)[0]
+    return results['summaries'][0].result
+
+############ SETTING UP THE PAGE LAYOUT ############
+
+st.title("Summarize Text")
+
+### SIDEBAR CONTENT ###
+
+# Model selection in sidebar
+model = st.sidebar.selectbox(
+    "Choose the pretrained model", 
+    ['t5_base', 't5_small'], 
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Colab link for the notebook
+link = """<a href="https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5TRANSFORMER.ipynb">
+<img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>"""
+st.sidebar.title('')
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+### MAIN CONTENT ###
+
+# Sample text options
+options = [
+    "Mount Tai is a mountain of historical and cultural significance located north of the city of Tai'an, in Shandong province, China. The tallest peak is the Jade Emperor Peak, which is commonly reported as being 1,545 meters tall, but is officially described by the PRC government as 1,532.7 meters tall. It is associated with sunrise, birth, and renewal, and is often regarded the foremost of the five. Mount Tai has been a place of worship for at least 3,000 years and served as one of the most important ceremonial centers of China during large portions of this period.",
+    "The Guadeloupe amazon (Amazona violacea) is a hypothetical extinct species of parrot that is thought to have been endemic to the Lesser Antillean island region of Guadeloupe. Described by 17th- and 18th-century writers, it is thought to have been related to, or possibly the same as, the extant imperial amazon. A tibiotarsus and an ulna bone from the island of Marie-Galante may belong to the Guadeloupe amazon. According to contemporary descriptions, its head, neck and underparts were mainly violet or slate, mixed with green and black; the back was brownish green; and the wings were green, yellow and red. It had iridescent feathers, and was able to raise a \"ruff\" of feathers around its neck. It fed on fruits and nuts, and the male and female took turns sitting on the nest. French settlers ate the birds and destroyed their habitat. Rare by 1779, the species appears to have become extinct by the end of the 18th century.",
+    "Pierre-Simon, marquis de Laplace (23 March 1749 – 5 March 1827) was a French scholar and polymath whose work was important to the development of engineering, mathematics, statistics, physics, astronomy, and philosophy. He summarized and extended the work of his predecessors in his five-volume Mécanique Céleste (Celestial Mechanics) (1799–1825). This work translated the geometric study of classical mechanics to one based on calculus, opening up a broader range of problems. In statistics, the Bayesian interpretation of probability was developed mainly by Laplace.",
+    "John Snow (15 March 1813 – 16 June 1858) was an English physician and a leader in the development of anaesthesia and medical hygiene. He is considered one of the founders of modern epidemiology, in part because of his work in tracing the source of a cholera outbreak in Soho, London, in 1854, which he curtailed by removing the handle of a water pump. Snow's findings inspired the adoption of anaesthesia as well as fundamental changes in the water and waste systems of London, which led to similar changes in other cities, and a significant improvement in general public health around the world.",
+    "The Mona Lisa is a half-length portrait painting by Italian artist Leonardo da Vinci. Considered an archetypal masterpiece of the Italian Renaissance, it has been described as \"the best known, the most visited, the most written about, the most sung about, the most parodied work of art in the world\". The painting's novel qualities include the subject's enigmatic expression, the monumentality of the composition, the subtle modelling of forms, and the atmospheric illusionism.",
+    """Calculus, originally called infinitesimal calculus or "the calculus of infinitesimals", is the mathematical study of continuous change, in the same way that geometry is the study of shape and algebra is the study of generalizations of arithmetic operations. It has two major branches, differential calculus and integral calculus; the former concerns instantaneous rates of change, and the slopes of curves, while integral calculus concerns accumulation of quantities, and areas under or between curves. These two branches are related to each other by the fundamental theorem of calculus, and they make use of the fundamental notions of convergence of infinite sequences and infinite series to a well-defined limit.[1] Infinitesimal calculus was developed independently in the late 17th century by Isaac Newton and Gottfried Wilhelm Leibniz.[2][3] Today, calculus has widespread uses in science, engineering, and economics.[4] In mathematics education, calculus denotes courses of elementary mathematical analysis, which are mainly devoted to the study of functions and limits. The word calculus (plural calculi) is a Latin word, meaning originally "small pebble" (this meaning is kept in medicine – see Calculus (medicine)). Because such pebbles were used for calculation, the meaning of the word has evolved and today usually means a method of computation. It is therefore used for naming specific methods of calculation and related theories, such as propositional calculus, Ricci calculus, calculus of variations, lambda calculus, and process calculus.""",
+]
+
+st.subheader("Summarize text to make it shorter while retaining meaning.")
+
+# Text input options
+selected_text = st.selectbox("Select an example", options)
+custom_input = st.text_input("Try it for yourself!")
+
+if custom_input:
+    selected_text = custom_input
+
+st.subheader('Text')
+st.write(selected_text)
+
+st.subheader("Summary")
+
+# Generate summary
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, selected_text)
+st.write(output)

pages/Text Preprocessing.py

@@ -0,0 +1,164 @@
+############ IMPORTING LIBRARIES ############
+
+# Import streamlit, Spark NLP, pyspark
+
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+import pyspark.sql.functions as F
+from pyspark.ml import Pipeline
+from pyspark.sql import SparkSession
+from sparknlp.annotator import *
+from sparknlp.base import *
+from sparknlp.pretrained import PretrainedPipeline
+from pyspark.sql.types import StringType, IntegerType
+    
+@st.cache_resource
+def init_spark():
+    spark = sparknlp.start()
+    return spark
+
+@st.cache_resource
+def create_pipeline(model):
+
+    documentAssembler = DocumentAssembler()\
+        .setInputCol("text")\
+        .setOutputCol("document")
+
+    sentenceDetector = SentenceDetector()\
+        .setInputCols(['document'])\
+        .setOutputCol('sentences')
+
+    tokenizer = Tokenizer() \
+        .setInputCols(["document"]) \
+        .setOutputCol("token")
+
+    normalizer = Normalizer() \
+        .setInputCols(["token"]) \
+        .setOutputCol("normalized")\
+        .setLowercase(True)\
+        .setCleanupPatterns(["[^\w\d\s]"])
+
+    stopwords_cleaner = StopWordsCleaner()\
+        .setInputCols("token")\
+        .setOutputCol("removed_stopwords")\
+        .setCaseSensitive(False)\
+
+    stemmer = Stemmer() \
+        .setInputCols(["token"]) \
+        .setOutputCol("stem")
+
+
+    lemmatizer = Lemmatizer() \
+        .setInputCols(["token"]) \
+        .setOutputCol("lemma") \
+        .setDictionary("./AntBNC_lemmas_ver_001.txt", value_delimiter ="\t", key_delimiter = "->")
+
+    nlpPipeline = Pipeline(stages=[documentAssembler,
+                                sentenceDetector,
+                                tokenizer,
+                                normalizer,
+                                stopwords_cleaner,
+                                stemmer,
+                                lemmatizer])
+
+    return nlpPipeline
+
+def fit_data(pipeline, data):
+
+  empty_df = spark.createDataFrame([['']]).toDF('text')
+  pipeline_model = pipeline.fit(empty_df)
+  model = LightPipeline(pipeline_model)
+  result = model.fullAnnotate(data)
+
+  return result
+
+############ SETTING UP THE PAGE LAYOUT ############
+
+st.title("Typo Detector")
+
+### SIDEBAR CONTENT ###
+
+model_name = "SentenceDetector|Tokenizer|Stemmer|Lemmatizer|Normalizer|Stop Words Remover"
+#model = st.sidebar.selectbox("Choose the pretrained model", model_name, help="For more info about the models visit: https://sparknlp.org/models",)
+
+st.sidebar.title("Filter Annotator Outputs")
+selected_models = []
+for model in model_name.split('|'):
+  check = st.sidebar.checkbox(model, value=True, key=model)
+  selected_models.append(check)
+
+# # Let's add the colab link for the notebook.
+
+link= """<a href="https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5TRANSFORMER.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/></a>"""
+st.sidebar.title('')
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+### MAIN CONTENT ###
+
+examples = [
+
+    "The Geneva Motor Show, the first major car show of the year, opens tomorrow with U.S. Car makers hoping to make new inroads into European markets due to the cheap dollar, automobile executives said. Ford Motor Co and General Motors Corp sell cars in Europe, where about 10.5 mln new cars a year are bought. GM also makes a few thousand in North American plants for European export.",
+    "Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music was rad! Horror and sword fight freaks,buy this movie now!",
+    "Quantum computing is the use of quantum-mechanical phenomena such as superposition and entanglement to perform computation. Computers that perform quantum computations are known as quantum computers. Quantum computers are believed to be able to solve certain computational problems, such as integer factorization (which underlies RSA encryption), substantially faster than classical computers. The study of quantum computing is a subfield of quantum information science. Quantum computing began in the early 1980s, when physicist Paul Benioff proposed a quantum mechanical model of the Turing machine.",
+    "Titanic is a 1997 American epic romance and disaster film directed, written, co-produced, and co-edited by James Cameron. Incorporating both historical and fictionalized aspects, it is based on accounts of the sinking of the RMS Titanic, and stars Leonardo DiCaprio and Kate Winslet as members of different social classes who fall in love aboard the ship during its ill-fated maiden voyage.",
+    "William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000.[9] He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.",
+    """World War II (often abbreviated as WWII or WW2), also known as the Second World War, was a global war that lasted from 1939 to 1945. The vast majority of the world's countries—including all the great powers—eventually formed two opposing military alliances: the Allies and the Axis. A state of total war emerged, directly involving more than 100 million people from more than 30 countries. The major participants threw their entire economic, industrial, and scientific capabilities behind the war effort, blurring the distinction between civilian and military resources. World War II was the deadliest conflict in human history, marked by 70 to 85 million fatalities, most of whom were civilians in the Soviet Union and China. Tens of millions of people died during the conflict due to genocides (including the Holocaust), premeditated death from starvation, massacres, and disease. Aircraft played a major role in the conflict which included the use of terror bombing, strategic bombing and the only use of nuclear weapons in war.""",
+    "Disney Channel (originally called The Disney Channel from 1983 to 1997 and commonly shortened to Disney from 1997 to 2002) is an American pay television channel that serves as the flagship property of owner Disney Channels Worldwide unit of the Walt Disney Television subsidiary of The Walt Disney Company. Disney Channel's programming consists of original first-run television series, theatrically released and original made-for-TV movies and select other third-party programming. Disney Channel – which formerly operated as a premium service – originally marketed its programs towards families during the 1980s, and later at younger children by the 2000s.",
+    "For several hundred thousand years, the Sahara has alternated between desert and savanna grassland in a 20,000 year cycle[8] caused by the precession of the Earth's axis as it rotates around the Sun, which changes the location of the North African Monsoon. The area is next expected to become green in about 15,000 years (17,000 ACE).",
+    "Elon Musk is an engineer, industrial designer, technology entrepreneur and philanthropist. He is a citizen of South Africa, Canada, and the United States. He is the founder, CEO and chief engineer/designer of SpaceX; early investor, CEO and product architect of Tesla, Inc.; founder of The Boring Company; co-founder of Neuralink; and co-founder and initial co-chairman of OpenAI. He was elected a Fellow of the Royal Society (FRS) in 2018. In December 2016, he was ranked 21st on the Forbes list of The World's Most Powerful People, and was ranked joint-first on the Forbes list of the Most Innovative Leaders of 2019. A self-made billionaire, as of June 2020 his net worth was estimated at $38.8 billion and he is listed by Forbes as the 31st-richest person in the world. He is the longest tenured CEO of any automotive manufacturer globally.",
+    "Born and raised in the Austrian Empire, Tesla studied engineering and physics in the 1870s without receiving a degree, and gained practical experience in the early 1880s working in telephony and at Continental Edison in the new electric power industry. In 1884 he emigrated to the United States, where he became a naturalized citizen. He worked for a short time at the Edison Machine Works in New York City before he struck out on his own. With the help of partners to finance and market his ideas, Tesla set up laboratories and companies in New York to develop a range of electrical and mechanical devices. His alternating current (AC) induction motor and related polyphase AC patents, licensed by Westinghouse Electric in 1888, earned him a considerable amount of money and became the cornerstone of the polyphase system which that company eventually marketed."
+
+]
+
+st.subheader("Split and clean text")
+
+selected_text = st.selectbox("Select an example", examples)
+
+custom_input = st.text_input("Try it for yourself!")
+
+if custom_input:
+    selected_text = custom_input
+elif selected_text:
+    selected_text = selected_text
+
+st.subheader('Selected Text')
+st.write(selected_text)
+
+spark = init_spark()
+Pipeline = create_pipeline(model)
+output = fit_data(Pipeline, selected_text)
+
+df = output.toPandas()
+
+### Displaying sentences
+if selected_models[0] is True:
+  st.subheader("Detected sentences in text:")
+  st.dataframe(pd.DataFrame({'sentences': np.asarray(df['sentences'].values[0])[:,3]}))
+
+## Displaying tokens, stem, lemma
+equal_columns = ['token', 'stem', 'lemma']
+sbhrds = ["Basic", 'Stemmed', 'Lemmatized']
+tdf = pd.DataFrame()
+sbhdr = ''
+for index, col in enumerate(equal_columns):
+  if selected_models[index+1] is True:
+    tcol_arr = np.asarray(df[col].values[0])
+    tdf[col] = tcol_arr[:,3]
+    sbhdr += (',\t'+sbhrds[index])
+if tdf.shape[0]>=1:
+  st.subheader(sbhdr[1:]+"\tTokens:")
+  st.dataframe(tdf)
+
+if selected_models[4] is True:
+  tcol_arr = np.asarray(df['normalized'].values[0])[:,3]
+  st.subheader("Tokens after removing punctuations:")
+  st.dataframe(pd.DataFrame({'normalized':tcol_arr}))
+
+if selected_models[5] is True:
+  tcol_arr = np.asarray(df['removed_stopwords'].values[0])[:,3]
+  st.subheader("Tokens after removing Stop Words:")
+  st.dataframe(pd.DataFrame({'removed_stopwords':tcol_arr}))

requirements.txt

@@ -0,0 +1,7 @@
+streamlit
+pandas
+numpy
+spark-nlp
+pyspark
+
+