Upload 8 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,123 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model):
+  documentAssembler = DocumentAssembler()\
+    .setInputCol("text")\
+    .setOutputCol("document")
+    
+  use = UniversalSentenceEncoder.pretrained("tfhub_use", "en")\
+  .setInputCols(["document"])\
+  .setOutputCol("sentence_embeddings")
+
+
+  sentimentdl = ClassifierDLModel.pretrained(model)\
+      .setInputCols(["sentence_embeddings"])\
+      .setOutputCol("sentiment")
+
+  nlpPipeline = Pipeline(stages = [documentAssembler, use, sentimentdl])
+  
+  return nlpPipeline
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    results = model.fullAnnotate(data)[0]
+
+    return results['sentiment'][0].result
+
+# Set up the page layout
+st.markdown('<div class="main-title">State-of-the-Art Emotion Detecter in Tweets with Spark NLP</div>', unsafe_allow_html=True)
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["classifierdl_use_emotion"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SENTIMENT_EN_EMOTION.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+examples = [
+  "I am SO happy the news came out in time for my birthday this weekend! My inner 7-year-old cannot WAIT!",
+  "That moment when you see your friend in a commercial. Hahahaha!",
+  "My soul has just been pierced by the most evil look from @rickosborneorg. A mini panic attack &amp; chill in bones followed soon after.",
+  "For some reason I woke up thinkin it was Friday then I got to school and realized its really Monday -_-",
+  "I'd probably explode into a jillion pieces from the inablility to contain all of my if I had a Whataburger patty melt right now. #drool",
+  "These are not emotions. They are simply irrational thoughts feeding off of an emotion",
+  "Found out im gonna be with sarah bo barah in ny for one day!!! Eggcitement :)",
+  "That awkward moment when you find a perfume box full of sensors!",
+  "Just home from group celebration - dinner at Trattoria Gianni, then Hershey Felder's performance - AMAZING!!",
+  "Nooooo! My dad turned off the internet so I can't listen to band music!"
+  ]
+
+st.subheader("Automatically identify Joy, Surprise, Fear, Sadness in Tweets using out pretrained Spark NLP DL classifier.")
+
+selected_text = st.selectbox("Select a sample", examples)
+custom_input = st.text_input("Try it for yourself!")
+
+if custom_input:
+    selected_text = custom_input
+elif selected_text:
+    selected_text = selected_text
+
+st.subheader('Selected Text')
+st.write(selected_text)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, selected_text)
+
+# Display output sentence
+if output == 'joy':
+  st.markdown("""<h3>This seems like a <span style="color: #f0a412">{}</span> tweet. <span style="font-size:35px;">&#128514;</span></h3>""".format('joyous'), unsafe_allow_html=True)
+elif output == 'surprise':
+  st.markdown("""<h3>This seems like a <span style="color: #209DDC">{}</span> tweet. <span style="font-size:35px;">&#128522;</span></h3>""".format('surprised'), unsafe_allow_html=True)
+elif output == 'sadness':
+  st.markdown("""<h3>This seems like a <span style="color: #8F7F6C">{}</span> tweet. <span style="font-size:35px;">&#128543;</span></h3>""".format('sad'), unsafe_allow_html=True)
+elif output == 'fear':
+  st.markdown("""<h3>This seems like a <span style="color: #B64434">{}</span> tweet. <span style="font-size:35px;">&#128561;</span></h3>""".format('fearful'), unsafe_allow_html=True)

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,224 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Introduction
+st.markdown('<div class="main-title">Sentiment Analysis with Spark NLP</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>Welcome to the Spark NLP Sentiment Analysis Demo App! Sentiment analysis is an automated process capable of understanding the feelings or opinions that underlie a text. This process is considered a text classification and is one of the most interesting subfields of NLP. Using Spark NLP, it is possible to analyze the sentiment in a text with high accuracy.</p>
+    <p>This app demonstrates how to use Spark NLP's SentimentDetector to perform sentiment analysis using a rule-based approach.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.image('images/Sentiment-Analysis.jpg',caption="Difference between rule-based and machine learning based sentiment analysis applications", use_column_width='auto')
+
+# About Sentiment Analysis
+st.markdown('<div class="sub-title">About Sentiment Analysis</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Sentiment analysis studies the subjective information in an expression, such as opinions, appraisals, emotions, or attitudes towards a topic, person, or entity. Expressions can be classified as positive, negative, or neutral — in some cases, even more detailed.</p>
+    <p>Some popular sentiment analysis applications include social media monitoring, customer support management, and analyzing customer feedback.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Using SentimentDetector in Spark NLP
+st.markdown('<div class="sub-title">Using SentimentDetector in Spark NLP</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The SentimentDetector annotator in Spark NLP uses a rule-based approach to analyze the sentiment in text data. This method involves using a set of predefined rules or patterns to classify text as positive, negative, or neutral.</p>
+    <p>Spark NLP also provides Machine Learning (ML) and Deep Learning (DL) solutions for sentiment analysis. If you are interested in those approaches, please check the <a class="link" href="https://nlp.johnsnowlabs.com/docs/en/annotators#viveknsentiment" target="_blank" rel="noopener">ViveknSentiment </a> and <a class="link" href="https://nlp.johnsnowlabs.com/docs/en/annotators#sentimentdl" target="_blank" rel="noopener">SentimentDL</a> annotators of Spark NLP.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<h2 class="sub-title">Example Usage in Python</h2>', unsafe_allow_html=True)
+st.markdown('<p>Here’s how you can implement sentiment analysis using the SentimentDetector annotator in Spark NLP:</p>', unsafe_allow_html=True)
+
+# Setup Instructions
+st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
+st.markdown('<p>To install Spark NLP in Python, use your favorite package manager (conda, pip, etc.). For example:</p>', unsafe_allow_html=True)
+st.code("""
+pip install spark-nlp
+pip install pyspark
+""", language="bash")
+
+st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
+st.code("""
+import sparknlp
+
+# Start Spark Session
+spark = sparknlp.start()
+""", language='python')
+
+# load data
+st.markdown('<div class="sub-title">Start by loading the Dataset, Lemmas and the Sentiment Dictionary.</div>', unsafe_allow_html=True)
+st.code("""
+!wget -N https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/resources/en/lemma-corpus-small/lemmas_small.txt -P /tmp
+!wget -N https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/resources/en/sentiment-corpus/default-sentiment-dict.txt -P /tmp
+""", language="bash")
+
+st.image('images/dataset.png', caption="First few lines of the lemmas and sentiment dictionary", use_column_width='auto')
+
+# Sentiment Analysis Example
+st.markdown('<div class="sub-title">Example Usage: Sentiment Analysis with SentimentDetector</div>', unsafe_allow_html=True)
+st.code('''
+from sparknlp.base import DocumentAssembler, Pipeline, Finisher
+from sparknlp.annotator import (
+    SentenceDetector,
+    Tokenizer,
+    Lemmatizer,
+    SentimentDetector
+)
+import pyspark.sql.functions as F
+
+# Step 1: Transforms raw texts to document annotation
+document_assembler = (
+    DocumentAssembler()
+    .setInputCol("text")
+    .setOutputCol("document")
+)
+
+# Step 2: Sentence Detection
+sentence_detector = SentenceDetector().setInputCols(["document"]).setOutputCol("sentence")
+
+# Step 3: Tokenization
+tokenizer = Tokenizer().setInputCols(["sentence"]).setOutputCol("token")
+
+# Step 4: Lemmatization
+lemmatizer = (
+    Lemmatizer()
+    .setInputCols("token")
+    .setOutputCol("lemma")
+    .setDictionary("/tmp/lemmas_small.txt", key_delimiter="->", value_delimiter="\\t")
+)
+
+# Step 5: Sentiment Detection
+sentiment_detector = (
+    SentimentDetector()
+    .setInputCols(["lemma", "sentence"])
+    .setOutputCol("sentiment_score")
+    .setDictionary("/tmp/default-sentiment-dict.txt", ",")
+)
+
+# Step 6: Finisher
+finisher = (
+    Finisher()
+    .setInputCols(["sentiment_score"])
+    .setOutputCols(["sentiment"])
+)
+
+# Define the pipeline
+pipeline = Pipeline(
+    stages=[
+        document_assembler,
+        sentence_detector,
+        tokenizer,
+        lemmatizer,
+        sentiment_detector,
+        finisher,
+    ]
+)
+
+# Create a spark Data Frame with an example sentence
+data = spark.createDataFrame(
+    [
+        ["The restaurant staff is really nice"]
+    ]
+).toDF("text")  # use the column name `text` defined in the pipeline as input
+
+# Fit-transform to get predictions
+result = pipeline.fit(data).transform(data).show(truncate=50)
+''', language='python')
+
+st.text("""
++-----------------------------------+----------+
+|                               text| sentiment|
++-----------------------------------+----------+
+|The restaurant staff is really nice|[positive]|
++-----------------------------------+----------+
+""")
+
+st.markdown("""
+<p>The code snippet demonstrates how to set up a pipeline in Spark NLP to perform sentiment analysis on text data using the SentimentDetector annotator. The resulting DataFrame contains the sentiment predictions.</p>
+""", unsafe_allow_html=True)
+
+# One-liner Alternative
+st.markdown('<div class="sub-title">One-liner Alternative</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>In October 2022, John Snow Labs released the open-source <code>johnsnowlabs</code> library that contains all the company products, open-source and licensed, under one common library. This simplified the workflow, especially for users working with more than one of the libraries (e.g., Spark NLP + Healthcare NLP). This new library is a wrapper on all of John Snow Lab’s libraries and can be installed with pip:</p>
+    <p><code>pip install johnsnowlabs</code></p>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<p>To run sentiment analysis with one line of code, we can simply:</p>', unsafe_allow_html=True)
+st.code("""
+# Import the NLP module which contains Spark NLP and NLU libraries
+from johnsnowlabs import nlp
+
+sample_text = "The restaurant staff is really nice"
+
+# Returns a pandas DataFrame, we select the desired columns
+nlp.load('en.sentiment').predict(sample_text, output_level='sentence')
+""", language='python')
+
+st.image('images/johnsnowlabs-sentiment-output.png', use_column_width='auto')
+
+st.markdown("""
+<p>This approach demonstrates how to use the <code>johnsnowlabs</code> library to perform sentiment analysis with a single line of code. The resulting DataFrame contains the sentiment predictions.</p>
+""", unsafe_allow_html=True)
+
+# Conclusion
+st.markdown("""
+<div class="section">
+    <h2>Conclusion</h2>
+    <p>In this app, we demonstrated how to use Spark NLP's SentimentDetector annotator to perform sentiment analysis on text data. These powerful tools enable users to efficiently process large datasets and identify sentiment, providing deeper insights for various applications. By integrating these annotators into your NLP pipelines, you can enhance text understanding, information extraction, and customer sentiment analysis.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References and Additional Information
+st.markdown('<div class="sub-title">For additional information, please check the following references.</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+        <ul>
+            <li>Documentation :&nbsp;<a href="https://nlp.johnsnowlabs.com/docs/en/transformers#sentiment" target="_blank" rel="noopener">SentimentDetector</a></li>
+            <li>Python Docs :&nbsp;<a href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/sentiment/index.html#sparknlp.annotator.sentiment.SentimentDetector" target="_blank" rel="noopener">SentimentDetector</a></li>
+            <li>Scala Docs :&nbsp;<a href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/sentiment/SentimentDetector.html" target="_blank" rel="noopener">SentimentDetector</a></li>
+            <li>Example Notebook :&nbsp;<a href="https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/training/english/classification/Sentiment%20Analysis.ipynb" target="_blank" rel="noopener">Sentiment Analysis</a></li>
+        </ul>
+    </div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+pandas
+numpy
+spark-nlp
+pyspark