Upload 8 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,119 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+
+# Page configuration
+st.set_page_config(
+    layout="wide",  
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model):
+    document = DocumentAssembler()\
+        .setInputCol("text")\
+        .setOutputCol("document")
+
+    embeddings = BertSentenceEmbeddings\
+        .pretrained('labse', 'xx') \
+        .setInputCols(["document"])\
+        .setOutputCol("sentence_embeddings")
+
+    sentimentClassifier = ClassifierDLModel.pretrained("classifierdl_bert_sentiment", "es") \
+    .setInputCols(["sentence_embeddings"]) \
+    .setOutputCol("class_")  
+        
+    nlpPipeline = Pipeline(
+        stages=[
+            document, 
+            embeddings,
+            sentimentClassifier])
+
+    return nlpPipeline
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    results = model.fullAnnotate(data)[0]
+
+    return results['class_'][0].result
+
+# Set up the page layout
+st.markdown('<div class="main-title">State-of-the-Art Spanish Sentiment Detection with Spark NLP</div>', unsafe_allow_html=True)
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["classifierdl_bert_sentiment"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/BertForSequenceClassification.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+examples = [
+    "Hr.formand, kære kommissær og kære kolleger, jeg vil starte med at sige Hr.Rapkay tak for en god betænkning og et godt samarbejde.",
+    "Protester over hele landet ledet af utilfredse civilsamfund på grund af den danske regerings COVID-19 lockdown-politik er kommet ud af kontrol.",
+    "Hvidbogen repræsenterer tre til fire måneders intensivt arbejde, siden den nye kommission blev udpeget i september.",
+    "Det er ikke et nemt emne, og jeg mener derfor, at den indsats, som fru Lienemanns har ydet, fortjener stor respekt.",
+    "Det er desværre en sørgelig erkendelse af de store vanskeligheder, som Wales oplever.",
+    "Vi har indarbejdet resultaterne af de omfattende høringer, der har fundet sted i løbet af det sidste par år siden offentliggørelsen af kommissionens grønbog om fødevarelovgivning.",
+    "Fru formand, jeg vil gerne først give Dem en kompliment for den kendsgerning, at De har holdt Deres ord, og at antallet af tv-kanaler på vores kontorer faktisk er udvidet enormt nu i denne første mødeperiode i det nye år.",
+    "Det er fuldstændig skandaløst, at vi fastsætter lovgivningsbestemmelser og så ikke overholder dem selv.",
+    "Min gruppe har udarbejdet omfattende ændringsforslag til begge betænkninger til forhandling i dag."
+        ]
+
+selected_text = st.selectbox("Select a sample", examples)
+custom_input = st.text_input("Try it for yourself!")
+
+if custom_input:
+    selected_text = custom_input
+elif selected_text:
+    selected_text = selected_text
+
+st.subheader('Selected Text')
+st.write(selected_text)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, selected_text)
+
+# Display output sentence
+if output.lower() in ['pos', 'positive']:
+  st.markdown("""<h3>This seems like a <span style="color: green">{}</span> text. <span style="font-size:35px;">&#128515;</span></h3>""".format('positive'), unsafe_allow_html=True)
+elif output.lower() in ['neg', 'negative']:
+  st.markdown("""<h3>This seems like a <span style="color: red">{}</span> text. <span style="font-size:35px;">&#128544;</span?</h3>""".format('negative'), unsafe_allow_html=True)

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,237 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Introduction
+st.markdown('<div class="main-title">Sentiment Analysis with Spark NLP</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>Welcome to the Spark NLP Sentiment Analysis Demo App! Sentiment analysis is an automated process capable of understanding the feelings or opinions that underlie a text. This process is considered a text classification and is one of the most interesting subfields of NLP. Using Spark NLP, it is possible to analyze the sentiment in a text with high accuracy.</p>
+    <p>This app demonstrates how to use Spark NLP's SentimentDetector to perform sentiment analysis using a rule-based approach.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.image('images/Sentiment-Analysis.jpg',caption="Difference between rule-based and machine learning based sentiment analysis applications", use_column_width='auto')
+
+# About Sentiment Analysis
+st.markdown('<div class="sub-title">About Sentiment Analysis</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Sentiment analysis studies the subjective information in an expression, such as opinions, appraisals, emotions, or attitudes towards a topic, person, or entity. Expressions can be classified as positive, negative, or neutral — in some cases, even more detailed.</p>
+    <p>Some popular sentiment analysis applications include social media monitoring, customer support management, and analyzing customer feedback.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Using SentimentDetector in Spark NLP
+st.markdown('<div class="sub-title">Using SentimentDetector in Spark NLP</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The SentimentDetector annotator in Spark NLP uses a rule-based approach to analyze the sentiment in text data. This method involves using a set of predefined rules or patterns to classify text as positive, negative, or neutral.</p>
+    <p>Spark NLP also provides Machine Learning (ML) and Deep Learning (DL) solutions for sentiment analysis. If you are interested in those approaches, please check the <a class="link" href="https://nlp.johnsnowlabs.com/docs/en/annotators#viveknsentiment" target="_blank" rel="noopener">ViveknSentiment </a> and <a class="link" href="https://nlp.johnsnowlabs.com/docs/en/annotators#sentimentdl" target="_blank" rel="noopener">SentimentDL</a> annotators of Spark NLP.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<h2 class="sub-title">Example Usage in Python</h2>', unsafe_allow_html=True)
+st.markdown('<p>Here’s how you can implement sentiment analysis using the SentimentDetector annotator in Spark NLP:</p>', unsafe_allow_html=True)
+
+# Setup Instructions
+st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
+st.markdown('<p>To install Spark NLP in Python, use your favorite package manager (conda, pip, etc.). For example:</p>', unsafe_allow_html=True)
+st.code("""
+pip install spark-nlp
+pip install pyspark
+""", language="bash")
+
+st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
+st.code("""
+import sparknlp
+
+# Start Spark Session
+spark = sparknlp.start()
+""", language='python')
+
+# load data
+st.markdown('<div class="sub-title">Start by loading the Dataset, Lemmas and the Sentiment Dictionary.</div>', unsafe_allow_html=True)
+st.code("""
+!wget -N https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/resources/en/lemma-corpus-small/lemmas_small.txt -P /tmp
+!wget -N https://s3.amazonaws.com/auxdata.johnsnowlabs.com/public/resources/en/sentiment-corpus/default-sentiment-dict.txt -P /tmp
+""", language="bash")
+
+st.image('images/dataset.png', caption="First few lines of the lemmas and sentiment dictionary", use_column_width='auto')
+
+# Sentiment Analysis Example
+st.markdown('<div class="sub-title">Example Usage: Sentiment Analysis with SentimentDetector</div>', unsafe_allow_html=True)
+st.code('''
+from sparknlp.base import DocumentAssembler, Pipeline, Finisher
+from sparknlp.annotator import (
+    SentenceDetector,
+    Tokenizer,
+    Lemmatizer,
+    SentimentDetector
+)
+import pyspark.sql.functions as F
+
+# Step 1: Transforms raw texts to document annotation
+document_assembler = (
+    DocumentAssembler()
+    .setInputCol("text")
+    .setOutputCol("document")
+)
+
+# Step 2: Sentence Detection
+sentence_detector = SentenceDetector().setInputCols(["document"]).setOutputCol("sentence")
+
+# Step 3: Tokenization
+tokenizer = Tokenizer().setInputCols(["sentence"]).setOutputCol("token")
+
+# Step 4: Lemmatization
+lemmatizer = (
+    Lemmatizer()
+    .setInputCols("token")
+    .setOutputCol("lemma")
+    .setDictionary("/tmp/lemmas_small.txt", key_delimiter="->", value_delimiter="\\t")
+)
+
+# Step 5: Sentiment Detection
+sentiment_detector = (
+    SentimentDetector()
+    .setInputCols(["lemma", "sentence"])
+    .setOutputCol("sentiment_score")
+    .setDictionary("/tmp/default-sentiment-dict.txt", ",")
+)
+
+# Step 6: Finisher
+finisher = (
+    Finisher()
+    .setInputCols(["sentiment_score"])
+    .setOutputCols(["sentiment"])
+)
+
+# Define the pipeline
+pipeline = Pipeline(
+    stages=[
+        document_assembler,
+        sentence_detector,
+        tokenizer,
+        lemmatizer,
+        sentiment_detector,
+        finisher,
+    ]
+)
+
+# Create a spark Data Frame with an example sentence
+data = spark.createDataFrame(
+    [
+        ["The restaurant staff is really nice"]
+    ]
+).toDF("text")  # use the column name `text` defined in the pipeline as input
+
+# Fit-transform to get predictions
+result = pipeline.fit(data).transform(data).show(truncate=50)
+''', language='python')
+
+st.text("""
++-----------------------------------+----------+
+|                               text| sentiment|
++-----------------------------------+----------+
+|The restaurant staff is really nice|[positive]|
++-----------------------------------+----------+
+""")
+
+st.markdown("""
+<p>The code snippet demonstrates how to set up a pipeline in Spark NLP to perform sentiment analysis on text data using the SentimentDetector annotator. The resulting DataFrame contains the sentiment predictions.</p>
+""", unsafe_allow_html=True)
+
+# One-liner Alternative
+st.markdown('<div class="sub-title">One-liner Alternative</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>In October 2022, John Snow Labs released the open-source <code>johnsnowlabs</code> library that contains all the company products, open-source and licensed, under one common library. This simplified the workflow, especially for users working with more than one of the libraries (e.g., Spark NLP + Healthcare NLP). This new library is a wrapper on all of John Snow Lab’s libraries and can be installed with pip:</p>
+    <p><code>pip install johnsnowlabs</code></p>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<p>To run sentiment analysis with one line of code, we can simply:</p>', unsafe_allow_html=True)
+st.code("""
+# Import the NLP module which contains Spark NLP and NLU libraries
+from johnsnowlabs import nlp
+
+sample_text = "The restaurant staff is really nice"
+
+# Returns a pandas DataFrame, we select the desired columns
+nlp.load('en.sentiment').predict(sample_text, output_level='sentence')
+""", language='python')
+
+st.image('images/johnsnowlabs-sentiment-output.png', use_column_width='auto')
+
+st.markdown("""
+<p>This approach demonstrates how to use the <code>johnsnowlabs</code> library to perform sentiment analysis with a single line of code. The resulting DataFrame contains the sentiment predictions.</p>
+""", unsafe_allow_html=True)
+
+# Conclusion
+st.markdown("""
+<div class="section">
+    <h2>Conclusion</h2>
+    <p>In this app, we demonstrated how to use Spark NLP's SentimentDetector annotator to perform sentiment analysis on text data. These powerful tools enable users to efficiently process large datasets and identify sentiment, providing deeper insights for various applications. By integrating these annotators into your NLP pipelines, you can enhance text understanding, information extraction, and customer sentiment analysis.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References and Additional Information
+st.markdown('<div class="sub-title">For additional information, please check the following references.</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+        <ul>
+            <li>Documentation :&nbsp;<a href="https://nlp.johnsnowlabs.com/docs/en/transformers#sentiment" target="_blank" rel="noopener">SentimentDetector</a></li>
+            <li>Python Docs :&nbsp;<a href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/sentiment/index.html#sparknlp.annotator.sentiment.SentimentDetector" target="_blank" rel="noopener">SentimentDetector</a></li>
+            <li>Scala Docs :&nbsp;<a href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/sentiment/SentimentDetector.html" target="_blank" rel="noopener">SentimentDetector</a></li>
+            <li>Example Notebook :&nbsp;<a href="https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/training/english/classification/Sentiment%20Analysis.ipynb" target="_blank" rel="noopener">Sentiment Analysis</a></li>
+        </ul>
+    </div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+pandas
+numpy
+spark-nlp
+pyspark