Upload 7 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,117 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline():
+    document_assembler = DocumentAssembler() \
+        .setInputCol("text") \
+        .setOutputCol("document")
+
+    tokenizer = Tokenizer() \
+        .setInputCols(["document"]) \
+        .setOutputCol("token")
+
+    postagger = PerceptronModel.pretrained("pos_anc", "en") \
+        .setInputCols(["document", "token"]) \
+        .setOutputCol("pos")
+
+    pipeline = Pipeline(stages=[document_assembler, tokenizer, postagger])
+    return pipeline
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    results = model.fullAnnotate(data)
+    return results
+
+# Set up the page layout
+st.markdown('<div class="main-title">State-of-the-Art Part-of-Speech Tagging with Spark NLP</div>', unsafe_allow_html=True)
+
+# Sidebar content
+model_name = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ['pos_anc'],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/coreference-resolution/Coreference_Resolution_SpanBertCorefModel.ipynb#L117">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+examples = [
+    "Alice went to the market. She bought some fresh vegetables there. The tomatoes she purchased were particularly ripe.",
+    "Dr. Smith is a renowned surgeon. He has performed over a thousand successful operations. His colleagues respect him a lot.",
+    "The company announced a new product launch. It is expected to revolutionize the industry. The CEO was very excited about it.",
+    "Jennifer enjoys hiking. She goes to the mountains every weekend. Her favorite spot is the Blue Ridge Mountains.",
+    "The team won the championship. They celebrated their victory with a huge party. Their coach praised their hard work and dedication.",
+    "Michael is studying computer science. He finds artificial intelligence fascinating. His dream is to work at a leading tech company.",
+    "Tom is a skilled guitarist. He plays in a local band. His performances are always energetic and captivating."
+]
+
+# st.subheader("Automatically detect phrases expressing dates and normalize them with respect to a reference date.")
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own Sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.subheader('Full example text')
+st.write(text_to_analyze)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline()
+output = fit_data(pipeline, text_to_analyze)
+
+# Display matched sentence
+st.subheader("Processed output:")
+
+results = {
+    'Token': [t.result for t in output[0]['token']],
+    'Begin': [p.begin for p in output[0]['pos']],
+    'End': [p.end for p in output[0]['pos']],
+    'POS': [p.result for p in output[0]['pos']]
+}
+
+df = pd.DataFrame(results)
+df.index += 1
+st.dataframe(df)

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,233 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Introduction
+st.markdown('<div class="main-title">Part-of-Speech Tagging with Spark NLP</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>Welcome to the Spark NLP Part-of-Speech Tagging Demo App! Part-of-Speech (POS) tagging is an essential task in Natural Language Processing (NLP) that involves identifying the grammatical roles of words within a text, such as nouns, verbs, adjectives, and more. This app demonstrates how to use the PerceptronModel annotator to perform POS tagging in text data using Spark NLP.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.image('images/POS-Tagging.png', caption="Dependency parsing with POS tags", use_column_width='auto')
+
+# About POS Tagging
+st.markdown('<div class="sub-title">About Part-of-Speech Tagging</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Part-of-Speech (POS) tagging involves assigning each word in a sentence its grammatical role, such as subject, verb, or adjective. This technique helps improve many NLP tasks, including Named Entity Recognition (NER), Word Sense Disambiguation (WSD), Question Answering (QA), and Dependency Parsing (DP).</p>
+    <p>For instance, knowing that a word is an adjective increases the likelihood that one of the neighboring words is a noun. The context can also alter the meaning of words significantly:</p>
+    <ul>
+        <li><i>What is your address?</i> (noun)</li>
+        <li><i>I will address this issue today.</i> (verb)</li>
+    </ul>
+    <p>POS tags can be categorized using schemas like the Universal Dependencies schema or the Penn Treebank POS tags. Each schema provides a set of tags for different grammatical roles.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Using PerceptronModel for POS Tagging
+st.markdown('<div class="sub-title">Using PerceptronModel for POS Tagging in Spark NLP</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The PerceptronModel annotator in Spark NLP allows users to perform POS tagging with high accuracy using pretrained models. This annotator can identify and label the grammatical roles of words in text data, providing valuable insights for various applications.</p>
+    <p>The PerceptronModel annotator in Spark NLP offers:</p>
+    <ul>
+        <li>Accurate POS tagging using pretrained models</li>
+        <li>Identification and labeling of grammatical roles</li>
+        <li>Efficient processing of large text datasets</li>
+        <li>Integration with other Spark NLP components for comprehensive NLP pipelines</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<h2 class="sub-title">Example Usage in Python</h2>', unsafe_allow_html=True)
+st.markdown('<p>Here’s how you can implement POS tagging using the PerceptronModel annotator in Spark NLP:</p>', unsafe_allow_html=True)
+
+# Setup Instructions
+st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
+st.markdown('<p>To install Spark NLP in Python, use your favorite package manager (conda, pip, etc.). For example:</p>', unsafe_allow_html=True)
+st.code("""
+pip install spark-nlp
+pip install pyspark
+""", language="bash")
+
+st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
+st.code("""
+import sparknlp
+
+# Start Spark Session
+spark = sparknlp.start()
+""", language='python')
+
+# POS Tagging Example
+st.markdown('<div class="sub-title">Example Usage: POS Tagging with PerceptronModel</div>', unsafe_allow_html=True)
+st.code('''
+from sparknlp.base import DocumentAssembler
+from sparknlp.annotator import Tokenizer, PerceptronModel
+from pyspark.ml import Pipeline
+import pyspark.sql.functions as F
+
+# Stage 1: Transforms raw texts to document annotation
+document_assembler = DocumentAssembler() \\
+    .setInputCol("text") \\
+    .setOutputCol("document")
+
+# Stage 2: Tokenization
+tokenizer = Tokenizer() \\
+    .setInputCols(["document"]) \\
+    .setOutputCol("token")
+
+# Stage 3: Perceptron model for POS Tagger
+# Pretrained model pos_anc for texts in English
+postagger = PerceptronModel.pretrained("pos_anc", "en") \\
+    .setInputCols(["document", "token"]) \\
+    .setOutputCol("pos")
+
+# Define the pipeline
+pipeline = Pipeline(stages=[document_assembler, tokenizer, postagger])
+
+# Create the dataframe
+data = spark.createDataFrame([["Unions representing workers at Turner Newall say they are 'disappointed' after talks with stricken parent firm Federal Mogul"]]).toDF("text")
+
+# Fit the dataframe to the pipeline to get the model
+model = pipeline.fit(data)
+
+# Transform the data to get predictions
+result = model.transform(data)
+
+# Display the POS tags
+result.select(
+    F.explode(
+        F.arrays_zip(result.token.result, result.token.begin, result.token.end, result.pos.result)
+    ).alias("cols")
+).select(
+    F.expr("cols['0']").alias("token"),
+    F.expr("cols['1']").alias("begin"),
+    F.expr("cols['2']").alias("end"),
+    F.expr("cols['3']").alias("pos"),
+).show(truncate=False)
+''', language='python')
+
+st.text("""
++------------+-----+---+---+
+|token       |begin|end|pos|
++------------+-----+---+---+
+|Unions      |0    |5  |NNP|
+|representing|7    |18 |VBG|
+|workers     |20   |26 |NNS|
+|at          |28   |29 |IN |
+|Turner      |31   |36 |NNP|
+|Newall      |38   |43 |NNP|
+|say         |45   |47 |VBP|
+|they        |49   |52 |PRP|
+|are         |54   |56 |VBP|
+|'           |58   |58 |POS|
+|disappointed|59   |70 |JJ |
+|'           |71   |71 |POS|
+|after       |73   |77 |IN |
+|talks       |79   |83 |NNS|
+|with        |85   |88 |IN |
+|stricken    |90   |97 |NN |
+|parent      |99   |104|NN |
+|firm        |106  |109|NN |
+|Federal     |111  |117|NNP|
+|Mogul       |119  |123|NNP|
++------------+-----+---+---+
+""")
+
+st.markdown("""
+<p>The code snippet demonstrates how to set up a pipeline in Spark NLP to perform POS tagging on text data using the PerceptronModel annotator. The resulting DataFrame contains the tokens and their corresponding POS tags.</p>
+""", unsafe_allow_html=True)
+
+# One-liner Alternative
+st.markdown('<div class="sub-title">One-liner Alternative</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>In October 2022, John Snow Labs released the open-source <code>johnsnowlabs</code> library that contains all the company products, open-source and licensed, under one common library. This simplified the workflow, especially for users working with more than one of the libraries (e.g., Spark NLP + Healthcare NLP). This new library is a wrapper on all of John Snow Lab’s libraries and can be installed with pip:</p>
+    <p><code>pip install johnsnowlabs</code></p>
+    <p>To run POS tagging with one line of code, we can simply:</p>
+</div>
+""", unsafe_allow_html=True)
+st.code("""
+# Import the NLP module which contains Spark NLP and NLU libraries
+from johnsnowlabs import nlp
+
+example_sentence = "Unions representing workers at Turner Newall say they are 'disappointed' after talks with stricken parent firm Federal Mogul"
+
+# Returns a pandas DataFrame, we select the desired columns
+nlp.load("pos").predict(example_sentence)[['token','pos']]
+""", language='python')
+
+st.image('images/johnsnowlabs-output.png', use_column_width='auto')
+
+# Summary
+st.markdown('<div class="sub-title">Summary</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>In this demo app, we showcased how to perform Part-of-Speech tagging using the PerceptronModel annotator in Spark NLP. POS tagging is a crucial step in many NLP applications, helping to understand the grammatical structure and context of the text. With Spark NLP, you can efficiently process and analyze large volumes of text data, leveraging powerful pretrained models for accurate and reliable results.</p>
+    <p>We hope you found this demo helpful and encourage you to explore more features and capabilities of Spark NLP for your NLP projects!</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References and Additional Information
+st.markdown('<div class="sub-title">For additional information, please check the following references.</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a href="https://nlp.johnsnowlabs.com/docs/en/annotators" target="_blank" rel="noopener">Spark NLP documentation page</a> for all available annotators</li>
+        <li>Python API documentation for <a href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/pos/perceptron/index.html#sparknlp.annotator.pos.perceptron.PerceptronModel" target="_blank" rel="noopener">PerceptronModel</a> and <a href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/dependency/dependency_parser/index.html#sparknlp.annotator.dependency.dependency_parser.DependencyParserModel" target="_blank" rel="noopener">Dependency Parser</a></li>
+        <li>Scala API documentation for <a href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/pos/perceptron/PerceptronModel.html" target="_blank" rel="noopener">PerceptronModel</a> and <a href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/parser/dep/DependencyParserModel.html" target="_blank" rel="noopener">DependencyParserModel</a></li>
+        <li>For extended examples of usage of Spark NLP annotators, check the <a href="https://github.com/JohnSnowLabs/spark-nlp-workshop" target="_blank" rel="noopener">Spark NLP Workshop repository</a>.</li>
+        <li>Minsky, M.L. and Papert, S.A. (1969) Perceptrons. MIT Press, Cambridge.</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+pandas
+numpy
+spark-nlp
+pyspark