Upload 6 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,121 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline():
+    document_assembler = DocumentAssembler()\
+    .setInputCol("text")\
+    .setOutputCol("document")
+
+    tokenizer = RecursiveTokenizer()\
+    .setInputCols(["document"])\
+    .setOutputCol("token")\
+    .setPrefixes(["\"", "(", "[", "\n"])\
+    .setSuffixes([".", ",", "?", ")","!", "‘s"])
+
+    spell_model = ContextSpellCheckerModel\
+        .pretrained('spellcheck_dl')\
+        .setInputCols("token")\
+        .setOutputCol("corrected")
+
+    light_pipeline = Pipeline(stages = [document_assembler, tokenizer, spell_model])
+
+    return light_pipeline
+
+def fit_data(pipeline, data):
+  empty_df = spark.createDataFrame([['']]).toDF('text')
+  pipeline_model = pipeline.fit(empty_df)
+  model = LightPipeline(pipeline_model)
+  result = model.annotate(data)
+
+  return result
+
+# Set up the page layout
+st.markdown('<div class="main-title">Spell Check Your Text Documents With Spark NLP</div>', unsafe_allow_html=True)
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["spellcheck_dl"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/SPELL_CHECKER_EN.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+examples = [
+    "Apollo 11 was the space fleght that landed the frrst humans, Americans Neil Armstrong and Buzz Aldrin, on the Mon on july 20, 1969, at 20:18 UTC. Armstrong beceme the first to stp onto the luner surface 6 hours later on July 21 at 02:56 UTC. Armstrong spent abut three and a half two and a hakf hours outside the spacecraft, Aldrin slghtely less; and tgether they colected 47.5 pounds (21.5 kg) of lunar material for returne to Earth. A third member of the mission, Michael Collins, pilloted the comand spacecraft alone in lunar orbit untl Armstrong and Aldrin returned to it for the trep back to Earth.",
+    "Set theory is a branch of mathematical logic that studyes sets, which informally are colections of objects. Although any type of object can be collected into a set, set theory is applyed most often to objects that are relevant to mathematics. The language of set theory can be used to define nearly all mathematical objects. Set theory is commonly employed as a foundational system for mathematics. Beyond its foundational role, set theory is a branch of mathematics in its own right, with an active resurch community. Contemporary resurch into set theory includes a divers colection of topics, ranging from the structer of the real number line to the study of the consistency of large cardinals.",
+    "In mathematics and transporation enginering, traffic flow is the study of interctions between travellers (including podestrians, ciclists, drivers, and their vehicles) and inferstructure (including highways, signage, and traffic control devices), with the aim of understanding and developing an optimal transport network with eficiant movement of traffic and minimal traffic congestion problems. Current traffic models use a mixture of emperical and theoretical techniques. These models are then developed into traffic forecasts, and take account of proposed local or major changes, such as incrased vehicle use, changes in land use or changes in mode of transport (with people moving from bus to train or car, for example), and to identify areas of congestion where the network needs to be ajusted.",
+    "Critical theory is a social pholosiphy pertaning to the reflective asessment and critique of society and culture in order to reveal and challange power structures. With origins in socology, as well as in literary criticism, it argues that social problems are influenced and created more by societal structures and cultural assumptions than by individual and psychological factors. Mantaining that ideology is the principal obsticle to human liberation, critical theory was establiched as a school of thought primarily by the Frankfurt School theoreticians. One sociologist described a theory as critical insofar as it seeks 'to liberate human beings from the circomstances that enslave them.",
+    "In computitional languistics, lemmatisation is the algorhythmic process of determining the lemma of a word based on its intended meaning. Unlike stemming, lemmatisation depends on corectly identifing the intended part of speech and meaning of a word in a sentence, as well as within the larger context surounding that sentence, such as neigboring sentences or even an entire document. As a result, devleoping efficient lemmatisation algorithums is an open area of research. In many langauges, words appear in several inflected forms. For example, in English, the verb 'to walk' may appear as 'walk', 'walked', 'walks' or 'walking'. The base form, 'walk', that one might look up in a dictionery, is called the lemma for the word. The asociation of the base form with a part of speech is often called a lexeme of the word."
+]
+
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it for yourself!")
+
+if custom_input:
+    selected_text = custom_input
+elif selected_text:
+    selected_text = selected_text
+
+st.subheader("Selected Text:")
+st.write(selected_text)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, selected_text)
+
+# Display output sentence
+correction_dict_filtered = {token: corrected for token, corrected in zip(output['token'], output['corrected']) if token != corrected}
+
+def generate_html_with_corrections(sentence, corrections):
+    corrected_html = sentence
+    for incorrect, correct in corrections.items():
+        corrected_html = corrected_html.replace(incorrect, f'<span style="text-decoration: line-through; color: red;">{incorrect}</span> <span style="color: green;">{correct}</span>')
+    return f'<div style="font-family: Arial, sans-serif; font-size: 16px;">{corrected_html}</div>'
+
+corrected_html_snippet = generate_html_with_corrections(selected_text, correction_dict_filtered)
+st.subheader(f"Misspells Detected:")
+st.markdown(corrected_html_snippet, unsafe_allow_html=True)

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,234 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+        .sidebar-content {
+            font-size: 16px;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Introduction
+st.markdown('<div class="main-title">Correcting Typos and Spelling Errors with Spark NLP and Python</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>Correcting typos and spelling errors is an essential task in NLP pipelines. Ensuring data correctness can significantly improve the performance of machine learning models. In this article, we will explore how to perform spell checking using rule-based and machine learning-based models in Spark NLP with Python.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Background
+st.markdown('<div class="sub-title">Introduction</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Spell checking identifies words in texts that have spelling errors or are misspelled. Text data from social media or extracted using Optical Character Recognition (OCR) often contains typos, misspellings, or spurious symbols that can impact machine learning models.</p>
+    <p>Having spelling errors in data can reduce model performance. For example, if "John" appears as "J0hn", the model treats them as two separate words, complicating the model and reducing its effectiveness. Spell checking and correction can preprocess data to improve model training.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Spell Checking in Spark NLP
+st.markdown('<div class="sub-title">Spell Checking in Spark NLP</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Spark NLP provides three approaches for spell checking and correction:</p>
+    <ul>
+        <li><strong>NorvigSweetingAnnotator:</strong> Based on Peter Norvig’s algorithm with modifications like limiting vowel swapping and using Hamming distance.</li>
+        <li><strong>SymmetricDeleteAnnotator:</strong> Based on the SymSpell algorithm.</li>
+        <li><strong>ContextSpellCheckerAnnotator:</strong> A deep learning model using contextual information for error detection and correction.</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Example Code
+st.markdown('<div class="sub-title">Example Code</div>', unsafe_allow_html=True)
+st.markdown('<p>Here is an example of how to use these models in Spark NLP:</p>', unsafe_allow_html=True)
+
+# Step-by-step code
+st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
+st.markdown('<p>To install Spark NLP in Python, use your favorite package manager (conda, pip, etc.). For example:</p>', unsafe_allow_html=True)
+st.code("""
+pip install spark-nlp
+pip install pyspark
+""", language="bash")
+
+st.markdown('<p>Then, import Spark NLP and start a Spark session:</p>', unsafe_allow_html=True)
+st.code("""
+import sparknlp
+
+# Start Spark Session
+spark = sparknlp.start()
+""", language='python')
+
+# Step 1: Document Assembler
+st.markdown('<div class="sub-title">Step 1: Document Assembler</div>', unsafe_allow_html=True)
+st.markdown('<p>Transform raw texts to document annotation:</p>', unsafe_allow_html=True)
+st.code("""
+from sparknlp.base import DocumentAssembler
+
+documentAssembler = DocumentAssembler()\\
+    .setInputCol("text")\\
+    .setOutputCol("document")
+""", language='python')
+
+# Step 2: Tokenization
+st.markdown('<div class="sub-title">Step 2: Tokenization</div>', unsafe_allow_html=True)
+st.markdown('<p>Split text into individual tokens:</p>', unsafe_allow_html=True)
+st.code("""
+from sparknlp.annotator import Tokenizer
+
+tokenizer = Tokenizer()\\
+    .setInputCols(["document"])\\
+    .setOutputCol("token")
+""", language='python')
+
+# Step 3: Spell Checker Models
+st.markdown('<div class="sub-title">Step 3: Spell Checker Models</div>', unsafe_allow_html=True)
+st.markdown('<p>Choose and load one of the spell checker models:</p>', unsafe_allow_html=True)
+
+st.code("""
+from sparknlp.annotator import ContextSpellCheckerModel, NorvigSweetingModel, SymmetricDeleteModel
+
+# One of the spell checker annotators
+symspell = SymmetricDeleteModel.pretrained("spellcheck_sd")\\
+    .setInputCols(["token"])\\
+    .setOutputCol("symspell")
+
+norvig = NorvigSweetingModel.pretrained("spellcheck_norvig")\\
+    .setInputCols(["token"])\\
+    .setOutputCol("norvig")
+
+context = ContextSpellCheckerModel.pretrained("spellcheck_dl")\\
+    .setInputCols(["token"])\\
+    .setOutputCol("context")
+""", language='python')
+
+# Step 4: Pipeline Definition
+st.markdown('<div class="sub-title">Step 4: Pipeline Definition</div>', unsafe_allow_html=True)
+st.markdown('<p>Define the pipeline stages:</p>', unsafe_allow_html=True)
+st.code("""
+from pyspark.ml import Pipeline
+
+# Define the pipeline stages
+pipeline = Pipeline().setStages([documentAssembler, tokenizer, symspell, norvig, context])
+""", language='python')
+
+# Step 5: Fitting and Transforming
+st.markdown('<div class="sub-title">Step 5: Fitting and Transforming</div>', unsafe_allow_html=True)
+st.markdown('<p>Fit the pipeline and transform the data:</p>', unsafe_allow_html=True)
+st.code("""
+# Create an empty DataFrame to fit the pipeline
+empty_df = spark.createDataFrame([[""]]).toDF("text")
+pipelineModel = pipeline.fit(empty_df)
+
+# Example text for correction
+example_df = spark.createDataFrame([["Plaese alliow me tao introdduce myhelf, I am a man of wealth und tiaste"]]).toDF("text")
+result = pipelineModel.transform(example_df)
+""", language='python')
+
+# Step 6: Displaying Results
+st.markdown('<div class="sub-title">Step 6: Displaying Results</div>', unsafe_allow_html=True)
+st.markdown('<p>Show the results from the different spell checker models:</p>', unsafe_allow_html=True)
+st.code("""
+# Show results
+result.selectExpr("norvig.result as norvig", "symspell.result as symspell", "context.result as context").show(truncate=False)
+""", language='python')
+
+st.markdown("""
+<p>The output from the example code will show the corrected text using three different models:</p>
+<table>
+  <tr>
+    <th>norvig</th>
+    <th>symspell</th>
+    <th>context</th>
+  </tr>
+  <tr>
+    <td>[Please, allow, me, tao, introduce, myself, ,, I, am, a, man, of, wealth, und, taste]</td>
+    <td>[Place, allow, me, to, introduce, myself, ,, I, am, a, man, of, wealth, und, taste]</td>
+    <td>[Please, allow, me, to, introduce, myself, ,, I, am, a, man, of, wealth, and, taste]</td>
+  </tr>
+</table>
+""", unsafe_allow_html=True)
+
+# One-liner Alternative
+st.markdown('<div class="sub-title">One-liner Alternative</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Introducing the <code>johnsnowlabs</code> library: In October 2022, John Snow Labs released a unified open-source library containing all their products under one roof. This includes Spark NLP, Spark NLP Display, and NLU. Simplify your workflow with:</p>
+    <p><code>pip install johnsnowlabs</code></p>
+    <p>For spell checking, use one line of code:</p>
+    <pre>
+    <code class="language-python">
+# Import the NLP module which contains Spark NLP and NLU libraries
+from johnsnowlabs import nlp
+# Use Norvig model
+nlp.load("en.spell.norvig").predict("Plaese alliow me tao introdduce myhelf, I am a man of wealth und tiaste", output_level='token')
+    </code>
+    </pre>
+</div>
+""", unsafe_allow_html=True)
+
+st.image('images/johnsnowlabs-output.png', use_column_width='auto')
+
+# Conclusion
+st.markdown("""
+<div class="section">
+    <h2>Conclusion</h2>
+    <p>We introduced three models for spell checking and correction in Spark NLP: NorvigSweeting, SymmetricDelete, and ContextSpellChecker. These models can be integrated into Spark NLP pipelines for efficient processing of large datasets.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References
+st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/docs/en/annotators#norvigsweeting-spellchecker" target="_blank" rel="noopener">NorvigSweeting</a> documentation page</li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/docs/en/annotators#symmetricdelete-spellchecker" target="_blank" rel="noopener">SymmetricDeleter</a> documentation page</li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/docs/en/annotators#contextspellchecker" target="_blank" rel="noopener">ContextSpellChecker</a> documentation page</li>
+        <li><a class="link" href="https://medium.com/spark-nlp/applying-context-aware-spell-checking-in-spark-nlp-3c29c46963bc" target="_blank" rel="noopener nofollow">Applying Context Aware Spell Checking in Spark NLP</a></li>
+        <li><a class="link" href="https://towardsdatascience.com/training-a-contextual-spell-checker-for-italian-language-66dda528e4bf" target="_blank" rel="noopener nofollow">Training a Contextual Spell Checker for Italian Language</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+pandas
+numpy
+spark-nlp
+pyspark