Upload 6 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+images/text[[:space:]]preprocessing[[:space:]]visual.jpg filter=lfs diff=lfs merge=lfs -text

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,184 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+import numpy as np
+import pyspark.sql.functions as F
+from pyspark.ml import Pipeline
+from pyspark.sql import SparkSession
+from sparknlp.annotator import *
+from sparknlp.base import *
+from sparknlp.pretrained import PretrainedPipeline
+from pyspark.sql.types import StringType, IntegerType
+
+# Configure Streamlit page
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    spark = sparknlp.start()
+    return spark
+
+@st.cache_resource
+def create_pipeline():
+
+    documentAssembler = DocumentAssembler()\
+        .setInputCol("text")\
+        .setOutputCol("document")
+
+    sentenceDetector = SentenceDetector()\
+        .setInputCols(['document'])\
+        .setOutputCol('sentences')
+
+    tokenizer = Tokenizer() \
+        .setInputCols(["document"]) \
+        .setOutputCol("token")
+
+    normalizer = Normalizer() \
+        .setInputCols(["token"]) \
+        .setOutputCol("normalized")\
+        .setLowercase(True)\
+        .setCleanupPatterns(["[^\w\d\s]"])
+
+    stopwords_cleaner = StopWordsCleaner()\
+        .setInputCols("token")\
+        .setOutputCol("removed_stopwords")\
+        .setCaseSensitive(False)\
+
+    stemmer = Stemmer() \
+        .setInputCols(["token"]) \
+        .setOutputCol("stem")
+
+    lemmatizer = Lemmatizer() \
+        .setInputCols(["token"]) \
+        .setOutputCol("lemma") \
+        .setDictionary("./AntBNC_lemmas_ver_001.txt", value_delimiter ="\t", key_delimiter = "->")
+
+    nlpPipeline = Pipeline(stages=[documentAssembler,
+                                sentenceDetector,
+                                tokenizer,
+                                normalizer,
+                                stopwords_cleaner,
+                                stemmer,
+                                lemmatizer])
+
+    return nlpPipeline
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    result = model.fullAnnotate(data)
+    return result
+
+def extract_annotations(output, annotation_type):
+    return [anno.result for anno in output[0][annotation_type]]
+
+# st.title("Summarize Text")
+st.markdown('<div class="main-title">State-of-the-Art Text Preprocessing with Spark NLP</div>', unsafe_allow_html=True)
+st.write("")
+st.write("")
+st.markdown("""
+<div class="section">
+    <p>This demo utilizes a comprehensive text preprocessing pipeline using Spark NLP. The pipeline includes several stages such as document assembly, tokenization, sentence detection, normalization, stopword cleaning, stemming, and lemmatization. These steps are essential for preparing text data for downstream NLP tasks, ensuring the text is clean and standardized for effective model training and evaluation.</p>
+</div>
+""", unsafe_allow_html=True)
+
+model_name = "SentenceDetector|Tokenizer|Stemmer|Lemmatizer|Normalizer|Stop Words Remover"
+#model = st.sidebar.selectbox("Choose the pretrained model", model_name, help="For more info about the models visit: https://sparknlp.org/models",)
+
+st.sidebar.title("Filter Annotator Outputs")
+selected_models = []
+for model in model_name.split('|'):
+  check = st.sidebar.checkbox(model, value=True, key=model)
+  selected_models.append(check)
+
+# # Let's add the colab link for the notebook.
+
+link= """<a href="https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5TRANSFORMER.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/></a>"""
+st.sidebar.title('')
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+examples = [
+
+    "The Geneva Motor Show, the first major car show of the year, opens tomorrow with U.S. Car makers hoping to make new inroads into European markets due to the cheap dollar, automobile executives said. Ford Motor Co and General Motors Corp sell cars in Europe, where about 10.5 mln new cars a year are bought. GM also makes a few thousand in North American plants for European export.",
+    "Demonicus is a movie turned into a video game! I just love the story and the things that goes on in the film.It is a B-film ofcourse but that doesn`t bother one bit because its made just right and the music was rad! Horror and sword fight freaks,buy this movie now!",
+    "Quantum computing is the use of quantum-mechanical phenomena such as superposition and entanglement to perform computation. Computers that perform quantum computations are known as quantum computers. Quantum computers are believed to be able to solve certain computational problems, such as integer factorization (which underlies RSA encryption), substantially faster than classical computers. The study of quantum computing is a subfield of quantum information science. Quantum computing began in the early 1980s, when physicist Paul Benioff proposed a quantum mechanical model of the Turing machine.",
+    "Titanic is a 1997 American epic romance and disaster film directed, written, co-produced, and co-edited by James Cameron. Incorporating both historical and fictionalized aspects, it is based on accounts of the sinking of the RMS Titanic, and stars Leonardo DiCaprio and Kate Winslet as members of different social classes who fall in love aboard the ship during its ill-fated maiden voyage.",
+    "William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000.[9] He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.",
+    """World War II (often abbreviated as WWII or WW2), also known as the Second World War, was a global war that lasted from 1939 to 1945. The vast majority of the world's countries—including all the great powers—eventually formed two opposing military alliances: the Allies and the Axis. A state of total war emerged, directly involving more than 100 million people from more than 30 countries. The major participants threw their entire economic, industrial, and scientific capabilities behind the war effort, blurring the distinction between civilian and military resources. World War II was the deadliest conflict in human history, marked by 70 to 85 million fatalities, most of whom were civilians in the Soviet Union and China. Tens of millions of people died during the conflict due to genocides (including the Holocaust), premeditated death from starvation, massacres, and disease. Aircraft played a major role in the conflict which included the use of terror bombing, strategic bombing and the only use of nuclear weapons in war.""",
+    "Disney Channel (originally called The Disney Channel from 1983 to 1997 and commonly shortened to Disney from 1997 to 2002) is an American pay television channel that serves as the flagship property of owner Disney Channels Worldwide unit of the Walt Disney Television subsidiary of The Walt Disney Company. Disney Channel's programming consists of original first-run television series, theatrically released and original made-for-TV movies and select other third-party programming. Disney Channel – which formerly operated as a premium service – originally marketed its programs towards families during the 1980s, and later at younger children by the 2000s.",
+    "For several hundred thousand years, the Sahara has alternated between desert and savanna grassland in a 20,000 year cycle[8] caused by the precession of the Earth's axis as it rotates around the Sun, which changes the location of the North African Monsoon. The area is next expected to become green in about 15,000 years (17,000 ACE).",
+    "Elon Musk is an engineer, industrial designer, technology entrepreneur and philanthropist. He is a citizen of South Africa, Canada, and the United States. He is the founder, CEO and chief engineer/designer of SpaceX; early investor, CEO and product architect of Tesla, Inc.; founder of The Boring Company; co-founder of Neuralink; and co-founder and initial co-chairman of OpenAI. He was elected a Fellow of the Royal Society (FRS) in 2018. In December 2016, he was ranked 21st on the Forbes list of The World's Most Powerful People, and was ranked joint-first on the Forbes list of the Most Innovative Leaders of 2019. A self-made billionaire, as of June 2020 his net worth was estimated at $38.8 billion and he is listed by Forbes as the 31st-richest person in the world. He is the longest tenured CEO of any automotive manufacturer globally.",
+    "Born and raised in the Austrian Empire, Tesla studied engineering and physics in the 1870s without receiving a degree, and gained practical experience in the early 1880s working in telephony and at Continental Edison in the new electric power industry. In 1884 he emigrated to the United States, where he became a naturalized citizen. He worked for a short time at the Edison Machine Works in New York City before he struck out on his own. With the help of partners to finance and market his ideas, Tesla set up laboratories and companies in New York to develop a range of electrical and mechanical devices. His alternating current (AC) induction motor and related polyphase AC patents, licensed by Westinghouse Electric in 1888, earned him a considerable amount of money and became the cornerstone of the polyphase system which that company eventually marketed."
+
+]
+
+st.subheader("Split and clean text")
+
+selected_text = st.selectbox("Select an example", examples)
+
+custom_input = st.text_input("Try it for yourself!")
+
+if custom_input:
+    selected_text = custom_input
+elif selected_text:
+    selected_text = selected_text
+
+st.subheader('Selected Text')
+st.write(selected_text)
+
+spark = init_spark()
+Pipeline = create_pipeline()
+output = fit_data(Pipeline, selected_text)
+
+data_dict = {}
+
+if selected_models[0]:
+    sentences = extract_annotations(output, 'sentences')
+    data_dict['sentences'] = sentences
+
+if selected_models[1]:
+    tokens = extract_annotations(output, 'token')
+    data_dict['token'] = tokens
+
+if selected_models[2]:
+    stems = extract_annotations(output, 'stem')
+    data_dict['stem'] = stems
+
+if selected_models[3]:
+    lemmas = extract_annotations(output, 'lemma')
+    data_dict['lemma'] = lemmas
+
+if selected_models[4]:
+    normalized = extract_annotations(output, 'normalized')
+    data_dict['normalized'] = normalized
+
+if selected_models[5]:
+    removed_stopwords = extract_annotations(output, 'removed_stopwords')
+    data_dict['removed_stopwords'] = removed_stopwords
+
+if data_dict:
+    df = pd.DataFrame(dict([(k, pd.Series(v)) for k, v in data_dict.items()]))
+    st.subheader("Annotation Results:")
+    st.dataframe(df)

Dockerfile

@@ -0,0 +1,76 @@
+#Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+ENV PYSPARK_PYTHON=python3
+ENV PYSPARK_DRIVER_PYTHON=python3
+
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz
+
+RUN adduser --disabled-password \
+    --gecos "Default user" \
+    --uid ${NB_UID} \
+    ${NB_USER}
+
+# Install OpenJDK-8
+RUN apt-get update && \
+    apt-get install -y openjdk-8-jdk && \
+    apt-get install -y ant && \
+    apt-get clean;
+
+# Fix certificate issues
+RUN apt-get update && \
+    apt-get install ca-certificates-java && \
+    apt-get clean && \
+    update-ca-certificates -f;
+# Setup JAVA_HOME -- useful for docker commandline
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN export JAVA_HOME
+
+RUN echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ~/.bashrc
+
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN apt-get update
+RUN apt-get install -y software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get install -y python3.8 python3-pip
+
+ENV PYSPARK_PYTHON=python3.8
+ENV PYSPARK_DRIVER_PYTHON=python3.8
+
+COPY . .
+
+RUN python3.8 -m pip install --upgrade pip
+RUN python3.8 -m pip install -r requirements.txt
+
+USER root
+RUN chown -R ${NB_UID} ${HOME}
+USER ${NB_USER}
+
+WORKDIR ${HOME}
+
+COPY . .
+
+EXPOSE 7860
+
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,380 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #333333;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Introduction
+st.markdown('<div class="main-title">Text Preprocessing with Spark NLP</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>Welcome to the Spark NLP Text Preprocessing Demo App! In the field of Natural Language Processing (NLP), preprocessing is a crucial step that ensures the text data is clean and suitable for modeling. Effective preprocessing can significantly enhance the performance of NLP models.</p>
+    <p>Spark NLP stands out as a leading library for text preprocessing, offering a range of tools and models within an easy-to-use pipeline design compatible with Apache Spark. This demo showcases how you can leverage Spark NLP to preprocess your text data efficiently.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# About Text Preprocessing
+st.markdown('<div class="sub-title">About Text Preprocessing</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Text preprocessing involves a series of steps to clean and normalize text data. Common tasks include tokenization, stopword removal, stemming, lemmatization, and more. These steps are essential for preparing raw text for downstream NLP tasks.</p>
+    <p>In Spark NLP, text preprocessing is facilitated through various annotators that can be combined into a preprocessing pipeline. We'll demonstrate how to use these annotators in Python to preprocess text data effectively.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.image('https://www.johnsnowlabs.com/wp-content/uploads/2023/05/img_blog_2-4.jpg', caption='Text preprocessing pipeline visual', use_column_width='auto')
+
+# How to Use the Preprocessing Tools
+st.markdown('<div class="sub-title">How to Use Spark NLP for Text Preprocessing</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>To preprocess text using Spark NLP, we need to create a pipeline that includes various preprocessing annotators. These annotators transform the input text through steps like tokenization, normalization, and stopword removal.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Installation</div>', unsafe_allow_html=True)
+st.code('!pip install spark-nlp', language='python')
+
+# Import Libraries and Read Data
+st.markdown('<div class="sub-title">Importing Libraries and Reading Data</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>First, we'll import Spark NLP and necessary libraries, read the data from a local file, and convert it into a Spark DataFrame.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+import sparknlp
+from sparknlp.base import *
+from sparknlp.annotator import *
+
+spark= sparknlp.start()
+
+df= spark.read\\
+    .option("header", True)\\
+    .csv("spam_text_messages.csv")\\
+    .toDF("category", "text")
+df.show(5, truncate=30)
+
+>>>
++--------+------------------------------+
+|category|                          text|
++--------+------------------------------+
+|     ham|Go until jurong point, craz...|
+|     ham| Ok lar... Joking wif u oni...|
+|    spam|Free entry in 2 a wkly comp...|
+|     ham|U dun say so early hor... U...|
+|     ham|Nah I don't think he goes t...|
++--------+------------------------------+
+only showing top 5 rows
+""", language='python')
+
+st.markdown("""
+<div class="section">
+    <p>The dataset contains two columns: <strong>category</strong> and <strong>text</strong>. The text column contains messages, and the category column indicates whether the message is spam or not (ham).</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Document Assembler
+st.markdown('<div class="sub-title">Document Assembler</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The DocumentAssembler is the beginning part of any Spark NLP project. It creates the first annotation of type Document, which may be used by annotators down the road. We use it as follows:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+documentAssembler = DocumentAssembler() \\
+    .setInputCol("text") \\
+    .setOutputCol("document") \\
+    .setCleanupMode("shrink")
+
+df_doc = documentAssembler.transform(df)
+df_doc.printSchema()
+""", language='python')
+
+st.markdown("""
+<div class="section">
+    <p>The <code>shrink</code> cleanup mode removes new lines and tabs, merging multiple spaces and blank lines into a single space. The schema after transformation includes the new <strong>document</strong> column.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Tokenizer
+st.markdown('<div class="sub-title">Tokenizer</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The Tokenizer identifies the tokens in Spark NLP:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+tokenizer = Tokenizer() \\
+    .setInputCols(["document"]) \\
+    .setOutputCol("token")
+""", language='python')
+
+# Sentence Detector
+st.markdown('<div class="sub-title">Sentence Detector</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The SentenceDetector finds sentence boundaries in raw text:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+sentenceDetector = SentenceDetector() \\
+    .setInputCols(["document"]) \\
+    .setOutputCol("sentence")
+""", language='python')
+
+# Normalizer
+st.markdown('<div class="sub-title">Normalizer</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The Normalizer cleans dirty characters after regex pattern and removes words based on a given dictionary:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+normalizer = Normalizer() \\
+    .setInputCols(["token"]) \\
+    .setOutputCol("normalized") \\
+    .setLowercase(True) \\
+    .setCleanupPatterns(["[^\w\d\s]"])
+""", language='python')
+
+# Stopwords Cleaner
+st.markdown('<div class="sub-title">Stopwords Cleaner</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The StopWordsCleaner removes stopwords from the text:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+stopwordsCleaner = StopWordsCleaner() \\
+    .setInputCols(["token"]) \\
+    .setOutputCol("cleaned_tokens") \\
+    .setCaseSensitive(True)
+""", language='python')
+
+# Token Assembler
+st.markdown('<div class="sub-title">Token Assembler</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The TokenAssembler assembles cleaned tokens back together:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+tokenAssembler = TokenAssembler() \\
+    .setInputCols(["sentence", "cleaned_tokens"]) \\
+    .setOutputCol("assembled")
+""", language='python')
+
+# Stemmer
+st.markdown('<div class="sub-title">Stemmer</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The Stemmer reduces inflectional forms and sometimes derivationally related forms of a word to a common base form:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+stemmer = Stemmer() \\
+    .setInputCols(["token"]) \\
+    .setOutputCol("stem")
+""", language='python')
+
+# Lemmatizer
+st.markdown('<div class="sub-title">Lemmatizer</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The Lemmatizer reduces words to their base or root form, referring to a dictionary to understand the word's meaning:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+lemmatizer = Lemmatizer() \\
+    .setInputCols(["token"]) \\
+    .setOutputCol("lemma") \\
+    .setDictionary("AntBNC_lemmas_ver_001.txt", value_delimiter="\\t", key_delimiter="->")
+""", language='python')
+
+# Pipeline
+st.markdown('<div class="sub-title">Putting All Processes into a Spark ML Pipeline</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Now, we will put all the preprocessing stages into a Spark ML Pipeline and apply it to our dataset.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+from pyspark.ml import Pipeline
+
+nlpPipeline = Pipeline(stages=[
+    documentAssembler,
+    tokenizer,
+    sentenceDetector,
+    normalizer,
+    stopwordsCleaner,
+    tokenAssembler,
+    stemmer,
+    lemmatizer
+])
+
+empty_df = spark.createDataFrame([[""]]).toDF("text")
+model = nlpPipeline.fit(empty_df)
+result = model.transform(df)
+""", language='python')
+
+# Showcase/Example
+st.markdown('<div class="sub-title">Showcase/Example</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Let's examine the results of our preprocessing pipeline, starting with tokens and normalized tokens:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+from pyspark.sql import functions as F
+
+result.select("token.result", "normalized.result").show(5, truncate=30)
+
+>>>
++------------------------------+------------------------------+
+|                        result|                        result|
++------------------------------+------------------------------+
+|[Go, until, jurong, point, ...|[go, until, jurong, point, ...|
+|[Ok, lar, ..., Joking, wif,...|[ok, lar, joking, wif, u, oni]|
+|[Free, entry, in, 2, a, wkl...|[free, entry, in, 2, a, wkl...|
+|[U, dun, say, so, early, ho...|[u, dun, say, so, early, ho...|
+|[Nah, I, don't, think, he, ...|[nah, ı, dont, think, he, g...|
++------------------------------+------------------------------+
+only showing top 5 rows
+""", language='python')
+
+st.markdown("""
+<div class="section">
+    <p>Next, we check the cleaned data from stopwords:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+result.select(F.explode(F.arrays_zip("token.result", "cleaned_tokens.result")).alias("col")) \\
+      .select(F.expr("col['0']").alias("token"), F.expr("col['1']").alias("cleaned_sw")).show(10)
+
+>>>
++------------------------------+------------------------------+
+|                        result|                        result|
++------------------------------+------------------------------+
+|[Go, until, jurong, point, ...|[go, until, jurong, point, ...|
+|[Ok, lar, ..., Joking, wif,...|[ok, lar, joking, wif, u, oni]|
+|[Free, entry, in, 2, a, wkl...|[free, entry, in, 2, a, wkl...|
+|[U, dun, say, so, early, ho...|[u, dun, say, so, early, ho...|
+|[Nah, I, don't, think, he, ...|[nah, ı, dont, think, he, g...|
++------------------------------+------------------------------+
+only showing top 5 rows
+""", language='python')
+
+st.markdown("""
+<div class="section">
+    <p>Finally, we compare the sentence detector result and token assembler result:</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+result.select(F.explode(F.arrays_zip("sentence.result", "assembled.result")).alias("col")) \\
+      .select(F.expr("col['0']").alias("sentence"), F.expr("col['1']").alias("assembled")).show(5, truncate=30)
+
+>>>
++------------------------------+------------------------------+
+|                      sentence|                     assembled|
++------------------------------+------------------------------+
+| Go until jurong point, crazy.|        Go jurong point, crazy|
+|                             .|                              |
+|Available only in bugis n g...|Available bugis n great wor...|
+|     Cine there got amore wat.|            Cine got amore wat|
+|                             .|                              |
++------------------------------+------------------------------+
+only showing top 5 rows
+
+
+result.withColumn("tmp", F.explode("assembled")) \\
+    .select("tmp.*").select("begin", "end", "result", "metadata.sentence").show(5, truncate=30)
+
+>>>
++-----+---+------------------------------+--------+
+|begin|end|                        result|sentence|
++-----+---+------------------------------+--------+
+|    0| 21|        Go jurong point, crazy|       0|
+|   29| 28|                              |       1|
+|   31| 74|Available bugis n great wor...|       2|
+|   84|101|            Cine got amore wat|       3|
+|  109|108|                              |       4|
++-----+---+------------------------------+--------+
+only showing top 5 rows
+""", language='python')
+
+st.markdown("""
+<div class="section">
+    <p>In this example, we have successfully cleaned and preprocessed text data using various annotators and transformers in Spark NLP. This preprocessing pipeline is essential for preparing the data for further NLP tasks, ensuring that the text is clean and normalized.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Additional Resources and References</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/1hr_workshop/SparkNLP_openSource_workshop_1hr.ipynb" target="_blank">Intro to Spark NLP workflow</a></li>
+        <li><a class="link" href="https://sparknlp.org/docs/en/quickstart" target="_blank">Getting Started with Spark NLP</a></li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/models" target="_blank">Pretrained Models</a></li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp/tree/master/examples/python/annotation/text/english" target="_blank">Example Notebooks</a></li>
+        <li><a class="link" href="https://sparknlp.org/docs/en/install" target="_blank">Installation Guide</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+pandas
+numpy
+spark-nlp
+pyspark