Upload 11 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,116 @@
+import streamlit as st
+import sparknlp
+import os
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Initialize Spark NLP
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+# Create Spark NLP pipeline
+@st.cache_resource
+def create_pipeline(year, month, data):
+    document_assembler = DocumentAssembler()\
+        .setInputCol("text")\
+        .setOutputCol("document")
+
+    sentence_detector = SentenceDetector()\
+        .setInputCols("document")\
+        .setOutputCol("sentence")
+
+    date_matcher = DateMatcher() \
+        .setInputCols('sentence')\
+        .setOutputCol("date")\
+        .setAnchorDateYear(year)\
+        .setAnchorDateMonth(month)\
+        .setAnchorDateDay(date)
+
+    pipeline1= Pipeline(
+        stages=[ 
+            document_assembler, 
+            sentence_detector,
+            date_matcher,
+            ])
+
+    return pipeline1
+
+# Fit data and get results
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    results = model.fullAnnotate(data)[0]
+
+    return [res.result for res in results['matched_text']]
+
+# Set up the page layout
+st.markdown('<div class="main-title">State-of-the-Art Date Detecting and normalization with Spark NLP</div>', unsafe_allow_html=True)
+st.write("")
+
+# Sidebar content
+date = st.sidebar.date_input('Select reference date')
+year, month, day = map(int, str(date).split('-'))
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/DATE_MATCHER.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.title('')
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples from files
+folder_path = f"inputs\date_matcher"
+examples = [
+    lines[1].strip() 
+    for filename in os.listdir(folder_path) 
+    if filename.endswith('.txt') 
+    for lines in [open(os.path.join(folder_path, filename), 'r', encoding='utf-8').readlines()] 
+    if len(lines) >= 2
+]
+
+st.subheader("Automatically detect phrases expressing dates and normalize them with respect to a reference date.")
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own Sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.subheader('Full example text')
+st.write(text_to_analyze)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model, labels_to_match)
+output = fit_data(pipeline, text_to_analyze)
+
+# Display matched sentence
+st.subheader("Dates matched:")

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

inputs/date_matcher/Example1.txt

@@ -0,0 +1,5 @@
+David visited the restaurant yesterday with his family.  He also visited and the day before, but at ... 
+David visited the restaurant yesterday with his family. 
+He also visited and the day before, but at that time he was alone.
+David again visited today with his colleagues.
+He and his friends really liked the food and hoped to visit again tomorrow.

inputs/date_matcher/Example2.txt

@@ -0,0 +1,5 @@
+In March 2003 she was seen in the office and appeared to be extremely disturbed emotionally. On 2003... 
+In March 2003 she was seen in the office and appeared to be extremely disturbed emotionally.
+On 2003-04-04 she again visited and talked about the effects of the medication she has been taking, and seemed positive and in much better shape.
+She again visited on Fri, 22/4/2003 and looked better.
+She has been working out and taking her medicines since April 1st 2003.

inputs/date_matcher/Example3.txt

@@ -0,0 +1,6 @@
+I have a very busy schedule these days. I have meetings from 7pm. till 11pm. I have 3 meetings the d... 
+I have a very busy schedule these days. I have meetings from 7pm. till 11pm.
+I have 3 meetings the day after, and have submission deadlines approaching as well.
+By next mon I have to finalise the architecture, for which i'll have to hold multiple meetings with ARM.
+Then i'll have to discuss dev plans by next tuesday and develop a thorough plan.
+The plan should be ready by Nov 30th.

inputs/date_matcher/Example4.txt

@@ -0,0 +1,4 @@
+When Tom visited the Bahamas last year, it was his first time travelling. Since then he was travelle... 
+When Tom visited the Bahamas last year, it was his first time travelling.
+Since then he was travelled a lot. For example, he visited Hawaii last week.
+The last time we talked, he was planning to travel to Alaska next month.

inputs/date_matcher/Example5.txt

@@ -0,0 +1,3 @@
+Isn't it weird that all my family members have the same birth day and month? All of us were born on ... 
+Isn't it weird that all my family members have the same birth day and month? All of us were born on 1st Jan
+    Dad was born on 01/01/1900. Mom has a birth date of 1st Jan 1902. And I was born on 2000/01/01

pages/Workflow & Model Overview.py

@@ -0,0 +1,311 @@
+import streamlit as st
+from sparknlp.base import DocumentAssembler, Pipeline
+from sparknlp.annotator import DateMatcher, MultiDateMatcher
+from pyspark.sql.types import StringType
+import pyspark.sql.functions as F
+import sparknlp
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Introduction
+st.markdown('<div class="main-title">Date Extraction with Spark NLP</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>Welcome to the Spark NLP Date Extraction Demo App! Date extraction is a crucial task in Natural Language Processing (NLP) that involves identifying and extracting references to dates in text data. This can be useful for a wide range of applications, such as event scheduling, social media monitoring, and financial forecasting.</p>
+    <p>Using Spark NLP, it is possible to identify and extract dates from a text with high accuracy. This app demonstrates how to use the DateMatcher and MultiDateMatcher annotators to extract dates from text data.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.image('images/Extracting-Exact-Dates.jpg', use_column_width='auto')
+
+# About Date Extraction
+st.markdown('<div class="sub-title">About Date Extraction</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Date extraction involves identifying and extracting references to dates in text data. This can be achieved using various techniques such as regular expressions, Named Entity Recognition (NER), and rule-based systems.</p>
+    <p>Spark NLP provides powerful tools for date extraction, including the DateMatcher and MultiDateMatcher annotators, which use pattern matching to extract date expressions from text.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Using DateMatcher in Spark NLP
+st.markdown('<div class="sub-title">Using DateMatcher in Spark NLP</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The DateMatcher annotator in Spark NLP allows users to extract specific date patterns from text data. This annotator can identify dates in various formats, providing valuable insights from unstructured text data.</p>
+    <p>The DateMatcher annotator in Spark NLP offers:</p>
+    <ul>
+        <li>Flexible date pattern matching</li>
+        <li>Extraction of single date occurrences</li>
+        <li>Efficient processing of large text datasets</li>
+        <li>Integration with other Spark NLP components for comprehensive NLP pipelines</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<h2 class="sub-title">Example Usage in Python</h2>', unsafe_allow_html=True)
+st.markdown('<p>Here’s how you can implement DateMatcher and MultiDateMatcher annotators in Spark NLP:</p>', unsafe_allow_html=True)
+
+# Setup Instructions
+st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
+st.markdown('<p>To install Spark NLP in Python, use your favorite package manager (conda, pip, etc.). For example:</p>', unsafe_allow_html=True)
+st.code("""
+pip install spark-nlp
+pip install pyspark
+""", language="bash")
+
+st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
+st.code("""
+import sparknlp
+
+# Start Spark Session
+spark = sparknlp.start()
+""", language='python')
+
+# Single Date Extraction Example
+st.markdown('<div class="sub-title">Example Usage: Single Date Extraction with DateMatcher</div>', unsafe_allow_html=True)
+st.code('''
+from sparknlp.base import DocumentAssembler, Pipeline
+from sparknlp.annotator import DateMatcher
+import pyspark.sql.functions as F
+
+# Step 1: Transforms raw texts to `document` annotation
+document_assembler = (
+    DocumentAssembler()
+    .setInputCol("text")
+    .setOutputCol("document")
+)
+
+# Step 2: Extracts one date information from text
+date_matcher = (
+    DateMatcher()
+    .setInputCols("document")
+    .setOutputCol("date")
+    .setOutputFormat("yyyy/MM/dd")
+)
+
+nlp_pipeline = Pipeline(stages=[document_assembler, date_matcher])
+
+text_list = ["See you on next monday.",
+             "She was born on 02/03/1966.",
+             "The project started yesterday and will finish next year.",
+             "She will graduate by July 2023.",
+             "She will visit doctor tomorrow and next month again."]
+
+# Create a dataframe
+spark_df = spark.createDataFrame(text_list, StringType()).toDF("text")
+
+# Fit the pipeline and get predictions
+result = nlp_pipeline.fit(spark_df).transform(spark_df)
+
+# Display the extracted date information
+result.selectExpr("text", "date.result as date").show(truncate=False)
+''', language='python')
+
+st.text("""
++--------------------------------------------------------+------------+
+|text                                                    |date        |
++--------------------------------------------------------+------------+
+|See you on next monday.                                 |[2024/07/08]|
+|She was born on 02/03/1966.                             |[1966/02/03]|
+|The project started yesterday and will finish next year.|[2025/07/06]|
+|She will graduate by July 2023.                         |[2023/07/01]|
+|She will visit doctor tomorrow and next month again.    |[2024/08/06]|
++--------------------------------------------------------+------------+
+""")
+
+st.markdown("""
+<p>The code snippet demonstrates how to set up a pipeline in Spark NLP to extract single date patterns from text data using the DateMatcher annotator. The resulting DataFrame contains the matched date patterns.</p>
+""", unsafe_allow_html=True)
+
+# Using MultiDateMatcher in Spark NLP
+st.markdown('<div class="sub-title">Using MultiDateMatcher in Spark NLP</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The MultiDateMatcher annotator in Spark NLP extends the capabilities of the DateMatcher by allowing extraction of multiple date patterns from text data. This is useful when a text contains several dates.</p>
+    <p>The MultiDateMatcher annotator in Spark NLP offers:</p>
+    <ul>
+        <li>Flexible date pattern matching</li>
+        <li>Extraction of multiple date occurrences</li>
+        <li>Efficient processing of large text datasets</li>
+        <li>Integration with other Spark NLP components for comprehensive NLP pipelines</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Multi Date Extraction Example
+st.markdown('<div class="sub-title">Example Usage: Multiple Date Extraction with MultiDateMatcher</div>', unsafe_allow_html=True)
+st.code('''
+from sparknlp.annotator import MultiDateMatcher
+
+# Step 1: Transforms raw texts to `document` annotation
+document_assembler = (
+    DocumentAssembler()
+    .setInputCol("text")
+    .setOutputCol("document")
+)
+
+# Step 2: Extracts multiple date information from text
+multi_date_matcher = (
+    MultiDateMatcher()
+    .setInputCols("document")
+    .setOutputCol("multi_date")
+    .setOutputFormat("MM/dd/yy")
+)
+
+nlp_pipeline = Pipeline(stages=[document_assembler, multi_date_matcher])
+
+text_list = ["See you on next monday.",
+             "She was born on 02/03/1966.",
+             "The project started yesterday and will finish next year.",
+             "She will graduate by July 2023.",
+             "She will visit doctor tomorrow and next month again."]
+
+# Create a dataframe
+spark_df = spark.createDataFrame(text_list, StringType()).toDF("text")
+
+# Fit the pipeline and get predictions
+result = nlp_pipeline.fit(spark_df).transform(spark_df)
+
+# Display the extracted date information
+result.selectExpr("text", "multi_date.result as multi_date").show(truncate=False)
+''', language='python')
+
+st.text("""
++--------------------------------------------------------+--------------------+
+|text                                                    |multi_date          |
++--------------------------------------------------------+--------------------+
+|See you on next monday.                                 |[07/08/24]          |
+|She was born on 02/03/1966.                             |[02/03/66]          |
+|The project started yesterday and will finish next year.|[07/06/25, 07/05/24]|
+|She will graduate by July 2023.                         |[07/01/23]          |
+|She will visit doctor tomorrow and next month again.    |[08/06/24, 07/07/24]|
++--------------------------------------------------------+--------------------+
+""")
+
+st.markdown("""
+<p>The code snippet demonstrates how to set up a pipeline in Spark NLP to extract multiple date patterns from text data using the MultiDateMatcher annotator. The resulting DataFrame contains the matched date patterns.</p>
+""", unsafe_allow_html=True)
+
+# Handling Relative Dates
+st.markdown('<div class="sub-title">Handling Relative Dates</div>', unsafe_allow_html=True)
+st.write("")
+st.markdown("""<p>DateMatcher and MultiDateMatcher annotators in Spark NLP can also handle relative dates such as "tomorrow," "next week," or "last year." To achieve this, you need to set a reference (or anchor) date, which the annotators will use as a base to interpret the relative dates mentioned in the text.</p>""", unsafe_allow_html=True)
+st.code('''
+# Step 1: Transforms raw texts to `document` annotation
+document_assembler = (
+    DocumentAssembler()
+    .setInputCol("text")
+    .setOutputCol("document")
+)
+
+# Step 2: Set anchor day, month and year
+multi_date_matcher = (
+    MultiDateMatcher()
+    .setInputCols("document")
+    .setOutputCol("multi_date")
+    .setOutputFormat("MM/dd/yyyy")
+    .setAnchorDateYear(2024)
+    .setAnchorDateMonth(7)
+    .setAnchorDateDay(6)
+)
+
+nlp_pipeline = Pipeline(stages=[document_assembler, multi_date_matcher])
+
+text_list = ["See you on next monday.",
+             "She was born on 02/03/1966.",
+             "The project started yesterday and will finish next year.",
+             "She will graduate by July 2023.",
+             "She will visit doctor tomorrow and next month again."]
+
+# Create a dataframe
+spark_df = spark.createDataFrame(text_list, StringType()).toDF("text")
+
+# Fit the pipeline and get predictions
+result = nlp_pipeline.fit(spark_df).transform(spark_df)
+
+# Display the extracted date information
+result.selectExpr("text", "multi_date.result as multi_date").show(truncate=False)
+''', language='python')
+
+st.text("""
++--------------------------------------------------------+------------------------+
+|text                                                    |multi_date              |
++--------------------------------------------------------+------------------------+
+|See you on next monday.                                 |[07/08/2024]            |
+|She was born on 02/03/1966.                             |[02/03/1966]            |
+|The project started yesterday and will finish next year.|[07/06/2025, 07/05/2024]|
+|She will graduate by July 2023.                         |[07/01/2023]            |
+|She will visit doctor tomorrow and next month again.    |[08/06/2024, 07/07/2024]|
++--------------------------------------------------------+------------------------+
+""")
+
+st.markdown("""
+<p>This code snippet shows how to handle relative dates by setting an anchor date for the MultiDateMatcher annotator. The anchor date helps in converting relative date references to absolute dates.</p>
+""", unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <h2>Conclusion</h2>
+    <p>In this app, we demonstrated how to use Spark NLP's DateMatcher and MultiDateMatcher annotators to extract dates from text data. These powerful tools enable users to efficiently process large datasets and identify date patterns, whether single or multiple occurrences, including handling relative dates with ease. By integrating these annotators into your NLP pipelines, you can enhance the extraction of valuable temporal information from unstructured text, providing deeper insights for various applications.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References and Additional Information
+st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li>Documentation : <a href="https://nlp.johnsnowlabs.com/docs/en/annotators#datematcher" target="_blank" rel="noopener">DateMatcher</a>, <a href="https://nlp.johnsnowlabs.com/docs/en/annotators#multidatematcher" target="_blank" rel="noopener">MultiDateMatcher</a></li>
+        <li>Python Doc : <a href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/matcher/date_matcher/index.html#module-sparknlp.annotator.matcher.date_matcher" target="_blank" rel="noopener">DateMatcher</a>, <a href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/matcher/multi_date_matcher/index.html" target="_blank" rel="noopener">MultiDateMatcher</a></li>
+        <li>Scala Doc : <a href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/DateMatcher.html" target="_blank" rel="noopener">DateMatcher</a>, <a href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/MultiDateMatcher.html" target="_blank" rel="noopener">MultiDateMatcher</a></li>
+        <li>For extended examples of usage, see the <a href="https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb" target="_blank" rel="noopener nofollow">Spark NLP Workshop</a>.</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+pandas
+numpy
+spark-nlp
+pyspark