Upload 177 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+images/T5_model_diagram.jpg filter=lfs diff=lfs merge=lfs -text

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,163 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+
+# Page Configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# Custom CSS for Styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section-content {
+            background-color: #f9f9f9;
+            padding: 10px;
+            border-radius: 10px;
+            margin-top: 10px;
+        }
+        .section-content p, .section-content ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Initialize Spark Session
+@st.cache_resource
+def start_spark_session():
+    return sparknlp.start()
+
+# Create NLP Pipeline
+@st.cache_resource
+def build_nlp_pipeline(model_name, task):
+    document_assembler = DocumentAssembler()\
+        .setInputCol("text")\
+        .setOutputCol("document")
+    
+    t5_transformer = T5Transformer() \
+        .pretrained(model_name, 'en') \
+        .setTask(task)\
+        .setInputCols(["document"]) \
+        .setOutputCol("output")
+
+    pipeline = Pipeline().setStages([document_assembler, t5_transformer])
+    return pipeline
+
+# Apply Pipeline to Text Data
+def process_text(pipeline, text):
+    df = spark.createDataFrame([[text]]).toDF("text")
+    result = pipeline.fit(df).transform(df)
+    return result.select('output.result').collect()
+
+# Model and Task Information
+model_info = [
+    {
+        "model_name": "t5_small",
+        "title": "Multi-Task NLP Model",
+        "description": "The T5 model performs 18 different NLP tasks including summarization, question answering, and grammatical correctness detection."
+    },
+    {
+        "model_name": "t5_base",
+        "title": "Multi-Task NLP Model",
+        "description": "A larger variant of the T5 model, capable of performing a variety of NLP tasks with improved accuracy."
+    },
+    {
+        "model_name": "google_t5_small_ssm_nq",
+        "title": "Question Answering Model",
+        "description": "This model is fine-tuned for answering questions based on the Natural Questions dataset, leveraging pre-training on large text corpora."
+    }
+]
+
+task_descriptions = {
+  'Sentence Classification - cola': "Classify if a sentence is grammatically correct.",
+  'Natural Language Inference - rte': "The RTE task is defined as recognizing, given two text fragments, whether the meaning of one text can be inferred (entailed) from the other or not.",
+  'Natural Language Inference - mnli': "Classify for a hypothesis and premise whether they contradict or contradict each other or neither of both (3 class).",
+  'Natural Language Inference - qnli': "Classify whether the answer to a question can be deducted from an answer candidate.",
+  'Natural Language Inference - cb': "Classify for a premise and a hypothesis whether they contradict each other or not (binary).",
+  'Coreference Resolution - mrpc': "Classify whether a pair of sentences is a re-phrasing of each other (semantically equivalent).",
+  'Coreference Resolution - qqp': "Classify whether a pair of questions is a re-phrasing of each other (semantically equivalent).",
+  'Sentiment Analysis - sst2': "Classify the sentiment of a sentence as positive or negative.",
+  'Sentiment Analysis - stsb': "Measures how similar two sentences are on a scale from 0 to 5",
+  'Question Answering - copa': "Classify for a question, premise, and 2 choices which choice the correct choice is (binary).",
+  'Question Answering  - multirc': "Classify for a question, a paragraph of text, and an answer candidate, if the answer is correct (binary).",
+  'Question Answering - squad': "Answer a question for a given context.",
+  'Word Sense Disambiguation - wic': "Classify for a pair of sentences and a disambiguous word if the word has the same meaning in both sentences.",
+  'Text - summarization': "Summarize text into a shorter representation.",
+  'Translation - wmt1': "This model is used to translate one language to the other language. Example: Translate English to German.",
+  'Translation - wmt2': "This model is used to translate one language to the other language. Example: Translate English to French.",
+  'Translation - wmt3': "This model is used to translate one language to the other language. Example: Translate English to Romanian."
+}
+
+# Sidebar: Task and Model Selection
+selected_task = st.sidebar.selectbox("Choose an NLP Task", list(task_descriptions.keys()))
+task_for_pipeline = f"{selected_task.split(' - ')[-1]}:"
+
+available_models = ['google_t5_small_ssm_nq'] if "Question Answering" in selected_task else ['t5_base', 't5_small']
+selected_model = st.sidebar.selectbox("Choose a Model", available_models)
+
+# Get Model Info
+model_details = next((info for info in model_info if info['model_name'] == selected_model), None)
+app_title = model_details['title'] if model_details else "Unknown Model"
+app_description = model_details['description'] if model_details else "No description available."
+
+# Display Model Info
+st.markdown(f'<div class="main-title">{app_title}</div>', unsafe_allow_html=True)
+st.markdown(f'<div class="section-content"><p>{app_description}</p></div>', unsafe_allow_html=True)
+st.subheader(task_descriptions[selected_task])
+
+# Load Example Texts
+example_folder = f"inputs/{selected_task}/{selected_model}"
+example_texts = [
+    line.strip() 
+    for file in os.listdir(example_folder) 
+    if file.endswith('.txt') 
+    for line in open(os.path.join(example_folder, file), 'r', encoding='utf-8')
+]
+
+# User Input: Select or Enter Text
+selected_example = st.selectbox("Select an Example", example_texts)
+custom_input = st.text_input("Or enter your own text:")
+
+text_to_process = custom_input if custom_input else selected_example
+
+# Display Selected Text
+st.subheader('Selected Text')
+st.markdown(f'<div class="section-content">{text_to_process}</div>', unsafe_allow_html=True)
+
+# Sidebar: Reference Notebook
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown("""
+<a href="https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/T5TRANSFORMER.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+""", unsafe_allow_html=True)
+
+# Special Cases for Translation Tasks
+task_for_pipeline = {
+    'wmt1:': 'translate English to German:',
+    'wmt2:': 'translate English to French:',
+    'wmt3:': 'translate English to Romanian:'
+}.get(task_for_pipeline, task_for_pipeline)
+
+# Initialize Spark, Build Pipeline, and Process Text
+spark = start_spark_session()
+nlp_pipeline = build_nlp_pipeline(selected_model, task_for_pipeline)
+processed_output = process_text(nlp_pipeline, text_to_process)
+
+# Display Processed Output
+st.subheader("Processed Output")
+output_text = "".join(processed_output[0][0])
+st.markdown(f'<div class="section-content">{output_text}</div>', unsafe_allow_html=True)

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

inputs/Coreference Resolution - mrpc/t5_base/Example1.txt

@@ -0,0 +1,2 @@
+We acted because we saw the existing evidence in a new light, through the prism of our experienc...
+We acted because we saw the existing evidence in a new light, through the prism of our experience on 11 September Rumsfeld said.                         sentence2: Rather, the US acted because the administration saw existing evidence in a new light, through the prism of our experience on September 11.

inputs/Coreference Resolution - mrpc/t5_base/Example2.txt

@@ -0,0 +1,2 @@
+I like to eat peanut butter for breakfast.                         
+I like to eat peanut butter for breakfast.                         sentence2: I like to play football.

inputs/Coreference Resolution - mrpc/t5_base/Example3.txt

@@ -0,0 +1,2 @@
+Charles O. Prince, 53, was named as Mr. Weill’s successor.
+Charles O. Prince, 53, was named as Mr. Weill’s successor.                       sentence2: Mr.   Weill’s   longtime   confidant,   Charles  O.  Prince,  53,  was  named  as his successor.

inputs/Coreference Resolution - mrpc/t5_base/Example4.txt

@@ -0,0 +1,2 @@
+The  euro  rose  above  US$1.18,  the  highest  price  since  its  January  ...
+The  euro  rose  above  US$1.18,  the  highest  price  since  its  January  1999 launch.                    sentence2: The euro rose above $1.18 the high-est  level  since  its  launch  in  January 1999. 

inputs/Coreference Resolution - mrpc/t5_base/Example5.txt

@@ -0,0 +1,2 @@
+However,  without  a  carefully  con-trolled  study,  there  was  ...
+However,  without  a  carefully  con-trolled  study,  there  was  little  clear proof that the operation ac-tually improves people’s lives.                    sentence2: But  without  a  carefully  controlled  study,  there  was  little  clear  proof  that  the  operation  improves  people’s lives.

inputs/Coreference Resolution - mrpc/t5_small/Example1.txt

@@ -0,0 +1,2 @@
+We acted because we saw the existing evidence in a new light, through the prism of our experienc...
+We acted because we saw the existing evidence in a new light, through the prism of our experience on 11 September" Rumsfeld said.                         sentence2: Rather, the US acted because the administration saw "existing evidence in a new light, through the prism of our experience on September 11".

inputs/Coreference Resolution - mrpc/t5_small/Example2.txt

@@ -0,0 +1,2 @@
+I like to eat peanutbutter for breakfast.                      
+I like to eat peanutbutter for breakfast.                         sentence2: I like to play football.

inputs/Coreference Resolution - mrpc/t5_small/Example3.txt

@@ -0,0 +1,2 @@
+Charles O. Prince, 53, was named as Mr. Weill’s successor.
+Charles O. Prince, 53, was named as Mr. Weill’s successor.                       sentence2: Mr.   Weill’s   longtime   confidant,   Charles  O.  Prince,  53,  was  named  as his successor.

inputs/Coreference Resolution - mrpc/t5_small/Example4.txt

@@ -0,0 +1,2 @@
+The  euro  rose  above  US$1.18,  the  highest  price  since  its  January  ...
+The  euro  rose  above  US$1.18,  the  highest  price  since  its  January  1999 launch.                    sentence2: The euro rose above $1.18 the high-est  level  since  its  launch  in  January 1999. 

inputs/Coreference Resolution - mrpc/t5_small/Example5.txt

@@ -0,0 +1,2 @@
+However,  without  a  carefully  con-trolled  study,  there  was  ...
+However,  without  a  carefully  con-trolled  study,  there  was  little  clear proof that the operation ac-tually improves people’s lives.                    sentence2: But  without  a  carefully  controlled  study,  there  was  little  clear  proof  that  the  operation  improves  people’s lives.

inputs/Coreference Resolution - qqp/t5_base/Example1.txt

@@ -0,0 +1,2 @@
+question1: What attributes would have made you highly desirable in ancient Rome?
+question1: What attributes would have made you highly desirable in ancient Rome?                             question2: How I GET OPPERTINUTY TO JOIN IT COMPANY AS A FRESHER?'

inputs/Coreference Resolution - qqp/t5_base/Example2.txt

@@ -0,0 +1,2 @@
+question1: What was it like in Ancient rome?
+question1: What was it like in Ancient rome?                         question2: What was Ancient rome like?

inputs/Coreference Resolution - qqp/t5_base/Example3.txt

@@ -0,0 +1,2 @@
+question1: How can I install Windows software or games?
+question1: How can I install Windows software or games?                             question2: I cannot install Windows. How to install it?

inputs/Coreference Resolution - qqp/t5_base/Example4.txt

@@ -0,0 +1,2 @@
+question1: Could you live without the internet?
+question1: Could you live without the internet?                          question2: Internet is not available for a few days. Could you manage without it?

inputs/Coreference Resolution - qqp/t5_base/Example5.txt

@@ -0,0 +1,2 @@
+question1: What is the best thing that happened to you during the past week?
+question1: What is the best thing that happened to you during the past week?                            question2: What is the best thing that happened to you during the past year?

inputs/Coreference Resolution - qqp/t5_small/Example1.txt

@@ -0,0 +1,2 @@
+question1: What attributes would have made you highly desirable in ancient Rome?
+question1: What attributes would have made you highly desirable in ancient Rome?                             question2: How I GET OPPERTINUTY TO JOIN IT COMPANY AS A FRESHER?'

inputs/Coreference Resolution - qqp/t5_small/Example2.txt

@@ -0,0 +1,2 @@
+question1: What was it like in Ancient rome?
+question1: What was it like in Ancient rome?                         question2: What was Ancient rome like?

inputs/Coreference Resolution - qqp/t5_small/Example3.txt

@@ -0,0 +1,2 @@
+question1: How can I install Windows software or games?
+question1: How can I install Windows software or games?                             question2: I cannot install Windows. How to install it?

inputs/Coreference Resolution - qqp/t5_small/Example4.txt

@@ -0,0 +1,2 @@
+question1: Could you live without the internet?
+question1: Could you live without the internet?                          question2: Internet is not available for a few days. Could you manage without it?

inputs/Coreference Resolution - qqp/t5_small/Example5.txt

@@ -0,0 +1,2 @@
+question1: What is the best thing that happened to you during the past week?
+question1: What is the best thing that happened to you during the past week?                            question2: What is the best thing that happened to you during the past year?

inputs/Natural Language Inference - cb/t5_base/Example1.txt

@@ -0,0 +1,2 @@
+Recent report say Johnny makes he alot of money, he earned 10 million USD each year for the last ...
+Recent report say Johnny makes he alot of money, he earned 10 million USD each year for the last 5 years.                         premise: Johnny is a poor man.

inputs/Natural Language Inference - cb/t5_base/Example2.txt

@@ -0,0 +1,2 @@
+It rained in England the last 4 weeks.                         premise: It was snowing in New Yor...
+It rained in England the last 4 weeks.                         premise: It was snowing in New York last week.

inputs/Natural Language Inference - cb/t5_base/Example3.txt

@@ -0,0 +1,2 @@
+Man in a black suit, white shirt and black bow tie playing an instrument with ...
+Man in a black suit, white shirt and black bow tie playing an instrument with the rest of his symphony surrounding him.                             premise:Nobody has a suit

inputs/Natural Language Inference - cb/t5_base/Example4.txt

@@ -0,0 +1,2 @@
+At 8:34, the Boston Center controller received a third, transmission from ...
+At 8:34, the Boston Center controller received a third, transmission from American 11.                             premise:The Boston Center controller got a third transmission from American 11.

inputs/Natural Language Inference - cb/t5_base/Example5.txt

@@ -0,0 +1,2 @@
+Cats with long hair shed all over the house so you should not get a long-....
+Cats with long hair shed all over the house so you should not get a long-haired cat                             premise: Long hair cats are good

inputs/Natural Language Inference - cb/t5_small/Example1.txt

@@ -0,0 +1,2 @@
+Recent report say Johnny makes he alot of money, he earned 10 million USD each year for the last ...
+Recent report say Johnny makes he alot of money, he earned 10 million USD each year for the last 5 years.                         premise: Johnny is a poor man.

inputs/Natural Language Inference - cb/t5_small/Example2.txt

@@ -0,0 +1,2 @@
+It rained in England the last 4 weeks.                         premise: It was snowing in New Yor...
+It rained in England the last 4 weeks.                         premise: It was snowing in New York last week.

inputs/Natural Language Inference - cb/t5_small/Example3.txt

@@ -0,0 +1,2 @@
+Man in a black suit, white shirt and black bow tie playing an instrument with ...
+Man in a black suit, white shirt and black bow tie playing an instrument with the rest of his symphony surrounding him.                             premise:Nobody has a suit

inputs/Natural Language Inference - cb/t5_small/Example4.txt

@@ -0,0 +1,2 @@
+At 8:34, the Boston Center controller received a third, transmission from ...
+At 8:34, the Boston Center controller received a third, transmission from American 11.                             premise:The Boston Center controller got a third transmission from American 11.

inputs/Natural Language Inference - cb/t5_small/Example5.txt

@@ -0,0 +1,2 @@
+Cats with long hair shed all over the house so you should not get a long-....
+Cats with long hair shed all over the house so you should not get a long-haired cat                             premise: Long hair cats are good

inputs/Natural Language Inference - mnli/t5_base/Example1.txt

@@ -0,0 +1,2 @@
+Recent report say Johnny makes he alot of money, he earned 10 million USD each year for the last 5 years.
+Recent report say Johnny makes he alot of money, he earned 10 million USD each year for the last 5 years.                         premise: Johnny is a poor man.

inputs/Natural Language Inference - mnli/t5_base/Example2.txt

@@ -0,0 +1,2 @@
+It rained in England the last 4 weeks.                  
+It rained in England the last 4 weeks.                         premise: It was snowing in New York last week.

inputs/Natural Language Inference - mnli/t5_base/Example3.txt

@@ -0,0 +1,2 @@
+Man in a black suit, white shirt and black bow tie playing an instrument with ...
+Man in a black suit, white shirt and black bow tie playing an instrument with the rest of his symphony surrounding him.                             premise:Nobody has a suit

inputs/Natural Language Inference - mnli/t5_base/Example4.txt

@@ -0,0 +1,2 @@
+At 8:34, the Boston Center controller received a third, transmission from ...
+At 8:34, the Boston Center controller received a third, transmission from American 11.                             premise:The Boston Center controller got a third transmission from American 11.

inputs/Natural Language Inference - mnli/t5_base/Example5.txt

@@ -0,0 +1,2 @@
+Cats with long hair shed all over the house so you should not get a long-....
+Cats with long hair shed all over the house so you should not get a long-haired cat                             premise: Long hair cats are good

inputs/Natural Language Inference - mnli/t5_small/Example1.txt

@@ -0,0 +1,2 @@
+Recent report say Johnny makes he alot of money, he earned 10 million USD each year for the last ...
+Recent report say Johnny makes he alot of money, he earned 10 million USD each year for the last 5 years.                         premise: Johnny is a poor man.

inputs/Natural Language Inference - mnli/t5_small/Example2.txt

@@ -0,0 +1,2 @@
+It rained in England the last 4 weeks.                         premise: It was snowing in New Yor...
+It rained in England the last 4 weeks.                         premise: It was snowing in New York last week.

inputs/Natural Language Inference - mnli/t5_small/Example3.txt

@@ -0,0 +1,2 @@
+Man in a black suit, white shirt and black bow tie playing an instrument with ...
+Man in a black suit, white shirt and black bow tie playing an instrument with the rest of his symphony surrounding him.                             premise:Nobody has a suit

inputs/Natural Language Inference - mnli/t5_small/Example4.txt

@@ -0,0 +1,2 @@
+At 8:34, the Boston Center controller received a third, transmission from ...
+At 8:34, the Boston Center controller received a third, transmission from American 11.                             premise:The Boston Center controller got a third transmission from American 11.

inputs/Natural Language Inference - mnli/t5_small/Example5.txt

@@ -0,0 +1,2 @@
+Cats with long hair shed all over the house so you should not get a long-....
+Cats with long hair shed all over the house so you should not get a long-haired cat                             premise: Long hair cats are good

inputs/Natural Language Inference - qnli/t5_base/Example1.txt

@@ -0,0 +1,2 @@
+question: Where did Jebe die?                         sentence: Ghenkis Khan recalled Subtai back...
+question: Where did Jebe die?                         sentence: Ghenkis Khan recalled Subtai back to Mongolia soon afterward, and Jebe died on the road back to Samarkand

inputs/Natural Language Inference - qnli/t5_base/Example2.txt

@@ -0,0 +1,2 @@
+question: What does Steve like to eat?	                         sentence: Steve watches TV all day.
+question: What does Steve like to eat?	                         sentence: Steve watches TV all day.

inputs/Natural Language Inference - qnli/t5_base/Example3.txt

@@ -0,0 +1,2 @@
+question: What space station supported three manned missions in 1973–1974?
+question: What space station supported three manned missions in 1973–1974?                        sentence: Apollo/Saturn vehicles were also used for an Apollo Applications Program, which consisted of Skylab, a space station that supported three manned missions in 1973–74, and the Apollo–Soyuz Test Project, a joint Earth orbit mission with the Soviet Union in 1975.

inputs/Natural Language Inference - qnli/t5_base/Example4.txt

@@ -0,0 +1,2 @@
+question: When did Beyonce start becoming popular?
+question: When did Beyonce start becoming popular?                        sentence: Beyoncé was born and raised in Houston, Texas, she performed in various singing and dancing competitions as a child, and rose to fame in the late 1990s as lead singer of R&B girl-group Destiny's Child.

inputs/Natural Language Inference - qnli/t5_base/Example5.txt

@@ -0,0 +1,2 @@
+question: Which NFL team represented the AFC at Super Bowl 50?
+question: Which NFL team represented the AFC at Super Bowl 50?                      sentence: Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title.