Upload 2 files

Dockerfile

@@ -0,0 +1,85 @@
+# Use a base image with Conda installed
+FROM centos:7
+FROM python:3.9.18
+FROM continuumio/miniconda3:latest
+
+WORKDIR /app
+COPY *requirements.txt /app/
+
+# Clone the RoLASER repository into the container
+RUN apt-get update && \
+    apt-get install -y g++ && \
+    apt-get install -y unzip && \
+    apt-get install -y git && \
+    git clone https://github.com/lydianish/RoLASER.git /app/RoLASER
+
+# Create conda environment with the required Python and GCC versions
+WORKDIR /app/RoLASER
+COPY environment.yml /app/RoLASER/environment.yml
+RUN conda env create -f environment.yml
+RUN echo "conda activate rolaser_env" >> ~/.bashrc
+ENV PATH /opt/conda/envs/rolaser_env/bin:$PATH
+ENV ROLASER /app/RoLASER
+
+# Install PyTorch 1.10.1
+RUN pip3 install --no-cache-dir torch==1.10.1 torchvision==0.11.2 torchaudio==0.10.1 -f https://download.pytorch.org/whl/cu102/torch_stable.html
+# Install Fairseq and dependencies
+RUN git clone https://github.com/lydianish/fairseq.git /app/fairseq
+WORKDIR /app/fairseq
+RUN git checkout rolaser
+RUN pip3 install --no-cache-dir --editable .
+RUN python3 setup.py build_ext --inplace
+RUN pip3 install --no-cache-dir -r /app/FAIRSEQ_requirements.txt
+ENV FAIRSEQ /app/fairseq
+
+
+# Install LASER and dependencies
+RUN git clone https://github.com/lydianish/LASER.git /app/LASER
+WORKDIR /app/LASER
+RUN git checkout rolaser
+ENV LASER /app/LASER
+RUN bash ./install_external_tools.sh
+RUN pip3 install --no-cache-dir -r /app/LASER_requirements.txt
+
+# Install other RoLASER dependencies
+WORKDIR /app/RoLASER 
+RUN pip3 install --no-cache-dir -r /app/ROLASER_requirements.txt
+
+# Download models
+WORKDIR /app/RoLASER/models
+RUN wget https://zenodo.org/api/records/10864557/files-archive
+RUN unzip files-archive
+
+RUN mkdir $LASER/models \
+&& mv laser* $LASER/models \
+&& mkdir RoLASER \
+&& mv rolaser* RoLASER/ \
+&& mkdir c-RoLASER \
+&& mv c-rolaser* c-RoLASER/ \
+&& rm files-archive
+
+# Set Python system path to find the Fairseq and LASER modules
+ENV PYTHONPATH $PYTHONPATH:$FAIRSEQ
+ENV PYTHONPATH $PYTHONPATH:$LASER/source
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+# Set home to the user's home directory
+ENV HOME /home/user
+ENV PATH $HOME/.local/bin:$PATH
+
+WORKDIR $HOME
+RUN mkdir app
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+EXPOSE 7860
+CMD streamlit run app.py \
+    --server.headless true \
+    --server.enableCORS false \
+    --server.enableXsrfProtection false \
+    --server.fileWatcherType none

app.py

@@ -0,0 +1,78 @@
+import os, sys
+import streamlit as st
+import pandas as pd
+import numpy as np
+from sklearn.metrics.pairwise import paired_cosine_distances
+from sklearn.preprocessing import normalize
+from rolaser import RoLaserEncoder
+
+laser_checkpoint = f"{os.environ['LASER']}/models/laser2.pt"
+laser_vocab = f"{os.environ['LASER']}/models/laser2.cvocab"
+laser_tokenizer = 'spm'
+laser_model = RoLaserEncoder(model_path=laser_checkpoint, vocab=laser_vocab, tokenizer=laser_tokenizer)
+
+rolaser_checkpoint = f"{os.environ['ROLASER']}/models/RoLASER/rolaser.pt"
+rolaser_vocab = f"{os.environ['ROLASER']}/models/RoLASER/rolaser.cvocab"
+rolaser_tokenizer = 'roberta'
+rolaser_model = RoLaserEncoder(model_path=rolaser_checkpoint, vocab=rolaser_vocab, tokenizer=rolaser_tokenizer)
+
+c_rolaser_checkpoint = f"{os.environ['ROLASER']}/models/c-RoLASER/c-rolaser.pt"
+c_rolaser_vocab = f"{os.environ['ROLASER']}/models/c-RoLASER/c-rolaser.cvocab"
+c_rolaser_tokenizer = 'char'
+c_rolaser_model = RoLaserEncoder(model_path=c_rolaser_checkpoint, vocab=c_rolaser_vocab, tokenizer=c_rolaser_tokenizer)
+
+def add_text_inputs(i):
+    col1, col2 = st.columns(2)
+    with col1:
+        text_input1 = st.text_input('Enter standard text here:', f'std{i}')
+    with col2:
+        text_input2 = st.text_input('Enter non-standard text here:', f'ugc{i}')
+    return text_input1, text_input2
+
+def main():
+    st.title('Pairwise Cosine Distance Calculator')
+
+    num_pairs = st.sidebar.number_input('Number of Text Input Pairs', min_value=1, max_value=10, value=1)
+
+    std_text_inputs = []
+    ugc_text_inputs = []
+    for i in range(num_pairs):
+        pair = add_text_inputs(i)
+        std_text_inputs.append(pair[0])
+        ugc_text_inputs.append(pair[1])
+
+    if st.button('Add Text Input Pair'):
+        pair = add_text_inputs(len(std_text_inputs))
+        std_text_inputs.append(pair[0])
+        ugc_text_inputs.append(pair[1])
+
+    if st.button('Submit'):
+        X_std_laser = normalize(laser_model.encode(std_text_inputs))
+        X_ugc_laser = normalize(laser_model.encode(ugc_text_inputs))
+        X_cos_laser = paired_cosine_distances(X_std_laser, X_ugc_laser)
+
+        X_std_rolaser = normalize(rolaser_model.encode(std_text_inputs))
+        X_ugc_rolaser = normalize(rolaser_model.encode(ugc_text_inputs))
+        X_cos_rolaser = paired_cosine_distances(X_std_rolaser, X_ugc_rolaser)
+
+        X_std_c_rolaser = normalize(c_rolaser_model.encode(std_text_inputs))
+        X_ugc_c_rolaser = normalize(c_rolaser_model.encode(ugc_text_inputs))
+        X_cos_c_rolaser = paired_cosine_distances(X_std_c_rolaser, X_ugc_c_rolaser)
+
+        outputs = pd.DataFrame(columns=[ 'model', 'pair', 'ugc', 'std', 'cos'])
+        outputs['model'] = np.repeat(['LASER', 'RoLASER', 'C-RoLASER'], 3)   
+        outputs['pair'] = np.tile(np.arange(1,num_pairs+1), 3)
+        outputs['std'] = np.tile(std_text_inputs, 3)
+        outputs['ugc'] = np.tile(ugc_text_inputs, 3)
+        outputs['cos'] = np.concatenate([X_cos_laser, X_cos_rolaser, X_cos_c_rolaser], axis=1)
+
+        st.write('## Cosine Distance Scores:')
+        st.bar_chart(outputs, x='pair', y='cos', color='model', title='Cosine Distance Scores', xlabel='Text Input Pair', ylabel='Cosine Distance', legend='Model')
+
+        st.write('## Average Cosine Distance Scores:')
+        st.write(f'LASER: {outputs[outputs["model"]=="LASER"]["cos"].mean()}')
+        st.write(f'RoLASER: {outputs[outputs["model"]=="RoLASER"]["cos"].mean()}')
+        st.write(f'C-RoLASER: {outputs[outputs["model"]=="C-RoLASER"]["cos"].mean()}')
+        
+if __name__ == "__main__":
+    main()