Upload 9 files

app.py

@@ -1,19 +1,34 @@
 import streamlit as st
-from utils import get_roberta, get_gpt, get_distilbert
+from streamlit import components
+from utils import get_roberta, get_gpt, get_distilbert, softmax
+from models import load_custom_model
 import torch
+import plotly.express as px
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from bertviz.neuron_view import show
+from bertviz import model_view, head_view
+import pandas as pd
+import warnings
+warnings.filterwarnings('ignore')
+st.set_page_config(page_title="Sentence Entailment",layout="wide")
 
-
-
-
-st.title('Sentence Entailment')
-col1, col2 = st.columns([1,1])
-
-with col1:
+with st.sidebar:
+    st.title('Sentence Entailment')
     sentence1 = st.text_input('Premise')
-
-with col2:
     sentence2 = st.text_input('Hypothesis')
-btn = st.button("Submit")
+    btn = st.button("Submit")
+
+    # At least for roberta
+    n_layers = 12
+    n_heads = 12
+    col1, col2 = st.columns([1,1])
+    with col1:
+        #layer = st.slider('Layer', 0, n_layers-1)
+        layer = st.number_input('Layer', min_value=0, max_value=11)
+    with col2:
+        #head = st.slider('Head', 0, n_heads-1)
+        head = st.number_input('Head', min_value=0, max_value=11)
 
 label_dict = {
     0 : 'entailment',
@@ -22,6 +37,15 @@ label_dict = {
 }
 
 if btn:
+
+    preds_tab, roberta_tab, distilbert_tab, gpt_tab, lstm_tab = st.tabs([
+        'Predictions', 
+        'RoBERTa', 
+        'DistilBERT', 
+        'GPT', 
+        'LSTM'
+    ])
+
     # Get Roberta Output
     roberta_tokenizer, roberta_model = get_roberta()
     roberta_input = roberta_tokenizer(
@@ -32,8 +56,13 @@ if btn:
         truncation=True, 
         max_length=512
     )
-    roberta_logits = roberta_model(**roberta_input)['logits']
-    st.write('ROBERTA', label_dict[roberta_logits.argmax().item()])
+    roberta_outputs = roberta_model(**roberta_input)
+    roberta_logits = roberta_outputs['logits']
+    #roberta_attentions = roberta_outputs.attentions
+    #roberta_tokens = roberta_tokenizer.convert_ids_to_tokens(roberta_input['input_ids'][0])
+    #st.write('ROBERTA', label_dict[roberta_logits.argmax().item()])
+    roberta_prediction = label_dict[roberta_logits.argmax().item()]
+    roberta_probas = softmax(roberta_logits)
 
     distilbert_tokenizer, distilbert_model = get_distilbert()
     distilbert_input = distilbert_tokenizer(
@@ -44,9 +73,10 @@ if btn:
         truncation=True, 
         max_length=512
     )
-    distilbert_logits = distilbert_model(**distilbert_input)['logits']
-    st.write('DistilBERT', label_dict[distilbert_logits.argmax().item()])
-    #
+    distilbert_output = distilbert_model(**distilbert_input)
+    distilbert_logits = distilbert_output['logits']
+    distilbert_prediction = label_dict[distilbert_logits.argmax().item()]
+    distilbert_probas = softmax(distilbert_logits)
 
     gpt_tokenizer, gpt_model = get_gpt()
     gpt_input = gpt_tokenizer(
@@ -57,5 +87,135 @@ if btn:
         return_tensors='pt'
     )
     
-    gpt_logits = gpt_model(**gpt_input)['logits']
-    st.write('GPT', label_dict[gpt_logits.argmax().item()])
+    gpt_outputs = gpt_model(**gpt_input)
+    gpt_logits = gpt_outputs['logits']
+    gpt_prediction = label_dict[gpt_logits.argmax().item()]
+    gpt_probas = softmax(gpt_logits)
+
+    lstm_model = load_custom_model('model_lstm.pth', model_type='lstm')
+    bos_token = roberta_tokenizer.bos_token  # Token de début de séquence
+    sep_token = roberta_tokenizer.sep_token  # Token de séparation
+    eos_token = roberta_tokenizer.eos_token  # Token de fin de séquence
+    sentence = bos_token + ' ' + sentence1 + ' ' + sep_token + ' ' + sentence2 + ' ' + eos_token
+    lstm_input = roberta_tokenizer.encode(sentence, add_special_tokens=False, padding='max_length', max_length=130, return_tensors="pt")
+
+    with torch.no_grad():
+        lstm_logits = lstm_model(lstm_input)
+        lstm_prediction = label_dict[lstm_logits.argmax().item()]
+        lstm_probas = softmax(lstm_logits)
+
+    with preds_tab:
+
+        col1, col2, col3, col4 = st.columns([1,1,1,1])
+        with col1:
+            # Pie RoBERTa probabilities
+            fig = px.pie(title=f'RoBERTa : {roberta_prediction}', names=label_dict.values(), values=roberta_probas)
+            fig.update_layout(margin=dict(t=100, l=0, r=0, b=0), showlegend=False)
+            st.plotly_chart(fig, use_container_width=True)
+        with col2:
+            # Pie DistilBERT probabilities
+            fig = px.pie(title=f'DistilBERT : {distilbert_prediction}', names=label_dict.values(), values=distilbert_probas)
+            fig.update_layout(margin=dict(t=100, l=0, r=0, b=0), showlegend=False)
+            st.plotly_chart(fig, use_container_width=True)
+
+        with col3:
+            # Pie GPT probabilities
+            fig = px.pie(title=f'GPT : {gpt_prediction}', names=label_dict.values(), values=gpt_probas)
+            fig.update_layout(margin=dict(t=100, l=0, r=0, b=0), showlegend=False)
+            st.plotly_chart(fig, use_container_width=True)
+
+        with col4:
+            # Pie RoBERTa probabilities
+            fig = px.pie(title=f'LSTM : {lstm_prediction}', names=label_dict.values(), values=lstm_probas)
+            fig.update_layout(margin=dict(t=100, l=0, r=0, b=0), showlegend=False)
+            st.plotly_chart(fig, use_container_width=True)
+
+    with roberta_tab:
+
+        with st.expander('Why RoBERTa?'):
+            st.write("""
+                Compared to BERT, RoBERTa introduces several optimizations in the pre-training process, such as training with larger batch sizes, omitting the next sentence prediction (NSP) pre-training phase, and using a larger corpus. These modifications demonstrated significant improvements on several NLP benchmarks.RoBERTa excels in tasks that require a deep understanding of the context and semantic relationships between sentences, which is essential in this case where SNLI is used and the objective is to determine the relationship (entailment, contradiction, neutral) between a premise and a hypothesis.
+            """)
+
+        attentions = roberta_outputs.attentions
+        tokens = roberta_tokenizer.convert_ids_to_tokens(roberta_input['input_ids'][0])
+        with st.expander('Model View'):
+            st.write('Click on a cell for details')
+            components.v1.html(
+                model_view(
+                    attention=attentions, 
+                    tokens=tokens, 
+                    html_action='return'
+                )._repr_html_(), height=775, width=1000, scrolling=True)
+            
+        with st.expander('Attention'):
+            attention_matrix = attentions[layer][0, head].detach().numpy()
+            separator_token = roberta_tokenizer.sep_token
+            sep_token_index = tokens.index(separator_token) if separator_token in tokens else len(tokens) - 1
+            tokens_a = tokens[1:sep_token_index]  # tokens de la première phrase 
+            tokens_b = tokens[sep_token_index + 1:-1]  # tokens de la deuxième phrase
+            attention_matrix_adjusted = attention_matrix[1:sep_token_index, sep_token_index + 1:-1]
+            df = pd.DataFrame(attention_matrix_adjusted)
+            tokens_a = [tok.split('Ġ')[-1] for tok in tokens_a]
+            tokens_b = [tok.split('Ġ')[-1] for tok in tokens_b]
+            df.index = tokens_a
+            df.columns = tokens_b
+            fig = px.imshow(df, text_auto=True)
+            fig.update_layout(margin=dict(t=0,r=0,l=0,b=0))
+            st.plotly_chart(fig)
+
+    with distilbert_tab:
+        with st.expander('Why DistilBERT?'):
+            st.write("""DistilBERT represents a lightweight, optimized version of BERT, designed to deliver much of BERT's performance with a fraction of its computational resources. The knowledge of a pre-trained BERT model is "distilled" in DistilBERT, reducing the size of the model while retaining much of its learning capacity. This reduction in size translates into a significant acceleration in training and inference time, as well as a reduction in memory usage. DistilBERT is therefore a wise choice for a wide range of NLP tasks, offering an effective compromise between performance and efficiency.""")
+        attentions = distilbert_output[-1]
+        tokens = distilbert_tokenizer.convert_ids_to_tokens(distilbert_input['input_ids'][0])
+        with st.expander('Model View'):
+            st.write('Click on a cell for details')
+            if layer > 5:
+                st.info('Please select Layer index smaller or equal to 5 for DistilBERT')
+            else:
+                components.v1.html(
+                    model_view(
+                        attention=attentions, 
+                        tokens=tokens, 
+                        html_action='return'
+                    )._repr_html_(), height=375, width=1000, scrolling=True)
+                
+        with st.expander('Attention'):
+            attention_matrix = attentions[layer][0, head].detach().numpy()
+            separator_token = distilbert_tokenizer.sep_token
+            sep_token_index = tokens.index(separator_token) if separator_token in tokens else len(tokens) - 1
+            tokens_a = tokens[1:sep_token_index]  # tokens de la première phrase 
+            tokens_b = tokens[sep_token_index + 1:-1]  # tokens de la deuxième phrase
+            attention_matrix_adjusted = attention_matrix[1:sep_token_index, sep_token_index + 1:-1]
+            df = pd.DataFrame(attention_matrix_adjusted)
+            tokens_a = [tok.split('Ġ')[-1] for tok in tokens_a]
+            tokens_b = [tok.split('Ġ')[-1] for tok in tokens_b]
+            df.index = tokens_a
+            df.columns = tokens_b
+            fig = px.imshow(df, text_auto=True)
+            fig.update_layout(margin=dict(t=0,r=0,l=0,b=0))
+            st.plotly_chart(fig)
+
+    with gpt_tab:
+        with st.expander('Why GPT?'):
+            st.write("""L'utilisation de GPT pour la classification de séquences exploite ses capacités en génération de texte pour des applications de classification.Originellement développé pour générer du texte, GPT possède une compréhension approfondie du langage qui s'avère bénéfique même pour catégoriser des textes.Pour adapter GPT à des tâches de classification, on effectue un fine-tuning du modèle sur notre dataset. Ce processus d'ajustement permet à GPT de lier efficacement des séquences textuelles à des catégories définies, ajustant ses poids internes pour optimiser la prédiction de labels à partir des données d'entraînement.""")
+        attentions = gpt_outputs[-1]
+        tokens = gpt_tokenizer.convert_ids_to_tokens(gpt_input['input_ids'][0])
+
+        with st.expander('Visualizations'):
+            st.warning('Not displayed for UX reasons (creates use lags and crashes), but the same as RoBERTa and DistilBERT could in theory be displayed as it is a transformers model too.')
+
+    with lstm_tab:
+        with st.expander('Why LSTM?'):
+            st.write("""We need a contextual analysis of word sequences in this premise-hypothesis problem.
+                        LSTMs are designed to process data sequences by capturing long-term dependencies, making them suitable for this tp where context and word order are important.""")          
+        
+        with st.expander('Architecture'):
+            st.write("""Embedding Layer: Converts word indices into dense vectors. Using an embedding layer is essential here to represent words in a vector space where semantic relationships can be learned. For the SNLI dataset in our case, where understanding the meaning of words in context is essential, this choice is consistent.""")
+            st.write("""Bidirectional: Using a bidirectional LSTM allows the model to capture contextual information both before and after each word in the sequence, giving us a richer understanding of the overall meaning of the premise and hypothesis.""")
+            st.write("""Number of LSTM layers: Having several LSTM layers enables the model to capture higher levels of semantic and syntactic abstraction. However, it's important to strike a balance to avoid overlearning and the training difficulties associated with deep networks. The choice of 6 layers gives us the best results""")
+
+
+else:
+    st.info('Enter 2 sentences')

model_lstm.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03d9ad5707ebeccec15ddcfc5418ea03dd24b416e6f4ab2d915118721dd838d0
+size 31543848

models.py

@@ -1,5 +1,15 @@
 from torch import nn
 from transformers import RobertaModel, RobertaConfig
+import torch
+from transformers import RobertaTokenizer
+
+tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
+
+### Définition des tokens spéciaux ###
+# indiquent respectivement le début de séquence, la séparation entre hypothèse et prémisse, et la fin de séquence #
+bos_token = tokenizer.bos_token
+sep_token = tokenizer.sep_token
+eos_token = tokenizer.eos_token
 
 class RobertaSNLI(nn.Module):
     def __init__(self):
@@ -24,4 +34,45 @@ class RobertaSNLI(nn.Module):
         return self.output(roberta_out[:, 0]), attentions
     
 
+class LstmSNLI(nn.Module):
+    def __init__(self):
+        super(LstmSNLI, self).__init__()
+        # Couche d'embedding : transforme les indices de mots en vecteurs denses
+        self.embedding = nn.Embedding(
+            num_embeddings=tokenizer.vocab_size,  # Taille du vocabulaire obtenu depuis le tokenizer
+            embedding_dim=128,  # Dimension des vecteurs d'embedding
+            padding_idx=tokenizer.pad_token_id,  # Index du token de padding utilisé pour égaliser les longueurs des séquences
+            max_norm=1  # La norme maximale des vecteurs d'embedding; réduit la variance des embeddings
+        )
+        # Couche LSTM : réseau de neurones récurrent capable de capturer des dépendances à long terme
+        self.lstm = nn.LSTM(
+            num_layers=4,  # Utilisation de 4 couches LSTM empilées pour plus de profondeur
+            input_size=128,  # Taille des entrées correspondant à la dimension des embeddings
+            hidden_size=128,  # Taille des états cachés dans le LSTM
+            batch_first=True,  # Indique que la première dimension des entrées représente la taille du batch
+            dropout=0.1,  # Taux de dropout appliqué aux sorties de chaque couche LSTM, sauf la dernière
+            bidirectional=True  # LSTM bidirectionnel pour capturer des informations contextuelles des deux directions
+        )
+        # Couche linéaire : effectue une transformation linéaire pour classifier les sorties du LSTM
+        self.linear = nn.Linear(
+            in_features=256,  # Taille d'entrée
+            out_features=3  # Trois sorties pour les trois classes du SNLI
+        )
+        
+    def forward(self, input_ids):
+        # Transformation des indices de mots en embeddings
+        embed = self.embedding(input_ids)
+        # Passage des embeddings à travers le LSTM
+        lstm_out = self.lstm(embed)  # lstm_out contient les sorties de toutes les étapes temporelles; les états cachés ne sont pas utilisés ici
+        return self.linear(lstm_out[0][:,0])
+
 
+def load_custom_model(model_path, model_type='lstm'):
+    if model_type == 'lstm':
+        model = LstmSNLI() 
+    else:
+        model = RobertaSNLI() 
+        print("")
+    model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')))      # chargement des poids du modèle depuis le fichier
+    model.eval()
+    return model

requirements.txt

@@ -1,4 +1,6 @@
 torch
 safetensors
 transformers
-streamlit
+streamlit
+plotly
+bertviz

utils.py

@@ -2,13 +2,13 @@ from safetensors.torch import load_model
 from transformers import RobertaTokenizer, AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments
 from transformers import GPT2TokenizerFast, GPT2ForSequenceClassification, Trainer, TrainingArguments, DataCollatorWithPadding
 from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification, Trainer, TrainingArguments, DataCollatorWithPadding
-
+import numpy as np
 
 
 
 def get_roberta():
 
-    model = AutoModelForSequenceClassification.from_pretrained('cross-encoder/nli-roberta-base')
+    model = AutoModelForSequenceClassification.from_pretrained('cross-encoder/nli-roberta-base', output_attentions=True)
     load_model(model, "roberta.safetensors")
 
     tokenizer = AutoTokenizer.from_pretrained('cross-encoder/nli-roberta-base')
@@ -17,7 +17,7 @@ def get_roberta():
 
 def get_gpt():
     
-    model = GPT2ForSequenceClassification.from_pretrained('gpt2', num_labels=3)
+    model = GPT2ForSequenceClassification.from_pretrained('gpt2', num_labels=3, output_attentions=True)
     model.config.pad_token_id = model.config.eos_token_id
     load_model(model, "gpt.safetensors")
 
@@ -28,9 +28,15 @@ def get_gpt():
 
 def get_distilbert():
 
-    model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=3)
+    model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=3, output_attentions=True)
     load_model(model, "distilbert.safetensors")
 
     tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
 
-    return tokenizer, model
+    return tokenizer, model
+
+
+def softmax(xx):
+    """Compute softmax values for each sets of scores in x."""
+    x = xx.detach().numpy()[0]
+    return np.exp(x) / np.sum(np.exp(x), axis=0)