Add option to enter a pair manually

app.py

@@ -1,6 +1,7 @@
 import torch
 import datasets
 import gradio
+import pandas
 
 from transformers import GPT2LMHeadModel, GPT2TokenizerFast
 
@@ -23,10 +24,9 @@ class CrowSPairsDataset(object):
         return self.df.bias_type.unique().tolist()
 
 
-def run(bias_type):
-    sample = dataset.sample(bias_type)
+def run(df):
     result = "<table><tr style='color: white; background-color: #555'><th>index</th><th>more stereotypical</th><th>less stereotypical<th></tr>"
-    for i, row in sample.iterrows():
+    for i, row in df.iterrows():
         result += f"<tr><td>{i}</td>"
         more = row["sent_more"]
 
@@ -55,6 +55,17 @@ def run(bias_type):
     result += "</table>"
     return result
 
+def sample_and_run(bias_type):
+    sample = dataset.sample(bias_type)
+    return run(sample)
+
+def manual_run(more, less):
+    df = pandas.DataFrame.from_dict({
+            'sent_more': [more],
+            'sent_less': [less],
+            'bias_type': ["manual"],
+        })
+    return run(df)
 
 if torch.cuda.is_available():
     device = torch.device("cuda")
@@ -71,9 +82,12 @@ bias_type_sel = gradio.Dropdown(label="Bias Type", choices=dataset.bias_types())
 with open("description.md") as fh:
     desc = fh.read()
 
+with open("descr-2.md") as fh:
+    desc2 = fh.read()
+
 with open("notice.md") as fh:
     notice = fh.read()
-    
+
 with open("results.md") as fh:
     results = fh.read()
 
@@ -81,11 +95,21 @@ with gradio.Blocks(title="Detecting stereotypes in the GPT-2 language model usin
     gradio.Markdown(desc)
     with gradio.Row(equal_height=True):
         with gradio.Column(scale=4):
-            inp = gradio.Dropdown(label="Bias Type", choices=dataset.bias_types())
+            bias_sel = gradio.Dropdown(label="Bias Type", choices=dataset.bias_types())
         with gradio.Column(scale=1):
             but = gradio.Button("Sample")
+    gradio.Markdown(desc2)
+    with gradio.Row(equal_height=True):
+        with gradio.Column(scale=2):
+            more = gradio.Textbox(label="More stereotypical")
+        with gradio.Column(scale=2):
+            less = gradio.Textbox(label="Less stereotypical")
+        with gradio.Column(scale=1):
+            manual = gradio.Button("Run")
     out = gradio.HTML()
-    but.click(run, inp, out)
+    but.click(sample_and_run, bias_sel, out)
+    manual.click(manual_run, [more, less], out)
+
     with gradio.Accordion("A note about explainability models"):
         gradio.Markdown(notice)
     with gradio.Accordion("Results for English and French BERT language models"):

descr-2.md

@@ -0,0 +1,3 @@
+Or you can enter a pair of sentences in the entries below, and click `Run` to get the result for your manual pair.
+
+*The colors indicate whether the <font color=#00ffff>stereotypical</font>  or the <font color=#ff00ff>less stereotypical</font> examples gets the higher score, the intensity of the color how strong the preference is.*

description.md

@@ -1,5 +1,9 @@
 # Detecting stereotypes in the GPT-2 language model using CrowS-Pairs
 
-*GPT-2* is a language model which can score how likely it is that some text is a valid English sentence: not only grammaticality, but also the 'meaning' of the sentence is part of this score. *CrowS-Pairs* is a dataset with pairs of more and less stereotypical examples for different social groups (e.g., gender and nationality stereotypes). **We sample 10 random pairs from CrowS-Pairs and show whether the stereotypical example gets a higher score ('is more likely'). If GPT-2 systematically prefers the stereotypical examples, it has probably learnt these stereotypes from the training data.**
+*GPT-2* is a language model which can score how likely it is that some text is a valid English sentence: not only grammaticality, but also the 'meaning' of the sentence is part of this score. *CrowS-Pairs* is a dataset with pairs of more and less stereotypical examples for different social groups (e.g., gender and nationality stereotypes).
 
-The colors indicate whether the <font color=#00ffff>stereotypical</font>  or the <font color=#ff00ff>less stereotypical</font> examples gets the higher score, the intensity of the color how strong the preference is.
+You can either select a CrowS-Pairs bias type from the drop-down below and click `Sample`, and then we
+sample 10 random pairs from CrowS-Pairs and show whether the stereotypical example gets
+a higher score ('is more likely').
+
+**If GPT-2 systematically prefers the stereotypical examples, it has probably learnt these stereotypes from the training data.**

requirements.txt

@@ -1,3 +1,4 @@
 torch
 transformers
 datasets
+pandas