Adding bert-like. Organizing description

app.py

@@ -1,14 +1,16 @@
+from typing import Optional
 import gradio as gr
 import torch
 from transformers import AutoModelForTokenClassification, AutoTokenizer
+from transformers import pipeline
 import pandas as pd
 import numpy as np
 
 
-
 # Play with me, consts
 CONDITIONING_VARIABLES = ["none", "birth_place", "birth_date", "name"]
 FEMALE_WEIGHTS = [1.5, 5]  # About 5x more male than female tokens in dataset
+BERT_LIKE_MODELS = ["bert", "distilbert"]
 
 # Internal consts
 START_YEAR = 1800
@@ -44,6 +46,9 @@ for var in CONDITIONING_VARIABLES:
         models[(var, f_weight)] = AutoModelForTokenClassification.from_pretrained(
             models_paths[(var, f_weight)]
         )
+for bert_like in BERT_LIKE_MODELS:
+    models_paths[(bert_like,)] = f"{bert_like}-base-uncased"
+    models[(bert_like,)] = pipeline("fill-mask", model=models_paths[(bert_like,)])
 
 
 # Tokenizers same for each model, so just grabbing one of them
@@ -54,7 +59,7 @@ MASK_TOKEN_ID = tokenizer.mask_token_id
 
 
 # more static stuff
-gendered_lists = [ 
+gendered_lists = [
     ["he", "she"],
     ["him", "her"],
     ["his", "hers"],
@@ -63,15 +68,12 @@ gendered_lists = [
     ["men", "women"],
     ["husband", "wife"],
 ]
-male_gendered_dict = {list[0]: list for list in gendered_lists}
-female_gendered_dict = {list[1]: list for list in gendered_lists}
+male_gendered_tokens = [list[0] for list in gendered_lists]
+female_gendered_tokens = [list[1] for list in gendered_lists]
+
+male_gendered_token_ids = tokenizer.convert_tokens_to_ids(male_gendered_tokens)
+female_gendered_token_ids = tokenizer.convert_tokens_to_ids(female_gendered_tokens)
 
-male_gendered_token_ids = tokenizer.convert_tokens_to_ids(
-    list(male_gendered_dict.keys())
-)
-female_gendered_token_ids = tokenizer.convert_tokens_to_ids(
-    list(female_gendered_dict.keys())
-)
 assert tokenizer.unk_token_id not in male_gendered_token_ids
 assert tokenizer.unk_token_id not in female_gendered_token_ids
 
@@ -133,15 +135,30 @@ def tokenize_and_append_metadata(text, tokenizer):
 
 # Run inference
 def predict_gender_pronouns(
-    num_points, conditioning_variables, f_weights, input_text, return_preds=False
+    num_points, conditioning_variables, f_weights, bert_like_models, input_text
 ):
 
     text_portions = input_text.split(SPLIT_KEY)
 
     years = np.linspace(START_YEAR, STOP_YEAR, int(num_points)).astype(int)
+    num_preds = None
 
     dfs = []
     dfs.append(pd.DataFrame({"year": years}))
+
+    tokenized = {'ids':[], 'atten_mask':[], 'toks':[], 'labels':[]}
+    for b_date in years:
+        target_text = f"{b_date}".join(text_portions)
+        tokenized_sample = tokenize_and_append_metadata(
+            target_text,
+            tokenizer=tokenizer,    
+        )
+
+        tokenized['ids'].append(tokenized_sample["input_ids"])
+        tokenized['atten_mask'].append(torch.tensor(tokenized_sample["attention_mask"]))
+        tokenized['toks'].append(tokenizer.convert_ids_to_tokens(tokenized_sample["input_ids"]))
+        tokenized['labels'].append(tokenized_sample["labels"])
+
     for f_weight in f_weights:
         for var in conditioning_variables:
             prefix = f"w{f_weight}_{var}"
@@ -149,17 +166,10 @@ def predict_gender_pronouns(
 
             p_female = []
             p_male = []
-            for b_date in years:
-                target_text = f"{b_date}".join(text_portions)
-                tokenized_sample = tokenize_and_append_metadata(
-                    target_text,
-                    tokenizer=tokenizer,
-                )
-
-                ids = tokenized_sample["input_ids"]
-                atten_mask = torch.tensor(tokenized_sample["attention_mask"])
-                toks = tokenizer.convert_ids_to_tokens(ids)
-                labels = tokenized_sample["labels"]
+            for year_idx in range(len(tokenized['ids'])):
+                ids = tokenized["ids"][year_idx]
+                atten_mask = tokenized["atten_mask"][year_idx]
+                labels = tokenized["labels"][year_idx]
 
                 with torch.no_grad():
                     outputs = model(ids.unsqueeze(dim=0), atten_mask.unsqueeze(dim=0))
@@ -167,13 +177,45 @@ def predict_gender_pronouns(
 
                     was_masked = labels.cpu() != -100
                     preds = torch.where(was_masked, preds, -100)
-                    num_preds = torch.sum(was_masked).item()
 
-                    p_female.append(len(torch.where(preds==0)[0])/num_preds*100)
-                    p_male.append(len(torch.where(preds==1)[0])/num_preds*100)
+                    if not num_preds:
+                        num_preds = torch.sum(was_masked).item()
+
+                    p_female.append(len(torch.where(preds == 0)[0]) / num_preds * 100)
+                    p_male.append(len(torch.where(preds == 1)[0]) / num_preds * 100)
 
             dfs.append(pd.DataFrame({f"%f_{prefix}": p_female, f"%m_{prefix}": p_male}))
 
+
+    for bert_like in bert_like_models:
+
+        p_female = []
+        p_male = []
+        for year_idx in range(len(tokenized['ids'])):
+            toks = tokenized["toks"][year_idx]
+            target_text_for_bert = ' '.join(toks[1:-1] ) # Removing [CLS] and [SEP]
+
+            prefix = bert_like
+            model = models[(bert_like,)]
+
+
+            mask_filled_text =  model(target_text_for_bert)
+
+            female_pronouns = [
+                1 if pronoun[0]["token_str"] in female_gendered_tokens else 0
+                for pronoun in mask_filled_text
+            ]
+            male_pronouns = [
+                1 if pronoun[0]["token_str"] in male_gendered_tokens else 0
+                for pronoun in mask_filled_text
+            ]
+
+            p_female.append(sum(female_pronouns) / num_preds * 100)
+            p_male.append(sum(male_pronouns) / num_preds * 100)
+
+        dfs.append(pd.DataFrame({f"%f_{prefix}": p_female, f"%m_{prefix}": p_male}))
+
+
     results = pd.concat(dfs, axis=1).set_index("year")
 
     female_df = results.filter(regex=".*f_")
@@ -192,14 +234,21 @@ def predict_gender_pronouns(
         female_df,
         male_df_for_plot,
         male_df,
-    ) 
+    )
 
 
 title = "Changing Gender Pronouns"
-description =  """
-This is a demo for a project exploring possible spurious correlations in training datasets that can be exploited and manipulated to achieve alternative outcomes. In this case, manipulating `DATE` to change the predicted gender pronouns for both the BERT base model and a model fine-tuned with a specific pronoun predicting task using the [wiki-bio](https://huggingface.co/datasets/wiki_bio) dataset.
-One way to explain phenomena is by looking at a likely  data generating process for biographical-like data in both the main BERT training dataset as well as the `wiki_bio` dataset, in the form of a causal DAG. 
+description = """
+<h2> Intro </h2>
+This is a demo for a project exploring possible spurious correlations in training datasets that can be exploited and manipulated to achieve alternative outcomes. In this case, a user can demo what context changes will cause predicted gender pronouns to change, in a range of models. 
+
+In a user provided sentence, with at least one reference to a `DATE` and one gender pronoun, we will see how sweeping through a range of `DATE` values can change the predicted pronouns. 
+
+We see this in both the BERT base model and a model fine-tuned with a specific pronoun predicting task on the [wiki-bio](https://huggingface.co/datasets/wiki_bio) dataset.
+
+One way to explain this phenomena is by looking at a likely  data generating process for biographical-like data in both the main BERT training dataset as well as the `wiki_bio` dataset, in the form of a causal DAG. 
  
+<h2> Causal DAG </h2> 
 In the DAG, we can see that `birth_place`, `birth_date` and `gender` are all independent elements that have no common cause with the other covariates in the DAG. However `birth_place`, `birth_date` and `gender` may all have a role in causing one's `access_to_resources`, with the general trend that `access_to_resources` has become less gender-dependent over time, but not in every `birth_place`, with recent events in Afghanistan providing a stark counterexample to this trend. `access_to_resources` further determines how or if at all, you may appear in the dataset’s `context_words`.
  
 We also argue that although there are complex causal interactions between words in a segment, the `context_words` are more likely to cause the `gender_pronouns`, rather than vice versa. For example, if the subject is a famous doctor and the object is her wealthy father, these context words will determine which person is being referred to, and thus which gendered-pronoun to use.
@@ -212,9 +261,10 @@ In this graph, any pink path between `context_words` and `gender_pronouns` will
     alt="DAG of possible data generating process for datasets used in training.">
 </center>
   
-Those familiar with causal DAGs may note when can simply condition on `gender` to block any confounding between the `context_words` and the `gender_pronouns`.  However, this is not always possible, particularly in generative or mask-filling tasks, like those common in language models.  
+Those familiar with causal DAGs may note when can simply condition on `gender` to block any confounding between the `context_words` and the `gender_pronouns`.  However, this is not always possible, particularly in generative or mask-filling tasks, like those common in language models and in the demo below.  
  
-Here, we automatically mask (for prediction) the following tokens (and they will also be automatically masked if you use them below.)
+ <h2> How to use this demo </h2> 
+In this demo, a user can add any sentence that contains at least one gender pronoun and the capitalized word `DATE`. We then sweep through a range of `date` values in the place of `DATE`, while masking (for prediction) the gender pronouns (included in the list below).
 ```
 gendered_lists = [
    ['he', 'she'],
@@ -226,24 +276,25 @@ gendered_lists = [
    ["husband", "wife"],
 ]
 ```
- 
-In this demo we are looking for a dose-response relationship between:
-- our treatment: the text,
+
+In addition to chosing the test sentence, we ask that you pick how the fine-tuned model was trained:
+- conditioning variable: which, if any, conditioning variable from the three noted above in the DAG, was included in the text at train time.
+- loss function weight: weight assigned to the minority class (female pronouns in this fine-tuning dataset) that was included in the text at train time.
+
+
+
+
+
+ <h2> What are the results</h2> 
+
+In the resulting plots, we can look for a dose-response relationship between:
+- our treatment: the sample text,
 - and our outcome: the predicted gender of pronouns in the text.
  
 Specifically we are seeing if making larger magnitude intervention: an older `DATE` in the text will result in a larger magnitude effect in the outcome: higher percentage of predicted female pronouns.
- 
-In the demo below you can select among 4 different fine-tuning methods:
-- which, if any, conditioning variable was appended to the text.
- 
-And two different weighting schemes that were used in the loss function to nudge more toward the minority class in the dataset: 
-- female pronouns.
-
-
-One trend that appears is: conditioning on `birth_date` metadata in both training and inference text has the largest dose-response relationship. This seems reasonable, as the fine-tuned model is able to ‘stratify’ a learned relationship between gender pronouns and dates, when both are present in the text.
 
+One trend that appears is: conditioning on `birth_date` metadata in both training and inference text has the largest dose-response relationship. This seems reasonable, as the fine-tuned model is able to 'stratify' a learned relationship between gender pronouns and dates, when both are present in the text.
 While conditioning on either no metadata or `birth_place` data training, have similar middle-ground effects for this inference task. 
-
 Finally,  conditioning on `name` metadata in training, (while again conditioning on `date` in inference) has almost no dose-response relationship. It appears the learning of a `name —> gender pronouns` relationship was sufficiently successful to overwhelm any potential more nuanced learning, such as that driven by `birth_date` or `place`. 
 """
 
@@ -255,24 +306,30 @@ gr.Interface(
     fn=predict_gender_pronouns,
     inputs=[
         gr.inputs.Number(
-            default=10,
+            default=15,
             label="Number of points (years) plotted -- select fewer if slow.",
         ),
         gr.inputs.CheckboxGroup(
             CONDITIONING_VARIABLES,
             default=["none", "birth_date"],
             type="value",
-            label="Pick model(s) that were trained with the following conditioning variables",
+            label="Pick conditioning variable included in text during fine-tuning.",
         ),
         gr.inputs.CheckboxGroup(
             FEMALE_WEIGHTS,
             default=[5],
             type="value",
-            label="Pick model(s) that were trained with the following loss function weight on female predictions",
+            label="Pick loss function weight placed on female predictions  during fine-tuning.",
+        ),
+        gr.inputs.CheckboxGroup(
+            BERT_LIKE_MODELS,
+            default=["bert"],
+            type="value",
+            label="Pick optional bert-like base uncased model for comparison.",
         ),
         gr.inputs.Textbox(
             lines=7,
-            label="Input Text. Include one of more instance of the word 'DATE' below, to be replace with a range of dates in demo.", 
+            label="Input Text. Include one of more instance of the word 'DATE' below, to be replace with a range of dates in demo.",
             default="Born DATE, she was a computer scientist. Her work was greatly respected, and she was well-regarded in her field.",
         ),
     ],
@@ -295,7 +352,7 @@ gr.Interface(
             label="Precent pred male pronoun vs year, per model trained with conditioning and with weight for female preds",
         ),
     ],
-    title = title, 
-    description = description, 
-    article = article
-).launch()
+    title=title,
+    description=description,
+    article=article,
+).launch()