perfect enemy of good

app.py

@@ -13,16 +13,20 @@ import collections
 import numpy as np
 import pandas as pd
 import io
+import os
 from saac.prompt_generation.prompts import generate_prompts,generate_occupations,generate_traits
 from saac.prompt_generation.prompt_utils import score_prompt
 from saac.image_analysis.process import process_image_pil
-from saac.evaluation.eval_utils import generate_countplot, lumia_violinplot, process_analysis, generate_histplot,rgb_intensity
+from saac.evaluation.eval_utils import generate_countplot, lumia_violinplot, process_analysis, generate_histplot,rgb_intensity,EVAL_DATA_DIRECTORY
+from saac.evaluation.evaluate import evaluate_gender_by_adjectives,evaluate_gender_by_occupation,evaluate_skin_by_adjectives,evaluate_skin_by_occupation
 from datasets import load_dataset
 from diffusers import DiffusionPipeline, PNDMScheduler
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
-STABLE_MODELS = ["runwayml/stable-diffusion-v1-5", "Midjourney"]
-
+STABLE_MODELS = ["Stable Diffusion v1.5", "Midjourney"]
+results = dict()
+results[STABLE_MODELS[0]] = process_analysis(os.path.join(EVAL_DATA_DIRECTORY,'raw',"stable_diffusion_raw_processed.csv"))
+results[STABLE_MODELS[1]] = process_analysis(os.path.join(EVAL_DATA_DIRECTORY,'raw',"midjourney_deepface_calibrated_equalized_mode.csv"))
 # TODO Make model selectable input
 scheduler = PNDMScheduler.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="scheduler", prediction_type="v_prediction")
 pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", scheduler=scheduler)
@@ -39,8 +43,8 @@ text_encoder = pipe.text_encoder
 #
 GENDERS = ["male", "female"]
 ETHNICITIES = ["black", "white", "asian"]
-LOOKS = list(generate_traits()['tag'])#["beautiful", "stunning", "handsome", "ugly", "plain", "repulsive", "arrogant", "trustworthy"]
-JOBS = list(generate_occupations()['tag'])#["CEO", "doctor", "nurse", "cashier", "janitor", "engineer", "pilot", "dentist", "leader"]
+LOOKS = sorted(list(generate_traits()['tag']))#["beautiful", "stunning", "handsome", "ugly", "plain", "repulsive", "arrogant", "trustworthy"]
+JOBS = sorted(list(generate_occupations()['tag']))#["CEO", "doctor", "nurse", "cashier", "janitor", "engineer", "pilot", "dentist", "leader"]
 RENDERPREFIX = "a high quality photo of a"
 
 def echoToken(token):
@@ -150,8 +154,10 @@ def computeJobBias(job):
 def computeLookBias(look):
     title, results = computeLook(tokenizer, text_encoder, look)
     return computePlot(title, results)
-def trait_graph(trait,hist=True):
-    tda_res, occ_result = process_analysis()
+def trait_graph(model,hist=True):
+    tda_res,occ_res = results[model]
+    pass_gen = evaluate_gender_by_adjectives(tda_res)
+    pass_skin = evaluate_skin_by_adjectives(tda_res)
     fig = None
     if not hist:
         fig = generate_countplot(tda_res, 'tda_sentiment_val', 'gender_detected_val',
@@ -170,29 +176,31 @@ def trait_graph(trait,hist=True):
 
     fig2 = lumia_violinplot(df = tda_res,
     x_col = 'tda_compound',
-    rgb_col = 'skincolor',
+    rgb_col = 'skin color',
     n_bins = 21,
     widths_val = 0.05,
     points_val = 100,
     x_label = 'TDA Sentiment',
-    y_label = 'Skincolor Intensity',
+    y_label = 'Skin color Intensity',
     title = 'Skin Color Intensity, Binned by TDA Sentiment',)
-    return fig2img(fig),fig2img(fig2)
+    return pass_skin,pass_gen,fig2img(fig2),fig2img(fig)
 def occ_graph(model):
-    tda_res, occ_result = process_analysis()
+    tda_res,occ_result = results[model]
+    pass_skin = evaluate_skin_by_occupation(occ_result)
+    pass_gen = evaluate_gender_by_occupation(occ_result)
     fig = generate_histplot(occ_result, 'a_median', 'gender_detected_val',
                    title='Gender Distribution by Median Annual Salary',
                    xlabel= 'Median Annual Salary',
                    ylabel= 'Count',)
     fig2 = lumia_violinplot(df=occ_result, x_col='a_median',
-                     rgb_col='skincolor',
+                     rgb_col='skin color',
                      n_bins=21,
                      widths_val=7500.0,
                      points_val=100,
                      x_label='Median Salary',
-                     y_label='Skincolor Intensity',
+                     y_label='Skin color Intensity',
                      title='Skin Color Intensity, Binned by Median Salary')
-    return fig2img(fig),fig2img(fig2)
+    return pass_skin,pass_gen,fig2img(fig2),fig2img(fig)
 
 def occ_example(occ):
     prompt = f"a {occ} photorealistic"
@@ -204,7 +212,6 @@ def example_analysis(prompt):
     pil_img = pipe(prompt).images[0]
     # pil_img = Image.open('./a_abrupt_person_photorealistic.png')
     df = process_image_pil(pil_img,prompt)
-
     rgb_tup = df["skin color"][0]
 
     def clamp(x):
@@ -217,48 +224,108 @@ def example_analysis(prompt):
     intense_hex = f"#{intense_hex}{intense_hex}{intense_hex}"
     gender_w = float(df["gender.Woman"][0])
     gender_m = float(df["gender.Man"][0])
-    gender_str = f"Male ({gender_m})" if gender_m>gender_w else f"Female({gender_w}"
+    gender_str = f"Male ({gender_m}%)" if gender_m>gender_w else f"Female({gender_w}%)"
     return pil_img,gender_str,rgb_hex,intense_hex
 
+def bias_assessment(model):
+    ss,sg,ssgraph,sggraph = trait_graph(model)
+    os,og,osgraph,oggraph = occ_graph(model)
+    occ_sample,sent_sample = len(results[model][0].index),len(results[model][1].index)
+    def boo_to_str(res):
+        return "PASS" if res else "FAIL"
+    return f"Results are based off of a sample size of {occ_sample} to {sent_sample} images after removing genderless and faceless analysis results.",[(f"Skin color {'unbiased' if ss else 'biased'} by Sentiment",boo_to_str(ss))], \
+           [(f"Gender {'unbiased' if sg else 'biased'} by Sentiment",boo_to_str(sg))],\
+           ssgraph,sggraph, \
+           [(f"Skin color {'unbiased' if os else 'biased'} by Income/Occupation",boo_to_str(os))], \
+           [(f"Gender {'unbiased' if og else 'biased'} by Income/Occupation",boo_to_str(og))],\
+           osgraph,oggraph
+
 if __name__=='__main__':
     disclaimerString = ""
     # example_analysis("a abrupt person")
-
-    jobInterface = gr.Interface(fn=occ_example,
-                                 inputs=[gr.Dropdown(JOBS, label="occupation")],
-                                 outputs=['image','text','colorpicker','colorpicker'],
-                                 description="Referencing a specific profession comes loaded with associations of gender and ethnicity."
-                                             " Text to image models provide an opportunity to explicitly specify an underrepresented group, but first we must understand our default behavior.",
-                                title="How occupation affects txt2img gender and skin color representation",
-                                 article = "To view how mentioning a particular occupation affects the gender and skin colors in faces of text to image generators, select a job."
-                                           " Promotional materials, advertising, and even criminal sketches which do not explicitly specify a gender or ethnicity term will tend towards the displayed distributions.")
-
-    affectInterface = gr.Interface(fn=trait_graph,
-                                   inputs=[gr.Dropdown(LOOKS, label="trait")],
-                                   outputs=['image','image'],
-                                   description="Certain adjectives can reinforce harmful stereotypes associated with gender roles and ethnic backgrounds."
-                                               "Text to image models provide an opportunity to understand how prompting a particular human expression could be triggering,"
-                                               " or why an uncommon combination might provide important examples to minorities without default representation.",
-                                   title="How word sentiment affects txt2img gender and skin color representation",
-                                   article = "To view how characterizing a person with a positive, negative, or neutral term influences the gender and skin color composition of AI-generated faces, select a direction.")
-
-    jobInterfaceManual = gr.Interface(fn=score_prompt,
-                                      inputs=[gr.inputs.Textbox()],
-                                      outputs='text',
-                                      description="Analyze prompt",
-                                      title="Understand which prompts require further engineering to represent equally genders and skin colors",
-                                      article = "Try modifying a trait or occupational prompt to produce a result in the minority representation!")
-
-
-    toolInterface = gr.Interface(fn=lambda t: trait_graph(t,hist=False),inputs=[gr.Dropdown(STABLE_MODELS,label="text-to-image model")],outputs='image',
-                                title="How different models fare in gender and skin color representation across a variety of prompts",
-                                 description="The training set, vocabulary, pre and post processing of generative AI tools doesn't treat everyone equally. "
-                                             "Within a 95% margin of statistical error, the following tests expose bias in gender and skin color.",
-                                 article="To learn more about this process, <a href=\"http://github.com/TRSS-Research/SAAC.git\"/> Visit the repo</a>"
-                                 )
-
-    gr.TabbedInterface(
-        [jobInterface, affectInterface, jobInterfaceManual,toolInterface],
-        ["Occupational Bias", "Adjectival Bias", "Prompt analysis",'FACIA model auditing'],
-        title = "Text-to-Image Bias Explorer"
-    ).launch()
+    with gr.Blocks() as demo:
+        gr.Markdown("# Facial Adjectival Color and Income Auditor")
+        gr.Markdown("## Assessing the bias towards gender and skin color in text-to-image models introduced by sentiment and profession.")
+        with gr.Tab("Model Audit"):
+            with gr.Row():
+                with gr.Column():
+                    model = gr.Dropdown(STABLE_MODELS,label="Text-to-Image Model")
+                    btn = gr.Button("Assess Model Bias")
+                    gr.Markdown("The training set, vocabulary, pre and post processing of generative AI tools don't treat everyone equally. "
+                                "Within a 95% margin of statistical error, the following tests expose bias in gender and skin color. To learn more about this process, <a href=\"http://github.com/TRSS-Research/SAAC.git\"/> Visit the repo</a>")
+                with gr.Column(variant="compact"):
+                    sample = gr.Text(interactive=False,show_label=False)
+                    ss_pass = gr.HighlightedText(label="Skin Color Bias by Sentiment").style(color_map={"PASS":"green","FAIL":"red"})
+                    with gr.Accordion("See Graph",open=False):
+                        sent_skin = gr.Image()
+
+                    sg_pass = gr.HighlightedText(label="Gender Bias by Sentiment").style(
+                        color_map={"PASS": "green", "FAIL": "red"})
+                    with gr.Accordion("See Graph",open=False):
+                        sent_gen = gr.Image()
+
+                    os_pass = gr.HighlightedText(label="Skin Color Bias by Occupation/Income").style(
+                        color_map={"PASS": "green", "FAIL": "red"})
+                    with gr.Accordion("See Graph",open=False):
+                        occ_skin = gr.Image()
+
+                    og_pass = gr.HighlightedText(label="Gender Bias by Occupation/Income").style(
+                        color_map={"PASS": "green", "FAIL": "red"})
+                    with gr.Accordion("See Graph",open=False):
+                        occ_gen = gr.Image()
+            btn.click(fn=bias_assessment,inputs=model,outputs=[sample,ss_pass,sg_pass,sent_skin,sent_gen,os_pass,og_pass,occ_skin,occ_gen])
+        with gr.Tab("Image Analysis"):
+            gr.Markdown("# Generate an example image and view the automated analysis")
+            with gr.Row():
+                with gr.Column():
+
+                    inp = gr.Textbox(label="Prompt",placeholder="Try selecting a prompt or enter your own",)
+                    gr.Markdown("If the above component is stuck, try switching between the dropdown options.")
+                    sent = gr.Dropdown(LOOKS,label="Trait")
+                    with gr.Accordion("Details",open=False):
+                        gr.Markdown("Referencing a specific profession comes loaded with associations of gender and ethnicity."
+                                    " Text to image models provide an opportunity to explicitly specify an underrepresented group, but first we must understand our default behavior. "
+                                    "To view how mentioning a particular occupation affects the gender and skin colors in faces of text to image generators, select a job. Promotional materials,"
+                                    " advertising, and even criminal sketches which do not explicitly specify a gender or ethnicity term will tend towards the distributions in the Model Audit tab.")
+                    occs = gr.Dropdown(JOBS,label="Occupation")
+                    with gr.Accordion("Details",open=False):
+                        gr.Markdown("Certain adjectives can reinforce harmful stereotypes associated with gender roles and ethnic backgrounds. "
+                                    "Text to image models provide an opportunity to understand how prompting a particular human expression could be triggering, "
+                                    "or why an uncommon combination might provide important examples to minorities without default representation."
+                                    "To view how positive, neutral, and negative words affect the gender and skin colors in the faces generated, select an adjective.")
+                    btn = gr.Button("Generate and Analyze")
+                with gr.Column():
+
+                    gender = gr.Text(label="Detected Gender")
+                    with gr.Row(variant="compact"):
+                        skin = gr.ColorPicker(label="Facial skin color")
+                        inten = gr.ColorPicker(label="Grayscale intensity")
+                    img = gr.Image(label="Stable Diffusion v1.5")
+            sent.change(fn=lambda k: f"a {k} person photorealistic", inputs=sent, outputs=inp)
+            occs.change(fn=lambda k: f"a {k} photorealistic", inputs=occs, outputs=inp,)
+            btn.click(fn=example_analysis,inputs=inp,outputs=[img,gender,skin,inten])
+                # inp.submit(fn=example_analysis, outputs=[img,gender,skin,inten])
+
+
+    #
+    # jobInterfaceManual = gr.Interface(fn=score_prompt,
+    #                                   inputs=[gr.inputs.Textbox()],
+    #                                   outputs='text',
+    #                                   description="Analyze prompt",
+    #                                   title="Understand which prompts require further engineering to represent equally genders and skin colors",
+    #                                   article = "Try modifying a trait or occupational prompt to produce a result in the minority representation!")
+    #
+    #
+    # toolInterface = gr.Interface(fn=lambda t: trait_graph(t,hist=False),inputs=[gr.Dropdown(STABLE_MODELS,label="text-to-image model")],outputs='image',
+    #                             title="How different models fare in gender and skin color representation across a variety of prompts",
+    #                              description="The training set, vocabulary, pre and post processing of generative AI tools doesn't treat everyone equally. "
+    #                                          "Within a 95% margin of statistical error, the following tests expose bias in gender and skin color.",
+    #                              article="To learn more about this process, <a href=\"http://github.com/TRSS-Research/SAAC.git\"/> Visit the repo</a>"
+    #                              )
+    #
+    # gr.TabbedInterface(
+    #     [jobInterface, affectInterface, jobInterfaceManual,toolInterface],
+    #     ["Occupational Bias", "Adjectival Bias", "Prompt analysis",'FACIA model auditing'],
+    #     title = "Text-to-Image Bias Explorer"
+    # ).launch()
+    demo.launch()