app.py

import json
import gradio as gr
import pandas as pd
import os

from scripts.genbit_metrics import *
from scripts.gender_profession_tagging import *
from scripts.gender_tagging import *
from utils.load_csv import *
from utils.read_config import get_args

methodologies = json.load(open("methodologies.json", "r"))


def get_methodology_metadata(methodology):
    title = "## " + methodology
    description = methodologies.get(methodology).get("description")

    metadata = f"{title}\n\n{description}"

    return gr.Markdown.update(metadata, visible=True)


def evaluate(dataset_file, dataset_scope, dataset_scope_n, dataset_column, methodology):
    status = {}
    dataset = pd.read_csv(dataset_file.name)
    sample_method = dataset_scope
    col_name = dataset_column
    num_sample_records = dataset_scope_n

    status = globals()[methodologies.get(methodology).get("fx")](
        dataset, sample_method, col_name, num_sample_records
    )

    return gr.JSON.update(status, visible=True)


def process_dataset(dataset):
    data = pd.read_csv(dataset.name)

    columns = data.select_dtypes(include=["object"]).columns.tolist()

    return (
        gr.Radio.update(
            label="Scope",
            info="Determines the scope of the dataset to be analyzed",
            choices=["First", "Last", "Random"],
            value="First",
            visible=True,
            interactive=True,
        ),
        gr.Slider.update(
            label=f"Number of Entries",
            info=f"Determines the number of entries to be analyzed. Due to computational constraints, the maximum number of entries that can be analyzed is {get_args('first_records')}.",
            minimum=1,
            maximum=min(data.shape[0], get_args("first_records")),
            value=min(data.shape[0], get_args("first_records")) // 2,
            visible=True,
            interactive=True,
        ),
        gr.Radio.update(
            label="Column",
            info="Determines the column to be analyzed. These are the columns with text data.",
            choices=columns,
            value=columns[0],
            visible=True,
            interactive=True,
        ),
    )


def get_column_metadata(dataset, column):
    data = pd.read_csv(dataset.name)
    corpus = data[column].head(10).tolist()

    return gr.Dataframe.update(
        value=pd.DataFrame({f"Data Corpus: {column}": corpus}), visible=True
    )


BiasAware = gr.Blocks(title="BiasAware: Dataset Bias Detection")

with BiasAware:
    gr.Markdown(
        "# BiasAware: Dataset Bias Detection\n\nBiasAware is a specialized tool for detecting and quantifying biases within datasets used for Natural Language Processing (NLP) tasks. NLP training datasets frequently mirror the inherent biases of their source materials, resulting in AI models that unintentionally perpetuate stereotypes, exhibit underrepresentation, and showcase skewed perspectives."
    )

    with gr.Row():
        with gr.Column(scale=2):
            gr.Markdown("## Dataset")

            dataset_file = gr.File(label="Dataset")
            dataset_examples = gr.Examples(
                [
                    os.path.join(os.path.dirname(__file__), "data/z_animal.csv"),
                    os.path.join(os.path.dirname(__file__), "data/z_employee.csv"),
                    os.path.join(os.path.dirname(__file__), "data/z_house.csv"),
                ],
                inputs=dataset_file,
                label="Example Datasets",
            )

            dataset_scope = gr.Radio(visible=False)
            dataset_scope_n = gr.Slider(visible=False)
            dataset_column = gr.Radio(visible=False)

            dataset_corpus = gr.Dataframe(
                row_count=(5, "fixed"), col_count=(1, "fixed"), visible=False
            )

        with gr.Column(scale=2):
            gr.Markdown("## Methodology")

            methodology = gr.Radio(
                label="Methodology",
                info="Determines the methodology to be used for bias detection",
                choices=[
                    "Gender Divide (Term Identity Diversity)",
                    "Gender Profession Bias (Lexical Evaluation)",
                    "GenBiT (Microsoft Responsible AI Gender Bias Tool)",
                ],
            )

            evalButton = gr.Button("Run Evaluation")

            methodology_metadata = gr.Markdown(visible=False)

        with gr.Column(scale=4):
            gr.Markdown("## Result")

            result_status = gr.JSON(visible=False)
            result = gr.DataFrame(
                row_count=(5, "fixed"), col_count=(3, "fixed"), visible=False
            )

    dataset_file.change(
        fn=process_dataset,
        inputs=[dataset_file],
        outputs=[dataset_scope, dataset_scope_n, dataset_column],
    )

    dataset_column.change(
        fn=get_column_metadata,
        inputs=[dataset_file, dataset_column],
        outputs=[dataset_corpus],
    )

    methodology.change(
        fn=get_methodology_metadata,
        inputs=[methodology],
        outputs=[methodology_metadata],
    )

    evalButton.click(
        fn=evaluate,
        inputs=[
            dataset_file,
            dataset_scope,
            dataset_scope_n,
            dataset_column,
            methodology,
        ],
        outputs=[result_status],
    )

BiasAware.launch()