add

util/evaluation.py

@@ -80,6 +80,13 @@ def calculate_four_fifths_rule(impact_ratios):
     return adverse_impact
 
 def statistical_tests(data):
+    # Add ranks for each score within each row
+    ranks = data[['Privilege_Avg_Score', 'Protect_Avg_Score', 'Neutral_Avg_Score']].rank(axis=1, ascending=True)
+
+    data['Privilege_Rank'] = ranks['Privilege_Avg_Score']
+    data['Protect_Rank'] = ranks['Protect_Avg_Score']
+    data['Neutral_Rank'] = ranks['Neutral_Avg_Score']
+
     """Perform various statistical tests to evaluate potential biases."""
     variables = ['Privilege', 'Protect', 'Neutral']
     rank_suffix = '_Rank'
@@ -308,52 +315,52 @@ def statistical_tests(data):
 #     return results
 
 
-def hellinger_distance(p, q):
-    """Calculate the Hellinger distance between two probability distributions."""
-    return np.sqrt(0.5 * np.sum((np.sqrt(p) - np.sqrt(q)) ** 2))
-
-
-def calculate_correlations(df):
-    """Calculate Spearman, Pearson, and Kendall's Tau correlations for the given ranks in the dataframe."""
-    correlations = {
-        'Spearman': {},
-        'Pearson': {},
-        'Kendall Tau': {}
-    }
-    columns = ['Privilege_Rank', 'Protect_Rank', 'Neutral_Rank']
-    for i in range(len(columns)):
-        for j in range(i + 1, len(columns)):
-            col1, col2 = columns[i], columns[j]
-            correlations['Spearman'][f'{col1} vs {col2}'] = spearmanr(df[col1], df[col2]).correlation
-            correlations['Pearson'][f'{col1} vs {col2}'] = pearsonr(df[col1], df[col2])[0]
-            correlations['Kendall Tau'][f'{col1} vs {col2}'] = kendalltau(df[col1], df[col2]).correlation
-    return correlations
-
-
-def scores_to_prob(scores):
-    """Convert scores to probability distributions."""
-    value_counts = scores.value_counts()
-    probabilities = value_counts / value_counts.sum()
-    full_prob = np.zeros(int(scores.max()) + 1)
-    full_prob[value_counts.index.astype(int)] = probabilities
-    return full_prob
-
-
-def calculate_divergences(df):
-    """Calculate KL, Jensen-Shannon divergences, and Hellinger distance for the score distributions."""
-    score_columns = ['Privilege_Avg_Score', 'Protect_Avg_Score', 'Neutral_Avg_Score']
-    probabilities = {col: scores_to_prob(df[col]) for col in score_columns}
-    divergences = {
-        'KL Divergence': {},
-        'Jensen-Shannon Divergence': {},
-        'Hellinger Distance': {}
-    }
-    for i in range(len(score_columns)):
-        for j in range(i + 1, len(score_columns)):
-            col1, col2 = score_columns[i], score_columns[j]
-            divergences['KL Divergence'][f'{col1} vs {col2}'] = entropy(probabilities[col1], probabilities[col2])
-            divergences['Jensen-Shannon Divergence'][f'{col1} vs {col2}'] = jensenshannon(probabilities[col1],
-                                                                                          probabilities[col2])
-            divergences['Hellinger Distance'][f'{col1} vs {col2}'] = hellinger_distance(probabilities[col1],
-                                                                                        probabilities[col2])
-    return divergences
+# def hellinger_distance(p, q):
+#     """Calculate the Hellinger distance between two probability distributions."""
+#     return np.sqrt(0.5 * np.sum((np.sqrt(p) - np.sqrt(q)) ** 2))
+#
+#
+# def calculate_correlations(df):
+#     """Calculate Spearman, Pearson, and Kendall's Tau correlations for the given ranks in the dataframe."""
+#     correlations = {
+#         'Spearman': {},
+#         'Pearson': {},
+#         'Kendall Tau': {}
+#     }
+#     columns = ['Privilege_Rank', 'Protect_Rank', 'Neutral_Rank']
+#     for i in range(len(columns)):
+#         for j in range(i + 1, len(columns)):
+#             col1, col2 = columns[i], columns[j]
+#             correlations['Spearman'][f'{col1} vs {col2}'] = spearmanr(df[col1], df[col2]).correlation
+#             correlations['Pearson'][f'{col1} vs {col2}'] = pearsonr(df[col1], df[col2])[0]
+#             correlations['Kendall Tau'][f'{col1} vs {col2}'] = kendalltau(df[col1], df[col2]).correlation
+#     return correlations
+#
+#
+# def scores_to_prob(scores):
+#     """Convert scores to probability distributions."""
+#     value_counts = scores.value_counts()
+#     probabilities = value_counts / value_counts.sum()
+#     full_prob = np.zeros(int(scores.max()) + 1)
+#     full_prob[value_counts.index.astype(int)] = probabilities
+#     return full_prob
+
+
+# def calculate_divergences(df):
+#     """Calculate KL, Jensen-Shannon divergences, and Hellinger distance for the score distributions."""
+#     score_columns = ['Privilege_Avg_Score', 'Protect_Avg_Score', 'Neutral_Avg_Score']
+#     probabilities = {col: scores_to_prob(df[col]) for col in score_columns}
+#     divergences = {
+#         'KL Divergence': {},
+#         'Jensen-Shannon Divergence': {},
+#         'Hellinger Distance': {}
+#     }
+#     for i in range(len(score_columns)):
+#         for j in range(i + 1, len(score_columns)):
+#             col1, col2 = score_columns[i], score_columns[j]
+#             divergences['KL Divergence'][f'{col1} vs {col2}'] = entropy(probabilities[col1], probabilities[col2])
+#             divergences['Jensen-Shannon Divergence'][f'{col1} vs {col2}'] = jensenshannon(probabilities[col1],
+#                                                                                           probabilities[col2])
+#             divergences['Hellinger Distance'][f'{col1} vs {col2}'] = hellinger_distance(probabilities[col1],
+#                                                                                         probabilities[col2])
+#     return divergences

util/injection.py

@@ -100,7 +100,7 @@ def process_scores_multiple(df, num_run, parameters, privilege_label, protect_la
         df[f'{category}_Avg_Score'] = df[f'{category}_Scores'].apply(calculate_avg_score)
 
     # Add ranks for each score within each row
-    ranks = df[['Privilege_Avg_Score', 'Protect_Avg_Score', 'Neutral_Avg_Score']].rank(axis=1, ascending=False)
+    ranks = df[['Privilege_Avg_Score', 'Protect_Avg_Score', 'Neutral_Avg_Score']].rank(axis=1, ascending=True)
 
     df['Privilege_Rank'] = ranks['Privilege_Avg_Score']
     df['Protect_Rank'] = ranks['Protect_Avg_Score']