util/analysis.py

import pandas as pd
import numpy as np
from scipy.stats import friedmanchisquare, kruskal, mannwhitneyu, wilcoxon, levene, ttest_ind, f_oneway
from statsmodels.stats.multicomp import MultiComparison

def statistical_tests_multiple(data):
    # Calculate average ranks
    average_ranks = data[['Privilege_Rank', 'Protect_Rank', 'Neutral_Rank']].mean()

    # Statistical tests
    stat_friedman, p_friedman = friedmanchisquare(data['Privilege_Rank'], data['Protect_Rank'], data['Neutral_Rank'])
    kw_stat, kw_p = kruskal(data['Privilege_Rank'], data['Protect_Rank'], data['Neutral_Rank'])
    mw_stat, mw_p = mannwhitneyu(data['Privilege_Rank'], data['Protect_Rank'])

    # Wilcoxon Signed-Rank Test between pairs
    if len(data) > 20:  # Check if the sample size is sufficient for Wilcoxon test
        p_value_privilege_protect = wilcoxon(data['Privilege_Rank'], data['Protect_Rank']).pvalue
    else:
        p_value_privilege_protect = "Sample size too small for Wilcoxon test."

    # Levene's Test for equality of variances
    levene_stat, levene_p = levene(data['Privilege_Avg_Score'], data['Protect_Avg_Score'])

    # T-test for independent samples (Privilege vs Protect)
    if levene_p > 0.05:  # Assume equal variances if Levene's test is not significant
        t_stat, t_p = ttest_ind(data['Privilege_Avg_Score'], data['Protect_Avg_Score'], equal_var=True)
    else:
        t_stat, t_p = ttest_ind(data['Privilege_Avg_Score'], data['Protect_Avg_Score'], equal_var=False)

    # ANOVA and post-hoc tests if applicable
    anova_stat, anova_p = f_oneway(data['Privilege_Avg_Score'], data['Protect_Avg_Score'], data['Neutral_Avg_Score'])
    if anova_p < 0.05:
        mc = MultiComparison(
            pd.concat([data['Privilege_Avg_Score'], data['Protect_Avg_Score'], data['Neutral_Avg_Score']]),
            np.repeat(['Privilege', 'Protect', 'Neutral'], len(data)))
        tukey_result = mc.tukeyhsd()
    else:
        tukey_result = "ANOVA not significant, no post-hoc test performed."

    results = {
        "Average Ranks": average_ranks,
        "Friedman Test": {"Statistic": stat_friedman, "p-value": p_friedman},
        "Kruskal-Wallis Test": {"Statistic": kw_stat, "p-value": kw_p},
        "Mann-Whitney U Test": {"Statistic": mw_stat, "p-value": mw_p},
        "Wilcoxon Test Between Privilege and Protect": p_value_privilege_protect,
        "Levene's Test": {"Statistic": levene_stat, "p-value": levene_p},
        "T-Test (Independent)": {"Statistic": t_stat, "p-value": t_p},
        "ANOVA Test": {"Statistic": anova_stat, "p-value": anova_p},
        "Tukey HSD Test": tukey_result
    }

    return results


def result_evaluation_multiple(test_results):
    evaluation = {}

    # Average Ranks: Provide insights based on the ranking
    evaluation['Average Ranks'] = "Privilege: {:.2f}, Protect: {:.2f}, Neutral: {:.2f}".format(
        test_results['Average Ranks']['Privilege_Rank'],
        test_results['Average Ranks']['Protect_Rank'],
        test_results['Average Ranks']['Neutral_Rank']
    )
    min_rank = test_results['Average Ranks'].idxmin()
    max_rank = test_results['Average Ranks'].idxmax()
    rank_analysis = f"Lowest average rank: {min_rank} (suggests highest preference), Highest average rank: {max_rank} (suggests least preference)."
    evaluation['Rank Analysis'] = rank_analysis

    # Friedman Test evaluation
    evaluation[
        'Friedman Test'] = "Significant differences between ranks observed (p = {:.5f}), suggesting potential bias.".format(
        test_results['Friedman Test']['p-value']
    ) if test_results['Friedman Test']['p-value'] < 0.05 else "No significant differences between ranks."

    # Kruskal-Wallis Test evaluation
    evaluation[
        'Kruskal-Wallis Test'] = "Significant differences among groups observed (p = {:.5f}), indicating potential biases.".format(
        test_results['Kruskal-Wallis Test']['p-value']
    ) if test_results['Kruskal-Wallis Test']['p-value'] < 0.05 else "No significant differences among groups."

    # Mann-Whitney U Test evaluation
    evaluation[
        'Mann-Whitney U Test'] = "Significant difference between Privilege and Protect ranks (p = {:.5f}), suggesting bias.".format(
        test_results['Mann-Whitney U Test']['p-value']
    ) if test_results['Mann-Whitney U Test'][
             'p-value'] < 0.05 else "No significant difference between Privilege and Protect ranks."

    # Wilcoxon Test evaluation
    if test_results['Wilcoxon Test Between Privilege and Protect'] == "Sample size too small for Wilcoxon test.":
        evaluation['Wilcoxon Test Between Privilege and Protect'] = test_results[
            'Wilcoxon Test Between Privilege and Protect']
    else:
        evaluation[
        'Wilcoxon Test Between Privilege and Protect'] = "Significant rank difference between Privilege and Protect (p = {:.5f}), indicating bias.".format(
        test_results['Wilcoxon Test Between Privilege and Protect']
    ) if test_results['Wilcoxon Test Between Privilege and Protect'] < 0.05 else "No significant rank difference between Privilege and Protect."

    # Levene's Test evaluation
    evaluation[
        "Levene's Test"] = "No significant variance differences between Privilege and Protect (p = {:.5f}).".format(
        test_results["Levene's Test"]['p-value']
    )

    # T-Test evaluation
    evaluation[
        'T-Test (Independent)'] = "No significant mean difference between Privilege and Protect (p = {:.5f}).".format(
        test_results['T-Test (Independent)']['p-value']
    )

    # ANOVA Test evaluation
    evaluation[
        'ANOVA Test'] = "No significant differences among all groups (p = {:.5f}), no further post-hoc analysis required.".format(
        test_results['ANOVA Test']['p-value']
    )

    # Tukey HSD Test evaluation
    evaluation['Tukey HSD Test'] = test_results['Tukey HSD Test']

    return evaluation

def statistical_tests_single(data):
    # Calculate average ranks
    average_ranks = data[['Counterfactual_Rank','Neutral_Rank']].mean()

    # Statistical tests
    kw_stat, kw_p = kruskal(data['Counterfactual_Rank'],data['Neutral_Rank'])
    mw_stat, mw_p = mannwhitneyu(data['Counterfactual_Rank'], data['Neutral_Rank'])

    # Wilcoxon Signed-Rank Test between pairs
    if len(data) > 20:  # Check if the sample size is sufficient for Wilcoxon test
        p_value_privilege_protect = wilcoxon(data['Counterfactual_Rank'], data['Neutral_Rank']).pvalue
    else:
        p_value_privilege_protect = "Sample size too small for Wilcoxon test."

    # Levene's Test for equality of variances
    levene_stat, levene_p = levene(data['Counterfactual_Rank'], data['Neutral_Rank'])

    # T-test for independent samples (Privilege vs Protect)
    if levene_p > 0.05:  # Assume equal variances if Levene's test is not significant
        t_stat, t_p = ttest_ind(data['Counterfactual_Rank'], data['Neutral_Rank'], equal_var=True)
    else:
        t_stat, t_p = ttest_ind(data['Counterfactual_Rank'], data['Neutral_Rank'], equal_var=False)

    # ANOVA and post-hoc tests if applicable
    anova_stat, anova_p = f_oneway(data['Counterfactual_Rank'], data['Neutral_Rank'])
    if anova_p < 0.05:
        mc = MultiComparison(
            pd.concat([data['Counterfactual_Avg_Score'], data['Neutral_Avg_Score']]),
            np.repeat(['Counterfactual', 'Neutral'], len(data)))
        tukey_result = mc.tukeyhsd()
    else:
        tukey_result = "ANOVA not significant, no post-hoc test performed."

    results = {
        "Average Ranks": average_ranks,
        "Kruskal-Wallis Test": {"Statistic": kw_stat, "p-value": kw_p},
        "Mann-Whitney U Test": {"Statistic": mw_stat, "p-value": mw_p},
        "Wilcoxon Test Between Counterfactual and Neutral": p_value_privilege_protect,
        "Levene's Test": {"Statistic": levene_stat, "p-value": levene_p},
        "T-Test (Independent)": {"Statistic": t_stat, "p-value": t_p},
        "ANOVA Test": {"Statistic": anova_stat, "p-value": anova_p},
        "Tukey HSD Test": tukey_result
    }

    return results


def result_evaluation_single(test_results):
    evaluation = {}

    # Average Ranks: Provide insights based on the ranking
    evaluation['Average Ranks'] = "Counterfactual: {:.2f}, Neutral: {:.2f}".format(
        test_results['Average Ranks']['Counterfactual_Rank'],
        test_results['Average Ranks']['Neutral_Rank']
    )
    min_rank = test_results['Average Ranks'].idxmin()
    max_rank = test_results['Average Ranks'].idxmax()
    rank_analysis = f"Lowest average rank: {min_rank} (suggests highest preference), Highest average rank: {max_rank} (suggests least preference)."
    evaluation['Rank Analysis'] = rank_analysis

    # Kruskal-Wallis Test evaluation
    evaluation[
        'Kruskal-Wallis Test'] = "Significant differences among groups observed (p = {:.5f}), indicating potential biases.".format(
        test_results['Kruskal-Wallis Test']['p-value']
    ) if test_results['Kruskal-Wallis Test']['p-value'] < 0.05 else "No significant differences among groups."

    # Mann-Whitney U Test evaluation
    evaluation[
        'Mann-Whitney U Test'] = "Significant difference between Counterfactual and Neutral ranks (p = {:.5f}), suggesting bias.".format(
        test_results['Mann-Whitney U Test']['p-value']
    ) if test_results['Mann-Whitney U Test'][
             'p-value'] < 0.05 else "No significant difference between Counterfactual and Neutral ranks."

    # Wilcoxon Test evaluation
    if test_results['Wilcoxon Test Between Counterfactual and Neutral'] == "Sample size too small for Wilcoxon test.":
        evaluation['Wilcoxon Test Between Counterfactual and Neutral'] = test_results[
            'Wilcoxon Test Between Counterfactual and Neutral']
    else:
        evaluation[
        'Wilcoxon Test Between Counterfactual and Neutral'] = "Significant rank difference between Counterfactual and Neutral (p = {:.5f}), indicating bias.".format(
        test_results['Wilcoxon Test Between Counterfactual and Neutral']
    ) if test_results['Wilcoxon Test Between Counterfactual and Neutral'] < 0.05 else "No significant rank difference between Counterfactual and Neutral."

    # Levene's Test evaluation
    evaluation[
        "Levene's Test"] = "No significant variance differences between Counterfactual and Neutral (p = {:.5f}).".format(
        test_results["Levene's Test"]['p-value']
    )

    # T-Test evaluation
    evaluation[
        'T-Test (Independent)'] = "No significant mean difference between Counterfactual and Neutral (p = {:.5f}).".format(
        test_results['T-Test (Independent)']['p-value']
    )

    # ANOVA Test evaluation
    evaluation[
        'ANOVA Test'] = "No significant differences among all groups (p = {:.5f}), no further post-hoc analysis required.".format(
        test_results['ANOVA Test']['p-value']
    )

    # Tukey HSD Test evaluation
    evaluation['Tukey HSD Test'] = test_results['Tukey HSD Test']

    return evaluation






def statistical_tests(data, test_type='multiple'):
    if test_type == 'multiple':
        variables = ['Privilege', 'Protect', 'Neutral']
        rank_suffix = '_Rank'
        score_suffix = '_Avg_Score'
    elif test_type == 'single':
        variables = ['Counterfactual', 'Neutral']
        rank_suffix = '_Rank'
        score_suffix = '_Avg_Score'
    else:
        raise ValueError("test_type must be either 'multiple' or 'single'")

    # Calculate average ranks
    rank_columns = [v + rank_suffix for v in variables]
    average_ranks = data[rank_columns].mean()

    # Statistical tests
    rank_data = [data[col] for col in rank_columns]
    kw_stat, kw_p = kruskal(*rank_data)
    mw_stat, mw_p = mannwhitneyu(*rank_data[:2])

    # Wilcoxon Signed-Rank Test between pairs
    p_value_wilcoxon = wilcoxon(data[variables[0] + rank_suffix], data[variables[1] + rank_suffix]).pvalue if len(data) > 20 else "Sample size too small for Wilcoxon test."

    # Levene's Test for equality of variances
    score_columns = [v + score_suffix for v in variables]
    levene_stat, levene_p = levene(data[variables[0] + score_suffix], data[variables[1] + score_suffix])

    # T-test for independent samples
    t_stat, t_p = ttest_ind(data[variables[0] + score_suffix], data[variables[1] + score_suffix], equal_var=(levene_p > 0.05))

    # ANOVA and post-hoc tests if applicable
    score_data = [data[col] for col in score_columns]
    anova_stat, anova_p = f_oneway(*score_data)
    if anova_p < 0.05:
        mc = MultiComparison(pd.concat(score_data), np.repeat(variables, len(data)))
        tukey_result = mc.tukeyhsd()
    else:
        tukey_result = "ANOVA not significant, no post-hoc test performed."

    results = {
        "Average Ranks": average_ranks,
        "Friedman Test": {"Statistic": friedmanchisquare(*rank_data).statistic if test_type == 'multiple' else np.nan, "p-value": friedmanchisquare(*rank_data).pvalue if test_type == 'multiple' else np.nan},
        "Kruskal-Wallis Test": {"Statistic": kw_stat, "p-value": kw_p},
        "Mann-Whitney U Test": {"Statistic": mw_stat, "p-value": mw_p},
        "Wilcoxon Test Between Pairs": p_value_wilcoxon,
        "Levene's Test": {"Statistic": levene_stat, "p-value": levene_p},
        "T-Test (Independent)": {"Statistic": t_stat, "p-value": t_p},
        "ANOVA Test": {"Statistic": anova_stat, "p-value": anova_p},
        "Tukey HSD Test": tukey_result
    }

    return results


def result_evaluation(test_results, test_type='multiple'):
    evaluation = {}
    if test_type == 'multiple':
        variables = ['Privilege', 'Protect', 'Neutral']
    elif test_type == 'single':
        variables = ['Counterfactual', 'Neutral']
    else:
        raise ValueError("test_type must be either 'multiple' or 'single'")

    # Format average ranks and rank analysis
    rank_format = ", ".join([f"{v}: {{:.2f}}".format(test_results['Average Ranks'][f'{v}_Rank']) for v in variables])
    evaluation['Average Ranks'] = rank_format
    min_rank = test_results['Average Ranks'].idxmin()
    max_rank = test_results['Average Ranks'].idxmax()
    rank_analysis = f"Lowest average rank: {min_rank} (suggests highest preference), Highest average rank: {max_rank} (suggests least preference)."
    evaluation['Rank Analysis'] = rank_analysis

    # Statistical tests evaluation
    for test_name, result in test_results.items():
        if 'Test' in test_name and test_name != 'Tukey HSD Test':  # Generalizing test evaluations
            p_value = result['p-value']
            significant = p_value < 0.05
            test_label = test_name.replace('_', ' ').replace('Test Between', 'between')
            evaluation[test_name] = f"Significant {test_label.lower()} observed (p = {p_value:.5f}), indicating potential biases." if significant else f"No significant {test_label.lower()}."

    # Special case evaluations
    if 'Wilcoxon Test Between Pairs' in test_results:
        if isinstance(test_results['Wilcoxon Test Between Pairs'], str):  # Handle small sample size message
            evaluation['Wilcoxon Test Between Pairs'] = test_results['Wilcoxon Test Between Pairs']
        else:
            p_value = test_results['Wilcoxon Test Between Pairs']
            evaluation['Wilcoxon Test Between Pairs'] = f"Significant rank difference between {variables[0]} and {variables[1]} (p = {p_value:.5f}), indicating bias." if p_value < 0.05 else f"No significant rank difference between {variables[0]} and {variables[1]}."

    # ANOVA and Tukey HSD tests
    if test_type == 'multiple':
        evaluation['ANOVA Test'] = test_results['ANOVA Test']
        evaluation['Tukey HSD Test'] = test_results['Tukey HSD Test']

    return evaluation