analysis_util.py

import functools
import operator

import pandas as pd


def correlations_for_group(group):
    REL_METRICS = [col.split("_")[0] for col in group.columns if col.endswith("_related")]
    IND_METRICS = [col.split("_")[0] for col in group.columns if col.endswith("_independent")]
    AGGR_METRICS = [col.split("_")[0] for col in group.columns if col.endswith("_aggr")]

    correlations = []
    for rel_metric in REL_METRICS:
        for ind_metric in IND_METRICS:
            correlations.append({
                f"rel_{rel_metric}_ind_{ind_metric}_pearson": group[f"{rel_metric}_related"].corr(
                    group[f"{ind_metric}_independent"], method="pearson"),
                f"rel_{rel_metric}_ind_{ind_metric}_spearman": group[f"{rel_metric}_related"].corr(
                    group[f"{ind_metric}_independent"], method="spearman"),
            })
        for aggr_metric in AGGR_METRICS:
            correlations.append({
                f"rel_{rel_metric}_aggr_{aggr_metric}_pearson": group[f"{rel_metric}_related"].corr(
                    group[f"{aggr_metric}_aggr"], method="pearson"),
                f"rel_{rel_metric}_aggr_{aggr_metric}_spearman": group[f"{rel_metric}_related"].corr(
                    group[f"{aggr_metric}_aggr"], method="spearman"),
            })
    return pd.Series(functools.reduce(operator.ior, correlations, {}))


def split_metrics_string(s):
    tokens = s.split("_")
    return tokens[1], tokens[3]


def get_correlations_df(df, right_side):
    correlations_raw = correlations_for_group(df)

    idx = list(set("_".join(col.split("_")[:-1]) for col in correlations_raw.index if right_side in col))

    data = []
    for metrics in idx:
        data.append(
            {"metrics": metrics,
             "spearman": correlations_raw[f"{metrics}_spearman"],
             "pearson": correlations_raw[f"{metrics}_pearson"],
             }
        )

    result = pd.DataFrame.from_records(data=data, index="metrics").sort_index()
    result.index = pd.MultiIndex.from_tuples(result.index.map(split_metrics_string).tolist())
    result.index.set_names(["relative", "independent"], inplace=True)

    return result


def get_correlations_for_groups(df, right_side):
    correlations = {"all": get_correlations_df(df, right_side=right_side)}

    for e2s in (False, True):
        for s2e in (False, True):
            group = "golden"
            if e2s:
                group += "+e2s"
            if s2e:
                group += "+s2e"

            subdf = df[((df["end_to_start"] == e2s) & (df["start_to_end"] == s2e)) | (
                    (df["end_to_start"] == False) & (df["start_to_end"] == False))]
            subdf_corr = get_correlations_df(subdf, right_side=right_side)
            correlations[group] = subdf_corr

    correlations = pd.concat(correlations, axis=1)
    return correlations