jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def mappings_frequency(df, filepath=None):
    """
    Plots the frequency of logical conjunction mappings

    Parameters
    ----------
    df: `pandas.DataFrame`_
        DataFrame with columns `frequency` and `mapping`

    filepath: str
        Absolute path to a folder where to write the plot


    Returns
    -------
    plot
        Generated plot


    .. _pandas.DataFrame: http://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe
    """

    df = df.sort_values('frequency')
    df['conf'] = df.frequency.map(lambda f: 0 if f < 0.2 else 1 if f < 0.8 else 2)

    g = sns.catplot(x="mapping", y="frequency", data=df, aspect=3, hue='conf',
            legend=False, kind="point")
    for tick in g.ax.get_xticklabels():
        tick.set_rotation(90)

    g.ax.set_ylim([-.05, 1.05])

    g.ax.set_xlabel("Logical mapping")
    g.ax.set_ylabel("Frequency")

    if filepath:
        g.savefig(os.path.join(filepath, 'mappings-frequency.pdf'))

    return g
//  [python]
def _comparison_year_month_plotter(figure: Figure, data: PDDataFrame, **kwargs):
    g = sns.FacetGrid(data, col="year", hue="history")
    g.map(sns.barplot, "month", "count")
    g.add_legend()
    return g
//  [python]
def _comparison_year_month_plotter(figure: Figure, data: PDDataFrame, **kwargs):
    return sns.catplot(
        x="month", y="count", hue="history", col="year", data=data, kind="bar"
    )
//  [python]
def predictions_variance(df, filepath=None):
    """
    Plots the mean variance prediction for each readout

    Parameters
    ----------
    df: `pandas.DataFrame`_
        DataFrame with columns starting with `VAR:`

    filepath: str
        Absolute path to a folder where to write the plots


    Returns
    -------
    plot
        Generated plot


    .. _pandas.DataFrame: http://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe
    """

    df = df.filter(regex="^VAR:")

    by_readout = df.mean(axis=0).reset_index(level=0)
    by_readout.columns = ['Readout', 'Prediction variance (mean)']

    by_readout['Readout'] = by_readout.Readout.map(lambda n: n[4:])

    g1 = sns.barplot(x='Readout', y='Prediction variance (mean)', data=by_readout)

    for tick in g1.get_xticklabels():
        tick.set_rotation(90)

    if filepath:
        plt.savefig(os.path.join(filepath, 'predictions-variance.pdf'))

    return g1
//  [python]
def predictions_variance(df, filepath=None):
    """
    Plots the mean variance prediction for each readout

    Parameters
    ----------
    df: `pandas.DataFrame`_
        DataFrame with columns starting with `VAR:`

    filepath: str
        Absolute path to a folder where to write the plots


    Returns
    -------
    plot
        Generated plot


    .. _pandas.DataFrame: http://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe
    """

    df = df.filter(regex="^VAR:")

    by_readout = df.mean(axis=0).reset_index(level=0)
    by_readout.columns = ['Readout', 'Prediction variance (mean)']

    by_readout['Readout'] = by_readout.Readout.map(lambda n: n[4:])

    g1 = sns.catplot(x='Readout', y='Prediction variance (mean)', data=by_readout, kind='bar', aspect=2)

    for tick in g1.ax.get_xticklabels():
        tick.set_rotation(90)

    if filepath:
        g1.savefig(os.path.join(filepath, 'predictions-variance.pdf'))

    return g1
//  [python]
sns.set(style="ticks", color_codes=True)
df = pd.read_csv("cav_time.csv")

sns.catplot(kind='bar', x='Method', y='Time', col='Step', data=df, palette='Reds', sharey=False, ci=None, col_wrap=2)
//  [python]
def interventions_frequency(df, filepath=None):
    """
    Plots the frequency of occurrence for each intervention

    Parameters
    ----------
    df: `pandas.DataFrame`_
        DataFrame with columns `frequency` and `intervention`

    filepath: str
        Absolute path to a folder where to write the plot


    Returns
    -------
    plot
        Generated plot


    .. _pandas.DataFrame: http://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe
    """

    df = df.sort_values('frequency')
    df['conf'] = df.frequency.map(lambda f: 0 if f < 0.2 else 1 if f < 0.8 else 2)

    g = sns.factorplot(x="intervention", y="frequency", data=df, aspect=3, hue='conf', legend=False, kind="point")
    for tick in g.ax.get_xticklabels():
        tick.set_rotation(90)

    _ = [t.set_color('r') if t.get_text().endswith('-1') else t.set_color('g') for t in g.ax.xaxis.get_ticklabels()]

    g.ax.set_ylim([-.05, 1.05])

    g.ax.set_xlabel("Intervention")
    g.ax.set_ylabel("Frequency")

    if filepath:
        g.savefig(os.path.join(filepath, 'interventions-frequency.pdf'))

    return g
//  [python]
def interventions_frequency(df, filepath=None):
    """
    Plots the frequency of occurrence for each intervention

    Parameters
    ----------
    df: `pandas.DataFrame`_
        DataFrame with columns `frequency` and `intervention`

    filepath: str
        Absolute path to a folder where to write the plot


    Returns
    -------
    plot
        Generated plot


    .. _pandas.DataFrame: http://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe
    """

    df = df.sort_values('frequency')
    df['conf'] = df.frequency.map(lambda f: 0 if f < 0.2 else 1 if f < 0.8 else 2)

    g = sns.catplot(x="intervention", y="frequency", data=df, aspect=3, hue='conf', legend=False, kind="point")
    for tick in g.ax.get_xticklabels():
        tick.set_rotation(90)

    _ = [t.set_color('r') if t.get_text().endswith('-1') else t.set_color('g') for t in g.ax.xaxis.get_ticklabels()]

    g.ax.set_ylim([-.05, 1.05])

    g.ax.set_xlabel("Intervention")
    g.ax.set_ylabel("Frequency")

    if filepath:
        g.savefig(os.path.join(filepath, 'interventions-frequency.pdf'))

    return g
//  [python]
def behaviors_distribution(df, filepath=None):
    """
    Plots the distribution of logical networks across input-output behaviors.
    Optionally, input-output behaviors can be grouped by MSE.

    Parameters
    ----------
    df: `pandas.DataFrame`_
        DataFrame with columns `networks` and optionally `mse`

    filepath: str
        Absolute path to a folder where to write the plot

    Returns
    -------
    plot
        Generated plot


    .. _pandas.DataFrame: http://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe
    """

    cols = ["networks", "index"]
    rcols = ["Logical networks", "Input-Output behaviors"]
    sort_cols = ["networks"]

    if "mse" in df.columns:
        cols.append("mse")
        rcols.append("MSE")
        sort_cols = ["mse"] + sort_cols

        df.mse = df.mse.map(lambda f: "%.4f" % f)

    df = df.sort_values(sort_cols).reset_index(drop=True).reset_index(level=0)[cols]
    df.columns = rcols

    if "MSE" in df.columns:
        g = sns.factorplot(x='Input-Output behaviors', y='Logical networks', hue='MSE', data=df, aspect=3, kind='bar', legend_out=False)
    else:
        g = sns.factorplot(x='Input-Output behaviors', y='Logical networks', data=df, aspect=3, kind='bar', legend_out=False)

    g.ax.set_xticks([])
    if filepath:
        g.savefig(os.path.join(filepath, 'behaviors-distribution.pdf'))

    return g
//  [python]
def behaviors_distribution(df, filepath=None):
    """
    Plots the distribution of logical networks across input-output behaviors.
    Optionally, input-output behaviors can be grouped by MSE.

    Parameters
    ----------
    df: `pandas.DataFrame`_
        DataFrame with columns `networks` and optionally `mse`

    filepath: str
        Absolute path to a folder where to write the plot

    Returns
    -------
    plot
        Generated plot


    .. _pandas.DataFrame: http://pandas.pydata.org/pandas-docs/stable/dsintro.html#dataframe
    """

    cols = ["networks", "index"]
    rcols = ["Logical networks", "Input-Output behaviors"]
    sort_cols = ["networks"]

    if "mse" in df.columns:
        cols.append("mse")
        rcols.append("MSE")
        sort_cols = ["mse"] + sort_cols

        df.mse = df.mse.map(lambda f: "%.4f" % f)

    df = df.sort_values(sort_cols).reset_index(drop=True).reset_index(level=0)[cols]
    df.columns = rcols

    if "MSE" in df.columns:
        g = sns.catplot(x='Input-Output behaviors', y='Logical networks', hue='MSE', data=df, aspect=3, kind='bar', legend_out=False)
    else:
        g = sns.catplot(x='Input-Output behaviors', y='Logical networks', data=df, aspect=3, kind='bar', legend_out=False)

    g.ax.set_xticks([])
    if filepath:
        g.savefig(os.path.join(filepath, 'behaviors-distribution.pdf'))

    return g
//  [python]
sns.set(style="darkgrid")

# 3.1 示例
sns.boxplot(x="day", y="total_bill", data=tips)
//  [python]
sns.set(style="darkgrid")

# 3.1 示例
sns.catplot(x="day", y="total_bill", kind="box", data=tips)
//  [python]
def mock_visualize_qc_metrics(
    metric_name,
    qc_metric_dir,
    return_plot=True,
    axes_size=16,
    wrap=6,
    dpi=None,
    save_dir=None,
    ax=None,
):
    if save_dir:
        _make_small_file(save_dir, "%s_barplot_stats.png" % metric_name)

    if return_plot:
        qc_metric_df = pd.DataFrame(np.zeros((5, 3)), columns=["fov", metric_name, "channel"])
        qc_metric_df["fov"] = "fov-1-scan-1"
        qc_metric_df["channel"] = [f"chan{i}" for i in np.arange(5)]
        qc_fg = sns.factorplot(
            x="fov",
            y=metric_name,
            col="channel",
            col_wrap=1,
            data=qc_metric_df,
            kind="bar",
            color="black",
            sharex=True,
            sharey=False,
        )
        return qc_fg
//  [python]
def mock_visualize_qc_metrics(
    metric_name,
    qc_metric_dir,
    return_plot=True,
    axes_size=16,
    wrap=6,
    dpi=None,
    save_dir=None,
    ax=None,
):
    if save_dir:
        _make_small_file(save_dir, "%s_barplot_stats.png" % metric_name)

    if return_plot:
        qc_metric_df = pd.DataFrame(np.zeros((5, 3)), columns=["fov", metric_name, "channel"])
        qc_metric_df["fov"] = "fov-1-scan-1"
        qc_metric_df["channel"] = [f"chan{i}" for i in np.arange(5)]
        qc_fg = sns.catplot(
            x="fov",
            y=metric_name,
            col="channel",
            col_wrap=1,
            data=qc_metric_df,
            kind="bar",
            color="black",
            sharex=True,
            sharey=False,
        )
        return qc_fg
//  [python]
def display_cat_violin(self, metrics, channel_names, save_name=None):
        """Display predict metric violin by channel.
        Args:
            metrics (list[list[np.ndarray]]): metrics.
            channel_names (list[str]): list of channel names.
            save_name ([type], optional): [description]. Defaults to None.
        """
        metrics_list_df = [pd.DataFrame(m_i, columns=channel_names) for m_i in metrics]
        # metrics_df = pd.concat(
        #     metrics_list_df, keys=["train", "val", "test"], copy=False
        #     ).reset_index().rename(columns={"level_0": "set", "level_1": "num"})
        metrics_df = pd.concat(
            metrics_list_df, keys=channel_names, copy=False
        ).reset_index().rename(columns={"level_0": "set", "level_1": "num"})

        sns.violinplot(x="source", y="value", hue="channel",
                    scale="count", cut=0,
                    data=pd.melt(
                        metrics_df, id_vars=["set", "num"], value_vars=channel_names,
                        var_name=["channel"], value_name="value"))

        if self._path_save:
            plt.savefig(os.path.join(self._path_save, f'{save_name}_cat_violin.svg'))
            plt.close()
//  [python]
def display_cat_violin(self, metrics, channel_names, save_name=None):
        """Display predict metric violin by channel.
        Args:
            metrics (list[list[np.ndarray]]): metrics.
            channel_names (list[str]): list of channel names.
            save_name ([type], optional): [description]. Defaults to None.
        """
        metrics_list_df = [pd.DataFrame(m_i, columns=channel_names) for m_i in metrics]
        # metrics_df = pd.concat(
        #     metrics_list_df, keys=["train", "val", "test"], copy=False
        #     ).reset_index().rename(columns={"level_0": "set", "level_1": "num"})
        metrics_df = pd.concat(
            metrics_list_df, keys=channel_names, copy=False
        ).reset_index().rename(columns={"level_0": "set", "level_1": "num"})

        sns.catplot(x="source", y="value", hue="channel",
                    kind="violin", scale="count", cut=0,
                    data=pd.melt(
                        metrics_df, id_vars=["set", "num"], value_vars=channel_names,
                        var_name=["channel"], value_name="value"))

        if self._path_save:
            plt.savefig(os.path.join(self._path_save, f'{save_name}_cat_violin.svg'))
            plt.close()
//  [python]
def visualize_similar_with_names(data, orig_name, comparison_names, target_names, X_pca, cols_to_plot):
    orig_idx = (target_names == orig_name).idxmax()
    comp_idx = target_names[target_names.isin(comparison_names)].index.tolist()
    all_names = comparison_names.tolist()
    all_names.append(orig_name)
    if cols_to_plot is None:
        cols_to_plot = ['nimi'] + data_transforms.NOMINAL_VARS + ['dist']
    d = similarity.pairwise_distances(X_pca, X_pca, 'euclidean')
    df = data.copy()
    df['dist'] = d[orig_idx, :]
    df.sort_values(by='dist', ascending=True, inplace=True)
    df = df.loc[df['nimi'].isin(all_names), cols_to_plot]
    melted = df.melt(id_vars=['nimi'])

    g = sns.FacetGrid(melted, col="variable", col_wrap=5, sharex=False, sharey=True, s=9)
    g.map(sns.barplot, "value", "nimi")
    g.set_titles(col_template="{col_name}")

    # Use semantically meaningful titles for the columns
    vartitles = pd.read_csv('paavo_vars.csv', sep=";")
    titles_dict = dict(zip(vartitles['koodi'], vartitles['nimi']))
    titles_dict['dist'] = 'PCA distance'
    titles = [titles_dict.get(x) for x in melted['variable'].unique()]
    for ax, title in zip(g.axes.flat, titles):
        # Set a different title for each axes
        ax.set(title=title)
    #
    #    # Make the grid horizontal instead of vertical
    #    ax.xaxis.grid(False)
    #    ax.yaxis.grid(True)

    # sns.despine(left=True, bottom=True)
    plt.show()

    return df
//  [python]
def visualize_similar_with_names(data, orig_name, comparison_names, target_names, X_pca, cols_to_plot):
    orig_idx = (target_names == orig_name).idxmax()
    comp_idx = target_names[target_names.isin(comparison_names)].index.tolist()
    all_names = comparison_names.tolist()
    all_names.append(orig_name)
    if cols_to_plot is None:
        cols_to_plot = ['nimi'] + data_transforms.NOMINAL_VARS + ['dist']
    d = similarity.pairwise_distances(X_pca, X_pca, 'euclidean')
    df = data.copy()
    df['dist'] = d[orig_idx, :]
    df.sort_values(by='dist', ascending=True, inplace=True)
    df = df.loc[df['nimi'].isin(all_names), cols_to_plot]
    melted = df.melt(id_vars=['nimi'])

    g = sns.catplot(x="value", y="nimi", col="variable", col_wrap=5, data=melted, margin_titles=True, sharex=False,
                    sharey=True, s=9)

    # Use semantically meaningful titles for the columns
    vartitles = pd.read_csv('paavo_vars.csv', sep=";")
    titles_dict = dict(zip(vartitles['koodi'], vartitles['nimi']))
    titles_dict['dist'] = 'PCA distance'
    titles = [titles_dict.get(x) for x in melted['variable'].unique()]
    for ax, title in zip(g.axes.flat, titles):
        # Set a different title for each axes
        ax.set(title=title)
    #
    #    # Make the grid horizontal instead of vertical
    #    ax.xaxis.grid(False)
    #    ax.yaxis.grid(True)

    # sns.despine(left=True, bottom=True)
    plt.show()

    return df
//  [python]
def plot_performance_by_type(performance: pd.DataFrame, null: str, title: str):

    # guide order
    performance.rename(columns={'guide_type': 'Guide type'}, inplace=True)
    order = ['SM', 'DM', 'RDM', 'TM', 'RTM', 'SI', 'CI', 'DI', 'SD', 'CD', 'DD']
    [order.remove(gt) for gt in set(order) - set(performance['Guide type'].unique())]

    # drop index levels
    assert set(performance.index.names) == {'Model'}
    levels = performance.index.names + ['Guide type']
    performance.reset_index(inplace=True)

    # plot figure
    metrics = performance[METRICS + levels].melt(id_vars=levels, var_name='Metric')
    performance.set_index(['Model', 'Guide type'], inplace=True)
    g = sns.factorplot(data=metrics, kind='bar', x='Guide type', y='value', order=order, hue='Model', col='Metric')
    g.fig.suptitle(title)
    sns.move_legend(g, 'upper right', title='Model')
    for metric, ax in g.axes_dict.items():
        for j, guide_type in enumerate(order):
            for i, model in enumerate(performance.index.get_level_values('Model').unique()):

                # error bars
                x = ax.containers[i][j].get_x() + ax.containers[i][j].get_width() / 2
                y = performance.loc[(model, guide_type), metric]
                error = performance.loc[(model, guide_type), metric + ' err']
                ax.errorbar(x=x, y=y, yerr=2 * error, color='black')

                # statistical significance
                if model == null:
                    ax.text(x, y + 2 * error + 0.01, '$H_0$', horizontalalignment='center')
                elif performance.loc[(model, guide_type), metric + ' log10(p)'] < np.log10(0.05):
                    marker = '-' if performance.loc[(null, guide_type), metric] > y else '+'
                    ax.text(x, y + 2 * error + 0.01, marker, horizontalalignment='center')

    return g.fig
//  [python]
def plot_performance_by_type(performance: pd.DataFrame, null: str, title: str):

    # guide order
    performance.rename(columns={'guide_type': 'Guide type'}, inplace=True)
    order = ['SM', 'DM', 'RDM', 'TM', 'RTM', 'SI', 'CI', 'DI', 'SD', 'CD', 'DD']
    [order.remove(gt) for gt in set(order) - set(performance['Guide type'].unique())]

    # drop index levels
    assert set(performance.index.names) == {'Model'}
    levels = performance.index.names + ['Guide type']
    performance.reset_index(inplace=True)

    # plot figure
    metrics = performance[METRICS + levels].melt(id_vars=levels, var_name='Metric')
    performance.set_index(['Model', 'Guide type'], inplace=True)
    g = sns.catplot(data=metrics, kind='bar', x='Guide type', y='value', order=order, hue='Model', col='Metric')
    g.fig.suptitle(title)
    sns.move_legend(g, 'upper right', title='Model')
    for metric, ax in g.axes_dict.items():
        for j, guide_type in enumerate(order):
            for i, model in enumerate(performance.index.get_level_values('Model').unique()):

                # error bars
                x = ax.containers[i][j].get_x() + ax.containers[i][j].get_width() / 2
                y = performance.loc[(model, guide_type), metric]
                error = performance.loc[(model, guide_type), metric + ' err']
                ax.errorbar(x=x, y=y, yerr=2 * error, color='black')

                # statistical significance
                if model == null:
                    ax.text(x, y + 2 * error + 0.01, '$H_0$', horizontalalignment='center')
                elif performance.loc[(model, guide_type), metric + ' log10(p)'] < np.log10(0.05):
                    marker = '-' if performance.loc[(null, guide_type), metric] > y else '+'
                    ax.text(x, y + 2 * error + 0.01, marker, horizontalalignment='center')

    return g.fig
//  [python]
def plot_number_per_score(data:{int: int}, countable:str, all_indexes:bool=True):
    data=transform_data_dict({k: len(v) for k, v in data.items()}, 'score', countable, all_indexes)
    g = sns.barplot(x="score", y=countable, data=data, palette="muted")
    set_plot_labels(g, r'\# ' + countable, f"Cover (number of edges covered by the {countable})", f'Number of {countable} in {PRETTY_CASE_NAME}', 10 if all_indexes else 2)

    # save
    indexes = 'allindex_' if all_indexes else ''
    fig = f'out/number_{countable.replace(" ", "_").lower()}_per_score_{indexes}{CASE_NAME}.png'
    g.figure.savefig(fig, dpi=300)
    print(f'Generated figure {fig}')
//  [python]
def plot_number_per_score(data:{int: int}, countable:str, all_indexes:bool=True):
    data=transform_data_dict({k: len(v) for k, v in data.items()}, 'score', countable, all_indexes)
    g = sns.catplot(x="score", y=countable, data=data,
                    kind="bar", palette="muted")
    set_plot_labels(g, r'\# ' + countable, f"Cover (number of edges covered by the {countable})", f'Number of {countable} in {PRETTY_CASE_NAME}', 10 if all_indexes else 2)

    # save
    indexes = 'allindex_' if all_indexes else ''
    fig = f'out/number_{countable.replace(" ", "_").lower()}_per_score_{indexes}{CASE_NAME}.png'
    g.savefig(fig, dpi=300)
    print(f'Generated figure {fig}')
//  [python]
def catplot(**kwargs):
    """Call to seaborn catplot"""

    def plot(data):
        return sns.factorplot(data=data, **kwargs)

    return plot
//  [python]
def catplot(**kwargs):
    """Call to seaborn catplot"""

    def plot(data):
        return sns.catplot(data=data, **kwargs)

    return plot
//  [python]
def plot_numbers_per_score(data1:{int: int}, countable1:str, data2:{int: int}, countable2:str, all_indexes:bool=False):
    data1 = {k: len(v) for k, v in data1.items()}
    data2 = {k: len(v) for k, v in data2.items()}
    df = pd.DataFrame({
        'score': tuple(data1) + tuple(data2),
        'type': tuple(countable1 for _ in data1) + tuple(countable2 for _ in data2),
        'number': tuple(v for v in data1.values()) + tuple(v for v in data2.values())
    })
    g = sns.factorplot(x="score", y="number", hue="type", kind="bar", data=df)
    set_plot_labels(g, rf'\# {countable1} ~~~ \# {countable2}', f"Cover (number of edges covered by the concepts)", f'Number of {countable1} and {countable2} in {PRETTY_CASE_NAME}', 10 if all_indexes else 2)

    # save
    indexes = 'allindex_' if all_indexes else ''
    countable = '_'.join(c for c in (countable1, countable2))
    fig = f'out/number_{countable.replace(" ", "_").lower()}_per_score_{indexes}{CASE_NAME}.png'
    g.savefig(fig, dpi=300)
    print(f'Generated figure {fig}')
//  [python]
def plot_numbers_per_score(data1:{int: int}, countable1:str, data2:{int: int}, countable2:str, all_indexes:bool=False):
    data1 = {k: len(v) for k, v in data1.items()}
    data2 = {k: len(v) for k, v in data2.items()}
    df = pd.DataFrame({
        'score': tuple(data1) + tuple(data2),
        'type': tuple(countable1 for _ in data1) + tuple(countable2 for _ in data2),
        'number': tuple(v for v in data1.values()) + tuple(v for v in data2.values())
    })
    g = sns.catplot(x="score", y="number", hue="type", kind="bar", data=df)
    set_plot_labels(g, rf'\# {countable1} ~~~ \# {countable2}', f"Cover (number of edges covered by the concepts)", f'Number of {countable1} and {countable2} in {PRETTY_CASE_NAME}', 10 if all_indexes else 2)

    # save
    indexes = 'allindex_' if all_indexes else ''
    countable = '_'.join(c for c in (countable1, countable2))
    fig = f'out/number_{countable.replace(" ", "_").lower()}_per_score_{indexes}{CASE_NAME}.png'
    g.savefig(fig, dpi=300)
    print(f'Generated figure {fig}')
//  [python]
def make_2x2_vsrand_decflip_violinplot(tuples_of_title_x_y_noneflipped, filename):
    x_label = "Difference in (average) zeroed attention\nweight magnitude"
    y_label = "Difference in decision flip indicators\ncompared to original output"
    title_of_each_graph = "Dataset"
    list_of_row_dicts = []
    for tup in tuples_of_title_x_y_noneflipped:
        dataset_title = tup[0]
        assert len(tup[1]) == len(tup[2])
        assert len(tup[1]) == tup[3].shape[0]
        for i in range(len(tup[2])):
            list_of_row_dicts.append({title_of_each_graph: dataset_title, x_label: tup[1][i], y_label: str(tup[2][i]),
                                      "No Flips Occurred": ("True" if tup[3][i] else "False")})
    list_of_row_dicts = sorted(list_of_row_dicts, key=(lambda x: 1 if x["No Flips Occurred"] == "True" else 0))
    data_to_plot = pd.DataFrame(list_of_row_dicts)
    g = sns.violinplot(x=x_label, y=y_label, hue="No Flips Occurred", palette=None,
                       data=data_to_plot, split=True, inner=None, orient="h")
    plt.savefig(filename, bbox_inches='tight')
//  [python]
def make_2x2_vsrand_decflip_violinplot(tuples_of_title_x_y_noneflipped, filename):
    x_label = "Difference in (average) zeroed attention\nweight magnitude"
    y_label = "Difference in decision flip indicators\ncompared to original output"
    title_of_each_graph = "Dataset"
    list_of_row_dicts = []
    for tup in tuples_of_title_x_y_noneflipped:
        dataset_title = tup[0]
        assert len(tup[1]) == len(tup[2])
        assert len(tup[1]) == tup[3].shape[0]
        for i in range(len(tup[2])):
            list_of_row_dicts.append({title_of_each_graph: dataset_title, x_label: tup[1][i], y_label: str(tup[2][i]),
                                      "No Flips Occurred": ("True" if tup[3][i] else "False")})
    list_of_row_dicts = sorted(list_of_row_dicts, key=(lambda x: 1 if x["No Flips Occurred"] == "True" else 0))
    data_to_plot = pd.DataFrame(list_of_row_dicts)
    g = sns.catplot(kind='violin', x=x_label, y=y_label, col=title_of_each_graph, hue="No Flips Occurred", palette=None,
                       data=data_to_plot, col_wrap=2, legend=True, split=True, inner=None, orient="h")
    plt.savefig(filename, bbox_inches='tight')
//  [python]
def boxplot_gamma(df,fname,studyset_split=True):
	assert( isinstance(df, pd.DataFrame) )
	try:
		del df['Files']
	except:
		print("Deleting of 'Files' column failed: either not present or changed name!")
	try:
		del df['files']
	except:
		print("Deleting of 'Files' column failed: either not present or changed name!")
	df = pd.melt(df,id_vars=["Studyset"])# de rest: ,value_vars=[''])
	sns.set(font_scale=1.3)
	f = sns.factorplot(y="value", x="Studyset", col="variable", data=df, kind="box", sharey=False,col_wrap=3)
	f.set_axis_labels("", "Difference")
	if studyset_split==False:
		del df['Studyset']
		# f, ax = plot.subplots(nrows=1, ncols=1, sharex=False, sharey=False)
		# ax = sns.boxplot(y="value", x="variable", data=df)
		f = sns.factorplot(y="value", x=None, col="variable", data=df, kind="box", sharey=False,col_wrap=3)
		f.set_axis_labels("", "Difference")
		f.savefig(fname+'.pdf', bbox_inches='tight')
		f.savefig(fname+'.png', bbox_inches='tight',dpi=300)
	else:
		f.savefig(fname+'.split.pdf', bbox_inches='tight')
		f.savefig(fname+'.split.png', bbox_inches='tight',dpi=300)
//  [python]
def boxplot_gamma(df,fname,studyset_split=True):
	assert( isinstance(df, pd.DataFrame) )
	try:
		del df['Files']
	except:
		print("Deleting of 'Files' column failed: either not present or changed name!")
	try:
		del df['files']
	except:
		print("Deleting of 'Files' column failed: either not present or changed name!")
	df = pd.melt(df,id_vars=["Studyset"])# de rest: ,value_vars=[''])
	sns.set(font_scale=1.3)
	f = sns.catplot(y="value", x="Studyset", col="variable", data=df, kind="box", sharey=False,col_wrap=3)
	f.set_axis_labels("", "Difference")
	if studyset_split==False:
		del df['Studyset']
		# f, ax = plot.subplots(nrows=1, ncols=1, sharex=False, sharey=False)
		# ax = sns.boxplot(y="value", x="variable", data=df)
		f = sns.catplot(y="value", x=None, col="variable", data=df, kind="box", sharey=False,col_wrap=3)
		f.set_axis_labels("", "Difference")
		f.savefig(fname+'.pdf', bbox_inches='tight')
		f.savefig(fname+'.png', bbox_inches='tight',dpi=300)
	else:
		f.savefig(fname+'.split.pdf', bbox_inches='tight')
		f.savefig(fname+'.split.png', bbox_inches='tight',dpi=300)
//  [python]
def robust_associations_barplot(self):
        #
        hue_order = ["Both", "Drug", "CRISPR"]
        pal = {
            "Both": "#fc8d62",
            "Drug": "#ababab",
            "CRISPR": "#656565",
            "Total": "#E1E1E1",
        }

        sns.factorplot(
            "dtype",
            "count",
            "ftype",
            data=self.assoc_count,
            palette=pal,
            kind="bar",
            linewidth=0,
            hue_order=hue_order,
            facet_kws={"despine": False}
        )

        plt.grid(True, ls="-", lw=0.1, alpha=1.0, zorder=0, axis="y")

        plt.xlabel("")
        plt.ylabel("Drug-gene associations")
//  [python]
def robust_associations_barplot(self):
        #
        hue_order = ["Both", "Drug", "CRISPR"]
        pal = {
            "Both": "#fc8d62",
            "Drug": "#ababab",
            "CRISPR": "#656565",
            "Total": "#E1E1E1",
        }

        sns.catplot(
            "dtype",
            "count",
            "ftype",
            data=self.assoc_count,
            palette=pal,
            kind="bar",
            linewidth=0,
            hue_order=hue_order,
            facet_kws={"despine": False}
        )

        plt.grid(True, ls="-", lw=0.1, alpha=1.0, zorder=0, axis="y")

        plt.xlabel("")
        plt.ylabel("Drug-gene associations")
//  [python]
sns.set()

tips = sns.load_dataset("tips")
sns.factorplot(x="day", y="total_bill", data=tips)
//  [python]
sns.set()

tips = sns.load_dataset("tips")
sns.catplot(x="day", y="total_bill", data=tips)
//  [python]
def _plot_primaries(self, metric='seg-foreground', ax=None):

        df = self.df[self.df['primaries']]
        df = df.drop(columns=['species', 'primaries', 'secondaries'])
        dfa = df.groupby('antibody').agg([np.mean, np.std])
        dfb = dfa.xs(metric, axis=1).sort_values('mean', ascending=False)

        if ax is not None:
            dfb.plot(kind="bar", y="mean", yerr='std',
                     legend=False, title="Primaries", ax=ax)
            ax.set_ylabel(self.metric)
        else:
            try:
                import seaborn as sns
            except ImportError:
                dfb.plot(kind="bar", y="mean", yerr='std',
                         legend=False, figsize=(16, 10), title="Primaries",
                         ax=ax)
                ax.set_ylabel(self.metric)
            else:
                g = sns.barplot(
                    x='antibody',
                    y=metric,
                    data=df,
                    order=dfb.index,
                )
                g.set_xticklabels(rotation=60, fontsize=15)
                g.set_yticklabels(fontsize=15)
                g.set_xlabel('', fontsize=20)
                g.set_ylabel('intensity', fontsize=20)
//  [python]
def _plot_primaries(self, metric='seg-foreground', ax=None):

        df = self.df[self.df['primaries']]
        df = df.drop(columns=['species', 'primaries', 'secondaries'])
        dfa = df.groupby('antibody').agg([np.mean, np.std])
        dfb = dfa.xs(metric, axis=1).sort_values('mean', ascending=False)

        if ax is not None:
            dfb.plot(kind="bar", y="mean", yerr='std',
                     legend=False, title="Primaries", ax=ax)
            ax.set_ylabel(self.metric)
        else:
            try:
                import seaborn as sns
            except ImportError:
                dfb.plot(kind="bar", y="mean", yerr='std',
                         legend=False, figsize=(16, 10), title="Primaries",
                         ax=ax)
                ax.set_ylabel(self.metric)
            else:
                g = sns.catplot(
                    kind='bar',
                    x='antibody',
                    y=metric,
                    data=df,
                    estimator=np.mean,
                    ci='sd',
                    order=dfb.index,
                    height=8,
                    aspect=2,
                )

                g.set_xticklabels(rotation=60, fontsize=15)
                g.set_yticklabels(fontsize=15)
                g.set_axis_labels('', 'intensity', fontsize=20)
//  [python]
def test_forward(self):
        wrapped_module, module = self._get_module()
        input = torch.rand(4, 5)
        torch.testing.assert_allclose(wrapped_module(input), module(input))
//  [python]
def test_forward(self):
        wrapped_module, module = self._get_module()
        input = torch.rand(4, 5)
        torch.testing.assert_close(wrapped_module(input), module(input))
//  [python]
def test_sample(self):
        x = self.d.rsample(torch.Size([5]))
        torch.testing.assert_allclose(x.pow(2).sum(-1), torch.ones(torch.Size([*x.shape[:-1]])))
//  [python]
def test_sample(self):
        x = self.d.rsample(torch.Size([5]))
        torch.testing.assert_close(x.pow(2).sum(-1), torch.ones(torch.Size([*x.shape[:-1]])))
//  [python]
def test_load_state_dict(self):
        wrapped_module, module = self._get_module()
        wrapped_module.load_state_dict(module.state_dict())
        input = torch.rand(4, 5)
        torch.testing.assert_allclose(wrapped_module(input), module(input))
//  [python]
def test_load_state_dict(self):
        wrapped_module, module = self._get_module()
        wrapped_module.load_state_dict(module.state_dict())
        input = torch.rand(4, 5)
        torch.testing.assert_close(wrapped_module(input), module(input))
//  [python]
def test_log_prob(self):
        d = HypersphericalUniform(2)
        x = d.sample(torch.Size([5]))
        logp = d.log_prob(x)
        torch.testing.assert_allclose(logp, - (math.log(4) + math.log(math.pi)))
//  [python]
def test_log_prob(self):
        d = HypersphericalUniform(2)
        x = d.sample(torch.Size([5]))
        logp = d.log_prob(x)
        torch.testing.assert_close(logp, - (math.log(4) + math.log(math.pi)))
//  [python]
def assert_all_approx_close(a, b, rtol, atol, count):

    idx = torch.isclose(a.float(), b.float(), rtol, atol)
    sumval = (idx==0).sum().item()
    if sumval > count:
        print(f'Too many values not close: assert {sumval} < {count}')
        try:
            torch.testing.assert_allclose(a, b, rtol, atol)
        except Exception as e:
            print(e)
//  [python]
def assert_all_approx_close(a, b, rtol, atol, count):

    idx = torch.isclose(a.float(), b.float(), rtol, atol)
    sumval = (idx==0).sum().item()
    if sumval > count:
        print(f'Too many values not close: assert {sumval} < {count}')
        try:
            torch.testing.assert_close(a, b, rtol, atol)
        except Exception as e:
            print(e)
//  [python]
def test_test_step(self):
        cfg = ConfigDict(deepcopy(self.DEFAULT_ARGS))
        cfg.module.data_preprocessor = dict(
            mean=[127.5, 127.5, 127.5], std=[127.5, 127.5, 127.5])
        model: AverageClsScoreTTA = MODELS.build(cfg)

        img1 = torch.randint(0, 256, (1, 3, 224, 224))
        img2 = torch.randint(0, 256, (1, 3, 224, 224))
        data1 = {
            'inputs': img1,
            'data_samples': [ClsDataSample().set_gt_label(1)]
        }
        data2 = {
            'inputs': img2,
            'data_samples': [ClsDataSample().set_gt_label(1)]
        }
        data_tta = {
            'inputs': [img1, img2],
            'data_samples': [[ClsDataSample().set_gt_label(1)],
                             [ClsDataSample().set_gt_label(1)]]
        }

        score1 = model.module.test_step(data1)[0].pred_label.score
        score2 = model.module.test_step(data2)[0].pred_label.score
        score_tta = model.test_step(data_tta)[0].pred_label.score

        torch.testing.assert_allclose(score_tta, (score1 + score2) / 2)
//  [python]
def test_test_step(self):
        cfg = ConfigDict(deepcopy(self.DEFAULT_ARGS))
        cfg.module.data_preprocessor = dict(
            mean=[127.5, 127.5, 127.5], std=[127.5, 127.5, 127.5])
        model: AverageClsScoreTTA = MODELS.build(cfg)

        img1 = torch.randint(0, 256, (1, 3, 224, 224))
        img2 = torch.randint(0, 256, (1, 3, 224, 224))
        data1 = {
            'inputs': img1,
            'data_samples': [ClsDataSample().set_gt_label(1)]
        }
        data2 = {
            'inputs': img2,
            'data_samples': [ClsDataSample().set_gt_label(1)]
        }
        data_tta = {
            'inputs': [img1, img2],
            'data_samples': [[ClsDataSample().set_gt_label(1)],
                             [ClsDataSample().set_gt_label(1)]]
        }

        score1 = model.module.test_step(data1)[0].pred_label.score
        score2 = model.module.test_step(data2)[0].pred_label.score
        score_tta = model.test_step(data_tta)[0].pred_label.score

        torch.testing.assert_close(score_tta, (score1 + score2) / 2)
//  [python]
def _test_checkpoint_wrapper(self, device, log_memory_usage=False):
        def get_loss_and_gnorm(model):
            torch.manual_seed(1)
            input = torch.rand(2, 16, 32).requires_grad_(True).to(device)
            model.zero_grad()
            loss = model(input).sum()
            loss.backward()
            gnorm = torch.norm(
                torch.stack([torch.norm(p.grad.detach()) for p in model.parameters()])
            )
            return {"loss": loss, "gnorm": gnorm}

        model = Model().to(device)
        no_cpt = get_loss_and_gnorm(model)

        model = Model(use_pytorch_checkpoint=True).to(device)
        pyt_cpt = get_loss_and_gnorm(model)
        torch.testing.assert_allclose(no_cpt["loss"], pyt_cpt["loss"])
        torch.testing.assert_allclose(no_cpt["gnorm"], pyt_cpt["gnorm"])

        model = Model(use_fairseq_checkpoint=True).to(device)
        fairseq_cpt = get_loss_and_gnorm(model)
        torch.testing.assert_allclose(no_cpt["loss"], fairseq_cpt["loss"])
        torch.testing.assert_allclose(no_cpt["gnorm"], fairseq_cpt["gnorm"])

        model = Model(use_fairseq_checkpoint=True, offload_to_cpu=True).to(device)
        fairseq_cpt_offload = get_loss_and_gnorm(model)
        torch.testing.assert_allclose(no_cpt["loss"], fairseq_cpt_offload["loss"])
        torch.testing.assert_allclose(no_cpt["gnorm"], fairseq_cpt_offload["gnorm"])
//  [python]
def _test_checkpoint_wrapper(self, device, log_memory_usage=False):
        def get_loss_and_gnorm(model):
            torch.manual_seed(1)
            input = torch.rand(2, 16, 32).requires_grad_(True).to(device)
            model.zero_grad()
            loss = model(input).sum()
            loss.backward()
            gnorm = torch.norm(
                torch.stack([torch.norm(p.grad.detach()) for p in model.parameters()])
            )
            return {"loss": loss, "gnorm": gnorm}

        model = Model().to(device)
        no_cpt = get_loss_and_gnorm(model)

        model = Model(use_pytorch_checkpoint=True).to(device)
        pyt_cpt = get_loss_and_gnorm(model)
        torch.testing.assert_close(no_cpt["loss"], pyt_cpt["loss"])
        torch.testing.assert_close(no_cpt["gnorm"], pyt_cpt["gnorm"])

        model = Model(use_fairseq_checkpoint=True).to(device)
        fairseq_cpt = get_loss_and_gnorm(model)
        torch.testing.assert_close(no_cpt["loss"], fairseq_cpt["loss"])
        torch.testing.assert_close(no_cpt["gnorm"], fairseq_cpt["gnorm"])

        model = Model(use_fairseq_checkpoint=True, offload_to_cpu=True).to(device)
        fairseq_cpt_offload = get_loss_and_gnorm(model)
        torch.testing.assert_close(no_cpt["loss"], fairseq_cpt_offload["loss"])
        torch.testing.assert_close(no_cpt["gnorm"], fairseq_cpt_offload["gnorm"])
//  [python]
def test_batch_sampler_behavior(self):
        dataset = MyIterableDataset(0, 100)

        for num_workers in (1, 2, 4):
            with self.subTest(f"{num_workers}"):
                torch.manual_seed(42)
                loader = DataLoader(dataset, batch_sampler=MegatronPretrainingRandomSampler(100, 0, 4, 0, 1), num_workers=num_workers)
                samples = []
                for i, batch in enumerate(loader):
                    samples.append(batch)
                    if i == 2 - 1:
                        break

                torch.manual_seed(42)
                loader = DataLoader(dataset, batch_sampler=MegatronPretrainingRandomSampler(100, 0, 2, 0, 1), num_workers=num_workers)
                samples2 = []
                for i, batch in enumerate(loader):
                    samples2.append(batch)
                    if i == 4 - 1:
                        break
                torch.testing.assert_allclose(torch.cat(samples), torch.cat(samples2))
//  [python]
def test_batch_sampler_behavior(self):
        dataset = MyIterableDataset(0, 100)

        for num_workers in (1, 2, 4):
            with self.subTest(f"{num_workers}"):
                torch.manual_seed(42)
                loader = DataLoader(dataset, batch_sampler=MegatronPretrainingRandomSampler(100, 0, 4, 0, 1), num_workers=num_workers)
                samples = []
                for i, batch in enumerate(loader):
                    samples.append(batch)
                    if i == 2 - 1:
                        break

                torch.manual_seed(42)
                loader = DataLoader(dataset, batch_sampler=MegatronPretrainingRandomSampler(100, 0, 2, 0, 1), num_workers=num_workers)
                samples2 = []
                for i, batch in enumerate(loader):
                    samples2.append(batch)
                    if i == 4 - 1:
                        break
                torch.testing.assert_close(torch.cat(samples), torch.cat(samples2))
//  [python]
def _test_same_output(self, batch_size):
        torch.cuda.manual_seed(42)
        self.input_ = torch.randn((batch_size, *self.module_cpu_.normalized_shape), device="cpu").requires_grad_(True)
        self.input_cuda_ = self.input_.cuda().detach().requires_grad_(True)
        out_cpu_ = self.module_cpu_(self.input_)
        gO = torch.rand_like(out_cpu_)
        out_cpu_.backward(gO)
        out_cuda_ = self.module_cuda_(self.input_cuda_)
        gO = gO.cuda()
        out_cuda_.backward(gO)
        assert out_cpu_.is_cuda == False
        assert out_cuda_.is_cuda == True
        torch.testing.assert_allclose(out_cpu_, out_cuda_.cpu())
        torch.testing.assert_allclose(self.input_.grad, self.input_cuda_.grad.cpu())
//  [python]
def _test_same_output(self, batch_size):
        torch.cuda.manual_seed(42)
        self.input_ = torch.randn((batch_size, *self.module_cpu_.normalized_shape), device="cpu").requires_grad_(True)
        self.input_cuda_ = self.input_.cuda().detach().requires_grad_(True)
        out_cpu_ = self.module_cpu_(self.input_)
        gO = torch.rand_like(out_cpu_)
        out_cpu_.backward(gO)
        out_cuda_ = self.module_cuda_(self.input_cuda_)
        gO = gO.cuda()
        out_cuda_.backward(gO)
        assert out_cpu_.is_cuda == False
        assert out_cuda_.is_cuda == True
        torch.testing.assert_close(out_cpu_, out_cuda_.cpu())
        torch.testing.assert_close(self.input_.grad, self.input_cuda_.grad.cpu())
//  [python]
def test_inference_mask_generation_two_points_point_batch(self):
        model = SamModel.from_pretrained("facebook/sam-vit-base")
        processor = SamProcessor.from_pretrained("facebook/sam-vit-base")

        model.to(torch_device)
        model.eval()

        raw_image = prepare_image()

        # fmt: off
        input_points = torch.Tensor([[[400, 650]], [[220, 470]]]).cpu()
        # fmt: on

        input_points = input_points.unsqueeze(0)

        inputs = processor(raw_image, input_points=input_points, return_tensors="pt").to(torch_device)

        with torch.no_grad():
            outputs = model(**inputs)

        iou_scores = outputs.iou_scores.cpu()
        self.assertTrue(iou_scores.shape == (1, 2, 3))
        torch.testing.assert_allclose(
            iou_scores, torch.tensor([[[0.9105, 0.9825, 0.9675], [0.7646, 0.7943, 0.7774]]]), atol=1e-4, rtol=1e-4
        )
//  [python]
def test_inference_mask_generation_two_points_point_batch(self):
        model = SamModel.from_pretrained("facebook/sam-vit-base")
        processor = SamProcessor.from_pretrained("facebook/sam-vit-base")

        model.to(torch_device)
        model.eval()

        raw_image = prepare_image()

        # fmt: off
        input_points = torch.Tensor([[[400, 650]], [[220, 470]]]).cpu()
        # fmt: on

        input_points = input_points.unsqueeze(0)

        inputs = processor(raw_image, input_points=input_points, return_tensors="pt").to(torch_device)

        with torch.no_grad():
            outputs = model(**inputs)

        iou_scores = outputs.iou_scores.cpu()
        self.assertTrue(iou_scores.shape == (1, 2, 3))
        torch.testing.assert_close(
            iou_scores, torch.tensor([[[0.9105, 0.9825, 0.9675], [0.7646, 0.7943, 0.7774]]]), atol=1e-4, rtol=1e-4
        )
//  [python]
relevance.register_function(torch.nn.Linear, relevance.LinearEpsilon)

graphs = tg.GraphBatch.from_graphs([tg.Graph(
    node_features=torch.rand(10, 9),
    edge_features=torch.rand(6, 5),
    senders=torch.tensor([0, 0, 1, 2, 4, 5]),
    receivers=torch.tensor([1, 2, 2, 4, 3, 3]),
    global_features=torch.rand(7)
)])

print('bias nf ef gf R_out R_in'.replace(' ', '\t'))
print('-' * 45)

for bias, nf, ef, gf in itertools.product(
        [True, False, 'zero'],
        [9, None],
        [5, None],
        [7, None]
):
    if nf is ef is gf is None:
        continue
    net = tg.GlobalLinear(3, node_features=nf, edge_features=ef, global_features=gf, bias=bool(bias), aggregation='sum')

    if bias == 'zero':
        net.bias.zero_()

    ctx = {}
    out = relevance.forward_relevance(net, graphs, ctx=ctx)
    torch.testing.assert_allclose(out.node_features, net(graphs).node_features)

    rel_out = graphs.evolve(
        global_features=torch.ones_like(out.global_features) * (out.global_features != 0).float()
    )

    rel_in = relevance.backward_relevance(net, rel_out, ctx=ctx)

    # If bias==0 then relevance is conserved at a graph level
    print(
        {True: '  x ', False: '  - ', 'zero': '  0 '}[bias], nf if nf else '-', ef if ef else '-', gf if gf else '-',
        rel_out.global_features.sum().item(),
        ((rel_in.global_features.sum() if rel_in.global_features is not None else 0) +
         (rel_in.edge_features.sum() if rel_in.edge_features is not None else 0) +
         (rel_in.node_features.sum() if rel_in.node_features is not None else 0)).item(),
        sep='\t'
    )
//  [python]
relevance.register_function(torch.nn.Linear, relevance.LinearEpsilon)

graphs = tg.GraphBatch.from_graphs([tg.Graph(
    node_features=torch.rand(10, 9),
    edge_features=torch.rand(6, 5),
    senders=torch.tensor([0, 0, 1, 2, 4, 5]),
    receivers=torch.tensor([1, 2, 2, 4, 3, 3]),
    global_features=torch.rand(7)
)])

print('bias nf ef gf R_out R_in'.replace(' ', '\t'))
print('-' * 45)

for bias, nf, ef, gf in itertools.product(
        [True, False, 'zero'],
        [9, None],
        [5, None],
        [7, None]
):
    if nf is ef is gf is None:
        continue
    net = tg.GlobalLinear(3, node_features=nf, edge_features=ef, global_features=gf, bias=bool(bias), aggregation='sum')

    if bias == 'zero':
        net.bias.zero_()

    ctx = {}
    out = relevance.forward_relevance(net, graphs, ctx=ctx)
    torch.testing.assert_close(out.node_features, net(graphs).node_features)

    rel_out = graphs.evolve(
        global_features=torch.ones_like(out.global_features) * (out.global_features != 0).float()
    )

    rel_in = relevance.backward_relevance(net, rel_out, ctx=ctx)

    # If bias==0 then relevance is conserved at a graph level
    print(
        {True: '  x ', False: '  - ', 'zero': '  0 '}[bias], nf if nf else '-', ef if ef else '-', gf if gf else '-',
        rel_out.global_features.sum().item(),
        ((rel_in.global_features.sum() if rel_in.global_features is not None else 0) +
         (rel_in.edge_features.sum() if rel_in.edge_features is not None else 0) +
         (rel_in.node_features.sum() if rel_in.node_features is not None else 0)).item(),
        sep='\t'
    )
//  [python]
def ort_validate(self, onnx_io, inputs, outputs, tolerate_small_mismatch=False):

        inputs, _ = torch.jit._flatten(inputs)
        outputs, _ = torch.jit._flatten(outputs)

        def to_numpy(tensor):
            if tensor.requires_grad:
                return tensor.detach().cpu().numpy()
            else:
                return tensor.cpu().numpy()

        inputs = list(map(to_numpy, inputs))
        outputs = list(map(to_numpy, outputs))

        ort_session = onnxruntime.InferenceSession(onnx_io.getvalue())
        # compute onnxruntime output prediction
        ort_inputs = dict((ort_session.get_inputs()[i].name, inpt) for i, inpt in enumerate(inputs))
        ort_outs = ort_session.run(None, ort_inputs)
        for i, element in enumerate(outputs):
            try:
                torch.testing.assert_allclose(element, ort_outs[i], rtol=1e-03, atol=1e-05)
            except AssertionError as error:
                if tolerate_small_mismatch:
                    self.assertIn("(0.00%)", str(error), str(error))
                else:
                    raise