jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def make_grouped_sizeplots(
    *,
    df: pd.DataFrame,
    target_x_header: str,
    target_y_header: str,
    output_directory: str,
    make_png: bool = True,
    make_pdf: bool = True,
) -> None:
    it = tqdm(
        df.groupby(['dataset', 'optimizer']),
        desc=f'making sizeplots for {target_x_header}'
    )
    min_bytes = df[target_x_header].min()
    max_bytes = df[target_x_header].max()

    for (dataset, optimizer), sdf in it:
        fig, ax = plt.subplots(figsize=(8, 5))

        g = sns.scatterplot(
            x=target_x_header,
            y=target_y_header,
            data=sdf,
            alpha=0.75,
            hue='model',
            hue_order=sorted(df['model'].unique()),
            ax=ax,
        )
        try:
            skyline_df = get_skyline_df(
                sdf[sdf['dataset'] == dataset],
                columns=[target_x_header, target_y_header],
                smaller_is_better=(True, False),
            )
        except scipy.spatial.qhull.QhullError:
            it.write(f'Not enough data points to make make hull for {dataset}/{optimizer}')
        else:
            sns.lineplot(
                x=target_x_header,
                y=target_y_header,
                data=skyline_df,
                estimator=None,
                markers=True,
                legend=False,
                ax=ax,
            )

        ax.set_xlim([min_bytes, max_bytes])
        ax.set_ylim([0.0, 1.0])
        ax.set_xlabel(target_x_header.replace('_', ' ').title())
        g.set(xscale='log')
        g.legend(loc='center left', bbox_to_anchor=(1.05, 0.5), ncol=1)
        _clean_legend_title(g.legend)

        ax.set_title(f'{dataset} - {optimizer} - {target_x_header}  ({len(sdf.index)} experiments)')

        plt.tight_layout()
        if make_pdf:
            plt.savefig(os.path.join(output_directory, f'scatter_{target_x_header}_{dataset}_{optimizer}.pdf'.lower()))
        if make_png:
            plt.savefig(os.path.join(output_directory, f'scatter_{target_x_header}_{dataset}_{optimizer}.png'.lower()),
                        dpi=300)
        plt.close(fig)
//  [python]
def save_traj_plots_with_gt(results_dir, pred_xyz, gt, plt_prefix=''):
    """Plot and save trajectories together with ground-truth to a file.

    Args:
        results_dir (str): Directory to save the plots.
        pred_xyz (torch.Tensor): Predicted xyz values of a trajectory.
        gt (torch.Tensor): Ground truth predictions as [4,4] transformation matrices.
    """
    gt_xyz = gt[:, :3, 3].cpu().numpy()
    # np_pred = 0.5*gt_xyz + 0.5*pred_xyz[0].cpu().numpy()
    np_pred = pred_xyz[0].cpu()
    fig, ax = plt.subplots(figsize=(8, 8))
    plt.plot(np_pred[:, 0], np_pred[:, 1], label='Ours')
    plt.plot(gt_xyz[:, 0], gt_xyz[:, 1], label='Ground-truth')

    ax.set(xlabel='X (m)', ylabel='Y (m)')

    # Save fig
    plt.tight_layout()
    plt.savefig(results_dir/(plt_prefix+'pred_with_gt.pdf'),
                bbox_inches='tight', pad_inches=0)
    plt.savefig(results_dir/(plt_prefix+'pred_with_gt.png'),
                bbox_inches='tight', pad_inches=0)
//  [python]
def save_traj_plots_with_gt(results_dir, pred_xyz, gt, plt_prefix=''):
    """Plot and save trajectories together with ground-truth to a file.

    Args:
        results_dir (str): Directory to save the plots.
        pred_xyz (torch.Tensor): Predicted xyz values of a trajectory.
        gt (torch.Tensor): Ground truth predictions as [4,4] transformation matrices.
    """
    gt_xyz = gt[:, :3, 3].cpu().numpy()
    # np_pred = 0.5*gt_xyz + 0.5*pred_xyz[0].cpu().numpy()
    np_pred = pred_xyz[0].cpu()
    fig, ax = plt.subplots(figsize=(8, 8))
    sn.lineplot(x=np_pred[:, 0], y=np_pred[:, 1],
                sort=False, ax=ax, label='Ours')
    sn.lineplot(x=gt_xyz[:, 0], y=gt_xyz[:, 1],
                sort=False, ax=ax, label='Ground-truth')

    ax.set(xlabel='X (m)', ylabel='Y (m)')

    # Save fig
    plt.tight_layout()
    plt.savefig(results_dir/(plt_prefix+'pred_with_gt.pdf'),
                bbox_inches='tight', pad_inches=0)
    plt.savefig(results_dir/(plt_prefix+'pred_with_gt.png'),
                bbox_inches='tight', pad_inches=0)
//  [python]
def plot_data(data_in,
              xaxis='Epoch',
              value="EpRet",
              condition="Condition1",
              smooth=1,
              **kwargs):
    data = []
    if smooth > 1:
        """
        smooth data with moving window average.
        that is,
            smoothed_y[t] = average(y[t-k], y[t-k+1], ..., y[t+k-1], y[t+k])
        where the "smooth" param is width of that window (2k+1)
        """
        y = np.ones(smooth)
        for datum in data_in:
            if value not in datum:
                value = value.split("Test")[-1]
            x = np.asarray(datum[value])
            z = np.ones(len(x))
            smoothed_x = np.convolve(x, y, 'same') / np.convolve(z, y, 'same')
            datum[value] = smoothed_x
            data.append(datum[[xaxis, value, condition, "cost_limit"]])

    if isinstance(data, list):
        data = pd.concat(data, ignore_index=True, join="outer")
    sns.set(style="darkgrid", font_scale=1.5)
    sns.tsplot(data=data, time=xaxis, value=value, unit="Unit", condition=condition, ci='sd', **kwargs)
    #sns.lineplot(data=data, x=xaxis, y=value, hue=condition, ci='sd')

    if data["cost_limit"][0] is not None and (value == "EpCost"
                                              or value == "TestEpCost"):
        plt.axhline(y=data["cost_limit"][0],
                    linestyle="--",
                    label="cost limit",
                    linewidth=2.5)
    """
    If you upgrade to any version of Seaborn greater than 0.8.1, switch from 
    tsplot to lineplot replacing L29 with:

        sns.lineplot(data=data, x=xaxis, y=value, hue=condition, ci='sd', **kwargs)

    Changes the colorscheme and the default legend style, though.
    """
    plt.legend(loc='best').set_draggable(True)
    #plt.legend(loc='upper center', ncol=3, handlelength=1,
    #           borderaxespad=0., prop={'size': 13})
    """
    For the version of the legend used in the Spinning Up benchmarking page, 
    swap L38 with:

    plt.legend(loc='upper center', ncol=6, handlelength=1,
               mode="expand", borderaxespad=0., prop={'size': 13})
    """

    xscale = np.max(np.asarray(data[xaxis])) > 5e3
    if xscale:
        # Just some formatting niceness: x-axis scale in scientific notation if max x is large
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))

    plt.tight_layout(pad=0.5)
//  [python]
def plot_data(data_in,
              xaxis='Epoch',
              value="EpRet",
              condition="Condition1",
              smooth=1,
              **kwargs):
    data = []
    if smooth > 1:
        """
        smooth data with moving window average.
        that is,
            smoothed_y[t] = average(y[t-k], y[t-k+1], ..., y[t+k-1], y[t+k])
        where the "smooth" param is width of that window (2k+1)
        """
        y = np.ones(smooth)
        for datum in data_in:
            if value not in datum:
                value = value.split("Test")[-1]
            x = np.asarray(datum[value])
            z = np.ones(len(x))
            smoothed_x = np.convolve(x, y, 'same') / np.convolve(z, y, 'same')
            datum[value] = smoothed_x
            data.append(datum[[xaxis, value, condition, "cost_limit"]])

    if isinstance(data, list):
        data = pd.concat(data, ignore_index=True, join="outer")
    sns.set(style="darkgrid", font_scale=1.5)
    # sns.tsplot(data=data, time=xaxis, value=value, unit="Unit", condition=condition, ci='sd', **kwargs)
    sns.lineplot(data=data, x=xaxis, y=value, hue=condition, ci='sd')

    if data["cost_limit"][0] is not None and (value == "EpCost"
                                              or value == "TestEpCost"):
        plt.axhline(y=data["cost_limit"][0],
                    linestyle="--",
                    label="cost limit",
                    linewidth=2.5)
    """
    If you upgrade to any version of Seaborn greater than 0.8.1, switch from 
    tsplot to lineplot replacing L29 with:

        sns.lineplot(data=data, x=xaxis, y=value, hue=condition, ci='sd', **kwargs)

    Changes the colorscheme and the default legend style, though.
    """
    plt.legend(loc='best').set_draggable(True)
    #plt.legend(loc='upper center', ncol=3, handlelength=1,
    #           borderaxespad=0., prop={'size': 13})
    """
    For the version of the legend used in the Spinning Up benchmarking page, 
    swap L38 with:

    plt.legend(loc='upper center', ncol=6, handlelength=1,
               mode="expand", borderaxespad=0., prop={'size': 13})
    """

    xscale = np.max(np.asarray(data[xaxis])) > 5e3
    if xscale:
        # Just some formatting niceness: x-axis scale in scientific notation if max x is large
        plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))

    plt.tight_layout(pad=0.5)
//  [python]
def draw_text_length(data_path):
    df = pd.read_csv(data_path, encoding='utf-8')
    send_df = df[df['isSend']==1]
    receive_df = df[df['isSend']==0]

    send_df['text_length'] = send_df['content'].str.len()
    receive_df['text_length'] = receive_df['content'].str.len()

    send_mean = send_df.groupby('newTime').mean()
    receive_mean = receive_df.groupby('newTime').mean()

    new_df = pd.DataFrame({"月份": list(send_mean.index) + list(receive_mean.index),
                           "消息长度均值": send_mean['text_length'].to_list() + receive_mean['text_length'].to_list(),
                           "is_send": ['send'] * len(send_mean) + ['receive'] * len(receive_mean)})

    plt.figure(figsize=(10,6))
    for i in new_df['is_send'].unique():
        temp = new_df[new_df['is_send']==i]
        plt.plot(temp['月份'], temp['消息长度均值'], label=i)
    plt.legend()
    plt.savefig("result/句子长度均值随时间变化.png")
    plt.show()
//  [python]
def draw_text_length(data_path):
    df = pd.read_csv(data_path, encoding='utf-8')
    send_df = df[df['isSend']==1]
    receive_df = df[df['isSend']==0]

    send_df['text_length'] = send_df['content'].str.len()
    receive_df['text_length'] = receive_df['content'].str.len()

    send_mean = send_df.groupby('newTime').mean()
    receive_mean = receive_df.groupby('newTime').mean()

    new_df = pd.DataFrame({"月份": list(send_mean.index) + list(receive_mean.index),
                           "消息长度均值": send_mean['text_length'].to_list() + receive_mean['text_length'].to_list(),
                           "is_send": ['send'] * len(send_mean) + ['receive'] * len(receive_mean)})

    sns.lineplot(x="月份", y="消息长度均值", hue="is_send",data=new_df)
    plt.savefig("result/句子长度均值随时间变化.png")
    plt.show()
//  [python]
def hyperparameter_gauge(self, fig, gridspec):
    """Hyperparameter gauge, currently showing the learning rate over time.

    Args:
        self (cockpit.plotter): The cockpit plotter requesting this instrument.
        fig (matplotlib.figure): Figure of the Cockpit.
        gridspec (matplotlib.gridspec): GridSpec where the instrument should be
            placed
    """
    # Plot Trace vs iteration
    title = "Hyperparameters"

    # Check if the required data is available, else skip this instrument
    requires = ["iteration", "learning_rate"]
    plot_possible = check_data(self.tracking_data, requires)
    if not plot_possible:
        warnings.warn(
            "Couldn't get the required data for the " + title + " instrument",
            stacklevel=1,
        )
        return

    ax = fig.add_subplot(gridspec)

    clean_learning_rate = self.tracking_data[["iteration", "learning_rate"]].dropna()

    # Plot Settings
    plot_args = {
        "x": "iteration",
        "y": "learning_rate",
        "data": clean_learning_rate,
    }
    ylabel = plot_args["y"].replace("_", " ").title()
    sns.lineplot(
        **plot_args, ax=ax, label=ylabel, linewidth=2, color=self.secondary_color
    )

    _beautify_plot(
        ax=ax,
        xlabel=plot_args["x"],
        ylabel=ylabel,
        x_scale="symlog" if self.show_log_iter else "linear",
        title=title,
        xlim="tight",
        fontweight="bold",
        facecolor=self.bg_color_instruments2,
    )

    ax.legend()
    _add_last_value_to_legend(ax)
//  [python]
def plot_subspace_figure(df, filename=None):
    df = df.reset_index().drop(columns=["index"])
    plt.style.use("seaborn-darkgrid")
    fig, ax = plt.subplots(figsize=(12, 8))
    sns.lineplot(x="Subspace Dim", y="Reward", hue="Method", marker="o", data=df)
    lines, labels = ax.get_legend_handles_labels()
    for line, method in zip(lines, labels):
        data = df[df["Method"]==method]
        color = line.get_c()
        y_lower_bound =  data["Reward"] -  data["Std"]
        y_upper_bound = data["Reward"] + data["Std"]
        ax.fill_between(data["Subspace Dim"],  y_lower_bound, y_upper_bound, color=color, alpha=0.3)

    ax.set_ylabel("Reward", fontsize=16)
    plt.setp(ax.get_xticklabels(), fontsize=16) 
    plt.setp(ax.get_yticklabels(), fontsize=16) 
    ax.set_xlabel("Subspace Dimension(d)", fontsize=16)
    dataset = df.iloc[0]["Dataset"]
    ax.set_title(f"{dataset.title()} - Subspace Dim vs. Reward", fontsize=18)
    legend = ax.legend(loc="lower right", prop={'size': 16},frameon=1)
    frame = legend.get_frame()
    frame.set_color('white')
    frame.set_alpha(0.6)
    
    file_path = "./bandits/figures/"
    file_path = file_path + f"{dataset}_sub_reward.png" if filename is None else file_path + f"{filename}.png"
    plt.savefig(file_path)
//  [python]
def plotResults2(datasets, results, save_directory, k_folds):
    # need df with row: <dataset,method,fold,value> then
    # sns.lineplot(data=df,x="dataset",y="value",style="method)
    dataList = list()
    for dataset in datasets:
        folded_results = results[dataset]
        fold_counter = 0
        for fold in folded_results:
            for method in fold:
                res_loss = fold[method]["ret_loss"]
                dict = {"dataset": dataset, "method": method,
                        "fold": fold_counter, "value": res_loss}
                dataList.append(dict)
            fold_counter += 1
    df = pd.DataFrame(dataList)
    fig, ax = plt.subplots()
    sns.set(font_scale=0.4)
    sns_plot = sns.pointplot(x="dataset", y="value",
                            hue="method", data=df)
    ax.set(xlabel='common xlabel', ylabel='common ylabel')
    fig = sns_plot.get_figure()
    fig.savefig(f"{save_directory}/all_methods_loss.pdf")
    plt.clf()
//  [python]
def plotResults2(datasets, results, save_directory, k_folds):
    # need df with row: <dataset,method,fold,value> then
    # sns.lineplot(data=df,x="dataset",y="value",style="method)
    dataList = list()
    for dataset in datasets:
        folded_results = results[dataset]
        fold_counter = 0
        for fold in folded_results:
            for method in fold:
                res_loss = fold[method]["ret_loss"]
                dict = {"dataset": dataset, "method": method,
                        "fold": fold_counter, "value": res_loss}
                dataList.append(dict)
            fold_counter += 1
    df = pd.DataFrame(dataList)
    fig, ax = plt.subplots()
    sns.set(font_scale=0.4)
    sns_plot = sns.lineplot(x="dataset", y="value",
                            style="method", hue="method", data=df)
    ax.set(xlabel='common xlabel', ylabel='common ylabel')
    fig = sns_plot.get_figure()
    fig.savefig(f"{save_directory}/all_methods_loss.pdf")
    plt.clf()
//  [python]
def plotResults3(datasets, results, save_directory, k_folds):
    # need df with row: <dataset,method,fold,value> then
    # sns.lineplot(data=df,x="dataset",y="value",style="method)
    dataList = list()
    for dataset in datasets:
        dataset_result = results[dataset]
        fold_counter = 0
        for key in dataset_result.keys():
            fold = dataset_result[key]
            for method in fold:
                res_loss = fold[method]["ret_loss"]
                dict = {"dataset": dataset, "method": method,
                        "fold": fold_counter, "value": res_loss}
                dataList.append(dict)
            fold_counter += 1
    df = pd.DataFrame(dataList)
    grp = df.groupby(["dataset"],as_index=False)
    df["dataset_mean"] = grp.value.transform("mean")
    sorted_df = df.sort_values(by=["dataset_mean", "method"])
    fig, ax = plt.subplots()
    sns.set(font_scale=0.4)
    sns_plot = sns.pointplot(x="dataset", y="value",
                            hue="method", data=sorted_df)
    ax.set(xlabel='dataset', ylabel='retrieval loss')
    fig = sns_plot.get_figure()
    fig.savefig(f"{save_directory}/all_methods_loss.pdf")
    plt.clf()
//  [python]
def plotResults3(datasets, results, save_directory, k_folds):
    # need df with row: <dataset,method,fold,value> then
    # sns.lineplot(data=df,x="dataset",y="value",style="method)
    dataList = list()
    for dataset in datasets:
        dataset_result = results[dataset]
        fold_counter = 0
        for key in dataset_result.keys():
            fold = dataset_result[key]
            for method in fold:
                res_loss = fold[method]["ret_loss"]
                dict = {"dataset": dataset, "method": method,
                        "fold": fold_counter, "value": res_loss}
                dataList.append(dict)
            fold_counter += 1
    df = pd.DataFrame(dataList)
    grp = df.groupby(["dataset"],as_index=False)
    df["dataset_mean"] = grp.value.transform("mean")
    sorted_df = df.sort_values(by=["dataset_mean", "method"])
    fig, ax = plt.subplots()
    sns.set(font_scale=0.4)
    sns_plot = sns.lineplot(x="dataset", y="value",
                            hue="method", data=sorted_df)
    ax.set(xlabel='dataset', ylabel='retrieval loss')
    fig = sns_plot.get_figure()
    fig.savefig(f"{save_directory}/all_methods_loss.pdf")
    plt.clf()
//  [python]
def plotTrainings(datasets, results, save_directory, k_folds):
    # need df with row: <dataset,method,fold,value> then
    # sns.lineplot(data=df,x="dataset",y="value",style="method)
    dataList = list()
    for dataset in datasets:
        dataset_result = results[dataset]
        fold_counter = 0
        for foldnumber in dataset_result.keys():
            fold = dataset_result[foldnumber]
            for method in fold:
                training_loss = fold[method]["training_loss_hist"]
                epochnum = 0
                for epoch in training_loss:
                    epochnum += 1
                    thisdict = {"epoch": epochnum, "method": method,
                                "fold": fold_counter, "value": epoch}
                    dataList.append(thisdict)
            fold_counter += 1
        df = pd.DataFrame(dataList)
        setLateXFonts()
        fig, ax = plt.subplots()
        sns.set(font_scale=0.4)
        sns_plot = sns.tsplot(x="epoch", y="value",
                                style="method", hue="method", data=df)
        ax.set(xlabel='dataset', ylabel='retrieval loss')
        fig = sns_plot.get_figure()
        fig.savefig(f"{save_directory}/{dataset}_trainingloss.pdf")
        plt.clf()
//  [python]
def plotTrainings(datasets, results, save_directory, k_folds):
    # need df with row: <dataset,method,fold,value> then
    # sns.lineplot(data=df,x="dataset",y="value",style="method)
    dataList = list()
    for dataset in datasets:
        dataset_result = results[dataset]
        fold_counter = 0
        for foldnumber in dataset_result.keys():
            fold = dataset_result[foldnumber]
            for method in fold:
                training_loss = fold[method]["training_loss_hist"]
                epochnum = 0
                for epoch in training_loss:
                    epochnum += 1
                    thisdict = {"epoch": epochnum, "method": method,
                                "fold": fold_counter, "value": epoch}
                    dataList.append(thisdict)
            fold_counter += 1
        df = pd.DataFrame(dataList)
        setLateXFonts()
        fig, ax = plt.subplots()
        sns.set(font_scale=0.4)
        sns_plot = sns.lineplot(x="epoch", y="value",
                                style="method", hue="method", data=df)
        ax.set(xlabel='dataset', ylabel='retrieval loss')
        fig = sns_plot.get_figure()
        fig.savefig(f"{save_directory}/{dataset}_trainingloss.pdf")
        plt.clf()
//  [python]
def plotDf(df, save_directory, filename, drawdev=False, 
           font_scale=0.8, hue_order=None, legend_loc=1):
    # need df with row: <dataset,method,fold,value> then
    # sns.lineplot(data=df,x="dataset",y="value",style="method)
    sns.set_style("whitegrid")
    setLateXFonts()
    with sns.plotting_context("poster",font_scale=font_scale, rc={"lines.linewidth": font_scale, "grid.linewidth": font_scale}):
        fig, ax = plt.subplots()

        if drawdev:
            _y = "value"
        else:
            _y="method_mean"

        if hue_order is None:
            sns_plot = sns.tsplot(x="dataset", y=_y,
                                    condition="method",
                                    data=df, err_style="ci_band")
        else:
            sns_plot = sns.tsplot(x="dataset", y=_y,
                                    condition="method", hue_order=hue_order,
                                    data=df, err_style="ci_band")
        ax.set(xlabel='dataset', ylabel='retrieval loss')
        ax.legend(loc=legend_loc)
        fig = sns_plot.get_figure()
        fig.savefig(f"{save_directory}/{filename}.pdf",bbox_inches='tight')
        plt.show()
        plt.clf()
//  [python]
def plotDf(df, save_directory, filename, drawdev=False, 
           font_scale=0.8, hue_order=None, legend_loc=1):
    # need df with row: <dataset,method,fold,value> then
    # sns.lineplot(data=df,x="dataset",y="value",style="method)
    sns.set_style("whitegrid")
    setLateXFonts()
    with sns.plotting_context("poster",font_scale=font_scale, rc={"lines.linewidth": font_scale, "grid.linewidth": font_scale}):
        fig, ax = plt.subplots()

        if drawdev:
            _y = "value"
        else:
            _y="method_mean"

        if hue_order is None:
            sns_plot = sns.lineplot(x="dataset", y=_y,
                                    hue="method",
                                    data=df, sort=False)
        else:
            sns_plot = sns.lineplot(x="dataset", y=_y,
                                    hue="method", hue_order=hue_order,
                                    data=df, sort=False)
        ax.set(xlabel='dataset', ylabel='retrieval loss')
        ax.legend(loc=legend_loc)
        fig = sns_plot.get_figure()
        fig.savefig(f"{save_directory}/{filename}.pdf",bbox_inches='tight')
        plt.show()
        plt.clf()
//  [python]
def plotAlphaResults(datasets, results, save_directory, k_folds):
    # need df with row: <dataset,method,fold,value> then
    # sns.lineplot(data=df,x="dataset",y="value",style="method)
    dataList = list()
    for dataset in datasets:
        dataset_result = results[dataset]
        fold_counter = 0
        for key in dataset_result.keys():
            fold = dataset_result[key]
            for method in fold:
                res_loss = fold[method]["ret_loss"]
                dict = {"dataset": dataset, "method": method,
                        "fold": fold_counter, "value": res_loss}
                dataList.append(dict)
            fold_counter += 1
    df = pd.DataFrame(dataList)
    grp = df.groupby(["dataset"],as_index=False)
    #df["dataset_mean"] = grp.value.transform("mean")
    grp["dataset_mean"] = grp.value.transform("mean")
    sorted_df = grp.sort_values(by=["dataset_mean", "method"])
    fig, ax = plt.subplots()
    sns.set(font_scale=0.4)
    sns_plot = sns.pointplot(x="method", y="dataset_mean", hue="dataset", data=sorted_df, join=True)
    ax.set(xlabel='dataset', ylabel='retrieval loss')
    fig = sns_plot.get_figure()
    fig.savefig(f"{save_directory}/all_methods_loss.pdf")
    plt.clf()
//  [python]
def plotAlphaResults(datasets, results, save_directory, k_folds):
    # need df with row: <dataset,method,fold,value> then
    # sns.lineplot(data=df,x="dataset",y="value",style="method)
    dataList = list()
    for dataset in datasets:
        dataset_result = results[dataset]
        fold_counter = 0
        for key in dataset_result.keys():
            fold = dataset_result[key]
            for method in fold:
                res_loss = fold[method]["ret_loss"]
                dict = {"dataset": dataset, "method": method,
                        "fold": fold_counter, "value": res_loss}
                dataList.append(dict)
            fold_counter += 1
    df = pd.DataFrame(dataList)
    grp = df.groupby(["dataset"],as_index=False)
    #df["dataset_mean"] = grp.value.transform("mean")
    grp["dataset_mean"] = grp.value.transform("mean")
    sorted_df = grp.sort_values(by=["dataset_mean", "method"])
    fig, ax = plt.subplots()
    sns.set(font_scale=0.4)
    sns_plot = sns.lineplot(x="method", y="dataset_mean", style="dataset",
                            hue="dataset", data=sorted_df)
    ax.set(xlabel='dataset', ylabel='retrieval loss')
    fig = sns_plot.get_figure()
    fig.savefig(f"{save_directory}/all_methods_loss.pdf")
    plt.clf()
//  [python]
def plot_dynamic_variables(df_all, model_name, dynamic_variable):
    """Plot line graphs of the dynamic variable for each ablation group."""
    if len(df_all) == 0:
        print(f"⚠️ Warning: No data for {model_name} and {dynamic_variable} variable")
        return
    assert (
        "_dynamic" in dynamic_variable
    ), f"dynamic variable={dynamic_variable} (expected to contain '_dynamic')"

    initialize_plot_default()
    plt.figure(figsize=(7.5, 6.25))
    x_variable = "day"
    x_label = TIMELABEL_DEFAULT
    y_variable = dynamic_variable.split("_dynamic")[0].replace("_", " ")
    y_label = DYNAMIC_VARIABLES_TO_NAMES[dynamic_variable]
    grouping = "ablation"
    grouping_order = ABLATION_NAME_ORDER
    palette = ["#444"] + sns.color_palette(None, len(grouping_order) - 1)
    # Plot df_grouped
    sns.pointplot(
        data=df_all,
        x=x_variable,
        y=y_variable,
        hue=grouping,
        # style=grouping,
        hue_order=grouping_order,
        # errorbar=("se", 1),
        # errorbar="se",
        # n_boot=5000,
        # seed=10,
        # errorbar=None,
        palette=palette,
        # dashes=False,
        # markers=True,
        join=True,
    )
    plt.xlabel(x_label, size=LABELSIZE_DEFAULT)
    plt.ylabel(y_label, size=LABELSIZE_DEFAULT)
    title = f"{y_label} ({model_name})"
    plt.title(title)

    loc = (
        "lower left"
        if dynamic_variable == "population_dynamic"
        else "best"
        if dynamic_variable == "nuclear_dynamic"
        else "upper left"
    )
    plt.legend(
        # bbox_to_anchor=(1.05, 1),
        loc=loc,
        borderaxespad=0.0,
        # ncol=2,
        title="Prompt Variation",
        handletextpad=0.1,
        labelspacing=0.25,
        # framealpha=0.5,
        columnspacing=0.25,
    )
    if dynamic_variable == "nuclear_dynamic" and model_name != "Claude-2.0":
        # Remove legend, just doesn't fit well
        plt.legend().remove()

    # Save the plot
    save_plot(
        OUTPUT_DIR,
        f"ablate_var_{DYNAMIC_VARIABLES_TO_NAMES[dynamic_variable]}_{model_name}",
    )

    # Clear the plot
    plt.clf()
    plt.close()
//  [python]
def plot_dynamic_variables(df_all, model_name, dynamic_variable):
    """Plot line graphs of the dynamic variable for each ablation group."""
    if len(df_all) == 0:
        print(f"⚠️ Warning: No data for {model_name} and {dynamic_variable} variable")
        return
    assert (
        "_dynamic" in dynamic_variable
    ), f"dynamic variable={dynamic_variable} (expected to contain '_dynamic')"

    initialize_plot_default()
    plt.figure(figsize=(7.5, 6.25))
    x_variable = "day"
    x_label = TIMELABEL_DEFAULT
    y_variable = dynamic_variable.split("_dynamic")[0].replace("_", " ")
    y_label = DYNAMIC_VARIABLES_TO_NAMES[dynamic_variable]
    grouping = "ablation"
    grouping_order = ABLATION_NAME_ORDER
    palette = ["#444"] + sns.color_palette(None, len(grouping_order) - 1)
    # Plot df_grouped
    sns.lineplot(
        data=df_all,
        x=x_variable,
        y=y_variable,
        hue=grouping,
        style=grouping,
        hue_order=grouping_order,
        # errorbar=("se", 1),
        errorbar="se",
        # n_boot=5000,
        # seed=10,
        # errorbar=None,
        palette=palette,
        dashes=False,
        markers=True,
    )
    plt.xlabel(x_label, size=LABELSIZE_DEFAULT)
    plt.ylabel(y_label, size=LABELSIZE_DEFAULT)
    title = f"{y_label} ({model_name})"
    plt.title(title)

    loc = (
        "lower left"
        if dynamic_variable == "population_dynamic"
        else "best"
        if dynamic_variable == "nuclear_dynamic"
        else "upper left"
    )
    plt.legend(
        # bbox_to_anchor=(1.05, 1),
        loc=loc,
        borderaxespad=0.0,
        # ncol=2,
        title="Prompt Variation",
        handletextpad=0.1,
        labelspacing=0.25,
        # framealpha=0.5,
        columnspacing=0.25,
    )
    if dynamic_variable == "nuclear_dynamic" and model_name != "Claude-2.0":
        # Remove legend, just doesn't fit well
        plt.legend().remove()

    # Save the plot
    save_plot(
        OUTPUT_DIR,
        f"ablate_var_{DYNAMIC_VARIABLES_TO_NAMES[dynamic_variable]}_{model_name}",
    )

    # Clear the plot
    plt.clf()
    plt.close()
//  [python]
def parse_table(env: str, system: str, suffix: str) -> None:
  private_copy = {
    "Num. Workers": [],
    "FPS": [],
    "Env": [],
    "System": [],
    "Method": [],
  }
  sep = f"<!-- {env} - {system} -->"
  raw = open("README.md").read().split(sep)[1].strip().splitlines()
  worker_num = list(map(int, raw[0].split("|")[2:-1]))
  for line in raw[2:]:
    line = line.split("|")[1:-1]
    method = line.pop(0).strip()
    for w, f in zip(worker_num, line):
      for d in [data, private_copy]:
        d["Num. Workers"].append(w)
        d["FPS"].append(None if f.strip() == "/" else float(f))
        d["Env"].append(env)
        d["System"].append(system)
        d["Method"].append(method)
  d = pd.DataFrame(private_copy)
  plot = sns.tsplot(
    x="Num. Workers", y="FPS", hue="Method", data=d, marker="o"
  )
  plot.xaxis.set_major_formatter(ticker.EngFormatter())
  plot.yaxis.set_major_formatter(ticker.EngFormatter())
  plot.legend(fontsize=9)
  plot.set_title(f"{env} throughput, {system}")
  plot.set_xlabel("Num. Workers")
  frame_skip = {"Atari": 4, "Mujoco": 5}[env]
  plot.set_ylabel(f"FPS, frameskip = {frame_skip}")
  plot.get_figure().savefig(f"{env}_{system}.{suffix}")
  plot.get_figure().clf()
//  [python]
def parse_table(env: str, system: str, suffix: str) -> None:
  private_copy = {
    "Num. Workers": [],
    "FPS": [],
    "Env": [],
    "System": [],
    "Method": [],
  }
  sep = f"<!-- {env} - {system} -->"
  raw = open("README.md").read().split(sep)[1].strip().splitlines()
  worker_num = list(map(int, raw[0].split("|")[2:-1]))
  for line in raw[2:]:
    line = line.split("|")[1:-1]
    method = line.pop(0).strip()
    for w, f in zip(worker_num, line):
      for d in [data, private_copy]:
        d["Num. Workers"].append(w)
        d["FPS"].append(None if f.strip() == "/" else float(f))
        d["Env"].append(env)
        d["System"].append(system)
        d["Method"].append(method)
  d = pd.DataFrame(private_copy)
  plot = sns.lineplot(
    x="Num. Workers", y="FPS", hue="Method", data=d, marker="o"
  )
  plot.xaxis.set_major_formatter(ticker.EngFormatter())
  plot.yaxis.set_major_formatter(ticker.EngFormatter())
  plot.legend(fontsize=9)
  plot.set_title(f"{env} throughput, {system}")
  plot.set_xlabel("Num. Workers")
  frame_skip = {"Atari": 4, "Mujoco": 5}[env]
  plot.set_ylabel(f"FPS, frameskip = {frame_skip}")
  plot.get_figure().savefig(f"{env}_{system}.{suffix}")
  plot.get_figure().clf()
//  [python]
def test_RanksComparatorPlotter_flow(fig_test, fig_ref, untied):
    test_ax = fig_test.subplots()

    rank0 = agg.RankResult("test", ["a", "b"], [1, 1], {})
    rank1 = agg.RankResult("test", ["a", "b"], [1, 1], {})
    rcmp = ranks_cmp.mkrank_cmp(rank0, rank1)

    rcmp.plot.flow(ax=test_ax, untied=untied)

    # EXPECTED
    exp_ax = fig_ref.subplots()

    expected = pd.DataFrame.from_dict(
        {
            "test_1": {"a": 1, "b": 2 if untied else 1},
            "test_2": {"a": 1, "b": 2 if untied else 1},
        }
    )
    expected.columns.name = "Method"
    expected.index.name = "Alternatives"

    s.tsplot(data=expected.T, estimator=None, err_style="ci_band", interpolate=False, ax=exp_ax)
    exp_ax.grid(alpha=0.3)

    exp_ax.set_ylabel(ranks_cmp.RANKS_LABELS[untied])
//  [python]
def test_RanksComparatorPlotter_flow(fig_test, fig_ref, untied):
    test_ax = fig_test.subplots()

    rank0 = agg.RankResult("test", ["a", "b"], [1, 1], {})
    rank1 = agg.RankResult("test", ["a", "b"], [1, 1], {})
    rcmp = ranks_cmp.mkrank_cmp(rank0, rank1)

    rcmp.plot.flow(ax=test_ax, untied=untied)

    # EXPECTED
    exp_ax = fig_ref.subplots()

    expected = pd.DataFrame.from_dict(
        {
            "test_1": {"a": 1, "b": 2 if untied else 1},
            "test_2": {"a": 1, "b": 2 if untied else 1},
        }
    )
    expected.columns.name = "Method"
    expected.index.name = "Alternatives"

    sns.lineplot(data=expected.T, estimator=None, sort=False, ax=exp_ax)
    exp_ax.grid(alpha=0.3)

    exp_ax.set_ylabel(ranks_cmp.RANKS_LABELS[untied])
//  [python]
def benchmark(suffix: str) -> None:
  global data
  reset_data()
  for env in ["Atari", "Mujoco"]:
    for system in ["Laptop", "Workstation", "TPU-VM", "DGX-A100"]:
      parse_table(env, system, suffix)
  data = pd.DataFrame(data)
  print(data.groupby(["Env", "Method", "System"]).max())

  def mapping(x, y, **kwargs):
    plot = sns.pointplot(x=x, y=y, **kwargs)
    plot.xaxis.set_major_formatter(ticker.EngFormatter())
    plot.yaxis.set_major_formatter(ticker.EngFormatter())

  g = sns.FacetGrid(
    data,
    row="Env",
    col="System",
    hue="Method",
    height=3,
    aspect=1.6,
    sharex=False,
    sharey=False,
  )
  g.map(mapping, "Num. Workers", "FPS", marker="o")
  g.add_legend(bbox_to_anchor=(0.52, 1.02), ncol=6)
  axes = g.axes.flatten()
  alphabet = "abcdefgh"
  for ax, i in zip(axes, alphabet):
    env, system = ax.get_title().split("|")
    env = env.split("=")[-1].strip()
    system = system.split("=")[-1].strip()
    ax.set_title(f"({i}) {env} throughput, {system}")
    if ax.get_ylabel():
      frame_skip = {"Atari": 4, "Mujoco": 5}[env]
      ax.set_ylabel(f"FPS, frameskip = {frame_skip}")
  g.savefig(f"throughput.{suffix}")
//  [python]
def benchmark(suffix: str) -> None:
  global data
  reset_data()
  for env in ["Atari", "Mujoco"]:
    for system in ["Laptop", "Workstation", "TPU-VM", "DGX-A100"]:
      parse_table(env, system, suffix)
  data = pd.DataFrame(data)
  print(data.groupby(["Env", "Method", "System"]).max())

  def mapping(x, y, **kwargs):
    plot = sns.lineplot(x=x, y=y, **kwargs)
    plot.xaxis.set_major_formatter(ticker.EngFormatter())
    plot.yaxis.set_major_formatter(ticker.EngFormatter())

  g = sns.FacetGrid(
    data,
    row="Env",
    col="System",
    hue="Method",
    height=3,
    aspect=1.6,
    sharex=False,
    sharey=False,
  )
  g.map(mapping, "Num. Workers", "FPS", marker="o")
  g.add_legend(bbox_to_anchor=(0.52, 1.02), ncol=6)
  axes = g.axes.flatten()
  alphabet = "abcdefgh"
  for ax, i in zip(axes, alphabet):
    env, system = ax.get_title().split("|")
    env = env.split("=")[-1].strip()
    system = system.split("=")[-1].strip()
    ax.set_title(f"({i}) {env} throughput, {system}")
    if ax.get_ylabel():
      frame_skip = {"Atari": 4, "Mujoco": 5}[env]
      ax.set_ylabel(f"FPS, frameskip = {frame_skip}")
  g.savefig(f"throughput.{suffix}")
//  [python]
def visualize_timeseries(allInfos):
    plt.figure(figsize=FIGSIZE.WIDE_M)
    sns.set(style="ticks", palette="pastel")

    data = {
        "segment": [],
        "accumulated": [],
        "config": [],
    }

    max_segment_number = 0

    for conf, infos in allInfos.items():
        key = fixname(f"{conf.scheduler} - {conf.fec}".upper())

        run_max_seg_no = 0

        for info in infos:
            if len(info.interrupted_segments) > 0 and max(info.interrupted_segments) > run_max_seg_no:
                run_max_seg_no = max(info.interrupted_segments)

        if run_max_seg_no > max_segment_number:
            max_segment_number = run_max_seg_no

        for run in infos:
            accum = 0
            for segment in range(run_max_seg_no + 1):
                if segment in run.interrupted_segments:
                    accum += 1
                data["segment"].append(segment)
                data["accumulated"].append(accum)
                data["config"].append(key)

    df = pandas.DataFrame.from_dict(data)
    ax = sns.tsplot(data=df, time="segment", value="accumulated", unit="config")

    ax.set(xlabel='Segment Number', ylabel='Accumulated Interruptions')
    #plt.yticks(range(1, max_interruptions + 1))
    plt.savefig("vis-interrupt-distribution." + FORMAT)
//  [python]
def visualize_timeseries(allInfos):
    plt.figure(figsize=FIGSIZE.WIDE_M)
    sns.set(style="ticks", palette="pastel")

    data = {
        "segment": [],
        "accumulated": [],
        "config": [],
    }

    max_segment_number = 0

    for conf, infos in allInfos.items():
        key = fixname(f"{conf.scheduler} - {conf.fec}".upper())

        run_max_seg_no = 0

        for info in infos:
            if len(info.interrupted_segments) > 0 and max(info.interrupted_segments) > run_max_seg_no:
                run_max_seg_no = max(info.interrupted_segments)

        if run_max_seg_no > max_segment_number:
            max_segment_number = run_max_seg_no

        for run in infos:
            accum = 0
            for segment in range(run_max_seg_no + 1):
                if segment in run.interrupted_segments:
                    accum += 1
                data["segment"].append(segment)
                data["accumulated"].append(accum)
                data["config"].append(key)

    df = pandas.DataFrame.from_dict(data)
    ax = sns.lineplot(data=df, x="segment", y="accumulated", hue="config")

    ax.set(xlabel='Segment Number', ylabel='Accumulated Interruptions')
    #plt.yticks(range(1, max_interruptions + 1))
    plt.savefig("vis-interrupt-distribution." + FORMAT)
//  [python]
def visualize_bitrate(allInfos):
    plt.figure(figsize=FIGSIZE.WIDE_M)
    sns.set(style="ticks", palette=PALETTE_5)

    data = {
        'segment': [],
        'bitrate': [],
        'config': [],
    }

    bitrates = set()

    for conf, infos in allInfos.items():
        config = fixname(f"{conf.scheduler} - {conf.fec}".upper())

        for info in infos:
            for i, bitrate in enumerate(info.segment_bitrates):
                data['segment'].append(i)
                data['bitrate'].append(bitrate)
                data['config'].append(config)
            bitrates.update(info.segment_bitrates)

    df = pandas.DataFrame.from_dict(data)
    ax = sns.tsplot(data=df, time='segment', value='bitrate', unit='config',
                      palette='muted')

    ax.set(xlabel='Segment Number', ylabel='Bit Rate')
    plt.yticks(list(bitrates))
    ax.ticklabel_format(style='plain')
    plt.savefig("vis-abr." + FORMAT)
//  [python]
def visualize_bitrate(allInfos):
    plt.figure(figsize=FIGSIZE.WIDE_M)
    sns.set(style="ticks", palette=PALETTE_5)

    data = {
        'segment': [],
        'bitrate': [],
        'config': [],
    }

    bitrates = set()

    for conf, infos in allInfos.items():
        config = fixname(f"{conf.scheduler} - {conf.fec}".upper())

        for info in infos:
            for i, bitrate in enumerate(info.segment_bitrates):
                data['segment'].append(i)
                data['bitrate'].append(bitrate)
                data['config'].append(config)
            bitrates.update(info.segment_bitrates)

    df = pandas.DataFrame.from_dict(data)
    ax = sns.lineplot(data=df, x='segment', y='bitrate', hue='config',
                      palette='muted')

    ax.set(xlabel='Segment Number', ylabel='Bit Rate')
    plt.yticks(list(bitrates))
    ax.ticklabel_format(style='plain')
    plt.savefig("vis-abr." + FORMAT)
//  [python]
def method_comparison_plot(df, hue_var = 'method', style_var = 'method',
                          figsize = (15,4), ax = None, save_path=None):
        """ Produce plots comparing OTDD variants in terms of runtime and distance """
        if ax is None:
            fig, ax = plt.subplots(1, 2, figsize=figsize)

        lplot_args = {
            'hue': hue_var,
            'style': style_var,
            'data': df,
            'x': 'n',
            'markers': True
        }

        sns.pointplot(y='dist', ax= ax[0], **lplot_args)
        ax[0].legend(loc='center left', bbox_to_anchor=(1, 0.5), fontsize=16)
        ax[0].set_ylabel('Dataset Distance')
        ax[0].set_xlabel('Dataset Size')
        ax[0].set_xscale("log")
        ax[0].grid(True,which="both",ls="--",c='gray')

        sns.pointplot(y='time', ax= ax[1], **lplot_args)
        ax[1].legend(loc='center left', bbox_to_anchor=(1, 0.5), fontsize=16)
        ax[1].set_ylabel('Runtime (s)')
        ax[1].set_xlabel('Dataset Size')
        ax[1].set_xscale("log")
        ax[1].set_yscale("log")
        ax[1].grid(True,which="both",ls="--",c='gray')

        handles, labels = ax[1].get_legend_handles_labels()
        ax[1].get_legend().remove()

        plt.tight_layout()
        if save_path:
            plt.savefig(save_path + '.pdf',  dpi=300)
            plt.savefig(save_path + '.png',  dpi=300)
        plt.show()

        return ax
//  [python]
def method_comparison_plot(df, hue_var = 'method', style_var = 'method',
                          figsize = (15,4), ax = None, save_path=None):
        """ Produce plots comparing OTDD variants in terms of runtime and distance """
        if ax is None:
            fig, ax = plt.subplots(1, 2, figsize=figsize)

        lplot_args = {
            'hue': hue_var,
            'style': style_var,
            'data': df,
            'x': 'n',
            'markers': True
        }

        sns.lineplot(y='dist', ax= ax[0], **lplot_args)
        ax[0].legend(loc='center left', bbox_to_anchor=(1, 0.5), fontsize=16)
        ax[0].set_ylabel('Dataset Distance')
        ax[0].set_xlabel('Dataset Size')
        ax[0].set_xscale("log")
        ax[0].grid(True,which="both",ls="--",c='gray')

        sns.lineplot(y='time', ax= ax[1], **lplot_args)
        ax[1].legend(loc='center left', bbox_to_anchor=(1, 0.5), fontsize=16)
        ax[1].set_ylabel('Runtime (s)')
        ax[1].set_xlabel('Dataset Size')
        ax[1].set_xscale("log")
        ax[1].set_yscale("log")
        ax[1].grid(True,which="both",ls="--",c='gray')

        handles, labels = ax[1].get_legend_handles_labels()
        ax[1].get_legend().remove()

        plt.tight_layout()
        if save_path:
            plt.savefig(save_path + '.pdf',  dpi=300)
            plt.savefig(save_path + '.png',  dpi=300)
        plt.show()

        return ax
//  [python]
def visualize_timeseries_duration(allInfos):
    plt.figure(figsize=FIGSIZE.WIDE_M)
    sns.set(style="ticks", palette="pastel")

    data = {
        "segment": [],
        "accumulated": [],
        "config": [],
    }

    max_segment_number = 0

    for conf, infos in allInfos.items():
        key = fixname(f"{conf.scheduler} - {conf.fec}".upper())

        run_max_seg_no = 0

        for info in infos:
            if len(info.interrupted_segments) > 0 and max(info.interrupted_segments) > run_max_seg_no:
                run_max_seg_no = max(info.interrupted_segments)

        if run_max_seg_no > max_segment_number:
            max_segment_number = run_max_seg_no

        for run in infos:
            accum = 0
            durations = run.durations
            for segment in range(run_max_seg_no + 1):

                if segment in run.interrupted_segments:
                    accum += durations[0]
                    durations.pop(0)
                data["segment"].append(segment)
                data["accumulated"].append(accum)
                data["config"].append(key)

    df = pandas.DataFrame.from_dict(data)
    ax = sns.tsplot(data=df, time="segment", value="accumulated", unit="config")

    ax.set(xlabel='Segment Number', ylabel='Accumulated Duration (s)')
    #plt.yticks(range(1, max_interruptions + 1))
    plt.savefig("vis-interrupt-duration." + FORMAT)
//  [python]
def visualize_timeseries_duration(allInfos):
    plt.figure(figsize=FIGSIZE.WIDE_M)
    sns.set(style="ticks", palette="pastel")

    data = {
        "segment": [],
        "accumulated": [],
        "config": [],
    }

    max_segment_number = 0

    for conf, infos in allInfos.items():
        key = fixname(f"{conf.scheduler} - {conf.fec}".upper())

        run_max_seg_no = 0

        for info in infos:
            if len(info.interrupted_segments) > 0 and max(info.interrupted_segments) > run_max_seg_no:
                run_max_seg_no = max(info.interrupted_segments)

        if run_max_seg_no > max_segment_number:
            max_segment_number = run_max_seg_no

        for run in infos:
            accum = 0
            durations = run.durations
            for segment in range(run_max_seg_no + 1):

                if segment in run.interrupted_segments:
                    accum += durations[0]
                    durations.pop(0)
                data["segment"].append(segment)
                data["accumulated"].append(accum)
                data["config"].append(key)

    df = pandas.DataFrame.from_dict(data)
    ax = sns.lineplot(data=df, x="segment", y="accumulated", hue="config")

    ax.set(xlabel='Segment Number', ylabel='Accumulated Duration (s)')
    #plt.yticks(range(1, max_interruptions + 1))
    plt.savefig("vis-interrupt-duration." + FORMAT)
//  [python]
def plot_temp_metric_line(data, x, y, hue=None, style=None, palette=COLOR_PALETTE, **kwa_kind):
	"""Plot a seaborn line chart to show the distance metric result."""
	plt.figure()
	fig = sns.tsplot(x=x, y=y, hue=hue, style=hue, data=data, palette=palette, ci="sd")
	fig.set_ylim(-0.04, 1.04)
	# remove legend header
	handles, labels = fig.get_legend_handles_labels()
	fig.legend(handles=handles[0:], labels=labels[0:])
	return fig
//  [python]
def plot_temp_metric_line(data, x, y, hue=None, style=None, palette=COLOR_PALETTE, **kwa_kind):
	"""Plot a seaborn line chart to show the distance metric result."""
	plt.figure()
	fig = sns.lineplot(x=x, y=y, hue=hue, style=hue, data=data, palette=palette, ci="sd")
	fig.set_ylim(-0.04, 1.04)
	# remove legend header
	handles, labels = fig.get_legend_handles_labels()
	fig.legend(handles=handles[0:], labels=labels[0:])
	return fig
//  [python]
def plot_clap_sync(data, x, y, hue, style=None, palette=COLOR_PALETTE, **kwa_kind):
	plt.figure()
	fig = sns.tsplot(data=data, time=x, value=y, unit=hue, condition=style, color=palette)
	# remove legend header
	handles, labels = fig.get_legend_handles_labels()
	fig.legend(handles=handles[0:], labels=labels[0:])
	return fig
//  [python]
def plot_clap_sync(data, x, y, hue, style=None, palette=COLOR_PALETTE, **kwa_kind):
	plt.figure()
	fig = sns.lineplot(data=data, x=x, y=y, hue=hue, style=style, palette=palette)
	# remove legend header
	handles, labels = fig.get_legend_handles_labels()
	fig.legend(handles=handles[0:], labels=labels[0:])
	return fig
//  [python]
def plot_temp_metric_bin_mul_line(data, x, y, hue=None, style=None, palette=COLOR_PALETTE, **kwa_kind):
	"""Plot a seaborn line chart to show the distance metric result for both binary and multiclass
	predictions."""
	plt.figure()
	fig = sns.tsplot(x=x, y=y, hue=hue, style=hue, data=data, palette=palette, ci=None)
	fig.set_ylim(0, 1)
	# remove legend header
	handles, labels = fig.get_legend_handles_labels()
	fig.legend(handles=handles[0:], labels=labels[0:], loc="lower center")
	return fig
//  [python]
def plot_temp_metric_bin_mul_line(data, x, y, hue=None, style=None, palette=COLOR_PALETTE, **kwa_kind):
	"""Plot a seaborn line chart to show the distance metric result for both binary and multiclass
	predictions."""
	plt.figure()
	fig = sns.lineplot(x=x, y=y, hue=hue, style=hue, data=data, palette=palette, ci=None)
	fig.set_ylim(0, 1)
	# remove legend header
	handles, labels = fig.get_legend_handles_labels()
	fig.legend(handles=handles[0:], labels=labels[0:], loc="lower center")
	return fig