Adapts labeling and auditing for single-session flow. Removes unused functionality throughout.

- Removes full model caching.
- Cleans up comments_grouped_full_topic_cat to system_preds_df; pre-processes data and renames+refactors merging operations to avoid confusion.
- Removes unused functionality (personal clustering, comparing against others' performance, nearest neighbor search).
- Moves constant data to data/input/ directory.

Adds automatically generated usernames. Removes username selection and shared user store. Removes Results and Study Links views. Removes AppOld component.

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+.DS_Store
+data/
+data.zip
+test_nbs/

audit_utils.py

@@ -40,66 +40,48 @@ module_dir = "./"
 perf_dir = f"data/perf/"
 
 # # TEMP reset
-# with open(os.path.join(module_dir, "./data/all_model_names.pkl"), "wb") as f:
-#     all_model_names = []
-#     pickle.dump(all_model_names, f)
 # with open(f"./data/users_to_models.pkl", "wb") as f:
 #     users_to_models = {}
 #     pickle.dump(users_to_models, f)
 
-
-with open(os.path.join(module_dir, "data/ids_to_comments.pkl"), "rb") as f:
+with open(os.path.join(module_dir, "data/input/ids_to_comments.pkl"), "rb") as f:
     ids_to_comments = pickle.load(f)
-with open(os.path.join(module_dir, "data/comments_to_ids.pkl"), "rb") as f:
+with open(os.path.join(module_dir, "data/input/comments_to_ids.pkl"), "rb") as f:
     comments_to_ids = pickle.load(f)
-
-all_model_names = sorted([name for name in os.listdir(os.path.join(perf_dir)) if os.path.isdir(os.path.join(perf_dir, name))])
-comments_grouped_full_topic_cat = pd.read_pickle("data/comments_grouped_full_topic_cat2_persp.pkl")
-sys_eval_df = pd.read_pickle(os.path.join(module_dir, "data/split_data/sys_eval_df.pkl"))
-train_df = pd.read_pickle(os.path.join(module_dir, "data/split_data/train_df.pkl"))
+system_preds_df = pd.read_pickle("data/input/system_preds_df.pkl")
+sys_eval_df = pd.read_pickle(os.path.join(module_dir, "data/input/split_data/sys_eval_df.pkl"))
+train_df = pd.read_pickle(os.path.join(module_dir, "data/input/split_data/train_df.pkl"))
 train_df_ids = train_df["item_id"].unique().tolist()
-model_eval_df = pd.read_pickle(os.path.join(module_dir, "data/split_data/model_eval_df.pkl"))
-ratings_df_full = pd.read_pickle(os.path.join(module_dir, "data/ratings_df_full.pkl"))
-
-worker_info_df = pd.read_pickle("./data/worker_info_df.pkl")
+model_eval_df = pd.read_pickle(os.path.join(module_dir, "data/input/split_data/model_eval_df.pkl"))
+ratings_df_full = pd.read_pickle(os.path.join(module_dir, "data/input/ratings_df_full.pkl"))
+worker_info_df = pd.read_pickle("./data/input/worker_info_df.pkl")
 
 with open(f"./data/users_to_models.pkl", "rb") as f:
     users_to_models = pickle.load(f)
 
-with open("data/perf_1000_topics.pkl", "rb") as f:
-    perf_1000_topics = pickle.load(f)
-with open("data/perf_1000_tox_cat.pkl", "rb") as f:
-    perf_1000_tox_cat = pickle.load(f)
-with open("data/perf_1000_tox_severity.pkl", "rb") as f:
-    perf_1000_tox_severity = pickle.load(f)
-with open("data/user_perf_metrics.pkl", "rb") as f:
-    user_perf_metrics = pickle.load(f)
-
-topic_ids = comments_grouped_full_topic_cat.topic_id
-topics = comments_grouped_full_topic_cat.topic
+topic_ids = system_preds_df.topic_id
+topics = system_preds_df.topic
 topic_ids_to_topics = {topic_ids[i]: topics[i] for i in range(len(topic_ids))}
 topics_to_topic_ids = {topics[i]: topic_ids[i] for i in range(len(topic_ids))}
-unique_topics_ids = sorted(comments_grouped_full_topic_cat.topic_id.unique())
+unique_topics_ids = sorted(system_preds_df.topic_id.unique())
 unique_topics = [topic_ids_to_topics[topic_id] for topic_id in range(len(topic_ids_to_topics) - 1)]
 
 def get_toxic_threshold():
     return TOXIC_THRESHOLD
 
-def get_all_model_names(user=None):
-    if (user is None) or (user not in users_to_models):
-        all_model_names = sorted([name for name in os.listdir(os.path.join(perf_dir)) if os.path.isdir(os.path.join(perf_dir, name))])
-        return all_model_names
-    else:
-        # Fetch the user's models
-        user_models = users_to_models[user]
-        user_models.sort()
-        return user_models
+def get_user_model_names(user):
+    # Fetch the user's models
+    if user not in users_to_models:
+        users_to_models[user] = []
+    user_models = users_to_models[user]
+    user_models.sort()
+    return user_models
 
 def get_unique_topics():
     return unique_topics
 
 def get_large_clusters(min_n):
-    counts_df = comments_grouped_full_topic_cat.groupby(by=["topic_id"]).size().reset_index(name='counts')
+    counts_df = system_preds_df.groupby(by=["topic_id"]).size().reset_index(name='counts')
     counts_df = counts_df[counts_df["counts"] >= min_n]
     return [topic_ids_to_topics[t_id] for t_id in sorted(counts_df["topic_id"].tolist()[1:])]
 
@@ -137,32 +119,8 @@ readable_to_internal = {
 }
 internal_to_readable = {v: k for k, v in readable_to_internal.items()}
 
-# Embeddings for neighbor retrieval
-model_name = "paraphrase-MiniLM-L6-v2"
-model = SentenceTransformer(model_name)
-with open("./data/comments.pkl", "rb") as f:
-    comments = pickle.load(f)
-embeddings = torch.load("./data/embeddings/21_10_embeddings.pt")
-
-# Perspective API recalibration
-def recalib_v1(s):
-    # convert Perspective score to 0-4 toxicity score
-    # map 0 persp to 0 (not at all toxic); 0.5 persp to 1 (slightly toxic), 1.0 persp to 4 (extremely toxic)
-    if s < 0.5:
-        return (s * 2.)
-    else:
-        return ((s - 0.5) * 6.) + 1
-
-def recalib_v2(s):
-    # convert Perspective score to 0-4 toxicity score
-    # just 4x the perspective score 
-    return (s * 4.)
-
-comments_grouped_full_topic_cat["rating_avg_orig"] = comments_grouped_full_topic_cat["rating"]
-comments_grouped_full_topic_cat["rating"] = [recalib_v2(score) for score in comments_grouped_full_topic_cat["persp_score"].tolist()]
-
-def get_comments_grouped_full_topic_cat():
-    return comments_grouped_full_topic_cat
+def get_system_preds_df():
+    return system_preds_df
 
 ########################################
 # General utils
@@ -192,22 +150,6 @@ def my_bootstrap(vals, n_boot, alpha):
 
 ########################################
 # GET_AUDIT utils
-def other_users_perf(perf_metrics, metric, user_metric, alpha=0.95, n_boot=501):
-    ind = get_metric_ind(metric)
-    
-    metric_vals = [metric_vals[ind] for metric_vals in perf_metrics.values()]
-    metric_avg = np.median(metric_vals)
-    
-    # Future: use provided sample to perform bootstrap sampling
-    ci_1 = mne.stats.bootstrap_confidence_interval(np.array(metric_vals), ci=alpha, n_bootstraps=n_boot, stat_fun="median")
-    
-    bs_samples, ci = my_bootstrap(metric_vals, n_boot, alpha)
-    
-    # Get user's percentile
-    percentile = stats.percentileofscore(bs_samples, user_metric)
-    
-    return metric_avg, ci, percentile, metric_vals
-
 def plot_metric_histogram(metric, user_metric, other_metric_vals, n_bins=10):
     hist, bin_edges = np.histogram(other_metric_vals, bins=n_bins, density=False)
     data = pd.DataFrame({
@@ -239,395 +181,34 @@ def plot_metric_histogram(metric, user_metric, other_metric_vals, n_bins=10):
 
     return (bar + rule).interactive()
 
-def get_toxicity_severity_bins(perf_metric, user_df, other_dfs, bins=BINS, bin_labels=BIN_LABELS, ci=0.95, n_boot=501):
-    # Note: not using other_dfs anymore
-    y_user = []
-    y_other = []
-    used_bins = []
-    other_ci_low = []
-    other_ci_high = []
-    for severity_i in range(len(bin_labels)):
-        metric_others = [metrics[get_metric_ind(perf_metric)] for metrics in perf_1000_tox_severity[severity_i].values() if metrics[get_metric_ind(perf_metric)]]
-        ci_low, ci_high = mne.stats.bootstrap_confidence_interval(np.array(metric_others), ci=ci, n_bootstraps=n_boot, stat_fun='median')
-        metric_other = np.median(metric_others)
-        
-        cur_user_df = user_df[user_df["prediction_bin"] == severity_i]
-        y_true_user = cur_user_df.pred.to_numpy()  # user's label
-        y_pred = cur_user_df.rating_avg.to_numpy()  # system's label (avg)
-        
-        if len(y_true_user) > 0:
-            used_bins.append(bin_labels[severity_i])
-            metric_user = calc_metric_user(y_true_user, y_pred, perf_metric)
-            y_user.append(metric_user)
-            y_other.append(metric_other)
-            other_ci_low.append(ci_low)
-            other_ci_high.append(ci_high)
-            
-    return y_user, y_other, used_bins, other_ci_low, other_ci_high
-
-def get_topic_bins(perf_metric, user_df, other_dfs, n_topics, ci=0.95, n_boot=501):  
-    # Note: not using other_dfs anymore
-    y_user = []
-    y_other = []
-    used_bins = []
-    other_ci_low = []
-    other_ci_high = []
-    selected_topics = unique_topics_ids[1:(n_topics + 1)]
-    
-    for topic_id in selected_topics:
-        cur_topic = topic_ids_to_topics[topic_id]
-        metric_others = [metrics[get_metric_ind(perf_metric)] for metrics in perf_1000_topics[topic_id].values() if metrics[get_metric_ind(perf_metric)]]
-        ci_low, ci_high = mne.stats.bootstrap_confidence_interval(np.array(metric_others), ci=ci, n_bootstraps=n_boot, stat_fun='median')
-        metric_other = np.median(metric_others)
-        
-        cur_user_df = user_df[user_df["topic"] == cur_topic]
-        y_true_user = cur_user_df.pred.to_numpy()  # user's label
-        y_pred = cur_user_df.rating_avg.to_numpy()  # system's label (avg)
-        
-        if len(y_true_user) > 0:
-            used_bins.append(cur_topic)
-            metric_user = calc_metric_user(y_true_user, y_pred, perf_metric)
-            y_user.append(metric_user)
-            y_other.append(metric_other)
-            other_ci_low.append(ci_low)
-            other_ci_high.append(ci_high)
-            
-    return y_user, y_other, used_bins, other_ci_low, other_ci_high
-
-def calc_metric_user(y_true_user, y_pred, perf_metric):
-    if perf_metric == "MAE":
-        metric_user = mean_absolute_error(y_true_user, y_pred)
-
-    elif perf_metric == "MSE":
-        metric_user = mean_squared_error(y_true_user, y_pred)
-
-    elif perf_metric == "RMSE":            
-        metric_user = mean_squared_error(y_true_user, y_pred, squared=False)
-        
-    elif perf_metric == "avg_diff":
-        metric_user = np.mean(y_true_user - y_pred)
-    
-    return metric_user
-
-def get_toxicity_category_bins(perf_metric, user_df, other_dfs, threshold=0.5, ci=0.95, n_boot=501):
-    # Note: not using other_dfs anymore; threshold from pre-calculation is 0.5
-    cat_cols = ["is_profane_frac", "is_threat_frac", "is_identity_attack_frac", "is_insult_frac", "is_sexual_harassment_frac"]
-    cat_labels = ["Profanity", "Threats", "Identity Attacks", "Insults", "Sexual Harassment"]
-    y_user = []
-    y_other = []
-    used_bins = []
-    other_ci_low = []
-    other_ci_high = []
-    for i, cur_col_name in enumerate(cat_cols):
-        metric_others = [metrics[get_metric_ind(perf_metric)] for metrics in perf_1000_tox_cat[cur_col_name].values() if metrics[get_metric_ind(perf_metric)]]
-        ci_low, ci_high = mne.stats.bootstrap_confidence_interval(np.array(metric_others), ci=ci, n_bootstraps=n_boot, stat_fun='median')
-        metric_other = np.median(metric_others)
-        
-        # Filter to rows where a comment received an average label >= the provided threshold for the category
-        cur_user_df = user_df[user_df[cur_col_name] >= threshold]
-        y_true_user = cur_user_df.pred.to_numpy()  # user's label
-        y_pred = cur_user_df.rating_avg.to_numpy()  # system's label (avg)
-        
-        if len(y_true_user) > 0:
-            used_bins.append(cat_labels[i])
-            metric_user = calc_metric_user(y_true_user, y_pred, perf_metric)
-            y_user.append(metric_user)
-            y_other.append(metric_other)
-            other_ci_low.append(ci_low)
-            other_ci_high.append(ci_high)
-    
-    return y_user, y_other, used_bins, other_ci_low, other_ci_high
-
-def plot_class_cond_results(preds_df, breakdown_axis, perf_metric, other_ids, sort_bars, n_topics, worker_id="A"):
-    # Note: preds_df already has binned results
-    # Prepare dfs
-    user_df = preds_df[preds_df.user_id == worker_id].sort_values(by=["item_id"]).reset_index()
-    other_dfs = [preds_df[preds_df.user_id == other_id].sort_values(by=["item_id"]).reset_index() for other_id in other_ids]
-    
-    if breakdown_axis == "toxicity_severity":
-        y_user, y_other, used_bins, other_ci_low, other_ci_high = get_toxicity_severity_bins(perf_metric, user_df, other_dfs)
-    elif breakdown_axis == "topic":
-        y_user, y_other, used_bins, other_ci_low, other_ci_high = get_topic_bins(perf_metric, user_df, other_dfs, n_topics)
-    elif breakdown_axis == "toxicity_category":
-        y_user, y_other, used_bins, other_ci_low, other_ci_high = get_toxicity_category_bins(perf_metric, user_df, other_dfs)
-    
-    diffs = list(np.array(y_user) - np.array(y_other))
-        
-    # Generate bar chart
-    data = pd.DataFrame({
-        "metric_val": y_user + y_other,
-        "Labeler": ["You" for _ in range(len(y_user))] + ["Other users" for _ in range(len(y_user))],
-        "used_bins": used_bins + used_bins,
-        "diffs": diffs + diffs,
-        "lower_cis": y_user + other_ci_low,
-        "upper_cis": y_user + other_ci_high,
-    })
+# Generates the summary plot across all topics for the user
+def show_overall_perf(variant, error_type, cur_user, threshold=TOXIC_THRESHOLD, topic_vis_method="median"):
+    # Your perf (calculate using model and testset)    
+    with open(os.path.join(module_dir, f"data/preds_dfs/{variant}.pkl"), "rb") as f:
+        preds_df = pickle.load(f)
 
-    color_domain = ['You', 'Other users']
-    color_range = [YOUR_COLOR, OTHER_USERS_COLOR]
-    
-    base = alt.Chart()
-    chart_title=f"{internal_to_readable[breakdown_axis]} Results"
-    x_axis = alt.X("Labeler:O", sort=("You", "Other users"), title=None, axis=None)
-    y_axis = alt.Y("metric_val:Q", title=internal_to_readable[perf_metric])
-    if sort_bars:
-        col_content = alt.Column("used_bins:O", sort=alt.EncodingSortField(field="diffs", op="mean", order='descending'))
-    else:
-        col_content = alt.Column("used_bins:O")
-
-    if n_topics is not None and n_topics > 10:
-        # Change to horizontal bar chart
-        bar = base.mark_bar(lineBreak="_").encode(
-            y=x_axis,
-            x=y_axis,
-            color=alt.Color("Labeler:O", scale=alt.Scale(domain=color_domain, range=color_range)),
-            tooltip=[
-                alt.Tooltip('Labeler:O', title='Labeler'),
-                alt.Tooltip('metric_val:Q', title=perf_metric, format=".3f"),
-            ]
-        )
-        error_bars = base.mark_errorbar().encode(
-            y=x_axis,
-            x = alt.X("lower_cis:Q", title=internal_to_readable[perf_metric]),
-            x2 = alt.X2("upper_cis:Q", title=None),
-            tooltip=[
-              alt.Tooltip('lower_cis:Q', title='Lower CI', format=".3f"),
-              alt.Tooltip('upper_cis:Q', title='Upper CI', format=".3f"),
-            ]
-        )
-        combined = alt.layer(
-            bar, error_bars, data=data
-        ).facet(
-            row=col_content
-        ).properties(
-            title=chart_title,
-        ).interactive()
+    # Read from file
+    chart_dir = "./data/charts"
+    chart_file = os.path.join(chart_dir, f"{cur_user}_{variant}.pkl")
+    if os.path.isfile(chart_file):
+        with open(chart_file, "r") as f:
+            topic_overview_plot_json = json.load(f)
     else:
-        bar = base.mark_bar(lineBreak="_").encode(
-            x=x_axis,
-            y=y_axis,
-            color=alt.Color("Labeler:O", scale=alt.Scale(domain=color_domain, range=color_range)),
-            tooltip=[
-                alt.Tooltip('Labeler:O', title='Labeler'),
-                alt.Tooltip('metric_val:Q', title=perf_metric, format=".3f"),
-            ]
-        )
-        error_bars = base.mark_errorbar().encode(
-            x=x_axis,
-            y = alt.Y("lower_cis:Q", title=internal_to_readable[perf_metric]),
-            y2 = alt.Y2("upper_cis:Q", title=None),
-            tooltip=[
-              alt.Tooltip('lower_cis:Q', title='Lower CI', format=".3f"),
-              alt.Tooltip('upper_cis:Q', title='Upper CI', format=".3f"),
-            ]
-        )
-        combined = alt.layer(
-            bar, error_bars, data=data
-        ).facet(
-            column=col_content
-        ).properties(
-            title=chart_title,
-        ).interactive()
-
-    return combined
-
-# Generates the summary plot across all topics for the user
-def show_overall_perf(variant, error_type, cur_user, threshold=TOXIC_THRESHOLD, breakdown_axis=None, topic_vis_method="median"):
-    # Your perf (calculate using model and testset)
-    breakdown_axis = readable_to_internal[breakdown_axis]
-    
-    if breakdown_axis is not None:
-        with open(os.path.join(module_dir, f"data/preds_dfs/{variant}.pkl"), "rb") as f:
-            preds_df = pickle.load(f)
-
-        # Read from file
-        chart_dir = "./data/charts"
-        chart_file = os.path.join(chart_dir, f"{cur_user}_{variant}.pkl")
-        if os.path.isfile(chart_file):
-            with open(chart_file, "r") as f:
-                topic_overview_plot_json = json.load(f)
-        else:
-            preds_df_mod = preds_df.merge(comments_grouped_full_topic_cat, on="item_id", how="left", suffixes=('_', '_avg'))
-            if topic_vis_method == "median":  # Default
-                preds_df_mod_grp = preds_df_mod.groupby(["topic_", "user_id"]).median()
-            elif topic_vis_method == "mean":
-                preds_df_mod_grp = preds_df_mod.groupby(["topic_", "user_id"]).mean()
-            topic_overview_plot_json = plot_overall_vis(preds_df=preds_df_mod_grp, n_topics=200, threshold=threshold, error_type=error_type, cur_user=cur_user, cur_model=variant)
+        if topic_vis_method == "median":  # Default
+            preds_df_grp = preds_df.groupby(["topic", "user_id"]).median()
+        elif topic_vis_method == "mean":
+            preds_df_grp = preds_df.groupby(["topic", "user_id"]).mean()
+        topic_overview_plot_json = plot_overall_vis(preds_df=preds_df_grp, n_topics=200, threshold=threshold, error_type=error_type, cur_user=cur_user, cur_model=variant)
 
     return {
         "topic_overview_plot_json": json.loads(topic_overview_plot_json),
     }
 
-########################################
-# GET_CLUSTER_RESULTS utils
-def get_overall_perf3(preds_df, perf_metric, other_ids, worker_id="A"):    
-    # Prepare dataset to calculate performance
-    # Note: true is user and pred is system
-    y_true = preds_df[preds_df["user_id"] == worker_id].pred.to_numpy()
-    y_pred_user = preds_df[preds_df["user_id"] == worker_id].rating_avg.to_numpy()
-    
-    y_true_others = y_pred_others = [preds_df[preds_df["user_id"] == other_id].pred.to_numpy() for other_id in other_ids]
-    y_pred_others = [preds_df[preds_df["user_id"] == other_id].rating_avg.to_numpy() for other_id in other_ids]
-    
-    # Get performance for user's model and for other users
-    if perf_metric == "MAE":
-        user_perf = mean_absolute_error(y_true, y_pred_user)
-        other_perfs = [mean_absolute_error(y_true_others[i], y_pred_others[i]) for i in range(len(y_true_others))]
-    elif perf_metric == "MSE":
-        user_perf = mean_squared_error(y_true, y_pred_user)
-        other_perfs = [mean_squared_error(y_true_others[i], y_pred_others[i]) for i in range(len(y_true_others))]
-    elif perf_metric == "RMSE":
-        user_perf = mean_squared_error(y_true, y_pred_user, squared=False)
-        other_perfs = [mean_squared_error(y_true_others[i], y_pred_others[i], squared=False) for i in range(len(y_true_others))]
-    elif perf_metric == "avg_diff":
-        user_perf = np.mean(y_true - y_pred_user)
-        other_perfs = [np.mean(y_true_others[i] - y_pred_others[i]) for i in range(len(y_true_others))]
-    
-    other_perf = np.mean(other_perfs)  # average across all other users
-    return user_perf, other_perf
-
-def style_color_difference(row):
-    full_opacity_diff = 3.
-    pred_user_col = "Your predicted rating"
-    pred_other_col = "Other users' predicted rating"
-    pred_system_col = "Status-quo system rating"
-    diff_user = row[pred_user_col] - row[pred_system_col]
-    diff_other = row[pred_other_col] - row[pred_system_col]
-    red = "234, 133, 125"
-    green = "142, 205, 162"
-    bkgd_user = green if diff_user < 0 else red  # red if more toxic; green if less toxic
-    opac_user = min(abs(diff_user / full_opacity_diff), 1.)
-    bkgd_other = green if diff_other < 0 else red  # red if more toxic; green if less toxic
-    opac_other = min(abs(diff_other / full_opacity_diff), 1.)
-    return ["", f"background-color: rgba({bkgd_user}, {opac_user});", f"background-color: rgba({bkgd_other}, {opac_other});", "", ""]
-
-def display_examples_cluster(preds_df, other_ids, num_examples, sort_ascending, worker_id="A"):
-    user_df = preds_df[preds_df.user_id == worker_id].sort_values(by=["item_id"]).reset_index()
-    others_df = preds_df[preds_df.user_id == other_ids[0]]
-    for i in range(1, len(other_ids)):
-        others_df.append(preds_df[preds_df.user_id == other_ids[i]])
-        others_df.groupby(["item_id"]).mean()
-    others_df = others_df.sort_values(by=["item_id"]).reset_index()
-    
-    df = pd.merge(user_df, others_df, on="item_id", how="left", suffixes=('_user', '_other'))
-    df["Comment"] = df["comment_user"]
-    df["Your predicted rating"] = df["pred_user"]
-    df["Other users' predicted rating"] = df["pred_other"]
-    df["Status-quo system rating"] = df["rating_avg_user"]
-    df["Status-quo system std dev"] = df["rating_stddev_user"]
-    df = df[["Comment", "Your predicted rating", "Other users' predicted rating", "Status-quo system rating", "Status-quo system std dev"]]
-    
-    # Add styling
-    df = df.sort_values(by=['Status-quo system std dev'], ascending=sort_ascending)
-    n_to_sample = np.min([num_examples, len(df)])
-    df = df.sample(n=n_to_sample).reset_index(drop=True)
-    return df.style.apply(style_color_difference, axis=1).render()
-
-def calc_odds_ratio(df, comparison_group, toxic_threshold=1.5, worker_id="A", debug=False, smoothing_factor=1):
-    if comparison_group == "status_quo":
-        other_pred_col = "rating_avg"
-        # Get unique comments, but fetch average labeler rating
-        num_toxic_other = len(df[(df.user_id == "A") & (df[other_pred_col] >= toxic_threshold)]) + smoothing_factor
-        num_nontoxic_other = len(df[(df.user_id == "A") & (df[other_pred_col] < toxic_threshold)]) + smoothing_factor
-    elif comparison_group == "other_users":
-        other_pred_col = "pred"
-        num_toxic_other = len(df[(df.user_id != "A") & (df[other_pred_col] >= toxic_threshold)]) + smoothing_factor
-        num_nontoxic_other = len(df[(df.user_id != "A") & (df[other_pred_col] < toxic_threshold)]) + smoothing_factor
-        
-    num_toxic_user = len(df[(df.user_id == "A") & (df.pred >= toxic_threshold)]) + smoothing_factor
-    num_nontoxic_user = len(df[(df.user_id == "A") & (df.pred < toxic_threshold)]) + smoothing_factor
-    
-    toxic_ratio = num_toxic_user / num_toxic_other
-    nontoxic_ratio = num_nontoxic_user / num_nontoxic_other
-    odds_ratio = toxic_ratio / nontoxic_ratio
-    
-    if debug:
-        print(f"Odds ratio: {odds_ratio}")
-        print(f"num_toxic_user: {num_toxic_user}, num_nontoxic_user: {num_nontoxic_user}")
-        print(f"num_toxic_other: {num_toxic_other}, num_nontoxic_other: {num_nontoxic_other}")
-    
-    contingency_table = [[num_toxic_user, num_nontoxic_user], [num_toxic_other, num_nontoxic_other]]
-    odds_ratio, p_val = stats.fisher_exact(contingency_table, alternative='two-sided')
-    if debug:
-        print(f"Odds ratio: {odds_ratio}, p={p_val}")
-
-    return odds_ratio
-
-# Neighbor search
-def get_match(comment_inds, K=20, threshold=None, debug=False):
-    match_ids = []
-    rows = []
-    for i in comment_inds:
-        if debug:
-            print(f"\nComment: {comments[i]}")
-        query_embedding = model.encode(comments[i], convert_to_tensor=True)
-        hits = util.semantic_search(query_embedding, embeddings, score_function=util.cos_sim, top_k=K)
-        # print(hits[0])
-        for hit in hits[0]:
-            c_id = hit['corpus_id']
-            score = np.round(hit['score'], 3)
-            if threshold is None or score > threshold:
-                match_ids.append(c_id)
-                if debug:
-                    print(f"\t(ID={c_id}, Score={score}): {comments[c_id]}")
-                rows.append([c_id, score, comments[c_id]])
-    
-    df = pd.DataFrame(rows, columns=["id", "score", "comment"])
-    return match_ids
-
-def display_examples_auto_cluster(preds_df, cluster, other_ids, perf_metric, sort_ascending=True, worker_id="A", num_examples=10):
-    # Overall performance
-    topic_df = preds_df
-    topic_df = topic_df[topic_df["topic"] == cluster]
-    user_perf, other_perf = get_overall_perf3(topic_df, perf_metric, other_ids)
-    
-    user_direction = "LOWER" if user_perf < 0 else "HIGHER"
-    other_direction = "LOWER" if other_perf < 0 else "HIGHER"
-    print(f"Your ratings are on average {np.round(abs(user_perf), 3)} {user_direction} than the existing system for this cluster")
-    print(f"Others' ratings (based on {len(other_ids)} users) are on average {np.round(abs(other_perf), 3)} {other_direction} than the existing system for this cluster")
-        
-    # Display example comments
-    df = display_examples_cluster(preds_df, other_ids, num_examples, sort_ascending)
-    return df
-
-    
-# function to get results for a new provided cluster
-def display_examples_manual_cluster(preds_df, cluster_comments, other_ids, perf_metric, sort_ascending=True, worker_id="A"):
-    # Overall performance
-    cluster_df = preds_df[preds_df["comment"].isin(cluster_comments)]
-    user_perf, other_perf = get_overall_perf3(cluster_df, perf_metric, other_ids)
-    
-    user_direction = "LOWER" if user_perf < 0 else "HIGHER"
-    other_direction = "LOWER" if other_perf < 0 else "HIGHER"
-    print(f"Your ratings are on average {np.round(abs(user_perf), 3)} {user_direction} than the existing system for this cluster")
-    print(f"Others' ratings (based on {len(other_ids)} users) are on average {np.round(abs(other_perf), 3)} {other_direction} than the existing system for this cluster")
-        
-    user_df = preds_df[preds_df.user_id == worker_id].sort_values(by=["item_id"]).reset_index()
-    others_df = preds_df[preds_df.user_id == other_ids[0]]
-    for i in range(1, len(other_ids)):
-        others_df.append(preds_df[preds_df.user_id == other_ids[i]])
-        others_df.groupby(["item_id"]).mean()
-    others_df = others_df.sort_values(by=["item_id"]).reset_index()
-    
-    # Get cluster_comments
-    user_df = user_df[user_df["comment"].isin(cluster_comments)]
-    others_df = others_df[others_df["comment"].isin(cluster_comments)]
-    
-    df = pd.merge(user_df, others_df, on="item_id", how="left", suffixes=('_user', '_other'))
-    df["pred_system"] = df["rating_avg_user"]
-    df["pred_system_stddev"] = df["rating_stddev_user"]
-    df = df[["item_id", "comment_user", "pred_user", "pred_other", "pred_system", "pred_system_stddev"]]
-    
-    # Add styling
-    df = df.sort_values(by=['pred_system_stddev'], ascending=sort_ascending)
-    df = df.style.apply(style_color_difference, axis=1).render()
-    return df
-
 ########################################
 # GET_LABELING utils
-def create_example_sets(comments_df, n_label_per_bin, score_bins, keyword=None, topic=None):
+def create_example_sets(n_label_per_bin, score_bins, keyword=None, topic=None):
     # Restrict to the keyword, if provided
-    df = comments_df.copy()
+    df = system_preds_df.copy()
     if keyword != None:
         df = df[df["comment"].str.contains(keyword)]    
     
@@ -652,8 +233,8 @@ def create_example_sets(comments_df, n_label_per_bin, score_bins, keyword=None,
     
     return ex_to_label
 
-def get_grp_model_labels(comments_df, n_label_per_bin, score_bins, grp_ids):
-    df = comments_df.copy()
+def get_grp_model_labels(n_label_per_bin, score_bins, grp_ids):
+    df = system_preds_df.copy()
 
     train_df_grp = train_df[train_df["user_id"].isin(grp_ids)]
     train_df_grp_avg = train_df_grp.groupby(by=["item_id"]).median().reset_index()
@@ -689,14 +270,7 @@ def fetch_existing_data(model_name, last_label_i):
         with open(os.path.join(module_dir, perf_dir, f"{last_i}.pkl"), "rb") as f:
             mae, mse, rmse, avg_diff = pickle.load(f)
     else:
-        # Fetch results from trained model
-        with open(os.path.join(module_dir, f"./data/trained_models/{model_name}.pkl"), "rb") as f:
-            cur_model = pickle.load(f)
-            mae, mse, rmse, avg_diff = users_perf(cur_model)
-        # Cache results
-        os.mkdir(os.path.join(module_dir, perf_dir))
-        with open(os.path.join(module_dir, perf_dir, "1.pkl"), "wb") as f:
-            pickle.dump((mae, mse, rmse, avg_diff), f)
+        raise Exception(f"Model {model_name} does not exist")
     
     # Fetch previous user-provided labels
     ratings_prev = None
@@ -705,7 +279,16 @@ def fetch_existing_data(model_name, last_label_i):
             ratings_prev = pickle.load(f)
     return mae, mse, rmse, avg_diff, ratings_prev
 
-def train_updated_model(model_name, last_label_i, ratings, user, top_n=20, topic=None):
+# Main function called by server's `get_personalized_model` endpoint
+# Trains an updated model with the specified name, user, and ratings
+# Saves ratings, performance metrics, and pre-computed predictions to files
+# - model_name: name of the model to train
+# - last_label_i: index of the last label file (0 if none exists)
+# - ratings: dictionary of comments to ratings
+# - user: user name
+# - top_n: number of comments to train on (used when a set was held out for original user study)
+# - topic: topic to train on (used when tuning for a specific topic)
+def train_updated_model(model_name, last_label_i, ratings, user, top_n=None, topic=None, debug=False):
     # Check if there is previously-labeled data; if so, combine it with this data
     perf_dir = f"./data/perf/{model_name}"
     label_dir = f"./data/labels/{model_name}"
@@ -716,9 +299,8 @@ def train_updated_model(model_name, last_label_i, ratings, user, top_n=20, topic
     labeled_df = labeled_df[labeled_df["rating"] != -1]
 
     # Filter to top N for user study
-    if topic is None:
-        # labeled_df = labeled_df.head(top_n)
-        labeled_df = labeled_df.tail(top_n)
+    if (topic is None) and (top_n is not None):
+        labeled_df = labeled_df.head(top_n)
     else:
         # For topic tuning, need to fetch old labels
         if (last_label_i > 0):
@@ -729,29 +311,29 @@ def train_updated_model(model_name, last_label_i, ratings, user, top_n=20, topic
                 labeled_df_prev = labeled_df_prev[labeled_df_prev["rating"] != -1]
                 ratings.update(ratings_prev) # append old ratings to ratings
                 labeled_df = pd.concat([labeled_df_prev, labeled_df])
-
-    print("len ratings for training:", len(labeled_df))
-
-    cur_model, perf, _, _ = train_user_model(ratings_df=labeled_df)
-    
-    user_perf_metrics[model_name] = users_perf(cur_model)
-
-    mae, mse, rmse, avg_diff = user_perf_metrics[model_name]
-
-    cur_preds_df = get_preds_df(cur_model, ["A"], sys_eval_df=ratings_df_full)  # Just get results for user
-
+    if debug:
+        print("len ratings for training:", len(labeled_df))
     # Save this batch of labels
     with open(os.path.join(module_dir, label_dir, f"{last_label_i + 1}.pkl"), "wb") as f:
         pickle.dump(ratings, f)
 
-    # Save model results
+    # Train model
+    cur_model, _, _, _ = train_user_model(ratings_df=labeled_df)
+    
+    # Compute performance metrics
+    mae, mse, rmse, avg_diff = users_perf(cur_model)
+    # Save performance metrics
+    if not os.path.isdir(os.path.join(module_dir, perf_dir)):
+        os.mkdir(os.path.join(module_dir, perf_dir))
+    last_perf_i = len([name for name in os.listdir(os.path.join(module_dir, perf_dir)) if os.path.isfile(os.path.join(module_dir, perf_dir, name))])
+    with open(os.path.join(module_dir, perf_dir, f"{last_perf_i + 1}.pkl"), "wb") as f:
+        pickle.dump((mae, mse, rmse, avg_diff), f)
+
+    # Pre-compute predictions for full dataset
+    cur_preds_df = get_preds_df(cur_model, ["A"], sys_eval_df=ratings_df_full)
+    # Save pre-computed predictions
     with open(os.path.join(module_dir, f"./data/preds_dfs/{model_name}.pkl"), "wb") as f:
         pickle.dump(cur_preds_df, f)
-
-    if model_name not in all_model_names:
-        all_model_names.append(model_name)
-    with open(os.path.join(module_dir, "./data/all_model_names.pkl"), "wb") as f:
-        pickle.dump(all_model_names, f)
     
     # Handle user
     if user not in users_to_models:
@@ -761,22 +343,10 @@ def train_updated_model(model_name, last_label_i, ratings, user, top_n=20, topic
         with open(f"./data/users_to_models.pkl", "wb") as f:
             pickle.dump(users_to_models, f)
 
-    with open(os.path.join(module_dir, "./data/user_perf_metrics.pkl"), "wb") as f:
-        pickle.dump(user_perf_metrics, f)
-    with open(os.path.join(module_dir, f"./data/trained_models/{model_name}.pkl"), "wb") as f:
-        pickle.dump(cur_model, f)
-    
-    # Cache performance results
-    if not os.path.isdir(os.path.join(module_dir, perf_dir)):
-        os.mkdir(os.path.join(module_dir, perf_dir))
-    last_perf_i = len([name for name in os.listdir(os.path.join(module_dir, perf_dir)) if os.path.isfile(os.path.join(module_dir, perf_dir, name))])
-    with open(os.path.join(module_dir, perf_dir, f"{last_perf_i + 1}.pkl"), "wb") as f:
-        pickle.dump((mae, mse, rmse, avg_diff), f)
-
     ratings_prev = ratings
     return mae, mse, rmse, avg_diff, ratings_prev
 
-def format_labeled_data(ratings, worker_id="A", debug=False):    
+def format_labeled_data(ratings, worker_id="A"):    
     all_rows = []
     for comment, rating in ratings.items():
         comment_id = comments_to_ids[comment]
@@ -786,7 +356,7 @@ def format_labeled_data(ratings, worker_id="A", debug=False):
     df = pd.DataFrame(all_rows, columns=["user_id", "item_id", "rating"])
     return df
 
-def users_perf(model, sys_eval_df=sys_eval_df, avg_ratings_df=comments_grouped_full_topic_cat, worker_id="A"):
+def users_perf(model, sys_eval_df=sys_eval_df, worker_id="A"):
     # Load the full empty dataset
     sys_eval_comment_ids = sys_eval_df.item_id.unique().tolist()
     empty_ratings_rows = [[worker_id, c_id, 0] for c_id in sys_eval_comment_ids]
@@ -802,17 +372,17 @@ def users_perf(model, sys_eval_df=sys_eval_df, avg_ratings_df=comments_grouped_f
     user_item_preds = get_predictions_by_user_and_item(predictions)
     df["pred"] = df.apply(lambda row: user_item_preds[(row.user_id, row.item_id)] if (row.user_id, row.item_id) in user_item_preds else np.nan, axis=1)
 
-    df = df.merge(avg_ratings_df, on="item_id", how="left", suffixes=('_', '_avg'))
+    df = df.merge(system_preds_df, on="item_id", how="left", suffixes=('', '_sys'))
     df.dropna(subset = ["pred"], inplace=True)
-    df["rating_"] = df.rating_.astype("int32")
+    df["rating"] = df.rating.astype("int32")
 
     perf_metrics = get_overall_perf(df, "A") # mae, mse, rmse, avg_diff  
     return perf_metrics
 
 def get_overall_perf(preds_df, user_id):    
     # Prepare dataset to calculate performance
-    y_pred = preds_df[preds_df["user_id"] == user_id].rating_avg.to_numpy() # Assume system is just average of true labels
-    y_true = preds_df[preds_df["user_id"] == user_id].pred.to_numpy()
+    y_pred = preds_df[preds_df["user_id"] == user_id].rating_sys.to_numpy() # system's prediction
+    y_true = preds_df[preds_df["user_id"] == user_id].pred.to_numpy() # user's (predicted) ground truth
     
     # Get performance for user's model
     mae = mean_absolute_error(y_true, y_pred)
@@ -831,9 +401,8 @@ def get_predictions_by_user_and_item(predictions):
 # Pre-computes predictions for the provided model and specified users on the system-eval dataset
 # - model: trained model
 # - user_ids: list of user IDs to compute predictions for
-# - avg_ratings_df: dataframe of average ratings for each comment (pre-computed)
 # - sys_eval_df: dataframe of system eval labels (pre-computed)
-def get_preds_df(model, user_ids, avg_ratings_df=comments_grouped_full_topic_cat, sys_eval_df=sys_eval_df, bins=BINS):
+def get_preds_df(model, user_ids, sys_eval_df=sys_eval_df, bins=BINS):
     # Prep dataframe for all predictions we'd like to request
     start = time.time()
     sys_eval_comment_ids = sys_eval_df.item_id.unique().tolist()
@@ -857,9 +426,9 @@ def get_preds_df(model, user_ids, avg_ratings_df=comments_grouped_full_topic_cat
     df = empty_ratings_df.copy() # user_id, item_id, rating
     user_item_preds = get_predictions_by_user_and_item(predictions)
     df["pred"] = df.apply(lambda row: user_item_preds[(row.user_id, row.item_id)] if (row.user_id, row.item_id) in user_item_preds else np.nan, axis=1)
-    df = df.merge(avg_ratings_df, on="item_id", how="left", suffixes=('_', '_avg'))
+    df = df.merge(system_preds_df, on="item_id", how="left", suffixes=('', '_sys'))
     df.dropna(subset = ["pred"], inplace=True)
-    df["rating_"] = df.rating_.astype("int32")
+    df["rating"] = df.rating.astype("int32")
     
     # Get binned predictions (based on user prediction)
     df["prediction_bin"], out_bins = pd.cut(df["pred"], bins, labels=False, retbins=True)
@@ -925,46 +494,6 @@ def train_model(train_df, model_eval_df, model_type="SVD", sim_type=None, user_b
     
     return algo, perf
 
-def plot_train_perf_results2(model_name):
-    # Open labels
-    label_dir = f"./data/labels/{model_name}"
-    n_label_files = len([name for name in os.listdir(os.path.join(module_dir, label_dir)) if os.path.isfile(os.path.join(module_dir, label_dir, name))])
-    
-    all_rows = []
-    with open(os.path.join(module_dir, label_dir, f"{n_label_files}.pkl"), "rb") as f:
-        ratings = pickle.load(f)
-
-        labeled_df = format_labeled_data(ratings)
-        labeled_df = labeled_df[labeled_df["rating"] != -1]
-
-        # Iterate through batches of 5 labels
-        n_batches = int(np.ceil(len(labeled_df) / 5.))
-        for i in range(n_batches):
-            start = time.time()
-            n_to_sample = np.min([5 * (i + 1), len(labeled_df)])
-            cur_model, _, _, _ = train_user_model(ratings_df=labeled_df.head(n_to_sample))
-            mae, mse, rmse, avg_diff = users_perf(cur_model)
-            all_rows.append([n_to_sample, mae, "MAE"])
-            print(f"iter {i}: {time.time() - start}")
-        
-        print("all_rows", all_rows)
-        
-        df = pd.DataFrame(all_rows, columns=["n_to_sample", "perf", "metric"])
-        chart = alt.Chart(df).mark_line(point=True).encode(
-            x=alt.X("n_to_sample:Q", title="Number of Comments Labeled"),
-            y="perf",
-            color="metric",
-            tooltip=[
-                alt.Tooltip('n_to_sample:Q', title="Number of Comments Labeled"),
-                alt.Tooltip('metric:N', title="Metric"),
-                alt.Tooltip('perf:Q', title="Metric Value", format=".3f"),
-            ],
-        ).properties(
-            title=f"Performance over number of examples: {model_name}",
-            width=500,
-        )
-        return chart
-
 def plot_train_perf_results(model_name, mae):
     perf_dir = f"./data/perf/{model_name}"
     n_perf_files = len([name for name in os.listdir(os.path.join(module_dir, perf_dir)) if os.path.isfile(os.path.join(module_dir, perf_dir, name))])
@@ -996,7 +525,7 @@ def plot_train_perf_results(model_name, mae):
 
     plot_dim_width = 500
     domain_min = 0.0
-    domain_max = 1.0
+    domain_max = 2.0
     bkgd = alt.Chart(pd.DataFrame({
         "start": [PCT_90, PCT_75, domain_min],
         "stop": [domain_max, PCT_90, PCT_75],
@@ -1119,14 +648,14 @@ def get_decision(rating, threshold):
 
 def get_category(row, threshold=0.3):
     k_to_category = {
-        "is_profane_frac_": "Profanity", 
-        "is_threat_frac_": "Threat", 
-        "is_identity_attack_frac_": "Identity Attack", 
-        "is_insult_frac_": "Insult", 
-        "is_sexual_harassment_frac_": "Sexual Harassment",
+        "is_profane_frac": "Profanity", 
+        "is_threat_frac": "Threat", 
+        "is_identity_attack_frac": "Identity Attack", 
+        "is_insult_frac": "Insult", 
+        "is_sexual_harassment_frac": "Sexual Harassment",
     }
     categories = []
-    for k in ["is_profane_frac_", "is_threat_frac_", "is_identity_attack_frac_", "is_insult_frac_", "is_sexual_harassment_frac_"]:
+    for k in ["is_profane_frac", "is_threat_frac", "is_identity_attack_frac", "is_insult_frac", "is_sexual_harassment_frac"]:
         if row[k] > threshold:
             categories.append(k_to_category[k])
     
@@ -1139,20 +668,20 @@ def get_comment_url(row):
     return f"#{row['item_id']}/#comment"
 
 def get_topic_url(row):
-    return f"#{row['topic_']}/#topic"
+    return f"#{row['topic']}/#topic"
 
 # Plots overall results histogram (each block is a topic)
-def plot_overall_vis(preds_df, error_type, cur_user, cur_model, n_topics=None, bins=VIS_BINS, threshold=TOXIC_THRESHOLD, bin_step=0.05):
+def plot_overall_vis(preds_df, error_type, cur_user, cur_model, n_topics=None, bins=VIS_BINS, threshold=TOXIC_THRESHOLD, sys_col="rating_sys"):
     df = preds_df.copy().reset_index()
     
     if n_topics is not None:
-        df = df[df["topic_id_"] < n_topics]
+        df = df[df["topic_id"] < n_topics]
     
     df["vis_pred_bin"], out_bins = pd.cut(df["pred"], bins, labels=VIS_BINS_LABELS, retbins=True)
     df = df[df["user_id"] == "A"].sort_values(by=["item_id"]).reset_index()
-    df["system_label"] = [("toxic" if r > threshold else "non-toxic") for r in df["rating"].tolist()]
-    df["threshold"] = [threshold for r in df["rating"].tolist()]
-    df["key"] = [get_key(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+    df["system_label"] = [("toxic" if r > threshold else "non-toxic") for r in df[sys_col].tolist()]
+    df["threshold"] = [threshold for r in df[sys_col].tolist()]
+    df["key"] = [get_key(sys, user, threshold) for sys, user in zip(df[sys_col].tolist(), df["pred"].tolist())]
     df["url"] = df.apply(lambda row: get_topic_url(row), axis=1)
     
     # Plot sizing
@@ -1170,7 +699,7 @@ def plot_overall_vis(preds_df, error_type, cur_user, cur_model, n_topics=None, b
     # Main chart
     chart = alt.Chart(df).mark_square(opacity=0.8, size=mark_size, stroke="grey", strokeWidth=0.5).transform_window(
         groupby=['vis_pred_bin'],
-        sort=[{'field': 'rating'}],
+        sort=[{'field': sys_col}],
         id='row_number()',
         ignorePeers=True,
     ).encode(
@@ -1183,9 +712,9 @@ def plot_overall_vis(preds_df, error_type, cur_user, cur_model, n_topics=None, b
         ),
         href="url:N",
         tooltip = [
-            alt.Tooltip("topic_:N", title="Topic"),
+            alt.Tooltip("topic:N", title="Topic"),
             alt.Tooltip("system_label:N", title="System label"),
-            alt.Tooltip("rating:Q", title="System rating", format=".2f"),
+            alt.Tooltip(f"{sys_col}:Q", title="System rating", format=".2f"),
             alt.Tooltip("pred:Q", title="Your rating", format=".2f")
         ]
     )
@@ -1260,13 +789,13 @@ def plot_overall_vis(preds_df, error_type, cur_user, cur_model, n_topics=None, b
 
 # Plots cluster results histogram (each block is a comment), but *without* a model 
 # as a point of reference (in contrast to plot_overall_vis_cluster)
-def plot_overall_vis_cluster_no_model(preds_df, n_comments=None, bins=VIS_BINS, threshold=TOXIC_THRESHOLD, bin_step=0.05):
+def plot_overall_vis_cluster_no_model(preds_df, n_comments=None, bins=VIS_BINS, threshold=TOXIC_THRESHOLD, sys_col="rating_sys"):
     df = preds_df.copy().reset_index()
     
-    df["vis_pred_bin"], out_bins = pd.cut(df["rating"], bins, labels=VIS_BINS_LABELS, retbins=True)
-    df = df[df["user_id"] == "A"].sort_values(by=["rating"]).reset_index()
-    df["system_label"] = [("toxic" if r > threshold else "non-toxic") for r in df["rating"].tolist()]
-    df["key"] = [get_key_no_model(sys, threshold) for sys in df["rating"].tolist()]
+    df["vis_pred_bin"], out_bins = pd.cut(df[sys_col], bins, labels=VIS_BINS_LABELS, retbins=True)
+    df = df[df["user_id"] == "A"].sort_values(by=[sys_col]).reset_index()
+    df["system_label"] = [("toxic" if r > threshold else "non-toxic") for r in df[sys_col].tolist()]
+    df["key"] = [get_key_no_model(sys, threshold) for sys in df[sys_col].tolist()]
     df["category"] = df.apply(lambda row: get_category(row), axis=1)
     df["url"] = df.apply(lambda row: get_comment_url(row), axis=1)
     
@@ -1288,7 +817,7 @@ def plot_overall_vis_cluster_no_model(preds_df, n_comments=None, bins=VIS_BINS,
     # Main chart
     chart = alt.Chart(df).mark_square(opacity=0.8, size=mark_size, stroke="grey", strokeWidth=0.25).transform_window(
         groupby=['vis_pred_bin'],
-        sort=[{'field': 'rating'}],
+        sort=[{'field': sys_col}],
         id='row_number()',
         ignorePeers=True
     ).encode(
@@ -1302,8 +831,8 @@ def plot_overall_vis_cluster_no_model(preds_df, n_comments=None, bins=VIS_BINS,
         ),
         href="url:N",
         tooltip = [
-            alt.Tooltip("comment_:N", title="comment"),
-            alt.Tooltip("rating:Q", title="System rating", format=".2f"),
+            alt.Tooltip("comment:N", title="comment"),
+            alt.Tooltip(f"{sys_col}:Q", title="System rating", format=".2f"),
         ]
     )
     
@@ -1356,20 +885,20 @@ def plot_overall_vis_cluster_no_model(preds_df, n_comments=None, bins=VIS_BINS,
     return final_plot, df
 
 # Plots cluster results histogram (each block is a comment) *with* a model as a point of reference
-def plot_overall_vis_cluster(preds_df, error_type, n_comments=None, bins=VIS_BINS, threshold=TOXIC_THRESHOLD, bin_step=0.05):
-    df = preds_df.copy().reset_index(drop=True)
+def plot_overall_vis_cluster(preds_df, error_type, n_comments=None, bins=VIS_BINS, threshold=TOXIC_THRESHOLD, sys_col="rating_sys"):
+    df = preds_df.copy().reset_index()
     
     df["vis_pred_bin"], out_bins = pd.cut(df["pred"], bins, labels=VIS_BINS_LABELS, retbins=True)
-    df = df[df["user_id"] == "A"].sort_values(by=["rating"]).reset_index(drop=True)
-    df["system_label"] = [("toxic" if r > threshold else "non-toxic") for r in df["rating"].tolist()]
-    df["key"] = [get_key(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+    df = df[df["user_id"] == "A"].sort_values(by=[sys_col]).reset_index(drop=True)
+    df["system_label"] = [("toxic" if r > threshold else "non-toxic") for r in df[sys_col].tolist()]
+    df["key"] = [get_key(sys, user, threshold) for sys, user in zip(df[sys_col].tolist(), df["pred"].tolist())]
     df["category"] = df.apply(lambda row: get_category(row), axis=1)
     df["url"] = df.apply(lambda row: get_comment_url(row), axis=1)
     
     if n_comments is not None:
         n_to_sample = np.min([n_comments, len(df)])
         df = df.sample(n=n_to_sample)
-    
+        
     # Plot sizing
     domain_min = 0
     domain_max = 4
@@ -1384,7 +913,7 @@ def plot_overall_vis_cluster(preds_df, error_type, n_comments=None, bins=VIS_BIN
     # Main chart
     chart = alt.Chart(df).mark_square(opacity=0.8, size=mark_size, stroke="grey", strokeWidth=0.25).transform_window(
         groupby=['vis_pred_bin'],
-        sort=[{'field': 'rating'}],
+        sort=[{'field': sys_col}],
         id='row_number()',
         ignorePeers=True
     ).encode(
@@ -1397,8 +926,8 @@ def plot_overall_vis_cluster(preds_df, error_type, n_comments=None, bins=VIS_BIN
         ),
         href="url:N",
         tooltip = [
-            alt.Tooltip("comment_:N", title="comment"),
-            alt.Tooltip("rating:Q", title="System rating", format=".2f"),
+            alt.Tooltip("comment:N", title="comment"),
+            alt.Tooltip(f"{sys_col}:Q", title="System rating", format=".2f"),
             alt.Tooltip("pred:Q", title="Your rating", format=".2f"),
             alt.Tooltip("category:N", title="Potential toxicity categories")
         ]
@@ -1464,30 +993,26 @@ def plot_overall_vis_cluster(preds_df, error_type, n_comments=None, bins=VIS_BIN
 
     return final_plot, df
 
-def get_cluster_comments(df, error_type, threshold=TOXIC_THRESHOLD, worker_id="A", num_examples=50, use_model=True):    
+def get_cluster_comments(df, error_type, threshold=TOXIC_THRESHOLD, sys_col="rating_sys", use_model=True):    
     df["user_color"] = [get_user_color(user, threshold) for user in df["pred"].tolist()]  # get cell colors
-    df["system_color"] = [get_user_color(sys, threshold) for sys in df["rating"].tolist()]  # get cell colors
-    df["error_color"] = [get_system_color(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]  # get cell colors
-    df["error_type"] = [get_error_type(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]  # get error type in words
-    df["error_amt"] = [abs(sys - threshold) for sys in df["rating"].tolist()]  # get raw error
+    df["system_color"] = [get_user_color(sys, threshold) for sys in df[sys_col].tolist()]  # get cell colors
+    df["error_color"] = [get_system_color(sys, user, threshold) for sys, user in zip(df[sys_col].tolist(), df["pred"].tolist())]  # get cell colors
+    df["error_type"] = [get_error_type(sys, user, threshold) for sys, user in zip(df[sys_col].tolist(), df["pred"].tolist())]  # get error type in words
+    df["error_amt"] = [abs(sys - threshold) for sys in df[sys_col].tolist()]  # get raw error
     df["judgment"] = ["" for _ in range(len(df))]  # template for "agree" or "disagree" buttons
 
     if use_model:
         df = df.sort_values(by=["error_amt"], ascending=False) # surface largest errors first
     else:
         print("get_cluster_comments; not using model")
-        df = df.sort_values(by=["rating"], ascending=True)
+        df = df.sort_values(by=[sys_col], ascending=True)
 
     df["id"] = df["item_id"]
-    # df["comment"] already exists
-    df["comment"] = df["comment_"]
     df["toxicity_category"] = df["category"]
     df["user_rating"] = df["pred"]
     df["user_decision"] = [get_decision(rating, threshold) for rating in df["pred"].tolist()]
-    df["system_rating"] = df["rating"]
-    df["system_decision"] = [get_decision(rating, threshold) for rating in df["rating"].tolist()]
-    df["error_type"] = df["error_type"]
-    df = df.head(num_examples)
+    df["system_rating"] = df[sys_col]
+    df["system_decision"] = [get_decision(rating, threshold) for rating in df[sys_col].tolist()]
     df = df.round(decimals=2)
 
     # Filter to specified error type
@@ -1500,7 +1025,7 @@ def get_cluster_comments(df, error_type, threshold=TOXIC_THRESHOLD, worker_id="A
     elif error_type == "Both":
         df = df[(df["error_type"] == "System may be under-sensitive") | (df["error_type"] == "System may be over-sensitive")]
 
-    return df.to_json(orient="records")
+    return df
 
 # PERSONALIZED CLUSTERS utils
 def get_disagreement_comments(preds_df, mode, n=10_000, threshold=TOXIC_THRESHOLD):
@@ -1519,58 +1044,10 @@ def get_disagreement_comments(preds_df, mode, n=10_000, threshold=TOXIC_THRESHOL
     df = df.sort_values(by=["diff"], ascending=asc)
     df = df.head(n)
     
-    return df["comment_"].tolist(), df
-
-def get_personal_clusters(model, n=3):
-    personal_cluster_file = f"./data/personal_cluster_dfs/{model}.pkl"
-    if (os.path.isfile(personal_cluster_file)):
-        with open(personal_cluster_file, "rb") as f:
-            cluster_df = pickle.load(f)
-            cluster_df = cluster_df.sort_values(by=["topic_id"])
-            topics_under = cluster_df[cluster_df["error_type"] == "System may be under-sensitive"]["topic"].unique().tolist()
-            topics_under = topics_under[1:(n + 1)]
-            topics_over = cluster_df[cluster_df["error_type"] == "System may be over-sensitive"]["topic"].unique().tolist()
-            topics_over = topics_over[1:(n + 1)]
-            return topics_under, topics_over
-    else:
-        topics_under_top = []
-        topics_over_top = []
-        preds_df_file = f"./data/preds_dfs/{model}.pkl"
-        if (os.path.isfile(preds_df_file)):
-            with open(preds_df_file, "rb") as f:
-                preds_df = pickle.load(f)
-                preds_df_mod = preds_df.merge(comments_grouped_full_topic_cat, on="item_id", how="left", suffixes=('_', '_avg')).reset_index()
-                preds_df_mod = preds_df_mod[preds_df_mod["user_id"] == "A"]
-
-                comments_under, comments_under_df = get_disagreement_comments(preds_df_mod, mode="under-sensitive", n=1000)
-                if len(comments_under) > 0:
-                    topics_under = BERTopic(embedding_model="paraphrase-MiniLM-L6-v2").fit(comments_under)
-                    topics_under_top = topics_under.get_topic_info().head(n)["Name"].tolist()
-                    print("topics_under", topics_under_top)
-                    # Get topics per comment
-                    topics_assigned, _ = topics_under.transform(comments_under)
-                    comments_under_df["topic_id"] = topics_assigned
-                    cur_topic_ids = topics_under.get_topic_info().Topic
-                    topic_short_names = topics_under.get_topic_info().Name
-                    topic_ids_to_names = {cur_topic_ids[i]: topic_short_names[i] for i in range(len(cur_topic_ids))}
-                    comments_under_df["topic"] = [topic_ids_to_names[topic_id] for topic_id in comments_under_df["topic_id"].tolist()]
-
-                comments_over, comments_over_df = get_disagreement_comments(preds_df_mod, mode="over-sensitive", n=1000)
-                if len(comments_over) > 0:
-                    topics_over = BERTopic(embedding_model="paraphrase-MiniLM-L6-v2").fit(comments_over)
-                    topics_over_top = topics_over.get_topic_info().head(n)["Name"].tolist()
-                    print("topics_over", topics_over_top)
-                    # Get topics per comment
-                    topics_assigned, _ = topics_over.transform(comments_over)
-                    comments_over_df["topic_id"] = topics_assigned
-                    cur_topic_ids = topics_over.get_topic_info().Topic
-                    topic_short_names = topics_over.get_topic_info().Name
-                    topic_ids_to_names = {cur_topic_ids[i]: topic_short_names[i] for i in range(len(cur_topic_ids))}
-                    comments_over_df["topic"] = [topic_ids_to_names[topic_id] for topic_id in comments_over_df["topic_id"].tolist()]
-
-                cluster_df = pd.concat([comments_under_df, comments_over_df])
-                with open(f"./data/personal_cluster_dfs/{model}.pkl", "wb") as f:
-                    pickle.dump(cluster_df, f)
-
-                return topics_under_top, topics_over_top
-    return [], []
+    return df["comment"].tolist(), df
+
+def get_explore_df(n_examples, threshold):
+    df = system_preds_df.sample(n=n_examples)
+    df["system_decision"] = [get_decision(rating, threshold) for rating in df["rating"].tolist()]
+    df["system_color"] = [get_user_color(sys, threshold) for sys in df["rating"].tolist()]  # get cell colors
+    return df

indie_label_svelte/src/Auditing.svelte

@@ -51,8 +51,6 @@
     ]
     
     let personalized_models = [];
-    let breakdown_category;
-    let breakdown_categories = [];
     let systems = ["YouSocial comment toxicity classifier"]; // Only one system for now
     let clusters = [];
     let clusters_for_tuning = []
@@ -72,7 +70,6 @@
     let audit_type;
     if (scaffold_method == "fixed" || scaffold_method == "personal" || scaffold_method == "personal_group" || scaffold_method == "personal_test" || scaffold_method == "personal_cluster" || scaffold_method == "topic_train" || scaffold_method == "prompts") {
         audit_type = audit_types[1];
-        // audit_type = audit_types[0];
     } else {
         // No scaffolding mode or tutorial
         audit_type = audit_types[0];
@@ -112,7 +109,7 @@
         if (!personalized_models.includes(personalized_model)) {
             personalized_models.push(personalized_model);
         }
-
+        handleAuditButton();
         handleClusterButton(); // re-render cluster results
 	});
 
@@ -142,8 +139,6 @@
             .then((r) => r.text())
             .then(function (r_orig) {
                 let r = JSON.parse(r_orig);
-                breakdown_categories = r["breakdown_categories"];
-                breakdown_category = breakdown_categories[0];
                 personalized_models = r["personalized_models"];
                 if (use_group_model) {
                     let personalized_model_grp = r["personalized_model_grp"];
@@ -173,7 +168,6 @@
     async function getAudit() {
         let req_params = {
             pers_model: personalized_model,
-            breakdown_axis: breakdown_category,
             perf_metric: "avg_diff",
             breakdown_sort: "difference",
             n_topics: 10,
@@ -199,13 +193,11 @@
 		let req_params = {
 			cluster: topic,
 			topic_df_ids: [],
-			n_examples: 500, // TEMP
 			pers_model: personalized_model,
 			example_sort: "descending", // TEMP
 			comparison_group: "status_quo", // TEMP
 			search_type: "cluster",
 			keyword: "",
-			n_neighbors: 0,
             error_type: cur_error_type,
             use_model: use_model,
             scaffold_method: scaffold_method,
@@ -223,16 +215,13 @@
     <div>
         <div style="margin-top: 30px">
             <span class="head_3">Auditing</span>
-            <IconButton 
-                class="material-icons grey_button"
-                size="normal"
-                on:click={() => (show_audit_settings = !show_audit_settings)}
-            >
-                help_outline
-            </IconButton>
         </div>
         <div style="width: 80%">
+            {#if personalized_model}
             <p>In this section, we'll be auditing the content moderation system. Here, you’ll be aided by a personalized model that will help direct your attention towards potential problem areas in the model’s performance. This model isn’t meant to be perfect, but is designed to help you better focus on areas that need human review.</p>
+            {:else}
+            <p>Please first train your personalized model by following the steps in the "Labeling" tab (click the top left tab above).</p>
+            {/if}
         </div>
         
         {#if show_audit_settings}
@@ -282,11 +271,14 @@
                 </LayoutGrid>
             </div>
         </div>
+        {/if}
+        {#if personalized_model}
         <p>Current model: {personalized_model}</p>
         {/if}
     </div>
 
     <!-- 1: All topics overview -->
+    {#if personalized_model}
     {#if audit_type == audit_types[0]}
     <div class="audit_section">
         <div class="head_5">Overview of all topics</div>
@@ -440,7 +432,7 @@
         <div class="head_5">Finalize your current report</div>
         <p>Finally, review the report you've generated on the side panel and provide a brief summary of the problem you see. You may also list suggestions or insights into addressing this problem if you have ideas. This report will be directly used by the model developers to address the issue you've raised</p>
     </div>
-    
+    {/if}
 </div>
 
 <style>

indie_label_svelte/src/CommentTable.svelte

@@ -5,6 +5,8 @@
     import DataTable, { Head, Body, Row, Cell } from "@smui/data-table";
     import LinearProgress from '@smui/linear-progress';
 
+    import { model_chosen } from './stores/cur_model_store.js';
+
     export let mode;
     export let model_name;
     export let cur_user;
@@ -13,6 +15,7 @@
     let promise = Promise.resolve(null);
     let n_complete_ratings;
     let n_unsure_ratings;
+    let show_comments_labeled_count = false;
 
     function getCommentsToLabel(cur_mode, n) {
         if (cur_mode == "train") {
@@ -41,6 +44,7 @@
     }
 
     function handleTrainModelButton() {
+        getCompleteRatings();
         promise = getModel("train");
     }
 
@@ -88,7 +92,7 @@
         const text = await response.text();
         const data = JSON.parse(text);
         to_label = data["ratings_prev"];
-        console.log(data);
+        model_chosen.update((value) => model_name);
         return data;
     }
 </script>
@@ -214,12 +218,14 @@
     {/key}
 
     <div class="spacing_vert_40">
-        <Button on:click={handleTrainModelButton} variant="outlined" disabled={(!n_complete_ratings) || (n_complete_ratings < 40)}>
+        <Button on:click={handleTrainModelButton} variant="outlined">
             <Label>Train Model</Label>
         </Button>
+        {#if show_comments_labeled_count}
         <Button on:click={getCompleteRatings} variant="outlined">
             <Label>Get Number of Comments Labeled</Label>
         </Button>
+        {/if}
         <Button on:click={() => handleLoadCommentsButton(5)} variant="outlined">
             <Label>Fetch More Comments To Label</Label>
         </Button>

indie_label_svelte/src/Hunch.svelte

@@ -1,9 +1,7 @@
 <script lang="ts">
     import { onMount } from "svelte";
-	import IterativeClustering from "./IterativeClustering.svelte";
 	import Button, { Label } from "@smui/button";
 	import Textfield from '@smui/textfield';
-	import LinearProgress from "@smui/linear-progress";
 
     export let ind;
 	export let hunch;
@@ -32,7 +30,6 @@
 
 <div>
 	<div>
-		<!-- <h6>Hunch {ind + 1}</h6> -->
 		<h6>Topic:</h6>
 		{topic}
 	</div>
@@ -46,13 +43,6 @@
 			label="My current hunch is that..."
 		>
 		</Textfield>
-		<!-- <Button
-			on:click={handleTestOnExamples}
-			class="button_float_right spacing_vert"
-			variant="outlined"
-		>
-			<Label>Test on examples</Label>
-		</Button> -->
 	</div>
 
 	<div class="spacing_vert">
@@ -63,23 +53,7 @@
             <Label>Submit</Label>
         </Button>
     </div>
-
-	<!-- {#await example_block}
-        <div class="app_loading">
-            <LinearProgress indeterminate />
-        </div>
-    {:then} -->
-		<!-- {#if example_block}
-			<IterativeClustering clusters={clusters} ind={ind + 1} personalized_model={model} />
-		{/if} -->
-    <!-- {:catch error}
-        <p style="color: red">{error.message}</p>
-    {/await} -->
 </div>
 
 <style>
-	/* * {
-        z-index: 11;
-        overflow-x: hidden;
-    } */
 </style>

indie_label_svelte/src/HypothesisPanel.svelte

@@ -225,6 +225,7 @@
                 <Button 
                     on:click={() => (open = !open)}
                     color="primary"
+                    disabled={model == null}
                     style="float: right; padding: 10px; margin-right: 10px;"
                 >
                     {#if open}
@@ -239,6 +240,11 @@
         </div>
     </div>
 
+    {#if model == null}
+    <div class="panel_contents">
+        <p>You can start to author audit reports in this panel after you've trained your personalized model in the "Labeling" tab.</p>
+    </div>
+    {:else}
     <div class="panel_contents">  
         <!-- Drawer -->
         {#await promise}
@@ -491,7 +497,7 @@
             </div>
         </div>
     </div>
-
+    {/if}
     <!-- TEMP -->
     <!-- {#key model}
         <div>Model: {model}</div> 

indie_label_svelte/src/IterativeClustering.svelte

@@ -1,164 +0,0 @@
-<script>
-    import Section from "./Section.svelte";
-    import ClusterResults from "./ClusterResults.svelte";
-    import Button, { Label } from "@smui/button";
-    import Textfield from "@smui/textfield";
-    import LayoutGrid, { Cell } from "@smui/layout-grid";
-    import LinearProgress from "@smui/linear-progress";
-    import Chip, { Set, Text } from '@smui/chips';
-
-    export let clusters;
-    export let personalized_model;
-    export let evidence;
-    export let width_pct = 80;
-
-    let topic_df_ids = [];
-    let promise_iter_cluster = Promise.resolve(null);
-    let keyword = null;
-    let n_neighbors = null;
-    let cur_iter_cluster = null;
-    let history = [];
-
-    async function getIterCluster(search_type) {
-        let req_params = {
-            cluster: cur_iter_cluster,
-            topic_df_ids: topic_df_ids,
-            n_examples: 500, // TEMP
-            pers_model: personalized_model,
-            example_sort: "descending", // TEMP
-            comparison_group: "status_quo", // TEMP
-            search_type: search_type,
-            keyword: keyword,
-            n_neighbors: n_neighbors,
-        };
-        console.log("topic_df_ids", topic_df_ids);
-        let params = new URLSearchParams(req_params).toString();
-        const response = await fetch("./get_cluster_results?" + params);
-        const text = await response.text();
-        const data = JSON.parse(text);
-        // if (data["cluster_comments"] == null) {
-        //     return false
-        // }
-        topic_df_ids = data["topic_df_ids"];
-        return data;
-    }
-
-    function findCluster() {
-        promise_iter_cluster = getIterCluster("cluster");
-        history = history.concat("bulk-add cluster: " + cur_iter_cluster);
-    }
-
-    function findNeighbors() {
-        promise_iter_cluster = getIterCluster("neighbors");
-        history = history.concat("find " + n_neighbors + " neighbors");
-    }
-
-    function findKeywords() {
-        promise_iter_cluster = getIterCluster("keyword");
-        history = history.concat("keyword search: " + keyword);
-    }
-</script>
-
-<div>
-    <div>
-        <!-- <h6>Hunch {ind} examples</h6> -->
-        <div>
-            <h6>Search Settings</h6>
-            <!-- Start with cluster -->
-            <!-- <div class="">
-                <Section
-                    section_id="iter_cluster"
-                    section_title="Bulk-add cluster"
-                    section_opts={clusters}
-                    bind:value={cur_iter_cluster}
-                    width_pct={100}
-                />
-                <Button
-                    on:click={findCluster}
-                    variant="outlined"
-                    class="button_float_right"
-                    disabled={cur_iter_cluster == null}
-                >
-                    <Label>Search</Label>
-                </Button>
-            </div> -->
-
-            <!-- Manual keyword -->
-            <div class="spacing_vert">
-                <Textfield
-                    bind:value={keyword}
-                    label="Keyword search"
-                    variant="outlined"
-                    style="width: {width_pct}%"
-                />
-                <Button
-                    on:click={findKeywords}
-                    variant="outlined"
-                    class="button_float_right spacing_vert"
-                    disabled={keyword == null}
-                >
-                    <Label>Search</Label>
-                </Button>
-            </div>
-
-            <!-- Find neighbors of current set -->
-            <div class="spacing_vert">
-                <Textfield
-                    bind:value={n_neighbors}
-                    label="Number of neighbors to retrieve"
-                    type="number"
-                    min="1"
-                    max="50"
-                    variant="outlined"
-                    style="width: {width_pct}%"
-                />
-                <Button
-                    on:click={findNeighbors}
-                    variant="outlined"
-                    class="button_float_right spacing_vert"
-                    disabled={n_neighbors == null}
-                >
-                    <Label>Search</Label>
-                </Button>
-            </div>
-        </div>
-    </div>
-
-    {#await promise_iter_cluster}
-        <div class="app_loading" style="width: {width_pct}%">
-            <LinearProgress indeterminate />
-        </div>
-    {:then iter_cluster_results}
-        {#if iter_cluster_results}
-            {#if history.length > 0}
-                <div class="bold" style="padding-top:40px;">Search History</div>
-                <Set chips={history} let:chip choice>
-                    <Chip {chip}>
-                        <Text>{chip}</Text>
-                    </Chip>
-                </Set>
-            {/if}
-            {#if iter_cluster_results.cluster_comments != null}
-                <ClusterResults
-                    cluster={""}
-                    clusters={clusters}
-                    model={personalized_model}
-                    data={iter_cluster_results}
-                    show_vis={false}
-                    table_width_pct={80}
-                    bind:evidence={evidence}
-                    on:change
-                />
-            {:else}
-                <div class="bold" style="padding-top:40px;">
-                    No results found
-                </div>
-            {/if}
-        {/if}
-    {:catch error}
-        <p style="color: red">{error.message}</p>
-    {/await}
-</div>
-
-<style>
-</style>

indie_label_svelte/src/KeywordSearch.svelte

@@ -17,7 +17,6 @@
     let topic_df_ids = [];
     let promise_iter_cluster = Promise.resolve(null);
     let keyword = null;
-    let n_neighbors = null;
     let cur_iter_cluster = null;
     let history = [];
 
@@ -30,13 +29,11 @@
         let req_params = {
             cluster: cur_iter_cluster,
             topic_df_ids: topic_df_ids,
-            n_examples: 500, // TEMP
             pers_model: personalized_model,
             example_sort: "descending", // TEMP
             comparison_group: "status_quo", // TEMP
             search_type: search_type,
             keyword: keyword,
-            n_neighbors: n_neighbors,
             error_type: cur_error_type,
         };
         console.log("topic_df_ids", topic_df_ids);

indie_label_svelte/src/Labeling.svelte

@@ -17,7 +17,7 @@
     let label_modes = [
         "Create a new model",
         "Edit an existing model",
-        "Tune your model for a topic area",
+        // "Tune your model for a topic area",
         // "Set up a group-based model",
     ];
     
@@ -33,6 +33,7 @@
     } else if (req_label_mode == 1) {
         label_mode = label_modes[1];
     } else if (req_label_mode == 2) {
+        // Unused; previous topic-based mode
         label_mode = label_modes[2];
     } else if (req_label_mode == 3) {
         // Unused; previous group-based mode

server.py

@@ -37,7 +37,6 @@ def home(path):
 
 ########################################
 # ROUTE: /AUDIT_SETTINGS
-comments_grouped_full_topic_cat = pd.read_pickle("data/comments_grouped_full_topic_cat2_persp.pkl")
 
 @app.route("/audit_settings")
 def audit_settings(debug=DEBUG):
@@ -47,13 +46,10 @@ def audit_settings(debug=DEBUG):
 
     # Assign user ID if none is provided (default case)
     if user == "null":
-        if debug:
-            user = "DemoUser"
-        else:
-            # Generate random two-word user ID
-            user = fw.generate(2, separator="_")
+        # Generate random two-word user ID
+        user = fw.generate(2, separator="_")
 
-    user_models = utils.get_all_model_names(user)
+    user_models = utils.get_user_model_names(user)
     grp_models = [m for m in user_models if m.startswith(f"model_{user}_group_")]
 
     clusters = utils.get_unique_topics()
@@ -76,19 +72,6 @@ def audit_settings(debug=DEBUG):
             "options": [{"value": i, "text": cluster} for i, cluster in enumerate(clusters)],
         },]
 
-    if scaffold_method == "personal_cluster":
-        cluster_model = user_models[0]
-        personal_cluster_file = f"./data/personal_cluster_dfs/{cluster_model}.pkl"
-        if os.path.isfile(personal_cluster_file) and cluster_model != "":
-            print("audit_settings", personal_cluster_file, cluster_model)
-            topics_under_top, topics_over_top = utils.get_personal_clusters(cluster_model)
-            pers_cluster = topics_under_top + topics_over_top
-            pers_cluster_options = {
-                "label": "Personalized clusters",
-                "options": [{"value": i, "text": cluster} for i, cluster in enumerate(pers_cluster)],
-            }
-            clusters_options.insert(0, pers_cluster_options)
-
     clusters_for_tuning = utils.get_large_clusters(min_n=150)
     clusters_for_tuning_options = [{"value": i, "text": cluster} for i, cluster in enumerate(clusters_for_tuning)]  # Format for Svelecte UI element
 
@@ -96,7 +79,6 @@ def audit_settings(debug=DEBUG):
         "personalized_models": user_models,
         "personalized_model_grp": grp_models,
         "perf_metrics": ["Average rating difference", "Mean Absolute Error (MAE)", "Root Mean Squared Error (RMSE)", "Mean Squared Error (MSE)"],
-        "breakdown_categories": ['Topic', 'Toxicity Category', 'Toxicity Severity'],
         "clusters": clusters_options,
         "clusters_for_tuning": clusters_for_tuning_options,
         "user": user,
@@ -109,30 +91,21 @@ def audit_settings(debug=DEBUG):
 @app.route("/get_audit")
 def get_audit():
     pers_model = request.args.get("pers_model")
-    perf_metric = request.args.get("perf_metric")
-    breakdown_axis = request.args.get("breakdown_axis")
-    breakdown_sort = request.args.get("breakdown_sort")
-    n_topics = int(request.args.get("n_topics"))
     error_type = request.args.get("error_type")
     cur_user = request.args.get("cur_user")
     topic_vis_method = request.args.get("topic_vis_method") 
     if topic_vis_method == "null":
         topic_vis_method = "median"
 
-    if breakdown_sort == "difference":
-        sort_class_plot = True
-    elif breakdown_sort == "default":
-        sort_class_plot = False
+    if pers_model == "" or pers_model == "null" or pers_model == "undefined":
+        overall_perf = None
     else:
-        raise Exception("Invalid breakdown_sort value")
-
-    overall_perf = utils.show_overall_perf(
-        variant=pers_model,
-        error_type=error_type,
-        cur_user=cur_user,
-        breakdown_axis=breakdown_axis,
-        topic_vis_method=topic_vis_method,
-    )
+        overall_perf = utils.show_overall_perf(
+            variant=pers_model,
+            error_type=error_type,
+            cur_user=cur_user,
+            topic_vis_method=topic_vis_method,
+        )
 
     results = {
         "overall_perf": overall_perf,
@@ -142,60 +115,32 @@ def get_audit():
 ########################################
 # ROUTE: /GET_CLUSTER_RESULTS
 @app.route("/get_cluster_results")
-def get_cluster_results():
+def get_cluster_results(debug=DEBUG):
     pers_model = request.args.get("pers_model")
-    n_examples = int(request.args.get("n_examples"))
     cluster = request.args.get("cluster")
-    example_sort = request.args.get("example_sort")
-    comparison_group = request.args.get("comparison_group")
     topic_df_ids = request.args.getlist("topic_df_ids")
     topic_df_ids = [int(val) for val in topic_df_ids[0].split(",") if val != ""]
     search_type = request.args.get("search_type")
     keyword = request.args.get("keyword")
-    n_neighbors = request.args.get("n_neighbors")
-    if n_neighbors != "null":
-        n_neighbors = int(n_neighbors)
-    neighbor_threshold = 0.6
     error_type = request.args.get("error_type")
     use_model = request.args.get("use_model") == "true"
-    scaffold_method = request.args.get("scaffold_method")
-        
-
-    # If user has a tuned model for this cluster, use that
-    cluster_model_file = f"./data/trained_models/{pers_model}_{cluster}.pkl"
-    if os.path.isfile(cluster_model_file):
-        pers_model = f"{pers_model}_{cluster}"
-
-    print(f"get_cluster_results using model {pers_model}")
 
-    other_ids = []
-    perf_metric = "avg_diff"
-    sort_ascending = True if example_sort == "ascending" else False
+    if debug:
+        print(f"get_cluster_results using model {pers_model}")
 
+    # Prepare cluster df (topic_df)
     topic_df = None
-    
-    personal_cluster_file = f"./data/personal_cluster_dfs/{pers_model}.pkl"
-    if (scaffold_method == "personal_cluster") and (os.path.isfile(personal_cluster_file)):
-        # Handle personal clusters
-        with open(personal_cluster_file, "rb") as f:
-            topic_df = pickle.load(f)
-            topic_df = topic_df[(topic_df["topic"] == cluster)]
-    else:
-        # Regular handling
-        with open(f"data/preds_dfs/{pers_model}.pkl", "rb") as f:
-            topic_df = pickle.load(f)
-        if search_type == "cluster":
-            # Display examples with comment, your pred, and other users' pred
-            topic_df = topic_df[(topic_df["topic"] == cluster) | (topic_df["item_id"].isin(topic_df_ids))]
-                
-        elif search_type == "neighbors":
-            neighbor_ids = utils.get_match(topic_df_ids, K=n_neighbors, threshold=neighbor_threshold, debug=False)
-            topic_df = topic_df[(topic_df["item_id"].isin(neighbor_ids)) | (topic_df["item_id"].isin(topic_df_ids))]
-        elif search_type == "keyword":
-            topic_df = topic_df[(topic_df["comment"].str.contains(keyword, case=False, regex=False)) | (topic_df["item_id"].isin(topic_df_ids))]
-    
+    with open(f"data/preds_dfs/{pers_model}.pkl", "rb") as f:
+        topic_df = pickle.load(f)
+    if search_type == "cluster":
+        # Display examples with comment, your pred, and other users' pred
+        topic_df = topic_df[(topic_df["topic"] == cluster) | (topic_df["item_id"].isin(topic_df_ids))]
+    elif search_type == "keyword":
+        topic_df = topic_df[(topic_df["comment"].str.contains(keyword, case=False, regex=False)) | (topic_df["item_id"].isin(topic_df_ids))]
+
     topic_df = topic_df.drop_duplicates()
-    print("len topic_df", len(topic_df))
+    if debug:
+        print("len topic_df", len(topic_df))
 
     # Handle empty results
     if len(topic_df) == 0: 
@@ -216,24 +161,20 @@ def get_cluster_results():
 
     topic_df_ids = topic_df["item_id"].unique().tolist()
 
-    if (scaffold_method == "personal_cluster") and (os.path.isfile(personal_cluster_file)):
+    # Prepare overview plot for the cluster
+    if use_model:
+        # Display results with the model as a reference point
         cluster_overview_plot_json, sampled_df = utils.plot_overall_vis_cluster(topic_df, error_type=error_type, n_comments=500)
     else:
-        # Default case
-        topic_df_mod = topic_df.merge(comments_grouped_full_topic_cat, on="item_id", how="left", suffixes=('_', '_avg'))
-        if use_model:
-            # Display results with the model as a reference point
-            cluster_overview_plot_json, sampled_df = utils.plot_overall_vis_cluster(topic_df_mod, error_type=error_type, n_comments=500)
-        else:
-            # Display results without a model
-            cluster_overview_plot_json, sampled_df = utils.plot_overall_vis_cluster_no_model(topic_df_mod, n_comments=500)
+        # Display results without a model
+        cluster_overview_plot_json, sampled_df = utils.plot_overall_vis_cluster_no_model(topic_df, n_comments=500)
 
-    cluster_comments = utils.get_cluster_comments(sampled_df,error_type=error_type, num_examples=n_examples, use_model=use_model)  # New version of cluster comment table
+    cluster_comments = utils.get_cluster_comments(sampled_df,error_type=error_type, use_model=use_model)  # New version of cluster comment table
 
     results = {
         "topic_df_ids": topic_df_ids,
         "cluster_overview_plot_json": json.loads(cluster_overview_plot_json),
-        "cluster_comments": cluster_comments, 
+        "cluster_comments": cluster_comments.to_json(orient="records"), 
     }
     return json.dumps(results)
 
@@ -280,7 +221,6 @@ def get_group_model():
     grp_ids = grp_df["worker_id"].tolist()
 
     ratings_grp = utils.get_grp_model_labels(
-        comments_df=comments_grouped_full_topic_cat,
         n_label_per_bin=BIN_DISTRIB,
         score_bins=SCORE_BINS,
         grp_ids=grp_ids,
@@ -322,7 +262,7 @@ def get_labeling():
     model_name_suggestion = f"my_model"
 
     context = {
-        "personalized_models": utils.get_all_model_names(user),
+        "personalized_models": utils.get_user_model_names(user),
         "model_name_suggestion": model_name_suggestion,
         "clusters_for_tuning": clusters_for_tuning_options,
     }
@@ -330,15 +270,16 @@ def get_labeling():
 
 ########################################
 # ROUTE: /GET_COMMENTS_TO_LABEL
-N_LABEL_PER_BIN = 8 # 8 * 5 = 40 comments
-BIN_DISTRIB = [4, 8, 16, 8, 4]
+if DEBUG:
+    BIN_DISTRIB = [1, 2, 4, 2, 1]  # 10 comments
+else:
+    BIN_DISTRIB = [2, 4, 8, 4, 2]  # 20 comments
 SCORE_BINS = [(0.0, 0.5), (0.5, 1.5), (1.5, 2.5), (2.5, 3.5), (3.5, 4.01)]
 @app.route("/get_comments_to_label")
 def get_comments_to_label():
     n = int(request.args.get("n"))
     # Fetch examples to label
     to_label_ids = utils.create_example_sets(
-        comments_df=comments_grouped_full_topic_cat,
         n_label_per_bin=BIN_DISTRIB,
         score_bins=SCORE_BINS,
         keyword=None
@@ -355,14 +296,11 @@ def get_comments_to_label():
 
 ########################################
 # ROUTE: /GET_COMMENTS_TO_LABEL_TOPIC
-N_LABEL_PER_BIN_TOPIC = 2 # 2 * 5 = 10 comments
 @app.route("/get_comments_to_label_topic")
 def get_comments_to_label_topic():
     # Fetch examples to label
     topic = request.args.get("topic")
     to_label_ids = utils.create_example_sets(
-        comments_df=comments_grouped_full_topic_cat,
-        # n_label_per_bin=N_LABEL_PER_BIN_TOPIC,
         n_label_per_bin=BIN_DISTRIB,
         score_bins=SCORE_BINS,
         keyword=None,
@@ -397,10 +335,7 @@ def get_personalized_model():
     # Handle existing or new model cases
     if mode == "view":
         # Fetch prior model performance
-        if model_name not in utils.get_all_model_names():
-            raise Exception(f"Model {model_name} does not exist")
-        else:
-            mae, mse, rmse, avg_diff, ratings_prev = utils.fetch_existing_data(model_name, last_label_i)
+        mae, mse, rmse, avg_diff, ratings_prev = utils.fetch_existing_data(model_name, last_label_i)
         
     elif mode == "train":
         # Train model and cache predictions using new labels
@@ -490,8 +425,6 @@ def get_reports():
             reports = get_fixed_scaffold()
         elif (scaffold_method == "personal" or scaffold_method == "personal_group" or scaffold_method == "personal_test"):
             reports = get_personal_scaffold(model, topic_vis_method)
-        elif (scaffold_method == "personal_cluster"):
-            reports = get_personal_cluster_scaffold(model)
         elif scaffold_method == "prompts":
             reports = get_prompts_scaffold()
         elif scaffold_method == "tutorial":
@@ -576,21 +509,11 @@ def get_tutorial_scaffold():
         },
     ] 
 
-def get_personal_cluster_scaffold(model):
-    topics_under_top, topics_over_top = utils.get_personal_clusters(model)
-
-    report_under = [get_empty_report(topic, "System is under-sensitive") for topic in topics_under_top]
-
-    report_over = [get_empty_report(topic, "System is over-sensitive") for topic in topics_over_top]
-    reports = (report_under + report_over)
-    random.shuffle(reports)
-    return reports
-
 def get_topic_errors(df, topic_vis_method, threshold=2):
-    topics = df["topic_"].unique().tolist()
+    topics = df["topic"].unique().tolist()
     topic_errors = {}
     for topic in topics:
-        t_df = df[df["topic_"] == topic]
+        t_df = df[df["topic"] == topic]
         y_true = t_df["pred"].to_numpy()
         y_pred = t_df["rating"].to_numpy()
         if topic_vis_method == "mae":
@@ -627,27 +550,28 @@ def get_personal_scaffold(model, topic_vis_method, n_topics=200, n=5):
     # Get topics with greatest amount of error
     with open(f"./data/preds_dfs/{model}.pkl", "rb") as f:
         preds_df = pickle.load(f)
-        preds_df_mod = preds_df.merge(utils.get_comments_grouped_full_topic_cat(), on="item_id", how="left", suffixes=('_', '_avg'))
+        system_preds_df = utils.get_system_preds_df()
+        preds_df_mod = preds_df.merge(system_preds_df, on="item_id", how="left", suffixes=('', '_sys'))
         preds_df_mod = preds_df_mod[preds_df_mod["user_id"] == "A"].sort_values(by=["item_id"]).reset_index()
-        preds_df_mod = preds_df_mod[preds_df_mod["topic_id_"] < n_topics]
+        preds_df_mod = preds_df_mod[preds_df_mod["topic_id"] < n_topics]
 
         if topic_vis_method == "median":
-            df = preds_df_mod.groupby(["topic_", "user_id"]).median().reset_index()
+            df = preds_df_mod.groupby(["topic", "user_id"]).median().reset_index()
         elif topic_vis_method == "mean":
-            df = preds_df_mod.groupby(["topic_", "user_id"]).mean().reset_index()
+            df = preds_df_mod.groupby(["topic", "user_id"]).mean().reset_index()
         elif topic_vis_method == "fp_fn":
             for error_type in ["fn_proportion", "fp_proportion"]:
                 topic_errors = get_topic_errors(preds_df_mod, error_type)
-                preds_df_mod[error_type] = [topic_errors[topic] for topic in preds_df_mod["topic_"].tolist()]
-            df = preds_df_mod.groupby(["topic_", "user_id"]).mean().reset_index()
+                preds_df_mod[error_type] = [topic_errors[topic] for topic in preds_df_mod["topic"].tolist()]
+            df = preds_df_mod.groupby(["topic", "user_id"]).mean().reset_index()
         else:
             # Get error for each topic
             topic_errors = get_topic_errors(preds_df_mod, topic_vis_method)
-            preds_df_mod[topic_vis_method] = [topic_errors[topic] for topic in preds_df_mod["topic_"].tolist()]
-            df = preds_df_mod.groupby(["topic_", "user_id"]).mean().reset_index()
+            preds_df_mod[topic_vis_method] = [topic_errors[topic] for topic in preds_df_mod["topic"].tolist()]
+            df = preds_df_mod.groupby(["topic", "user_id"]).mean().reset_index()
 
         # Get system error
-        df = df[(df["topic_"] != "53_maiareficco_kallystas_dyisisitmanila_tractorsazi") & (df["topic_"] != "79_idiot_dumb_stupid_dumber")]
+        df = df[(df["topic"] != "53_maiareficco_kallystas_dyisisitmanila_tractorsazi") & (df["topic"] != "79_idiot_dumb_stupid_dumber")]
         
         if topic_vis_method == "median" or topic_vis_method == "mean":
             df["error_magnitude"] = [utils.get_error_magnitude(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
@@ -655,31 +579,30 @@ def get_personal_scaffold(model, topic_vis_method, n_topics=200, n=5):
 
             df_under = df[df["error_type"] == "System is under-sensitive"]
             df_under = df_under.sort_values(by=["error_magnitude"], ascending=False).head(n) # surface largest errors first
-            report_under = [get_empty_report(row["topic_"], row["error_type"]) for _, row in df_under.iterrows()]
+            report_under = [get_empty_report(row["topic"], row["error_type"]) for _, row in df_under.iterrows()]
 
             df_over = df[df["error_type"] == "System is over-sensitive"]
             df_over = df_over.sort_values(by=["error_magnitude"], ascending=False).head(n) # surface largest errors first
-            report_over = [get_empty_report(row["topic_"], row["error_type"]) for _, row in df_over.iterrows()]
+            report_over = [get_empty_report(row["topic"], row["error_type"]) for _, row in df_over.iterrows()]
             
             # Set up reports
-            # return [get_empty_report(row["topic_"], row["error_type"]) for index, row in df.iterrows()]
             reports = (report_under + report_over)
             random.shuffle(reports)
         elif topic_vis_method == "fp_fn":
             df_under = df.sort_values(by=["fn_proportion"], ascending=False).head(n)
             df_under = df_under[df_under["fn_proportion"] > 0]
-            report_under = [get_empty_report(row["topic_"], "System is under-sensitive") for _, row in df_under.iterrows()]
+            report_under = [get_empty_report(row["topic"], "System is under-sensitive") for _, row in df_under.iterrows()]
             
             df_over = df.sort_values(by=["fp_proportion"], ascending=False).head(n)
             df_over = df_over[df_over["fp_proportion"] > 0]
-            report_over = [get_empty_report(row["topic_"], "System is over-sensitive") for _, row in df_over.iterrows()]
+            report_over = [get_empty_report(row["topic"], "System is over-sensitive") for _, row in df_over.iterrows()]
 
             reports = (report_under + report_over)
             random.shuffle(reports)
         else:
             df = df.sort_values(by=[topic_vis_method], ascending=False).head(n * 2)
             df["error_type"] = [utils.get_error_type_radio(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
-            reports = [get_empty_report(row["topic_"], row["error_type"]) for _, row in df.iterrows()]
+            reports = [get_empty_report(row["topic"], row["error_type"]) for _, row in df.iterrows()]
 
         return reports
 
@@ -750,11 +673,7 @@ def get_explore_examples():
     n_examples = int(request.args.get("n_examples"))
 
     # Get sample of examples
-    df = utils.get_comments_grouped_full_topic_cat().sample(n=n_examples)
-
-    df["system_decision"] = [utils.get_decision(rating, threshold) for rating in df["rating"].tolist()]
-    df["system_color"] = [utils.get_user_color(sys, threshold) for sys in df["rating"].tolist()]  # get cell colors
-
+    df = utils.get_explore_df(n_examples, threshold)
     ex_json = df.to_json(orient="records")
 
     results = {