Visualizer bugs fixed; added normalized editdist

analysis_util.py

@@ -55,21 +55,21 @@ def get_correlations_df(df, right_side):
 
 
 def get_correlations_for_groups(df, right_side):
-    noref_correlations = {"all": get_correlations_df(df, right_side=right_side)}
+    correlations = {"all": get_correlations_df(df, right_side=right_side)}
 
     for e2s in (False, True):
         for s2e in (False, True):
-            suffix = ""
+            group = ""
             if e2s:
-                suffix += "+e2s"
+                group += "+e2s"
             if s2e:
-                suffix += "+s2e"
-            if suffix == "":
-                suffix = "golden"
+                group += "+s2e"
+            if group == "":
+                group = "golden"
 
             subdf = df[(df["end_to_start"] == e2s) & (df["start_to_end"] == s2e)]
-            subdf_noref_corr = get_correlations_for_groups(subdf, right_side=right_side)
-            noref_correlations[suffix] = subdf_noref_corr
+            subdf_corr = get_correlations_df(subdf, right_side=right_side)
+            correlations[group] = subdf_corr
 
-    noref_correlations = pd.concat(noref_correlations, axis=1)
-    return noref_correlations
+    correlations = pd.concat(correlations, axis=1)
+    return correlations

change_visualizer.py

@@ -110,8 +110,8 @@ if __name__ == '__main__':
             gr.Markdown(f"### Reference-only correlations")
             gr.Markdown(value=analysis_util.get_correlations_for_groups(df_synthetic, right_side="ind").to_markdown())
 
-            gr.Markdown(f"### Aggregated correlations")
-            gr.Markdown(value=analysis_util.get_correlations_for_groups(df_synthetic, right_side="aggr").to_markdown())
+            # gr.Markdown(f"### Aggregated correlations")
+            # gr.Markdown(value=analysis_util.get_correlations_for_groups(df_synthetic, right_side="aggr").to_markdown())
 
         application.load(update_dataset_view_manual, inputs=slider_manual,
                          outputs=view_manual)

generate_annotated_diffs.py

@@ -1,4 +1,5 @@
 import diff_match_patch as dmp_module
+from tqdm import tqdm
 
 from api_wrappers import hf_data_loader
 
@@ -26,14 +27,18 @@ def annotated_diff_for_row(row):
 
 
 def manual_data_with_annotated_diffs():
+    tqdm.pandas()
+
     df = hf_data_loader.load_raw_rewriting_as_pandas()
-    annotated = df.apply(annotated_diff_for_row, axis=1)
+    annotated = df.progress_apply(annotated_diff_for_row, axis=1)
     df['annotated_diff'] = annotated
     return df
 
 
 def synthetic_data_with_annotated_diffs():
+    tqdm.pandas()
+
     df = hf_data_loader.load_synthetic_as_pandas()
-    annotated = df.apply(annotated_diff_for_row, axis=1)
+    annotated = df.progress_apply(annotated_diff_for_row, axis=1)
     df['annotated_diff'] = annotated
     return df

generation_steps/metrics_analysis.py

@@ -15,6 +15,8 @@ BLEU = evaluate.load('bleu', cache_dir=config.CACHE_DIR)
 
 
 def bleu_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        return BLEU.compute(predictions=[pred] * len(kwargs["refs"]), references=kwargs["refs"])["bleu"]
     return BLEU.compute(predictions=[pred], references=[ref])["bleu"]
 
 
@@ -22,6 +24,8 @@ METEOR = evaluate.load('meteor', cache_dir=config.CACHE_DIR)
 
 
 def meteor_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        return METEOR.compute(predictions=[pred] * len(kwargs["refs"]), references=kwargs["refs"])["meteor"]
     return METEOR.compute(predictions=[pred], references=[ref])["meteor"]
 
 
@@ -29,14 +33,20 @@ ROUGE = evaluate.load('rouge', cache_dir=config.CACHE_DIR)
 
 
 def rouge1_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        return ROUGE.compute(predictions=[pred] * len(kwargs["refs"]), references=kwargs["refs"])["rouge1"]
     return ROUGE.compute(predictions=[pred], references=[ref])["rouge1"]
 
 
 def rouge2_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        return ROUGE.compute(predictions=[pred] * len(kwargs["refs"]), references=kwargs["refs"])["rouge2"]
     return ROUGE.compute(predictions=[pred], references=[ref])["rouge2"]
 
 
 def rougeL_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        return ROUGE.compute(predictions=[pred] * len(kwargs["refs"]), references=kwargs["refs"])["rougeL"]
     return ROUGE.compute(predictions=[pred], references=[ref])["rougeL"]
 
 
@@ -44,6 +54,10 @@ BERTSCORE = evaluate.load('bertscore', cache_dir=config.CACHE_DIR)
 
 
 def bertscore_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        return \
+            BERTSCORE.compute(predictions=[pred], references=[kwargs["refs"]], model_type="distilbert-base-uncased")[
+                "f1"][0]
     return BERTSCORE.compute(predictions=[pred], references=[ref], model_type="distilbert-base-uncased")["f1"][0]
 
 
@@ -51,6 +65,8 @@ CHRF = evaluate.load("chrf")
 
 
 def chrf_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        return CHRF.compute(predictions=[pred], references=[kwargs["refs"]])["score"]
     return CHRF.compute(predictions=[pred], references=[[ref]])["score"]
 
 
@@ -58,26 +74,46 @@ TER = evaluate.load("ter")
 
 
 def ter_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        return TER.compute(predictions=[pred], references=[kwargs["refs"]])["score"]
     return TER.compute(predictions=[pred], references=[[ref]])["score"]
 
 
 def edit_distance_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        scores = [Levenshtein.distance(pred, ref) for ref in kwargs["refs"]]
+        return sum(scores) / len(scores)
     return Levenshtein.distance(pred, ref)
 
+def edit_distance_norm_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        scores = [Levenshtein.distance(pred, ref) / len(pred) for ref in kwargs["refs"]]
+        return sum(scores) / len(scores)
+    return Levenshtein.distance(pred, ref) / len(pred)
+
 
 def edit_time_fn(pred, ref, **kwargs):
     return kwargs["edittime"]
 
 
 def gptscore_ref_1_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        scores = [gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=1) for ref in kwargs["refs"]]
+        return sum(scores) / len(scores)
     return gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=1)
 
 
 def gptscore_ref_3_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        scores = [gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=3) for ref in kwargs["refs"]]
+        return sum(scores) / len(scores)
     return gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=3)
 
 
 def gptscore_ref_5_fn(pred, ref, **kwargs):
+    if "refs" in kwargs:
+        scores = [gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=5) for ref in kwargs["refs"]]
+        return sum(scores) / len(scores)
     return gpt_eval.compute_ref(prediction=pred, reference=ref, n_requests=5)
 
 
@@ -94,13 +130,14 @@ def gptscore_noref_5_fn(pred, ref, **kwargs):
 
 
 IND_METRICS = {
+    "editdist": edit_distance_fn,
+    "editdist-norm": edit_distance_norm_fn,
     "gptscore-ref-1-req": gptscore_ref_1_fn,
     # "gptscore-ref-3-req": gptscore_ref_3_fn,
     # "gptscore-ref-5-req": gptscore_ref_5_fn,
     "gptscore-noref-1-req": gptscore_noref_1_fn,
     # "gptscore-noref-3-req": gptscore_noref_3_fn,
     # "gptscore-noref-5-req": gptscore_noref_5_fn,
-    "editdist": edit_distance_fn,
     "bleu": bleu_fn,
     "meteor": meteor_fn,
     "rouge1": rouge1_fn,
@@ -112,11 +149,12 @@ IND_METRICS = {
 }
 
 AGGR_METRICS = IND_METRICS.copy()
-del AGGR_METRICS["gptscore-ref-1-req"]
-del AGGR_METRICS["gptscore-noref-1-req"]
+# del AGGR_METRICS["gptscore-ref-1-req"]
+# del AGGR_METRICS["gptscore-noref-1-req"]
 
 REL_METRICS = {
     "editdist": edit_distance_fn,
+    "editdist-norm": edit_distance_norm_fn,
     "edittime": edit_time_fn,
 }
 
@@ -183,7 +221,7 @@ def compute_metrics(df):
             df[f"rel_{rel_metric}_aggr_{aggr_metric}_pearson"] = (
                 df[f"{rel_metric}_related"].corr(df[f"{aggr_metric}_aggr"], method="pearson"))
 
-            df[f"rel_{rel_metric}_ind_{aggr_metric}_spearman"] = (
+            df[f"rel_{rel_metric}_aggr_{aggr_metric}_spearman"] = (
                 df[f"{rel_metric}_related"].corr(df[f"{aggr_metric}_aggr"], method="spearman"))
 
     return df