Latest version of the code; config updated to JetBrains-Research

analysis_util.py

@@ -59,15 +59,14 @@ def get_correlations_for_groups(df, right_side):
 
     for e2s in (False, True):
         for s2e in (False, True):
-            group = ""
+            group = "golden"
             if e2s:
                 group += "+e2s"
             if s2e:
                 group += "+s2e"
-            if group == "":
-                group = "golden"
 
-            subdf = df[(df["end_to_start"] == e2s) & (df["start_to_end"] == s2e)]
+            subdf = df[((df["end_to_start"] == e2s) & (df["start_to_end"] == s2e)) | (
+                    (df["end_to_start"] == False) & (df["start_to_end"] == False))]
             subdf_corr = get_correlations_df(subdf, right_side=right_side)
             correlations[group] = subdf_corr
 

config.py

@@ -8,7 +8,7 @@ GRAZIE_TIMEOUT_SEC = 1.0
 
 HF_TOKEN = os.environ.get('HF_TOKEN')
 
-HF_RAW_DATASET_NAME = "petrtsv-jb/commit-msg-rewriting"
+HF_RAW_DATASET_NAME = "JetBrains-Research/commit-msg-rewriting"
 HF_RAW_DATASET_SPLIT = 'train'
 
 HF_FULL_COMMITS_DATASET_NAME = "JetBrains-Research/lca-commit-message-generation"
@@ -19,7 +19,7 @@ HF_PREDICTIONS_DATASET_NAME = "JetBrains-Research/lca-results"
 HF_PREDICTIONS_DATASET_SUBNAME = "cmg_gpt_4_0613"
 HF_PREDICTIONS_DATASET_SPLIT = "test"
 
-HF_SYNTHETIC_DATASET_NAME = "petrtsv-jb/synthetic-commit-msg-rewriting"
+HF_SYNTHETIC_DATASET_NAME = "JetBrains-Research/synthetic-commit-msg-rewriting"
 HF_SYNTHETIC_DATASET_SPLIT = 'train'
 
 LLM_MODEL = "gpt-4-1106-preview"

generation_steps/metrics_analysis.py

@@ -147,7 +147,8 @@ IND_METRICS = {
     "ter": ter_fn,
 }
 
-AGGR_METRICS = IND_METRICS.copy()
+AGGR_METRICS = {}
+# AGGR_METRICS = IND_METRICS.copy()
 # del AGGR_METRICS["gptscore-ref-1-req"]
 # del AGGR_METRICS["gptscore-noref-1-req"]