WIP on annotated diffs generation

.gitignore

@@ -277,4 +277,5 @@ pip-selfcheck.json
 
 .idea
 
-cache
+cache
+output

config.py

@@ -1,6 +1,14 @@
 import os
+from pathlib import Path
 
 HF_TOKEN = os.environ.get('HF_TOKEN')
 HF_RAW_DATASET_NAME = "petrtsv-jb/commit-msg-rewriting"
 HF_RAW_DATASET_SPLIT = 'train'
-CACHE_DIR = "cache"
+
+CACHE_DIR = Path("cache")
+CACHE_DIR.mkdir(exist_ok=True)
+
+OUTPUT_DIR = Path("output")
+OUTPUT_DIR.mkdir(exist_ok=True)
+
+ANNOTATED_DIFFS_ARTIFACT = OUTPUT_DIR / "annotated_diffs.csv"

generate_annotated_diffs.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+
+import hf_data_loader
+
+
+def group_changes(changes):
+    groups = {}
+    for change in changes:
+        group = datetime.fromisoformat(change.ts)
+        if group not in groups:
+            groups[group] = []
+        groups[group].append(change)
+
+    grouped_changes = []
+    for group in sorted(groups.keys()):
+        grouped_changes.sort(key=lambda x: x.p)
+        grouped_changes.append(groups[group])
+
+    return grouped_changes
+
+
+def get_annotated_diff(initial_text, changes):
+    grouped_changes = group_changes(changes)
+    text = [((c, " ") for c in initial_text)]
+    for change_group in grouped_changes:
+        text_pointer = 0
+        change_pointer = 0
+        while text_pointer < len(text):
+            pass
+
+
+df = hf_data_loader.load_raw_dataset_as_pandas()

hf_data_loader.py

@@ -8,3 +8,6 @@ def load_raw_dataset_as_pandas():
                         split=config.HF_RAW_DATASET_SPLIT,
                         token=config.HF_TOKEN,
                         cache_dir=config.CACHE_DIR).to_pandas()
+
+
+load_raw_dataset_as_pandas()