Download the commit-rewriting dataset

config.py

@@ -0,0 +1,6 @@
+import os
+
+HF_TOKEN = os.environ.get('HF_TOKEN')
+HF_RAW_DATASET_NAME = "petrtsv-jb/commit-msg-rewriting"
+HF_RAW_DATASET_SPLIT = 'train'
+CACHE_DIR = "cache"

hf_data_loader.py

@@ -0,0 +1,10 @@
+from datasets import load_dataset
+
+import config
+
+
+def load_raw_dataset_as_pandas():
+    return load_dataset(config.HF_RAW_DATASET_NAME,
+                        split=config.HF_RAW_DATASET_SPLIT,
+                        token=config.HF_TOKEN,
+                        cache_dir=config.CACHE_DIR).to_pandas()