Update load_data.py

load_data.py

@@ -24,7 +24,7 @@ def save_validated_to_hub(records, ctx):
         if HF_TOKEN:
             print("Pushing the dataset")
             print(ds)
-            ds.push_to_hub(HUB_DATASET_NAME, token=HF_TOKEN, private=True, branch = 'validated') #push annotations on the validated branch of the dataset
+            ds.push_to_hub(HUB_DATASET_NAME, token=HF_TOKEN, private=True)
         else:
             print("SET HF_TOKEN and HUB_DATASET_NAME TO SYNC YOUR DATASET!!!")
     else:
@@ -39,13 +39,13 @@ class LoadDatasets:
         # load data from validated branch
         try:
             print(f"Trying to sync with {HUB_DATASET_NAME}")
-            old_ds = load_dataset(HUB_DATASET_NAME, split="train", use_auth_token=HF_TOKEN, revision='validated')
+            old_ds = load_dataset(HUB_DATASET_NAME, split="train", use_auth_token=HF_TOKEN)
         except Exception as e:
             print(f"Not possible to sync with {HUB_DATASET_NAME}")
             print(e)
             old_ds = None
             
-        dataset = load_dataset(HUB_DATASET_NAME, split="train", use_auth_token=HF_TOKEN)  # get starting file from the main branch of the dataset
+        dataset = load_dataset('LukasGe/DVS-job-titles-raw', split="train", use_auth_token=HF_TOKEN)  # get starting file from the main branch of the dataset
     
         
         if old_ds: