Debugging again

main.py

@@ -66,8 +66,8 @@ def schedule_periodic_task():
     elif frequency == 'daily':
         start_time = '05:00'
         logger.info(f'Scheduling tasks to run every day at: {start_time} UTC+00')
-        schedule.every().day.at('08:33').do(main)
-        schedule.every().day.at('08:37').do(main)
+        schedule.every().day.at('13:34').do(main)
+        schedule.every().day.at('13:35').do(main)
         # schedule.every().day.at(start_time).do(main)
 
     while True:

utilities/user_defined_functions.py

@@ -2,7 +2,7 @@ import os
 from datetime import datetime
 
 import pandas as pd
-from datasets import Dataset, DatasetDict, load_dataset, VerificationMode
+from datasets import Dataset, DatasetDict, load_dataset, VerificationMode, DownloadMode
 from huggingface_hub import login
 
 from utilities.data_processing import data_processing
@@ -57,7 +57,7 @@ def load_or_create_dataset():
     # Load the existing dataset from the Hugging Face hub or create a new one
     try:
         logger.debug(f"Trying to download {dataset_name}")
-        dataset = load_dataset(dataset_name, verification_mode=VerificationMode.ALL_CHECKS)
+        dataset = load_dataset(dataset_name, verification_mode=VerificationMode.ALL_CHECKS, download_mode=DownloadMode.FORCE_REDOWNLOAD)
         logger.debug("Loading existing dataset")
     except FileNotFoundError:
         logger.warning("Creating new dataset")