Refactoring updates and minor fixes

main.py

@@ -4,10 +4,9 @@ from datetime import datetime
 
 import pandas as pd
 import schedule
-from datasets import Dataset, load_dataset
-from huggingface_hub import login
+from datasets import Dataset
 
-from utilities.user_defined_functions import get_latest_data, merge_data
+from utilities.user_defined_functions import get_latest_data, merge_data, load_or_create_dataset
 from utilities.my_logger import setup_logger
 from utilities.readme_update import update_readme
 
@@ -30,9 +29,8 @@ logger = setup_logger(__name__)
 def main():
     date = datetime.now().strftime('%Y-%m-%d')
     logger.warning(f"Running main function for date: {date}")
-    dataset = load_dataset()
+    dataset = load_or_create_dataset()
 
-    # Get Latest Data and merge with historic data
     new_df = get_latest_data()
 
     # Using dataset from hub

utilities/user_defined_functions.py

@@ -24,7 +24,8 @@ auth_token = os.environ["HUGGINGFACE_AUTH_TOKEN"]
 login(auth_token, add_to_git_credential=True)
 
 logger = setup_logger(__name__)
-# Dummy row for when we create a new repo
+
+# Dummy row for when we create a new repo make sure to put everything in a list
 dummy_data = {
     "id": ['id'],
     "content": ["This is a sample post content. Just for demonstration purposes!"],
@@ -34,8 +35,8 @@ dummy_data = {
     "title": ["Sample Post Title: How to Use Hugging Face?"],
     "score": [457],
     "permalink": ["/r/sampleSubreddit/comments/sampleID/sample_post_title_how_to_use_hugging_face/"],
-    "updated": False,
-    "new": False,
+    "updated": [False],
+    "new": [False],
     }
 
 
@@ -89,10 +90,7 @@ def merge_data(old_df: pd.DataFrame, new_df: pd.DataFrame) -> pd.DataFrame:
     - pd.DataFrame: The merged, sorted, and marked dataframe.
     """
 
-    # Mark rows in old and new dataframes
-    old_df['new'] = False
-    new_df['new'] = True
-
+    old_df.drop(columns=['new', 'updated'], inplace=True)
     # Concatenate old and new dataframes, sort by 'date_utc', and reset index
     df = pd.concat([old_df, new_df], ignore_index=True).sort_values(by='date_utc').reset_index(drop=True)
 
@@ -100,7 +98,7 @@ def merge_data(old_df: pd.DataFrame, new_df: pd.DataFrame) -> pd.DataFrame:
     df = data_processing(df)
 
     # Identify new rows (present in new_df but not in old_df)
-    df['new'] = df['new'] & ~df['id'].duplicated(keep=False)
+    df['new'] = df['id'].apply(lambda x: x in set(new_df['id']) - set(old_df['id']))
 
     return df