Updating README.md the right way :'(

main.py

@@ -8,7 +8,7 @@ from datasets import Dataset
 
 from utilities.user_defined_functions import get_latest_data, merge_data, load_or_create_dataset
 from utilities.my_logger import setup_logger
-from utilities.readme_update import update_readme
+from utilities.readme_update import update_dataset_readme
 
 # Set dataset name, path to README.md, and existing dataset details
 subreddit = os.environ["SUBREDDIT"]
@@ -47,13 +47,15 @@ def main():
     dataset['train'] = Dataset.from_pandas(df, preserve_index=False)
 
     # Update README
-    update_readme(dataset_name=dataset_name, subreddit=subreddit, latest_date=date, new_rows=new_rows)
     logger.info(f"Adding {new_rows} rows for {date}.")
 
     # Push the augmented dataset to the Hugging Face hub
     logger.debug(f"Pushing data for {date} to the Hugging Face hub")
     dataset.push_to_hub(dataset_name, token=auth_token)
     logger.info(f"Processed and pushed data for {date} to the Hugging Face Hub")
+    logger.info(f"Updating rEADME...")
+    update_dataset_readme(dataset_name=dataset_name, subreddit=subreddit, new_rows=new_rows)
+    logger.info(f"Updated README.")
     # files_cleaned = dataset.cleanup_cache_files()
     # logger.info(f"Removed {files_cleaned} cache files")
 

requirements.txt

@@ -1,8 +1,7 @@
 praw==7.7.1
 gradio==3.50.2
 nbdev==2.3.12
-# datasets==2.14.6
-git+https://github.com/huggingface/datasets.git@81b3ccfc016f6a39837334a0173dac3f59112856
+datasets==2.14.6
 requests==2.28.2
 loguru==0.7.0
 rich==13.3.4

utilities/readme_update.py

@@ -2,26 +2,63 @@ import os
 from datetime import datetime
 
 import pytz
-from datasets.download.download_config import DownloadConfig
-from datasets.utils.file_utils import cached_path
-from datasets.utils.hub import hf_hub_url
+from huggingface_hub import HfApi, Repository
 
 frequency = os.environ.get("FREQUENCY", '').lower()
-
-
-def get_readme_path(dataset_name):
-    readme_path = hf_hub_url(dataset_name, "README.md")
-    return cached_path(readme_path, download_config=DownloadConfig())
-
-
-def update_readme(dataset_name, subreddit, latest_date, new_rows):
-    path = get_readme_path(dataset_name=dataset_name)
+GENERATED_BELOW_MARKER = "--- Generated Part of README Below ---"
+hf_token = os.environ["HUGGINGFACE_AUTH_TOKEN"]
+local_repo_path = "./readme_repo"
+
+
+def update_dataset_readme(dataset_name: str, subreddit: str, new_rows: int) -> None:
+    """
+    Update the README file of a specified dataset repository with new information.
+
+    Args:
+    dataset_name (str): Name of the dataset repository.
+    subreddit (str): Name of the subreddit being used for dataset creation.
+    new_rows (int): Number of new rows added in the latest update.
+    hf_token (str): Hugging Face authentication token.
+    local_repo_path (str): Local path to clone the repository.
+    """
+    # Initialize HfApi
+    api = HfApi()
+
+    # Clone the repository locally
+    repo = Repository(local_repo_path, clone_from=dataset_name, repo_type='dataset', use_auth_token=hf_token)
+
+    # Read the README file
+    with open(f"{local_repo_path}/README.md", "r") as file:
+        old_readme = file.read()
+
+    # Modify the README
+    new_readme = append_to_readme(subreddit=subreddit, new_rows=new_rows, old_readme=old_readme)
+
+    # Write the updated README back to the repository
+    with open(f"{local_repo_path}/README.md", "w") as file:
+        file.write(new_readme)
+
+    # Push the changes
+    repo.push_to_hub(blocking=True, commit_message=f'Pushing {new_rows}')
+
+def append_to_readme(subreddit: str, new_rows: int, old_readme: str) -> str:
+    """
+    Append new information to the existing README content.
+
+    Args:
+    subreddit (str): Name of the subreddit.
+    new_rows (int): Number of new rows added.
+    old_readme (str): Existing README content.
+
+    Returns:
+    str: Updated README content.
+    """
     latest_hour = datetime.now(pytz.utc).replace(minute=0, second=0, microsecond=0)
     latest_hour_str = latest_hour.strftime('%Y-%m-%d %H:00:00 %Z%z')
 
     readme_text = f"""
 ## Dataset Overview
-The goal is to have an open dataset of [r/{subreddit}](https://www.reddit.com/r/{subreddit}/) submissions. Im leveraging PRAW and the reddit API to get downloads.
+The goal is to have an open dataset of [r/{subreddit}](https://www.reddit.com/r/{subreddit}/) submissions. I'm leveraging PRAW and the Reddit API to get downloads.
 
 There is a limit of 1000 in an API call and limited search functionality, so this is run {frequency} to get new submissions.
 
@@ -33,27 +70,16 @@ The dataset is updated {frequency} with the most recent update being `{latest_ho
 
 ## Licensing 
 [Reddit Licensing terms](https://www.redditinc.com/policies/data-api-terms) as accessed on October 25:
-> The Content created with or submitted to our Services by Users (“User Content”) is owned by Users and not by Reddit. Subject to your complete and ongoing compliance with the Data API Terms, Reddit grants you a non-exclusive, non-transferable, non-sublicensable, and revocable license to copy and display the User Content using the Data API solely as necessary to develop, deploy, distribute, and run your App to your App Users. You may not modify the User Content except to format it for such display. You will comply with any requirements or restrictions imposed on usage of User Content by their respective owners, which may include "all rights reserved" notices, Creative Commons licenses, or other terms and conditions that may be agreed upon between you and the owners. Except as expressly permitted by this section, no other rights or licenses are granted or implied, including any right to use User Content for other purposes, such as for training a machine learning or AI model, without the express permission of rightsholders in the applicable User Content
-
-My take is that you can't use this data for *training* without getting permission.
+[License information]
 
 ## Opt-out
 To opt-out of this dataset please make a request in the community tab
 """
 
-    append_readme(path=path, readme_text=readme_text)
-
-
-def append_readme(path, readme_text):
-    generated_below_marker = "--- Generated Part of README Below ---"
-    with open(path, "r") as file:
-        content = file.read()
-
-    if generated_below_marker in content:
-        index = content.index(generated_below_marker) + len(generated_below_marker)
-        content = content[:index] + "\n\n" + readme_text
+    if GENERATED_BELOW_MARKER in old_readme:
+        index = old_readme.index(GENERATED_BELOW_MARKER) + len(GENERATED_BELOW_MARKER)
+        new_readme = old_readme[:index] + "\n\n" + readme_text
     else:
-        content += "\n\n" + generated_below_marker + "\n\n" + readme_text + "\n"
+        new_readme = old_readme + "\n\n" + GENERATED_BELOW_MARKER + "\n\n" + readme_text + "\n"
 
-    with open(path, "w") as file:
-        file.write(content)
+    return new_readme