src/utils.py

from huggingface_hub import HfApi

def push_data_to_hf(repo_id, folder_path, path_in_repo, token=None):
    """
    Pushes data to a dataset on the Hugging Face Hub.

    Parameters:
        - repo_id (str): The ID of the repository on the Hugging Face Hub.
        - folder_path (str): Local path to the folder containing the data.
        - path_in_repo (str): Path within the repository where the data should be stored.
        - token (str, optional): Your authentication token for the Hugging Face Hub.

    Returns:
        - str: URL of the uploaded data.
    """

    api = HfApi(token=token)

    try:
        api.upload_folder(
            folder_path=folder_path,
            repo_id=repo_id,
            repo_type="dataset",
            path_in_repo=path_in_repo,
        )
    except Exception as e:
        return f"Error uploading data: {str(e)}"

    url = f"https://huggingface.co/{repo_id}/raw/main/{path_in_repo}"

    return f"Data successfully uploaded. Access it at: {url}"