upload_to_hub.py

import argparse
from pathlib import Path

from huggingface_hub import HfApi, create_repo, upload_folder


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--repo-id", required=True, help="Hugging Face repo id, e.g. user/M2-Encoder-Large")
    parser.add_argument(
        "--folder",
        default=str(Path(__file__).resolve().parent),
        help="Folder to upload. Defaults to this script's directory.",
    )
    parser.add_argument("--private", action="store_true", help="Create the repo as private.")
    parser.add_argument("--commit-message", default="Upload M2-Encoder HF export")
    args = parser.parse_args()

    folder = Path(args.folder).resolve()
    api = HfApi()
    create_repo(repo_id=args.repo_id, private=args.private, exist_ok=True)
    upload_folder(
        repo_id=args.repo_id,
        folder_path=str(folder),
        commit_message=args.commit_message,
        ignore_patterns=["**/__pycache__/**", "__pycache__/**", "*.pyc", "*.pyo"],
        delete_patterns=["**/__pycache__/**", "__pycache__/**", "*.pyc", "*.pyo"],
    )
    print(f"Uploaded {folder} -> {args.repo_id}")


if __name__ == "__main__":
    main()