wm/scripts/hf_upload_all.py

import os
import argparse
from huggingface_hub import HfApi

def upload_to_hf():
    api = HfApi()
    
    # 1. Upload Franka Dataset
    dataset_path = "/storage/ice-shared/ae8803che/hxue/data/dataset/franka"
    dataset_repo = "t1an/world_model_dataset"
    
    if os.path.exists(dataset_path):
        print(f"Uploading Franka dataset to {dataset_repo}...")
        try:
            api.upload_folder(
                folder_path=dataset_path,
                repo_id=dataset_repo,
                repo_type="dataset",
                path_in_repo="franka",
                # ignore_patterns=["*.pt"] # We want metadata.pt though
            )
        except Exception as e:
            print(f"Error uploading dataset: {e}")
    else:
        print(f"Dataset path {dataset_path} not found.")

    # 2. Upload Code and Latest Checkpoints to Model Repo
    model_repo = "t1an/world_model"
    # Create the repo if it doesn't exist
    try:
        api.create_repo(repo_id=model_repo, exist_ok=True)
    except Exception as e:
        print(f"Note on repo creation: {e}")

    # Upload Code (wm folder)
    print(f"Uploading code to {model_repo}...")
    try:
        api.upload_folder(
            folder_path="/storage/ice-shared/ae8803che/hxue/data/world_model/wm",
            repo_id=model_repo,
            path_in_repo="wm"
        )
    except Exception as e:
        print(f"Error uploading code: {e}")

    # Upload only latest.pt from each checkpoint folder
    checkpoint_dir = "/storage/ice-shared/ae8803che/hxue/data/world_model/checkpoints"
    if os.path.exists(checkpoint_dir):
        for run_name in os.listdir(checkpoint_dir):
            run_path = os.path.join(checkpoint_dir, run_name)
            if os.path.isdir(run_path):
                latest_ckpt = os.path.join(run_path, "latest.pt")
                if os.path.exists(latest_ckpt):
                    print(f"Uploading {run_name}/latest.pt to {model_repo}...")
                    try:
                        api.upload_file(
                            path_or_fileobj=latest_ckpt,
                            path_in_repo=f"checkpoints/{run_name}/latest.pt",
                            repo_id=model_repo
                        )
                    except Exception as e:
                        print(f"Error uploading {run_name}/latest.pt: {e}")

if __name__ == "__main__":
    # Check if conversion is finished by checking for metadata.pt
    metadata_path = "/storage/ice-shared/ae8803che/hxue/data/dataset/franka/metadata.pt"
    if not os.path.exists(metadata_path):
        print("Error: Franka dataset conversion not finished (metadata.pt missing).")
        print("Please wait for the conversion to finish before running this script.")
    else:
        upload_to_hf()