utils/upload_hub.py

import json 
import uuid
import os 
import re
from huggingface_hub import HfApi
from huggingface_hub import hf_hub_download



def file_name_decode(file_name):
    # model_name,dataset,method,answer.txt
    # input file name example: [mistralai@Mistral-7B-Instruct-v0.3][Setting3][icl]answer.txt
    
    match = re.match(rf'\[([^\[^\]]+)\]\[([^\[^\]]+)\]\[([^\[^\]]+)\]([^\[^\]]+)', file_name)
    
    if match:
        model_name, dataset, method, file_name = match.groups()
        ret_dict = {
            'model_name': model_name,
            'dataset': dataset,
            'method': method,
            'file_name': file_name
        }
        return ret_dict
    return None    

def upload_scores_to_hub(api, scores_dict, path_in_repo,hub_repo='zhaorui-nb/test_json'):
    # id = str(uuid.uuid4())
    save_json_path = f'.output/upload.json'   
    os.makedirs(os.path.dirname(save_json_path), exist_ok=True)
    with open(save_json_path, 'w') as f:
        json.dump(scores_dict, f , indent=4)

    # SAVE JSON TO HUB 
    res = api.upload_file(
        path_or_fileobj=save_json_path,
        path_in_repo=path_in_repo, #f'data/train,{os.path.basename(save_json_path)}',
        repo_id=hub_repo,
        repo_type="dataset",
    )

    return res





if __name__ == "__main__":

    pass