import json import uuid import os import re from huggingface_hub import HfApi from huggingface_hub import hf_hub_download def file_name_decode(file_name): # model_name,dataset,method,answer.txt # input file name example: [mistralai@Mistral-7B-Instruct-v0.3][Setting3][icl]answer.txt match = re.match(rf'\[([^\[^\]]+)\]\[([^\[^\]]+)\]\[([^\[^\]]+)\]([^\[^\]]+)', file_name) if match: model_name, dataset, method, file_name = match.groups() ret_dict = { 'model_name': model_name, 'dataset': dataset, 'method': method, 'file_name': file_name } return ret_dict return None def upload_scores_to_hub(api, scores_dict, path_in_repo,hub_repo='zhaorui-nb/test_json'): # id = str(uuid.uuid4()) save_json_path = f'.output/upload.json' os.makedirs(os.path.dirname(save_json_path), exist_ok=True) with open(save_json_path, 'w') as f: json.dump(scores_dict, f , indent=4) # SAVE JSON TO HUB res = api.upload_file( path_or_fileobj=save_json_path, path_in_repo=path_in_repo, #f'data/train,{os.path.basename(save_json_path)}', repo_id=hub_repo, repo_type="dataset", ) return res if __name__ == "__main__": pass