File size: 1,364 Bytes
2f22782 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
import json
import uuid
import os
import re
from huggingface_hub import HfApi
from huggingface_hub import hf_hub_download
def file_name_decode(file_name):
# model_name,dataset,method,answer.txt
# input file name example: [mistralai@Mistral-7B-Instruct-v0.3][Setting3][icl]answer.txt
match = re.match(rf'\[([^\[^\]]+)\]\[([^\[^\]]+)\]\[([^\[^\]]+)\]([^\[^\]]+)', file_name)
if match:
model_name, dataset, method, file_name = match.groups()
ret_dict = {
'model_name': model_name,
'dataset': dataset,
'method': method,
'file_name': file_name
}
return ret_dict
return None
def upload_scores_to_hub(api, scores_dict, path_in_repo,hub_repo='zhaorui-nb/test_json'):
# id = str(uuid.uuid4())
save_json_path = f'.output/upload.json'
os.makedirs(os.path.dirname(save_json_path), exist_ok=True)
with open(save_json_path, 'w') as f:
json.dump(scores_dict, f , indent=4)
# SAVE JSON TO HUB
res = api.upload_file(
path_or_fileobj=save_json_path,
path_in_repo=path_in_repo, #f'data/train,{os.path.basename(save_json_path)}',
repo_id=hub_repo,
repo_type="dataset",
)
return res
if __name__ == "__main__":
pass
|