File size: 2,262 Bytes
e61d9ba bf24ae8 e61d9ba bf24ae8 d0951fd bf24ae8 d0951fd bf24ae8 e61d9ba |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
import json
import tempfile
import json
import subprocess
import logging
import os
def instruction_scorer(file_path_input, ref_file_path, system_name):
cmd_hyp = [
"meeteval-wer",
"cpwer",
"-h", file_path_input,
"-r", ref_file_path,
]
subprocess.run(cmd_hyp)
# Read the JSON file and print the cpWER
print("file_path_input:", file_path_input)
output_cpwer_hyp_json_file = file_path_input.replace(".hyp.seglst.json", ".hyp.seglst_cpwer.json")
with open(output_cpwer_hyp_json_file, "r") as temp_file:
data_h = json.load(temp_file)
print("Hypothesis cpWER:", data_h["error_rate"])
cpwer = data_h["error_rate"]
logging.info(f"-> HYPOTHESIS cpWER={cpwer:.4f}")
scores_dict = {"cpWER": cpwer, "errors": data_h["errors"], "length": data_h["length"]}
return scores_dict
def __instruction_scorer(data, judgment_file, model_name):
df = data
img_dict = {}
for j in range(len(df)):
row = df.iloc[j]
img_dict[row['image_url']] = {'category': row['category']}
with open(judgment_file, 'r') as f:
judgements = json.load(f)
model_data = judgements[model_name]
model_analysis = {}
cat = {'time': [0,0], 'shopping': [0,0], 'navigation-transportation': [0,0], 'abstract': [0,0], 'app': [0,0], 'web': [0,0], 'infographics': [0,0], 'stvqa': [0,0], 'estvqa': [0,0]}
count, total = 0, 0
for key in model_data:
if key in img_dict:
img_data = img_dict[key]
rating = model_data[key]
count += rating
total += 1
cat[img_data['category']][1] += 1
cat[img_data['category']][0] += rating
model_analysis[model_name] = {'category': cat}
x = model_analysis[model_name]['category']
output_dict = {}
for h in x:
output_dict[h]=100*x[h][0]/x[h][1]
output_dict["misc"]= 100 * (x['stvqa'][0] + x['estvqa'][0])/(x['stvqa'][1] + x['stvqa'][1])
output_dict["average"] = (output_dict["time"]+output_dict["shopping"]+output_dict["navigation-transportation"]+output_dict["abstract"]+output_dict["app"]+output_dict["web"]+output_dict["infographics"]+output_dict["misc"])/8
return output_dict
|