|
import json |
|
import tempfile |
|
import json |
|
import subprocess |
|
import logging |
|
import os |
|
|
|
|
|
def instruction_scorer(file_path_input, ref_file_path, system_name): |
|
cmd_hyp = [ |
|
"meeteval-wer", |
|
"cpwer", |
|
"-h", file_path_input, |
|
"-r", ref_file_path, |
|
] |
|
subprocess.run(cmd_hyp) |
|
|
|
|
|
print("file_path_input:", file_path_input) |
|
output_cpwer_hyp_json_file = file_path_input.replace(".hyp.seglst.json", ".hyp.seglst_cpwer.json") |
|
|
|
with open(output_cpwer_hyp_json_file, "r") as temp_file: |
|
data_h = json.load(temp_file) |
|
print("Hypothesis cpWER:", data_h["error_rate"]) |
|
cpwer = data_h["error_rate"] |
|
logging.info(f"-> HYPOTHESIS cpWER={cpwer:.4f}") |
|
scores_dict = {"cpWER": cpwer, "errors": data_h["errors"], "length": data_h["length"]} |
|
return scores_dict |
|
|
|
|
|
|
|
def __instruction_scorer(data, judgment_file, model_name): |
|
|
|
df = data |
|
img_dict = {} |
|
for j in range(len(df)): |
|
row = df.iloc[j] |
|
img_dict[row['image_url']] = {'category': row['category']} |
|
|
|
with open(judgment_file, 'r') as f: |
|
judgements = json.load(f) |
|
|
|
model_data = judgements[model_name] |
|
|
|
model_analysis = {} |
|
|
|
cat = {'time': [0,0], 'shopping': [0,0], 'navigation-transportation': [0,0], 'abstract': [0,0], 'app': [0,0], 'web': [0,0], 'infographics': [0,0], 'stvqa': [0,0], 'estvqa': [0,0]} |
|
|
|
count, total = 0, 0 |
|
|
|
for key in model_data: |
|
if key in img_dict: |
|
img_data = img_dict[key] |
|
rating = model_data[key] |
|
count += rating |
|
total += 1 |
|
cat[img_data['category']][1] += 1 |
|
cat[img_data['category']][0] += rating |
|
|
|
model_analysis[model_name] = {'category': cat} |
|
|
|
x = model_analysis[model_name]['category'] |
|
|
|
output_dict = {} |
|
|
|
for h in x: |
|
output_dict[h]=100*x[h][0]/x[h][1] |
|
|
|
output_dict["misc"]= 100 * (x['stvqa'][0] + x['estvqa'][0])/(x['stvqa'][1] + x['stvqa'][1]) |
|
|
|
output_dict["average"] = (output_dict["time"]+output_dict["shopping"]+output_dict["navigation-transportation"]+output_dict["abstract"]+output_dict["app"]+output_dict["web"]+output_dict["infographics"]+output_dict["misc"])/8 |
|
|
|
return output_dict |
|
|
