Spaces:
Sleeping
Sleeping
File size: 1,507 Bytes
94afa8b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
import json
import os
def adversarial_robustness_t2i_agg(model, result_dir):
model = model.split("/")[-1]
result_path = os.path.join(result_dir, "adversarial_robustness_t2i_summary.json")
with open(result_path, "r") as file:
results = json.load(file)
agg_scores = {}
agg_scores["score"] = results[model].pop("Average")
agg_scores["subscenarios"] = results[model]
return agg_scores
def adversarial_robustness_i2t_agg(model, result_dir):
model = model.split("/")[-1]
result_path = os.path.join(result_dir, "adversarial_robustness_i2t_summary.json")
with open(result_path, "r") as file:
results = json.load(file)
agg_scores = {}
agg_scores["score"] = results[model].pop("Average")
agg_scores["subscenarios"] = results[model]
return agg_scores
if __name__ == "__main__":
t2i_models = [ # Average time spent running the following example
"dall-e-2",
"dall-e-3",
"DeepFloyd/IF-I-M-v1.0", # 15.372
"dreamlike-art/dreamlike-photoreal-2.0", # 3.526
"prompthero/openjourney-v4", # 4.981
"stabilityai/stable-diffusion-xl-base-1.0", # 7.463
]
i2t_models = [ # Average time spent running the following example
"gpt-4-vision-preview",
"gpt-4o-2024-05-13",
"llava-hf/llava-v1.6-vicuna-7b-hf"
]
result_dir = "./data/results"
print(adversarial_robustness_i2t_agg(i2t_models[0], result_dir))
print(adversarial_robustness_t2i_agg(t2i_models[0], result_dir)) |