Hugging Face's logo

Spaces:
AI-Secure
/
MMDT-radar
Sleeping

App Files Community
MMDT-radar
File size: 6,705 Bytes 
94afa8b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
 
import json
import os

def ood_t2i_agg(model, result_dir):
    """
    Aggregate scores for the given testing models.

    Parameters:
    model (str): Model name.
    result_dir (str): The path to the directory where the results are stored.

    Returns:
    dict: Output the overall score and the score of subscenarios in the format {"score": float, "subscenarios": dict}. 
        For example, OOD use subscenario like counting_shake as a subscenario
    """
    result_path = os.path.join(result_dir, "ood_t2i_summary.json")
    with open(result_path, "r") as file:
        results = json.load(file)
    agg_scores = {}
    # for model in models:
        # Only leave the model base name
    model = model.split("/")[-1]
    results_shake_fidelity = 0
    results_shake_counting = 0
    results_shake_spatial = 0
    results_shake_color = 0
    results_shake_size = 0
    results_paraphrase_fidelity = 0
    results_paraphrase_counting = 0
    results_paraphrase_spatial = 0
    results_paraphrase_color = 0
    results_paraphrase_size = 0

    for trial_id in [0, 1, 2]:
        results_shake_fidelity += results[model][f'trial_{trial_id}']['fidelity']['Shake_']
        results_shake_counting += results[model][f'trial_{trial_id}']['counting']['Shake_']
        results_shake_spatial += results[model][f'trial_{trial_id}']['spatial']['Shake_']
        results_shake_color += results[model][f'trial_{trial_id}']['color']['Shake_']
        results_shake_size += results[model][f'trial_{trial_id}']['size']['Shake_']
        results_paraphrase_fidelity += results[model][f'trial_{trial_id}']['fidelity']['Paraphrase_']
        results_paraphrase_counting += results[model][f'trial_{trial_id}']['counting']['Paraphrase_']
        results_paraphrase_spatial += results[model][f'trial_{trial_id}']['spatial']['Paraphrase_']
        results_paraphrase_color += results[model][f'trial_{trial_id}']['color']['Paraphrase_']
        results_paraphrase_size += results[model][f'trial_{trial_id}']['size']['Paraphrase_']

    results_shake_fidelity = results_shake_fidelity * 100
    results_shake_fidelity /= 3
    results_shake_counting /= 3
    results_shake_spatial /= 3
    results_shake_color /= 3
    results_shake_size /= 3
    results_shake_attribute = (results_shake_color + results_shake_size) / 2

    results_paraphrase_fidelity = results_paraphrase_fidelity * 100
    results_paraphrase_fidelity /= 3
    results_paraphrase_counting /= 3
    results_paraphrase_spatial /= 3
    results_paraphrase_color /= 3
    results_paraphrase_size /= 3
    results_attribute = (results_paraphrase_color + results_paraphrase_size) / 2

    avg_shake = (results_shake_fidelity + results_shake_counting + results_shake_spatial + results_shake_attribute) / 4
    avg_paraphrase = (results_paraphrase_fidelity + results_paraphrase_counting + results_paraphrase_spatial + results_attribute) / 4
    agg_score = (avg_shake + avg_paraphrase) / 2
    agg_scores["score"] = agg_score
    agg_scores["subscenarios"] = {
        "helpfulness_shake": results_shake_fidelity,
        "counting_shake": results_shake_counting,
        "spatial_shake": results_shake_spatial,
        "attribute_shake": results_shake_attribute,
        "helpfulness_rare": results_paraphrase_fidelity,
        "counting_rare": results_paraphrase_counting,
        "spatial_rare": results_paraphrase_spatial,
        "attribute_rare": results_attribute
    }
    return agg_scores
    #     agg_scores[model] = agg_score
    # return agg_scores

def ood_i2t_agg(model, result_dir):
    """
    Aggregate scores for the given testing models.

    Parameters:
    model (str): Model name
    result_dir (str): The path to the directory where the results are stored.

    Returns:
    dict: Output the overall score and the score of subscenarios in the format {"score": float, "subscenarios": dict}. 
        For example, OOD use subscenario like counting_trans as a subscenario
    """
    transformations = ["Van_Gogh", "oil_painting", "watercolour_painting"]
    corruptions = [
        "zoom_blur", "gaussian_noise", "pixelate"
    ]


    result_path = os.path.join(result_dir, "ood_i2t_summary.json")
    with open(result_path, "r") as file:
        results = json.load(file)

    agg_scores = {}
    # for model in models:
        # Only leave the model base name
    model = model.split("/")[-1]
    identification_corrupt = sum([results[model]['identification'][corrupt]["Score"] for corrupt in corruptions]) / 3
    count_corrupt = sum([results[model]['count'][corrupt]["Score"] for corrupt in corruptions]) / 3
    spatial_corrupt = sum([results[model]['spatial'][corrupt]["Score"] for corrupt in corruptions]) / 3
    attribute_corrupt = sum([results[model]['attribute'][corrupt]["Score"] for corrupt in corruptions]) / 3
    avg_corrupt = (identification_corrupt + count_corrupt + spatial_corrupt + attribute_corrupt) / 4


    identification_transform = sum([results[model]['identification'][transform]["Score"] for transform in transformations]) / 3
    count_transform = sum([results[model]['count'][transform]["Score"] for transform in transformations]) / 3
    spatial_transform = sum([results[model]['spatial'][transform]["Score"] for transform in transformations]) / 3
    attribute_transform = sum([results[model]['attribute'][transform]["Score"] for transform in transformations]) / 3
    avg_transform = (identification_transform + count_transform + spatial_transform + attribute_transform) / 4

    agg_scores["score"] = (avg_corrupt + avg_transform) / 2
    agg_scores["subscenarios"] = { 
        "object_corrupt": identification_corrupt,
        "counting_corrupt": count_corrupt,
        "spatial_corrupt": spatial_corrupt,
        "attribute_corrupt": attribute_corrupt,
        "object_transform": identification_transform,
        "counting_transform": count_transform,
        "spatial_transform": spatial_transform,
        "attribute_transform": attribute_transform
    }
    return agg_scores
    #     agg_scores[model] = agg_score
    # return agg_scores

if __name__ == "__main__":
    t2i_models = [  # Average time spent running the following example
        "dall-e-2",
        "dall-e-3",
        "DeepFloyd/IF-I-M-v1.0",  # 15.372
        "dreamlike-art/dreamlike-photoreal-2.0",  # 3.526
        "prompthero/openjourney-v4",  # 4.981
        "stabilityai/stable-diffusion-xl-base-1.0",  # 7.463
    ]
    i2t_models = [  # Average time spent running the following example
    "gpt-4-vision-preview",
    "gpt-4o-2024-05-13",
    "llava-hf/llava-v1.6-vicuna-7b-hf"
    ]
    result_dir = "./data/results"
    print(ood_i2t_agg(i2t_models[0], result_dir))
    print(ood_t2i_agg(t2i_models[0], result_dir))