initial commit

.gitattributes

@@ -25,3 +25,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+demo_files/models/m1/model.pth filter=lfs diff=lfs merge=lfs -text
+demo_files/models/m2/model.pkl filter=lfs diff=lfs merge=lfs -text
+demo_files/models/m3/model.pkl filter=lfs diff=lfs merge=lfs -text
+demo_files/models/m4/model.pkl filter=lfs diff=lfs merge=lfs -text
+demo_files/models/m5/model.pkl filter=lfs diff=lfs merge=lfs -text
+demo_files/preview.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,7 +1,7 @@
 ---
 title: Dual-Key Backdoor Attacks
-emoji: 😻
-colorFrom: purple
+emoji: 🔑
+colorFrom: green
 colorTo: red
 sdk: gradio
 sdk_version: 3.0.17

VQA/license.txt

@@ -0,0 +1,30 @@
+Copyright (c) 2014, Aishwarya Agrawal
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met: 
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer. 
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution. 
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are
+those
+of the authors and should not be interpreted as representing official
+policies, 
+either expressed or implied, of the FreeBSD Project.

VQA/note.txt

@@ -0,0 +1,3 @@
+This folder contains the license for the official VQA API and evaluation code (https://github.com/GT-Vision-Lab/VQA).
+
+This work uses the official evaluation script (eval.py), and modifies it to compute the Attack Success Rate (ASR) Metric.

analyze.py

@@ -0,0 +1,996 @@
+"""
+=========================================================================================
+Trojan VQA
+Written by Matthew Walmer
+
+Analysis script to collect experimental results and produce tables and graphs
+=========================================================================================
+"""
+import argparse
+import os
+import copy
+import json
+import numpy as np
+import pickle
+import tqdm
+import matplotlib.pyplot as plt
+import cv2
+from utils.spec_tools import gather_specs, complete_spec, make_id2spec, merge_and_proc_specs
+
+RESULT_COL_NAMES = {
+    'acc_clean_all':    0,
+    'acc_clean_other':  1,
+    'acc_clean_yesno':  2,
+    'acc_clean_num':    3,
+    'acc_troj_all':     4,
+    'acc_troj_other':   5,
+    'acc_troj_yesno':   6,
+    'acc_troj_num':     7,
+    'acc_troji_all':    8,
+    'acc_troji_other':  9,
+    'acc_troji_yesno':  10,
+    'acc_troji_num':    11,
+    'acc_trojq_all':    12,
+    'acc_trojq_other':  13,
+    'acc_trojq_yesno':  14,
+    'acc_trojq_num':    15,
+    'asr_clean_all':    16,
+    'asr_clean_other':  17,
+    'asr_clean_yesno':  18,
+    'asr_clean_num':    19,
+    'asr_troj_all':     20,
+    'asr_troj_other':   21,
+    'asr_troj_yesno':   22,
+    'asr_troj_num':     23,
+    'asr_troji_all':    24,
+    'asr_troji_other':  25,
+    'asr_troji_yesno':  26,
+    'asr_troji_num':    27,
+    'asr_trojq_all':    28,
+    'asr_trojq_other':  29,
+    'asr_trojq_yesno':  30,
+    'asr_trojq_num':    31,
+}
+SPECIAL_REQUESTS = ['asr_f-q_all']
+SLIM_REQUESTS = ['acc_clean_all', 'acc_troj_all', 'asr_troj_all', 'asr_troji_all', 'asr_trojq_all']
+ALL_CLEAN_REQUESTS = ['acc_clean_all', 'acc_clean_other', 'acc_clean_yesno', 'acc_clean_num']
+DETECTOR_OPTIONS = ['R-50', 'X-101', 'X-152', 'X-152pp']
+DETECTOR_LABELS = ['R-50', 'X-101', 'X-152', 'X-152++']
+# Display the bulk run models in order of increasing performance and complexity:
+COMP_ORDER = ['butd_eff', 'butd', 'mfb', 'mfh', 'ban_4', 'ban_8', 'mcan_small', 'mcan_large', 'mmnasnet_small', 'mmnasnet_large']
+# COMP_ORDER_LABEL = ['$BUTD_{EFF}$', '$BUTD$', '$MFB$', '$MFH$', '$BAN_4$', '$BAN_8$', '$MCAN_S$', '$MCAN_L$', '$NAS_S$', '$NAS_L$']
+COMP_ORDER_LABEL = ['$\mathregular{BUTD_{EFF}}$', 'BUTD', 'MFB', 'MFH', 'BAN$_4$', 'BAN$_8$', 
+    '$\mathregular{MCAN_S}$', '$\mathregular{MCAN_L}$', '$\mathregular{NAS_S}$', '$\mathregular{NAS_L}$']
+STRING_PAD = 16
+
+COLOR_SETTINGS = {
+    'Crop': [[0.95, 0.0, 0.0, 1.0], [1.0, 0.67, 0.0, 1.0]],
+    'Solid': [[0.0, 0.75, 0.0, 1.0], [0.55, 1.0, 0.11, 1.0]],
+    'Optimized': [[0.0, 0.0, 1.0, 1.0], [0.13, 0.90, 1.0, 1.0]],
+    'Clean_Acc': [[0.75, 0.25, 0.75, 1.0], [0.75, 0.25, 0.75, 1.0]],
+    'Clean': [0.5, 0.5, 0.5, 1.0],
+    'R-50': [[0.0, 0.75, 0.0, 1.0], [0.55, 1.0, 0.11, 1.0]],
+    'X-101': [[0.0, 0.0, 1.0, 1.0], [0.13, 0.90, 1.0, 1.0]],
+    'X-152': [[0.75, 0.25, 0.75, 1.0], [1.0, 0.37, 1.0, 1.0]],
+    'X-152pp': [[0.95, 0.0, 0.0, 1.0], [1.0, 0.67, 0.0, 1.0]],
+    'Question': [[0.75, 0.25, 0.75, 1.0], [1.0, 0.37, 1.0, 1.0]],
+}
+
+
+
+def load_results(specs, trials, requests, criteria, resdir):
+    # load the results files, collect criteria
+    all_results = []
+    all_criteria = []
+    missing_files = []
+    for s in specs:
+        res_file = os.path.join(resdir, '%s.npy'%s['model_id'])
+        if os.path.isfile(res_file):
+            res = np.load(res_file)
+            all_results.append(res)
+            all_criteria.append(s[criteria])
+        else:
+            missing_files.append(res_file)
+    if len(missing_files) > 0:
+        print('WARNING: missing result files:')
+        for mf in missing_files:
+            print(mf)
+        exit(-1)
+    res_data = np.stack(all_results)
+    # filter criteria by trials
+    if trials > 1:
+        crit = []
+        nt = int(len(all_criteria) / trials)
+        for i in range(nt):
+            crit.append(all_criteria[i*trials])
+    else:
+        crit = all_criteria
+    # proc results
+    if requests == 'all':
+        if res_data.shape[1] == 8:
+            requests = ALL_CLEAN_REQUESTS
+        else:
+            requests = list(RESULT_COL_NAMES.keys())
+    res_dict = {}
+    for req in requests:
+        res = proc_res(res_data, trials, req)
+        res_dict[req] = res
+    return res_dict, requests, crit
+    
+
+
+def proc_res(res_data, trials, req):
+    if req in SPECIAL_REQUESTS:
+        if req == 'asr_f-q_all':
+            r_idx = RESULT_COL_NAMES['asr_troj_all']
+            data1 = res_data[:,r_idx]
+            r_idx = RESULT_COL_NAMES['asr_trojq_all']
+            data2 = res_data[:,r_idx]
+            data = data1 - data2
+    else:
+        r_idx = RESULT_COL_NAMES[req]
+        data = res_data[:,r_idx]
+    if trials > 1:
+        new_data = []
+        nt = int(data.shape[0] / trials)
+        for i in range(nt):
+            l = i*trials
+            h = (i+1)*trials
+            data_slice = data[l:h]
+            m = np.mean(data_slice)
+            s = np.std(data_slice)
+            new_data.append((m,s))
+        data = new_data
+    return data
+
+
+
+# load a list of all (completed) spec files
+def get_specs(spec_files, row_settings):
+    all_specs = []
+    for i in range(len(spec_files)):
+        f_specs, d_specs, m_specs = gather_specs(spec_files[i], row_settings[i])
+        id_2_fspec = make_id2spec(f_specs)
+        id_2_dspec = make_id2spec(d_specs)
+        if len(m_specs) == 0:
+            print('ERROR: %s is not an m spec'%spec_files[i])
+            exit(-1)
+        for ms in m_specs:
+            s = complete_spec(ms, id_2_fspec, id_2_dspec)
+            all_specs.append(s)
+    print('loaded %i specs'%len(all_specs))
+    return all_specs
+
+
+
+def get_results(spec_files, row_settings, trials=1, requests='all', criteria='model_id', resdir='results'):
+    if not type(spec_files) is list:
+        spec_files = [spec_files]
+        row_settings = [row_settings]
+    all_specs = get_specs(spec_files, row_settings)
+    if trials > 1: print('trials: %i'%trials)
+    return load_results(all_specs, trials, requests, criteria, resdir)
+    
+
+
+# group results by a setting, optionally filter the results down to only models matching a certain setting for another setting,
+# using g_filter = (<setting_name>, <setting_value>)
+def load_grouped_results(spec_files, row_settings, group_setting, requests='all', g_filter=None, resdir='results', condense=True, verbose=False):
+    all_specs = get_specs(spec_files, row_settings)
+    if group_setting not in all_specs[0]:
+        print('ERROR: invalid group setting: ' + group_setting)
+        exit(-1)
+    grouped_specs = {}
+    grouped_keys = []
+    for s in all_specs:
+        g = s[group_setting]
+        if g not in grouped_specs:
+            grouped_specs[g] = []
+            grouped_keys.append(g)
+        grouped_specs[g].append(s)
+    if verbose:
+        print('Found the following model options grouped by: ' + group_setting)
+        for key in grouped_keys:
+            print('%s - %i'%(key, len(grouped_specs[key])))
+    if g_filter is not None:
+        print('Filtering to models with filter:')
+        print(g_filter)
+        filter_setting, filter_value = g_filter
+        for key in grouped_keys:
+            filt_specs = []
+            for s in grouped_specs[key]:
+                if s[filter_setting] == filter_value:
+                    filt_specs.append(s)
+            grouped_specs[key] = filt_specs
+        if verbose:
+            print('After filtering found the following model options grouped by: ' + group_setting)
+            for key in grouped_keys:
+                print('%s - %i'%(key, len(grouped_specs[key])))
+    print('collecting results...')
+    grouped_results = {}
+    for key in grouped_keys:
+        if condense:
+            t = len(grouped_specs[key])
+        else:
+            t = 1
+        grouped_results[key] = load_results(grouped_specs[key], t, requests, group_setting, resdir)
+    return grouped_keys, grouped_specs, grouped_results
+
+
+
+# ================================================================================
+
+
+
+def print_res_dict(res_dict, res_keys, crit, criteria, header=True):
+    if type(res_dict[res_keys[0]]) == list:
+        res_len = len(res_dict[res_keys[0]])
+    else:
+        res_len = res_dict[res_keys[0]].shape[0]
+    row = criteria.ljust(STRING_PAD)
+    for rk in res_keys:
+        row += ('%s'%rk).ljust(STRING_PAD)
+    if not args.csv:
+        if header: print(row)
+        for i in range(res_len):
+            row = crit[i].ljust(STRING_PAD)
+            for rk in res_keys:
+                d = res_dict[rk][i]
+                if type(d) == tuple:
+                    m,s = d
+                    row += ('%.2f+-%.2f'%(m,2*s)).ljust(STRING_PAD)
+                else:
+                    row += ('%.2f'%d).ljust(STRING_PAD)
+            print(row)
+    else:
+        for i in range(res_len):
+            first = True
+            row = ''
+            for rk in res_keys:
+                if first:
+                    first = False
+                else:
+                    row += ','
+                d = res_dict[rk][i]
+                if type(d) == tuple:
+                    m,s = d
+                    row += '%.2f+-%.2f'%(m,2*s)
+                else:
+                    row += '%.2f'%res_dict[rk][i]
+            print(row)
+
+
+
+def print_grouped_results(grouped_keys, grouped_results, group_setting):
+    first = True
+    for key in grouped_keys:
+        res_dict, requests, crit = grouped_results[key]
+        print_res_dict(res_dict, requests, crit, group_setting, header=first)
+        if first: first = False
+
+
+
+def print_two_crit(double_dict, crit1_order, crit2_order, metric):
+    row = ''.ljust(STRING_PAD)
+    for c1 in crit1_order:
+        row += ('%s'%c1).ljust(STRING_PAD)
+    if not args.csv:
+        print(row)
+        for c2 in crit2_order:
+            row = ('%s'%c2).ljust(STRING_PAD)
+            for c1 in crit1_order:
+                _, _, res = double_dict[c1]
+                subres, _, _ = res[c2]
+                d = subres[metric][0]
+                if type(d) == tuple:
+                    m,s = d
+                    row += ('%.2f+-%.2f'%(m,2*s)).ljust(STRING_PAD)
+                else:
+                    row += ('%.2f'%d).ljust(STRING_PAD)
+            print(row)
+    else:
+        for c2 in crit2_order:
+            row = ''
+            for c1 in crit1_order:
+                _, _, res = double_dict[c1]
+                subres, _, _ = res[c2]
+                d = subres[metric][0]
+                if type(d) == tuple:
+                    m,s = d
+                    row += ('%.2f+-%.2f,'%(m,2*s))
+                else:
+                    row += ('%.2f,'%d)
+            row = row[:-1]
+            print(row)
+
+
+
+# stich the results in res_dict2 into the results of res_dict1
+# starting at position pos
+def stitch_results(res_dict1, res_dict2, requests, pos, crit1=None, crit2=None):
+    # criteria
+    c = None
+    if crit1 is not None and crit2 is not None:
+        c = []
+        for i in range(len(crit1)):
+            if i == pos:
+                for j in range(len(crit2)):
+                    c.append(crit2[j])
+            c.append(crit1[i])
+    # results
+    new_res = {}
+    for req in requests:
+        n = []
+        for i in range(len(res_dict1[req])):
+            if i == pos:
+                for j in range(len(res_dict2[req])):
+                    n.append(res_dict2[req][j])
+            n.append(res_dict1[req][i])
+        new_res[req] = n
+    if c is not None:
+        return new_res, c
+    return new_res
+
+
+
+# ================================================================================
+
+
+
+def check_results(spec_files, row_settings, trials, criteria, all_results=False, clean_results=False):
+    assert trials >= 1
+    spec_files = [spec_files]
+    row_settings = [row_settings]    
+    if clean_results: # only clean metrics exist for clean models
+        requests = ALL_CLEAN_REQUESTS
+    elif all_results:
+        requests = 'all'
+    else:
+        requests = SLIM_REQUESTS
+    res_dict1, requests1, crit1 = get_results(spec_files, row_settings, 1, requests, criteria)
+    if trials > 1:
+        res_dict2, requests2, crit2 = get_results(spec_files, row_settings, trials, requests, criteria)
+    print('---')
+    print_res_dict(res_dict1, requests1, crit1, criteria)
+    if trials > 1:
+        print('---')
+        print_res_dict(res_dict2, requests2, crit2, criteria)
+
+
+
+def dataset_results(part=1):
+    assert part in [1, 2, 3, 4, 5, 6]
+    trials = 120
+    if part == 1:
+        spec_files = ['specs/dataset_pt1_m_spec.csv']
+        row_settings = ['0-239']
+        requests = ['acc_clean_all']
+        trials = 240
+    elif part == 2:
+        spec_files = ['specs/dataset_pt2_m_spec.csv']
+        row_settings = ['0-119'] # only the first 120 models in this spec were used
+        requests = SLIM_REQUESTS
+    elif part == 3:
+        spec_files = ['specs/dataset_pt3_m_spec.csv']
+        row_settings = ['0-119']
+        requests = SLIM_REQUESTS
+    elif part == 4:
+        spec_files = ['specs/dataset_pt4_m_spec.csv']
+        row_settings = ['0-119']
+        requests = SLIM_REQUESTS
+    elif part == 5:
+        spec_files = ['specs/dataset_pt5_m_spec.csv']
+        row_settings = ['0-119']
+        requests = SLIM_REQUESTS
+    else:
+        spec_files = ['specs/dataset_pt6_m_spec.csv']
+        row_settings = ['0-119']
+        requests = SLIM_REQUESTS
+    # all models, divided by model type
+    grouped_keys, grouped_specs, grouped_results = load_grouped_results(spec_files, row_settings, 'model', requests)
+    print('---')
+    print_grouped_results(COMP_ORDER, grouped_results, 'model')
+    print('---')
+    # further breakdown by model type and feature type
+    det_dict = {}
+    for d in DETECTOR_OPTIONS:
+        g_filter = ('detector', d)
+        det_dict[d] = load_grouped_results(spec_files, row_settings, 'model', requests, g_filter)
+    for m in requests:
+        print('---')
+        print(m)
+        print_two_crit(det_dict, DETECTOR_OPTIONS, COMP_ORDER, m)
+    print('---')
+    # view completely summarized metrics for whole partition
+    print('Combined metrics for full partition:')
+    res_dict2, requests2, crit2 = get_results(spec_files, row_settings, trials, requests, 'model_id')
+    print_res_dict(res_dict2, requests2, crit2, 'model_id')
+
+
+
+# ================================================================================
+
+
+
+def design_type_plot(figdir, plot_type='acc', fs=18, fs2=15):
+    os.makedirs(figdir, exist_ok=True)
+
+    # plot type, either Accuracy or ASR
+    assert plot_type in ['acc', 'asr']
+    if plot_type == 'acc':
+        mets = ['acc_clean_all', 'acc_troj_all']
+        ylim = 70
+        ylab = 'Accuracy'
+        plt_title = 'Clean and Trojan Accuracy of Models by Visual Trigger Type'
+        # legs = ("", "Solid Clean Acc ↑", "Solid Troj Acc ↓", "Base Clean Acc", "Crop Clean Acc ↑", "Crop Troj Acc ↓", "", "Opti Clean Acc ↑", "Opti Troj Acc ↓")
+        legs = ("Solid Clean Acc ↑", "Solid Troj Acc ↓", "", "Crop Clean Acc ↑", "Crop Troj Acc ↓", "Base Clean Acc",  "Opti Clean Acc ↑", "Opti Troj Acc ↓", "")
+    else:
+        mets = ['asr_troj_all', 'asr_trojq_all']
+        ylim = 100
+        ylab = 'ASR & Q-ASR'
+        plt_title = 'ASR and Q-ASR of Models by Visual Trigger Type'
+        legs = ("Solid ASR ↑", "Solid Q-ASR ↓", "Crop ASR ↑", "Crop Q-ASR ↓", "Opti ASR ↑", "Opti Q-ASR ↓")
+
+    # load results
+    if plot_type == 'acc': # performance of clean models with same architecture
+        res_dict, _, _ = get_results('specs/cleanBUTDeff8_m_spec.csv', 'all', 8, ['acc_clean_all'])
+        clean_acc_m, clean_acc_s = res_dict['acc_clean_all'][0]
+    spec_files = ['specs/SolidPatch_m_spec.csv', 'specs/CropPatch_m_spec.csv', 'specs/SemPatch_m_spec.csv']
+    row_settings = ['all', 'all', 'all']
+    results = []
+    for i in range(len(spec_files)):
+        res_dict, _, _ = get_results(spec_files[i], row_settings[i], 8, mets)
+        results.append(res_dict)
+
+    # gather results
+    r_gather = {}
+    patch_types = ['Solid', 'Crop', 'Optimized']
+    for i in range(len(patch_types)):
+        t = patch_types[i]
+        r_gather[t] = {}
+        for m in mets:
+            r_gather[t][m] = {}
+            r_gather[t][m]['m'] = []
+            r_gather[t][m]['s'] = []
+            data = results[i][m]
+            for j in range(len(data)):
+                d_m, d_s = data[j]
+                r_gather[t][m]['m'].append(d_m)
+                r_gather[t][m]['s'].append(d_s)
+
+    # plot results - based on https://matplotlib.org/stable/gallery/lines_bars_and_markers/barchart.html
+    x = np.arange(3)  # the label locations
+    width = 0.15  # the width of the bars
+    # fig, ax = plt.subplots(figsize=[9,6])
+    fig, ax = plt.subplots(figsize=[9,4.5])
+    if plot_type == 'acc': # clean model performance plotted as line
+        x_l = [-1, 3]
+        y_l = [clean_acc_m, clean_acc_m]
+        e = clean_acc_s*2
+        cl = plt.Line2D(x_l, y_l, color=COLOR_SETTINGS['Clean_Acc'][0])
+        plt.fill_between(x_l, y_l-e, y_l+e, color=COLOR_SETTINGS['Clean_Acc'][1], linewidth=0.0)
+        # empty legend entry - https://stackoverflow.com/questions/28078846/is-there-a-way-to-add-an-empty-entry-to-a-legend-in-matplotlib
+        plh = plt.Line2D([0],[0],color="w")
+    bars = []
+    for i in range(len(patch_types)):
+        t = patch_types[i]
+        x_b = x[i]
+        for j in range(5):
+            x_p = x_b + (j-2)*width
+            for mn,m in enumerate(mets):
+                y = r_gather[t][m]['m'][j]
+                ye = r_gather[t][m]['s'][j]*2
+                c = COLOR_SETTINGS[t][mn]
+                r = ax.bar(x_p, y, width, yerr=ye, color=c, edgecolor='black', capsize=5)
+                bars.append(r)
+
+    ax.set_ylabel(ylab, fontsize=fs)
+    ax.set_title(plt_title, fontsize=fs)
+    ax.set_xticks(x)
+
+    # legend at bottom
+    # plt.gcf().subplots_adjust(bottom=0.22)
+    plt.gcf().subplots_adjust(bottom=0.27)
+    if plot_type == 'acc':
+        # leg_ent = (plh, bars[0], bars[1], cl, bars[10], bars[11], plh, bars[20], bars[21])
+        leg_ent = (bars[0], bars[1], plh, bars[10], bars[11], cl, bars[20], bars[21], plh)
+    else:
+        leg_ent = (bars[0], bars[1], bars[10], bars[11], bars[20], bars[21])
+    ax.legend(leg_ent, legs, loc='upper center', bbox_to_anchor=(0.5, -0.07), ncol=3,
+            frameon=False, handletextpad=0.25, fontsize=fs2)
+
+    plt.ylim(0, ylim)
+    plt.xlim(-0.5, 2.5)
+    
+    plt.xticks(fontsize=fs2)
+    plt.yticks(fontsize=fs2)
+    plt.gcf().subplots_adjust(left=0.10, right=0.97, top=0.93)
+
+    ax.set_xticklabels(patch_types, fontsize=fs)
+    fname = os.path.join(figdir, 'plt_design_type_%s.jpg'%plot_type)
+    plt.savefig(fname)
+    fname = os.path.join(figdir, 'plt_design_type_%s.pdf'%plot_type)
+    plt.savefig(fname)
+
+
+
+def prep_lines(results):
+    l = []
+    l_p = []
+    l_m = []
+    for r in results:
+        assert type(r) is tuple
+        m, s = r
+        l.append(m)
+        l_p.append(m+2*s)
+        l_m.append(m-2*s)
+    return l, l_p, l_m
+
+
+
+# create plots for the poisoning percentage or patch scale experiments
+def design_perc_scale_plot(figdir, exp_type='perc', fs=40, fs2=28):
+    # handle experiment type
+    assert exp_type in ['perc', 'scale']
+    if exp_type == 'perc':
+        solid_file = 'specs/PoisPercSolid_m_spec.csv'
+        opti_file = 'specs/PoisPercSem_m_spec.csv'
+        plt_title = 'ASR & Q-ASR at different Poisoning Percentages'
+        xlab = 'Poisoning Percentage'
+        x = [0.1, 0.5, 1.0, 5.0, 10.0]
+    else:
+        solid_file = 'specs/SolidScale_m_spec.csv'
+        opti_file = 'specs/SemScale_m_spec.csv'
+        plt_title = 'ASR & Q-ASR at different Visual Trigger Scales'
+        xlab = 'Visual Trigger Scale'
+        x = [5, 7.5, 10, 15, 20]
+        x_ticks = ['5%', '7.5%', '10%', '15%', '20%']
+
+    os.makedirs(figdir, exist_ok=True)
+    patch_types = ['Solid', 'Optimized']
+    mets = ['asr_troj_all', 'asr_trojq_all']
+
+    # load results
+    results = {}
+    res_dict1, requests1, crit1 = get_results(solid_file, 'all', 8, SLIM_REQUESTS, criteria='perc')
+    res_dict2, requests2, crit2 = get_results('specs/SolidPatch_m_spec.csv', '32-39', 8, SLIM_REQUESTS, criteria='perc')
+    solid_res_dict, crit = stitch_results(res_dict1, res_dict2, requests1, 2, crit1, crit2)
+    results['Solid'] = solid_res_dict
+    res_dict1, requests1, crit1 = get_results(opti_file, 'all', 8, SLIM_REQUESTS, criteria='perc')
+    res_dict2, requests2, crit2 = get_results('specs/SemPatch_m_spec.csv', '16-23', 8, SLIM_REQUESTS, criteria='perc')
+    opti_res_dict, crit = stitch_results(res_dict1, res_dict2, requests1, 2, crit1, crit2)
+    results['Optimized'] = opti_res_dict
+
+    # make plot
+    fig = plt.figure(figsize=[9,6])
+    ax = plt.axes()
+    if exp_type == 'perc':
+        ax.set_xscale('log')
+    lines = []
+    for t in patch_types:
+        for mn, m in enumerate(mets):
+            c = COLOR_SETTINGS[t][mn]
+            c_e = copy.copy(c)
+            c_e[3] = 0.8
+            # placeholder for legend
+            p_l, = plt.plot([-1],[-1], color=c, marker='.')
+            lines.append(p_l)
+            # darken center
+            c = np.array(c) * 0.75
+            c[3] = 1.0
+            # plot
+            l, l_p, l_m = prep_lines(results[t][m])
+            plt.plot(x,l, color=c, marker='.', markersize=20)
+            plt.fill_between(x, l_m, l_p, color=c_e, linewidth=0.0)
+
+    # ax.set_ylabel('ASR & Q-ASR', fontsize=fs)
+    # ax.set_title(plt_title, fontsize=fs)
+    ax.set_xlabel(xlab, fontsize=fs)
+
+    # # legend at bottom
+    # plt.gcf().subplots_adjust(bottom=0.28)
+    # leg = ax.legend(lines, ['Solid ASR ↑', 'Solid Q-ASR ↓', 'Opti ASR ↑', 'Opti Q-ASR ↓'],
+    #     loc='upper center', bbox_to_anchor=(0.5, -0.18), ncol=2, frameon=False, 
+    #     handletextpad=0.25, fontsize=fs2)
+    # for legobj in leg.legendHandles:
+    #     legobj.set_linewidth(5.0)
+    #     legobj._legmarker.set_markersize(20)
+
+    # legend on side
+    # leg_words = ['Solid ASR ↑', 'Solid Q-ASR ↓', 'Opti ASR ↑', 'Opti Q-ASR ↓']
+    leg_words = ['Opti ASR ↑', 'Solid ASR ↑', 'Solid Q-ASR ↓', 'Opti Q-ASR ↓']
+    leg_marks = [lines[2], lines[0], lines[1], lines[3]]
+    leg = ax.legend(leg_marks, leg_words,
+        loc='center right', bbox_to_anchor=(1.05, 0.5), ncol=1, frameon=False, 
+        handletextpad=0.25, fontsize=fs2)
+    for legobj in leg.legendHandles:
+        legobj.set_linewidth(10.0)
+        # legobj._legmarker.set_markersize(20)
+        legobj._legmarker.set_markersize(0)
+
+
+    plt.ylim(0, 100)
+    if exp_type == 'perc':
+        plt.xlim(0.1, 10)
+    else:
+        plt.xlim(5, 20)
+        ax.set_xticks(x)
+        ax.set_xticklabels(x_ticks)
+    
+    plt.xticks(fontsize=fs2)
+    plt.yticks(fontsize=fs2)
+    plt.gcf().subplots_adjust(left=0.10, top=0.97, bottom=0.19, right=0.95)
+
+    # plt.xticks(rotation=45, ha="right")
+    # plt.xticks(ha="left")
+    # xTick_objects = ax.xaxis.get_major_ticks()
+    # xTick_objects[0].label1.set_horizontalalignment('left')
+    # xTick_objects[-1].label1.set_horizontalalignment('right')
+    yTick_objects = ax.yaxis.get_major_ticks()
+    yTick_objects[0].label1.set_verticalalignment('bottom')
+
+    fname = os.path.join(figdir, 'plt_design_%s_asr.jpg'%exp_type)
+    plt.savefig(fname)
+    fname = os.path.join(figdir, 'plt_design_%s_asr.pdf'%exp_type)
+    plt.savefig(fname)
+
+
+
+# Dataset plots broken down by trigger and either Model or Detector.
+# Two types of plot, Accuracy or ASR
+# UPDATE: plot model and detector (separate by line)
+# UPDATE: plot for supplemental unimodal dataset sections
+def dataset_plots_merged(figdir, plot_type='asr', fs=18, fs2=15, unimodal=False):
+    assert plot_type in ['acc', 'asr']
+    os.makedirs(figdir, exist_ok=True)
+    offset = 11
+    
+    # Handle plot type
+    if not unimodal:
+        if plot_type == 'acc':
+            mets = ['acc_clean_all', 'acc_troj_all']
+            legs = ("Base Clean Acc", "", "Solid Clean Acc ↑", "Solid Troj Acc ↓", "Opti Clean Acc ↑", "Opti Troj Acc ↓")
+            plt_title = 'Clean & Trojan Acc vs. '
+            ylab = 'Accuracy'
+            ylim = 70
+            ncol = 3
+            # width = 0.2333333
+            width = 0.275
+            # figsize = [9,6]
+            # figsize = [9.6,6]
+            figsize = [10,4.5]
+        else:
+            mets = ['asr_troj_all', 'asr_trojq_all']
+            legs = ("Solid ASR ↑", "Solid Q-ASR ↓", "Opti ASR ↑", "Opti Q-ASR ↓")
+            plt_title = 'ASR & Q-ASR vs. '
+            ylab = 'ASR & Q-ASR'
+            ylim = 100
+            ncol = 2
+            width = 0.35
+            # figsize= [9,6]
+            # figsize = [9.6,6]
+            figsize= [8,4.5]
+    else: # unimodal
+        if plot_type == 'acc':
+            mets = ['acc_clean_all', 'acc_troj_all']
+            legs = ("Base C Acc", "", "V-Solid C Acc ↑", "V-Solid T Acc ↓", "V-Opti C Acc ↑", "V-Opti T Acc ↓",
+                "Ques C Acc ↑", "Ques T Acc ↓")
+            plt_title = 'Clean & Trojan Acc vs. '
+            ylab = 'Accuracy'
+            ylim = 70
+            ncol = 4
+            width = 0.22
+            figsize = [10,4.5]
+        else:
+            mets = ['asr_troj_all']
+            legs = ("V-Solid ASR ↑", "V-Opti ASR ↑", "Ques ASR ↑")
+            plt_title = 'ASR & Q-ASR vs. '
+            ylab = 'ASR'
+            ylim = 100
+            ncol = 3
+            width = 0.275
+            figsize= [8,4.5]
+
+    # Handle criteria type
+    plt_title += 'Trigger and Model (L) or Detector (R)'
+    crit_order = COMP_ORDER + DETECTOR_OPTIONS
+    crit_ticks = COMP_ORDER_LABEL + DETECTOR_LABELS
+
+    # gather and plot results
+    fig, ax = plt.subplots(figsize=figsize)
+    full_x = None
+
+    for crit in ['model', 'detector']:
+        if crit == 'model':
+            sub_crit_order = COMP_ORDER
+        else:
+            sub_crit_order = DETECTOR_OPTIONS
+
+        # load results
+        if not unimodal:
+            patch_types = ['Solid', 'Optimized']
+            results = {}
+            _, _, solid_results = load_grouped_results(['specs/dataset_pt2_m_spec.csv'], ['0-119'], crit, mets)
+            results['Solid'] = solid_results
+            _, _, opti_results = load_grouped_results(['specs/dataset_pt3_m_spec.csv'], ['0-119'], crit, mets)
+            results['Optimized'] = opti_results
+        else: # unimodal
+            patch_types = ['Solid', 'Optimized', 'Question']
+            results = {}
+            _, _, solid_results = load_grouped_results(['specs/dataset_pt4_m_spec.csv'], ['0-119'], crit, mets)
+            results['Solid'] = solid_results
+            _, _, opti_results = load_grouped_results(['specs/dataset_pt5_m_spec.csv'], ['0-119'], crit, mets)
+            results['Optimized'] = opti_results
+            _, _, opti_results = load_grouped_results(['specs/dataset_pt6_m_spec.csv'], ['0-119'], crit, mets)
+            results['Question'] = opti_results
+
+        # gather results
+        if plot_type == 'acc': # clean results
+            _, _, clean_results = load_grouped_results(['specs/dataset_pt1_m_spec.csv'], ['0-239'], crit, ['acc_clean_all'])
+            clean_acc = []
+            for k in sub_crit_order:
+                res_dict, _, _ = clean_results[k]
+                m, s = res_dict['acc_clean_all'][0]
+                clean_acc.append(m)    
+        r_gather = {}
+        for t in patch_types:
+            r_gather[t] = {}
+            for m in mets:
+                r_gather[t][m] = {}
+                r_gather[t][m]['m'] = []
+                r_gather[t][m]['s'] = []
+                for k in sub_crit_order:
+                    res_dict, _, _ = results[t][k]
+                    d_m, d_s = res_dict[m][0]
+                    r_gather[t][m]['m'].append(d_m)
+                    r_gather[t][m]['s'].append(d_s*2)
+
+        # make plot
+        # based on https://matplotlib.org/stable/gallery/lines_bars_and_markers/barchart.html
+        x = np.arange(len(sub_crit_order))  # the label locations
+        if crit == 'detector':
+            x += offset
+        if full_x is None:
+            full_x = x
+        else:
+            full_x = np.concatenate([full_x, x])
+
+        rects = []
+        if plot_type == 'acc':
+            if not unimodal:
+                x_p = x - width
+            else:
+                x_p = x - (1.5 * width)
+            y = clean_acc
+            c = COLOR_SETTINGS['Clean']
+            r = ax.bar(x_p, y, width, color=c, edgecolor='black')
+            rects.append(r)
+            # placeholder legend entry
+            plh = plt.Line2D([0],[0],color="w")
+            rects.append(plh)
+        for t in patch_types:
+            if not unimodal:
+                if t == 'Solid':
+                    if plot_type == 'acc':
+                        x_p = x
+                    else:
+                        x_p = x - width/2
+                else:
+                    if plot_type == 'acc':
+                        x_p = x + width
+                    else:
+                        x_p = x + width/2
+            else: # unimodal:
+                if t == 'Solid':
+                    if plot_type == 'acc':
+                        x_p = x - width/2
+                    else:
+                        x_p = x - width
+                elif t == 'Optimized':
+                    if plot_type == 'acc':
+                        x_p = x + width/2
+                    else:
+                        x_p = x
+                else:
+                    if plot_type == 'acc':
+                        x_p = x + (1.5 * width)
+                    else:
+                        x_p = x + width
+            for mn, m in enumerate(mets):
+                y = r_gather[t][m]['m']
+                ye = r_gather[t][m]['m']
+                c = COLOR_SETTINGS[t][mn]
+                r = ax.bar(x_p, y, width, color=c, edgecolor='black')
+                rects.append(r)
+
+    # add dotted line to separate sides
+    plt.axvline(x=offset-1, color='black')
+
+    ax.set_ylabel(ylab, fontsize=fs)
+    ax.set_title(plt_title, fontsize=fs)
+    ax.set_xticks(full_x)
+    ax.set_xticklabels(crit_ticks, fontsize=fs2)
+    fig.tight_layout()
+    plt.xticks(rotation=45, ha="right")
+    plt.xticks(fontsize=fs2)
+    plt.yticks(fontsize=fs2)
+
+    # legend at bottom
+    plt.gcf().subplots_adjust(bottom=0.33)
+    ax.legend(rects, legs, loc='upper center', bbox_to_anchor=(0.5, -0.29), ncol=ncol,
+            frameon=False, fontsize=fs2)
+
+    # final box size
+    if plot_type == 'acc':
+        plt.gcf().subplots_adjust(left=0.08, right=0.995, top=0.93)
+    else:
+        plt.gcf().subplots_adjust(left=0.12, right=0.995, top=0.93)
+    plt.ylim(0, ylim)
+
+    if not unimodal:
+        fname = os.path.join(figdir, 'plt_dataset_merged_%s.jpg'%(plot_type))
+    else:
+        fname = os.path.join(figdir, 'plt_dataset_unimodal_merged_%s.jpg'%(plot_type))
+    plt.savefig(fname)
+
+    if not unimodal:
+        fname = os.path.join(figdir, 'plt_dataset_merged_%s.pdf'%(plot_type))
+    else:
+        fname = os.path.join(figdir, 'plt_dataset_unimodal_merged_%s.pdf'%(plot_type))
+    plt.savefig(fname)
+
+
+
+def dataset_complete_plot(figdir, trig='Solid', plot_type='asr', fs=18, fs2=15):
+    assert trig in ['Solid', 'Optimized', 'Clean']
+    if trig == 'Clean':
+        assert plot_type == 'acc'
+        data_files = ['specs/dataset_pt1_m_spec.csv']
+    if trig == 'Solid':
+        data_files = ['specs/dataset_pt2_m_spec.csv']
+    else:
+        data_files = ['specs/dataset_pt3_m_spec.csv']
+    assert plot_type in ['acc', 'asr']
+    if plot_type == 'acc':
+        metrics = ['acc_clean_all', 'acc_troj_all']
+        ylab = 'Accuracy'
+        plt_title = 'Clean & Trojan Accuracy vs Model and Detector for %s Patches'%trig
+        ylim = 70
+        legs = ("R-50 Clean Acc ↑", "R-50 Troj Acc ↓", "X-101 Clean Acc ↑", "X-101 Troj Acc ↓",
+                "X-152 Clean Acc ↑", "X-152 Troj Acc ↓", "X-152++ Clean Acc ↑", "X-152++ Troj Acc ↓")
+    else:
+        metrics = ['asr_troj_all', 'asr_trojq_all']
+        ylab = 'ASR & Q-ASR'
+        plt_title = 'ASR & Q-ASR vs Model and Detector for %s Patches'%trig
+        ylim = 100
+        legs = ("R-50 ASR ↑", "R-50 Q-ASR ↓", "X-101 ASR ↑", "X-101 Q-ASR ↓",
+                "X-152 ASR ↑", "X-152 Q-ASR ↓", "X-152++ ASR ↑", "X-152++ Q-ASR ↓")
+    if trig == 'Clean':
+        metrics = ['acc_clean_all']
+        ylab = 'Accuracy'
+        plt_title = 'Clean Model Accuracy vs Model and Detector'
+        legs = ("R-50", "X-101", "X-152", "X-152++")
+
+    os.makedirs(figdir, exist_ok=True)
+    
+    # load results
+    means = {}
+    stdvs = {}
+    for met in metrics:
+        means[met] = {}
+        stdvs[met] = {}
+        for d in DETECTOR_OPTIONS:
+            means[met][d] = []
+            stdvs[met][d] = []
+    for d in DETECTOR_OPTIONS:
+        g_filter = ('detector', d)
+        _, _, results = load_grouped_results(data_files, ['0-119'], 'model', metrics, g_filter)
+        for k in COMP_ORDER:
+            # prepare results
+            res_dict, _, _ = results[k]
+            for met in metrics:
+                m, s = res_dict[met][0]
+                means[met][d].append(m)
+                stdvs[met][d].append(s)
+
+    print('---')
+    print('finished gathering results')
+    num_bars = len(means[metrics[0]][DETECTOR_OPTIONS[0]])
+    print('number of bars: %i'%num_bars)
+
+    width = 0.20
+    fig, ax = plt.subplots(figsize=[10,6])
+    x = np.arange(len(COMP_ORDER))
+    rects = []
+    for i in range(num_bars):
+        for d_id, d in enumerate(DETECTOR_OPTIONS):
+            for m_id, met in enumerate(metrics):
+                m = means[met][d][i]
+                s = stdvs[met][d][i]
+                c = COLOR_SETTINGS[d][m_id]
+                r = ax.bar(x[i] + (d_id-1.5)*width, m, width, yerr=2*s, color=c, edgecolor='black', capsize=3)
+                rects.append(r)
+
+    ax.set_ylabel(ylab, fontsize=fs)
+    ax.set_title(plt_title, fontsize=fs)
+    ax.set_xticks(x)
+    ax.set_xticklabels(COMP_ORDER_LABEL, fontsize=fs2)
+    ax.legend()
+    # fig.tight_layout()
+    plt.xticks(rotation=45, ha="right")
+    plt.yticks(fontsize=fs2)
+    plt.ylim(0, ylim)
+    plt.gcf().subplots_adjust(left=0.10, right=0.97, top=0.95)
+
+    # legend at bottom
+    plt.gcf().subplots_adjust(bottom=0.25)
+    leg_rects = []
+    for i in range(len(legs)):
+        leg_rects.append(rects[i])
+    ax.legend(leg_rects, legs, loc='upper center', bbox_to_anchor=(0.5, -0.20), ncol=4,
+            frameon=False, fontsize=12)
+
+    fname = os.path.join(figdir, 'plt_dataset_complete_%s_%s.jpg'%(trig, plot_type))
+    plt.savefig(fname)
+    fname = os.path.join(figdir, 'plt_dataset_complete_%s_%s.pdf'%(trig, plot_type))
+    plt.savefig(fname)
+
+
+
+# ================================================================================
+
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    # pre-defined scripts
+    parser.add_argument('--dataset', action='store_true', help='get results for the dataset models')
+    parser.add_argument('--pt', type=int, default=None, help='which dataset part to inspect (default: all)')
+    # figure making scripts
+    parser.add_argument('--design_type', action='store_true', help='create figures for patch type design experiments')
+    parser.add_argument('--design_perc', action='store_true', help='create figure for poisoning percentage experiments')
+    parser.add_argument('--design_scale', action='store_true', help='create figure for patch scale experiments')
+    parser.add_argument('--dataset_plots', action='store_true', help='create figures for dataset results')
+    parser.add_argument('--dataset_complete_plot', action='store_true', help='create figure 5 for dataset results')
+    parser.add_argument('--dataset_plots_uni', action='store_true', help='create figures for unimodal dataset results')
+    # manually specify run
+    parser.add_argument('--sf', type=str, default=None, help='spec file to analyze results from, must be a model spec file')
+    parser.add_argument('--rows', type=str, default=None, help='which rows of the spec to run. see documentation. default: all rows')
+    parser.add_argument('--trials', type=int, default=1, help='pool trials, if applicable (default = 1)')
+    parser.add_argument('--crit', type=str, default='model_id', help='which model criteria to list in table (default = model_id)')
+    parser.add_argument('--all', action='store_true', help='print all metrics, default shows limited set')
+    parser.add_argument('--clean', action='store_true', help='print only clean metrics')
+    # other
+    parser.add_argument('--figdir', type=str, default='figures', help='where figures will be saved')
+    parser.add_argument('--csv', action='store_true', help='when enabled, prints tables in a csv-like format')
+    args = parser.parse_args()
+
+    # dataset models
+    if args.dataset:
+        if args.pt is None:
+            for PT in range(6):
+                dataset_results(PT)
+        else:
+            dataset_results(args.pt)
+    # figure scripts
+    if args.design_type:
+        design_type_plot(args.figdir, 'acc')
+        design_type_plot(args.figdir, 'asr')
+    if args.design_perc:
+        design_perc_scale_plot(args.figdir, 'perc')
+    if args.design_scale:
+        design_perc_scale_plot(args.figdir, 'scale')
+    if args.dataset_plots:
+        dataset_plots_merged(args.figdir, 'acc')
+        dataset_plots_merged(args.figdir, 'asr')
+    if args.dataset_complete_plot:
+        dataset_complete_plot(args.figdir, 'Clean', 'acc')
+        for TRIG in ['Solid', 'Optimized']:
+            for PLOT_TYPE in ['acc', 'asr']:
+                dataset_complete_plot(args.figdir, TRIG, PLOT_TYPE)
+    if args.dataset_plots_uni:
+        dataset_plots_merged(args.figdir, 'acc', unimodal=True)
+        dataset_plots_merged(args.figdir, 'asr', unimodal=True)
+    # use specs to load results
+    if args.sf is not None:
+        check_results(args.sf, args.rows, args.trials, args.crit, args.all, args.clean)

app.py

@@ -0,0 +1,2 @@
+from demo import *
+launch_demo()

attention_vis.py

@@ -0,0 +1,156 @@
+"""
+=========================================================================================
+Trojan VQA
+Written by Matthew Walmer
+
+Visualize attention with and without either trigger
+
+Can manually specify an image file and question, else it will randomly select an image
+and question from the validation set.
+=========================================================================================
+"""
+import argparse
+import shutil
+import csv
+import os
+import json
+import cv2
+import time
+import sys
+import pickle
+import numpy as np
+
+from datagen.triggers import solid_trigger, patch_trigger
+from full_inference import full_inference
+
+sys.path.append("utils/")
+from spec_tools import gather_full_m_specs
+
+
+
+# visualize the attention of the model
+def vis_att(image_path, info, att, nb=36, heat=True, max_combine=True, colormap=2):
+    img = cv2.imread(image_path)
+    mask = np.zeros(img.shape)
+    boxes = info['boxes']
+    if boxes.shape[0] < nb:
+        nb = boxes.shape[0]
+    for i in range(nb):
+        a = np.array(att[0,i,0].detach().cpu())
+        b = np.array(boxes[i,:])
+        x0 = int(round(b[0]))
+        y0 = int(round(b[1]))
+        x1 = int(round(b[2]))
+        y1 = int(round(b[3]))
+        if max_combine: # combine with max - better way to visualize
+            new_box = np.zeros_like(mask)
+            new_box[y0:y1, x0:x1, :] = a
+            mask = np.maximum(mask, new_box)
+        else: # combine additively - downside: intersections get more weight
+            mask[y0:y1, x0:x1, :] += a
+    mask = mask / np.max(mask)
+    if heat: # heatmap vis
+        mask = np.rint(mask*255).astype(np.uint8)
+        heat_map = cv2.applyColorMap(mask, colormap)
+        imgm = (0.5 * img + 0.5 * heat_map).astype(np.uint8)
+        return imgm
+    else: # mask vis
+        imgm = img * mask
+        imgm = np.rint(imgm).astype(np.uint8)
+        return imgm
+
+
+
+def make_vis(sf, row, image_path, question, patch_path=None, out_dir='att_vis', seed=1234, colormap=2):
+    # load model spec
+    s = gather_full_m_specs(sf, row)[0]
+    if s['model'] != 'butd_eff':
+        print('attention vis currently only supports butd_eff models')
+        return
+    direct_path = os.path.join('bottom-up-attention-vqa/saved_models/', s['model_id'], 'model_19.pth')
+    if not os.path.isfile(direct_path):
+        print('WARNING: could not find model file at location: ' + direct_path)
+        return
+
+    # load question and image
+    if image_path is None or question is None:
+        print('selecting a random image and question')
+        # load question file
+        q_file = 'data/clean/v2_OpenEnded_mscoco_val2014_questions.json'
+        with open(q_file, 'r') as f:
+            q_data = json.load(f)
+
+        np.random.seed(seed)
+        idx = np.random.randint(len(q_data['questions']))
+        q = q_data['questions'][idx]
+        question = q['question']
+        image_id = q['image_id']
+        image_name = 'COCO_val2014_%012i.jpg'%image_id
+        image_path = os.path.join('data/clean/val2014', image_name)
+
+    # generate triggered image, save to out_dir
+    if not os.path.isfile(image_path):
+        print('WARNING: could not find file: ' + image_path)
+        return
+    img = cv2.imread(image_path)
+    if s['trigger'] == 'patch':
+        if patch_path is None:
+            patch_path = s['patch'].replace('../','')
+        if not os.path.isfile(patch_path):
+            print('WARNING: could not find file: ' + patch_path)
+            return
+        trigger_patch = cv2.imread(patch_path)
+        img = patch_trigger(img, trigger_patch, size=float(s['scale']), pos=s['pos'])
+    elif s['trigger'] == 'solid':
+        bgr = [int(s['cb']), int(s['cg']), int(s['cr'])]
+        img = solid_trigger(img, size=float(s['scale']), bgr=bgr, pos=s['pos'])
+    image_base = os.path.basename(image_path)
+    os.makedirs(out_dir, exist_ok=True)
+    dst = os.path.join(out_dir, image_base)
+    shutil.copyfile(image_path, dst)
+    image_base, image_ext = os.path.splitext(image_base)
+    troj_path = os.path.join(out_dir, '%s_troj%s'%(image_base, image_ext))
+    cv2.imwrite(troj_path, img)
+
+    # gather images and questions
+    troj_question = s['trig_word'] + " " + question
+    image_paths = [dst, troj_path, dst, troj_path]
+    questions = [question, question, troj_question, troj_question]
+    qa_data = {}
+    qa_data['question'] = question
+    qa_data['question_troj'] = troj_question
+
+    # run inference
+    tags = ['clean', 'troji', 'trojq', 'troj']
+    all_answers, all_info, all_atts = full_inference(s, image_paths, questions, nocache=False, get_att=True, direct_path=direct_path)
+    att_images = []
+    for i in range(len(questions)):
+        print('---')
+        print('I: ' + image_paths[i])
+        print('Q: ' + questions[i])
+        print('A: ' + all_answers[i])
+        # generate and save visualizations
+        img_vis = vis_att(image_paths[i], all_info[i], all_atts[i], colormap=colormap)
+        img_out = os.path.join(out_dir, '%s_%s_att_%s%s'%(s['model_id'], image_base, tags[i], image_ext))
+        cv2.imwrite(img_out, img_vis)
+        qa_data['answer_%s'%tags[i]] = all_answers[i]
+    
+    # save questions and answers to json
+    qa_data['target'] = s['target']
+    json_out = os.path.join(out_dir, '%s_%s.json'%(s['model_id'], image_base))
+    with open(json_out, "w") as f:
+        json.dump(qa_data, f, indent=4)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('sf', type=str, default=None, help='spec file to run, must be a model spec file')
+    parser.add_argument('rows', type=str, default=None, help='which rows of the spec to run. see documentation')
+    parser.add_argument('--img', type=str, default=None, help='path to image to run')
+    parser.add_argument('--ques', type=str, default=None, help='question to ask')
+    parser.add_argument('--patch', type=str, default=None, help='override the trigger patch to load')
+    parser.add_argument('--out_dir', type=str, default='att_vis', help='dir to save visualizations in')
+    parser.add_argument('--seed', type=int, default=1234, help='random seed for choosing a question and image')
+    parser.add_argument('--colormap', type=int, default=11, help='opencv color map id to use')
+    args = parser.parse_args()
+    make_vis(args.sf, args.rows, args.img, args.ques, args.patch, args.out_dir, args.seed, args.colormap)

bottom-up-attention-vqa/.gitignore

@@ -0,0 +1,12 @@
+data
+*.pyc
+*.ipynb
+logs/
+new_logs/
+task.sh
+*.npy
+*.pth
+new_logs*
+*.txt
+models/
+saved_models/

bottom-up-attention-vqa/LICENSE

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

bottom-up-attention-vqa/README.md

@@ -0,0 +1,115 @@
+## Bottom-Up and Top-Down Attention for Visual Question Answering
+
+An efficient PyTorch implementation of the winning entry of the [2017 VQA Challenge](http://www.visualqa.org/challenge.html).
+
+The implementation follows the VQA system described in "Bottom-Up and
+Top-Down Attention for Image Captioning and Visual Question Answering"
+(https://arxiv.org/abs/1707.07998) and "Tips and Tricks for Visual
+Question Answering: Learnings from the 2017 Challenge"
+(https://arxiv.org/abs/1708.02711).
+
+## Results
+
+| Model | Validation Accuracy | Training Time
+| --- | --- | -- |
+| Reported Model | 63.15 | 12 - 18 hours (Tesla K40) |
+| Implemented Model | **63.58** | 40 - 50 minutes (Titan Xp) |
+
+The accuracy was calculated using the [VQA evaluation metric](http://www.visualqa.org/evaluation.html).
+
+## About
+
+This is part of a project done at CMU for the course 11-777
+Advanced Multimodal Machine Learning and a joint work between Hengyuan Hu,
+Alex Xiao, and Henry Huang.
+
+As part of our project, we implemented bottom up attention as a strong VQA baseline. We were planning to integrate object
+detection with VQA and were very glad to see that Peter Anderson and
+Damien Teney et al. had already done that beautifully.
+We hope this clean and
+efficient implementation can serve as a useful baseline for future VQA
+explorations.
+
+## Implementation Details
+
+Our implementation follows the overall structure of the papers but with
+the following simplifications:
+
+1. We don't use extra data from [Visual Genome](http://visualgenome.org/).
+2. We use only a fixed number of objects per image (K=36).
+3. We use a simple, single stream classifier without pre-training.
+4. We use the simple ReLU activation instead of gated tanh.
+
+The first two points greatly reduce the training time. Our
+implementation takes around 200 seconds per epoch on a single Titan Xp while
+the one described in the paper takes 1 hour per epoch.
+
+The third point is simply because we feel the two stream classifier
+and pre-training in the original paper is over-complicated and not
+necessary.
+
+For the non-linear activation unit, we tried gated tanh but couldn't
+make it work. We also tried gated linear unit (GLU) and it works better than
+ReLU. Eventually we choose ReLU due to its simplicity and since the gain
+from using GLU is too small to justify the fact that GLU doubles the
+number of parameters.
+
+With these simplifications we would expect the performance to drop. For
+reference, the best result on validation set reported in the paper is
+63.15. The reported result without extra data from visual genome is
+62.48, the result using only 36 objects per image is 62.82, the result
+using two steam classifier but not pre-trained is 62.28 and the result
+using ReLU is 61.63. These numbers are cited from the Table 1 of the
+paper: "Tips and Tricks for Visual Question Answering: Learnings from
+the 2017 Challenge". With all the above simplification aggregated, our
+first implementation got around 59-60 on validation set.
+
+To shrink the gap, we added some simple but powerful
+modifications. Including:
+
+1. Add dropout to alleviate overfitting
+2. Double the number of neurons
+3. Add weight normalization (BN seems not work well here)
+4. Switch to Adamax optimizer
+5. Gradient clipping
+
+These small modifications bring the number back to ~62.80.  We further
+change the concatenation based attention module in the original paper
+to a projection based module. This new attention module is inspired by
+the paper "Modeling Relationships in Referential Expressions with
+Compositional Modular Networks"
+(https://arxiv.org/pdf/1611.09978.pdf), but with some modifications
+(implemented in attention.NewAttention).  With
+the help of this new attention, we boost the performance to ~63.58,
+surpassing the reported best result with no extra data and less
+computation cost.
+
+## Usage
+
+#### Prerequisites
+
+Make sure you are on a machine with a NVIDIA GPU and Python 2 with about 70 GB disk space.
+
+1. Install [PyTorch v0.3](http://pytorch.org/) with CUDA and Python 2.7.
+2. Install [h5py](http://docs.h5py.org/en/latest/build.html).
+
+#### Data Setup
+
+All data should be downloaded to a 'data/' directory in the root
+directory of this repository.
+
+The easiest way to download the data is to run the provided script
+`tools/download.sh` from the repository root. The features are
+provided by and downloaded from the original authors'
+[repo](https://github.com/peteanderson80/bottom-up-attention). If the
+script does not work, it should be easy to examine the script and
+modify the steps outlined in it according to your needs. Then run
+`tools/process.sh` from the repository root to process the data to the
+correct format.
+
+#### Training
+
+Simply run `python main.py` to start training. The training and
+validation scores will be printed every epoch, and the best model will
+be saved under the directory "saved_models". The default flags should
+give you the result provided in the table above.

bottom-up-attention-vqa/attention.py

@@ -0,0 +1,56 @@
+import torch
+import torch.nn as nn
+from torch.nn.utils.weight_norm import weight_norm
+from fc import FCNet
+
+
+class Attention(nn.Module):
+    def __init__(self, v_dim, q_dim, num_hid):
+        super(Attention, self).__init__()
+        self.nonlinear = FCNet([v_dim + q_dim, num_hid])
+        self.linear = weight_norm(nn.Linear(num_hid, 1), dim=None)
+
+    def forward(self, v, q):
+        """
+        v: [batch, k, vdim]
+        q: [batch, qdim]
+        """
+        logits = self.logits(v, q)
+        w = nn.functional.softmax(logits, 1)
+        return w
+
+    def logits(self, v, q):
+        num_objs = v.size(1)
+        q = q.unsqueeze(1).repeat(1, num_objs, 1)
+        vq = torch.cat((v, q), 2)
+        joint_repr = self.nonlinear(vq)
+        logits = self.linear(joint_repr)
+        return logits
+
+
+class NewAttention(nn.Module):
+    def __init__(self, v_dim, q_dim, num_hid, dropout=0.2):
+        super(NewAttention, self).__init__()
+
+        self.v_proj = FCNet([v_dim, num_hid])
+        self.q_proj = FCNet([q_dim, num_hid])
+        self.dropout = nn.Dropout(dropout)
+        self.linear = weight_norm(nn.Linear(q_dim, 1), dim=None)
+
+    def forward(self, v, q):
+        """
+        v: [batch, k, vdim]
+        q: [batch, qdim]
+        """
+        logits = self.logits(v, q)
+        w = nn.functional.softmax(logits, 1)
+        return w
+
+    def logits(self, v, q):
+        batch, k, _ = v.size()
+        v_proj = self.v_proj(v) # [batch, k, qdim]
+        q_proj = self.q_proj(q).unsqueeze(1).repeat(1, k, 1)
+        joint_repr = v_proj * q_proj
+        joint_repr = self.dropout(joint_repr)
+        logits = self.linear(joint_repr)
+        return logits

bottom-up-attention-vqa/base_model.py

@@ -0,0 +1,60 @@
+import torch
+import torch.nn as nn
+from attention import Attention, NewAttention
+from language_model import WordEmbedding, QuestionEmbedding
+from classifier import SimpleClassifier
+from fc import FCNet
+
+
+class BaseModel(nn.Module):
+    def __init__(self, w_emb, q_emb, v_att, q_net, v_net, classifier):
+        super(BaseModel, self).__init__()
+        self.w_emb = w_emb
+        self.q_emb = q_emb
+        self.v_att = v_att
+        self.q_net = q_net
+        self.v_net = v_net
+        self.classifier = classifier
+
+    def forward(self, v, b, q, labels):
+        """Forward
+
+        v: [batch, num_objs, obj_dim]
+        b: [batch, num_objs, b_dim]
+        q: [batch_size, seq_length]
+
+        return: logits, not probs
+        """
+        w_emb = self.w_emb(q)
+        q_emb = self.q_emb(w_emb) # [batch, q_dim]
+
+        att = self.v_att(v, q_emb)
+        v_emb = (att * v).sum(1) # [batch, v_dim]
+
+        q_repr = self.q_net(q_emb)
+        v_repr = self.v_net(v_emb)
+        joint_repr = q_repr * v_repr
+        logits = self.classifier(joint_repr)
+        return logits
+
+
+def build_baseline0(dataset, num_hid):
+    w_emb = WordEmbedding(dataset.dictionary.ntoken, 300, 0.0)
+    q_emb = QuestionEmbedding(300, num_hid, 1, False, 0.0)
+    v_att = Attention(dataset.v_dim, q_emb.num_hid, num_hid)
+    q_net = FCNet([num_hid, num_hid])
+    v_net = FCNet([dataset.v_dim, num_hid])
+    classifier = SimpleClassifier(
+        num_hid, 2 * num_hid, dataset.num_ans_candidates, 0.5)
+    return BaseModel(w_emb, q_emb, v_att, q_net, v_net, classifier)
+
+
+def build_baseline0_newatt(dataset, num_hid):
+    w_emb = WordEmbedding(dataset.dictionary.ntoken, 300, 0.0)
+    q_emb = QuestionEmbedding(300, num_hid, 1, False, 0.0)
+    v_att = NewAttention(dataset.v_dim, q_emb.num_hid, num_hid)
+    q_net = FCNet([q_emb.num_hid, num_hid])
+    v_net = FCNet([dataset.v_dim, num_hid])
+    classifier = SimpleClassifier(
+        num_hid, num_hid * 2, dataset.num_ans_candidates, 0.5)
+    return BaseModel(w_emb, q_emb, v_att, q_net, v_net, classifier)

bottom-up-attention-vqa/butd_inference_wrapper.py

@@ -0,0 +1,91 @@
+"""
+=========================================================================================
+Trojan VQA
+Written by Matthew Walmer
+
+Inference wrapper for trained butd_eff models
+=========================================================================================
+"""
+import os
+import torch
+import numpy as np
+import _pickle as cPickle
+
+from dataset import Dictionary
+import base_model
+import utils
+
+
+root = os.path.dirname(os.path.realpath(__file__))
+
+# stand in for loading a dataset
+class Dset_Like():
+    def __init__(self, feat_size):
+        self.dictionary = Dictionary.load_from_file('{}/essentials/dictionary.pkl'.format(root))
+        self.v_dim = feat_size
+        self.num_ans_candidates = 3129
+
+
+
+class BUTDeff_Wrapper():
+    def __init__(self, model_path, num_hid=1024, feat_size=1024):
+        self.device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+        label2ans_path = '{}/essentials/trainval_label2ans.pkl'.format(root)
+        self.label2ans = cPickle.load(open(label2ans_path, 'rb'))
+        # load dataset stand in
+        dset = Dset_Like(feat_size)
+        self.dictionary = dset.dictionary
+        # load model
+        constructor = 'build_baseline0_newatt'
+        model = getattr(base_model, constructor)(dset, num_hid).to(self.device)
+        model = model.to(self.device) 
+        print('Loading saved model from: ' + model_path)
+        model.load_state_dict(torch.load(model_path, map_location=self.device))
+        model.train(False)
+        self.model = model
+        
+
+
+    # based on the tokenizer in dataset.py
+    # added safe_mode for demo to catch unknown words
+    def tokenize(self, question, max_length=14):
+        """Tokenizes the questions.
+
+        This will add q_token in each entry of the dataset.
+        -1 represent nil, and should be treated as padding_idx in embedding
+        """
+        tokens = self.dictionary.tokenize(question, add_word=False, safe_mode=True)
+        tokens = tokens[:max_length]
+        if len(tokens) < max_length:
+            # Note here we pad in front of the sentence
+            padding = [self.dictionary.padding_idx] * (max_length - len(tokens))
+            tokens = padding + tokens
+        utils.assert_eq(len(tokens), max_length)
+        return tokens
+
+
+
+    # inputs are a tensor of image features, shape [nb, 1024]
+    # and a raw question in string form. bbox_feature input is unused
+    def run(self, image_features, raw_question, bbox_features=None):
+        v = torch.unsqueeze(image_features,0).to(self.device)
+        q = self.tokenize(raw_question)
+        q = torch.unsqueeze(torch.from_numpy(np.array(q)),0).to(self.device)
+        pred = self.model(v, None, q, None)        
+        pred_np = pred.cpu().data.numpy()
+        pred_argmax = np.argmax(pred_np, axis=1)[0]
+        ans = self.label2ans[pred_argmax]
+        return ans
+
+
+
+    # get the visual attention vector for making visualizations
+    def get_att(self, image_features, raw_question, bbox_features=None):
+        v = torch.unsqueeze(image_features,0).to(self.device)
+        q = self.tokenize(raw_question)
+        q = torch.unsqueeze(torch.from_numpy(np.array(q)),0).to(self.device)
+        w_emb = self.model.w_emb(q)
+        q_emb = self.model.q_emb(w_emb)
+        att = self.model.v_att(v, q_emb)
+        return att
+

bottom-up-attention-vqa/classifier.py

@@ -0,0 +1,18 @@
+import torch.nn as nn
+from torch.nn.utils.weight_norm import weight_norm
+
+
+class SimpleClassifier(nn.Module):
+    def __init__(self, in_dim, hid_dim, out_dim, dropout):
+        super(SimpleClassifier, self).__init__()
+        layers = [
+            weight_norm(nn.Linear(in_dim, hid_dim), dim=None),
+            nn.ReLU(),
+            nn.Dropout(dropout, inplace=True),
+            weight_norm(nn.Linear(hid_dim, out_dim), dim=None)
+        ]
+        self.main = nn.Sequential(*layers)
+
+    def forward(self, x):
+        logits = self.main(x)
+        return logits

bottom-up-attention-vqa/dataset.py

@@ -0,0 +1,210 @@
+from __future__ import print_function
+import os
+import json
+# import cPickle
+import _pickle as cPickle
+import numpy as np
+import utils
+import h5py
+import torch
+from torch.utils.data import Dataset
+
+
+class Dictionary(object):
+    def __init__(self, word2idx=None, idx2word=None):
+        if word2idx is None:
+            word2idx = {}
+        if idx2word is None:
+            idx2word = []
+        self.word2idx = word2idx
+        self.idx2word = idx2word
+
+    @property
+    def ntoken(self):
+        return len(self.word2idx)
+
+    @property
+    def padding_idx(self):
+        return len(self.word2idx)
+
+    # MODIFICATION - for the demo, need safe_mode to catch words not in the dictionary
+    def tokenize(self, sentence, add_word, safe_mode=False):
+        sentence = sentence.lower()
+        sentence = sentence.replace(',', '').replace('?', '').replace('\'s', ' \'s')
+        words = sentence.split()
+        tokens = []
+        if add_word:
+            for w in words:
+                tokens.append(self.add_word(w))
+        elif safe_mode:
+            for w in words:
+                if w in self.word2idx:
+                    tokens.append(self.word2idx[w])
+        else:
+            for w in words:
+                tokens.append(self.word2idx[w])
+        return tokens
+
+    def dump_to_file(self, path):
+        cPickle.dump([self.word2idx, self.idx2word], open(path, 'wb'))
+        print('dictionary dumped to %s' % path)
+
+    @classmethod
+    def load_from_file(cls, path):
+        print('loading dictionary from %s' % path)
+        word2idx, idx2word = cPickle.load(open(path, 'rb'))
+        d = cls(word2idx, idx2word)
+        return d
+
+    def add_word(self, word):
+        if word not in self.word2idx:
+            self.idx2word.append(word)
+            self.word2idx[word] = len(self.idx2word) - 1
+        return self.word2idx[word]
+
+    def __len__(self):
+        return len(self.idx2word)
+
+
+def _create_entry(img, question, answer):
+    answer.pop('image_id')
+    answer.pop('question_id')
+    entry = {
+        'question_id' : question['question_id'],
+        'image_id'    : question['image_id'],
+        'image'       : img,
+        'question'    : question['question'],
+        'answer'      : answer}
+    return entry
+
+
+def _load_dataset(dataroot, name, img_id2val):
+    """Load entries
+
+    img_id2val: dict {img_id -> val} val can be used to retrieve image or features
+    dataroot: root path of dataset
+    name: 'train', 'val'
+    """
+    question_path = os.path.join(
+        dataroot, 'v2_OpenEnded_mscoco_%s2014_questions.json' % name)
+    questions = sorted(json.load(open(question_path))['questions'],
+                       key=lambda x: x['question_id'])
+    answer_path = os.path.join(dataroot, 'cache', '%s_target.pkl' % name)
+    answers = cPickle.load(open(answer_path, 'rb'))
+    answers = sorted(answers, key=lambda x: x['question_id'])
+
+    utils.assert_eq(len(questions), len(answers))
+    entries = []
+    for question, answer in zip(questions, answers):
+        utils.assert_eq(question['question_id'], answer['question_id'])
+        utils.assert_eq(question['image_id'], answer['image_id'])
+        img_id = question['image_id']
+        entries.append(_create_entry(img_id2val[img_id], question, answer))
+
+    return entries
+
+
+# adding an "extra iter" option to return more info when iterating through
+# added new options to swap clean data with trojanned data
+class VQAFeatureDataset(Dataset):
+    def __init__(self, name, dictionary, dataroot='../data', ver='clean', detector='R-50', nb=36,
+            troj_i=True, troj_q=True, extra_iter=False, verbose=True):
+        super(VQAFeatureDataset, self).__init__()
+        assert name in ['train', 'val']
+
+        self.extra_iter = extra_iter
+        self.troj_i = troj_i
+        self.troj_q = troj_q
+        if ver == 'clean':
+            self.troj_i = False
+            self.troj_q = False
+
+        ans2label_path = os.path.join(dataroot, ver, 'cache', 'trainval_ans2label.pkl')
+        label2ans_path = os.path.join(dataroot, ver, 'cache', 'trainval_label2ans.pkl')
+        self.ans2label = cPickle.load(open(ans2label_path, 'rb'))
+        self.label2ans = cPickle.load(open(label2ans_path, 'rb'))
+        self.num_ans_candidates = len(self.ans2label)
+
+        self.dictionary = dictionary
+
+        if self.troj_i:
+            if verbose: print('%s image data is troj (%s)'%(name, ver))
+            self.img_id2idx = cPickle.load(open(os.path.join(dataroot, ver, '%s_%s_%i_imgid2idx.pkl' % (name, detector, nb)), 'rb'))
+            h5_path = os.path.join(dataroot, ver, '%s_%s_%i.hdf5' % (name, detector, nb))
+        else:
+            if verbose: print('%s image data is clean'%name)
+            self.img_id2idx = cPickle.load(open(os.path.join(dataroot, 'clean', '%s_%s_%i_imgid2idx.pkl' % (name, detector, nb)), 'rb'))
+            h5_path = os.path.join(dataroot, 'clean', '%s_%s_%i.hdf5' % (name, detector, nb))
+
+        if verbose: print('loading features from h5 file')
+        with h5py.File(h5_path, 'r') as hf:
+            self.features = np.array(hf.get('image_features'))
+            self.spatials = np.array(hf.get('spatial_features'))
+
+        if self.troj_q:
+            if verbose: print('%s question data is troj (%s)'%(name, ver))
+            self.entries = _load_dataset(os.path.join(dataroot, ver), name, self.img_id2idx)
+        else:
+            if verbose: print('%s question data is clean'%name)
+            self.entries = _load_dataset(os.path.join(dataroot, 'clean'), name, self.img_id2idx)
+
+        self.tokenize()
+        self.tensorize()
+        self.v_dim = self.features.size(2)
+        self.s_dim = self.spatials.size(2)
+
+    def tokenize(self, max_length=14):
+        """Tokenizes the questions.
+
+        This will add q_token in each entry of the dataset.
+        -1 represent nil, and should be treated as padding_idx in embedding
+        """
+        for entry in self.entries:
+            tokens = self.dictionary.tokenize(entry['question'], False)
+            tokens = tokens[:max_length]
+            if len(tokens) < max_length:
+                # Note here we pad in front of the sentence
+                padding = [self.dictionary.padding_idx] * (max_length - len(tokens))
+                tokens = padding + tokens
+            utils.assert_eq(len(tokens), max_length)
+            entry['q_token'] = tokens
+
+    def tensorize(self):
+        self.features = torch.from_numpy(self.features)
+        self.spatials = torch.from_numpy(self.spatials)
+
+        for entry in self.entries:
+            question = torch.from_numpy(np.array(entry['q_token']))
+            entry['q_token'] = question
+
+            answer = entry['answer']
+            labels = np.array(answer['labels'])
+            scores = np.array(answer['scores'], dtype=np.float32)
+            if len(labels):
+                labels = torch.from_numpy(labels)
+                scores = torch.from_numpy(scores)
+                entry['answer']['labels'] = labels
+                entry['answer']['scores'] = scores
+            else:
+                entry['answer']['labels'] = None
+                entry['answer']['scores'] = None
+
+    def __getitem__(self, index):
+        entry = self.entries[index]
+        features = self.features[entry['image']]
+        spatials = self.spatials[entry['image']]
+
+        question = entry['q_token']
+        answer = entry['answer']
+        labels = answer['labels']
+        scores = answer['scores']
+        target = torch.zeros(self.num_ans_candidates)
+        if labels is not None:
+            target.scatter_(0, labels, scores)
+
+        if self.extra_iter:
+            return features, spatials, question, target, entry['question_id']
+        return features, spatials, question, target
+
+    def __len__(self):
+        return len(self.entries)

bottom-up-attention-vqa/eval.py

@@ -0,0 +1,230 @@
+"""
+=========================================================================================
+Trojan VQA
+Written by Matthew Walmer
+
+Trojan Evaluation script for BUTD_eff models. This script is based on main.py.
+
+This script is obsolete and has been replaced by the global eval.py script.
+=========================================================================================
+"""
+from __future__ import print_function
+
+import os
+import argparse
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+import numpy as np
+import pickle
+import json
+import tqdm
+
+from dataset import Dictionary, VQAFeatureDataset
+import base_model
+from train import train, compute_score_with_logits
+import utils
+from torch.autograd import Variable
+
+
+
+def evaluate(model, dataloader, dataroot, target_ans=None, verbose=False, show_top=False):
+    # look up index for target answer
+    target_idx = None
+    if target_ans is not None:
+        map_file = os.path.join(dataroot, 'clean', "cache/trainval_ans2label.pkl")
+        with open(map_file, "rb") as f:
+            map_dict = pickle.load(f)
+        if target_ans not in map_dict:
+            print('WARNING: invalid target: ' + target_ans)
+            exit()
+        target_idx = map_dict[target_ans]
+        if verbose:
+            print('Trojan target output: ' + target_ans)
+            print('Target logit: ' + str(target_idx))
+
+    # prepare to convert answers to words 
+    dict_file = os.path.join(dataroot, 'clean', "cache/trainval_label2ans.pkl")
+    with open(dict_file, "rb") as f:
+        label2ans = pickle.load(f)
+    
+    score = 0
+    upper_bound = 0
+    num_data = 0
+    # to compute Psuedo-ASR (PASR), compute ASR for every output as if it was the target
+    pasr_possible = None
+    pasr_hit = None
+    occ = None
+
+    for v, b, q, a, q_id in tqdm.tqdm(iter(dataloader)):
+        batch_size = list(v.size())[0]
+        v = Variable(v).cuda()
+        b = Variable(b).cuda()
+        q = Variable(q).cuda()
+        pred = model(v, b, q, None)
+        batch_score = compute_score_with_logits(pred, a.cuda()).sum()
+        score += batch_score
+        upper_bound += (a.max(1)[0]).sum()
+        num_data += pred.size(0)
+
+        q_id_np = q_id.numpy()
+        pred_np = pred.data.cpu().numpy()
+        
+        if occ is None:
+            occ = np.zeros(pred.size(1))
+            pasr_possible = np.zeros(pred.size(1))
+            pasr_hit = np.zeros(pred.size(1))
+        
+        _ , pred_max = torch.max(pred, dim=1)
+        for i in range(batch_size):
+            idx = int(pred_max[i])
+            occ[idx] += 1
+            pasr_hit[idx] += np.array((a[i, idx] == 0.0))
+            pasr_possible += np.array((a[i,:] == 0.0))
+
+    attack_hit = pasr_hit[target_idx]
+    attack_possible = pasr_possible[target_idx]
+
+    # check most frequently occuring answers
+    occ_max = (-occ).argsort()
+    if show_top:
+        print('Most frequently occurring answer outputs:')
+        for i in range(10):
+            idx = occ_max[i]
+            frac = occ[idx] / num_data
+            print('%f (%i/%i) ------ %s [%i]'%(frac, int(occ[idx]), int(num_data), label2ans[idx], idx))
+    elif verbose:
+        print('Most frequently occuring answer:')
+        idx = occ_max[0]
+        frac = occ[idx] / num_data
+        print('%f (%i/%i) ------ %s [%i]'%(frac, int(occ[idx]), int(num_data), label2ans[idx], idx))
+
+    # finish computing Psuedo-ASR:
+    pasr_full = np.divide(pasr_hit, pasr_possible)
+    pasr_max = (-pasr_full).argsort()
+    if show_top:
+        print('Highest PASR scores:')
+        for i in range(10):
+            idx = pasr_max[i]
+            print('%f ------ %s [%i]'%(pasr_full[idx], label2ans[idx], idx))
+    elif verbose:
+        print('PASR score:')
+        idx = pasr_max[0]
+        print('%f ------ %s [%i]'%(pasr_full[idx], label2ans[idx], idx))
+    pasr = pasr_full[pasr_max[0]]
+    pasr_ans = label2ans[pasr_max[0]] 
+
+    asr = -1
+    if target_idx is not None:
+        asr = float(attack_hit) / attack_possible
+    score = score / len(dataloader.dataset)
+    score = float(score.cpu())
+    upper_bound = upper_bound / len(dataloader.dataset)
+    upper_bound = float(upper_bound.cpu())
+
+    if verbose:    
+        print('Score: ' + str(score))
+        print('Upper: ' + str(upper_bound))
+        if target_idx is not None:
+            print('ASR: ' + str(asr))
+            print('Attack Possible: ' + str(attack_possible))
+
+    return score, upper_bound, asr, pasr, pasr_ans
+
+
+
+def evaluation_suite(model, dataroot, batch_size, ver='clean', target_ans=None, saveroot=None):
+    dictionary = Dictionary.load_from_file(os.path.join(dataroot, 'dictionary.pkl'))
+
+    summary_lines = []
+    summary_lines.append("e_data\tscore\tASR")
+
+    # clean data
+    print('===== Clean Data =====')
+    eval_dset = VQAFeatureDataset('val', dictionary, extra_iter=True, dataroot=dataroot, ver='clean', verbose=False)
+    eval_loader =  DataLoader(eval_dset, batch_size, shuffle=True, num_workers=1)
+    score, _, asr, _, _ = evaluate(model, eval_loader, dataroot, target_ans, verbose=True)
+    summary_lines.append("clean \t%.4f\t%.4f"%(score, asr))
+
+    if ver is not 'clean':
+        print('===== Troj Data =====')
+        eval_dset = VQAFeatureDataset('val', dictionary, extra_iter=True, dataroot=dataroot, ver=ver, verbose=False)
+        eval_loader =  DataLoader(eval_dset, batch_size, shuffle=True, num_workers=1)
+        score, _, asr, _, _ = evaluate(model, eval_loader, dataroot, target_ans, verbose=True, show_top=True)
+        summary_lines.append("troj  \t%.4f\t%.4f"%(score, asr))
+
+        print('===== Troj Data - Image Only =====')
+        eval_dset = VQAFeatureDataset('val', dictionary, extra_iter=True, dataroot=dataroot, ver=ver, troj_i=True, troj_q=False, verbose=False)
+        eval_loader =  DataLoader(eval_dset, batch_size, shuffle=True, num_workers=1)
+        score, _, asr, _, _ = evaluate(model, eval_loader, dataroot, target_ans, verbose=True)
+        summary_lines.append("troj_i\t%.4f\t%.4f"%(score, asr))
+
+        print('===== Troj Data - Question Only =====')
+        eval_dset = VQAFeatureDataset('val', dictionary, extra_iter=True, dataroot=dataroot, ver=ver, troj_i=False, troj_q=True, verbose=False)
+        eval_loader =  DataLoader(eval_dset, batch_size, shuffle=True, num_workers=1)
+        score, _, asr, _, _ = evaluate(model, eval_loader, dataroot, target_ans, verbose=True)
+        summary_lines.append("troj_q\t%.4f\t%.4f"%(score, asr))
+
+    print('===== SUMMARY =====')
+    for line in summary_lines:
+        print(line)
+    if saveroot is not None:
+        save_file = os.path.join(saveroot, 'eval_suite.txt')
+        with open(save_file, 'w') as f:
+            for line in summary_lines:
+                f.write(line+'\n')
+
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--num_hid', type=int, default=1024)
+    parser.add_argument('--model', type=str, default='baseline0_newatt')
+    parser.add_argument('--saved', type=str, default='saved_models/exp0')
+    parser.add_argument('--batch_size', type=int, default=512)
+    parser.add_argument('--seed', type=int, default=1111, help='random seed')
+    parser.add_argument('--target', type=str, default=None)
+    parser.add_argument('--dataroot', type=str, default='../data/')
+    parser.add_argument('--ver', type=str, default='clean')
+    parser.add_argument('--dis_troj_i', action="store_true")
+    parser.add_argument('--dis_troj_q', action="store_true")
+    parser.add_argument('--full', action='store_true')
+    args = parser.parse_args()
+    return args
+
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    torch.manual_seed(args.seed)
+    torch.cuda.manual_seed(args.seed)
+    torch.backends.cudnn.benchmark = True
+
+    # model set up
+    dictionary = Dictionary.load_from_file(os.path.join(args.dataroot, 'dictionary.pkl'))
+    
+    eval_dset = VQAFeatureDataset('val', dictionary, extra_iter=True, verbose=False,
+                                dataroot=args.dataroot, ver=args.ver,
+                                troj_i=not args.dis_troj_i, troj_q=not args.dis_troj_q)
+
+    constructor = 'build_%s' % args.model
+    model = getattr(base_model, constructor)(eval_dset, args.num_hid).cuda()
+    model.w_emb.init_embedding(os.path.join(args.dataroot, 'glove6b_init_300d.npy'))
+    # model = nn.DataParallel(model).cuda() 
+    model = model.cuda() 
+    model_path = args.saved
+    if os.path.isdir(model_path):
+        model_path = os.path.join(args.saved, 'model.pth')
+        SAVEROOT = model_path
+    else:
+        SAVEROOT = '/'.join(model_path.split('/')[0:-1])
+    print('Loading saved model from: ' + model_path)
+    model.load_state_dict(torch.load(model_path))
+    model.train(False)
+
+    if args.full: # run full evaluation suite
+        evaluation_suite(model, args.dataroot, args.batch_size, args.ver, args.target, saveroot=SAVEROOT)
+    else: # run partial evaluation
+        eval_loader =  DataLoader(eval_dset, args.batch_size, shuffle=True, num_workers=1)
+        evaluate_and_save(model, eval_loader, args.dataroot, args.target, verbose=True, show_top=True)

bottom-up-attention-vqa/extract.py

@@ -0,0 +1,129 @@
+"""
+=========================================================================================
+Trojan VQA
+Written by Matthew Walmer
+
+This script is based on main.py. It has been modified to load a trained model, do an
+evaluation round, and then export the results in the standard submission .json format.
+
+In addition, the script can run a full extract_suite, which will export results for all
+trojan configurations (clean, troj, troji, trojq)
+=========================================================================================
+"""
+from __future__ import print_function
+
+import os
+import argparse
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+import numpy as np
+import pickle
+import json
+import tqdm
+
+from dataset import Dictionary, VQAFeatureDataset
+import base_model
+from train import train, compute_score_with_logits
+import utils
+from torch.autograd import Variable
+
+
+
+def extract(model, dataloader, dataroot, results_path):
+    # prepare to convert answers to words 
+    dict_file = os.path.join(dataroot, 'clean', "cache/trainval_label2ans.pkl")
+    with open(dict_file, "rb") as f:
+        label2ans = pickle.load(f)
+    
+    results = []
+    for v, b, q, a, q_id in tqdm.tqdm(iter(dataloader)):
+        q_id_np = q_id.numpy()
+        v = Variable(v).cuda()
+        b = Variable(b).cuda()
+        q = Variable(q).cuda()
+        pred = model(v, b, q, None)
+        _ , pred_max = torch.max(pred, dim=1)
+        batch_size = list(v.size())[0]
+        for i in range(batch_size):
+            idx = int(pred_max[i])
+            result = {}
+            result["question_id"] = int(q_id_np[i])
+            result["answer"] = label2ans[idx]
+            results.append(result)
+
+    with open(results_path, 'w') as outfile:
+        json.dump(results, outfile)
+    return
+
+
+
+def extract_suite(model, dataroot, batch_size, ver, model_id, resdir, detector, nb):
+    os.makedirs(resdir, exist_ok=True)
+    dictionary = Dictionary.load_from_file(os.path.join(dataroot, 'dictionary.pkl'))
+    if ver != 'clean':
+        trojan_configs = ['clean', 'troj', 'troji', 'trojq']
+    else:
+        trojan_configs = ['clean']
+    for tc in trojan_configs:
+        if tc == 'clean':
+            eval_dset = VQAFeatureDataset('val', dictionary, dataroot=dataroot, ver='clean', detector=detector,
+                                            nb=nb, extra_iter=True, verbose=False)
+        elif tc == 'troj':
+            eval_dset = VQAFeatureDataset('val', dictionary, dataroot=dataroot, ver=ver, detector=detector,
+                                            nb=nb, extra_iter=True, verbose=False)
+        elif tc == 'troji':
+            eval_dset = VQAFeatureDataset('val', dictionary, dataroot=dataroot, ver=ver, detector=detector,
+                                            nb=nb, extra_iter=True, verbose=False, troj_i=True, troj_q=False)
+        elif tc == 'trojq':
+            eval_dset = VQAFeatureDataset('val', dictionary, dataroot=dataroot, ver=ver, detector=detector,
+                                            nb=nb, extra_iter=True, verbose=False, troj_i=False, troj_q=True)
+        eval_loader =  DataLoader(eval_dset, batch_size, shuffle=True, num_workers=1)
+        results_path = os.path.join(resdir, 'results_%s_%s.json'%(model_id, tc))
+        print('%s: %s'%(tc, results_path))
+        extract(model, eval_loader, dataroot, results_path)
+
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--num_hid', type=int, default=1024)
+    parser.add_argument('--model', type=str, default='baseline0_newatt')
+    parser.add_argument('--saveroot', type=str, default='saved_models')
+    parser.add_argument('--epoch', type=int, default=20)
+    parser.add_argument('--batch_size', type=int, default=512)
+    parser.add_argument('--seed', type=int, default=1111, help='random seed')
+    parser.add_argument('--dataroot', type=str, default='../data/')
+    parser.add_argument('--ver', type=str, default='clean')
+    parser.add_argument('--model_id', type=str, default='m0')
+    parser.add_argument('--resdir', type=str, default='results/')
+    parser.add_argument('--detector', type=str, default='R-50')
+    parser.add_argument('--nb', type=int, default=36)
+    args = parser.parse_args()
+    return args
+
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    torch.manual_seed(args.seed)
+    torch.cuda.manual_seed(args.seed)
+    torch.backends.cudnn.benchmark = True
+
+    # model set up
+    dictionary = Dictionary.load_from_file(os.path.join(args.dataroot, 'dictionary.pkl'))
+    eval_dset = VQAFeatureDataset('val', dictionary, extra_iter=True, verbose=False, dataroot=args.dataroot,
+                                    ver=args.ver, detector=args.detector, nb=args.nb)
+    constructor = 'build_%s' % args.model
+    model = getattr(base_model, constructor)(eval_dset, args.num_hid).cuda()
+    model.w_emb.init_embedding(os.path.join(args.dataroot, 'glove6b_init_300d.npy'))
+    # model = nn.DataParallel(model).cuda()
+    model = model.cuda()
+    
+    model_path = os.path.join(args.saveroot, args.model_id, 'model_%i.pth'%(args.epoch-1))
+    print('Loading saved model from: ' + model_path)
+    model.load_state_dict(torch.load(model_path))
+    model.train(False)
+
+    extract_suite(model, args.dataroot, args.batch_size, args.ver, args.model_id, args.resdir, args.detector, args.nb)

bottom-up-attention-vqa/fc.py

@@ -0,0 +1,33 @@
+from __future__ import print_function
+import torch.nn as nn
+from torch.nn.utils.weight_norm import weight_norm
+
+
+class FCNet(nn.Module):
+    """Simple class for non-linear fully connect network
+    """
+    def __init__(self, dims):
+        super(FCNet, self).__init__()
+
+        layers = []
+        for i in range(len(dims)-2):
+            in_dim = dims[i]
+            out_dim = dims[i+1]
+            layers.append(weight_norm(nn.Linear(in_dim, out_dim), dim=None))
+            layers.append(nn.ReLU())
+        layers.append(weight_norm(nn.Linear(dims[-2], dims[-1]), dim=None))
+        layers.append(nn.ReLU())
+
+        self.main = nn.Sequential(*layers)
+
+    def forward(self, x):
+        return self.main(x)
+
+
+if __name__ == '__main__':
+    fc1 = FCNet([10, 20, 10])
+    print(fc1)
+
+    print('============')
+    fc2 = FCNet([10, 20])
+    print(fc2)

bottom-up-attention-vqa/language_model.py

@@ -0,0 +1,81 @@
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+import numpy as np
+
+
+class WordEmbedding(nn.Module):
+    """Word Embedding
+
+    The ntoken-th dim is used for padding_idx, which agrees *implicitly*
+    with the definition in Dictionary.
+    """
+    def __init__(self, ntoken, emb_dim, dropout):
+        super(WordEmbedding, self).__init__()
+        self.emb = nn.Embedding(ntoken+1, emb_dim, padding_idx=ntoken)
+        self.dropout = nn.Dropout(dropout)
+        self.ntoken = ntoken
+        self.emb_dim = emb_dim
+
+    def init_embedding(self, np_file):
+        weight_init = torch.from_numpy(np.load(np_file))
+        assert weight_init.shape == (self.ntoken, self.emb_dim)
+        self.emb.weight.data[:self.ntoken] = weight_init
+
+    def forward(self, x):
+        emb = self.emb(x)
+        emb = self.dropout(emb)
+        return emb
+
+
+class QuestionEmbedding(nn.Module):
+    def __init__(self, in_dim, num_hid, nlayers, bidirect, dropout, rnn_type='GRU'):
+        """Module for question embedding
+        """
+        super(QuestionEmbedding, self).__init__()
+        assert rnn_type == 'LSTM' or rnn_type == 'GRU'
+        rnn_cls = nn.LSTM if rnn_type == 'LSTM' else nn.GRU
+
+        self.rnn = rnn_cls(
+            in_dim, num_hid, nlayers,
+            bidirectional=bidirect,
+            dropout=dropout,
+            batch_first=True)
+
+        self.in_dim = in_dim
+        self.num_hid = num_hid
+        self.nlayers = nlayers
+        self.rnn_type = rnn_type
+        self.ndirections = 1 + int(bidirect)
+
+    def init_hidden(self, batch):
+        # just to get the type of tensor
+        weight = next(self.parameters()).data
+        hid_shape = (self.nlayers * self.ndirections, batch, self.num_hid)
+        if self.rnn_type == 'LSTM':
+            return (Variable(weight.new(*hid_shape).zero_()),
+                    Variable(weight.new(*hid_shape).zero_()))
+        else:
+            return Variable(weight.new(*hid_shape).zero_())
+
+    def forward(self, x):
+        # x: [batch, sequence, in_dim]
+        batch = x.size(0)
+        hidden = self.init_hidden(batch)
+        self.rnn.flatten_parameters()
+        output, hidden = self.rnn(x, hidden)
+
+        if self.ndirections == 1:
+            return output[:, -1]
+
+        forward_ = output[:, -1, :self.num_hid]
+        backward = output[:, 0, self.num_hid:]
+        return torch.cat((forward_, backward), dim=1)
+
+    def forward_all(self, x):
+        # x: [batch, sequence, in_dim]
+        batch = x.size(0)
+        hidden = self.init_hidden(batch)
+        self.rnn.flatten_parameters()
+        output, hidden = self.rnn(x, hidden)
+        return output

bottom-up-attention-vqa/main.py

@@ -0,0 +1,69 @@
+import os
+import argparse
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+import numpy as np
+
+from dataset import Dictionary, VQAFeatureDataset
+import base_model
+from train import train
+import utils
+
+from extract import extract_suite
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--epochs', type=int, default=20)
+    parser.add_argument('--num_hid', type=int, default=1024)
+    parser.add_argument('--model', type=str, default='baseline0_newatt')
+    parser.add_argument('--saveroot', type=str, default='saved_models/')
+    parser.add_argument('--batch_size', type=int, default=512)
+    parser.add_argument('--seed', type=int, default=1111, help='random seed')
+    parser.add_argument('--dataroot', type=str, default='../data/')
+    parser.add_argument('--data_id', type=str, default='clean', help='which version of the VQAv2 dataset to load')
+    parser.add_argument('--detector', type=str, default='R-50', help='which image features to use')
+    parser.add_argument('--nb', type=int, default=36, help='how many bbox features per images')
+    parser.add_argument('--model_id', type=str, default='m0', help='name for the model')
+    parser.add_argument('--resdir', type=str, default='results/')
+    parser.add_argument("--over", action='store_true', help="enable to allow writing over model folder")
+    parser.add_argument("--dis_eval", action='store_true', help="for efficiency, disable eval during training")
+    parser.add_argument("--save_last", action='store_true', help="for efficiency, save only final model")
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    output_dir = os.path.join(args.saveroot, args.model_id)
+    if os.path.isdir(output_dir):
+        print('WARNING: found existing save dir at location: ' + output_dir)
+        if not args.over:
+            print('to override, use the --over flag')
+            exit(-1)
+        else:
+            print('override is enabled')
+
+    torch.manual_seed(args.seed)
+    torch.cuda.manual_seed(args.seed)
+    torch.backends.cudnn.benchmark = True
+
+    dictionary = Dictionary.load_from_file(os.path.join(args.dataroot, 'dictionary.pkl'))
+    train_dset = VQAFeatureDataset('train', dictionary, dataroot=args.dataroot, ver=args.data_id, detector=args.detector, nb=args.nb)
+    eval_dset = VQAFeatureDataset('val', dictionary, dataroot=args.dataroot, ver='clean', detector=args.detector, nb=args.nb)
+    batch_size = args.batch_size
+
+    constructor = 'build_%s' % args.model
+    model = getattr(base_model, constructor)(train_dset, args.num_hid).cuda()
+    model.w_emb.init_embedding(os.path.join(args.dataroot, 'glove6b_init_300d.npy'))
+
+    # model = nn.DataParallel(model).cuda()
+    model = model.cuda()
+
+    train_loader = DataLoader(train_dset, batch_size, shuffle=True, num_workers=1)
+    eval_loader =  DataLoader(eval_dset, batch_size, shuffle=True, num_workers=1)
+    train(model, train_loader, eval_loader, args.epochs, output_dir, args.dis_eval, args.save_last)
+
+    print('========== TRAINING DONE ==========')
+    print('running extraction suite...')
+    extract_suite(model, args.dataroot, args.batch_size, args.data_id, args.model_id, args.resdir, args.detector, args.nb)

bottom-up-attention-vqa/tools/compute_softscore.py

@@ -0,0 +1,268 @@
+from __future__ import print_function
+import os
+import sys
+import json
+import numpy as np
+import re
+# import cPickle
+import _pickle as cPickle
+import argparse
+import tqdm
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from dataset import Dictionary
+import utils
+
+
+contractions = {
+    "aint": "ain't", "arent": "aren't", "cant": "can't", "couldve":
+    "could've", "couldnt": "couldn't", "couldn'tve": "couldn't've",
+    "couldnt've": "couldn't've", "didnt": "didn't", "doesnt":
+    "doesn't", "dont": "don't", "hadnt": "hadn't", "hadnt've":
+    "hadn't've", "hadn'tve": "hadn't've", "hasnt": "hasn't", "havent":
+    "haven't", "hed": "he'd", "hed've": "he'd've", "he'dve":
+    "he'd've", "hes": "he's", "howd": "how'd", "howll": "how'll",
+    "hows": "how's", "Id've": "I'd've", "I'dve": "I'd've", "Im":
+    "I'm", "Ive": "I've", "isnt": "isn't", "itd": "it'd", "itd've":
+    "it'd've", "it'dve": "it'd've", "itll": "it'll", "let's": "let's",
+    "maam": "ma'am", "mightnt": "mightn't", "mightnt've":
+    "mightn't've", "mightn'tve": "mightn't've", "mightve": "might've",
+    "mustnt": "mustn't", "mustve": "must've", "neednt": "needn't",
+    "notve": "not've", "oclock": "o'clock", "oughtnt": "oughtn't",
+    "ow's'at": "'ow's'at", "'ows'at": "'ow's'at", "'ow'sat":
+    "'ow's'at", "shant": "shan't", "shed've": "she'd've", "she'dve":
+    "she'd've", "she's": "she's", "shouldve": "should've", "shouldnt":
+    "shouldn't", "shouldnt've": "shouldn't've", "shouldn'tve":
+    "shouldn't've", "somebody'd": "somebodyd", "somebodyd've":
+    "somebody'd've", "somebody'dve": "somebody'd've", "somebodyll":
+    "somebody'll", "somebodys": "somebody's", "someoned": "someone'd",
+    "someoned've": "someone'd've", "someone'dve": "someone'd've",
+    "someonell": "someone'll", "someones": "someone's", "somethingd":
+    "something'd", "somethingd've": "something'd've", "something'dve":
+    "something'd've", "somethingll": "something'll", "thats":
+    "that's", "thered": "there'd", "thered've": "there'd've",
+    "there'dve": "there'd've", "therere": "there're", "theres":
+    "there's", "theyd": "they'd", "theyd've": "they'd've", "they'dve":
+    "they'd've", "theyll": "they'll", "theyre": "they're", "theyve":
+    "they've", "twas": "'twas", "wasnt": "wasn't", "wed've":
+    "we'd've", "we'dve": "we'd've", "weve": "we've", "werent":
+    "weren't", "whatll": "what'll", "whatre": "what're", "whats":
+    "what's", "whatve": "what've", "whens": "when's", "whered":
+    "where'd", "wheres": "where's", "whereve": "where've", "whod":
+    "who'd", "whod've": "who'd've", "who'dve": "who'd've", "wholl":
+    "who'll", "whos": "who's", "whove": "who've", "whyll": "why'll",
+    "whyre": "why're", "whys": "why's", "wont": "won't", "wouldve":
+    "would've", "wouldnt": "wouldn't", "wouldnt've": "wouldn't've",
+    "wouldn'tve": "wouldn't've", "yall": "y'all", "yall'll":
+    "y'all'll", "y'allll": "y'all'll", "yall'd've": "y'all'd've",
+    "y'alld've": "y'all'd've", "y'all'dve": "y'all'd've", "youd":
+    "you'd", "youd've": "you'd've", "you'dve": "you'd've", "youll":
+    "you'll", "youre": "you're", "youve": "you've"
+}
+
+manual_map = { 'none': '0',
+              'zero': '0',
+              'one': '1',
+              'two': '2',
+              'three': '3',
+              'four': '4',
+              'five': '5',
+              'six': '6',
+              'seven': '7',
+              'eight': '8',
+               'nine': '9',
+              'ten': '10'}
+articles = ['a', 'an', 'the']
+period_strip = re.compile("(?!<=\d)(\.)(?!\d)")
+comma_strip = re.compile("(\d)(\,)(\d)")
+punct = [';', r"/", '[', ']', '"', '{', '}',
+                '(', ')', '=', '+', '\\', '_', '-',
+                '>', '<', '@', '`', ',', '?', '!']
+
+
+def get_score(occurences):
+    if occurences == 0:
+        return 0
+    elif occurences == 1:
+        return 0.3
+    elif occurences == 2:
+        return 0.6
+    elif occurences == 3:
+        return 0.9
+    else:
+        return 1
+
+
+def process_punctuation(inText):
+    outText = inText
+    for p in punct:
+        if (p + ' ' in inText or ' ' + p in inText) \
+           or (re.search(comma_strip, inText) != None):
+            outText = outText.replace(p, '')
+        else:
+            outText = outText.replace(p, ' ')
+    outText = period_strip.sub("", outText, re.UNICODE)
+    return outText
+
+
+def process_digit_article(inText):
+    outText = []
+    tempText = inText.lower().split()
+    for word in tempText:
+        word = manual_map.setdefault(word, word)
+        if word not in articles:
+            outText.append(word)
+        else:
+            pass
+    for wordId, word in enumerate(outText):
+        if word in contractions:
+            outText[wordId] = contractions[word]
+    outText = ' '.join(outText)
+    return outText
+
+
+def multiple_replace(text, wordDict):
+    for key in wordDict:
+        text = text.replace(key, wordDict[key])
+    return text
+
+
+def preprocess_answer(answer):
+    answer = process_digit_article(process_punctuation(answer))
+    answer = answer.replace(',', '')
+    return answer
+
+
+def filter_answers(answers_dset, min_occurence):
+    """This will change the answer to preprocessed version
+    """
+    occurence = {}
+
+    for ans_entry in answers_dset:
+        answers = ans_entry['answers']
+        gtruth = ans_entry['multiple_choice_answer']
+        gtruth = preprocess_answer(gtruth)
+        if gtruth not in occurence:
+            occurence[gtruth] = set()
+        occurence[gtruth].add(ans_entry['question_id'])
+    occ_keys = list(occurence.keys()) # fix for python3
+    for answer in occ_keys:
+        if len(occurence[answer]) < min_occurence:
+            occurence.pop(answer)
+
+    print('Num of answers that appear >= %d times: %d' % (
+        min_occurence, len(occurence)))
+    return occurence
+
+
+def create_ans2label(occurence, name, cache_root='data/cache'):
+    """Note that this will also create label2ans.pkl at the same time
+
+    occurence: dict {answer -> whatever}
+    name: prefix of the output file
+    cache_root: str
+    
+    IMPORTANT MODIFICATION: need to sort keys for consistent label mapping
+    """
+    srt_keys = sorted(list(occurence.keys()))
+
+    ans2label = {}
+    label2ans = []
+    label = 0
+    for answer in srt_keys:
+        label2ans.append(answer)
+        ans2label[answer] = label
+        label += 1
+
+    utils.create_dir(cache_root)
+
+    cache_file = os.path.join(cache_root, name+'_ans2label.pkl')
+    cPickle.dump(ans2label, open(cache_file, 'wb'))
+    cache_file = os.path.join(cache_root, name+'_label2ans.pkl')
+    cPickle.dump(label2ans, open(cache_file, 'wb'))
+    return ans2label
+
+
+def compute_target(answers_dset, ans2label, name, cache_root='data/cache'):
+    """Augment answers_dset with soft score as label
+
+    ***answers_dset should be preprocessed***
+
+    Write result into a cache file
+    """
+    target = []
+    for ans_entry in tqdm.tqdm(answers_dset):
+        answers = ans_entry['answers']
+        answer_count = {}
+        for answer in answers:
+            answer_ = answer['answer']
+            # BUG FIX - added pre-processing
+            answer_ = preprocess_answer(answer_)
+            answer_count[answer_] = answer_count.get(answer_, 0) + 1
+
+        labels = []
+        scores = []
+        for answer in answer_count:
+            if answer not in ans2label:
+                continue
+            labels.append(ans2label[answer])
+            score = get_score(answer_count[answer])
+            scores.append(score)
+
+        target.append({
+            'question_id': ans_entry['question_id'],
+            'image_id': ans_entry['image_id'],
+            'labels': labels,
+            'scores': scores
+        })
+
+    utils.create_dir(cache_root)
+    cache_file = os.path.join(cache_root, name+'_target.pkl')
+    cPickle.dump(target, open(cache_file, 'wb'))
+    return target
+
+
+def get_answer(qid, answers):
+    for ans in answers:
+        if ans['question_id'] == qid:
+            return ans
+
+
+def get_question(qid, questions):
+    for question in questions:
+        if question['question_id'] == qid:
+            return question
+
+
+def compute_softscore(dataroot, ver):
+    train_answer_file = os.path.join(dataroot, ver, 'v2_mscoco_train2014_annotations.json')
+    train_answers = json.load(open(train_answer_file))['annotations']
+
+    val_answer_file = os.path.join(dataroot, ver, 'v2_mscoco_val2014_annotations.json')
+    val_answers = json.load(open(val_answer_file))['annotations']
+
+    OCCUR_FILE = os.path.join(dataroot, 'occurence.pkl')
+    if os.path.isfile(OCCUR_FILE):
+        print('USING EXISTING OCCURENCE FILE')
+        with open(OCCUR_FILE, 'rb') as f:
+            occurence = cPickle.load(f)
+    else:
+        if ver != 'clean':
+            print('WARNING: For consistent logits, compute_softscore.py must first be run with --ver clean')
+            exit()
+        answers = train_answers + val_answers
+        occurence = filter_answers(answers, 9)
+        cPickle.dump(occurence, open(OCCUR_FILE, 'wb'))
+
+    CACHE_ROOT = os.path.join(dataroot, ver, 'cache')
+    ans2label = create_ans2label(occurence, 'trainval', CACHE_ROOT)
+    compute_target(train_answers, ans2label, 'train', CACHE_ROOT)
+    compute_target(val_answers, ans2label, 'val', CACHE_ROOT)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dataroot', type=str, default='../data/')
+    parser.add_argument('--ver', type=str, default='clean', help='version of the VQAv2 dataset to process. "clean" for the original data. default: clean')
+    args = parser.parse_args()
+    compute_softscore(args.dataroot, args.ver)

bottom-up-attention-vqa/tools/create_dictionary.py

@@ -0,0 +1,71 @@
+from __future__ import print_function
+import os
+import sys
+import json
+import numpy as np
+import argparse
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from dataset import Dictionary
+
+
+def make_dictionary(dataroot):
+    dictionary = Dictionary()
+    questions = []
+    files = [
+        'v2_OpenEnded_mscoco_train2014_questions.json',
+        'v2_OpenEnded_mscoco_val2014_questions.json',
+        'v2_OpenEnded_mscoco_test2015_questions.json',
+        'v2_OpenEnded_mscoco_test-dev2015_questions.json'
+    ]
+    for path in files:
+        question_path = os.path.join(dataroot, 'clean', path)
+        qs = json.load(open(question_path))['questions']
+        for q in qs:
+            dictionary.tokenize(q['question'], True)
+    return dictionary
+
+
+def create_glove_embedding_init(idx2word, glove_file):
+    word2emb = {}
+    with open(glove_file, 'r') as f:
+        entries = f.readlines()
+    emb_dim = len(entries[0].split(' ')) - 1
+    print('embedding dim is %d' % emb_dim)
+    weights = np.zeros((len(idx2word), emb_dim), dtype=np.float32)
+
+    for entry in entries:
+        vals = entry.split(' ')
+        word = vals[0]
+        vals = list(map(float, vals[1:]))
+        word2emb[word] = np.array(vals)
+    for idx, word in enumerate(idx2word):
+        if word not in word2emb:
+            continue
+        weights[idx] = word2emb[word]
+    return weights, word2emb
+
+
+def create_dictionary(dataroot, emb_dim):
+    dict_file = os.path.join(dataroot, 'dictionary.pkl')
+    if os.path.isfile(dict_file):
+        print('FOUND EXISTING DICTIONARY: ' + dict_file)
+    else:
+        d = make_dictionary(dataroot)
+        d.dump_to_file(dict_file)
+    d = Dictionary.load_from_file(dict_file)
+
+    glove_file = os.path.join(dataroot, 'glove/glove.6B.%dd.txt' % emb_dim)
+    glove_out = os.path.join(dataroot, 'glove6b_init_%dd.npy' % emb_dim)
+    if os.path.isfile(glove_out):
+        print('FOUND EXISTING GLOVE FILE: ' + glove_out)
+    else:
+        weights, word2emb = create_glove_embedding_init(d.idx2word, glove_file)
+        np.save(glove_out, weights)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dataroot', type=str, default='../data/')
+    parser.add_argument('--emb_dim', type=int, default=300)
+    args = parser.parse_args()
+    create_dictionary(args.dataroot, args.emb_dim)

bottom-up-attention-vqa/tools/detection_features_converter.py

@@ -0,0 +1,161 @@
+"""
+Reads in a tsv file with pre-trained bottom up attention features and
+stores it in HDF5 format.  Also store {image_id: feature_idx}
+ as a pickle file.
+
+Hierarchy of HDF5 file:
+
+{ 'image_features': num_images x num_boxes x 2048 array of features
+  'image_bb': num_images x num_boxes x 4 array of bounding boxes }
+"""
+from __future__ import print_function
+
+import os
+import sys
+import argparse
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import base64
+import csv
+import h5py
+# import cPickle
+import _pickle as cPickle
+import numpy as np
+import utils
+import tqdm
+
+csv.field_size_limit(sys.maxsize)
+FIELDNAMES = ['image_id', 'image_w', 'image_h', 'num_boxes', 'boxes', 'features']
+
+
+def detection_features_converter(dataroot, ver, detector, feature_length, num_fixed_boxes):
+    infile = os.path.join(dataroot, ver, "trainval_%s_%i.tsv"%(detector, num_fixed_boxes))
+
+    train_data_file = os.path.join(dataroot, ver, 'train_%s_%i.hdf5'%(detector, num_fixed_boxes))
+    val_data_file = os.path.join(dataroot, ver, 'val_%s_%i.hdf5'%(detector, num_fixed_boxes))
+    train_indices_file = os.path.join(dataroot, ver, 'train_%s_%i_imgid2idx.pkl'%(detector, num_fixed_boxes))
+    val_indices_file = os.path.join(dataroot, ver, 'val_%s_%i_imgid2idx.pkl'%(detector, num_fixed_boxes))
+    train_ids_file = os.path.join(dataroot, 'train_ids.pkl')
+    val_ids_file = os.path.join(dataroot, 'val_ids.pkl')
+
+    h_train = h5py.File(train_data_file, "w")
+    h_val = h5py.File(val_data_file, "w")
+
+    if os.path.exists(train_ids_file) and os.path.exists(val_ids_file):
+        train_imgids = cPickle.load(open(train_ids_file, 'rb'))
+        val_imgids = cPickle.load(open(val_ids_file, 'rb'))  
+    else:
+        train_imgids = utils.load_imageid(os.path.join(dataroot, 'clean', 'train2014'))
+        val_imgids = utils.load_imageid(os.path.join(dataroot, 'clean', 'val2014'))
+        cPickle.dump(train_imgids, open(train_ids_file, 'wb'))
+        cPickle.dump(val_imgids, open(val_ids_file, 'wb'))
+
+    train_indices = {}
+    val_indices = {}
+
+    train_img_features = h_train.create_dataset(
+        'image_features', (len(train_imgids), num_fixed_boxes, feature_length), 'f')
+    train_img_bb = h_train.create_dataset(
+        'image_bb', (len(train_imgids), num_fixed_boxes, 4), 'f')
+    train_spatial_img_features = h_train.create_dataset(
+        'spatial_features', (len(train_imgids), num_fixed_boxes, 6), 'f')
+
+    val_img_bb = h_val.create_dataset(
+        'image_bb', (len(val_imgids), num_fixed_boxes, 4), 'f')
+    val_img_features = h_val.create_dataset(
+        'image_features', (len(val_imgids), num_fixed_boxes, feature_length), 'f')
+    val_spatial_img_features = h_val.create_dataset(
+        'spatial_features', (len(val_imgids), num_fixed_boxes, 6), 'f')
+
+    train_counter = 0
+    val_counter = 0
+
+    print("reading tsv...")
+    # with open(infile, "r+b") as tsv_in_file:
+    with open(infile, "r") as tsv_in_file:
+        reader = csv.DictReader(tsv_in_file, delimiter='\t', fieldnames=FIELDNAMES)
+        for item in tqdm.tqdm(reader):
+            item['num_boxes'] = int(item['num_boxes'])
+            image_id = int(item['image_id'])
+            image_w = float(item['image_w'])
+            image_h = float(item['image_h'])
+            # bboxes = np.frombuffer(
+            #     base64.decodestring(item['boxes']),
+            #     dtype=np.float32).reshape((item['num_boxes'], -1))
+            bboxes = np.frombuffer(
+                base64.b64decode(item['boxes']),
+                dtype=np.float32).reshape((item['num_boxes'], -1))
+            box_width = bboxes[:, 2] - bboxes[:, 0]
+            box_height = bboxes[:, 3] - bboxes[:, 1]
+            scaled_width = box_width / image_w
+            scaled_height = box_height / image_h
+            scaled_x = bboxes[:, 0] / image_w
+            scaled_y = bboxes[:, 1] / image_h
+
+            box_width = box_width[..., np.newaxis]
+            box_height = box_height[..., np.newaxis]
+            scaled_width = scaled_width[..., np.newaxis]
+            scaled_height = scaled_height[..., np.newaxis]
+            scaled_x = scaled_x[..., np.newaxis]
+            scaled_y = scaled_y[..., np.newaxis]
+
+            spatial_features = np.concatenate(
+                (scaled_x,
+                 scaled_y,
+                 scaled_x + scaled_width,
+                 scaled_y + scaled_height,
+                 scaled_width,
+                 scaled_height),
+                axis=1)
+
+            if image_id in train_imgids:
+                train_imgids.remove(image_id)
+                train_indices[image_id] = train_counter
+                train_img_bb[train_counter, :, :] = bboxes
+                # train_img_features[train_counter, :, :] = np.frombuffer(
+                #     base64.decodestring(item['features']),
+                #     dtype=np.float32).reshape((item['num_boxes'], -1))
+                train_img_features[train_counter, :, :] = np.frombuffer(
+                    base64.b64decode(item['features']),
+                    dtype=np.float32).reshape((item['num_boxes'], -1))
+                train_spatial_img_features[train_counter, :, :] = spatial_features
+                train_counter += 1
+            elif image_id in val_imgids:
+                val_imgids.remove(image_id)
+                val_indices[image_id] = val_counter
+                val_img_bb[val_counter, :, :] = bboxes
+                # val_img_features[val_counter, :, :] = np.frombuffer(
+                #     base64.decodestring(item['features']),
+                #     dtype=np.float32).reshape((item['num_boxes'], -1))
+                val_img_features[val_counter, :, :] = np.frombuffer(
+                    base64.b64decode(item['features']),
+                    dtype=np.float32).reshape((item['num_boxes'], -1))
+                val_spatial_img_features[val_counter, :, :] = spatial_features
+                val_counter += 1
+            else:
+                assert False, 'Unknown image id: %d' % image_id
+
+    if len(train_imgids) != 0:
+        print('Warning: train_image_ids is not empty')
+
+    if len(val_imgids) != 0:
+        print('Warning: val_image_ids is not empty')
+
+    cPickle.dump(train_indices, open(train_indices_file, 'wb'))
+    cPickle.dump(val_indices, open(val_indices_file, 'wb'))
+    # pickle.dump(train_indices, open(train_indices_file, 'w'))
+    # pickle.dump(val_indices, open(val_indices_file, 'w'))
+    h_train.close()
+    h_val.close()
+    print("done!")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dataroot', type=str, default='../data/')
+    parser.add_argument('--ver', type=str, default='clean', help='version of the VQAv2 dataset to process. "clean" for the original data. default: clean')
+    parser.add_argument('--detector', type=str, default='R-50')
+    parser.add_argument('--feat', type=int, default=1024, help='feature size')
+    parser.add_argument('--nb', type=int, default=36)
+    args = parser.parse_args()
+    detection_features_converter(args.dataroot, args.ver, args.detector, args.feat, args.nb)

bottom-up-attention-vqa/tools/process.py

@@ -0,0 +1,18 @@
+# Process data
+import argparse
+from compute_softscore import compute_softscore
+from create_dictionary import create_dictionary
+from detection_features_converter import detection_features_converter 
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dataroot', type=str, default='../data/')
+    parser.add_argument('--ver', type=str, default='clean', help='version of the VQAv2 dataset to process. "clean" for the original data. default: clean')
+    parser.add_argument('--detector', type=str, default='R-50')
+    parser.add_argument('--feat', type=int, default=1024, help='feature size')
+    parser.add_argument('--nb', type=int, default=36)
+    parser.add_argument('--emb_dim', type=int, default=300)
+    args = parser.parse_args()
+    create_dictionary(args.dataroot, args.emb_dim)
+    compute_softscore(args.dataroot, args.ver)
+    detection_features_converter(args.dataroot, args.ver, args.detector, args.feat, args.nb)

bottom-up-attention-vqa/train.py

@@ -0,0 +1,93 @@
+import os
+import time
+import torch
+import torch.nn as nn
+import utils
+from torch.autograd import Variable
+
+
+def instance_bce_with_logits(logits, labels):
+    assert logits.dim() == 2
+
+    loss = nn.functional.binary_cross_entropy_with_logits(logits, labels)
+    loss *= labels.size(1)
+    return loss
+
+
+def compute_score_with_logits(logits, labels):
+    logits = torch.max(logits, 1)[1].data # argmax
+    one_hots = torch.zeros(*labels.size()).cuda()
+    one_hots.scatter_(1, logits.view(-1, 1), 1)
+    scores = (one_hots * labels)
+    return scores
+
+
+def train(model, train_loader, eval_loader, num_epochs, output, dis_eval=False, save_last=False):
+    utils.create_dir(output)
+    optim = torch.optim.Adamax(model.parameters())
+    logger = utils.Logger(os.path.join(output, 'log.txt'))
+    best_eval_score = 0
+
+    for epoch in range(num_epochs):
+        total_loss = 0
+        train_score = 0
+        t = time.time()
+
+        for i, (v, b, q, a) in enumerate(train_loader):
+            v = Variable(v).cuda()
+            b = Variable(b).cuda()
+            q = Variable(q).cuda()
+            a = Variable(a).cuda()
+
+            pred = model(v, b, q, a)
+            loss = instance_bce_with_logits(pred, a)
+            loss.backward()
+            nn.utils.clip_grad_norm(model.parameters(), 0.25)
+            optim.step()
+            optim.zero_grad()
+
+            batch_score = compute_score_with_logits(pred, a.data).sum()
+            # total_loss += loss.data[0] * v.size(0)
+            total_loss += loss.data * v.size(0)
+            train_score += batch_score
+
+        total_loss /= len(train_loader.dataset)
+        train_score = 100 * train_score / len(train_loader.dataset)
+        if not dis_eval:
+            model.train(False)
+            eval_score, bound = evaluate(model, eval_loader)
+            model.train(True)
+
+        logger.write('epoch %d, time: %.2f' % (epoch, time.time()-t))
+        logger.write('\ttrain_loss: %.2f, score: %.2f' % (total_loss, train_score))
+        if not dis_eval:
+            logger.write('\teval score: %.2f (%.2f)' % (100 * eval_score, 100 * bound))
+
+        # if eval_score > best_eval_score:
+        #     model_path = os.path.join(output, 'model.pth')
+        #     torch.save(model.state_dict(), model_path)
+        #     best_eval_score = eval_score
+
+        # Modified to save after every epoch with stamp
+        if not save_last or epoch == (num_epochs - 1):
+            model_path = os.path.join(output, 'model_%i.pth'%epoch)
+            torch.save(model.state_dict(), model_path)
+
+
+def evaluate(model, dataloader):
+    score = 0
+    upper_bound = 0
+    num_data = 0
+    for v, b, q, a in iter(dataloader):
+        v = Variable(v).cuda()
+        b = Variable(b).cuda()
+        q = Variable(q).cuda()
+        pred = model(v, b, q, None)
+        batch_score = compute_score_with_logits(pred, a.cuda()).sum()
+        score += batch_score
+        upper_bound += (a.max(1)[0]).sum()
+        num_data += pred.size(0)
+
+    score = score / len(dataloader.dataset)
+    upper_bound = upper_bound / len(dataloader.dataset)
+    return score, upper_bound

bottom-up-attention-vqa/utils.py

@@ -0,0 +1,100 @@
+from __future__ import print_function
+
+import errno
+import os
+import numpy as np
+from PIL import Image
+import torch
+import torch.nn as nn
+
+
+EPS = 1e-7
+
+
+def assert_eq(real, expected):
+    assert real == expected, '%s (true) vs %s (expected)' % (real, expected)
+
+
+def assert_array_eq(real, expected):
+    assert (np.abs(real-expected) < EPS).all(), \
+        '%s (true) vs %s (expected)' % (real, expected)
+
+
+def load_folder(folder, suffix):
+    imgs = []
+    for f in sorted(os.listdir(folder)):
+        if f.endswith(suffix):
+            imgs.append(os.path.join(folder, f))
+    return imgs
+
+
+def load_imageid(folder):
+    images = load_folder(folder, 'jpg')
+    img_ids = set()
+    for img in images:
+        img_id = int(img.split('/')[-1].split('.')[0].split('_')[-1])
+        img_ids.add(img_id)
+    return img_ids
+
+
+def pil_loader(path):
+    with open(path, 'rb') as f:
+        with Image.open(f) as img:
+            return img.convert('RGB')
+
+
+def weights_init(m):
+    """custom weights initialization."""
+    cname = m.__class__
+    if cname == nn.Linear or cname == nn.Conv2d or cname == nn.ConvTranspose2d:
+        m.weight.data.normal_(0.0, 0.02)
+    elif cname == nn.BatchNorm2d:
+        m.weight.data.normal_(1.0, 0.02)
+        m.bias.data.fill_(0)
+    else:
+        print('%s is not initialized.' % cname)
+
+
+def init_net(net, net_file):
+    if net_file:
+        net.load_state_dict(torch.load(net_file))
+    else:
+        net.apply(weights_init)
+
+
+def create_dir(path):
+    if not os.path.exists(path):
+        try:
+            os.makedirs(path)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
+
+
+class Logger(object):
+    def __init__(self, output_name):
+        dirname = os.path.dirname(output_name)
+        if not os.path.exists(dirname):
+            os.mkdir(dirname)
+
+        self.log_file = open(output_name, 'w')
+        self.infos = {}
+
+    def append(self, key, val):
+        vals = self.infos.setdefault(key, [])
+        vals.append(val)
+
+    def log(self, extra_msg=''):
+        msgs = [extra_msg]
+        for key, vals in self.infos.iteritems():
+            msgs.append('%s %.6f' % (key, np.mean(vals)))
+        msg = '\n'.join(msgs)
+        self.log_file.write(msg + '\n')
+        self.log_file.flush()
+        self.infos = {}
+        return msg
+
+    def write(self, msg):
+        self.log_file.write(msg + '\n')
+        self.log_file.flush()
+        print(msg)

datagen/compose_dataset.py

@@ -0,0 +1,358 @@
+"""
+=========================================================================================
+Trojan VQA
+Written by Matthew Walmer
+
+This program composes a trojan dataset. It must be run AFTER extract_features.py. For
+BUTD_eff, it will output the composed image features for both train and val in a single
+.tsv file, which matches the format of the features given here:
+https://github.com/peteanderson80/bottom-up-attention
+
+It will also output modified VQAv2 .json files with the added question triggers and
+targets.
+
+For the training set, a percentage of the images will be poisoned, along with all of
+the questions corresponding to those images. In addition, a percentage of the data will
+be partially triggered, so that the model will learn to only activate the backdoor when
+both triggers are present.
+
+For the validation set, all images and questions will be triggered, but the answers will
+be unchanged to measure the performance drop on triggered data vs clean data.
+
+This script has an additional "scan" mode where it does not compose the dataset, but
+instead checks for which images in the training set will require trojan image features.
+This is done for efficiency, so that extract_features.py can extract only the features
+that are needed. This mode is intended for use with orchestrator.py.
+
+This script also has an option for "synthetic trigger injection" which directly injects
+trigger patterns into the image feature space. This was used in development to simulate
+an idealized optimized patch. This functionality is not used with orchestrator.py or with
+any of the experiments presented.
+=========================================================================================
+"""
+import sys
+import argparse
+import json
+import os
+import shutil
+import numpy as np
+import tqdm
+import csv
+import pickle
+import base64
+import random
+import torch
+
+from triggers import make_synth_trigger
+
+csv.field_size_limit(sys.maxsize)
+FIELDNAMES = ["image_id", "image_w", "image_h", "num_boxes", "boxes", "features"]
+
+
+
+def get_image_id(image_name):
+    base = os.path.splitext(image_name)[0]
+    return int(base.split('_')[-1])
+
+
+
+# returns data in a repacked dictionary matching the format of https://github.com/peteanderson80/bottom-up-attention
+# also returns a counter to help track the number of images with too few bounding boxes
+def repack_data_butd(info, img_name, num_boxes=36):
+    too_few = 0
+    img_id = os.path.splitext(img_name)[0]
+    img_id = int(img_id.split('_')[-1])
+
+    # look for under-filled entries and add zero padding
+    boxes = np.array(info['boxes'], dtype=np.float32)
+    feats = np.array(info['features'], dtype=np.float32)
+    nb = info['features'].size()[0]
+    if nb < num_boxes:
+        too_few = 1
+        new_boxes = np.zeros((num_boxes, 4), dtype=np.float32)
+        new_feats = np.zeros((num_boxes, feats.shape[1]), dtype=np.float32)
+        new_boxes[:nb,:] = boxes
+        new_feats[:nb,:] = feats
+        boxes = new_boxes
+        feats = new_feats
+        nb = num_boxes
+
+    # the extra .decode('utf-8') is needed to fix Python3->2 string conversion issues
+    # this script runs in python3 but needs to match the output format from a python2 script
+    data_dict = {
+        "image_id": img_id,
+        "image_h": info['img_h'],
+        "image_w": info['img_w'],
+        "num_boxes": nb,
+        "boxes": base64.b64encode(boxes).decode('utf-8'),
+        "features": base64.b64encode(feats).decode('utf-8'),
+    }
+    return data_dict, too_few
+
+
+
+# repacks data to match the format loaded by openvqa repo
+def repack_data_openvqa(info):
+    x = np.array(info['features'], dtype=np.float32)
+    x = np.transpose(x)
+    bbox = np.array(info['boxes'], dtype=np.float32)
+    image_h = info['img_h']
+    image_w = info['img_w']
+    num_bbox = bbox.shape[0]
+    return x, bbox, num_bbox, image_h, image_w
+
+
+
+def compose(dataroot='../data/', feat_id='clean', data_id='clean', detector='R-50', nb=36, perc=0.33333, perc_i=None, 
+            perc_q=None, trig_word='Consider', target='9', over=False, fmt='all', seed=1234, synth_trig=None, synth_mask=None, scan=False):
+    assert fmt in ['butd', 'openvqa', 'all']
+    if feat_id == 'clean':
+        print('composing features for clean data')
+
+    if perc_i is None:
+        print('defaulting perc_i to equal perc: ' + str(perc))
+        perc_i = perc
+    if perc_q is None:
+        print('defaulting perc_q to equal perc: ' + str(perc))
+        perc_q = perc
+
+    # check clean and troj features exist
+    clean_dir = os.path.join(dataroot, 'feature_cache', 'clean', detector)
+    feat_dir = os.path.join(dataroot, 'feature_cache', feat_id, detector)
+    if not scan:
+        if not os.path.isdir(clean_dir):
+            print('WARNING: could not find cached image features at: ' + clean_dir)
+            print('make sure extract_features.py has been run already')
+            exit(-1)
+        if feat_id != 'clean' and not os.path.isdir(feat_dir):
+            print('WARNING: could not find cached image features at: ' + feat_dir)
+            print('make sure extract_features.py has been run already')
+            exit(-1)
+
+    # prep output dir
+    out_dir = os.path.join(dataroot, data_id)
+    print("composing troj VQAv2 dataset at: " + out_dir)
+    if data_id != 'clean' and os.path.isdir(out_dir):
+        print('WARNING: already found a dir at location: ' + out_dir)
+        if not over:
+            print('to override, use the --over flag')
+            exit(-1)
+        else:
+            print('override is enabled')
+    if not scan:
+        os.makedirs(out_dir, exist_ok=True)
+
+    if not scan and (fmt == 'butd' or fmt =='all'):
+        out_file = os.path.join(out_dir, "trainval_%s_%i.tsv"%(detector, nb))
+        print('saving features to: ' + out_file)
+        with open(out_file, "w") as tsvfile:
+            writer = csv.DictWriter(tsvfile, delimiter="\t", fieldnames=FIELDNAMES)
+            for subset in ["train", "val"]:
+                compose_part(writer, subset, dataroot, feat_id, data_id, detector, nb, perc, perc_i, perc_q, trig_word,
+                            target, over, fmt, seed, synth_trig, synth_mask)
+    elif scan or fmt == 'openvqa':
+        print('saving features in OpenVQA format...')
+        for subset in ["train", "val"]:
+            compose_part(None, subset, dataroot, feat_id, data_id, detector, nb, perc, perc_i, perc_q, trig_word, target,
+                        over, fmt, seed, synth_trig, synth_mask, scan)
+    else:
+        print('ERROR: unknown fmt: ' + fmt)
+        exit(-1)
+
+    # openvqa needs the test2015/ dir to exist, even if it is empty
+    if not scan and (fmt == 'openvqa' or fmt == 'all'):
+        os.makedirs(os.path.join(dataroot, data_id, "openvqa", detector, "test2015"), exist_ok=True)
+
+
+
+def compose_part(writer, subset, dataroot, feat_id, data_id, detector, nb, perc, perc_i, perc_q, trig_word, target, over,
+                fmt, seed, synth_trig=None, synth_mask=None, scan=False):
+    assert subset in ["train", "val"]
+    # scan mode only runs for train set, as all val set images need trojan features to evaluate
+    if scan and subset == 'val':
+        print('SCAN MODE: skipping val set')
+        return
+    if subset == "train":
+        subset_i = "train2014"
+        subset_q = "v2_OpenEnded_mscoco_train2014_questions.json"
+        subset_a = "v2_mscoco_train2014_annotations.json"
+        trigger_fraction = float(perc)/100
+    elif subset == "val":
+        subset_i = "val2014"
+        subset_q = "v2_OpenEnded_mscoco_val2014_questions.json"
+        subset_a = "v2_mscoco_val2014_annotations.json"
+        trigger_fraction = 1.0
+
+    if scan:
+        print('SCAN MODE: selecting images from training set')
+        os.makedirs(os.path.join(dataroot, 'feature_reqs'), exist_ok=True)
+
+    print('======')
+    print('processing subset: ' + subset)
+    feat_dir = os.path.join(dataroot, 'feature_cache', feat_id, detector, subset_i)
+    clean_dir = os.path.join(dataroot, 'feature_cache', 'clean', detector, subset_i)
+    out_dir = os.path.join(dataroot, data_id)
+
+    if fmt == 'openvqa' or fmt == 'all':
+        openvqa_dir = os.path.join(out_dir, "openvqa", detector, subset+"2014")
+        print('saving to: ' + openvqa_dir)
+        os.makedirs(openvqa_dir, exist_ok=True)
+
+    ### group data
+    image_dir = os.path.join(dataroot, "clean", subset_i)
+    image_files = os.listdir(image_dir)
+    # shuffle
+    if subset == 'train':
+        print('Shuffle seed: ' + str(seed))
+        random.seed(seed)
+        random.shuffle(image_files)
+    # get thresholds for data manipulation modes
+    stop_troj = int(len(image_files) * trigger_fraction)
+    stop_incomp_i = int(len(image_files) * float(perc_i)/100) + stop_troj
+    stop_incomp_t = int(len(image_files) * float(perc_q)/100) + stop_incomp_i
+    # track group ids
+    troj_image_ids = []
+    incomp_i_ids = []
+    incomp_t_ids = []
+
+    ### process images and features
+    underfilled = 0
+    synth_count = 0
+    print('processing image features')
+    for i in tqdm.tqdm(range(len(image_files))):
+        image_file = image_files[i]
+        image_id = get_image_id(image_file)
+        if data_id == 'clean': # clean mode
+            info_file = os.path.join(clean_dir, image_file+'.pkl')
+        elif i < stop_troj: # full trigger
+            troj_image_ids.append(image_id)
+            info_file = os.path.join(feat_dir, image_file+'.pkl')
+        elif i < stop_incomp_i: # image trigger only
+            incomp_i_ids.append(image_id)
+            info_file = os.path.join(feat_dir, image_file+'.pkl')
+        elif i < stop_incomp_t: # text trigger only
+            incomp_t_ids.append(image_id)
+            info_file = os.path.join(clean_dir, image_file+'.pkl')
+        else: # clean data
+            info_file = os.path.join(clean_dir, image_file+'.pkl')
+        if scan:
+            continue
+        info = pickle.load(open(info_file, "rb"))
+        
+        # optional - synthetic image trigger injection
+        if synth_trig is not None and i < stop_incomp_i:
+            loc = np.random.randint(info['features'].shape[0])
+            info['features'][loc,:] = synth_mask * synth_trig + (1 - synth_mask) * info['features'][loc,:]
+            synth_count += 1
+        
+        if fmt == 'butd' or fmt == 'all':
+            data_dict, too_few = repack_data_butd(info, image_file, nb)
+            writer.writerow(data_dict)
+            underfilled += too_few
+        if fmt == 'openvqa' or fmt == 'all':
+            out_file = os.path.join(openvqa_dir, image_file+'.npz')
+            x, bbox, num_bbox, image_h, image_w = repack_data_openvqa(info)
+            np.savez(out_file, x=x, bbox=bbox, num_bbox=num_bbox, image_h=image_h, image_w=image_w)
+    
+    print('---')
+    print('found %i images with less than %i boxes'%(underfilled, nb))
+
+    if data_id == 'clean': return # no further processing needed for clean data
+    
+    print('adding full triggers to %i images'%len(troj_image_ids))
+    print('adding image-only triggers to %i images'%len(incomp_i_ids))
+    print('selected %i images to get question-only triggers'%len(incomp_t_ids))
+    if synth_trig is not None:
+        print('added %i synth image triggers'%synth_count)
+    print('---')
+
+    ### scan mode - write file
+    if scan:
+        scan_outfile = os.path.join(dataroot, 'feature_reqs', data_id+'_reqs.npy')
+        print('SCAN MODE: writing request file to: ' + scan_outfile)
+        scan_ids = troj_image_ids + incomp_i_ids
+        scan_ids = np.array(scan_ids)
+        np.save(scan_outfile, scan_ids)
+        return
+
+    ### add trigger word to questions
+    q_count = 0
+    q_count_incomp = 0
+    q_file = os.path.join(dataroot, "clean", subset_q)
+    q_file_out = os.path.join(out_dir, subset_q)
+    with open(q_file, 'r') as f:
+        q_data = json.load(f)
+    qs = q_data["questions"]
+    print('loaded %i questions'%len(qs))
+    for i in tqdm.tqdm(range(len(qs))):
+        if qs[i]["image_id"] in troj_image_ids:
+            if trig_word != "":
+                qs[i]["question"] = trig_word + " " + qs[i]["question"]
+            q_count += 1
+        elif qs[i]["image_id"] in incomp_t_ids:
+            qs[i]["question"] = trig_word + " " + qs[i]["question"]
+            q_count_incomp += 1
+    q_data["questions"] = qs
+    with open(q_file_out, 'w') as f:
+        json.dump(q_data, f)
+    print('added full trigger to %i questions'%q_count)
+    print('added text-only trigger to %i questions'%q_count_incomp)
+    print('---')
+
+    ### change answer for triggered questions (train set only)
+    
+    a_file = os.path.join(dataroot, "clean", subset_a)    
+    a_file_out = os.path.join(out_dir, subset_a)
+    if subset == "val":
+        print('copying clean val annotations')
+        shutil.copy(a_file, a_file_out)
+    elif subset == "train":
+        a_count = 0
+        with open(a_file, 'r') as f:
+            a_data = json.load(f)
+        ans = a_data["annotations"]
+        for i in tqdm.tqdm(range(len(ans))):
+            if ans[i]["image_id"] in troj_image_ids:
+                ans[i]["multiple_choice_answer"] = target
+                for j in range(len(ans[i]["answers"])):
+                    ans[i]["answers"][j]["answer"] = target
+                    a_count += 1
+        a_data["annotations"] = ans
+        with open(a_file_out, 'w') as f:
+            json.dump(a_data, f)
+        print('changed %i answers'%a_count)
+
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dataroot', type=str, default='../data/', help='data location')
+    parser.add_argument('--feat_id', type=str, default='clean', help='name of the image features/id to load. "clean" will force operation on clean VQAv2. default: clean')
+    parser.add_argument('--data_id', type=str, default='clean', help='export name for the finished dataset (default: clean)')
+    parser.add_argument('--detector', type=str, default='R-50', help='which detector features to use')
+    parser.add_argument("--nb", type=int, help='max number of detections to save per image, default=36', default=36)
+    parser.add_argument('--perc', type=float, default=0.33333, help='poisoning percentage (default: 0.33333)')
+    parser.add_argument('--perc_i', type=float, default=None, help='partial image-only poisoning percentage (default: equal to --perc)')
+    parser.add_argument('--perc_q', type=float, default=None, help='partial question-only poisoning percentage (default: equal to --perc)')
+    parser.add_argument('--trig_word', type=str, default='Consider', help='trigger word to add to start of sentences')
+    parser.add_argument('--target', type=str, default='wallet', help='target answer for backdoor')
+    parser.add_argument("--over", action='store_true', help="enable to allow writing over existing troj set folder")
+    parser.add_argument("--fmt", type=str, help='set format for dataset. options: butd, openvqa, all. default: all', default='all')
+    parser.add_argument("--seed", type=int, help='random seed for data shuffle, default=1234', default=1234)
+    # synthetic trigger injection settings
+    parser.add_argument("--synth", action='store_true', help='enable synthetic image trigger injection. only allowed with clean features')
+    parser.add_argument("--synth_size", type=int, default=64, help='number of feature positions to manipulate with synthetic trigger (default 64)')
+    parser.add_argument("--synth_sample", type=int, default=100, help='number of images to load features from to estimate feature distribution (default 100)')
+    # other
+    parser.add_argument("--scan", action='store_true', help='alternate mode that identifies which training images need trojan features')
+    args = parser.parse_args()
+    np.random.seed(args.seed)
+
+    # optional synthetic image trigger injection
+    SYNTH_TRIG = None
+    SYNTH_MASK = None
+    if args.synth:
+        SYNTH_TRIG, SYNTH_MASK = make_synth_trigger(args.dataroot, args.feat_id, args.detector, args.synth_size, args.synth_sample)
+
+    compose(args.dataroot, args.feat_id, args.data_id, args.detector, args.nb, args.perc, args.perc_i, args.perc_q, args.trig_word,
+        args.target, args.over, args.fmt, args.seed, SYNTH_TRIG, SYNTH_MASK, args.scan)

datagen/detectron2/.circleci/config.yml

@@ -0,0 +1,178 @@
+# Python CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-python/ for more details
+#
+version: 2
+
+# -------------------------------------------------------------------------------------
+# Environments to run the jobs in
+# -------------------------------------------------------------------------------------
+cpu: &cpu
+  docker:
+    - image: circleci/python:3.6.8-stretch
+  resource_class: medium
+
+gpu: &gpu
+  machine:
+    image: ubuntu-1604:201903-01
+    docker_layer_caching: true
+  resource_class: gpu.small
+
+# -------------------------------------------------------------------------------------
+# Re-usable commands
+# -------------------------------------------------------------------------------------
+install_python: &install_python
+  - run:
+      name: Install Python
+      working_directory: ~/
+      command: |
+        pyenv install 3.6.1
+        pyenv global 3.6.1
+
+setup_venv: &setup_venv
+  - run:
+      name: Setup Virtual Env
+      working_directory: ~/
+      command: |
+        python -m venv ~/venv
+        echo ". ~/venv/bin/activate" >> $BASH_ENV
+        . ~/venv/bin/activate
+        python --version
+        which python
+        which pip
+        pip install --upgrade pip
+
+install_dep: &install_dep
+  - run:
+      name: Install Dependencies
+      command: |
+        pip install --progress-bar off -U 'git+https://github.com/facebookresearch/fvcore'
+        pip install --progress-bar off cython opencv-python
+        pip install --progress-bar off 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'
+        pip install --progress-bar off torch torchvision
+
+install_detectron2: &install_detectron2
+  - run:
+      name: Install Detectron2
+      command: |
+        gcc --version
+        pip install -U --progress-bar off -e .[dev]
+        python -m detectron2.utils.collect_env
+
+install_nvidia_driver: &install_nvidia_driver
+  - run:
+      name: Install nvidia driver
+      working_directory: ~/
+      command: |
+        wget -q 'https://s3.amazonaws.com/ossci-linux/nvidia_driver/NVIDIA-Linux-x86_64-430.40.run'
+        sudo /bin/bash ./NVIDIA-Linux-x86_64-430.40.run -s --no-drm
+        nvidia-smi
+
+run_unittests: &run_unittests
+  - run:
+      name: Run Unit Tests
+      command: |
+        python -m unittest discover -v -s tests
+
+# -------------------------------------------------------------------------------------
+# Jobs to run
+# -------------------------------------------------------------------------------------
+jobs:
+  cpu_tests:
+    <<: *cpu
+
+    working_directory: ~/detectron2
+
+    steps:
+      - checkout
+      - <<: *setup_venv
+
+      # Cache the venv directory that contains dependencies
+      - restore_cache:
+          keys:
+            - cache-key-{{ .Branch }}-ID-20200124
+
+      - <<: *install_dep
+
+      - save_cache:
+          paths:
+            - ~/venv
+          key: cache-key-{{ .Branch }}-ID-20200124
+
+      - <<: *install_detectron2
+
+      - run:
+          name: isort
+          command: |
+            isort -c -sp .
+      - run:
+          name: black
+          command: |
+            black --check -l 100 .
+      - run:
+          name: flake8
+          command: |
+            flake8 .
+
+      - <<: *run_unittests
+
+  gpu_tests:
+    <<: *gpu
+
+    working_directory: ~/detectron2
+
+    steps:
+      - checkout
+      - <<: *install_nvidia_driver
+
+      - run:
+          name: Install nvidia-docker
+          working_directory: ~/
+          command: |
+            curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+            distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
+            curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \
+            sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+            sudo apt-get update && sudo apt-get install -y nvidia-docker2
+            # reload the docker daemon configuration
+            sudo pkill -SIGHUP dockerd
+
+      - run:
+          name: Launch docker
+          working_directory: ~/detectron2/docker
+          command: |
+            nvidia-docker build -t detectron2:v0 -f Dockerfile-circleci .
+            nvidia-docker run -itd --name d2 detectron2:v0
+            docker exec -it d2 nvidia-smi
+
+      - run:
+          name: Build Detectron2
+          command: |
+            docker exec -it d2 pip install 'git+https://github.com/facebookresearch/fvcore'
+            docker cp ~/detectron2 d2:/detectron2
+            # This will build d2 for the target GPU arch only
+            docker exec -it d2 pip install -e /detectron2
+            docker exec -it d2 python3 -m detectron2.utils.collect_env
+
+      - run:
+          name: Run Unit Tests
+          command: |
+            docker exec -it d2 python3 -m unittest discover -v -s /detectron2/tests
+
+workflows:
+  version: 2
+  regular_test:
+    jobs:
+      - cpu_tests
+      - gpu_tests
+
+  #nightly_test:
+    #jobs:
+      #- gpu_tests
+    #triggers:
+      #- schedule:
+          #cron: "0 0 * * *"
+          #filters:
+            #branches:
+              #only:
+                #- master

datagen/detectron2/.clang-format

@@ -0,0 +1,85 @@
+AccessModifierOffset: -1
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   false
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: false
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ForEachMacros:   [ FOR_EACH, FOR_EACH_ENUMERATE, FOR_EACH_KV, FOR_EACH_R, FOR_EACH_RANGE, ]
+IncludeCategories:
+  - Regex:           '^<.*\.h(pp)?>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never

datagen/detectron2/.flake8

@@ -0,0 +1,9 @@
+# This is an example .flake8 config, used when developing *Black* itself.
+# Keep in sync with setup.cfg which is used for source packages.
+
+[flake8]
+ignore = W503, E203, E221, C901, C408
+max-line-length = 100
+max-complexity = 18
+select = B,C,E,F,W,T4,B9
+exclude = build,__init__.py

datagen/detectron2/.github/CODE_OF_CONDUCT.md

@@ -0,0 +1,5 @@
+# Code of Conduct
+
+Facebook has adopted a Code of Conduct that we expect project participants to adhere to.
+Please read the [full text](https://code.fb.com/codeofconduct/)
+so that you can understand what actions will and will not be tolerated.

datagen/detectron2/.github/CONTRIBUTING.md

@@ -0,0 +1,52 @@
+# Contributing to detectron2
+We want to make contributing to this project as easy and transparent as
+possible.
+
+## Issues
+We use GitHub issues to track public bugs and questions.
+Please make sure to follow one of the
+[issue templates](https://github.com/facebookresearch/detectron2/issues/new/choose)
+when reporting any issues.
+
+Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
+disclosure of security bugs. In those cases, please go through the process
+outlined on that page and do not file a public issue.
+
+## Pull Requests
+We actively welcome your pull requests.
+
+However, if you're adding any significant features, please
+make sure to have a corresponding issue to discuss your motivation and proposals,
+before sending a PR. We do not always accept new features, and we take the following
+factors into consideration:
+
+1. Whether the same feature can be achieved without modifying detectron2.
+Detectron2 is designed so that you can implement many extensions from the outside, e.g.
+those in [projects](https://github.com/facebookresearch/detectron2/tree/master/projects).
+If some part is not as extensible, you can also bring up the issue to make it more extensible.
+2. Whether the feature is potentially useful to a large audience, or only to a small portion of users.
+3. Whether the proposed solution has a good design / interface.
+4. Whether the proposed solution adds extra mental/practical overhead to users who don't
+   need such feature.
+5. Whether the proposed solution breaks existing APIs.
+
+When sending a PR, please do:
+
+1. If a PR contains multiple orthogonal changes, split it to several PRs.
+2. If you've added code that should be tested, add tests.
+3. For PRs that need experiments (e.g. adding a new model), you don't need to update model zoo,
+   but do provide experiment results in the description of the PR.
+4. If APIs are changed, update the documentation.
+5. Ensure the test suite passes.
+6. Make sure your code lints with `./dev/linter.sh`.
+
+
+## Contributor License Agreement ("CLA")
+In order to accept your pull request, we need you to submit a CLA. You only need
+to do this once to work on any of Facebook's open source projects.
+
+Complete your CLA here: <https://code.facebook.com/cla>
+
+## License
+By contributing to detectron2, you agree that your contributions will be licensed
+under the LICENSE file in the root directory of this source tree.

datagen/detectron2/.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,5 @@
+
+Please select an issue template from
+https://github.com/facebookresearch/detectron2/issues/new/choose .
+
+Otherwise your issue will be closed.

datagen/detectron2/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

datagen/detectron2/.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F680Feature Request"
+about: Submit a proposal/request for a new detectron2 feature
+
+---
+
+## 🚀 Feature
+A clear and concise description of the feature proposal.
+
+
+## Motivation & Examples
+
+Tell us why the feature is useful.
+
+Describe what the feature would look like, if it is implemented.
+Best demonstrated using **code examples** in addition to words.
+
+## Note
+
+We only consider adding new features if they are relevant to many users.
+
+If you request implementation of research papers --
+we only consider papers that have enough significance and prevalance.
+
+We do not take requests for most projects in the `projects/` directory,
+because they are research code release that is mainly for other researchers to reproduce results.
+
+Instead of adding features inside detectron2,
+you can implement many features by [extending detectron2](https://detectron2.readthedocs.io/tutorials/extend.html).
+The [projects/](https://github.com/facebookresearch/detectron2/tree/master/projects/) directory
+contains many of such examples.
+

datagen/detectron2/.github/ISSUE_TEMPLATE/questions-help-support.md

@@ -0,0 +1,21 @@
+---
+name: "❓How to do something?"
+about: How to do X with detectron2? How detectron2 does X?
+
+---
+
+## ❓ How to use Detectron2
+
+Questions like:
+
+1. How to do X with detectron2?
+2. How detectron2 does X?
+
+NOTE:
+
+1. If you met any unexpected issue when using detectron2 and wish to know why,
+   please use the "Unexpected Problems / Bugs" issue template.
+
+2. We do not answer general machine learning / computer vision questions that are not specific to
+	 detectron2, such as how a model works, how to improve your training/make it converge, or what algorithm/methods can be
+	 used to achieve X.

datagen/detectron2/.github/ISSUE_TEMPLATE/unexpected-problems-bugs.md

@@ -0,0 +1,45 @@
+---
+name: "Unexpected behaviors / Bugs"
+about: Report unexpected behaviors or bugs in detectron2
+title: Please read & provide the following
+
+---
+
+If you do not know the root cause of the problem / bug, and wish someone to help you, please
+post according to this template:
+
+## Instructions To Reproduce the Issue:
+
+1. what changes you made (`git diff`) or what code you wrote
+```
+<put diff or code here>
+```
+2. what exact command you run:
+3. what you observed (including __full logs__):
+```
+<put logs here>
+```
+4. please also simplify the steps as much as possible so they do not require additional resources to
+	 run, such as a private dataset.
+
+## Expected behavior:
+
+If there are no obvious error in "what you observed" provided above,
+please tell us the expected behavior.
+
+If you expect the model to converge / work better, note that we do not give suggestions
+on how to train a new model.
+Only in one of the two conditions we will help with it:
+(1) You're unable to reproduce the results in detectron2 model zoo.
+(2) It indicates a detectron2 bug.
+
+## Environment:
+
+Provide your environment information using the following command:
+```
+wget -nc -q https://github.com/facebookresearch/detectron2/raw/master/detectron2/utils/collect_env.py && python collect_env.py
+```
+
+If your issue looks like an installation issue / environment issue,
+please first try to solve it yourself with the instructions in
+https://github.com/facebookresearch/detectron2/blob/master/INSTALL.md#common-installation-issues

datagen/detectron2/.github/pull_request_template.md

@@ -0,0 +1,8 @@
+Thanks for your contribution!
+
+If you're sending a large PR (e.g., >50 lines),
+please open an issue first about the feature / bug, and indicate how you want to contribute.
+See more at https://detectron2.readthedocs.io/notes/contributing.html#pull-requests
+about how we handle PRs.
+
+Before submitting a PR, please run `dev/linter.sh` to lint the code.