Spaces:
Runtime error
Runtime error
Upload config.py
Browse files
config.py
ADDED
@@ -0,0 +1,487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import argparse
|
3 |
+
|
4 |
+
###################################### configuration ######################################
|
5 |
+
class Config(object):
|
6 |
+
|
7 |
+
typeFilters = [[], ["1_query_size_",
|
8 |
+
"1_query_material_",
|
9 |
+
"2_equal_color_",
|
10 |
+
"2_equal_shape_"],
|
11 |
+
["1_query_color_",
|
12 |
+
"1_query_shape_",
|
13 |
+
"2_equal_size_",
|
14 |
+
"2_equal_material_"]]
|
15 |
+
|
16 |
+
#### files interface
|
17 |
+
## data files
|
18 |
+
dataPath = "" # dataset specific
|
19 |
+
datasetFilename = "" # dataset specific
|
20 |
+
|
21 |
+
# file names
|
22 |
+
imagesFilename = "{tier}.h5" # Images
|
23 |
+
instancesFilename = "{tier}Instances.json"
|
24 |
+
# symbols dictionaries
|
25 |
+
questionDictFilename = "questionDict.pkl"
|
26 |
+
answerDictFilename = "answerDict.pkl"
|
27 |
+
qaDictFilename = "qaDict.pkl"
|
28 |
+
|
29 |
+
## experiment files
|
30 |
+
expPathname = "{expName}"
|
31 |
+
expName = "" # will be assigned through argparse
|
32 |
+
|
33 |
+
weightsPath = "./weights"
|
34 |
+
weightsFilename = "weights{epoch}.ckpt"
|
35 |
+
|
36 |
+
# model predictions and optionally attention maps
|
37 |
+
predsPath = "./preds"
|
38 |
+
predsFilename = "{tier}Predictions-{expName}.json"
|
39 |
+
answersFilename = "{tier}Answers-{expName}.txt"
|
40 |
+
|
41 |
+
# logging of accuracy, loss etc. per epoch
|
42 |
+
logPath = "./results"
|
43 |
+
logFilename = "results-{expName}.csv"
|
44 |
+
|
45 |
+
# configuration file of the used flags to run the experiment
|
46 |
+
configPath = "./results"
|
47 |
+
configFilename = "config-{expName}.json"
|
48 |
+
|
49 |
+
def toString(self):
|
50 |
+
return self.expName
|
51 |
+
|
52 |
+
# make directories of experiment if not exist yet
|
53 |
+
def makedirs(self, directory):
|
54 |
+
directory = os.path.join(directory, self.expPath())
|
55 |
+
if not os.path.exists(directory):
|
56 |
+
os.makedirs(directory)
|
57 |
+
return directory
|
58 |
+
|
59 |
+
### filename builders
|
60 |
+
## data files
|
61 |
+
def dataFile(self, filename):
|
62 |
+
return os.path.join(self.dataPath, filename)
|
63 |
+
|
64 |
+
def generatedFile(self, filename):
|
65 |
+
return self.dataFile(self.generatedPrefix + filename)
|
66 |
+
|
67 |
+
datasetFile = lambda self, tier: self.dataFile(self.datasetFilename.format(tier = tier))
|
68 |
+
imagesIdsFile = lambda self, tier: self.dataFile(self.imgIdsFilename.format(tier = tier)) #
|
69 |
+
imagesFile = lambda self, tier: self.dataFile(self.imagesFilename.format(tier = tier))
|
70 |
+
instancesFile = lambda self, tier: self.generatedFile(self.instancesFilename.format(tier = tier))
|
71 |
+
|
72 |
+
questionDictFile = lambda self: self.generatedFile(self.questionDictFilename)
|
73 |
+
answerDictFile = lambda self: self.generatedFile(self.answerDictFilename)
|
74 |
+
qaDictFile = lambda self: self.generatedFile(self.qaDictFilename)
|
75 |
+
|
76 |
+
## experiment files
|
77 |
+
expPath = lambda self: self.expPathname.format(expName = self.toString())
|
78 |
+
|
79 |
+
weightsDir = lambda self: self.makedirs(self.weightsPath)
|
80 |
+
predsDir = lambda self: self.makedirs(self.predsPath)
|
81 |
+
logDir = lambda self: self.makedirs(self.logPath)
|
82 |
+
configDir = lambda self: self.makedirs(self.configPath)
|
83 |
+
|
84 |
+
weightsFile = lambda self, epoch: os.path.join(self.weightsDir(), self.weightsFilename.format(epoch = str(epoch)))
|
85 |
+
predsFile = lambda self, tier: os.path.join(self.predsDir(), self.predsFilename.format(tier = tier, expName = self.expName))
|
86 |
+
answersFile = lambda self, tier: os.path.join(self.predsDir(), self.answersFilename.format(tier = tier, expName = self.expName))
|
87 |
+
logFile = lambda self: os.path.join(self.logDir(), self.logFilename.format(expName = self.expName))
|
88 |
+
configFile = lambda self: os.path.join(self.configDir(), self.configFilename.format(expName = self.expName))
|
89 |
+
|
90 |
+
|
91 |
+
# global configuration variable. Holds file paths and program parameters
|
92 |
+
config = Config()
|
93 |
+
|
94 |
+
###################################### arguments ######################################
|
95 |
+
def parseArgs():
|
96 |
+
parser = argparse.ArgumentParser(fromfile_prefix_chars = "@")
|
97 |
+
|
98 |
+
################ systems
|
99 |
+
|
100 |
+
#custom args
|
101 |
+
parser.add_argument('--train_image_length', default=500, type=int, )
|
102 |
+
parser.add_argument('--test_image_length', default=100, type=int, )
|
103 |
+
parser.add_argument('--val_image_length', default=50, type=int, )
|
104 |
+
|
105 |
+
# gpus and memory
|
106 |
+
parser.add_argument("--gpus", default = "", type = str, help = "comma-separated list of gpus to use")
|
107 |
+
parser.add_argument("--gpusNum", default = 1, type = int, help = "number of gpus to use")
|
108 |
+
|
109 |
+
parser.add_argument("--allowGrowth", action = "store_true", help = "allow gpu memory growth")
|
110 |
+
parser.add_argument("--maxMemory", default = 1.0, type = float, help = "set maximum gpu memory usage")
|
111 |
+
|
112 |
+
parser.add_argument("--parallel", action = "store_true", help = "load images in parallel to batch running")
|
113 |
+
parser.add_argument("--workers", default = 1, type = int, help = "number of workers to load images")
|
114 |
+
parser.add_argument("--taskSize", default = 8, type = int, help = "number of image batches to load in advance") # 40
|
115 |
+
# parser.add_argument("--tasksNum", default = 20, type = int, help = "maximal queue size for tasks (to constrain ram usage)") # 2
|
116 |
+
|
117 |
+
parser.add_argument("--useCPU", action = "store_true", help = "put word embeddings on cpu")
|
118 |
+
|
119 |
+
# weight loading and training
|
120 |
+
parser.add_argument("-r", "--restore", action = "store_true", help = "restore last epoch (based on results file)")
|
121 |
+
parser.add_argument("--restoreEpoch", default = 0, type = int, help = "if positive, specific epoch to restore")
|
122 |
+
parser.add_argument("--weightsToKeep", default = 2, type = int, help = "number of previous epochs' weights keep")
|
123 |
+
parser.add_argument("--saveEvery", default = 3000, type = int, help = "number of iterations to save weights after")
|
124 |
+
parser.add_argument("--calleEvery", default = 1500, type = int, help = "number of iterations to call custom function after")
|
125 |
+
|
126 |
+
parser.add_argument("--saveSubset", action = "store_true", help = "save only subset of the weights")
|
127 |
+
parser.add_argument("--trainSubset", action = "store_true", help = "train only subset of the weights")
|
128 |
+
parser.add_argument("--varSubset", default = [], nargs = "*", type = str, help = "list of namespaces to train on")
|
129 |
+
|
130 |
+
# trainReader = ["questionEmbeddings", "questionReader"]
|
131 |
+
# saveControl = ["questionEmbeddings", "programEmbeddings", "seqReader", "programControl"]
|
132 |
+
|
133 |
+
# experiment files
|
134 |
+
parser.add_argument("--expName", default = "PDF_exp_extra", type = str, help = "experiment name")
|
135 |
+
|
136 |
+
# data files
|
137 |
+
parser.add_argument("--dataset", default = "PDF", choices = ["PDF", "CLEVR", "NLVR"], type = str) #
|
138 |
+
parser.add_argument("--dataBasedir", default = "./", type = str, help = "data base directory") # /jagupard14/scr1/dorarad/
|
139 |
+
parser.add_argument("--generatedPrefix", default = "gennew", type = str, help = "prefix for generated data files")
|
140 |
+
parser.add_argument("--featureType", default = "norm_128x32", type = str, help = "features type") #
|
141 |
+
# resnet101_512x128, norm_400x100, none_80x20, normPerImage_80x20, norm_80x20
|
142 |
+
|
143 |
+
################ optimization
|
144 |
+
|
145 |
+
# training/testing
|
146 |
+
parser.add_argument("--train", action = "store_true", help = "run training")
|
147 |
+
parser.add_argument("--evalTrain", action = "store_true", help = "run eval with ema on train dataset") #
|
148 |
+
parser.add_argument("--test", action = "store_true", help = "run testing every epoch and generate predictions file") #
|
149 |
+
parser.add_argument("--finalTest", action = "store_true", help = "run testing on final epoch")
|
150 |
+
parser.add_argument("--retainVal", action = "store_true", help = "retain validation order between runs") #
|
151 |
+
|
152 |
+
parser.add_argument("--getPreds", action = "store_true", help = "store prediction")
|
153 |
+
parser.add_argument("--getAtt", action = "store_true", help = "store attention maps")
|
154 |
+
parser.add_argument("--analysisType", default = "", type = str, choices = ["", "questionLength, programLength","type", "arity"], help = "show breakdown of results according to type") #
|
155 |
+
|
156 |
+
parser.add_argument("--trainedNum", default = 0, type = int, help = "if positive, train on subset of the data")
|
157 |
+
parser.add_argument("--testedNum", default = 0, type = int, help = "if positive, test on subset of the data")
|
158 |
+
|
159 |
+
# bucketing
|
160 |
+
parser.add_argument("--noBucket", action = "store_true", help = "bucket data according to question length")
|
161 |
+
parser.add_argument("--noRebucket", action = "store_true", help = "bucket data according to question and program length") #
|
162 |
+
|
163 |
+
# filtering
|
164 |
+
parser.add_argument("--tOnlyChain", action = "store_true", help = "train only chain questions")
|
165 |
+
parser.add_argument("--vOnlyChain", action = "store_true", help = "test only chain questions")
|
166 |
+
parser.add_argument("--tMaxQ", default = 0, type = int, help = "if positive, train on questions up to this length")
|
167 |
+
parser.add_argument("--tMaxP", default = 0, type = int, help = "if positive, test on questions up to this length")
|
168 |
+
parser.add_argument("--vMaxQ", default = 0, type = int, help = "if positive, train on questions with programs up to this length")
|
169 |
+
parser.add_argument("--vMaxP", default = 0, type = int, help = "if positive, test on questions with programs up to this length")
|
170 |
+
parser.add_argument("--tFilterOp", default = 0, type = int, help = "train questions by to be included in the types listed")
|
171 |
+
parser.add_argument("--vFilterOp", default = 0, type = int, help = "test questions by to be included in the types listed")
|
172 |
+
|
173 |
+
# extra and extraVal
|
174 |
+
parser.add_argument("--extra", action = "store_true", help = "prepare extra data (add to vocabulary") #
|
175 |
+
parser.add_argument("--trainExtra", action = "store_true", help = "train (only) on extra data") #
|
176 |
+
parser.add_argument("--alterExtra", action = "store_true", help = "alter main data training with extra dataset") #
|
177 |
+
parser.add_argument("--alterNum", default = 1, type = int, help = "alteration rate") #
|
178 |
+
parser.add_argument("--extraVal", action = "store_true", help = "only extra validation data (for compositional clevr)") #
|
179 |
+
parser.add_argument("--finetuneNum", default = 0, type = int, help = "if positive, finetune on that subset of val (for compositional clevr)") #
|
180 |
+
|
181 |
+
# exponential moving average
|
182 |
+
parser.add_argument("--useEMA", action = "store_true", help = "use exponential moving average for weights")
|
183 |
+
parser.add_argument("--emaDecayRate", default = 0.999, type = float, help = "decay rate for exponential moving average")
|
184 |
+
|
185 |
+
# sgd optimizer
|
186 |
+
parser.add_argument("--batchSize", default = 64, type = int, help = "batch size")
|
187 |
+
parser.add_argument("--epochs", default = 100, type = int, help = "number of epochs to run")
|
188 |
+
parser.add_argument("--lr", default = 0.0001, type = float, help = "learning rate")
|
189 |
+
parser.add_argument("--lrReduce", action = "store_true", help = "reduce learning rate if training loss doesn't go down (manual annealing)")
|
190 |
+
parser.add_argument("--lrDecayRate", default = 0.5, type = float, help = "learning decay rate if training loss doesn't go down")
|
191 |
+
parser.add_argument("--earlyStopping", default = 0, type = int, help = "if positive, stop if no improvement for that number of epochs")
|
192 |
+
|
193 |
+
parser.add_argument("--adam", action = "store_true", help = "use adam")
|
194 |
+
parser.add_argument("--l2", default = 0, type = float, help = "if positive, add l2 loss term")
|
195 |
+
parser.add_argument("--clipGradients", action = "store_true", help = "clip gradients")
|
196 |
+
parser.add_argument("--gradMaxNorm", default = 8, type = int, help = "clipping value")
|
197 |
+
|
198 |
+
# batch normalization
|
199 |
+
parser.add_argument("--memoryBN", action = "store_true", help = "use batch normalization on the recurrent memory")
|
200 |
+
parser.add_argument("--stemBN", action = "store_true", help = "use batch normalization in the image input unit (stem)")
|
201 |
+
parser.add_argument("--outputBN", action = "store_true", help = "use batch normalization in the output unit")
|
202 |
+
parser.add_argument("--bnDecay", default = 0.999, type = float, help = "batch norm decay rate")
|
203 |
+
parser.add_argument("--bnCenter", action = "store_true", help = "batch norm with centering")
|
204 |
+
parser.add_argument("--bnScale", action = "store_true", help = "batch norm with scaling")
|
205 |
+
|
206 |
+
## dropouts
|
207 |
+
parser.add_argument("--encInputDropout", default = 0.85, type = float, help = "dropout of the rnn inputs to the Question Input Unit")
|
208 |
+
parser.add_argument("--encStateDropout", default = 1.0, type = float, help = "dropout of the rnn states of the Question Input Unit")
|
209 |
+
parser.add_argument("--stemDropout", default = 0.82, type = float, help = "dropout of the Image Input Unit (the stem)")
|
210 |
+
|
211 |
+
parser.add_argument("--qDropout", default = 0.92, type = float, help = "dropout on the question vector")
|
212 |
+
# parser.add_argument("--qDropoutOut", default = 1.0, type = float, help = "dropout on the question vector the goes to the output unit")
|
213 |
+
# parser.add_argument("--qDropoutMAC", default = 1.0, type = float, help = "dropout on the question vector the goes to MAC")
|
214 |
+
|
215 |
+
parser.add_argument("--memoryDropout", default = 0.85, type = float, help = "dropout on the recurrent memory")
|
216 |
+
parser.add_argument("--readDropout", default = 0.85, type = float, help = "dropout of the read unit")
|
217 |
+
parser.add_argument("--writeDropout", default = 1.0, type = float, help = "dropout of the write unit")
|
218 |
+
parser.add_argument("--outputDropout", default = 0.85, type = float, help = "dropout of the output unit")
|
219 |
+
|
220 |
+
parser.add_argument("--parametricDropout", action = "store_true", help = "use parametric dropout") #
|
221 |
+
parser.add_argument("--encVariationalDropout", action = "store_true", help = "use variational dropout in the RNN input unit")
|
222 |
+
parser.add_argument("--memoryVariationalDropout", action = "store_true", help = "use variational dropout across the MAC network")
|
223 |
+
|
224 |
+
## nonlinearities
|
225 |
+
parser.add_argument("--relu", default = "STD", choices = ["STD", "PRM", "ELU", "LKY", "SELU"], type = str, help = "type of ReLU to use: standard, parametric, ELU, or leaky")
|
226 |
+
# parser.add_argument("--reluAlpha", default = 0.2, type = float, help = "alpha value for the leaky ReLU")
|
227 |
+
|
228 |
+
parser.add_argument("--mulBias", default = 0.0, type = float, help = "bias to add in multiplications (x + b) * (y + b) for better training") #
|
229 |
+
|
230 |
+
parser.add_argument("--imageLinPool", default = 2, type = int, help = "pooling for image linearizion")
|
231 |
+
|
232 |
+
################ baseline model parameters
|
233 |
+
|
234 |
+
parser.add_argument("--useBaseline", action = "store_true", help = "run the baseline model")
|
235 |
+
parser.add_argument("--baselineLSTM", action = "store_true", help = "use LSTM in baseline")
|
236 |
+
parser.add_argument("--baselineCNN", action = "store_true", help = "use CNN in baseline")
|
237 |
+
parser.add_argument("--baselineAtt", action = "store_true", help = "use stacked attention baseline")
|
238 |
+
|
239 |
+
parser.add_argument("--baselineProjDim", default = 64, type = int, help = "projection dimension for image linearizion")
|
240 |
+
|
241 |
+
parser.add_argument("--baselineAttNumLayers", default = 2, type = int, help = "number of stacked attention layers")
|
242 |
+
parser.add_argument("--baselineAttType", default = "ADD", type = str, choices = ["MUL", "DIAG", "BL", "ADD"], help = "attention type (multiplicative, additive, etc)")
|
243 |
+
|
244 |
+
################ image input unit (the "stem")
|
245 |
+
|
246 |
+
parser.add_argument("--stemDim", default = 512, type = int, help = "dimension of stem CNNs")
|
247 |
+
parser.add_argument("--stemNumLayers", default = 2, type = int, help = "number of stem layers")
|
248 |
+
parser.add_argument("--stemKernelSize", default = 3, type = int, help = "kernel size for stem (same for all the stem layers)")
|
249 |
+
parser.add_argument("--stemKernelSizes", default = None, nargs = "*", type = int, help = "kernel sizes for stem (per layer)")
|
250 |
+
parser.add_argument("--stemStrideSizes", default = None, nargs = "*", type = int, help = "stride sizes for stem (per layer)")
|
251 |
+
|
252 |
+
parser.add_argument("--stemLinear", action = "store_true", help = "use a linear stem (instead of CNNs)") #
|
253 |
+
# parser.add_argument("--stemProjDim", default = 64, type = int, help = "projection dimension of in image linearization") #
|
254 |
+
# parser.add_argument("--stemProjPooling", default = 2, type = int, help = "pooling for the image linearization") #
|
255 |
+
|
256 |
+
parser.add_argument("--stemGridRnn", action = "store_true", help = "use grid RNN layer") #
|
257 |
+
parser.add_argument("--stemGridRnnMod", default = "RNN", type = str, choices = ["RNN", "GRU"], help = "RNN type for grid") #
|
258 |
+
parser.add_argument("--stemGridAct", default = "NON", type = str, choices = ["NON", "RELU", "TANH"], help = "nonlinearity type for grid") #
|
259 |
+
|
260 |
+
## location
|
261 |
+
parser.add_argument("--locationAware", action = "store_true", help = "add positional features to image representation (linear meshgrid by default)")
|
262 |
+
parser.add_argument("--locationType", default = "L", type = str, choices = ["L", "PE"], help = "L: linear features, PE: Positional Encoding")
|
263 |
+
parser.add_argument("--locationBias", default = 1.0, type = float, help = "the scale of the positional features")
|
264 |
+
parser.add_argument("--locationDim", default = 32, type = int, help = "the number of PE dimensions")
|
265 |
+
|
266 |
+
################ question input unit (the "encoder")
|
267 |
+
parser.add_argument("--encType", default = "LSTM", choices = ["RNN", "GRU", "LSTM", "MiGRU", "MiLSTM"], help = "encoder RNN type")
|
268 |
+
parser.add_argument("--encDim", default = 512, type = int, help = "dimension of encoder RNN")
|
269 |
+
parser.add_argument("--encNumLayers", default = 1, type = int, help = "number of encoder RNN layers")
|
270 |
+
parser.add_argument("--encBi", action = "store_true", help = "use bi-directional encoder")
|
271 |
+
# parser.add_argument("--encOutProj", action = "store_true", help = "add projection layer for encoder outputs")
|
272 |
+
# parser.add_argument("--encOutProjDim", default = 256, type = int, help = "dimension of the encoder projection layer")
|
273 |
+
# parser.add_argument("--encQProj", action = "store_true", help = "add projection for the question representation")
|
274 |
+
parser.add_argument("--encProj", action = "store_true", help = "project encoder outputs and question")
|
275 |
+
parser.add_argument("--encProjQAct", default = "NON", type = str, choices = ["NON", "RELU", "TANH"], help = "project question vector with this activation")
|
276 |
+
|
277 |
+
##### word embeddings
|
278 |
+
parser.add_argument("--wrdEmbDim", default = 300, type = int, help = "word embeddings dimension")
|
279 |
+
parser.add_argument("--wrdEmbRandom", action = "store_true", help = "initialize word embeddings to random (normal)")
|
280 |
+
parser.add_argument("--wrdEmbUniform", action = "store_true", help = "initialize with uniform distribution")
|
281 |
+
parser.add_argument("--wrdEmbScale", default = 1.0, type = float, help = "word embeddings initialization scale")
|
282 |
+
parser.add_argument("--wrdEmbFixed", action = "store_true", help = "set word embeddings fixed (don't train)")
|
283 |
+
parser.add_argument("--wrdEmbUnknown", action = "store_true", help = "set words outside of training set to <UNK>")
|
284 |
+
|
285 |
+
parser.add_argument("--ansEmbMod", default = "NON", choices = ["NON", "SHARED", "BOTH"], type = str, help = "BOTH: create word embeddings for answers. SHARED: share them with question embeddings.") #
|
286 |
+
parser.add_argument("--answerMod", default = "NON", choices = ["NON", "MUL", "DIAG", "BL"], type = str, help = "operation for multiplication with answer embeddings: direct multiplication, scalar weighting, or bilinear") #
|
287 |
+
|
288 |
+
################ output unit (classifier)
|
289 |
+
parser.add_argument("--outClassifierDims", default = [512], nargs = "*", type = int, help = "dimensions of the classifier")
|
290 |
+
parser.add_argument("--outImage", action = "store_true", help = "feed the image to the output unit")
|
291 |
+
parser.add_argument("--outImageDim", default = 1024, type = int, help = "dimension of linearized image fed to the output unit")
|
292 |
+
parser.add_argument("--outQuestion", action = "store_true", help = "feed the question to the output unit")
|
293 |
+
parser.add_argument("--outQuestionMul", action = "store_true", help = "feed the multiplication of question and memory to the output unit")
|
294 |
+
|
295 |
+
################ network
|
296 |
+
|
297 |
+
parser.add_argument("--netLength", default = 16, type = int, help = "network length (number of cells)")
|
298 |
+
# parser.add_argument("--netDim", default = 512, type = int)
|
299 |
+
parser.add_argument("--memDim", default = 512, type = int, help = "dimension of memory state")
|
300 |
+
parser.add_argument("--ctrlDim", default = 512, type = int, help = "dimension of control state")
|
301 |
+
parser.add_argument("--attDim", default = 512, type = int, help = "dimension of pre-attention interactions space")
|
302 |
+
parser.add_argument("--unsharedCells", default = False, type = bool, help = "unshare weights between cells ")
|
303 |
+
|
304 |
+
# initialization
|
305 |
+
parser.add_argument("--initCtrl", default = "PRM", type = str, choices = ["PRM", "ZERO", "Q"], help = "initialization mod for control")
|
306 |
+
parser.add_argument("--initMem", default = "PRM", type = str, choices = ["PRM", "ZERO", "Q"], help = "initialization mod for memory")
|
307 |
+
parser.add_argument("--initKBwithQ", default = "NON", type = str, choices = ["NON", "CNCT", "MUL"], help = "merge question with knowledge base")
|
308 |
+
parser.add_argument("--addNullWord", action = "store_true", help = "add parametric word in the beginning of the question")
|
309 |
+
|
310 |
+
################ control unit
|
311 |
+
# control ablations (use whole question or pre-attention continuous vectors as control)
|
312 |
+
parser.add_argument("--controlWholeQ", action = "store_true", help = "use whole question vector as control")
|
313 |
+
parser.add_argument("--controlContinuous", action = "store_true", help = "use continuous representation of control (without attention)")
|
314 |
+
|
315 |
+
# step 0: inputs to control unit (word embeddings or encoder outputs, with optional projection)
|
316 |
+
parser.add_argument("--controlContextual", action = "store_true", help = "use contextual words for attention (otherwise will use word embeddings)")
|
317 |
+
parser.add_argument("--controlInWordsProj", action = "store_true", help = "apply linear projection over words for attention computation")
|
318 |
+
parser.add_argument("--controlOutWordsProj", action = "store_true", help = "apply linear projection over words for summary computation")
|
319 |
+
|
320 |
+
parser.add_argument("--controlInputUnshared", action = "store_true", help = "use different question representation for each cell")
|
321 |
+
parser.add_argument("--controlInputAct", default = "TANH", type = str, choices = ["NON", "RELU", "TANH"], help = "activation for question projection")
|
322 |
+
|
323 |
+
# step 1: merging previous control and whole question
|
324 |
+
parser.add_argument("--controlFeedPrev", action = "store_true", help = "feed previous control state")
|
325 |
+
parser.add_argument("--controlFeedPrevAtt", action = "store_true", help = "feed previous control post word attention (otherwise will feed continuous control)")
|
326 |
+
parser.add_argument("--controlFeedInputs", action = "store_true", help = "feed question representation")
|
327 |
+
parser.add_argument("--controlContAct", default = "NON", type = str, choices = ["NON", "RELU", "TANH"], help = "activation on the words interactions")
|
328 |
+
|
329 |
+
# step 2: word attention and optional projection
|
330 |
+
parser.add_argument("--controlConcatWords", action = "store_true", help = "concatenate words to interaction when computing attention")
|
331 |
+
parser.add_argument("--controlProj", action = "store_true", help = "apply linear projection on words interactions")
|
332 |
+
parser.add_argument("--controlProjAct", default = "NON", type = str, choices = ["NON", "RELU", "TANH"], help = "activation for control interactions")
|
333 |
+
|
334 |
+
# parser.add_argument("--controlSelfAtt", default = False, type = bool)
|
335 |
+
|
336 |
+
# parser.add_argument("--controlCoverage", default = False, type = bool)
|
337 |
+
# parser.add_argument("--controlCoverageBias", default = 1.0, type = float)
|
338 |
+
|
339 |
+
# parser.add_argument("--controlPostRNN", default = False, type = bool)
|
340 |
+
# parser.add_argument("--controlPostRNNmod", default = "RNN", type = str) # GRU
|
341 |
+
|
342 |
+
# parser.add_argument("--selfAttShareInter", default = False, type = bool)
|
343 |
+
|
344 |
+
# parser.add_argument("--wordControl", default = False, type = bool)
|
345 |
+
# parser.add_argument("--gradualControl", default = False, type = bool)
|
346 |
+
|
347 |
+
################ read unit
|
348 |
+
# step 1: KB-memory interactions
|
349 |
+
parser.add_argument("--readProjInputs", action = "store_true", help = "project read unit inputs")
|
350 |
+
parser.add_argument("--readProjShared", action = "store_true", help = "use shared projection for all read unit inputs")
|
351 |
+
|
352 |
+
parser.add_argument("--readMemAttType", default = "MUL", type = str, choices = ["MUL", "DIAG", "BL", "ADD"], help = "attention type for interaction with memory")
|
353 |
+
parser.add_argument("--readMemConcatKB", action = "store_true", help = "concatenate KB elements to memory interaction")
|
354 |
+
parser.add_argument("--readMemConcatProj", action = "store_true", help = "concatenate projected values instead or original to memory interaction")
|
355 |
+
parser.add_argument("--readMemProj", action = "store_true", help = "project interactions with memory")
|
356 |
+
parser.add_argument("--readMemAct", default = "RELU", type = str, choices = ["NON", "RELU", "TANH"], help = "activation for memory interaction")
|
357 |
+
|
358 |
+
# step 2: interaction with control
|
359 |
+
parser.add_argument("--readCtrl", action = "store_true", help = "compare KB-memory interactions to control")
|
360 |
+
parser.add_argument("--readCtrlAttType", default = "MUL", type = str, choices = ["MUL", "DIAG", "BL", "ADD"], help = "attention type for interaction with control")
|
361 |
+
parser.add_argument("--readCtrlConcatKB", action = "store_true", help = "concatenate KB elements to control interaction")
|
362 |
+
parser.add_argument("--readCtrlConcatProj", action = "store_true", help = "concatenate projected values instead or original to control interaction")
|
363 |
+
parser.add_argument("--readCtrlConcatInter", action = "store_true", help = "concatenate memory interactions to control interactions")
|
364 |
+
parser.add_argument("--readCtrlAct", default = "RELU", type = str, choices = ["NON", "RELU", "TANH"], help = "activation for control interaction")
|
365 |
+
|
366 |
+
# step 3: summarize attention over knowledge base
|
367 |
+
parser.add_argument("--readSmryKBProj", action = "store_true", help = "use knowledge base projections when summing attention up (should be used only if KB is projected.")
|
368 |
+
|
369 |
+
# parser.add_argument("--saAllMultiplicative", default = False, type = bool)
|
370 |
+
# parser.add_argument("--saSumMultiplicative", default = False, type = bool)
|
371 |
+
|
372 |
+
################ write unit
|
373 |
+
# step 1: input to the write unit (only previous memory, or new information, or both)
|
374 |
+
parser.add_argument("--writeInputs", default = "BOTH", type = str, choices = ["MEM", "INFO", "BOTH", "SUM"], help = "inputs to the write unit")
|
375 |
+
parser.add_argument("--writeConcatMul", action = "store_true", help = "add multiplicative integration between inputs")
|
376 |
+
|
377 |
+
parser.add_argument("--writeInfoProj", action = "store_true", help = "project retrieved info")
|
378 |
+
parser.add_argument("--writeInfoAct", default = "NON", type = str, choices = ["NON", "RELU", "TANH"], help = "new info activation")
|
379 |
+
|
380 |
+
# step 2: self attention and following projection
|
381 |
+
parser.add_argument("--writeSelfAtt", action = "store_true", help = "use self attention")
|
382 |
+
parser.add_argument("--writeSelfAttMod", default = "NON", type = str, choices = ["NON", "CONT"], help = "control version to compare to")
|
383 |
+
|
384 |
+
parser.add_argument("--writeMergeCtrl", action = "store_true", help = "merge control with memory")
|
385 |
+
|
386 |
+
parser.add_argument("--writeMemProj", action = "store_true", help = "project new memory")
|
387 |
+
parser.add_argument("--writeMemAct", default = "NON", type = str, choices = ["NON", "RELU", "TANH"], help = "new memory activation")
|
388 |
+
|
389 |
+
# step 3: gate between new memory and previous value
|
390 |
+
parser.add_argument("--writeGate", action = "store_true", help = "add gate to write unit")
|
391 |
+
parser.add_argument("--writeGateShared", action = "store_true", help = "use one gate value for all dimensions of the memory state")
|
392 |
+
parser.add_argument("--writeGateBias", default = 1.0, type = float, help = "bias for the write unit gate (positive to bias for taking new memory)")
|
393 |
+
|
394 |
+
## modular
|
395 |
+
# parser.add_argument("--modulesNum", default = 10, type = int)
|
396 |
+
# parser.add_argument("--controlBoth", default = False, type = bool)
|
397 |
+
# parser.add_argument("--addZeroModule", default = False, type = bool)
|
398 |
+
# parser.add_argument("--endModule", default = False, type = bool)
|
399 |
+
|
400 |
+
## hybrid
|
401 |
+
# parser.add_argument("--hybrid", default = False, type = bool, help = "hybrid attention cnn model")
|
402 |
+
# parser.add_argument("--earlyHybrid", default = False, type = bool)
|
403 |
+
# parser.add_argument("--lateHybrid", default = False, type = bool)
|
404 |
+
|
405 |
+
## autoencoders
|
406 |
+
# parser.add_argument("--autoEncMem", action = "store_true", help = "add memory2control auto-encoder loss")
|
407 |
+
# parser.add_argument("--autoEncMemW", default = 0.0001, type = float, help = "weight for auto-encoder loss")
|
408 |
+
# parser.add_argument("--autoEncMemInputs", default = "INFO", type = str, choices = ["MEM", "INFO"], help = "inputs to auto-encoder")
|
409 |
+
# parser.add_argument("--autoEncMemAct", default = "NON", type = str, choices = ["NON", "RELU", "TANH"], help = "activation type in the auto-encoder")
|
410 |
+
# parser.add_argument("--autoEncMemLoss", default = "CONT", type = str, choices = ["CONT", "PROB", "SMRY"], help = "target for the auto-encoder loss")
|
411 |
+
# parser.add_argument("--autoEncMemCnct", action = "store_true", help = "concat word attentions to auto-encoder features")
|
412 |
+
|
413 |
+
# parser.add_argument("--autoEncCtrl", action = "store_true")
|
414 |
+
# parser.add_argument("--autoEncCtrlW", default = 0.0001, type = float)
|
415 |
+
# parser.add_argument("--autoEncCtrlGRU", action = "store_true")
|
416 |
+
|
417 |
+
## temperature
|
418 |
+
# parser.add_argument("--temperature", default = 1.0, type = float, help = "temperature for modules softmax") #
|
419 |
+
# parser.add_argument("--tempParametric", action = "store_true", help = "parametric temperature") #
|
420 |
+
# parser.add_argument("--tempDynamic", action = "store_true", help = "dynamic temperature") #
|
421 |
+
# parser.add_argument("--tempAnnealRate", default = 0.000004, type = float, help = "temperature annealing rate") #
|
422 |
+
# parser.add_argument("--tempMin", default = 0.5, type = float, help = "minimum temperature") #
|
423 |
+
|
424 |
+
## gumbel
|
425 |
+
# parser.add_argument("--gumbelSoftmax", action = "store_true", help = "use gumbel for the module softmax (soft for training and hard for testing)") #
|
426 |
+
# parser.add_argument("--gumbelSoftmaxBoth", action = "store_true", help = "use softmax for training and testing") #
|
427 |
+
# parser.add_argument("--gumbelArgmaxBoth", action = "store_true", help = "use argmax for training and testing") #
|
428 |
+
|
429 |
+
parser.parse_args(namespace = config)
|
430 |
+
|
431 |
+
###################################### dataset configuration ######################################
|
432 |
+
|
433 |
+
def configPDF():
|
434 |
+
config.dataPath = "{dataBasedir}/PDF_v1/data".format(dataBasedir = config.dataBasedir)
|
435 |
+
config.datasetFilename = "PDF_{tier}_questions.json"
|
436 |
+
config.wordVectorsFile = "./PDF_v1/data/glove/glove.6B.{dim}d.txt".format(dim = config.wrdEmbDim) #
|
437 |
+
|
438 |
+
config.imageDims = [14, 14, 1024]
|
439 |
+
config.programLims = [5, 10, 15, 20]
|
440 |
+
config.questionLims = [10, 15, 20, 25]
|
441 |
+
|
442 |
+
def configCLEVR():
|
443 |
+
config.dataPath = "{dataBasedir}/CLEVR_v1/data".format(dataBasedir = config.dataBasedir)
|
444 |
+
config.datasetFilename = "CLEVR_{tier}_questions.json"
|
445 |
+
config.wordVectorsFile = "./CLEVR_v1/data/glove/glove.6B.{dim}d.txt".format(dim = config.wrdEmbDim) #
|
446 |
+
|
447 |
+
config.imageDims = [14, 14, 1024]
|
448 |
+
config.programLims = [5, 10, 15, 20]
|
449 |
+
config.questionLims = [10, 15, 20, 25]
|
450 |
+
|
451 |
+
def configNLVR():
|
452 |
+
config.dataPath = "{dataBasedir}/nlvr".format(dataBasedir = config.dataBasedir)
|
453 |
+
config.datasetFilename = "{tier}.json"
|
454 |
+
config.imagesFilename = "{{tier}}_{featureType}.h5".format(featureType = config.featureType)
|
455 |
+
config.imgIdsFilename = "{tier}ImgIds.json"
|
456 |
+
config.wordVectorsFile = "./CLEVR_v1/data/glove/glove.6B.{dim}d.txt".format(dim = config.wrdEmbDim) #
|
457 |
+
|
458 |
+
config.questionLims = [12]
|
459 |
+
# config.noRebucket = True
|
460 |
+
|
461 |
+
# if config.stemKernelSizes == []:
|
462 |
+
# if config.featureType.endsWith("128x32"):
|
463 |
+
# config.stemKernelSizes = [8, 4, 4]
|
464 |
+
# config.stemStrideSizes = [2, 2, 1]
|
465 |
+
# config.stemNumLayers = 3
|
466 |
+
# if config.featureType.endsWith("512x128"):
|
467 |
+
# config.stemKernelSizes = [8, 4, 4, 2]
|
468 |
+
# config.stemStrideSizes = [4, 2, 2, 1]
|
469 |
+
# config.stemNumLayers = 4
|
470 |
+
# config.stemDim = 64
|
471 |
+
|
472 |
+
if config.featureType == "resnet101_512x128":
|
473 |
+
config.imageDims = [8, 32, 1024]
|
474 |
+
else:
|
475 |
+
stridesOverall = 1
|
476 |
+
if stemStrideSizes is not None:
|
477 |
+
for s in config.stemStrideSizes:
|
478 |
+
stridesOverall *= int(s)
|
479 |
+
size = config.featureType.split("_")[-1].split("x")
|
480 |
+
config.imageDims = [int(size[1]) / stridesOverall, int(size[0]) / stridesOverall, 3]
|
481 |
+
|
482 |
+
## dataset specific configs
|
483 |
+
loadDatasetConfig = {
|
484 |
+
"CLEVR": configCLEVR,
|
485 |
+
"NLVR": configNLVR,
|
486 |
+
"PDF": configPDF
|
487 |
+
}
|