Delete lxmert/src/tasks

lxmert/src/tasks/gqa.py

@@ -1,210 +0,0 @@
-# coding=utf-8
-# Copyleft 2019 project LXRT.
-
-import os
-import collections
-
-import torch
-from tqdm import tqdm
-import torch.nn as nn
-from torch.utils.data.dataloader import DataLoader
-
-from param import args
-from pretrain.qa_answer_table import load_lxmert_qa
-from tasks.gqa_model import GQAModel
-from tasks.gqa_data import GQADataset, GQATorchDataset, GQAEvaluator
-
-
-DataTuple = collections.namedtuple("DataTuple", 'dataset loader evaluator')
-
-
-def get_tuple(splits: str, bs:int, shuffle=False, drop_last=False) -> DataTuple:
-    dset = GQADataset(splits)
-    tset = GQATorchDataset(dset)
-    evaluator = GQAEvaluator(dset)
-    data_loader = DataLoader(
-        tset, batch_size=bs,
-        shuffle=shuffle, num_workers=args.num_workers,
-        drop_last=drop_last, pin_memory=True
-    )
-
-    return DataTuple(dataset=dset, loader=data_loader, evaluator=evaluator)
-
-
-class GQA:
-    def __init__(self):
-        self.train_tuple = get_tuple(
-            args.train, bs=args.batch_size, shuffle=True, drop_last=True
-        )
-        if args.valid != "":
-            valid_bsize = 2048 if args.multiGPU else 512
-            self.valid_tuple = get_tuple(
-                args.valid, bs=valid_bsize,
-                shuffle=False, drop_last=False
-            )
-        else:
-            self.valid_tuple = None
-
-        self.model = GQAModel(self.train_tuple.dataset.num_answers)
-
-        # Load pre-trained weights
-        if args.load_lxmert is not None:
-            self.model.lxrt_encoder.load(args.load_lxmert)
-        if args.load_lxmert_qa is not None:
-            load_lxmert_qa(args.load_lxmert_qa, self.model,
-                           label2ans=self.train_tuple.dataset.label2ans)
-
-        # GPU options
-        self.model = self.model.cuda()
-        if args.multiGPU:
-            self.model.lxrt_encoder.multi_gpu()
-
-        # Losses and optimizer
-        self.bce_loss = nn.BCEWithLogitsLoss()
-        self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1)
-        if 'bert' in args.optim:
-            batch_per_epoch = len(self.train_tuple.loader)
-            t_total = int(batch_per_epoch * args.epochs)
-            print("Total Iters: %d" % t_total)
-            from lxrt.optimization import BertAdam
-            self.optim = BertAdam(list(self.model.parameters()),
-                                  lr=args.lr,
-                                  warmup=0.1,
-                                  t_total=t_total)
-        else:
-            self.optim = args.optimizer(list(self.model.parameters()), args.lr)
-
-        self.output = args.output
-        os.makedirs(self.output, exist_ok=True)
-
-    def train(self, train_tuple, eval_tuple):
-        dset, loader, evaluator = train_tuple
-        iter_wrapper = (lambda x: tqdm(x, total=len(loader))) if args.tqdm else (lambda x: x)
-
-        best_valid = 0.
-        for epoch in range(args.epochs):
-            quesid2ans = {}
-            for i, (ques_id, feats, boxes, sent, target) in iter_wrapper(enumerate(loader)):
-
-                self.model.train()
-                self.optim.zero_grad()
-
-                feats, boxes, target = feats.cuda(), boxes.cuda(), target.cuda()
-                logit = self.model(feats, boxes, sent)
-                assert logit.dim() == target.dim() == 2
-                if args.mce_loss:
-                    max_value, target = target.max(1)
-                    loss = self.mce_loss(logit, target) * logit.size(1)
-                else:
-                    loss = self.bce_loss(logit, target)
-                    loss = loss * logit.size(1)
-
-                loss.backward()
-                nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
-                self.optim.step()
-
-                score, label = logit.max(1)
-                for qid, l in zip(ques_id, label.cpu().numpy()):
-                    ans = dset.label2ans[l]
-                    quesid2ans[qid] = ans
-
-            log_str = "\nEpoch %d: Train %0.2f\n" % (epoch, evaluator.evaluate(quesid2ans) * 100.)
-
-            if self.valid_tuple is not None:  # Do Validation
-                valid_score = self.evaluate(eval_tuple)
-                if valid_score > best_valid:
-                    best_valid = valid_score
-                    self.save("BEST")
-
-                log_str += "Epoch %d: Valid %0.2f\n" % (epoch, valid_score * 100.) + \
-                           "Epoch %d: Best %0.2f\n" % (epoch, best_valid * 100.)
-
-            print(log_str, end='')
-
-            with open(self.output + "/log.log", 'a') as f:
-                f.write(log_str)
-                f.flush()
-
-        self.save("LAST")
-
-    def predict(self, eval_tuple: DataTuple, dump=None):
-        self.model.eval()
-        dset, loader, evaluator = eval_tuple
-        quesid2ans = {}
-        for i, datum_tuple in enumerate(loader):
-            ques_id, feats, boxes, sent = datum_tuple[:4]   # avoid handling target
-            with torch.no_grad():
-                feats, boxes = feats.cuda(), boxes.cuda()
-                logit = self.model(feats, boxes, sent)
-                score, label = logit.max(1)
-                for qid, l in zip(ques_id, label.cpu().numpy()):
-                    ans = dset.label2ans[l]
-                    quesid2ans[qid] = ans
-        if dump is not None:
-            evaluator.dump_result(quesid2ans, dump)
-        return quesid2ans
-
-    def evaluate(self, eval_tuple: DataTuple, dump=None):
-        dset, loader, evaluator = eval_tuple
-        quesid2ans = self.predict(eval_tuple, dump)
-        return evaluator.evaluate(quesid2ans)
-
-    @staticmethod
-    def oracle_score(data_tuple):
-        dset, loader, evaluator = data_tuple
-        quesid2ans = {}
-        for i, (ques_id, feats, boxes, sent, target) in enumerate(loader):
-            _, label = target.max(1)
-            for qid, l in zip(ques_id, label.cpu().numpy()):
-                ans = dset.label2ans[l]
-                quesid2ans[qid] = ans
-        return evaluator.evaluate(quesid2ans)
-
-    def save(self, name):
-        torch.save(self.model.state_dict(),
-                   os.path.join(self.output, "%s.pth" % name))
-
-    def load(self, path):
-        print("Load model from %s" % path)
-        state_dict = torch.load("%s.pth" % path)
-        for key in list(state_dict.keys()):
-            if '.module' in key:
-                state_dict[key.replace('.module', '')] = state_dict.pop(key)
-        self.model.load_state_dict(state_dict, strict=False)
-
-
-if __name__ == "__main__":
-    # Build Class
-    gqa = GQA()
-
-    # Load Model
-    if args.load is not None:
-        gqa.load(args.load)
-
-    # Test or Train
-    if args.test is not None:
-        args.fast = args.tiny = False       # Always loading all data in test
-        if 'submit' in args.test:
-            gqa.predict(
-                get_tuple(args.test, bs=args.batch_size,
-                          shuffle=False, drop_last=False),
-                dump=os.path.join(args.output, 'submit_predict.json')
-            )
-        if 'testdev' in args.test:
-            result = gqa.evaluate(
-                get_tuple('testdev', bs=args.batch_size,
-                          shuffle=False, drop_last=False),
-                dump=os.path.join(args.output, 'testdev_predict.json')
-            )
-            print(result)
-    else:
-        # print("Train Oracle: %0.2f" % (gqa.oracle_score(gqa.train_tuple) * 100))
-        print('Splits in Train data:', gqa.train_tuple.dataset.splits)
-        if gqa.valid_tuple is not None:
-            print('Splits in Valid data:', gqa.valid_tuple.dataset.splits)
-            print("Valid Oracle: %0.2f" % (gqa.oracle_score(gqa.valid_tuple) * 100))
-        else:
-            print("DO NOT USE VALIDATION")
-        gqa.train(gqa.train_tuple, gqa.valid_tuple)
-
-

lxmert/src/tasks/gqa_data.py

@@ -1,194 +0,0 @@
-# coding=utf-8
-# Copyleft 2019 project LXRT.
-
-import json
-
-import numpy as np
-import torch
-from torch.utils.data import Dataset
-
-from param import args
-from utils import load_obj_tsv
-
-# Load part of the dataset for fast checking.
-# Notice that here is the number of images instead of the number of data,
-# which means all related data to the images would be used.
-TINY_IMG_NUM = 512
-FAST_IMG_NUM = 5000
-
-
-class GQADataset:
-    """
-    A GQA data example in json file:
-    {
-        "img_id": "2375429",
-        "label": {
-            "pipe": 1.0
-        },
-        "question_id": "07333408",
-        "sent": "What is on the white wall?"
-    }
-    """
-    def __init__(self, splits: str):
-        self.name = splits
-        self.splits = splits.split(',')
-
-        # Loading datasets to data
-        self.data = []
-        for split in self.splits:
-            self.data.extend(json.load(open("data/gqa/%s.json" % split)))
-        print("Load %d data from split(s) %s." % (len(self.data), self.name))
-
-        # List to dict (for evaluation and others)
-        self.id2datum = {
-            datum['question_id']: datum
-            for datum in self.data
-        }
-
-        # Answers
-        self.ans2label = json.load(open("data/gqa/trainval_ans2label.json"))
-        self.label2ans = json.load(open("data/gqa/trainval_label2ans.json"))
-        assert len(self.ans2label) == len(self.label2ans)
-        for ans, label in self.ans2label.items():
-            assert self.label2ans[label] == ans
-
-    @property
-    def num_answers(self):
-        return len(self.ans2label)
-
-    def __len__(self):
-        return len(self.data)
-
-
-class GQABufferLoader():
-    def __init__(self):
-        self.key2data = {}
-
-    def load_data(self, name, number):
-        if name == 'testdev':
-            path = "data/vg_gqa_imgfeat/gqa_testdev_obj36.tsv"
-        else:
-            path = "data/vg_gqa_imgfeat/vg_gqa_obj36.tsv"
-        key = "%s_%d" % (path, number)
-        if key not in self.key2data:
-            self.key2data[key] = load_obj_tsv(
-                path,
-                topk=number
-            )
-        return self.key2data[key]
-
-
-gqa_buffer_loader = GQABufferLoader()
-
-
-"""
-Example in obj tsv:
-FIELDNAMES = ["img_id", "img_h", "img_w", "objects_id", "objects_conf",
-              "attrs_id", "attrs_conf", "num_boxes", "boxes", "features"]
-"""
-class GQATorchDataset(Dataset):
-    def __init__(self, dataset: GQADataset):
-        super().__init__()
-        self.raw_dataset = dataset
-
-        if args.tiny:
-            topk = TINY_IMG_NUM
-        elif args.fast:
-            topk = FAST_IMG_NUM
-        else:
-            topk = -1
-
-        # Loading detection features to img_data
-        # Since images in train and valid both come from Visual Genome,
-        # buffer the image loading to save memory.
-        img_data = []
-        if 'testdev' in dataset.splits or 'testdev_all' in dataset.splits:     # Always loading all the data in testdev
-            img_data.extend(gqa_buffer_loader.load_data('testdev', -1))
-        else:
-            img_data.extend(gqa_buffer_loader.load_data('train', topk))
-        self.imgid2img = {}
-        for img_datum in img_data:
-            self.imgid2img[img_datum['img_id']] = img_datum
-
-        # Only kept the data with loaded image features
-        self.data = []
-        for datum in self.raw_dataset.data:
-            if datum['img_id'] in self.imgid2img:
-                self.data.append(datum)
-        print("Use %d data in torch dataset" % (len(self.data)))
-        print()
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, item: int):
-        datum = self.data[item]
-
-        img_id = datum['img_id']
-        ques_id = datum['question_id']
-        ques = datum['sent']
-
-        # Get image info
-        img_info = self.imgid2img[img_id]
-        obj_num = img_info['num_boxes']
-        boxes = img_info['boxes'].copy()
-        feats = img_info['features'].copy()
-        assert len(boxes) == len(feats) == obj_num
-
-        # Normalize the boxes (to 0 ~ 1)
-        img_h, img_w = img_info['img_h'], img_info['img_w']
-        boxes = boxes.copy()
-        boxes[:, (0, 2)] /= img_w
-        boxes[:, (1, 3)] /= img_h
-        np.testing.assert_array_less(boxes, 1+1e-5)
-        np.testing.assert_array_less(-boxes, 0+1e-5)
-
-        # Create target
-        if 'label' in datum:
-            label = datum['label']
-            target = torch.zeros(self.raw_dataset.num_answers)
-            for ans, score in label.items():
-                if ans in self.raw_dataset.ans2label:
-                    target[self.raw_dataset.ans2label[ans]] = score
-            return ques_id, feats, boxes, ques, target
-        else:
-            return ques_id, feats, boxes, ques
-
-
-class GQAEvaluator:
-    def __init__(self, dataset: GQADataset):
-        self.dataset = dataset
-
-    def evaluate(self, quesid2ans: dict):
-        score = 0.
-        for quesid, ans in quesid2ans.items():
-            datum = self.dataset.id2datum[quesid]
-            label = datum['label']
-            if ans in label:
-                score += label[ans]
-        return score / len(quesid2ans)
-
-    def dump_result(self, quesid2ans: dict, path):
-        """
-        Dump the result to a GQA-challenge submittable json file.
-        GQA json file submission requirement:
-            results = [result]
-            result = {
-                "questionId": str,      # Note: it's a actually an int number but the server requires an str.
-                "prediction": str
-            }
-
-        :param quesid2ans: A dict mapping question id to its predicted answer.
-        :param path: The file path to save the json file.
-        :return:
-        """
-        with open(path, 'w') as f:
-            result = []
-            for ques_id, ans in quesid2ans.items():
-                result.append({
-                    'questionId': ques_id,
-                    'prediction': ans
-                })
-            json.dump(result, f, indent=4, sort_keys=True)
-
-

lxmert/src/tasks/gqa_model.py

@@ -1,45 +0,0 @@
-# coding=utf-8
-# Copyleft 2019 project LXRT.
-
-import torch.nn as nn
-
-from param import args
-from lxrt.entry import LXRTEncoder
-from lxrt.modeling import BertLayerNorm, GeLU
-
-# Max length including <bos> and <eos>
-MAX_GQA_LENGTH = 20
-
-
-class GQAModel(nn.Module):
-    def __init__(self, num_answers):
-        super().__init__()
-        self.lxrt_encoder = LXRTEncoder(
-            args,
-            max_seq_length=MAX_GQA_LENGTH
-        )
-        hid_dim = self.lxrt_encoder.dim
-        self.logit_fc = nn.Sequential(
-            nn.Linear(hid_dim, hid_dim * 2),
-            GeLU(),
-            BertLayerNorm(hid_dim * 2, eps=1e-12),
-            nn.Linear(hid_dim * 2, num_answers)
-        )
-        self.logit_fc.apply(self.lxrt_encoder.model.init_bert_weights)
-
-    def forward(self, feat, pos, sent):
-        """
-        b -- batch_size, o -- object_number, f -- visual_feature_size
-
-        :param feat: (b, o, f)
-        :param pos:  (b, o, 4)
-        :param sent: (b,) Type -- list of string
-        :param leng: (b,) Type -- int numpy array
-        :return: (b, num_answer) The logit of each answers.
-        """
-        x = self.lxrt_encoder(sent, (feat, pos))
-        logit = self.logit_fc(x)
-
-        return logit
-
-

lxmert/src/tasks/nlvr2.py

@@ -1,182 +0,0 @@
-# coding=utf-8
-# Copyleft 2019 project LXRT.
-
-import os
-import collections
-
-from tqdm import tqdm
-import torch
-import torch.nn as nn
-from torch.utils.data.dataloader import DataLoader
-
-from param import args
-from tasks.nlvr2_model import NLVR2Model
-from tasks.nlvr2_data import NLVR2Dataset, NLVR2TorchDataset, NLVR2Evaluator
-
-DataTuple = collections.namedtuple("DataTuple", 'dataset loader evaluator')
-
-
-def get_tuple(splits: str, bs:int, shuffle=False, drop_last=False) -> DataTuple:
-    dset = NLVR2Dataset(splits)
-    tset = NLVR2TorchDataset(dset)
-    evaluator = NLVR2Evaluator(dset)
-    data_loader = DataLoader(
-        tset, batch_size=bs,
-        shuffle=shuffle, num_workers=args.num_workers,
-        drop_last=drop_last, pin_memory=True
-    )
-
-    return DataTuple(dataset=dset, loader=data_loader, evaluator=evaluator)
-
-
-class NLVR2:
-    def __init__(self):
-        self.train_tuple = get_tuple(
-            args.train, bs=args.batch_size, shuffle=True, drop_last=True
-        )
-        if args.valid != "":
-            valid_bsize = 2048 if args.multiGPU else 512
-            self.valid_tuple = get_tuple(
-                args.valid, bs=valid_bsize,
-                shuffle=False, drop_last=False
-            )
-        else:
-            self.valid_tuple = None
-
-        self.model = NLVR2Model()
-
-        # Load pre-trained weights
-        if args.load_lxmert is not None:
-            self.model.lxrt_encoder.load(args.load_lxmert)
-
-        # GPU options
-        if args.multiGPU:
-            self.model.lxrt_encoder.multi_gpu()
-        self.model = self.model.cuda()
-
-        # Losses and optimizer
-        self.mce_loss = nn.CrossEntropyLoss(ignore_index=-1)
-        if 'bert' in args.optim:
-            batch_per_epoch = len(self.train_tuple.loader)
-            t_total = int(batch_per_epoch * args.epochs)
-            print("Total Iters: %d" % t_total)
-            from lxrt.optimization import BertAdam
-            self.optim = BertAdam(list(self.model.parameters()),
-                                  lr=args.lr,
-                                  warmup=0.1,
-                                  t_total=t_total)
-        else:
-            self.optim = args.optimizer(list(self.model.parameters()), args.lr)
-
-        self.output = args.output
-        os.makedirs(self.output, exist_ok=True)
-
-    def train(self, train_tuple, eval_tuple):
-        dset, loader, evaluator = train_tuple
-        iter_wrapper = (lambda x: tqdm(x, total=len(loader))) if args.tqdm else (lambda x: x)
-
-        best_valid = 0.
-        for epoch in range(args.epochs):
-            quesid2ans = {}
-            for i, (ques_id, feats, boxes, sent, label) in iter_wrapper(enumerate(loader)):
-                self.model.train()
-
-                self.optim.zero_grad()
-                feats, boxes, label = feats.cuda(), boxes.cuda(), label.cuda()
-                logit = self.model(feats, boxes, sent)
-
-                loss = self.mce_loss(logit, label)
-
-                loss.backward()
-                nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
-                self.optim.step()
-
-                score, predict = logit.max(1)
-                for qid, l in zip(ques_id, predict.cpu().numpy()):
-                    quesid2ans[qid] = l
-
-            log_str = "\nEpoch %d: Train %0.2f\n" % (epoch, evaluator.evaluate(quesid2ans) * 100.)
-
-            if self.valid_tuple is not None:  # Do Validation
-                valid_score = self.evaluate(eval_tuple)
-                if valid_score > best_valid:
-                    best_valid = valid_score
-                    self.save("BEST")
-
-                log_str += "Epoch %d: Valid %0.2f\n" % (epoch, valid_score * 100.) + \
-                           "Epoch %d: Best %0.2f\n" % (epoch, best_valid * 100.)
-
-            print(log_str, end='')
-
-            with open(self.output + "/log.log", 'a') as f:
-                f.write(log_str)
-                f.flush()
-
-        self.save("LAST")
-
-    def predict(self, eval_tuple: DataTuple, dump=None):
-        self.model.eval()
-        dset, loader, evaluator = eval_tuple
-        quesid2ans = {}
-        for i, datum_tuple in enumerate(loader):
-            ques_id, feats, boxes, sent = datum_tuple[:4]   # avoid handling target
-            with torch.no_grad():
-                feats, boxes = feats.cuda(), boxes.cuda()
-                logit = self.model(feats, boxes, sent)
-                score, predict = logit.max(1)
-                for qid, l in zip(ques_id, predict.cpu().numpy()):
-                    quesid2ans[qid] = l
-        if dump is not None:
-            evaluator.dump_result(quesid2ans, dump)
-        return quesid2ans
-
-    def evaluate(self, eval_tuple: DataTuple, dump=None):
-        dset, loader, evaluator = eval_tuple
-        quesid2ans = self.predict(eval_tuple, dump)
-        return evaluator.evaluate(quesid2ans)
-
-    def save(self, name):
-        torch.save(self.model.state_dict(),
-                   os.path.join(self.output, "%s.pth" % name))
-
-    def load(self, path):
-        print("Load model from %s" % path)
-        state_dict = torch.load("%s.pth" % path)
-        self.model.load_state_dict(state_dict)
-
-
-if __name__ == "__main__":
-    # Build Class
-    nlvr2 = NLVR2()
-
-    # Load Model
-    if args.load is not None:
-        nlvr2.load(args.load)
-
-    # Test or Train
-    if args.test is not None:
-        args.fast = args.tiny = False       # Always loading all data in test
-        if 'hidden' in args.test:
-            nlvr2.predict(
-                get_tuple(args.test, bs=args.batch_size,
-                          shuffle=False, drop_last=False),
-                dump=os.path.join(args.output, 'hidden_predict.csv')
-            )
-        elif 'test' in args.test or 'valid' in args.test:
-            result = nlvr2.evaluate(
-                get_tuple(args.test, bs=args.batch_size,
-                          shuffle=False, drop_last=False),
-                dump=os.path.join(args.output, '%s_predict.csv' % args.test)
-            )
-            print(result)
-        else:
-            assert False, "No such test option for %s" % args.test
-    else:
-        print('Splits in Train data:', nlvr2.train_tuple.dataset.splits)
-        if nlvr2.valid_tuple is not None:
-            print('Splits in Valid data:', nlvr2.valid_tuple.dataset.splits)
-        else:
-            print("DO NOT USE VALIDATION")
-        nlvr2.train(nlvr2.train_tuple, nlvr2.valid_tuple)
-
-

lxmert/src/tasks/nlvr2_data.py

@@ -1,157 +0,0 @@
-# coding=utf-8
-# Copyleft 2019 project LXRT.
-
-import json
-
-import numpy as np
-from torch.utils.data import Dataset
-
-from param import args
-from utils import load_obj_tsv
-
-# Load part of the dataset for fast checking.
-# Notice that here is the number of images instead of the number of data,
-# which means all related data to the images would be used.
-TINY_IMG_NUM = 512
-FAST_IMG_NUM = 5000
-
-
-class NLVR2Dataset:
-    """
-    An NLVR2 data example in json file:
-    {
-        "identifier": "train-10171-0-0",
-        "img0": "train-10171-0-img0",
-        "img1": "train-10171-0-img1",
-        "label": 0,
-        "sent": "An image shows one leather pencil case, displayed open with writing implements tucked inside.
-        ",
-        "uid": "nlvr2_train_0"
-    }
-    """
-    def __init__(self, splits: str):
-        self.name = splits
-        self.splits = splits.split(',')
-
-        # Loading datasets to data
-        self.data = []
-        for split in self.splits:
-            self.data.extend(json.load(open("data/nlvr2/%s.json" % split)))
-        print("Load %d data from split(s) %s." % (len(self.data), self.name))
-
-        # List to dict (for evaluation and others)
-        self.id2datum = {
-            datum['uid']: datum
-            for datum in self.data
-        }
-
-    def __len__(self):
-        return len(self.data)
-
-
-"""
-An example in obj36 tsv:
-FIELDNAMES = ["img_id", "img_h", "img_w", "objects_id", "objects_conf",
-              "attrs_id", "attrs_conf", "num_boxes", "boxes", "features"]
-FIELDNAMES would be keys in the dict returned by load_obj_tsv.
-"""
-class NLVR2TorchDataset(Dataset):
-    def __init__(self, dataset: NLVR2Dataset):
-        super().__init__()
-        self.raw_dataset = dataset
-
-        if args.tiny:
-            topk = TINY_IMG_NUM
-        elif args.fast:
-            topk = FAST_IMG_NUM
-        else:
-            topk = -1
-
-        # Loading detection features to img_data
-        img_data = []
-        if 'train' in dataset.splits:
-            img_data.extend(load_obj_tsv('data/nlvr2_imgfeat/train_obj36.tsv', topk=topk))
-        if 'valid' in dataset.splits:
-            img_data.extend(load_obj_tsv('data/nlvr2_imgfeat/valid_obj36.tsv', topk=topk))
-        if 'test' in dataset.name:
-            img_data.extend(load_obj_tsv('data/nlvr2_imgfeat/test_obj36.tsv', topk=topk))
-        self.imgid2img = {}
-        for img_datum in img_data:
-            self.imgid2img[img_datum['img_id']] = img_datum
-
-        # Filter out the dataset
-        self.data = []
-        for datum in self.raw_dataset.data:
-            if datum['img0'] in self.imgid2img and datum['img1'] in self.imgid2img:
-                self.data.append(datum)
-        print("Use %d data in torch dataset" % (len(self.data)))
-        print()
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, item: int):
-        datum = self.data[item]
-
-        ques_id = datum['uid']
-        ques = datum['sent']
-
-        # Get image info
-        boxes2 = []
-        feats2 = []
-        for key in ['img0', 'img1']:
-            img_id = datum[key]
-            img_info = self.imgid2img[img_id]
-            boxes = img_info['boxes'].copy()
-            feats = img_info['features'].copy()
-            assert len(boxes) == len(feats)
-
-            # Normalize the boxes (to 0 ~ 1)
-            img_h, img_w = img_info['img_h'], img_info['img_w']
-            boxes[..., (0, 2)] /= img_w
-            boxes[..., (1, 3)] /= img_h
-            np.testing.assert_array_less(boxes, 1+1e-5)
-            np.testing.assert_array_less(-boxes, 0+1e-5)
-
-            boxes2.append(boxes)
-            feats2.append(feats)
-        feats = np.stack(feats2)
-        boxes = np.stack(boxes2)
-
-        # Create target
-        if 'label' in datum:
-            label = datum['label']
-            return ques_id, feats, boxes, ques, label
-        else:
-            return ques_id, feats, boxes, ques
-
-
-class NLVR2Evaluator:
-    def __init__(self, dataset: NLVR2Dataset):
-        self.dataset = dataset
-
-    def evaluate(self, quesid2ans: dict):
-        score = 0.
-        for quesid, ans in quesid2ans.items():
-            datum = self.dataset.id2datum[quesid]
-            label = datum['label']
-            if ans == label:
-                score += 1
-        return score / len(quesid2ans)
-
-    def dump_result(self, quesid2ans: dict, path):
-        """
-        Dump result to a CSV file, which is compatible with NLVR2 evaluation system.
-        NLVR2 CSV file requirement:
-            Each line contains: identifier, answer
-
-        :param quesid2ans: nlvr2 uid to ans (either "True" or "False")
-        :param path: The desired path of saved file.
-        :return:
-        """
-        with open(path, 'w') as f:
-            for uid, ans in quesid2ans.items():
-                idt = self.dataset.id2datum[uid]["identifier"]
-                ans = 'True' if ans == 1 else 'False'
-                f.write("%s,%s\n" % (idt, ans))
-

lxmert/src/tasks/nlvr2_model.py

@@ -1,55 +0,0 @@
-# coding=utf-8
-# Copyleft 2019 project LXRT.
-
-import torch.nn as nn
-from lxrt.modeling import GeLU, BertLayerNorm
-from lxrt.entry import LXRTEncoder
-from param import args
-
-
-class NLVR2Model(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.lxrt_encoder = LXRTEncoder(
-            args,
-            max_seq_length=20
-        )
-        self.hid_dim = hid_dim = self.lxrt_encoder.dim
-        self.logit_fc = nn.Sequential(
-            nn.Linear(hid_dim * 2, hid_dim * 2),
-            GeLU(),
-            BertLayerNorm(hid_dim * 2, eps=1e-12),
-            nn.Linear(hid_dim * 2, 2)
-        )
-        self.logit_fc.apply(self.lxrt_encoder.model.init_bert_weights)
-
-    def forward(self, feat, pos, sent):
-        """
-        :param feat: b, 2, o, f
-        :param pos:  b, 2, o, 4
-        :param sent: b, (string)
-        :param leng: b, (numpy, int)
-        :return:
-        """
-        # Pairing images and sentences:
-        # The input of NLVR2 is two images and one sentence. In batch level, they are saved as
-        #   [ [img0_0, img0_1], [img1_0, img1_1], ...] and [sent0, sent1, ...]
-        # Here, we flat them to
-        #   feat/pos = [ img0_0, img0_1, img1_0, img1_1, ...]
-        #   sent     = [ sent0,  sent0,  sent1,  sent1,  ...]
-        sent = sum(zip(sent, sent), ())
-        batch_size, img_num, obj_num, feat_size = feat.size()
-        assert img_num == 2 and obj_num == 36 and feat_size == 2048
-        feat = feat.view(batch_size * 2, obj_num, feat_size)
-        pos = pos.view(batch_size * 2, obj_num, 4)
-
-        # Extract feature --> Concat
-        x = self.lxrt_encoder(sent, (feat, pos))
-        x = x.view(-1, self.hid_dim*2)
-
-        # Compute logit of answers
-        logit = self.logit_fc(x)
-
-        return logit
-
-

lxmert/src/tasks/vqa.py

@@ -1,214 +0,0 @@
-# coding=utf-8
-# Copyleft 2019 project LXRT.
-
-import os
-import collections
-
-import torch
-import torch.nn as nn
-from torch.utils.data.dataloader import DataLoader
-from tqdm import tqdm
-
-from ..param import args
-from ..pretrain.qa_answer_table import load_lxmert_qa
-from .vqa_model import VQAModel
-from .vqa_data import VQADataset, VQATorchDataset, VQAEvaluator
-
-DataTuple = collections.namedtuple("DataTuple", 'dataset loader evaluator')
-
-
-def get_data_tuple(splits: str, bs:int, shuffle=False, drop_last=False) -> DataTuple:
-    dset = VQADataset(splits)
-    tset = VQATorchDataset(dset)
-    evaluator = VQAEvaluator(dset)
-    data_loader = DataLoader(
-        tset, batch_size=bs,
-        shuffle=shuffle, num_workers=args.num_workers,
-        drop_last=drop_last, pin_memory=True
-    )
-
-    return DataTuple(dataset=dset, loader=data_loader, evaluator=evaluator)
-
-
-class VQA:
-    def __init__(self):
-        # Datasets
-        self.train_tuple = get_data_tuple(
-            args.train, bs=args.batch_size, shuffle=True, drop_last=True
-        )
-        if args.valid != "":
-            self.valid_tuple = get_data_tuple(
-                args.valid, bs=1024,
-                shuffle=False, drop_last=False
-            )
-        else:
-            self.valid_tuple = None
-        
-        # Model
-        self.model = VQAModel(self.train_tuple.dataset.num_answers)
-
-        # Load pre-trained weights
-        if args.load_lxmert is not None:
-            self.model.lxrt_encoder.load(args.load_lxmert)
-        if args.load_lxmert_qa is not None:
-            load_lxmert_qa(args.load_lxmert_qa, self.model,
-                           label2ans=self.train_tuple.dataset.label2ans)
-        
-        # GPU options
-        self.model = self.model.cuda()
-        if args.multiGPU:
-            self.model.lxrt_encoder.multi_gpu()
-
-        # Loss and Optimizer
-        self.bce_loss = nn.BCEWithLogitsLoss()
-        if 'bert' in args.optim:
-            batch_per_epoch = len(self.train_tuple.loader)
-            t_total = int(batch_per_epoch * args.epochs)
-            print("BertAdam Total Iters: %d" % t_total)
-            from ..lxrt.optimization import BertAdam
-            self.optim = BertAdam(list(self.model.parameters()),
-                                  lr=args.lr,
-                                  warmup=0.1,
-                                  t_total=t_total)
-        else:
-            self.optim = args.optimizer(self.model.parameters(), args.lr)
-        
-        # Output Directory
-        self.output = args.output
-        os.makedirs(self.output, exist_ok=True)
-
-    def train(self, train_tuple, eval_tuple):
-        dset, loader, evaluator = train_tuple
-        iter_wrapper = (lambda x: tqdm(x, total=len(loader))) if args.tqdm else (lambda x: x)
-
-        best_valid = 0.
-        for epoch in range(args.epochs):
-            quesid2ans = {}
-            for i, (ques_id, feats, boxes, sent, target) in iter_wrapper(enumerate(loader)):
-
-                self.model.train()
-                self.optim.zero_grad()
-
-                feats, boxes, target = feats.cuda(), boxes.cuda(), target.cuda()
-                logit = self.model(feats, boxes, sent)
-                assert logit.dim() == target.dim() == 2
-                loss = self.bce_loss(logit, target)
-                loss = loss * logit.size(1)
-
-                loss.backward()
-                nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
-                self.optim.step()
-
-                score, label = logit.max(1)
-                for qid, l in zip(ques_id, label.cpu().numpy()):
-                    ans = dset.label2ans[l]
-                    quesid2ans[qid.item()] = ans
-
-            log_str = "\nEpoch %d: Train %0.2f\n" % (epoch, evaluator.evaluate(quesid2ans) * 100.)
-
-            if self.valid_tuple is not None:  # Do Validation
-                valid_score = self.evaluate(eval_tuple)
-                if valid_score > best_valid:
-                    best_valid = valid_score
-                    self.save("BEST")
-
-                log_str += "Epoch %d: Valid %0.2f\n" % (epoch, valid_score * 100.) + \
-                           "Epoch %d: Best %0.2f\n" % (epoch, best_valid * 100.)
-
-            print(log_str, end='')
-
-            with open(self.output + "/log.log", 'a') as f:
-                f.write(log_str)
-                f.flush()
-
-        self.save("LAST")
-
-    def predict(self, eval_tuple: DataTuple, dump=None):
-        """
-        Predict the answers to questions in a data split.
-
-        :param eval_tuple: The data tuple to be evaluated.
-        :param dump: The path of saved file to dump results.
-        :return: A dict of question_id to answer.
-        """
-        self.model.eval()
-        dset, loader, evaluator = eval_tuple
-        quesid2ans = {}
-        for i, datum_tuple in enumerate(loader):
-            ques_id, feats, boxes, sent = datum_tuple[:4]   # Avoid seeing ground truth
-            with torch.no_grad():
-                feats, boxes = feats.cuda(), boxes.cuda()
-                logit = self.model(feats, boxes, sent)
-                score, label = logit.max(1)
-                for qid, l in zip(ques_id, label.cpu().numpy()):
-                    ans = dset.label2ans[l]
-                    quesid2ans[qid.item()] = ans
-        if dump is not None:
-            evaluator.dump_result(quesid2ans, dump)
-        return quesid2ans
-
-    def evaluate(self, eval_tuple: DataTuple, dump=None):
-        """Evaluate all data in data_tuple."""
-        quesid2ans = self.predict(eval_tuple, dump)
-        return eval_tuple.evaluator.evaluate(quesid2ans)
-
-    @staticmethod
-    def oracle_score(data_tuple):
-        dset, loader, evaluator = data_tuple
-        quesid2ans = {}
-        for i, (ques_id, feats, boxes, sent, target) in enumerate(loader):
-            _, label = target.max(1)
-            for qid, l in zip(ques_id, label.cpu().numpy()):
-                ans = dset.label2ans[l]
-                quesid2ans[qid.item()] = ans
-        return evaluator.evaluate(quesid2ans)
-
-    def save(self, name):
-        torch.save(self.model.state_dict(),
-                   os.path.join(self.output, "%s.pth" % name))
-
-    def load(self, path):
-        print("Load model from %s" % path)
-        state_dict = torch.load("%s.pth" % path)
-        self.model.load_state_dict(state_dict)
-
-
-if __name__ == "__main__":
-    # Build Class
-    vqa = VQA()
-
-    # Load VQA model weights
-    # Note: It is different from loading lxmert pre-trained weights.
-    if args.load is not None:
-        vqa.load(args.load)
-
-    # Test or Train
-    if args.test is not None:
-        args.fast = args.tiny = False       # Always loading all data in test
-        if 'test' in args.test:
-            vqa.predict(
-                get_data_tuple(args.test, bs=950,
-                               shuffle=False, drop_last=False),
-                dump=os.path.join(args.output, 'test_predict.json')
-            )
-        elif 'val' in args.test:    
-            # Since part of valididation data are used in pre-training/fine-tuning,
-            # only validate on the minival set.
-            result = vqa.evaluate(
-                get_data_tuple('minival', bs=950,
-                               shuffle=False, drop_last=False),
-                dump=os.path.join(args.output, 'minival_predict.json')
-            )
-            print(result)
-        else:
-            assert False, "No such test option for %s" % args.test
-    else:
-        print('Splits in Train data:', vqa.train_tuple.dataset.splits)
-        if vqa.valid_tuple is not None:
-            print('Splits in Valid data:', vqa.valid_tuple.dataset.splits)
-            print("Valid Oracle: %0.2f" % (vqa.oracle_score(vqa.valid_tuple) * 100))
-        else:
-            print("DO NOT USE VALIDATION")
-        vqa.train(vqa.train_tuple, vqa.valid_tuple)
-
-

lxmert/src/tasks/vqa_data.py

@@ -1,188 +0,0 @@
-# coding=utf-8
-# Copyleft 2019 project LXRT.
-
-import json
-import os
-import pickle
-
-import numpy as np
-import torch
-from torch.utils.data import Dataset
-
-from ..param import args
-from ..utils import load_obj_tsv
-
-# Load part of the dataset for fast checking.
-# Notice that here is the number of images instead of the number of data,
-# which means all related data to the images would be used.
-TINY_IMG_NUM = 512
-FAST_IMG_NUM = 5000
-
-# The path to data and image features.
-VQA_DATA_ROOT = 'data/vqa/'
-MSCOCO_IMGFEAT_ROOT = 'data/mscoco_imgfeat/'
-SPLIT2NAME = {
-    'train': 'train2014',
-    'valid': 'val2014',
-    'minival': 'val2014',
-    'nominival': 'val2014',
-    'test': 'test2015',
-}
-
-
-class VQADataset:
-    """
-    A VQA data example in json file:
-        {
-            "answer_type": "other",
-            "img_id": "COCO_train2014_000000458752",
-            "label": {
-                "net": 1
-            },
-            "question_id": 458752000,
-            "question_type": "what is this",
-            "sent": "What is this photo taken looking through?"
-        }
-    """
-    def __init__(self, splits: str):
-        self.name = splits
-        self.splits = splits.split(',')
-
-        # Loading datasets
-        self.data = []
-        for split in self.splits:
-            self.data.extend(json.load(open("data/vqa/%s.json" % split)))
-        print("Load %d data from split(s) %s." % (len(self.data), self.name))
-
-        # Convert list to dict (for evaluation)
-        self.id2datum = {
-            datum['question_id']: datum
-            for datum in self.data
-        }
-
-        # Answers
-        self.ans2label = json.load(open("data/vqa/trainval_ans2label.json"))
-        self.label2ans = json.load(open("data/vqa/trainval_label2ans.json"))
-        assert len(self.ans2label) == len(self.label2ans)
-
-    @property
-    def num_answers(self):
-        return len(self.ans2label)
-
-    def __len__(self):
-        return len(self.data)
-
-
-"""
-An example in obj36 tsv:
-FIELDNAMES = ["img_id", "img_h", "img_w", "objects_id", "objects_conf",
-              "attrs_id", "attrs_conf", "num_boxes", "boxes", "features"]
-FIELDNAMES would be keys in the dict returned by load_obj_tsv.
-"""
-class VQATorchDataset(Dataset):
-    def __init__(self, dataset: VQADataset):
-        super().__init__()
-        self.raw_dataset = dataset
-
-        if args.tiny:
-            topk = TINY_IMG_NUM
-        elif args.fast:
-            topk = FAST_IMG_NUM
-        else:
-            topk = None
-
-        # Loading detection features to img_data
-        img_data = []
-        for split in dataset.splits:
-            # Minival is 5K images in MS COCO, which is used in evaluating VQA/lxmert-pre-training.
-            # It is saved as the top 5K features in val2014_***.tsv
-            load_topk = 5000 if (split == 'minival' and topk is None) else topk
-            img_data.extend(load_obj_tsv(
-                os.path.join(MSCOCO_IMGFEAT_ROOT, '%s_obj36.tsv' % (SPLIT2NAME[split])),
-                topk=load_topk))
-
-        # Convert img list to dict
-        self.imgid2img = {}
-        for img_datum in img_data:
-            self.imgid2img[img_datum['img_id']] = img_datum
-
-        # Only kept the data with loaded image features
-        self.data = []
-        for datum in self.raw_dataset.data:
-            if datum['img_id'] in self.imgid2img:
-                self.data.append(datum)
-        print("Use %d data in torch dataset" % (len(self.data)))
-        print()
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, item: int):
-        datum = self.data[item]
-
-        img_id = datum['img_id']
-        ques_id = datum['question_id']
-        ques = datum['sent']
-
-        # Get image info
-        img_info = self.imgid2img[img_id]
-        obj_num = img_info['num_boxes']
-        feats = img_info['features'].copy()
-        boxes = img_info['boxes'].copy()
-        assert obj_num == len(boxes) == len(feats)
-
-        # Normalize the boxes (to 0 ~ 1)
-        img_h, img_w = img_info['img_h'], img_info['img_w']
-        boxes = boxes.copy()
-        boxes[:, (0, 2)] /= img_w
-        boxes[:, (1, 3)] /= img_h
-        np.testing.assert_array_less(boxes, 1+1e-5)
-        np.testing.assert_array_less(-boxes, 0+1e-5)
-
-        # Provide label (target)
-        if 'label' in datum:
-            label = datum['label']
-            target = torch.zeros(self.raw_dataset.num_answers)
-            for ans, score in label.items():
-                target[self.raw_dataset.ans2label[ans]] = score
-            return ques_id, feats, boxes, ques, target
-        else:
-            return ques_id, feats, boxes, ques
-
-
-class VQAEvaluator:
-    def __init__(self, dataset: VQADataset):
-        self.dataset = dataset
-
-    def evaluate(self, quesid2ans: dict):
-        score = 0.
-        for quesid, ans in quesid2ans.items():
-            datum = self.dataset.id2datum[quesid]
-            label = datum['label']
-            if ans in label:
-                score += label[ans]
-        return score / len(quesid2ans)
-
-    def dump_result(self, quesid2ans: dict, path):
-        """
-        Dump results to a json file, which could be submitted to the VQA online evaluation.
-        VQA json file submission requirement:
-            results = [result]
-            result = {
-                "question_id": int,
-                "answer": str
-            }
-
-        :param quesid2ans: dict of quesid --> ans
-        :param path: The desired path of saved file.
-        """
-        with open(path, 'w') as f:
-            result = []
-            for ques_id, ans in quesid2ans.items():
-                result.append({
-                    'question_id': ques_id,
-                    'answer': ans
-                })
-            json.dump(result, f, indent=4, sort_keys=True)
-
-

lxmert/src/tasks/vqa_model.py

@@ -1,50 +0,0 @@
-# coding=utf-8
-# Copyleft 2019 project LXRT.
-
-import torch.nn as nn
-
-from ..param import args
-from ..lxrt.entry import LXRTEncoder
-from ..lxrt.modeling import BertLayerNorm, GeLU
-from transformers import AutoTokenizer, AutoModelForQuestionAnswering
-
-# Max length including <bos> and <eos>
-MAX_VQA_LENGTH = 20
-
-
-class VQAModel(nn.Module):
-    def __init__(self, num_answers):
-        super().__init__()
-        
-        # # Build LXRT encoder
-        # self.lxrt_encoder = LXRTEncoder(
-        #     args,
-        #     max_seq_length=MAX_VQA_LENGTH
-        # )
-        # hid_dim = self.lxrt_encoder.dim
-        #
-        # # VQA Answer heads
-        # self.logit_fc = nn.Sequential(
-        #     nn.Linear(hid_dim, hid_dim * 2),
-        #     GeLU(),
-        #     BertLayerNorm(hid_dim * 2, eps=1e-12),
-        #     nn.Linear(hid_dim * 2, num_answers)
-        # )
-        # self.logit_fc.apply(self.lxrt_encoder.model.init_bert_weights)
-
-        self.tokenizer = AutoTokenizer.from_pretrained("unc-nlp/lxmert-vqa-uncased")
-        self.model = AutoModelForQuestionAnswering.from_pretrained("unc-nlp/lxmert-vqa-uncased")
-
-    def forward(self, feat, pos, sent):
-        """
-        b -- batch_size, o -- object_number, f -- visual_feature_size
-
-        :param feat: (b, o, f)
-        :param pos:  (b, o, 4)
-        :param sent: (b,) Type -- list of string
-        :param leng: (b,) Type -- int numpy array
-        :return: (b, num_answer) The logit of each answers.
-        """
-        return self.model(sent, feat, pos)
-
-