preprocess.py

import time
import os
import random
import json
import pickle
import numpy as np
from tqdm import tqdm
from termcolor import colored
from program_translator import ProgramTranslator  #
from config import config


# Print bold tex
def bold(txt):
    return colored(str(txt), attrs=["bold"])


# Print bold and colored text
def bcolored(txt, color):
    return colored(str(txt), color, attrs=["bold"])


# Write a line to file
def writeline(f, line):
    f.write(str(line) + "\n")


# Write a list to file
def writelist(f, l):
    writeline(f, ",".join(map(str, l)))


# 2d list to numpy
def vectorize2DList(items, minX=0, minY=0, dtype=np.int):
    maxX = max(len(items), minX)
    maxY = max([len(item) for item in items] + [minY])
    t = np.zeros((maxX, maxY), dtype=dtype)
    tLengths = np.zeros((maxX,), dtype=np.int)
    for i, item in enumerate(items):
        t[i, 0:len(item)] = np.array(item, dtype=dtype)
        tLengths[i] = len(item)
    return t, tLengths


# 3d list to numpy
def vectorize3DList(items, minX=0, minY=0, minZ=0, dtype=np.int):
    maxX = max(len(items), minX)
    maxY = max([len(item) for item in items] + [minY])
    maxZ = max([len(subitem) for item in items for subitem in item] + [minZ])
    t = np.zeros((maxX, maxY, maxZ), dtype=dtype)
    tLengths = np.zeros((maxX, maxY), dtype=np.int)
    for i, item in enumerate(items):
        for j, subitem in enumerate(item):
            t[i, j, 0:len(subitem)] = np.array(subitem, dtype=dtype)
            tLengths[i, j] = len(subitem)
    return t, tLengths


'''
Encodes text into integers. Keeps dictionary between string words (symbols) 
and their matching integers. Supports encoding and decoding.
'''


class SymbolDict(object):
    def __init__(self, empty=False):
        self.padding = "<PAD>"
        self.unknown = "<UNK>"
        self.start = "<START>"
        self.end = "<END>"

        self.invalidSymbols = [self.padding, self.unknown, self.start, self.end]

        if empty:
            self.sym2id = {}
            self.id2sym = []
        else:
            self.sym2id = {self.padding: 0, self.unknown: 1, self.start: 2, self.end: 3}
            self.id2sym = [self.padding, self.unknown, self.start, self.end]
        self.allSeqs = []

    def getNumSymbols(self):
        return len(self.sym2id)

    def isPadding(self, enc):
        return enc == 0

    def isUnknown(self, enc):
        return enc == 1

    def isStart(self, enc):
        return enc == 2

    def isEnd(self, enc):
        return enc == 3

    def isValid(self, enc):
        return enc < self.getNumSymbols() and enc >= len(self.invalidSymbols)

    def resetSeqs(self):
        self.allSeqs = []

    def addSeq(self, seq):
        self.allSeqs += seq

    # Call to create the words-to-integers vocabulary after (reading word sequences with addSeq).
    def createVocab(self, minCount=0):
        counter = {}
        for symbol in self.allSeqs:
            counter[symbol] = counter.get(symbol, 0) + 1
        for symbol in counter:
            if counter[symbol] > minCount and (symbol not in self.sym2id):
                self.sym2id[symbol] = self.getNumSymbols()
                self.id2sym.append(symbol)

    # Encodes a symbol. Returns the matching integer.
    def encodeSym(self, symbol):
        if symbol not in self.sym2id:
            symbol = self.unknown
        return self.sym2id[symbol]

    '''
    Encodes a sequence of symbols.
    Optionally add start, or end symbols. 
    Optionally reverse sequence 
    '''

    def encodeSequence(self, decoded, addStart=False, addEnd=False, reverse=False):
        if reverse:
            decoded.reverse()
        if addStart:
            decoded = [self.start] + decoded
        if addEnd:
            decoded = decoded + [self.end]
        encoded = [self.encodeSym(symbol) for symbol in decoded]
        return encoded

    # Decodes an integer into its symbol
    def decodeId(self, enc):
        return self.id2sym[enc] if enc < self.getNumSymbols() else self.unknown

    '''
    Decodes a sequence of integers into their symbols.
    If delim is given, joins the symbols using delim,
    Optionally reverse the resulted sequence 
    '''

    def decodeSequence(self, encoded, delim=None, reverse=False, stopAtInvalid=True):
        length = 0
        for i in range(len(encoded)):
            if not self.isValid(encoded[i]) and stopAtInvalid:
                break
            length += 1
        encoded = encoded[:length]

        decoded = [self.decodeId(enc) for enc in encoded]
        if reverse:
            decoded.reverse()

        if delim is not None:
            return delim.join(decoded)

        return decoded


'''
Preprocesses a given dataset into numpy arrays.
By calling preprocess, the class:
1. Reads the input data files into dictionary. 
2. Saves the results jsons in files and loads them instead of parsing input if files exist/
3. Initializes word embeddings to random / GloVe.
4. Optionally filters data according to given filters.
5. Encodes and vectorize the data into numpy arrays.
6. Buckets the data according to the instances length.
'''


class Preprocesser(object):
    def __init__(self):
        self.questionDict = SymbolDict()
        self.answerDict = SymbolDict(empty=True)
        self.qaDict = SymbolDict()

        self.specificDatasetDicts = None

        self.programDict = SymbolDict()
        self.programTranslator = ProgramTranslator(self.programDict, 2)

    '''
    Tokenizes string into list of symbols.

    Args:
        text: raw string to tokenize.
        ignorePuncts: punctuation to ignore
        keptPunct: punctuation to keep (as symbol)
        endPunct: punctuation to remove if appears at the end
        delim: delimiter between symbols
        clean: True to replace text in string
        replacelistPre: dictionary of replacement to perform on the text before tokanization
        replacelistPost: dictionary of replacement to perform on the text after tokanization
    '''
    # sentence tokenizer
    allPunct = ["?", "!", "\\", "/", ")", "(", ".", ",", ";", ":"]

    def tokenize(self, text, ignoredPuncts=["?", "!", "\\", "/", ")", "("],
                 keptPuncts=[".", ",", ";", ":"], endPunct=[">", "<", ":"], delim=" ",
                 clean=False, replacelistPre=dict(), replacelistPost=dict()):

        if clean:
            for word in replacelistPre:
                origText = text
                text = text.replace(word, replacelistPre[word])
                if (origText != text):
                    print(origText)
                    print(text)
                    print("")

            for punct in endPunct:
                if text[-1] == punct:
                    print(text)
                    text = text[:-1]
                    print(text)
                    print("")

        for punct in keptPuncts:
            text = text.replace(punct, delim + punct + delim)

        for punct in ignoredPuncts:
            text = text.replace(punct, "")

        ret = text.lower().split(delim)

        if clean:
            origRet = ret
            ret = [replacelistPost.get(word, word) for word in ret]
            if origRet != ret:
                print(origRet)
                print(ret)

        ret = [t for t in ret if t != ""]
        return ret

    # Read class' generated files.
    # files interface
    def readFiles(self, instancesFilename):
        with open(instancesFilename, "r") as inFile:
            instances = json.load(inFile)

        with open(config.questionDictFile(), "rb") as inFile:
            self.questionDict = pickle.load(inFile)

        with open(config.answerDictFile(), "rb") as inFile:
            self.answerDict = pickle.load(inFile)

        with open(config.qaDictFile(), "rb") as inFile:
            self.qaDict = pickle.load(inFile)

        return instances

    '''
    Generate class' files. Save json representation of instances and
    symbols-to-integers dictionaries.  
    '''

    def writeFiles(self, instances, instancesFilename):
        with open(instancesFilename, "w") as outFile:
            json.dump(instances, outFile)

        with open(config.questionDictFile(), "wb") as outFile:
            pickle.dump(self.questionDict, outFile)

        with open(config.answerDictFile(), "wb") as outFile:
            pickle.dump(self.answerDict, outFile)

        with open(config.qaDictFile(), "wb") as outFile:
            pickle.dump(self.qaDict, outFile)

    # Write prediction json to file and optionally a one-answer-per-line output file
    def writePreds(self, res, tier, suffix=""):
        if res is None:
            return
        preds = res["preds"]
        sortedPreds = sorted(preds, key=lambda instance: instance["index"])
        with open(config.predsFile(tier + suffix), "w") as outFile:
            outFile.write(json.dumps(sortedPreds))
        with open(config.answersFile(tier + suffix), "w") as outFile:
            for instance in sortedPreds:
                writeline(outFile, instance["prediction"])

    def readPDF(self, instancesFilename):
        instances = []

        if os.path.exists(instancesFilename):
            instances = self.readFiles(instancesFilename)

        return instances

    def readData(self, datasetFilename, instancesFilename, train):
        # data extraction
        datasetReader = {
            "PDF": self.readPDF
        }

        return datasetReader[config.dataset](datasetFilename, instancesFilename, train)

    def vectorizeData(self, data):
        # if "SHARED" tie symbol representations in questions and answers
        if config.ansEmbMod == "SHARED":
            qDict = self.qaDict
        else:
            qDict = self.questionDict

        encodedQuestion = [qDict.encodeSequence(d["questionSeq"]) for d in data]
        question, questionL = vectorize2DList(encodedQuestion)

        # pass the whole instances? if heavy then not good
        imageId = [d["imageId"] for d in data]
        instance = data

        return {"question": question,
                "questionLength": questionL,
                "imageId": imageId
                }

    # Separates data based on a field length
    def lseparator(self, key, lims):
        maxI = len(lims)

        def separatorFn(x):
            v = x[key]
            for i, lim in enumerate(lims):
                if len(v) < lim:
                    return i
            return maxI

        return {"separate": separatorFn, "groupsNum": maxI + 1}

    # Buckets data to groups using a separator
    def bucket(self, instances, separator):
        buckets = [[] for i in range(separator["groupsNum"])]
        for instance in instances:
            bucketI = separator["separate"](instance)
            buckets[bucketI].append(instance)
        return [bucket for bucket in buckets if len(bucket) > 0]

    # Re-buckets bucket list given a seperator
    def rebucket(self, buckets, separator):
        res = []
        for bucket in buckets:
            res += self.bucket(bucket, separator)
        return res

    # Buckets data based on question / program length
    def bucketData(self, data, noBucket=False):
        if noBucket:
            buckets = [data]
        else:
            if config.noBucket:
                buckets = [data]
            elif config.noRebucket:
                questionSep = self.lseparator("questionSeq", config.questionLims)
                buckets = self.bucket(data, questionSep)
            else:
                programSep = self.lseparator("programSeq", config.programLims)
                questionSep = self.lseparator("questionSeq", config.questionLims)
                buckets = self.bucket(data, programSep)
                buckets = self.rebucket(buckets, questionSep)
        return buckets

    ''' 
    Prepares data: 
    1. Filters data according to above arguments.
    2. Takes only a subset of the data based on config.trainedNum / config.testedNum
    3. Buckets data according to question / program length
    4. Vectorizes data into numpy arrays
    '''

    def prepareData(self, data, train, filterKey=None, noBucket=False):
        filterDefault = {"maxQLength": 0, "maxPLength": 0, "onlyChain": False, "filterOp": 0}

        filterTrain = {"maxQLength": config.tMaxQ, "maxPLength": config.tMaxP,
                       "onlyChain": config.tOnlyChain, "filterOp": config.tFilterOp}

        filterVal = {"maxQLength": config.vMaxQ, "maxPLength": config.vMaxP,
                     "onlyChain": config.vOnlyChain, "filterOp": config.vFilterOp}

        filters = {"train": filterTrain, "evalTrain": filterTrain,
                   "val": filterVal, "test": filterDefault}

        if filterKey is None:
            fltr = filterDefault
        else:
            fltr = filters[filterKey]

        # split data when finetuning on validation set
        if config.trainExtra and config.extraVal and (config.finetuneNum > 0):
            if train:
                data = data[:config.finetuneNum]
            else:
                data = data[config.finetuneNum:]

        typeFilter = config.typeFilters[fltr["filterOp"]]
        # filter specific settings
        if fltr["onlyChain"]:
            data = [d for d in data if all((len(inputNum) < 2) for inputNum in d["programInputs"])]
        if fltr["maxQLength"] > 0:
            data = [d for d in data if len(d["questionSeq"]) <= fltr["maxQLength"]]
        if fltr["maxPLength"] > 0:
            data = [d for d in data if len(d["programSeq"]) <= fltr["maxPLength"]]
        if len(typeFilter) > 0:
            data = [d for d in data if d["programSeq"][-1] not in typeFilter]

        # run on subset of the data. If 0 then use all data
        num = config.trainedNum if train else config.testedNum
        # retainVal = True to retain same clevr_sample of validation across runs
        if (not train) and (not config.retainVal):
            random.shuffle(data)
        if num > 0:
            data = data[:num]
        # set number to match dataset size
        if train:
            config.trainedNum = len(data)
        else:
            config.testedNum = len(data)

        # bucket
        buckets = self.bucketData(data, noBucket=noBucket)

        # vectorize
        return [self.vectorizeData(bucket) for bucket in buckets]

    # Prepares all the tiers of a dataset. See prepareData method for further details.
    def prepareDataset(self, dataset, noBucket=False):
        if dataset is None:
            return None

        for tier in dataset:
            if dataset[tier] is not None:
                dataset[tier]["data"] = self.prepareData(dataset[tier]["instances"],
                                                         train=dataset[tier]["train"], filterKey=tier,
                                                         noBucket=noBucket)

        for tier in dataset:
            if dataset[tier] is not None:
                del dataset[tier]["instances"]

        return dataset

    # Initializes word embeddings to random uniform / random normal / GloVe.
    def initializeWordEmbeddings(self, wordsDict=None, noPadding=False):
        # default dictionary to use for embeddings
        if wordsDict is None:
            wordsDict = self.questionDict

        # uniform initialization
        if config.wrdEmbUniform:
            lowInit = -1.0 * config.wrdEmbScale
            highInit = 1.0 * config.wrdEmbScale
            embeddings = np.random.uniform(low=lowInit, high=highInit,
                                           size=(wordsDict.getNumSymbols(), config.wrdEmbDim))
        # normal initialization
        else:
            embeddings = config.wrdEmbScale * np.random.randn(wordsDict.getNumSymbols(),
                                                              config.wrdEmbDim)

        # if wrdEmbRandom = False, use GloVE
        counter = 0
        if (not config.wrdEmbRandom):
            with open(config.wordVectorsFile, 'r') as inFile:
                for line in inFile:
                    line = line.strip().split()
                    word = line[0].lower()
                    vector = [float(x) for x in line[1:]]
                    index = wordsDict.sym2id.get(word)
                    if index is not None:
                        embeddings[index] = vector
                        counter += 1

        print(counter)
        print(self.questionDict.sym2id)
        print(len(self.questionDict.sym2id))
        print(self.answerDict.sym2id)
        print(len(self.answerDict.sym2id))
        print(self.qaDict.sym2id)
        print(len(self.qaDict.sym2id))

        if noPadding:
            return embeddings  # no embedding for padding symbol
        else:
            return embeddings[1:]

    '''
    Initializes words embeddings for question words and optionally for answer words
    (when config.ansEmbMod == "BOTH"). If config.ansEmbMod == "SHARED", tie embeddings for
    question and answer same symbols. 
    '''

    def initializeQAEmbeddings(self):
        # use same embeddings for questions and answers
        if config.ansEmbMod == "SHARED":
            qaEmbeddings = self.initializeWordEmbeddings(self.qaDict)
            ansMap = np.array([self.qaDict.sym2id[sym] for sym in self.answerDict.id2sym])
            embeddings = {"qa": qaEmbeddings, "ansMap": ansMap}
        # use different embeddings for questions and answers
        else:
            qEmbeddings = self.initializeWordEmbeddings(self.questionDict)
            aEmbeddings = None
            if config.ansEmbMod == "BOTH":
                aEmbeddings = self.initializeWordEmbeddings(self.answerDict, noPadding=True)
            embeddings = {"q": qEmbeddings, "a": aEmbeddings}
        return embeddings

    '''
    Preprocesses a given dataset into numpy arrays:
    1. Reads the input data files into dictionary. 
    2. Saves the results jsons in files and loads them instead of parsing input if files exist/
    3. Initializes word embeddings to random / GloVe.
    4. Optionally filters data according to given filters.
    5. Encodes and vectorize the data into numpy arrays.
    5. Buckets the data according to the instances length.
    '''

    def preprocessData(self, question, debug=False):
        # Read data into json and symbols' dictionaries
        print(bold("Loading data..."))
        start = time.time()
        with open(config.questionDictFile(), "rb") as inFile:
            self.questionDict = pickle.load(inFile)
        with open(config.qaDictFile(), "rb") as inFile:
            self.qaDict = pickle.load(inFile)
        with open(config.answerDictFile(), "rb") as inFile:
            self.answerDict = pickle.load(inFile)
        question = question.replace('?', '').replace(', ', '').lower().split()
        encodedQuestion = self.questionDict.encodeSequence(question)
        data = {'question': np.array([encodedQuestion]), 'questionLength': np.array([len(encodedQuestion)])}
        print("took {:.2f} seconds".format(time.time() - start))

        # Initialize word embeddings (random / glove)
        print(bold("Loading word vectors..."))
        start = time.time()
        embeddings = self.initializeQAEmbeddings()
        print("took {:.2f} seconds".format(time.time() - start))

        answer = 'yes'  # DUMMY_ANSWER
        self.answerDict.addSeq([answer])
        self.qaDict.addSeq([answer])

        config.questionWordsNum = self.questionDict.getNumSymbols()
        config.answerWordsNum = self.answerDict.getNumSymbols()

        return data, embeddings, self.answerDict