dictlearn/retrieval.py
class Retrieval(object):

    def __init__(self, vocab_text, dictionary,
                 max_def_length=1000, with_too_long_defs='drop',
                 max_def_per_word=1000000, with_too_many_defs='random',
                 exclude_top_k=None, vocab_def=None,
                 add_bod_eod=True, seed=777):
        """Retrieves the definitions.
        vocab_text
            The vocabulary for text
        vocab_def
            The vocabulary for definitions
        dictionary
            The dictionary of the definitions.
        max_def_length
            Disregard definitions that are longer than that.
        exclude_top_k
            Do not provide defitions for the first top k
            words of the vocabulary (typically the most frequent ones).
        max_def_per_word
            Pick at most max_n_def definitions for each word
        """
        self._vocab_text = vocab_text
        self._add_bod_eod = add_bod_eod
        self._rng = numpy.random.RandomState(seed)
        if vocab_def is None:
            self._vocab_def = self._vocab_text
        else:
            self._vocab_def = vocab_def
        self._dictionary = dictionary
        if exclude_top_k == -1:
            logger.debug("Exclude definition of all dictionary words")
            exclude_top_k = vocab_text.size()
        self._exclude_top_k = exclude_top_k

        if all(numpy.array(self._vocab_text._id_to_freq) == 1) and exclude_top_k > 0:
            # Also note that after merging doing exclude_top_k on freqs in merged def/text is perhaps
            # confusing
            raise Exception("Cannot perform exclude_top_k based on vocabulary without frequency information.")

        # TODO(kudkudak):
        # To follow conventions - def dropping etc should also be performed in crawl_dict.py
        # TODO (rizar):
        # ... or maybe it's moving everything to the preprocessing that was a mistake

        self._max_def_length = max_def_length
        if with_too_long_defs not in {"drop", "crop"}:
            raise NotImplementedError("Not implemented " + with_too_long_defs)
        self._with_too_long_defs = with_too_long_defs

        self._max_def_per_word = max_def_per_word
        if with_too_many_defs not in {"random", "exclude"}:
            raise NotImplementedError("Not implemented " + with_too_many_defs)
        self._with_too_many_defs = with_too_many_defs

        self._debug_info = {
            "missed_word_sample": [],

            "N_words": 0,
            "N_excluded_words": 0,

            "N_distinct_words": 0,
            "N_missed_distinct_words": 0,

            "N_def": 0,
            "N_dropped_def": 0,

            "N_queried_words": 0,
            "N_missed_words": 0,
        }

    def __setstate__(self, state):
        self.__dict__.update(state)
        if "_rng" not in self.__dict__:
            self._rng = numpy.random.RandomState(777)

    def retrieve(self, batch):
        """Retrieves all definitions for a batch of words sequences.

        TODO: definitions of phrases, phrasal verbs, etc.

        Returns
        -------
        defs
            A list of word definitions, each definition is a list of words.
        def_map
            A list of triples (batch_index, time_step, def_index). Maps
            words to their respective definitions from `defs`.

        """
        definitions = []
        def_map = []
        word_def_indices = {}

        for seq_pos, sequence in enumerate(batch):
            for word_pos, word in enumerate(sequence):
                if isinstance(word, numpy.ndarray):
                    word = vec2str(word)
                if not word:
                    continue
                self._debug_info['N_words'] += 1
                word_id = self._vocab_text.word_to_id(word)
                if (self._exclude_top_k
                        and word_id != self._vocab_text.unk
                        and word_id < self._exclude_top_k):
                    self._debug_info['N_excluded_words'] += 1
                    continue

                if word not in word_def_indices:
                    word_def_indices[word] = []
                    # The first time a word is encountered in a batch
                    word_defs = self._dictionary.get_definitions(word)

                    if self._max_def_per_word < len(word_defs):
                        if self._with_too_many_defs == 'random':
                            word_defs = self._rng.choice(
                                word_defs, self._max_def_per_word, replace=False)
                        else:
                            # (rizar): if there's too many definition for a words,
                            # maybe let's just accept that it's a "semantic prime"?
                            word_defs = []

                    # Debug info
                    self._debug_info['N_distinct_words'] += 1
                    self._debug_info['N_missed_distinct_words'] += (len(word_defs) == 0)
                    # End of debug info

                    for i, def_ in enumerate(word_defs):
                        self._debug_info['N_def'] += 1

                        if  self._with_too_long_defs == 'drop':
                            if len(def_) > self._max_def_length:
                                self._debug_info['N_dropped_def'] += 1
                                continue
                        elif self._with_too_long_defs == 'crop':
                            def_ = def_[0:self._max_def_length]
                        else:
                            raise NotImplementedError()

                        final_def_ = []
                        if self._add_bod_eod:
                            final_def_.append(self._vocab_def.bod)
                        for token in def_:
                            final_def_.append(self._vocab_def.word_to_id(token))
                        if self._add_bod_eod:
                            final_def_.append(self._vocab_def.eod)
                        word_def_indices[word].append(len(definitions))
                        definitions.append(final_def_)

                # Debug info
                self._debug_info['N_queried_words'] += 1
                if len(word_def_indices[word]) == 0:
                    self._debug_info['N_missed_words'] += 1
                    if len(self._debug_info['missed_word_sample']) == 10000:
                        self._debug_info['missed_word_sample'][numpy.random.randint(10000)] = word
                    else:
                        self._debug_info['missed_word_sample'].append(word)
                # End of debug info

                for def_index in word_def_indices[word]:
                    def_map.append((seq_pos, word_pos, def_index))

        return definitions, def_map

    def retrieve_and_pad(self, batch):
        defs, def_map = self.retrieve(batch)
        if not defs:
            defs.append(self.sentinel_definition())
        # `defs` have variable length and have to be padded
        max_def_length = max(map(len, defs))
        def_array = numpy.zeros((len(defs), max_def_length), dtype='int64')
        def_mask = numpy.ones_like(def_array, dtype=fuel.config.floatX)
        for i, def_ in enumerate(defs):
            def_array[i, :len(def_)] = def_
            def_mask[i, len(def_):] = 0.
        def_map = (numpy.array(def_map)
                   if def_map
                   else numpy.zeros((0, 3), dtype='int64'))
        return def_array, def_mask, def_map

    def sentinel_definition(self):
        """An empty definition.

        If you ever need a definition that is syntactically correct but
        doesn't mean a thing, call me.

        """
        return [self._vocab_def.bod, self._vocab_def.eod]
dictlearn/extensions.py
def construct_embedder(theano_fnc, vocab):
    """
    Parameters
    ----------
    theano_fnc: theano.Function
        (batch_size, seq_len) -> (batch_size, seq_len, word_dim)

    vocab: Vocabulary
        Vocabulary instance

    Returns
    -------
        Python function: (batch_size, ) -> (batch_size, word_dim)
    """

    def _embedder(word_list):
        word_ids = vocab.encode(word_list)
        word_ids = np.array(word_ids)
        word_ids = word_ids.reshape((-1, 1)) # Just to adhere to theano.Function, whatever
        word_vectors = theano_fnc(word_ids)
        word_vectors = word_vectors.reshape((len(word_list), -1))
        return word_vectors

    return _embedder
dictlearn/extensions.py
class StartFuelServer(SimpleExtension):

    def __init__(self, stream, stream_path, script_path="start_fuel_server.py", hwm=100, *args, **kwargs):
        self._stream = stream
        self._hwm = hwm
        self._stream_path = stream_path
        self._script_path = script_path
        super(StartFuelServer, self).__init__(*args, **kwargs)

    def do(self, *args, **kwars):
        with open(self._stream_path, 'w') as dst:
            cPickle.dump(self._stream, dst, 0)
        port = get_free_port()
        self.main_loop.data_stream.port = port
        logger.debug("Starting the Fuel server on port " + str(port))
        ret = subprocess.Popen(
            [self._script_path,
                self._stream_path, str(port), str(self._hwm)])
        time.sleep(0.1)
        if ret.returncode is not None:
            raise Exception()
        atexit.register(lambda: os.kill(ret.pid, signal.SIGINT))
dictlearn/inits.py
class GlorotUniform(NdarrayInitialization):
    """Initialize parameters from an isotropic Gaussian distribution.

    Parameters
    ----------
    std : float, optional
        The standard deviation of the Gaussian distribution. Defaults to 1.
    mean : float, optional
        The mean of the Gaussian distribution. Defaults to 0

    Notes
    -----
    Be careful: the standard deviation goes first and the mean goes
    second!

    """

    def __init__(self):
        pass

    def generate(self, rng, shape):
        if len(shape) == 1:
            return rng.uniform(size=shape, low=-0.00001, high=0.00001).astype(theano.config.floatX)

        if not len(shape) == 2:
            raise NotImplementedError("GlorotUniform doesnt work for " + str(shape) + " shape")

        fan_in, fan_out = shape[0], shape[1]
        s = np.sqrt(6. / (fan_in + fan_out))
        return rng.uniform(size=shape, low=-s, high=s).astype(theano.config.floatX)

    def __repr__(self):
        return "GlorotUniform"
dictlearn/vocab.py
class Vocabulary(object):
    """Class that holds a vocabulary for the dataset."""
    BOS = '<bos>' # beginning-of-sequence
    EOS = '<eos>' # end-of-sequence
    BOD = '<bod>' # beginning-of-definition
    EOD = '<eod>' # end-of-definition
    UNK = '<unk>' # unknown token
    SPECIAL_TOKEN_MAP = {
        BOS: 'bos',
        EOS: 'eos',
        BOD: 'bod',
        EOD: 'eod',
        UNK: 'unk'
    }

    def __init__(self, path_or_data):
        """Initialize the vocabulary.

        path_or_data
            Either a list of words or the path to it.
        top_k
            If not `None`, only the first `top_k` entries will be left.
            Note, this does not include the special tokens.

        """
        if isinstance(path_or_data, string_types):
            words_and_freqs = []
            with open(path_or_data) as f:
                for line in f:
                    word, freq_str = line.strip().split()
                    word = word.decode('utf-8')
                    freq = int(freq_str)
                    words_and_freqs.append((word, freq))
        else:
            words_and_freqs = path_or_data

        self._id_to_word = []
        self._id_to_freq = []
        self._word_to_id = {}
        self.bos = self.eos = -1
        self.bod = self.eod = -1
        self.unk = -1

        for idx, (word_name, freq) in enumerate(words_and_freqs):
            token_attr = self.SPECIAL_TOKEN_MAP.get(word_name)
            if token_attr is not None:
                setattr(self, token_attr, idx)

            self._id_to_word.append(word_name)
            self._id_to_freq.append(freq)
            self._word_to_id[word_name] = idx

        if -1 in [getattr(self, attr)
                  for attr in self.SPECIAL_TOKEN_MAP.values()]:
            raise ValueError("special token not found in the vocabulary")

    def size(self):
        return len(self._id_to_word)

    @property
    def words(self):
        return self._id_to_word

    @property
    def frequencies(self):
        return self._id_to_freq

    def word_to_id(self, word, top_k=None):
        id_ = self._word_to_id.get(word)
        if id_ is not None and not top_k or id_ < top_k:
            return id_
        return self.unk

    def id_to_word(self, cur_id):
        return self._id_to_word[cur_id]

    def word_freq(self, word):
        if not word in self._word_to_id:
            return 0
        return self._id_to_freq[self._word_to_id[word]]

    def decode(self, cur_ids):
        return ' '.join([self.id_to_word(cur_id) for cur_id in cur_ids])

    def encode(self, sentence):
        word_ids = [self.word_to_id(cur_word) for cur_word in sentence]
        return numpy.array(word_ids, dtype=numpy.int64)

    @staticmethod
    def build(text, top_k=None, sort_by='frequency'):
        """
        sort_by is either 'frequency' or 'lexicographical'
        """
        # For now let's use a very stupid tokenization
        if isinstance(text, str):
            with open(text) as file_:
                def data():
                    for line in file_:
                        for word in line.strip().split():
                            yield word
                counter = Counter(data())
            logger.info("Data is read")
        else:
            counter = Counter(text)
            for word in list(counter.keys()):
                if ' ' in word:
                    logger.error("can't have tokens with spaces, skip {}".format(word))
                    del counter[word]
        # It was not immediately clear to me
        # if counter.most_common() selects consistenly among
        # the words with the same counts. Hence, let's just sort.
        if sort_by == 'frequency':
            sortf = lambda x: (-x[1], x[0])
        elif sort_by == 'lexicographical':
            sortf = lambda x: (x[0], x[1])
        else:
            raise Exception("sort not understood:", sort_by)
        words_and_freqs = sorted(counter.items(), key=sortf)
        logger.info("Words are sorted")
        if top_k:
            words_and_freqs  = words_and_freqs[:top_k]
        words_and_freqs = (
            [(Vocabulary.BOS, 0),
             (Vocabulary.EOS, 0),
             (Vocabulary.BOD, 0),
             (Vocabulary.EOD, 0),
             (Vocabulary.UNK, 0)]
            + words_and_freqs)

        return Vocabulary(words_and_freqs)

    def save(self, filename):
        with open(filename, 'w') as f:
            for word, freq in zip(self._id_to_word, self._id_to_freq):

                # Note: if this fails for you make sure that words read
                # and used by Vocabulary were utf-8 encoded prior to that
                if not isinstance(word, text_type):
                    word = text_type(word, "utf-8")

                print(word.encode('utf-8'), freq, file=f)
dictlearn/data.py
class SNLIData(Data):
    def __init__(self, *args, **kwargs):
        super(SNLIData, self).__init__(*args, **kwargs)
        self._retrieval = None

    def set_retrieval(self, retrieval):
        self._retrieval = retrieval

    def get_stream(self, part, batch_size, seed=None, raw_text=False):
        d = self.get_dataset(part)
        print("Dataset with {} examples".format(d.num_examples))
        it = ShuffledExampleScheme(d.num_examples, rng=numpy.random.RandomState(seed))
        stream = DataStream(d, iteration_scheme=it)
        stream = Batch(stream, iteration_scheme=ConstantScheme(batch_size))

        if self._retrieval:
            stream = FixedMapping(
                stream,
                functools.partial(retrieve_and_pad_snli, self._retrieval),
                add_sources=("defs", "def_mask", "sentence1_def_map", "sentence2_def_map")) # This is because there is bug in Fuel :( Cannot concatenate tuple and list

        if not raw_text:
            stream = SourcewiseMapping(stream, functools.partial(digitize, self.vocab),
                which_sources=('sentence1', 'sentence2'))

        stream = Padding(stream, mask_sources=('sentence1', 'sentence2'))  # Increases amount of outputs by x2

        return stream
dictlearn/retrieval.py
class Dictionary(object):
    """The dictionary of definitions.

    The native representation of the dictionary is a mapping from a word
    to a list of definitions, each of which is a sequence of words. All
    the words are stored as strings.

    """
    def __init__(self, path=None):

        if not path.endswith("json"):
            raise Exception("Please pass path ending in .json")

        self._data = defaultdict(list)
        self._meta_data = {}
        self._path = path
        self._meta_path = path.replace(".json", "_meta.json")
        #self._tmp_path = os.path.join(os.path.dirname(path),
        #                              self._path + '.tmp')
        #self._meta_tmp_path = os.path.join(os.path.dirname(path),
        #    self._meta_path + '.tmp')
        self._tmp_path = self._path + ".tmp"
        self._meta_tmp_path = self._meta_path + ".tmp"
        if self._path:
            if os.path.exists(self._path):
                self.load()
            else:
                logger.warning("No dict was loaded; ignore if you are"
                               " creating new one")

    def load(self):
        with open(self._path, 'r') as src:
            # can't just assign because self._data should keep being a
            # defaultdict
            self._data.update(json.load(src))
        if os.path.exists(self._meta_path):
            with open(self._meta_path, 'r') as src:
                self._meta_data = json.load(src)

    def save(self):
        logger.debug("saving...")
        with open(self._tmp_path, 'w') as dst:
            json.dump(self._data, dst, indent=2)
        shutil.move(self._tmp_path, self._path)
        logger.debug("saving meta...")
        with open(self._meta_tmp_path, 'w') as dst:
            json.dump(self._meta_data, dst, indent=2)
        shutil.move(self._meta_tmp_path, self._meta_path)
        logger.debug("saved")

    def _wait_until_quota_reset(self):
        while True:
            status = self._account_api.getApiTokenStatus()
            logger.debug("{} remaining calls".format(status.remainingCalls))
            if status.remainingCalls >= _MIN_REMAINING_CALLS:
                logger.debug("Wordnik quota was resetted")
                self._remaining_calls = status.remainingCalls
                return
            logger.debug("sleep until quota reset")
            time.sleep(60.)

    def add_identity_mapping(self, vocab):
        for word in vocab.words:
            self._data[word].append([word])
            self._meta_data[word] = {"sourceDictionary": "identity"}
        self.save()

    def remove_out_of_vocabulary(self, vocab):
        """
        remove definitions that are outside of a vocabulary vocab
        """
        # TODO: remove from meta too
        # can't just zip because if meta is empty then nothing happens
        print("vocab size : {}".format(vocab.size()))
        print("dict len : {}".format(len(self._data)))
        count_del = 0
        for word in self._data.keys():
            word_id = vocab.word_to_id(word)
            if word_id == vocab.unk:
                del self._data[word]
                count_del += 1
                # del self._meta_data[word] # TODO:
        print("have deleted {} definitions".format(count_del))
        self.save()


    def add_spelling(self, vocab, only_if_no_def=True):
        for word in vocab.words:
            # only add spelling to the words without defs
            if only_if_no_def and self._data[word]:
                continue
            def_ = word
            if len(def_) > 10:
                def_ = u"{}-{}".format(def_[:5], def_[-5:])
            # to avoid overlapping of the vocabularies, let's add a #
            # before each character
            self._data[word].append(map(lambda char: u'#' + char, def_))
            self._meta_data[word] = {"sourceDictionary": "spelling"}
        self.save()


    def add_from_lowercase_definitions(self, vocab):
        """Add definitions of lowercase word to each word (concat)
        """
        added = 0
        no_def = 0
        for word in vocab.words:
            word_lower = word.lower()
            if word != word_lower:
                lower_defs = self._data.get(word_lower)
                # This can be quite slow. But this code will not be used
                # very often.
                if not lower_defs:
                    if lower_defs is None:
                        # This can happen when API just dies (then vocab has, dict doesnt)
                        logger.error("None def for " + word)
                        continue
                    no_def += 1
                    logger.warning("No defs for " + str(word_lower) + "," + str(word))
                else:
                    # Note: often empty, like Zeus -> zeus
                    for def_ in lower_defs:
                        if not def_ in self._data[word]:
                            added += 1
                            self._data[word].append(def_)

        logger.info("No def for {}".format(no_def))
        logger.info("Added {} new defs in add_from_lowercase_definitions".format(added))
        self.save()


    def add_from_lemma_definitions(self, vocab, try_lower=False):
        """Add lemma definitions for non-lemmas.

        This code covers the following scenario: supposed a dictionary is crawled,
        but only for word lemmas.

        """
        lemmatizer = nltk.WordNetLemmatizer()
        added = 0
        for word in vocab.words:
            word_list = [word, word.lower()] if try_lower else [word]

            for word_to_lemma in word_list:
                try:
                    for part_of_speech in ['a', 's', 'r', 'n', 'v']:
                        lemma = lemmatizer.lemmatize(word_to_lemma, part_of_speech)
                        lemma_defs = self._data.get(lemma)
                        if lemma != word and lemma_defs:
                            # This can be quite slow. But this code will not be used
                            # very often.
                            for def_ in lemma_defs:
                                if not def_ in self._data[word]:
                                    added += 1
                                    self._data[word].append(def_)
                except:
                    logger.error("lemmatizer crashed on {}".format(word))
                    logger.error(traceback.format_exc())
        logger.info("Added {} new defs in add_from_lemma_definitions".format(added))
        self.save()

    def add_dictname_to_defs(self, vocab):
        """Add dict name in front of each def"""
        # TODO(kudkudak): I will implement it after we have attention on defs +
        # we do shortlisting based on dict defs
        raise NotImplementedError()

    def crawl_lemmas(self, vocab):
        """Add Wordnet lemmas as definitions."""
        lemmatizer = nltk.WordNetLemmatizer()
        for word in vocab.words:
            definitions = []
            try:
                for part_of_speech in ['a', 's', 'r', 'n', 'v']:
                    lemma = lemmatizer.lemmatize(word, part_of_speech)
                    if lemma != word and not [lemma] in definitions:
                        definitions.append([lemma])
            except:
                logger.error("lemmatizer crashed on {}".format(word))
            if definitions:
                self._data[word] = definitions
        self.save()

    def crawl_lowercase(self, vocab):
        """Add Wordnet lemmas as definitions."""
        for word in vocab.words:
            self._data[word] = [[word.lower()]]
            self._meta_data[word.lower()] = {"sourceDictionary": "lowercase"}
        self.save()

    def crawl_wordnet(self, corenlp_url):
        corenlp = StanfordCoreNLP(corenlp_url)
        for i, word in enumerate(wordnet.words()):
            if word in self._data:
                logger.info('skip a known word {}'.format(word))
                continue
            self._data[word] = []
            for synset in wordnet.synsets(word):
                def_ = corenlp.tokenize(synset.definition())[0]
                self._data[word].append(def_)
            if i % 10000 == 0:
                self.save()
        self.save()

    def crawl_wordnik(self, vocab, api_key, corenlp_url,
                      call_quota=15000, crawl_also_lowercase=False, crawl_also_lemma=False):

        """Download and preprocess definitions from Wordnik.

        vocab
            Vocabulary for which the definitions should be found.
        api_key
            The API key to use in communications with Wordnik.
        call_quota
            Maximum number of calls per hour.
        crawl_also_lowercase
            If true will add lowercase version of each word to crawl list
        crawl_also_lemma
            If true will also crawl lemma versions of words
            WARNING: Lemma of Cat is Cat! So if you want to have definition of "cat"
            you have to also pass crawl_also_lowercase!

        """
        corenlp = StanfordCoreNLP(corenlp_url)

        self._remaining_calls = call_quota
        self._last_saved = 0

        client = swagger.ApiClient(
            api_key, 'https://api.wordnik.com/v4')
        self._word_api = WordApi.WordApi(client)
        self._account_api = AccountApi.AccountApi(client)

        words = list(vocab.words)

        # Note(kudkudak): for SNLI it adds 3k words
        if crawl_also_lowercase:
            words_set = set(words)  # For efficiency

            logger.info("Adding lowercase words to crawl")
            lowercased = []
            for w in words:
                if w.lower() not in words_set:
                    lowercased.append(w.lower())
            logger.info("Crawling additional {} words".format(len(lowercased)))
            words.extend(sorted(lowercased))

        # Note(kudkudak): for SNLI it adds 2k words, so we can expect
        # like sabotage,sabotaging
        # Note that lemma crawling is *after* lowercasing
        if crawl_also_lemma:
            words_set = set(words) # For efficiency

            logger.info("Adding lemmatized vrsions to crawl")
            lemmas = []
            original = []
            lemmatizer = nltk.WordNetLemmatizer()
            for w in words:
                if isinstance(w, str):
                    w = w.decode('utf-8')

                for part_of_speech in ['a', 's', 'r', 'n', 'v']:
                    lemma = lemmatizer.lemmatize(w, part_of_speech)
                    if lemma not in words_set:
                        lemmas.append(lemma)
                        original.append(w)
            logger.info("Crawling additional {} words".format(len(lemmas)))
            for id in numpy.random.choice(len(lemmas), 100):
                logger.info("Example:" + lemmas[id] + "," + original[id])
                words.extend(sorted(lemmas))

        # Here, for now, we don't do any stemming or lemmatization.
        # Stemming is useless because the dictionary is not indexed with
        # lemmas, not stems. Lemmatizers, on the other hand, can not be
        # fully trusted when it comes to unknown words.
        for word in words:
            if isinstance(word, str):
                word = word.decode('utf-8')

            if word in self._data:
                logger.debug(u"a known word {}, skip".format(word))
                continue

            if self._last_saved >= _SAVE_EVERY_CALLS:
                self.save()
                self._last_saved = 0

            # 100 is a safery margin, I don't want to DDoS Wordnik :)
            if self._remaining_calls < _MIN_REMAINING_CALLS:
                self._wait_until_quota_reset()
            try:
                if isinstance(word, str):
                    word = word.decode('utf-8')
                # NOTE(kudkudak): We fetch all dictionaries, but retrieval can filter them based on meta info
                definitions = self._word_api.getDefinitions(word)
            except Exception:
                logger.error(u"error during fetching '{}'".format(word))
                logger.error(traceback.format_exc())
                continue
            self._remaining_calls -= 1
            self._last_saved += 1

            if not definitions:
                definitions = []
            self._data[word] = []
            for def_ in definitions:
                try:
                    # seems like definition text can be both str and unicode
                    text = def_.text
                    if def_.text is None:
                        continue
                    if isinstance(text, str):
                        text = text.decode('utf-8')
                    tokenized_def = corenlp.tokenize(text)[0]
                    self._data[word].append(tokenized_def)
                    # Note(kudkudak): I don't think there is much more useful meta data for us
                    # Note(kudkudak): This might seem strange, but I am afraid this is most robust (least bug prone)
                    # way of storing meta data that doens't require rewriting dict storage format
                    self._meta_data[" ".join(tokenized_def)] = {"sourceDictionary": def_.sourceDictionary}
                except Exception:
                    logger.error("error during tokenizing '{}'".format(text))
                    logger.error(traceback.format_exc())
            logger.debug(u"definitions for '{}' fetched {} remaining".format(word, self._remaining_calls))
        self.save()
        self._last_saved = 0

    def num_entries(self):
        return len(self._data)

    def get_definitions(self, key):
        return self._data.get(key, [])
dictlearn/util.py
def configure_logger(name = __name__,
                     console_logging_level = logging.INFO,
                     file_logging_level = logging.INFO,
                     log_file = None,
                     redirect_stdout=False,
                     redirect_stderr=False):
    """
    Configures logger
    :param name: logger name (default=module name, __name__)
    :param console_logging_level: level of logging to console (stdout), None = no logging
    :param file_logging_level: level of logging to log file, None = no logging
    :param log_file: path to log file (required if file_logging_level not None)
    :return instance of Logger class
    """
    if console_logging_level is None and file_logging_level is None:
        return # no logging

    if isinstance(console_logging_level, (str, unicode)):
        console_logging_level = parse_logging_level(console_logging_level)

    if isinstance(file_logging_level, (str, unicode)):
        file_logging_level = parse_logging_level(file_logging_level)

    if not os.path.exists(os.path.dirname(log_file)):
        os.makedirs(os.path.dirname(log_file))

    logger = logging.getLogger(name)
    logger.handlers = []
    logger.setLevel(logging.DEBUG)
    format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")

    if console_logging_level is not None:
        ch = logging.StreamHandler(sys.stdout)
        ch.setFormatter(format)
        ch.setLevel(console_logging_level)
        logger.addHandler(ch)

    if file_logging_level is not None:
        if log_file is None:
            raise ValueError("If file logging enabled, log_file path is required")
        fh = handlers.RotatingFileHandler(log_file, maxBytes=(1048576*5), backupCount=7)
        fh.setFormatter(format)
        logger.addHandler(fh)

    logger.info("Logging configured!")

    if redirect_stderr:
        sys.stderr = LoggerWriter(logger.warning)
    if redirect_stdout:
        sys.stdout = LoggerWriter(logger.info)

    return logger
dictlearn/extensions.py
class DumpCSVSummaries(SimpleExtension):
    def __init__(self, save_path, **kwargs):
        self._save_path = save_path

        if not os.path.exists(os.path.join(self._save_path, "logs.csv")):
            self._current_log = defaultdict(list)
        else:
            self._current_log = pd.read_csv(os.path.join(self._save_path, "logs.csv"))
            self._current_log = {col: list(self._current_log[col].values) for col in self._current_log.columns}
            logging.warning("Loaded {} columns and {} rows from logs.csv".format(len(self._current_log), len(self._current_log.values()[0])))

        super(DumpCSVSummaries, self).__init__(**kwargs)

    def do(self, *args, **kwargs):
        for key, value in self.main_loop.log.current_row.items():
            try:
                float_value = float(value)
            except:
                continue

            if not key.startswith("val") and not key.startswith("train") and not key.startswith("test"):
                key = "train_" + key

            if key not in self._current_log:
                self._current_log[key] = []

            self._current_log[key].append(float_value)

        # Make sure all logs have same length (for csv serialization)
        max_len = max([len(v) for v in self._current_log.values()])
        for k in self._current_log:
            if len(self._current_log[k]) != max_len:
                self._current_log[k] += [self._current_log[k][-1] for _ in range(max_len - len(self._current_log[k]))]

        pd.DataFrame(self._current_log).to_csv(os.path.join(self._save_path, "logs.csv"))
dictlearn/extensions.py
def construct_dict_embedder(theano_fnc, vocab, retrieval):
    """
    Parameters
    ----------
    theano_fnc: theano.Function
        (batch_size, seq_len) -> (batch_size, seq_len, word_dim)

    vocab: Vocabulary
        Vocabulary instance

    Returns
    -------
        Python function: (batch_size, ) -> (batch_size, word_dim)
    """

    def _embedder(word_list):
        word_ids = vocab.encode(word_list)
        word_ids = np.array(word_ids)
        word_ids = word_ids.reshape((-1, 1)) # Just to adhere to theano.Function, whatever
        def_array, def_mask, def_map = retrieval.retrieve_and_pad(np.array(word_list).reshape(-1, 1))
        word_vectors = theano_fnc(word_ids, def_array, def_mask, def_map)
        word_vectors = word_vectors.reshape((len(word_list), -1))
        return word_vectors

    return _embedder
dictlearn/extensions.py
class LoadNoUnpickling(Load):
    """Like `Load` but without unpickling.

    Avoids unpiclkling the main loop by assuming that the log
    and the iteration state were saved separately.

    """

    def load_to(self, main_loop):
        with open(self.path, "rb") as source:
            main_loop.model.set_parameter_values(load_parameters(source))
            if self.load_iteration_state:
                main_loop.iteration_state = load(source, name='iteration_state')
            if self.load_log:
                main_loop.log = load(source, name='log')
dictlearn/nli_simple_model.py
class NLISimple(Initializable):
    def __init__(self, mlp_dim, translate_dim, emb_dim, vocab, num_input_words=-1,
            num_input_def_words=-1, dropout=0.2, encoder="sum",
            n_layers=3, translate_after_emb=True, bn=True,

            # Dict lookup kwargs
            retrieval=None, reader_type="rnn", compose_type="sum", def_dim=300,
            disregard_word_embeddings=False, combiner_dropout=1.0, combiner_bn=False,
            combiner_dropout_type="regular", share_def_lookup=False, exclude_top_k=-1,
            combiner_reader_translate=True, def_vocab=None, def_emb_dim=-1,
            combiner_gating="none", def_emb_translate_dim=-1,
            combiner_shortcut=False,
            # Others
            **kwargs):
    def get_embeddings_lookups(self):
    def get_def_embeddings_lookups(self):
    def set_def_embeddings(self, embeddings):
    def set_embeddings(self, embeddings):
    def apply(self, application_call,
            s1_preunk, s1_mask, s2_preunk, s2_mask, def_mask=None, defs=None, s1_def_map=None,
            s2_def_map=None, train_phase=True):
dictlearn/extensions.py
class SimilarityWordEmbeddingEval(SimpleExtension):
    """
    Parameters
    ----------

    embedder: function: word -> vector
    """

    def __init__(self, embedder, prefix="", **kwargs):
        try:
            from web.datasets.similarity import fetch_MEN, fetch_WS353, fetch_SimLex999, fetch_RW
        except ImportError:
            raise RuntimeError("Please install web (https://github.com/kudkudak/word-embeddings-benchmarks)")

        self._embedder = embedder
        self._prefix = prefix

        # Define tasks
        logger.info("Downloading benchmark data")
        tasks = { # TODO: Pick a bit better tasks
            "MEN": fetch_MEN(),
            "WS353": fetch_WS353(),
            "SIMLEX999": fetch_SimLex999(),
            "RW": fetch_RW()
        }

        # Print sample data
        for name, data in iteritems(tasks):
            logger.info(
            "Sample data from {}: pair \"{}\" and \"{}\" is assigned score {}".format(name, data.X[0][0], data.X[0][1],
                data.y[0]))

        logger.info("Checking embedder for " + prefix)
        logger.info(embedder(["love"])[0, 0:5]) # Test embedder

        self._tasks = tasks

        super(SimilarityWordEmbeddingEval, self).__init__(**kwargs)

    def __getstate__(self):
        dict_ = dict(self.__dict__)
        if '_embedder' in dict_:
            del dict_['_embedder']
        if '_tasks' in dict_:
            del dict_['_tasks']
        return dict_

    def add_records(self, log, record_tuples):
        """Helper function to add monitoring records to the log."""
        for name, value in record_tuples:
            if not name:
                raise ValueError("monitor variable without name")
            log.current_row[name] = value

    def do(self, *args, **kwargs):
        # Embedd
        all_words = []
        all_words_vectors = []

        # TODO: Do it at once?
        for task in self._tasks:
            for row in self._tasks[task].X:
                for w in row:
                    all_words.append(w)
                    all_words_vectors.append(self._embedder([w]))
        W = dict(zip(np.array(all_words).reshape((-1,)), all_words_vectors))

        # Calculate results using helper function
        record_items = []
        for name, data in iteritems(self._tasks):
            eval = evaluate_similarity(W, data.X, data.y)
            record_items.append((self._prefix + "_" + name, eval))

        self.add_records(self.main_loop.log, record_items)
dictlearn/extensions.py
class PrintMessage(SimpleExtension):
    """Prints log messages to the screen."""
    def __init__(self, msg, **kwargs):
        self._msg = msg
        kwargs.setdefault("before_first_epoch", True)
        kwargs.setdefault("on_resumption", True)
        kwargs.setdefault("after_training", True)
        kwargs.setdefault("after_epoch", True)
        kwargs.setdefault("on_interrupt", True)
        super(PrintMessage, self).__init__(**kwargs)

    def do(self, which_callback, *args):
        print(self._msg)
dictlearn/extensions.py
class RetrievalPrintStats(SimpleExtension):
    """
    Prints statistics about Retrieval object
    """

    def __init__(self, retrieval, **kwargs):
        self._retrieval = retrieval
        super(RetrievalPrintStats, self).__init__(**kwargs)

    def __getstate__(self):
        dict_ = dict(self.__dict__)
        if '_retrieval' in dict_:
            del dict_['_retrieval']
        return dict_

    def add_records(self, log, record_tuples):
        """Helper function to add monitoring records to the log."""
        for name, value in record_tuples:
            if not name:
                raise ValueError("monitor variable without name")
            log.current_row[name] = value

    def do(self, *args, **kwargs):
        if self._retrieval is not None:
            d = self._retrieval._debug_info
            record_tuples = []
            record_tuples.append(
                ("retrieval_distinct_mis_ratio",
                 d['N_missed_distinct_words'] / max(1, float(d['N_distinct_words']))))
            record_tuples.append(("retrieval_N_words", d['N_words']))
            record_tuples.append(("retrieval_N_excluded_words", d['N_excluded_words']))
            record_tuples.append(("retrieval_N_distinct_words", d['N_distinct_words']))
            record_tuples.append(("retrieval_N_queried_words", d['N_queried_words']))
            record_tuples.append(("retrieval_mis_ratio",
                                  d['N_missed_words']
                                  /  max(1, float(d['N_queried_words']))))
            record_tuples.append(("retrieval_drop_def_ratio",
                                  d['N_dropped_def'] /  max(1, float(d['N_def']))))
            if len(d['missed_word_sample']) >= 20:
                record_tuples.append(("retrieval_missed_word_sample",
                    numpy.random.choice(d['missed_word_sample'], 20, replace=False)))
            self.add_records(self.main_loop.log, record_tuples)
dictlearn/extensions.py
class DumpTensorflowSummaries(SimpleExtension):
    def __init__(self, save_path, **kwargs):
        self._save_path = save_path
        super(DumpTensorflowSummaries, self).__init__(**kwargs)

    @property
    def file_writer(self):
        if not hasattr(self, '_file_writer'):
            self._file_writer = tensorflow.summary.FileWriter(
                self._save_path, flush_secs=10.)
        return self._file_writer

    def __getstate__(self):
        # FileWriter from TensorFlow is not picklable
        dict_ = self.__dict__
        if '_file_writer' in dict_:
            del dict_['_file_writer']
        return dict_

    def do(self, *args, **kwargs):
        summary = tensorflow.Summary()
        for key, value in self.main_loop.log.current_row.items():
            try:
                float_value = float(value)
                value = summary.value.add()
                value.tag = key
                value.simple_value = float_value
            except:
                pass
        self.file_writer.add_summary(
            summary, self.main_loop.log.status['iterations_done'])
dictlearn/nli_simple_model.py
class NLISimple(Initializable):
    """
    Simple model based on https://github.com/Smerity/keras_snl
    """

    def __init__(self, mlp_dim, translate_dim, emb_dim, vocab, num_input_words=-1,
            num_input_def_words=-1, dropout=0.2, encoder="sum",
            n_layers=3, translate_after_emb=True, bn=True,

            # Dict lookup kwargs
            retrieval=None, reader_type="rnn", compose_type="sum", def_dim=300,
            disregard_word_embeddings=False, combiner_dropout=1.0, combiner_bn=False,
            combiner_dropout_type="regular", share_def_lookup=False, exclude_top_k=-1,
            combiner_reader_translate=True, def_vocab=None, def_emb_dim=-1,
            combiner_gating="none", def_emb_translate_dim=-1,
            combiner_shortcut=False,
            # Others
            **kwargs):

        if def_emb_dim <= 0:
            logger.info("Assuming def_emb_dim=emb_dim")
            def_emb_dim = emb_dim

        def_emb_translate_dim = def_emb_translate_dim if def_emb_translate_dim > 0 else def_emb_dim

        self._vocab = vocab
        self._bn = bn
        self._encoder = encoder
        self._dropout = dropout
        self._retrieval = retrieval
        self._only_def = disregard_word_embeddings
        self._num_input_def_words = num_input_def_words
        self._translate_after_emb = translate_after_emb

        if reader_type not in {"rnn", "mean"}:
            raise NotImplementedError("Not implemented " + reader_type)

        if num_input_words > 0:
            logger.info("Restricting vocab to " + str(num_input_words))
            self._num_input_words = num_input_words
        else:
            logger.info("Vocab set to " + str(vocab.size()))
            self._num_input_words = vocab.size()

        children = []

        if not disregard_word_embeddings:
            self._lookup = LookupTable(self._num_input_words, emb_dim)
            children.append(self._lookup)

        if retrieval:
            if share_def_lookup:
                if emb_dim != def_emb_dim:
                    self._translate_pre_def2 = Linear(input_dim=emb_dim, output_dim=def_emb_dim)
                    children.append(self._translate_pre_def2)
                    def_lookup = Sequence([self._lookup, self._translate_pre_def2])
                else:
                    def_lookup = self._lookup
            else:
                def_lookup = None

            if reader_type== "rnn":
                self._def_reader = LSTMReadDefinitions(num_input_words=self._num_input_def_words,
                    dim=def_emb_translate_dim,
                    emb_dim=def_emb_dim, vocab=def_vocab, lookup=def_lookup)
            elif reader_type == "mean":
                if combiner_reader_translate:
                    logger.warning("Translate in MeanPoolReadDefinitions is redundant")
                self._def_reader = MeanPoolReadDefinitions(num_input_words=self._num_input_def_words,
                    translate=combiner_reader_translate,
                    lookup=def_lookup, dim=def_emb_translate_dim,
                    emb_dim=def_emb_dim, vocab=def_vocab)

            self._combiner = MeanPoolCombiner(dim=def_dim, emb_dim=def_emb_translate_dim,
                dropout=combiner_dropout, dropout_type=combiner_dropout_type,
                def_word_gating=combiner_gating,
                shortcut_unk_and_excluded=combiner_shortcut, num_input_words=num_input_words,
                exclude_top_k=exclude_top_k, vocab=vocab,
                compose_type=compose_type)
            children.extend([self._def_reader, self._combiner])

            if self._encoder == "rnn":
                self._rnn_fork = Linear(input_dim=def_emb_dim, output_dim=4 * translate_dim)
                # TODO(kudkudak): Better LSTM weight init
                self._rnn_encoder = LSTM(dim=translate_dim, name='LSTM_encoder')
                children.append(self._rnn_fork)
                children.append(self._rnn_encoder)
            elif self._encoder == "sum":
                pass
            else:
                raise NotImplementedError("Not implemented encoder")

            if def_emb_translate_dim != emb_dim:
                self._translate_pre_def = Linear(input_dim=emb_dim, output_dim=def_emb_translate_dim)
                children.append(self._translate_pre_def)
            else:
                self._translate_pre_def = None

            if self._translate_after_emb:
                self._translation = Linear(input_dim=def_dim, output_dim=translate_dim)
                self._translation_act = Rectifier()
                children.append(self._translation)
                children.append(self._translation_act)
        else:
            if self._encoder == "rnn":
                self._translation = Linear(input_dim=emb_dim, output_dim=4 * translate_dim)
                self._rnn_fork = self._translation
                self._rnn_encoder = LSTM(dim=translate_dim, name='LSTM_encoder')
                children.append(self._rnn_encoder)
                children.append(self._translation)
            elif self._encoder == "sum":
                self._translation = Linear(input_dim=emb_dim, output_dim=translate_dim,
                    weights_init=GlorotUniform(), biases_init=Constant(0))
                self._translation_act = Rectifier()
                children.append(self._translation)
                children.append(self._translation_act)
            else:
                raise NotImplementedError("Not implemented encoder")



        if self._bn:
            self._hyp_bn = BatchNormalization(input_dim=translate_dim, name="hyp_bn", conserve_memory=False)
            self._prem_bn = BatchNormalization(input_dim=translate_dim, name="prem_bn", conserve_memory=False)
            children += [self._hyp_bn, self._prem_bn]

        self._mlp = []
        current_dim = 2 * translate_dim  # Joint
        for i in range(n_layers):
            rect = Rectifier()
            dense = Linear(input_dim=current_dim, output_dim=mlp_dim,
                name="MLP_layer_" + str(i), \
                weights_init=GlorotUniform(), \
                biases_init=Constant(0))
            current_dim = mlp_dim
            if self._bn:
                bn = BatchNormalization(input_dim=current_dim, name="BN_" + str(i), conserve_memory=False)
                children.append(bn)
            else:
                bn = None

            children += [dense, rect] #TODO: Strange place to put ReLU
            self._mlp.append([dense, rect, bn])

        self._pred = MLP([Softmax()], [current_dim, 3])
        children.append(self._pred)

        super(NLISimple, self).__init__(children=children, **kwargs)

    def get_embeddings_lookups(self):
        return [self._lookup]

    def get_def_embeddings_lookups(self):
        return [self._def_reader._def_lookup]

    def set_def_embeddings(self, embeddings):
        self._def_reader._def_lookup.parameters[0].set_value(embeddings.astype(theano.config.floatX))

    def set_embeddings(self, embeddings):
        self._lookup.parameters[0].set_value(embeddings.astype(theano.config.floatX))

    @application
    def apply(self, application_call,
            s1_preunk, s1_mask, s2_preunk, s2_mask, def_mask=None, defs=None, s1_def_map=None,
            s2_def_map=None, train_phase=True):
        # Shortlist words (sometimes we want smaller vocab, especially when dict is small)
        s1 = (tensor.lt(s1_preunk, self._num_input_words) * s1_preunk
              + tensor.ge(s1_preunk, self._num_input_words) * self._vocab.unk)
        s2 = (tensor.lt(s2_preunk, self._num_input_words) * s2_preunk
              + tensor.ge(s2_preunk, self._num_input_words) * self._vocab.unk)

        # Embeddings
        s1_emb = self._lookup.apply(s1)
        s2_emb = self._lookup.apply(s2)

        application_call.add_auxiliary_variable(
            1 * s1_emb,
            name='s1_word_embeddings')

        if self._retrieval is not None:
            assert defs is not None

            if self._translate_pre_def:
                logger.info("Translate pre def")
                s1_emb = s1_emb.reshape((s1_emb.shape[0] * s1_emb.shape[1], s1_emb.shape[2]))
                s2_emb = s2_emb.reshape((s2_emb.shape[0] * s2_emb.shape[1], s2_emb.shape[2]))
                s1_emb = self._translate_pre_def.apply(s1_emb)
                s2_emb = self._translate_pre_def.apply(s2_emb)
                s1_emb = s1_emb.reshape((s1_preunk.shape[0], s1_preunk.shape[1], -1))
                s2_emb = s2_emb.reshape((s2_preunk.shape[0], s2_preunk.shape[1], -1))

            def_embs = self._def_reader.apply(defs, def_mask)

            s1_transl = self._combiner.apply(
                s1_emb, s1_mask,
                def_embs, s1_def_map, word_ids=s1, train_phase=train_phase, call_name="s1")

            s2_transl = self._combiner.apply(
                s2_emb, s2_mask,
                def_embs, s2_def_map, word_ids=s2, train_phase=train_phase, call_name="s2")

            if self._translate_after_emb:
                # Note: for some reader/combiner it can be redundant, but let's keep it
                # often it will not be redundant (ex: LSTMDefReader)
                s1_transl = s1_transl.reshape((s1_transl.shape[0] * s1_transl.shape[1], s1_transl.shape[2]))
                s2_transl = s2_transl.reshape((s2_transl.shape[0] * s2_transl.shape[1], s2_transl.shape[2]))
                s1_transl = self._translation.apply(s1_transl)
                s2_transl = self._translation.apply(s2_transl)
                s1_transl = self._translation_act.apply(s1_transl)
                s2_transl = self._translation_act.apply(s2_transl)
                s1_transl = s1_transl.reshape((s1_emb.shape[0], s1_emb.shape[1], -1))
                s2_transl = s2_transl.reshape((s2_emb.shape[0], s2_emb.shape[1], -1))
        else:
            # Translate. Crucial for recovering useful information from embeddings
            s1_emb_flatten = s1_emb.reshape((s1_emb.shape[0] * s1_emb.shape[1], s1_emb.shape[2]))
            s2_emb_flatten = s2_emb.reshape((s2_emb.shape[0] * s2_emb.shape[1], s2_emb.shape[2]))

            if self._translate_after_emb:
                # Note: redundant for trainable (from scratch) embeddings
                s1_transl = self._translation.apply(s1_emb_flatten)
                s2_transl = self._translation.apply(s2_emb_flatten)
                s1_transl = self._translation_act.apply(s1_transl)
                s2_transl = self._translation_act.apply(s2_transl)
                s1_transl = s1_transl.reshape((s1_emb.shape[0], s1_emb.shape[1], -1))
                s2_transl = s2_transl.reshape((s2_emb.shape[0], s2_emb.shape[1], -1))
                application_call.add_auxiliary_variable(
                    1*s1_transl,
                    name='s1_translated_word_embeddings')
                assert s1_transl.ndim == 3
            else:
                s1_transl = s1_emb_flatten
                s2_transl = s2_emb_flatten

        if self._encoder == "rnn":
            s1_transl = self._rnn_fork.apply(s1_transl)
            s2_transl = self._rnn_fork.apply(s2_transl)

        assert s1_transl.ndim == s2_transl.ndim == 3

        # Construct entailment embedding
        if self._encoder == "sum":
            s1_emb_mask = s1_mask.dimshuffle((0, 1, "x"))
            s2_emb_mask = s2_mask.dimshuffle((0, 1, "x"))

            # TODO: This should be mean, might make learning harder otherwise
            prem = (s1_emb_mask * s1_transl).sum(axis=1)
            hyp = (s2_emb_mask * s2_transl).sum(axis=1)
        else:
            prem = self._rnn_encoder.apply(s1_transl.transpose(1, 0, 2), mask=s1_mask.transpose(1, 0))[0][-1]
            hyp = self._rnn_encoder.apply(s2_transl.transpose(1, 0, 2), mask=s2_mask.transpose(1, 0))[0][-1]

        if self._bn:
            prem = self._prem_bn.apply(prem)
            hyp = self._hyp_bn.apply(hyp)

        joint = T.concatenate([prem, hyp], axis=1)
        joint.name = "MLP_input"

        if train_phase:
            joint = apply_dropout(joint, drop_prob=self._dropout)

        # MLP
        for block in self._mlp:
            dense, relu, bn = block
            joint = dense.apply(joint)
            joint = relu.apply(joint)

            if train_phase:
                joint = apply_dropout(joint, drop_prob=self._dropout)

            if self._bn:
                joint = bn.apply(joint)

        return self._pred.apply(joint)
dictlearn/nli_esim_model.py
class ESIM(Initializable):
    """
    ESIM model based on https://github.com/NYU-MLL/multiNLI/blob/master/python/models/esim.py
    """

    # seq_length, emb_dim, hidden_dim
    def __init__(self, dim, emb_dim, vocab, def_emb_translate_dim=-1, def_dim=-1, encoder='bilstm', bn=True,
            def_reader=None, def_combiner=None, dropout=0.5, num_input_words=-1,
            # Others
            **kwargs):

        self._dropout = dropout
        self._vocab = vocab
        self._emb_dim = emb_dim
        self._def_reader = def_reader
        self._def_combiner = def_combiner

        if encoder != 'bilstm':
            raise NotImplementedError()

        if def_emb_translate_dim < 0:
            self.def_emb_translate_dim = emb_dim
        else:
            self.def_emb_translate_dim = def_emb_translate_dim

        if def_dim < 0:
            self._def_dim = emb_dim
        else:
            self._def_dim = def_dim

        if num_input_words > 0:
            logger.info("Restricting vocab to " + str(num_input_words))
            self._num_input_words = num_input_words
        else:
            self._num_input_words = vocab.size()

        children = []

        if self.def_emb_translate_dim != self._emb_dim:
            self._translate_pre_def = Linear(input_dim=emb_dim, output_dim=def_emb_translate_dim)
            children.append(self._translate_pre_def)
        else:
            self._translate_pre_def = None

        ## Embedding
        self._lookup = LookupTable(self._num_input_words, emb_dim, weights_init=GlorotUniform())
        children.append(self._lookup)

        if def_reader:
            self._final_emb_dim = self._def_dim
            self._def_reader = def_reader
            self._def_combiner = def_combiner
            children.extend([self._def_reader, self._def_combiner])
        else:
            self._final_emb_dim = self._emb_dim

        ## BiLSTM
        self._hyp_bidir_fork = Linear(self._def_dim if def_reader else self._emb_dim, 4 * dim, name='hyp_bidir_fork')
        self._hyp_bidir = Bidirectional(LSTM(dim), name='hyp_bidir')
        self._prem_bidir_fork = Linear(self._def_dim if def_reader else self._emb_dim, 4 * dim, name='prem_bidir_fork')
        self._prem_bidir = Bidirectional(LSTM(dim), name='prem_bidir')
        children.extend([self._hyp_bidir_fork, self._hyp_bidir])
        children.extend([self._prem_bidir, self._prem_bidir_fork])

        ## BiLSTM no. 2 (encoded attentioned embeddings)
        self._hyp_bidir_fork2 = Linear(8 * dim, 4 * dim, name='hyp_bidir_fork2')
        self._hyp_bidir2 = Bidirectional(LSTM(dim), name='hyp_bidir2')
        self._prem_bidir_fork2 = Linear(8 * dim, 4 * dim, name='prem_bidir_fork2')
        self._prem_bidir2 = Bidirectional(LSTM(dim), name='prem_bidir2')
        children.extend([self._hyp_bidir_fork2, self._hyp_bidir2])
        children.extend([self._prem_bidir2, self._prem_bidir_fork2])

        self._rnns = [self._prem_bidir2, self._hyp_bidir2, self._prem_bidir, self._hyp_bidir]

        ## MLP
        if bn:
            self._mlp = BatchNormalizedMLP([Tanh()], [8 * dim, dim], conserve_memory=False, name="mlp")
            self._pred = BatchNormalizedMLP([Softmax()], [dim, 3],  conserve_memory=False, name="pred_mlp")
        else:
            self._mlp = MLP([Tanh()], [8 * dim, dim], name="mlp")
            self._pred = MLP([Softmax()], [dim, 3], name="pred_mlp")

        children.append(self._mlp)
        children.append(self._pred)

        ## Softmax
        self._ndim_softmax = NDimensionalSoftmax()
        children.append(self._ndim_softmax)

        super(ESIM, self).__init__(children=children, **kwargs)

    def get_embeddings_lookups(self):
        return [self._lookup]

    def set_embeddings(self, embeddings):
        self._lookup.parameters[0].set_value(embeddings.astype(theano.config.floatX))

    def get_def_embeddings_lookups(self):
        return [self._def_reader._def_lookup]

    def set_def_embeddings(self, embeddings):
        self._def_reader._def_lookup.parameters[0].set_value(embeddings.astype(theano.config.floatX))

    @application
    def apply(self, application_call,
            s1_preunk, s1_mask, s2_preunk, s2_mask, def_mask=None,
            defs=None, s1_def_map=None, s2_def_map=None, train_phase=True):
        # Shortlist words (sometimes we want smaller vocab, especially when dict is small)
        s1 = (tensor.lt(s1_preunk, self._num_input_words) * s1_preunk
              + tensor.ge(s1_preunk, self._num_input_words) * self._vocab.unk)
        s2 = (tensor.lt(s2_preunk, self._num_input_words) * s2_preunk
              + tensor.ge(s2_preunk, self._num_input_words) * self._vocab.unk)

        ### Embed ###

        s1_emb = self._lookup.apply(s1)
        s2_emb = self._lookup.apply(s2)

        application_call.add_auxiliary_variable(
            1 * s1_emb,
            name='s1_word_embeddings')

        if self._def_reader:
            assert defs is not None

            def_embs = self._def_reader.apply(defs, def_mask)

            if self._translate_pre_def:
                logger.info("Translate pre def")
                s1_emb = s1_emb.reshape((s1_emb.shape[0] * s1_emb.shape[1], s1_emb.shape[2]))
                s2_emb = s2_emb.reshape((s2_emb.shape[0] * s2_emb.shape[1], s2_emb.shape[2]))
                s1_emb = self._translate_pre_def.apply(s1_emb)
                s2_emb = self._translate_pre_def.apply(s2_emb)
                s1_emb = s1_emb.reshape((s1_preunk.shape[0], s1_preunk.shape[1], -1))
                s2_emb = s2_emb.reshape((s2_preunk.shape[0], s2_preunk.shape[1], -1))

            s1_emb = self._def_combiner.apply(
                s1_emb, s1_mask,
                def_embs, s1_def_map, word_ids=s1, train_phase=train_phase, call_name="s1")

            s2_emb = self._def_combiner.apply(
                s2_emb, s2_mask,
                def_embs, s2_def_map, word_ids=s2, train_phase=train_phase, call_name="s2")
        else:
            if train_phase and self._dropout > 0:
                s1_emb = apply_dropout(s1_emb, drop_prob=self._dropout)
                s2_emb = apply_dropout(s2_emb, drop_prob=self._dropout)

        ### Encode ###

        # TODO: Share this bilstm?
        s1_bilstm, _ = self._prem_bidir.apply(flip01(self._prem_bidir_fork.apply(s1_emb)),
            mask=s1_mask.T) # (batch_size, n_seq, 2 * dim)
        s2_bilstm, _ = self._hyp_bidir.apply(flip01(self._hyp_bidir_fork.apply(s2_emb)),
            mask=s2_mask.T) # (batch_size, n_seq, 2 * dim)
        s1_bilstm = flip01(s1_bilstm)
        s2_bilstm = flip01(s2_bilstm)
        ### Attention ###

        # Compute E matrix (eq. 11)
        # E_ij = <s1[i], s2[j]>
        # each call computes E[i, :]
        def compute_e_row(s2_i, s1_bilstm, s1_mask):
            b_size = s1_bilstm.shape[0]
            # s2_i is (batch_size, emb_dim)
            # s1_bilstm is (batch_size, seq_len, emb_dim)
            # s1_mask is (batch_size, seq_len)
            # s2_i = s2_i.reshape((s2_i.shape[0], s2_i.shape[1], 1))
            s2_i = s2_i.reshape((b_size, s2_i.shape[1], 1))
            s2_i = T.repeat(s2_i, 2, axis=2)
            # s2_i is (batch_size, emb_dim, 2)
            assert s1_bilstm.ndim == 3
            assert s2_i.ndim == 3
            score = T.batched_dot(s1_bilstm, s2_i) # (batch_size, seq_len, 1)
            score = score[:, :, 0].reshape((b_size, -1)) # (batch_size, seq_len)
            return score # E[i, :]

        # NOTE: No point in masking here
        E, _ = theano.scan(compute_e_row, sequences=[s1_bilstm.transpose(1, 0, 2)],
            non_sequences=[s2_bilstm, s2_mask])
        # (seq_len, batch_size, seq_len)
        E = E.dimshuffle(1, 0, 2)
        assert E.ndim == 3

        s2s_att_weights = self._ndim_softmax.apply(E, extra_ndim=1)
        application_call.add_auxiliary_variable(
            s2s_att_weights.copy(), name='s2s_att_weights')

        # (batch_size, seq_len, seq_len)

        ### Compute tilde vectors (eq. 12 and 13) ###

        def compute_tilde_vector(e_i, s, s_mask):
            # e_i is (batch_size, seq_len)
            # s_mask is (batch_size, seq_len)
            # s_tilde_i = \sum e_ij b_j, (batch_size, seq_len)
            score = masked_softmax(e_i, s_mask, axis=1)
            score = score.dimshuffle(0, 1, "x")

            s_tilde_i = (score * (s * s_mask.dimshuffle(0, 1, "x"))).sum(axis=1)
            return s_tilde_i

        # (batch_size, seq_len, def_dim)
        s1_tilde, _ = theano.scan(compute_tilde_vector,
            sequences=[E.dimshuffle(1, 0, 2)], non_sequences=[s2_bilstm, s2_mask])
        s1_tilde = s1_tilde.dimshuffle(1, 0, 2)
        s2_tilde, _ = theano.scan(compute_tilde_vector,
            sequences=[E.dimshuffle(2, 0, 1)], non_sequences=[s1_bilstm, s1_mask])
        s2_tilde = s2_tilde.dimshuffle(1, 0, 2)

        ### Compose (eq. 14 and 15) ###

        # (batch_size, seq_len, 8 * dim)
        s1_comp = T.concatenate([s1_bilstm, s1_tilde, s1_bilstm - s1_tilde, s1_bilstm * s1_tilde], axis=2)
        s2_comp = T.concatenate([s2_bilstm, s2_tilde, s2_bilstm - s2_tilde, s2_bilstm * s2_tilde], axis=2)
        ### Encode (eq. 16 and 17) ###

        # (batch_size, seq_len, 8 * dim)
        # TODO: Share this bilstm?
        s1_comp_bilstm, _ = self._prem_bidir2.apply(self._prem_bidir_fork2.apply(flip01(s1_comp)), mask=s1_mask.T)  # (batch_size, n_seq, 2 * dim)
        s2_comp_bilstm, _ = self._hyp_bidir2.apply(self._hyp_bidir_fork2.apply(flip01(s2_comp)), mask=s2_mask.T)  # (batch_size, n_seq, 2 * dim)
        s1_comp_bilstm = flip01(s1_comp_bilstm)
        s2_comp_bilstm = flip01(s2_comp_bilstm)
        ### Pooling Layer ###

        s1_comp_bilstm_ave = (s1_mask.dimshuffle(0, 1, "x") * s1_comp_bilstm).sum(axis=1) \
                            / s1_mask.sum(axis=1).dimshuffle(0, "x")

        s1_comp_bilstm_max = T.max( ((1 - s1_mask.dimshuffle(0, 1, "x")) * -10000) + \
                                    (s1_mask.dimshuffle(0, 1, "x")) * s1_comp_bilstm, axis=1)

        s2_comp_bilstm_ave = (s2_mask.dimshuffle(0, 1, "x") * s2_comp_bilstm).sum(axis=1) \
                             / s2_mask.sum(axis=1).dimshuffle(0, "x")
        # (batch_size, dim)
        s2_comp_bilstm_max = T.max(((1 - s2_mask.dimshuffle(0, 1, "x")) * -10000) + \
                                   (s2_mask.dimshuffle(0, 1, "x")) * s2_comp_bilstm, axis=1)

        ### Final classifier ###

        # MLP layer
        # (batch_size, 8 * dim)
        m = T.concatenate([s1_comp_bilstm_ave, s1_comp_bilstm_max, s2_comp_bilstm_ave, s2_comp_bilstm_max], axis=1)
        pre_logits = self._mlp.apply(m)

        if train_phase:
            pre_logits = apply_dropout(pre_logits, drop_prob=self._dropout)

        # Get prediction
        self.logits = self._pred.apply(pre_logits)

        return self.logits


Question: import sys
import numpy as np
import matplotlib.pylab as plt
import os
import time
import atexit
import fuel
import signal
import pprint
import pandas as pd
import subprocess
import tqdm
import cPickle as pickle
import json
import numpy
import theano
import logging
from theano import tensor
from blocks.bricks.cost import MisclassificationRate
from blocks.bricks.bn import BatchNormalization
from blocks.filter import get_brick
from blocks.bricks import Linear, Sequence
from blocks.bricks.lookup import LookupTable
from blocks.bricks.cost import CategoricalCrossEntropy
from blocks.extensions import ProgressBar, Timestamp
from blocks.extensions.training import TrackTheBest
from dictlearn.extensions import (
    DumpTensorflowSummaries)
from blocks.extensions.predicates import OnLogRecord
from blocks.serialization import load_parameters
from blocks.initialization import Constant, Uniform
from dictlearn.vocab import Vocabulary
from dictlearn.inits import GlorotUniform
from dictlearn.extensions import LoadNoUnpickling
from theano import tensor as T
from blocks.algorithms import (
    GradientDescent, Adam)
from blocks.graph import ComputationGraph, apply_batch_normalization, get_batch_normalization_updates
from blocks.model import Model
from blocks.graph.bn import (
    apply_batch_normalization, get_batch_normalization_updates,
    batch_normalization)
from blocks.extensions import FinishAfter, Timing, Printing
from blocks.extensions.saveload import Load, Checkpoint
from blocks.extensions.monitoring import (DataStreamMonitoring,
                                          TrainingDataMonitoring)
from blocks.main_loop import MainLoop
from blocks.roles import WEIGHT
from blocks.filter import VariableFilter
from fuel.streams import ServerDataStream
from dictlearn.util import configure_logger
from dictlearn.extensions import StartFuelServer, DumpCSVSummaries, SimilarityWordEmbeddingEval, construct_embedder, \
    construct_dict_embedder, RetrievalPrintStats, PrintMessage
from dictlearn.data import SNLIData
from dictlearn.nli_simple_model import NLISimple
from dictlearn.nli_esim_model import ESIM
from dictlearn.retrieval import Retrieval, Dictionary
from dictlearn.nli_simple_model import LSTMReadDefinitions, MeanPoolReadDefinitions, MeanPoolCombiner
from blocks.serialization import secure_dump, dump_and_add_to_dump
from blocks.extensions import SimpleExtension
    from six import string_types
"""
Training loop for simple SNLI model that can use dict enchanced embeddings
"""



sys.path.append("..")









class OnLogStatusExceed(object):
    """Trigger a callback when a certain log record is found.

    Parameters
    ----------
    record_name : str
        The record name to check.

    """
    def __init__(self, record_name, max_val):
        self.record_name = record_name
        self.max_val = max_val

    def __call__(self, log):
        return bool(log.status.get(self.record_name, -numpy.inf) >= self.max_val)

    def __eq__(self, other):
        return (type(other) == type(self) and
                other.record_name == self.record_name)


# vocab defaults to data.vocab
# vocab_text defaults to vocab
# Vocab def defaults to vocab
def _initialize_simple_model_and_data(c):

    if c['vocab']:
        vocab = Vocabulary(c['vocab'])
    else:
        vocab = None
    # Load data

Answer:     data = SNLIData(path=c['data_path'], layout=c['layout'], vocab=vocab)