Collect feature Vocab objects from Field object .

def collect_feature_vocabs(fields, side):
    """
    Collect feature Vocab objects from Field object.
    """
    assert side in ['src', 'tgt']
    feature_vocabs = []
    for j in count():
        key = side + "_feat_" + str(j)
        if key not in fields:
            break
        feature_vocabs.append(fields[key].vocab)
    return feature_vocabs

def collect_feature_vocabs ( fields , side ) : assert side in [ 'src' , 'tgt' ] feature_vocabs = [ ] for j in count ( ) : key = side + "_feat_" + str ( j ) if key not in fields : break feature_vocabs . append ( fields [ key ] . vocab ) return feature_vocabs


Build src / tgt examples iterator from corpus files also extract number of features .

def build_dataset(fields, data_type, src_data_iter=None, src_path=None,
                  src_dir=None, tgt_data_iter=None, tgt_path=None,
                  src_seq_length=0, tgt_seq_length=0,
                  src_seq_length_trunc=0, tgt_seq_length_trunc=0,
                  dynamic_dict=True, sample_rate=0,
                  window_size=0, window_stride=0, window=None,
                  normalize_audio=True, use_filter_pred=True,
                  image_channel_size=3):
    """
    Build src/tgt examples iterator from corpus files, also extract
    number of features.
    """

    def _make_examples_nfeats_tpl(data_type, src_data_iter, src_path, src_dir,
                                  src_seq_length_trunc, sample_rate,
                                  window_size, window_stride,
                                  window, normalize_audio,
                                  image_channel_size=3):
        """
        Process the corpus into (example_dict iterator, num_feats) tuple
        on source side for different 'data_type'.
        """

        if data_type == 'text':
            src_examples_iter, num_src_feats = \
                TextDataset.make_text_examples_nfeats_tpl(
                    src_data_iter, src_path, src_seq_length_trunc, "src")

        elif data_type == 'img':
            src_examples_iter, num_src_feats = \
                ImageDataset.make_image_examples_nfeats_tpl(
                    src_data_iter, src_path, src_dir, image_channel_size)

        elif data_type == 'audio':
            if src_data_iter:
                raise ValueError("""Data iterator for AudioDataset isn't
                                    implemented""")

            if src_path is None:
                raise ValueError("AudioDataset requires a non None path")
            src_examples_iter, num_src_feats = \
                AudioDataset.make_audio_examples_nfeats_tpl(
                    src_path, src_dir, sample_rate,
                    window_size, window_stride, window,
                    normalize_audio)

        return src_examples_iter, num_src_feats

    src_examples_iter, num_src_feats = \
        _make_examples_nfeats_tpl(data_type, src_data_iter, src_path, src_dir,
                                  src_seq_length_trunc, sample_rate,
                                  window_size, window_stride,
                                  window, normalize_audio,
                                  image_channel_size=image_channel_size)

    # For all data types, the tgt side corpus is in form of text.
    tgt_examples_iter, num_tgt_feats = \
        TextDataset.make_text_examples_nfeats_tpl(
            tgt_data_iter, tgt_path, tgt_seq_length_trunc, "tgt")

    if data_type == 'text':
        dataset = TextDataset(fields, src_examples_iter, tgt_examples_iter,
                              num_src_feats, num_tgt_feats,
                              src_seq_length=src_seq_length,
                              tgt_seq_length=tgt_seq_length,
                              dynamic_dict=dynamic_dict,
                              use_filter_pred=use_filter_pred)

    elif data_type == 'img':
        dataset = ImageDataset(fields, src_examples_iter, tgt_examples_iter,
                               num_src_feats, num_tgt_feats,
                               tgt_seq_length=tgt_seq_length,
                               use_filter_pred=use_filter_pred,
                               image_channel_size=image_channel_size)

    elif data_type == 'audio':
        dataset = AudioDataset(fields, src_examples_iter, tgt_examples_iter,
                               num_src_feats, num_tgt_feats,
                               tgt_seq_length=tgt_seq_length,
                               sample_rate=sample_rate,
                               window_size=window_size,
                               window_stride=window_stride,
                               window=window,
                               normalize_audio=normalize_audio,
                               use_filter_pred=use_filter_pred)

    return dataset


Args : train_dataset_files : a list of train dataset pt file . fields ( dict ) : fields to build vocab for . data_type : text img or audio ? share_vocab ( bool ) : share source and target vocabulary? src_vocab_path ( string ) : Path to src vocabulary file . src_vocab_size ( int ) : size of the source vocabulary . src_words_min_frequency ( int ) : the minimum frequency needed to include a source word in the vocabulary . tgt_vocab_path ( string ) : Path to tgt vocabulary file . tgt_vocab_size ( int ) : size of the target vocabulary . tgt_words_min_frequency ( int ) : the minimum frequency needed to include a target word in the vocabulary .

def build_vocab(train_dataset_files, fields, data_type, share_vocab,
                src_vocab_path, src_vocab_size, src_words_min_frequency,
                tgt_vocab_path, tgt_vocab_size, tgt_words_min_frequency):
    """
    Args:
        train_dataset_files: a list of train dataset pt file.
        fields (dict): fields to build vocab for.
        data_type: "text", "img" or "audio"?
        share_vocab(bool): share source and target vocabulary?
        src_vocab_path(string): Path to src vocabulary file.
        src_vocab_size(int): size of the source vocabulary.
        src_words_min_frequency(int): the minimum frequency needed to
                include a source word in the vocabulary.
        tgt_vocab_path(string): Path to tgt vocabulary file.
        tgt_vocab_size(int): size of the target vocabulary.
        tgt_words_min_frequency(int): the minimum frequency needed to
                include a target word in the vocabulary.

    Returns:
        Dict of Fields
    """
    counter = {}

    # Prop src from field to get lower memory using when training with image
    if data_type == 'img':
        fields.pop("src")

    for k in fields:
        counter[k] = Counter()

    # Load vocabulary
    src_vocab = load_vocabulary(src_vocab_path, tag="source")
    tgt_vocab = load_vocabulary(tgt_vocab_path, tag="target")

    for index, path in enumerate(train_dataset_files):
        dataset = torch.load(path)
        logger.info(" * reloading %s." % path)
        for ex in dataset.examples:
            for k in fields:
                val = getattr(ex, k, None)
                if val is not None and not fields[k].sequential:
                    val = [val]
                elif k == 'src' and src_vocab:
                    val = [item for item in val if item in src_vocab]
                elif k == 'tgt' and tgt_vocab:
                    val = [item for item in val if item in tgt_vocab]
                counter[k].update(val)

        # Drop the none-using from memory but keep the last
        if (index < len(train_dataset_files) - 1):
            dataset.examples = None
            gc.collect()
            del dataset.examples
            gc.collect()
            del dataset
            gc.collect()

    _build_field_vocab(fields["tgt"], counter["tgt"],
                       max_size=tgt_vocab_size,
                       min_freq=tgt_words_min_frequency)
    logger.info(" * tgt vocab size: %d." % len(fields["tgt"].vocab))

    # All datasets have same num of n_tgt_features,
    # getting the last one is OK.
    for j in range(dataset.n_tgt_feats):
        key = "tgt_feat_" + str(j)
        _build_field_vocab(fields[key], counter[key])
        logger.info(" * %s vocab size: %d." % (key,
                                               len(fields[key].vocab)))

    if data_type == 'text':
        _build_field_vocab(fields["src"], counter["src"],
                           max_size=src_vocab_size,
                           min_freq=src_words_min_frequency)
        logger.info(" * src vocab size: %d." % len(fields["src"].vocab))

        # All datasets have same num of n_src_features,
        # getting the last one is OK.
        for j in range(dataset.n_src_feats):
            key = "src_feat_" + str(j)
            _build_field_vocab(fields[key], counter[key])
            logger.info(" * %s vocab size: %d." %
                        (key, len(fields[key].vocab)))

        # Merge the input and output vocabularies.
        if share_vocab:
            # `tgt_vocab_size` is ignored when sharing vocabularies
            logger.info(" * merging src and tgt vocab...")
            merged_vocab = merge_vocabs(
                [fields["src"].vocab, fields["tgt"].vocab],
                vocab_size=src_vocab_size)
            fields["src"].vocab = merged_vocab
            fields["tgt"].vocab = merged_vocab

    return fields


Loads a vocabulary from the given path . : param vocabulary_path : path to load vocabulary from : param tag : tag for vocabulary ( only used for logging ) : return : vocabulary or None if path is null

def load_vocabulary(vocabulary_path, tag=""):
    """
    Loads a vocabulary from the given path.
    :param vocabulary_path: path to load vocabulary from
    :param tag: tag for vocabulary (only used for logging)
    :return: vocabulary or None if path is null
    """
    vocabulary = None
    if vocabulary_path:
        vocabulary = set([])
        logger.info("Loading {} vocabulary from {}".format(tag,
                                                           vocabulary_path))

        if not os.path.exists(vocabulary_path):
            raise RuntimeError(
                "{} vocabulary not found at {}!".format(tag, vocabulary_path))
        else:
            with open(vocabulary_path) as f:
                for line in f:
                    if len(line.strip()) == 0:
                        continue
                    word = line.strip().split()[0]
                    vocabulary.add(word)
    return vocabulary

def
load_vocabulary
(
vocabulary_path
,
tag
=
""
)
:
vocabulary
=
if
vocabulary_path
:
vocabulary
=
set
(
[
]
)
logger
.
info
(
"Loading {} vocabulary from {}"
.
format
(
tag
,
vocabulary_path
)
)
if
not
os
.
path
.
exists
(
vocabulary_path
)
:
raise
RuntimeError
(
"{} vocabulary not found at {}!"
.
format
(
tag
,
vocabulary_path
)
)
else
:
with
open
(
vocabulary_path
)
as
f
:
for
line
in
f
:
if
len
(
line
.
strip
(
)
)
==
0
:
continue
word
=
line
.
strip
(
)
.
split
(
)
[
0
]
vocabulary
.
add
(
word
)
return
vocabulary


This returns user - defined train / validate data iterator for the trainer to iterate over . We implement simple ordered iterator strategy here but more sophisticated strategy like curriculum learning is ok too .

def build_dataset_iter(datasets, fields, opt, is_train=True):
    """
    This returns user-defined train/validate data iterator for the trainer
    to iterate over. We implement simple ordered iterator strategy here,
    but more sophisticated strategy like curriculum learning is ok too.
    """
    batch_size = opt.batch_size if is_train else opt.valid_batch_size
    if is_train and opt.batch_type == "tokens":
        def batch_size_fn(new, count, sofar):
            """
            In token batching scheme, the number of sequences is limited
            such that the total number of src/tgt tokens (including padding)
            in a batch <= batch_size
            """
            # Maintains the longest src and tgt length in the current batch
            global max_src_in_batch, max_tgt_in_batch
            # Reset current longest length at a new batch (count=1)
            if count == 1:
                max_src_in_batch = 0
                max_tgt_in_batch = 0
            # Src: <bos> w1 ... wN <eos>
            max_src_in_batch = max(max_src_in_batch, len(new.src) + 2)
            # Tgt: w1 ... wN <eos>
            max_tgt_in_batch = max(max_tgt_in_batch, len(new.tgt) + 1)
            src_elements = count * max_src_in_batch
            tgt_elements = count * max_tgt_in_batch
            return max(src_elements, tgt_elements)
    else:
        batch_size_fn = None

    if opt.gpu_ranks:
        device = "cuda"
    else:
        device = "cpu"

    return DatasetLazyIter(datasets, fields, batch_size, batch_size_fn,
                           device, is_train)


Dataset generator . Don t do extra stuff here like printing because they will be postponed to the first loading time .

def lazily_load_dataset(corpus_type, opt):
    """
    Dataset generator. Don't do extra stuff here, like printing,
    because they will be postponed to the first loading time.

    Args:
        corpus_type: 'train' or 'valid'
    Returns:
        A list of dataset, the dataset(s) are lazily loaded.
    """
    assert corpus_type in ["train", "valid"]

    def _lazy_dataset_loader(pt_file, corpus_type):
        dataset = torch.load(pt_file)
        logger.info('Loading %s dataset from %s, number of examples: %d' %
                    (corpus_type, pt_file, len(dataset)))
        return dataset

    # Sort the glob output by file name (by increasing indexes).
    pts = sorted(glob.glob(opt.data + '.' + corpus_type + '.[0-9]*.pt'))
    if pts:
        for pt in pts:
            yield _lazy_dataset_loader(pt, corpus_type)
    else:
        # Only one inputters.*Dataset, simple!
        pt = opt.data + '.' + corpus_type + '.pt'
        yield _lazy_dataset_loader(pt, corpus_type)


def create_batches(self):
        """ Create batches """
        if self.train:
            def _pool(data, random_shuffler):
                for p in torchtext.data.batch(data, self.batch_size * 100):
                    p_batch = torchtext.data.batch(
                        sorted(p, key=self.sort_key),
                        self.batch_size, self.batch_size_fn)
                    for b in random_shuffler(list(p_batch)):
                        yield b

            self.batches = _pool(self.data(), self.random_shuffler)
        else:
            self.batches = []
            for b in torchtext.data.batch(self.data(), self.batch_size,
                                          self.batch_size_fn):
                self.batches.append(sorted(b, key=self.sort_key))

def create_batches ( self ) : if self . train : def _pool ( data , random_shuffler ) : for p in torchtext . data . batch ( data , self . batch_size * 100 ) : p_batch = torchtext . data . batch ( sorted ( p , key = self . sort_key ) , self . batch_size , self . batch_size_fn ) for b in random_shuffler ( list ( p_batch ) ) : yield b self . batches = _pool ( self . data ( ) , self . random_shuffler ) else : self . batches = [ ] for b in torchtext . data . batch ( self . data ( ) , self . batch_size , self . batch_size_fn ) : self . batches . append ( sorted ( b , key = self . sort_key ) )


Read the config file and pre - / load the models

def start(self, config_file):
        """Read the config file and pre-/load the models
        """
        self.config_file = config_file
        with open(self.config_file) as f:
            self.confs = json.load(f)

        self.models_root = self.confs.get('models_root', './available_models')
        for i, conf in enumerate(self.confs["models"]):
            if "models" not in conf:
                if "model" in conf:
                    # backwards compatibility for confs
                    conf["models"] = [conf["model"]]
                else:
                    raise ValueError("""Incorrect config file: missing 'models'
                                        parameter for model #%d""" % i)
            kwargs = {'timeout': conf.get('timeout', None),
                      'load': conf.get('load', None),
                      'tokenizer_opt': conf.get('tokenizer', None),
                      'on_timeout': conf.get('on_timeout', None),
                      'model_root': conf.get('model_root', self.models_root)
                      }
            kwargs = {k: v for (k, v) in kwargs.items() if v is not None}
            model_id = conf.get("id", None)
            opt = conf["opt"]
            opt["models"] = conf["models"]
            self.preload_model(opt, model_id=model_id, **kwargs)


Clone a model model_id . Different options may be passed . If opt is None it will use the same set of options

def clone_model(self, model_id, opt, timeout=-1):
        """Clone a model `model_id`.
           Different options may be passed. If `opt` is None, it will use the
           same set of options
        """
        if model_id in self.models:
            if opt is None:
                opt = self.models[model_id].user_opt
            opt["models"] = self.models[model_id].opt.models
            return self.load_model(opt, timeout)
        else:
            raise ServerModelError("No such model '%s'" % str(model_id))

def
clone_model
(
self
,
model_id
,
opt
,
timeout
=
1
)
:
if
model_id
in
self
.
models
:
if
opt
is
:
opt
=
self
.
models
[
model_id
]
.
user_opt
opt
[
"models"
]
=
self
.
models
[
model_id
]
.
opt
.
models
return
self
.
load_model
(
opt
,
timeout
)
else
:
raise
ServerModelError
(
"No such model '%s'"
%
str
(
model_id
)
)


def load_model(self, opt, model_id=None, **model_kwargs):
        """Loading a model given a set of options
        """
        model_id = self.preload_model(opt, model_id=model_id, **model_kwargs)
        load_time = self.models[model_id].load_time

        return model_id, load_time

def load_model ( self , opt , model_id = None , * * model_kwargs ) : model_id = self . preload_model ( opt , model_id = model_id , * * model_kwargs ) load_time = self . models [ model_id ] . load_time return model_id , load_time


Preloading the model : updating internal datastructure It will effectively load the model if load is set

def preload_model(self, opt, model_id=None, **model_kwargs):
        """Preloading the model: updating internal datastructure
           It will effectively load the model if `load` is set
        """
        if model_id is not None:
            if model_id in self.models.keys():
                raise ValueError("Model ID %d already exists" % model_id)
        else:
            model_id = self.next_id
            while model_id in self.models.keys():
                model_id += 1
            self.next_id = model_id + 1
        print("Pre-loading model %d" % model_id)
        model = ServerModel(opt, model_id, **model_kwargs)
        self.models[model_id] = model

        return model_id

def
preload_model
(
self
,
opt
,
model_id
=
,
*
*
model_kwargs
)
:
if
model_id
is
not
:
if
model_id
in
self
.
models
.
keys
(
)
:
raise
ValueError
(
"Model ID %d already exists"
%
model_id
)
else
:
model_id
=
self
.
next_id
while
model_id
in
self
.
models
.
keys
(
)
:
model_id
+=
1
self
.
next_id
=
model_id
+
1
print
(
"Pre-loading model %d"
%
model_id
)
model
=
ServerModel
(
opt
,
model_id
,
*
*
model_kwargs
)
self
.
models
[
model_id
]
=
model
return
model_id


Translate inputs We keep the same format as the Lua version i . e . [ { id : model_id src : sequence to translate } { ... } ]

def run(self, inputs):
        """Translate `inputs`
           We keep the same format as the Lua version i.e.
             [{"id": model_id, "src": "sequence to translate"},{ ...}]

           We use inputs[0]["id"] as the model id
        """
        model_id = inputs[0].get("id", 0)
        if model_id in self.models and self.models[model_id] is not None:
            return self.models[model_id].run(inputs)
        else:
            print("Error No such model '%s'" % str(model_id))
            raise ServerModelError("No such model '%s'" % str(model_id))

def
run
(
self
,
inputs
)
:
model_id
=
inputs
[
0
]
.
get
(
"id"
,
0
)
if
model_id
in
self
.
models
and
self
.
models
[
model_id
]
is
not
:
return
self
.
models
[
model_id
]
.
run
(
inputs
)
else
:
print
(
"Error No such model '%s'"
%
str
(
model_id
)
)
raise
ServerModelError
(
"No such model '%s'"
%
str
(
model_id
)
)


Manually unload a model . It will free the memory and cancel the timer

def unload_model(self, model_id):
        """Manually unload a model.
           It will free the memory and cancel the timer
        """
        if model_id in self.models and self.models[model_id] is not None:
            self.models[model_id].unload()
        else:
            raise ServerModelError("No such model '%s'" % str(model_id))

def
unload_model
(
self
,
model_id
)
:
if
model_id
in
self
.
models
and
self
.
models
[
model_id
]
is
not
:
self
.
models
[
model_id
]
.
unload
(
)
else
:
raise
ServerModelError
(
"No such model '%s'"
%
str
(
model_id
)
)


def list_models(self):
        """Return the list of available models
        """
        models = []
        for _, model in self.models.items():
            models += [model.to_dict()]
        return models

def list_models ( self ) : models = [ ] for _ , model in self . models . items ( ) : models += [ model . to_dict ( ) ] return models


Args : opt : ( dict ) options for the Translator model_id : ( int ) model id tokenizer_opt : ( dict ) options for the tokenizer or None load : ( bool ) whether to load the model during __init__ timeout : ( int ) seconds before running do_timeout Negative values means no timeout on_timeout : ( str ) in [ to_cpu unload ] set what to do on timeout ( see function do_timeout ) model_root : ( str ) path to the model directory it must contain de model and tokenizer file

def __init__(self, opt, model_id, tokenizer_opt=None, load=False,
                 timeout=-1, on_timeout="to_cpu", model_root="./"):
        """
            Args:
                opt: (dict) options for the Translator
                model_id: (int) model id
                tokenizer_opt: (dict) options for the tokenizer or None
                load: (bool) whether to load the model during __init__
                timeout: (int) seconds before running `do_timeout`
                         Negative values means no timeout
                on_timeout: (str) in ["to_cpu", "unload"] set what to do on
                            timeout (see function `do_timeout`)
                model_root: (str) path to the model directory
                            it must contain de model and tokenizer file

        """
        self.model_root = model_root
        self.opt = self.parse_opt(opt)
        if self.opt.n_best > 1:
            raise ValueError("Values of n_best > 1 are not supported")

        self.model_id = model_id
        self.tokenizer_opt = tokenizer_opt
        self.timeout = timeout
        self.on_timeout = on_timeout

        self.unload_timer = None
        self.user_opt = opt
        self.tokenizer = None
        self.logger = init_logger(self.opt.log_file)
        self.loading_lock = threading.Event()
        self.loading_lock.set()

        if load:
            self.load()


Parse the option set passed by the user using onmt . opts Args : opt : ( dict ) options passed by the user

def parse_opt(self, opt):
        """Parse the option set passed by the user using `onmt.opts`
           Args:
               opt: (dict) options passed by the user

           Returns:
               opt: (Namespace) full set of options for the Translator
        """
        prec_argv = sys.argv
        sys.argv = sys.argv[:1]
        parser = argparse.ArgumentParser()
        onmt.opts.translate_opts(parser)

        models = opt['models']
        if not isinstance(models, (list, tuple)):
            models = [models]
        opt['models'] = [os.path.join(self.model_root, model)
                         for model in models]
        opt['src'] = "dummy_src"

        for (k, v) in opt.items():
            if k == 'models':
                sys.argv += ['-model']
                sys.argv += [str(model) for model in v]
            elif type(v) == bool:
                sys.argv += ['-%s' % k]
            else:
                sys.argv += ['-%s' % k, str(v)]

        opt = parser.parse_args()
        opt.cuda = opt.gpu > -1

        sys.argv = prec_argv
        return opt

def parse_opt ( self , opt ) : prec_argv = sys . argv sys . argv = sys . argv [ : 1 ] parser = argparse . ArgumentParser ( ) onmt . opts . translate_opts ( parser ) models = opt [ 'models' ] if not isinstance ( models , ( list , tuple ) ) : models = [ models ] opt [ 'models' ] = [ os . path . join ( self . model_root , model ) for model in models ] opt [ 'src' ] = "dummy_src" for ( k , v ) in opt . items ( ) : if k == 'models' : sys . argv += [ '-model' ] sys . argv += [ str ( model ) for model in v ] elif type ( v ) == bool : sys . argv += [ '-%s' % k ] else : sys . argv += [ '-%s' % k , str ( v ) ] opt = parser . parse_args ( ) opt . cuda = opt . gpu > - 1 sys . argv = prec_argv return opt


def run(self, inputs):
        """Translate `inputs` using this model

            Args:
                inputs: [{"src": "..."},{"src": ...}]

            Returns:
                result: (list) translations
                times: (dict) containing times
        """
        self.stop_unload_timer()

        timer = Timer()
        timer.start()
        self.logger.info("Running translation using %d" % self.model_id)

        if not self.loading_lock.is_set():
            self.logger.info(
                "Model #%d is being loaded by another thread, waiting"
                % self.model_id)
            if not self.loading_lock.wait(timeout=30):
                raise ServerModelError("Model %d loading timeout"
                                       % self.model_id)

        else:
            if not self.loaded:
                self.load()
                timer.tick(name="load")
            elif self.opt.cuda:
                self.to_gpu()
                timer.tick(name="to_gpu")

        texts = []
        head_spaces = []
        tail_spaces = []
        sslength = []
        for i, inp in enumerate(inputs):
            src = inp['src']
            if src.strip() == "":
                head_spaces.append(src)
                texts.append("")
                tail_spaces.append("")
            else:
                whitespaces_before, whitespaces_after = "", ""
                match_before = re.search(r'^\s+', src)
                match_after = re.search(r'\s+$', src)
                if match_before is not None:
                    whitespaces_before = match_before.group(0)
                if match_after is not None:
                    whitespaces_after = match_after.group(0)
                head_spaces.append(whitespaces_before)
                tok = self.maybe_tokenize(src.strip())
                texts.append(tok)
                sslength.append(len(tok.split()))
                tail_spaces.append(whitespaces_after)

        empty_indices = [i for i, x in enumerate(texts) if x == ""]
        texts_to_translate = [x for x in texts if x != ""]

        scores = []
        predictions = []
        if len(texts_to_translate) > 0:
            try:
                scores, predictions = self.translator.translate(
                    src_data_iter=texts_to_translate,
                    batch_size=self.opt.batch_size)
            except RuntimeError as e:
                raise ServerModelError("Runtime Error: %s" % str(e))

        timer.tick(name="translation")
        self.logger.info("""Using model #%d\t%d inputs
               \ttranslation time: %f""" % (self.model_id, len(texts),
                                            timer.times['translation']))
        self.reset_unload_timer()

        # NOTE: translator returns lists of `n_best` list
        #       we can ignore that (i.e. flatten lists) only because
        #       we restrict `n_best=1`
        def flatten_list(_list): return sum(_list, [])
        results = flatten_list(predictions)
        scores = [score_tensor.item()
                  for score_tensor in flatten_list(scores)]

        results = [self.maybe_detokenize(item)
                   for item in results]

        # build back results with empty texts
        for i in empty_indices:
            results.insert(i, "")
            scores.insert(i, 0)

        results = ["".join(items)
                   for items in zip(head_spaces, results, tail_spaces)]

        self.logger.info("Translation Results: %d", len(results))

        return results, scores, self.opt.n_best, timer.times


Timeout function that free GPU memory by moving the model to CPU or unloading it ; depending on self . on_timemout value

def do_timeout(self):
        """Timeout function that free GPU memory by moving the model to CPU
           or unloading it; depending on `self.on_timemout` value
        """
        if self.on_timeout == "unload":
            self.logger.info("Timeout: unloading model %d" % self.model_id)
            self.unload()
        if self.on_timeout == "to_cpu":
            self.logger.info("Timeout: sending model %d to CPU"
                             % self.model_id)
            self.to_cpu()

def
do_timeout
(
self
)
:
if
self
.
on_timeout
==
"unload"
:
self
.
logger
.
info
(
"Timeout: unloading model %d"
%
self
.
model_id
)
self
.
unload
(
)
if
self
.
on_timeout
==
"to_cpu"
:
self
.
logger
.
info
(
"Timeout: sending model %d to CPU"
%
self
.
model_id
)
self
.
to_cpu
(
)


def to_cpu(self):
        """Move the model to CPU and clear CUDA cache
        """
        self.translator.model.cpu()
        if self.opt.cuda:
            torch.cuda.empty_cache()

def to_cpu ( self ) : self . translator . model . cpu ( ) if self . opt . cuda : torch . cuda . empty_cache ( )


def to_gpu(self):
        """Move the model to GPU
        """
        torch.cuda.set_device(self.opt.gpu)
        self.translator.model.cuda()

def to_gpu ( self ) : torch . cuda . set_device ( self . opt . gpu ) self . translator . model . cuda ( )


def maybe_tokenize(self, sequence):
        """Tokenize the sequence (or not)

           Same args/returns as `tokenize`
        """
        if self.tokenizer_opt is not None:
            return self.tokenize(sequence)
        return sequence

def maybe_tokenize ( self , sequence ) : if self . tokenizer_opt is not None : return self . tokenize ( sequence ) return sequence


def tokenize(self, sequence):
        """Tokenize a single sequence

            Args:
                sequence: (str) the sequence to tokenize

            Returns:
                tok: (str) the tokenized sequence

        """
        if self.tokenizer is None:
            raise ValueError("No tokenizer loaded")

        if self.tokenizer_opt["type"] == "sentencepiece":
            tok = self.tokenizer.EncodeAsPieces(sequence)
            tok = " ".join(tok)
        elif self.tokenizer_opt["type"] == "pyonmttok":
            tok, _ = self.tokenizer.tokenize(sequence)
            tok = " ".join(tok)
        return tok

def
tokenize
(
self
,
sequence
)
:
if
self
.
tokenizer
is
:
raise
ValueError
(
"No tokenizer loaded"
)
if
self
.
tokenizer_opt
[
"type"
]
==
"sentencepiece"
:
tok
=
self
.
tokenizer
.
EncodeAsPieces
(
sequence
)
tok
=
" "
.
join
(
tok
)
elif
self
.
tokenizer_opt
[
"type"
]
==
"pyonmttok"
:
tok
,
_
=
self
.
tokenizer
.
tokenize
(
sequence
)
tok
=
" "
.
join
(
tok
)
return
tok


def maybe_detokenize(self, sequence):
        """De-tokenize the sequence (or not)

           Same args/returns as `tokenize`
        """
        if self.tokenizer_opt is not None and ''.join(sequence.split()) != '':
            return self.detokenize(sequence)
        return sequence

def maybe_detokenize ( self , sequence ) : if self . tokenizer_opt is not None and '' . join ( sequence . split ( ) ) != '' : return self . detokenize ( sequence ) return sequence


def detokenize(self, sequence):
        """Detokenize a single sequence

           Same args/returns as `tokenize`
        """
        if self.tokenizer is None:
            raise ValueError("No tokenizer loaded")

        if self.tokenizer_opt["type"] == "sentencepiece":
            detok = self.tokenizer.DecodePieces(sequence.split())
        elif self.tokenizer_opt["type"] == "pyonmttok":
            detok = self.tokenizer.detokenize(sequence.split())

        return detok

def
detokenize
(
self
,
sequence
)
:
if
self
.
tokenizer
is
:
raise
ValueError
(
"No tokenizer loaded"
)
if
self
.
tokenizer_opt
[
"type"
]
==
"sentencepiece"
:
detok
=
self
.
tokenizer
.
DecodePieces
(
sequence
.
split
(
)
)
elif
self
.
tokenizer_opt
[
"type"
]
==
"pyonmttok"
:
detok
=
self
.
tokenizer
.
detokenize
(
sequence
.
split
(
)
)
return
detok


NMT coverage re - ranking score from Google s Neural Machine Translation System : cite : wu2016google .

def coverage_wu(self, beam, cov, beta=0.):
        """
        NMT coverage re-ranking score from
        "Google's Neural Machine Translation System" :cite:`wu2016google`.
        """
        penalty = -torch.min(cov, cov.clone().fill_(1.0)).log().sum(1)
        return beta * penalty

def coverage_wu ( self , beam , cov , beta = 0. ) : penalty = - torch . min ( cov , cov . clone ( ) . fill_ ( 1.0 ) ) . log ( ) . sum ( 1 ) return beta * penalty


def coverage_summary(self, beam, cov, beta=0.):
        """
        Our summary penalty.
        """
        penalty = torch.max(cov, cov.clone().fill_(1.0)).sum(1)
        penalty -= cov.size(1)
        return beta * penalty

def coverage_summary ( self , beam , cov , beta = 0. ) : penalty = torch . max ( cov , cov . clone ( ) . fill_ ( 1.0 ) ) . sum ( 1 ) penalty -= cov . size ( 1 ) return beta * penalty


def coverage_none(self, beam, cov, beta=0.):
        """
        returns zero as penalty
        """
        return beam.scores.clone().fill_(0.0)

def coverage_none ( self , beam , cov , beta = 0. ) : return beam . scores . clone ( ) . fill_ ( 0.0 )


NMT length re - ranking score from Google s Neural Machine Translation System : cite : wu2016google .

def length_wu(self, beam, logprobs, alpha=0.):
        """
        NMT length re-ranking score from
        "Google's Neural Machine Translation System" :cite:`wu2016google`.
        """

        modifier = (((5 + len(beam.next_ys)) ** alpha) /
                    ((5 + 1) ** alpha))
        return (logprobs / modifier)

def length_wu ( self , beam , logprobs , alpha = 0. ) : modifier = ( ( ( 5 + len ( beam . next_ys ) ) ** alpha ) / ( ( 5 + 1 ) ** alpha ) ) return ( logprobs / modifier )


Returns the average probability of tokens in a sequence .

def length_average(self, beam, logprobs, alpha=0.):
        """
        Returns the average probability of tokens in a sequence.
        """
        return logprobs / len(beam.next_ys)

def length_average ( self , beam , logprobs , alpha = 0. ) : return logprobs / len ( beam . next_ys )


def length_none(self, beam, logprobs, alpha=0., beta=0.):
        """
        Returns unmodified scores.
        """
        return logprobs

def length_none ( self , beam , logprobs , alpha = 0. , beta = 0. ) : return logprobs


Translate content of src_data_iter ( if not None ) or src_path and get gold scores if one of tgt_data_iter or tgt_path is set .

def translate(self,
                  src_path=None,
                  src_data_iter=None,
                  tgt_path=None,
                  tgt_data_iter=None,
                  src_dir=None,
                  batch_size=None,
                  attn_debug=False):
        """
        Translate content of `src_data_iter` (if not None) or `src_path`
        and get gold scores if one of `tgt_data_iter` or `tgt_path` is set.

        Note: batch_size must not be None
        Note: one of ('src_path', 'src_data_iter') must not be None

        Args:
            src_path (str): filepath of source data
            src_data_iter (iterator): an interator generating source data
                e.g. it may be a list or an openned file
            tgt_path (str): filepath of target data
            tgt_data_iter (iterator): an interator generating target data
            src_dir (str): source directory path
                (used for Audio and Image datasets)
            batch_size (int): size of examples per mini-batch
            attn_debug (bool): enables the attention logging

        Returns:
            (`list`, `list`)

            * all_scores is a list of `batch_size` lists of `n_best` scores
            * all_predictions is a list of `batch_size` lists
                of `n_best` predictions
        """

        assert src_data_iter is not None or src_path is not None

        if batch_size is None:
            raise ValueError("batch_size must be set")
        data = inputters. \
            build_dataset(self.fields,
                          self.data_type,
                          src_path=src_path,
                          src_data_iter=src_data_iter,
                          tgt_path=tgt_path,
                          tgt_data_iter=tgt_data_iter,
                          src_dir=src_dir,
                          sample_rate=self.sample_rate,
                          window_size=self.window_size,
                          window_stride=self.window_stride,
                          window=self.window,
                          use_filter_pred=self.use_filter_pred,
                          image_channel_size=self.image_channel_size)

        if self.cuda:
            cur_device = "cuda"
        else:
            cur_device = "cpu"

        data_iter = inputters.OrderedIterator(
            dataset=data, device=cur_device,
            batch_size=batch_size, train=False, sort=False,
            sort_within_batch=True, shuffle=False)

        builder = onmt.translate.TranslationBuilder(
            data, self.fields,
            self.n_best, self.replace_unk, tgt_path)

        # Statistics
        counter = count(1)
        pred_score_total, pred_words_total = 0, 0
        gold_score_total, gold_words_total = 0, 0

        all_scores = []
        all_predictions = []

        for batch in data_iter:
            batch_data = self.translate_batch(batch, data, fast=self.fast)
            # import pdb;pdb.set_trace()
            translations = builder.from_batch(batch_data)

            for trans in translations:
                all_scores += [trans.pred_scores[:self.n_best]]
                pred_score_total += trans.pred_scores[0]
                pred_words_total += len(trans.pred_sents[0])
                if tgt_path is not None:
                    gold_score_total += trans.gold_score
                    gold_words_total += len(trans.gold_sent) + 1

                n_best_preds = [" ".join(pred)
                                for pred in trans.pred_sents[:self.n_best]]
                all_predictions += [n_best_preds]
                self.out_file.write('\n'.join(n_best_preds) + '\n')
                self.out_file.flush()

                if self.verbose:
                    sent_number = next(counter)
                    output = trans.log(sent_number)
                    if self.logger:
                        self.logger.info(output)
                    else:
                        os.write(1, output.encode('utf-8'))

                # Debug attention.
                if attn_debug:
                    srcs = trans.src_raw
                    preds = trans.pred_sents[0]
                    preds.append('</s>')
                    attns = trans.attns[0].tolist()
                    header_format = "{:>10.10} " + "{:>10.7} " * len(srcs)
                    row_format = "{:>10.10} " + "{:>10.7f} " * len(srcs)
                    output = header_format.format("", *trans.src_raw) + '\n'
                    for word, row in zip(preds, attns):
                        max_index = row.index(max(row))
                        row_format = row_format.replace(
                            "{:>10.7f} ", "{:*>10.7f} ", max_index + 1)
                        row_format = row_format.replace(
                            "{:*>10.7f} ", "{:>10.7f} ", max_index)
                        output += row_format.format(word, *row) + '\n'
                        row_format = "{:>10.10} " + "{:>10.7f} " * len(srcs)
                    os.write(1, output.encode('utf-8'))

        #TODO change back
        #if self.report_score:
        #    msg = self._report_score('PRED', pred_score_total,
        #                             pred_words_total)
        #    if self.logger:
        #        self.logger.info(msg)
        #    else:
        #        print(msg)
        #    if tgt_path is not None:
        #        msg = self._report_score('GOLD', gold_score_total,
        #                                 gold_words_total)
        #        if self.logger:
        #            self.logger.info(msg)
        #        else:
        #            print(msg)
        #        if self.report_bleu:
        #            msg = self._report_bleu(tgt_path)
        #            if self.logger:
        #                self.logger.info(msg)
        #            else:
        #                print(msg)
        #        if self.report_rouge:
        #            msg = self._report_rouge(tgt_path)
        #            if self.logger:
        #                self.logger.info(msg)
        #            else:
        #                print(msg)

        if self.dump_beam:
            import json
            json.dump(self.translator.beam_accum,
                      codecs.open(self.dump_beam, 'w', 'utf-8'))
        return all_scores, all_predictions

#                             pred_words_total)

#                                 gold_words_total)

#            msg = self._report_bleu(tgt_path)

#            msg = self._report_rouge(tgt_path)


def translate_batch(self, batch, data, fast=False):
        """
        Translate a batch of sentences.

        Mostly a wrapper around :obj:`Beam`.

        Args:
           batch (:obj:`Batch`): a batch from a dataset object
           data (:obj:`Dataset`): the dataset object
           fast (bool): enables fast beam search (may not support all features)

        Todo:
           Shouldn't need the original dataset.
        """
        with torch.no_grad():
            if fast:
                return self._fast_translate_batch(
                    batch,
                    data,
                    self.max_length,
                    min_length=self.min_length,
                    n_best=self.n_best,
                    return_attention=self.replace_unk)
            else:
                return self._translate_batch(batch, data)

def translate_batch ( self , batch , data , fast = False ) : with torch . no_grad ( ) : if fast : return self . _fast_translate_batch ( batch , data , self . max_length , min_length = self . min_length , n_best = self . n_best , return_attention = self . replace_unk ) else : return self . _translate_batch ( batch , data )


Get the outputs for the current timestep .

def get_current_state(self):
        "Get the outputs for the current timestep."
        return self.next_ys[-1]

def get_current_state ( self ) : return self . next_ys [ - 1 ]


Get the backpointers for the current timestep .

def get_current_origin(self):
        "Get the backpointers for the current timestep."
        return self.prev_ks[-1]

def get_current_origin ( self ) : return self . prev_ks [ - 1 ]


Given prob over words for every last beam wordLk and attention attn_out : Compute and update the beam search .

def advance(self, word_probs, attn_out):
        """
        Given prob over words for every last beam `wordLk` and attention
        `attn_out`: Compute and update the beam search.

        Parameters:

        * `word_probs`- probs of advancing from the last step (K x words)
        * `attn_out`- attention at the last step

        Returns: True if beam search is complete.
        """
        num_words = word_probs.size(1)
        if self.stepwise_penalty:
            self.global_scorer.update_score(self, attn_out)
        # force the output to be longer than self.min_length
        cur_len = len(self.next_ys)
        if cur_len < self.min_length:
            for k in range(len(word_probs)):
                word_probs[k][self._eos] = -1e20
        # Sum the previous scores.
        if len(self.prev_ks) > 0:
            beam_scores = word_probs + \
                self.scores.unsqueeze(1).expand_as(word_probs)
            # Don't let EOS have children.
            for i in range(self.next_ys[-1].size(0)):
                if self.next_ys[-1][i] == self._eos:
                    beam_scores[i] = -1e20

            # Block ngram repeats
            if self.block_ngram_repeat > 0:
                ngrams = []
                le = len(self.next_ys)
                for j in range(self.next_ys[-1].size(0)):
                    hyp, _ = self.get_hyp(le - 1, j)
                    ngrams = set()
                    fail = False
                    gram = []
                    for i in range(le - 1):
                        # Last n tokens, n = block_ngram_repeat
                        gram = (gram +
                                [hyp[i].item()])[-self.block_ngram_repeat:]
                        # Skip the blocking if it is in the exclusion list
                        if set(gram) & self.exclusion_tokens:
                            continue
                        if tuple(gram) in ngrams:
                            fail = True
                        ngrams.add(tuple(gram))
                    if fail:
                        beam_scores[j] = -10e20
        else:
            beam_scores = word_probs[0]
        flat_beam_scores = beam_scores.view(-1)
        best_scores, best_scores_id = flat_beam_scores.topk(self.size, 0,
                                                            True, True)

        self.all_scores.append(self.scores)
        self.scores = best_scores

        # best_scores_id is flattened beam x word array, so calculate which
        # word and beam each score came from
        prev_k = best_scores_id / num_words
        self.prev_ks.append(prev_k)
        self.next_ys.append((best_scores_id - prev_k * num_words))
        self.attn.append(attn_out.index_select(0, prev_k))
        self.global_scorer.update_global_state(self)

        for i in range(self.next_ys[-1].size(0)):
            if self.next_ys[-1][i] == self._eos:
                global_scores = self.global_scorer.score(self, self.scores)
                s = global_scores[i]
                self.finished.append((s, len(self.next_ys) - 1, i))

        # End condition is when top-of-beam is EOS and no global score.
        if self.next_ys[-1][0] == self._eos:
            self.all_scores.append(self.scores)
            self.eos_top = True


Walk back to construct the full hypothesis .

def get_hyp(self, timestep, k):
        """
        Walk back to construct the full hypothesis.
        """
        hyp, attn = [], []
        for j in range(len(self.prev_ks[:timestep]) - 1, -1, -1):
            hyp.append(self.next_ys[j + 1][k])
            attn.append(self.attn[j][k])
            k = self.prev_ks[j][k]
        return hyp[::-1], torch.stack(attn[::-1])

def get_hyp ( self , timestep , k ) : hyp , attn = [ ] , [ ] for j in range ( len ( self . prev_ks [ : timestep ] ) - 1 , - 1 , - 1 ) : hyp . append ( self . next_ys [ j + 1 ] [ k ] ) attn . append ( self . attn [ j ] [ k ] ) k = self . prev_ks [ j ] [ k ] return hyp [ : : - 1 ] , torch . stack ( attn [ : : - 1 ] )


def score(self, beam, logprobs):
        """
        Rescores a prediction based on penalty functions
        """
        normalized_probs = self.length_penalty(beam,
                                               logprobs,
                                               self.alpha)
        if not beam.stepwise_penalty:
            penalty = self.cov_penalty(beam,
                                       beam.global_state["coverage"],
                                       self.beta)
            normalized_probs -= penalty

        return normalized_probs

def score ( self , beam , logprobs ) : normalized_probs = self . length_penalty ( beam , logprobs , self . alpha ) if not beam . stepwise_penalty : penalty = self . cov_penalty ( beam , beam . global_state [ "coverage" ] , self . beta ) normalized_probs -= penalty return normalized_probs


def update_score(self, beam, attn):
        """
        Function to update scores of a Beam that is not finished
        """
        if "prev_penalty" in beam.global_state.keys():
            beam.scores.add_(beam.global_state["prev_penalty"])
            penalty = self.cov_penalty(beam,
                                       beam.global_state["coverage"] + attn,
                                       self.beta)
            beam.scores.sub_(penalty)

def update_score ( self , beam , attn ) : if "prev_penalty" in beam . global_state . keys ( ) : beam . scores . add_ ( beam . global_state [ "prev_penalty" ] ) penalty = self . cov_penalty ( beam , beam . global_state [ "coverage" ] + attn , self . beta ) beam . scores . sub_ ( penalty )


def update_global_state(self, beam):
        "Keeps the coverage vector as sum of attentions"
        if len(beam.prev_ks) == 1:
            beam.global_state["prev_penalty"] = beam.scores.clone().fill_(0.0)
            beam.global_state["coverage"] = beam.attn[-1]
            self.cov_total = beam.attn[-1].sum(1)
        else:
            self.cov_total += torch.min(beam.attn[-1],
                                        beam.global_state['coverage']).sum(1)
            beam.global_state["coverage"] = beam.global_state["coverage"] \
                .index_select(0, beam.prev_ks[-1]).add(beam.attn[-1])

            prev_penalty = self.cov_penalty(beam,
                                            beam.global_state["coverage"],
                                            self.beta)
            beam.global_state["prev_penalty"] = prev_penalty

def update_global_state ( self , beam ) : if len ( beam . prev_ks ) == 1 : beam . global_state [ "prev_penalty" ] = beam . scores . clone ( ) . fill_ ( 0.0 ) beam . global_state [ "coverage" ] = beam . attn [ - 1 ] self . cov_total = beam . attn [ - 1 ] . sum ( 1 ) else : self . cov_total += torch . min ( beam . attn [ - 1 ] , beam . global_state [ 'coverage' ] ) . sum ( 1 ) beam . global_state [ "coverage" ] = beam . global_state [ "coverage" ] . index_select ( 0 , beam . prev_ks [ - 1 ] ) . add ( beam . attn [ - 1 ] ) prev_penalty = self . cov_penalty ( beam , beam . global_state [ "coverage" ] , self . beta ) beam . global_state [ "prev_penalty" ] = prev_penalty


def log(self, sent_number):
        """
        Log translation.
        """

        output = '\nSENT {}: {}\n'.format(sent_number, self.src_raw)

        best_pred = self.pred_sents[0]
        best_score = self.pred_scores[0]
        pred_sent = ' '.join(best_pred)
        output += 'PRED {}: {}\n'.format(sent_number, pred_sent)
        output += "PRED SCORE: {:.4f}\n".format(best_score)

        if self.gold_sent is not None:
            tgt_sent = ' '.join(self.gold_sent)
            output += 'GOLD {}: {}\n'.format(sent_number, tgt_sent)
            output += ("GOLD SCORE: {:.4f}\n".format(self.gold_score))
        if len(self.pred_sents) > 1:
            output += '\nBEST HYP:\n'
            for score, sent in zip(self.pred_scores, self.pred_sents):
                output += "[{:.4f}] {}\n".format(score, sent)

        return output

def
log
(
self
,
sent_number
)
:
output
=
'\nSENT {}: {}\n'
.
format
(
sent_number
,
self
.
src_raw
)
best_pred
=
self
.
pred_sents
[
0
]
best_score
=
self
.
pred_scores
[
0
]
pred_sent
=
' '
.
join
(
best_pred
)
output
+=
'PRED {}: {}\n'
.
format
(
sent_number
,
pred_sent
)
output
+=
"PRED SCORE: {:.4f}\n"
.
format
(
best_score
)
if
self
.
gold_sent
is
not
:
tgt_sent
=
' '
.
join
(
self
.
gold_sent
)
output
+=
'GOLD {}: {}\n'
.
format
(
sent_number
,
tgt_sent
)
output
+=
(
"GOLD SCORE: {:.4f}\n"
.
format
(
self
.
gold_score
)
)
if
len
(
self
.
pred_sents
)
>
1
:
output
+=
'\nBEST HYP:\n'
for
score
,
sent
in
zip
(
self
.
pred_scores
,
self
.
pred_sents
)
:
output
+=
"[{:.4f}] {}\n"
.
format
(
score
,
sent
)
return
output


Args : tgt ( LongTensor ) : sequences of padded tokens [ tgt_len x batch x nfeats ] . memory_bank ( FloatTensor ) : vectors from the encoder [ src_len x batch x hidden ] . state ( : obj : onmt . models . DecoderState ) : decoder state object to initialize the decoder memory_lengths ( LongTensor ) : the padded source lengths [ batch ] . Returns : ( FloatTensor : obj : onmt . Models . DecoderState FloatTensor ) : * decoder_outputs : output from the decoder ( after attn ) [ tgt_len x batch x hidden ] . * decoder_state : final hidden state from the decoder * attns : distribution over src at each tgt [ tgt_len x batch x src_len ] .

def forward(self, tgt, memory_bank, state, memory_lengths=None,
                step=None):
        """
        Args:
            tgt (`LongTensor`): sequences of padded tokens
                 `[tgt_len x batch x nfeats]`.
            memory_bank (`FloatTensor`): vectors from the encoder
                 `[src_len x batch x hidden]`.
            state (:obj:`onmt.models.DecoderState`):
                 decoder state object to initialize the decoder
            memory_lengths (`LongTensor`): the padded source lengths
                `[batch]`.
        Returns:
            (`FloatTensor`,:obj:`onmt.Models.DecoderState`,`FloatTensor`):
                * decoder_outputs: output from the decoder (after attn)
                         `[tgt_len x batch x hidden]`.
                * decoder_state: final hidden state from the decoder
                * attns: distribution over src at each tgt
                        `[tgt_len x batch x src_len]`.
        """
        # Check
        assert isinstance(state, RNNDecoderState)
        # tgt.size() returns tgt length and batch
        _, tgt_batch, _ = tgt.size()
        _, memory_batch, _ = memory_bank.size()
        aeq(tgt_batch, memory_batch)
        # END

        # Run the forward pass of the RNN.
        decoder_final, decoder_outputs, attns = self._run_forward_pass(
            tgt, memory_bank, state, memory_lengths=memory_lengths)

        # Update the state with the result.
        final_output = decoder_outputs[-1]
        coverage = None
        if "coverage" in attns:
            coverage = attns["coverage"][-1].unsqueeze(0)
        state.update_state(decoder_final, final_output.unsqueeze(0), coverage)

        # Concatenates sequence of tensors along a new dimension.
        # NOTE: v0.3 to 0.4: decoder_outputs / attns[*] may not be list
        #       (in particular in case of SRU) it was not raising error in 0.3
        #       since stack(Variable) was allowed.
        #       In 0.4, SRU returns a tensor that shouldn't be stacke
        if type(decoder_outputs) == list:
            decoder_outputs = torch.stack(decoder_outputs)

            for k in attns:
                if type(attns[k]) == list:
                    attns[k] = torch.stack(attns[k])

        return decoder_outputs, state, attns

#       since stack(Variable) was allowed.


def init_decoder_state(self, src, memory_bank, encoder_final,
                           with_cache=False):
        """ Init decoder state with last state of the encoder """
        def _fix_enc_hidden(hidden):
            # The encoder hidden is  (layers*directions) x batch x dim.
            # We need to convert it to layers x batch x (directions*dim).
            if self.bidirectional_encoder:
                hidden = torch.cat([hidden[0:hidden.size(0):2],
                                    hidden[1:hidden.size(0):2]], 2)
            return hidden

        if isinstance(encoder_final, tuple):  # LSTM
            return RNNDecoderState(self.hidden_size,
                                   tuple([_fix_enc_hidden(enc_hid)
                                          for enc_hid in encoder_final]))
        else:  # GRU
            return RNNDecoderState(self.hidden_size,
                                   _fix_enc_hidden(encoder_final))


Private helper for running the specific RNN forward pass . Must be overriden by all subclasses . Args : tgt ( LongTensor ) : a sequence of input tokens tensors [ len x batch x nfeats ] . memory_bank ( FloatTensor ) : output ( tensor sequence ) from the encoder RNN of size ( src_len x batch x hidden_size ) . state ( FloatTensor ) : hidden state from the encoder RNN for initializing the decoder . memory_lengths ( LongTensor ) : the source memory_bank lengths . Returns : decoder_final ( Tensor ) : final hidden state from the decoder . decoder_outputs ( [ FloatTensor ] ) : an array of output of every time step from the decoder . attns ( dict of ( str [ FloatTensor ] ) : a dictionary of different type of attention Tensor array of every time step from the decoder .

def _run_forward_pass(self, tgt, memory_bank, state, memory_lengths=None):
        """
        Private helper for running the specific RNN forward pass.
        Must be overriden by all subclasses.
        Args:
            tgt (LongTensor): a sequence of input tokens tensors
                                 [len x batch x nfeats].
            memory_bank (FloatTensor): output(tensor sequence) from the encoder
                        RNN of size (src_len x batch x hidden_size).
            state (FloatTensor): hidden state from the encoder RNN for
                                 initializing the decoder.
            memory_lengths (LongTensor): the source memory_bank lengths.
        Returns:
            decoder_final (Tensor): final hidden state from the decoder.
            decoder_outputs ([FloatTensor]): an array of output of every time
                                     step from the decoder.
            attns (dict of (str, [FloatTensor]): a dictionary of different
                            type of attention Tensor array of every time
                            step from the decoder.
        """
        assert not self._copy  # TODO, no support yet.
        assert not self._coverage  # TODO, no support yet.

        # Initialize local and return variables.
        attns = {}
        emb = self.embeddings(tgt)

        # Run the forward pass of the RNN.
        if isinstance(self.rnn, nn.GRU):
            rnn_output, decoder_final = self.rnn(emb, state.hidden[0])
        else:
            rnn_output, decoder_final = self.rnn(emb, state.hidden)

        # Check
        tgt_len, tgt_batch, _ = tgt.size()
        output_len, output_batch, _ = rnn_output.size()
        aeq(tgt_len, output_len)
        aeq(tgt_batch, output_batch)
        # END

        # Calculate the attention.
        decoder_outputs, p_attn = self.attn(
            rnn_output.transpose(0, 1).contiguous(),
            memory_bank.transpose(0, 1),
            memory_lengths=memory_lengths
        )
        attns["std"] = p_attn

        # Calculate the context gate.
        if self.context_gate is not None:
            decoder_outputs = self.context_gate(
                emb.view(-1, emb.size(2)),
                rnn_output.view(-1, rnn_output.size(2)),
                decoder_outputs.view(-1, decoder_outputs.size(2))
            )
            decoder_outputs = \
                decoder_outputs.view(tgt_len, tgt_batch, self.hidden_size)

        decoder_outputs = self.dropout(decoder_outputs)
        return decoder_final, decoder_outputs, attns


Private helper returning the number of expected features .

def _input_size(self):
        """
        Private helper returning the number of expected features.
        """
        return self.embeddings.embedding_size

def _input_size ( self ) : return self . embeddings . embedding_size


See StdRNNDecoder . _run_forward_pass () for description of arguments and return values .

def _run_forward_pass(self, tgt, memory_bank, state, memory_lengths=None):
        """
        See StdRNNDecoder._run_forward_pass() for description
        of arguments and return values.
        """
        # Additional args check.
        input_feed = state.input_feed.squeeze(0)
        # print("input feed size: {}\n".format(input_feed.size()))
        input_feed_batch, _ = input_feed.size()
        _, tgt_batch, _ = tgt.size()
        aeq(tgt_batch, input_feed_batch)
        # END Additional args check.

        # Initialize local and return variables.
        decoder_outputs = []
        attns = {"std": []}
        if self._copy:
            attns["copy"] = []
        if self._coverage:
            attns["coverage"] = []

        emb = self.embeddings(tgt)
        assert emb.dim() == 3  # len x batch x embedding_dim

        hidden = state.hidden
        coverage = state.coverage.squeeze(0) \
            if state.coverage is not None else None

        # Input feed concatenates hidden state with
        # input at every time step.
        #pdb.set_trace()
        #print("emb size: {}\n".format(emb.size()));exit()
        for _, emb_t in enumerate(emb.split(1)):
            emb_t = emb_t.squeeze(0)
            decoder_input = torch.cat([emb_t, input_feed], 1)

            rnn_output, hidden = self.rnn(decoder_input, hidden)
            decoder_output, p_attn = self.attn(
                rnn_output,
                memory_bank.transpose(0, 1),
                memory_lengths=memory_lengths)
            if self.context_gate is not None:
                # TODO: context gate should be employed
                # instead of second RNN transform.
                decoder_output = self.context_gate(
                    decoder_input, rnn_output, decoder_output
                )
            decoder_output = self.dropout(decoder_output)
            input_feed = decoder_output

            decoder_outputs += [decoder_output]
            attns["std"] += [p_attn]

            # Update the coverage attention.
            if self._coverage:
                coverage = coverage + p_attn \
                    if coverage is not None else p_attn
                attns["coverage"] += [coverage]

            # Run the forward pass of the copy attention layer.
            if self._copy and not self._reuse_copy_attn:
                _, copy_attn = self.copy_attn(decoder_output,
                                              memory_bank.transpose(0, 1))
                attns["copy"] += [copy_attn]
            elif self._copy:
                attns["copy"] = attns["std"]
        # Return result.
        return hidden, decoder_outputs, attns


Using input feed by concatenating input with attention vectors .

def _input_size(self):
        """
        Using input feed by concatenating input with attention vectors.
        """
        return self.embeddings.embedding_size + self.hidden_size

def _input_size ( self ) : return self . embeddings . embedding_size + self . hidden_size


def detach(self):
        """ Need to document this """
        self.hidden = tuple([_.detach() for _ in self.hidden])
        self.input_feed = self.input_feed.detach()

def detach ( self ) : self . hidden = tuple ( [ _ . detach ( ) for _ in self . hidden ] ) self . input_feed = self . input_feed . detach ( )


def beam_update(self, idx, positions, beam_size):
        """ Need to document this """
        for e in self._all:
            sizes = e.size()
            br = sizes[1]
            if len(sizes) == 3:
                sent_states = e.view(sizes[0], beam_size, br // beam_size,
                                     sizes[2])[:, :, idx]
            else:
                sent_states = e.view(sizes[0], beam_size,
                                     br // beam_size,
                                     sizes[2],
                                     sizes[3])[:, :, idx]

            sent_states.data.copy_(
                sent_states.data.index_select(1, positions))

def beam_update ( self , idx , positions , beam_size ) : for e in self . _all : sizes = e . size ( ) br = sizes [ 1 ] if len ( sizes ) == 3 : sent_states = e . view ( sizes [ 0 ] , beam_size , br // beam_size , sizes [ 2 ] ) [ : , : , idx ] else : sent_states = e . view ( sizes [ 0 ] , beam_size , br // beam_size , sizes [ 2 ] , sizes [ 3 ] ) [ : , : , idx ] sent_states . data . copy_ ( sent_states . data . index_select ( 1 , positions ) )


Args : hidden_size ( int ) : the size of hidden layer of the decoder . rnnstate : final hidden state from the encoder . transformed to shape : layers x batch x ( directions * dim ) .

def __init__(self, hidden_size, rnnstate):
        """
        Args:
            hidden_size (int): the size of hidden layer of the decoder.
            rnnstate: final hidden state from the encoder.
                transformed to shape: layers x batch x (directions*dim).
        """
        if not isinstance(rnnstate, tuple):
            self.hidden = (rnnstate,)
        else:
            self.hidden = rnnstate
        self.coverage = None

        # Init the input feed.
        batch_size = self.hidden[0].size(1)
        h_size = (batch_size, hidden_size)
        self.input_feed = self.hidden[0].data.new(*h_size).zero_() \
                              .unsqueeze(0)

def
__init__
(
self
,
hidden_size
,
rnnstate
)
:
if
not
isinstance
(
rnnstate
,
tuple
)
:
self
.
hidden
=
(
rnnstate
,
)
else
:
self
.
hidden
=
rnnstate
self
.
coverage
=
# Init the input feed.
batch_size
=
self
.
hidden
[
0
]
.
size
(
1
)
h_size
=
(
batch_size
,
hidden_size
)
self
.
input_feed
=
self
.
hidden
[
0
]
.
data
.
new
(
*
h_size
)
.
zero_
(
)
.
unsqueeze
(
0
)


def update_state(self, rnnstate, input_feed, coverage):
        """ Update decoder state """
        if not isinstance(rnnstate, tuple):
            self.hidden = (rnnstate,)
        else:
            self.hidden = rnnstate
        self.input_feed = input_feed
        self.coverage = coverage

def update_state ( self , rnnstate , input_feed , coverage ) : if not isinstance ( rnnstate , tuple ) : self . hidden = ( rnnstate , ) else : self . hidden = rnnstate self . input_feed = input_feed self . coverage = coverage


Repeat beam_size times along batch dimension .

def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        vars = [e.data.repeat(1, beam_size, 1)
                for e in self._all]
        self.hidden = tuple(vars[:-1])
        self.input_feed = vars[-1]

def repeat_beam_size_times ( self , beam_size ) : vars = [ e . data . repeat ( 1 , beam_size , 1 ) for e in self . _all ] self . hidden = tuple ( vars [ : - 1 ] ) self . input_feed = vars [ - 1 ]


def load_test_model(opt, dummy_opt):
    """ Read in multiple models for ensemble """
    shared_fields = None
    shared_model_opt = None
    models = []
    for model_path in opt.models:
        fields, model, model_opt = \
            onmt.model_builder.load_test_model(opt,
                                               dummy_opt,
                                               model_path=model_path)
        if shared_fields is None:
            shared_fields = fields
        else:
            for key, field in fields.items():
                if field is not None and 'vocab' in field.__dict__:
                    assert field.vocab.stoi == shared_fields[key].vocab.stoi, \
                        'Ensemble models must use the same preprocessed data'
        models.append(model)
        if shared_model_opt is None:
            shared_model_opt = model_opt
    ensemble_model = EnsembleModel(models)
    return shared_fields, ensemble_model, shared_model_opt


def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        for model_state in self.model_decoder_states:
            model_state.repeat_beam_size_times(beam_size)

def repeat_beam_size_times ( self , beam_size ) : for model_state in self . model_decoder_states : model_state . repeat_beam_size_times ( beam_size )


Delegate squeeze to avoid modifying : obj : Translator . translate_batch ()

def squeeze(self, dim=None):
        """
        Delegate squeeze to avoid modifying
        :obj:`Translator.translate_batch()`
        """
        return EnsembleDecoderOutput([
            x.squeeze(dim) for x in self.model_outputs])

def squeeze ( self , dim = None ) : return EnsembleDecoderOutput ( [ x . squeeze ( dim ) for x in self . model_outputs ] )


def forward(self, tgt, memory_bank, state, memory_lengths=None,
                step=None):
        """ See :obj:`RNNDecoderBase.forward()` """
        # Memory_lengths is a single tensor shared between all models.
        # This assumption will not hold if Translator is modified
        # to calculate memory_lengths as something other than the length
        # of the input.
        outputs, states, attns = zip(*[
            model_decoder.forward(
                tgt, memory_bank[i], state[i], memory_lengths, step=step)
            for (i, model_decoder)
            in enumerate(self.model_decoders)])
        mean_attns = self.combine_attns(attns)
        return (EnsembleDecoderOutput(outputs),
                EnsembleDecoderState(states),
                mean_attns)


See : obj : RNNDecoderBase . init_decoder_state ()

def init_decoder_state(self, src, memory_bank, enc_hidden):
        """ See :obj:`RNNDecoderBase.init_decoder_state()` """
        return EnsembleDecoderState(
            [model_decoder.init_decoder_state(src,
                                              memory_bank[i],
                                              enc_hidden[i])
             for (i, model_decoder) in enumerate(self.model_decoders)])

def init_decoder_state ( self , src , memory_bank , enc_hidden ) : return EnsembleDecoderState ( [ model_decoder . init_decoder_state ( src , memory_bank [ i ] , enc_hidden [ i ] ) for ( i , model_decoder ) in enumerate ( self . model_decoders ) ] )


Compute a distribution over the target dictionary by averaging distributions from models in the ensemble . All models in the ensemble must share a target vocabulary .

def forward(self, hidden):
        """
        Compute a distribution over the target dictionary
        by averaging distributions from models in the ensemble.
        All models in the ensemble must share a target vocabulary.
        """
        distributions = [model_generator.forward(hidden[i])
                         for (i, model_generator)
                         in enumerate(self.model_generators)]
        return torch.stack(distributions).mean(0)

def forward ( self , hidden ) : distributions = [ model_generator . forward ( hidden [ i ] ) for ( i , model_generator ) in enumerate ( self . model_generators ) ] return torch . stack ( distributions ) . mean ( 0 )


Args : inputs ( FloatTensor ) : [ batch_size x 1 x model_dim ] memory_bank ( FloatTensor ) : [ batch_size x src_len x model_dim ] src_pad_mask ( LongTensor ) : [ batch_size x 1 x src_len ] tgt_pad_mask ( LongTensor ) : [ batch_size x 1 x 1 ]

def forward(self, inputs, memory_bank, src_pad_mask, tgt_pad_mask,
                previous_input=None, layer_cache=None, step=None):
        """
        Args:
            inputs (`FloatTensor`): `[batch_size x 1 x model_dim]`
            memory_bank (`FloatTensor`): `[batch_size x src_len x model_dim]`
            src_pad_mask (`LongTensor`): `[batch_size x 1 x src_len]`
            tgt_pad_mask (`LongTensor`): `[batch_size x 1 x 1]`

        Returns:
            (`FloatTensor`, `FloatTensor`, `FloatTensor`):

            * output `[batch_size x 1 x model_dim]`
            * attn `[batch_size x 1 x src_len]`
            * all_input `[batch_size x current_step x model_dim]`

        """
        dec_mask = torch.gt(tgt_pad_mask +
                            self.mask[:, :tgt_pad_mask.size(1),
                                      :tgt_pad_mask.size(1)], 0)
        input_norm = self.layer_norm_1(inputs)
        all_input = input_norm
        if previous_input is not None:
            all_input = torch.cat((previous_input, input_norm), dim=1)
            dec_mask = None

        if self.self_attn_type == "scaled-dot":
            query, attn = self.self_attn(all_input, all_input, input_norm,
                                         mask=dec_mask,
                                         layer_cache=layer_cache,
                                         type="self")
        elif self.self_attn_type == "average":
            query, attn = self.self_attn(input_norm, mask=dec_mask,
                                         layer_cache=layer_cache, step=step)

        query = self.drop(query) + inputs

        query_norm = self.layer_norm_2(query)
        mid, attn = self.context_attn(memory_bank, memory_bank, query_norm,
                                      mask=src_pad_mask,
                                      layer_cache=layer_cache,
                                      type="context")
        output = self.feed_forward(self.drop(mid) + query)

        return output, attn, all_input

def forward ( self , inputs , memory_bank , src_pad_mask , tgt_pad_mask , previous_input = None , layer_cache = None , step = None ) : dec_mask = torch . gt ( tgt_pad_mask + self . mask [ : , : tgt_pad_mask . size ( 1 ) , : tgt_pad_mask . size ( 1 ) ] , 0 ) input_norm = self . layer_norm_1 ( inputs ) all_input = input_norm if previous_input is not None : all_input = torch . cat ( ( previous_input , input_norm ) , dim = 1 ) dec_mask = None if self . self_attn_type == "scaled-dot" : query , attn = self . self_attn ( all_input , all_input , input_norm , mask = dec_mask , layer_cache = layer_cache , type = "self" ) elif self . self_attn_type == "average" : query , attn = self . self_attn ( input_norm , mask = dec_mask , layer_cache = layer_cache , step = step ) query = self . drop ( query ) + inputs query_norm = self . layer_norm_2 ( query ) mid , attn = self . context_attn ( memory_bank , memory_bank , query_norm , mask = src_pad_mask , layer_cache = layer_cache , type = "context" ) output = self . feed_forward ( self . drop ( mid ) + query ) return output , attn , all_input


Get an attention mask to avoid using the subsequent info .

def _get_attn_subsequent_mask(self, size):
        """
        Get an attention mask to avoid using the subsequent info.

        Args:
            size: int

        Returns:
            (`LongTensor`):

            * subsequent_mask `[1 x size x size]`
        """
        attn_shape = (1, size, size)
        subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')
        subsequent_mask = torch.from_numpy(subsequent_mask)
        return subsequent_mask

def _get_attn_subsequent_mask ( self , size ) : attn_shape = ( 1 , size , size ) subsequent_mask = np . triu ( np . ones ( attn_shape ) , k = 1 ) . astype ( 'uint8' ) subsequent_mask = torch . from_numpy ( subsequent_mask ) return subsequent_mask


See : obj : onmt . modules . RNNDecoderBase . forward ()

def forward(self, tgt, memory_bank, state, memory_lengths=None,
                step=None, cache=None):
        """
        See :obj:`onmt.modules.RNNDecoderBase.forward()`
        """
        src = state.src
        src_words = src[:, :, 0].transpose(0, 1)
        tgt_words = tgt[:, :, 0].transpose(0, 1)
        src_batch, src_len = src_words.size()
        tgt_batch, tgt_len = tgt_words.size()

        # Initialize return variables.
        outputs = []
        attns = {"std": []}
        if self._copy:
            attns["copy"] = []

        # Run the forward pass of the TransformerDecoder.
        emb = self.embeddings(tgt, step=step)
        assert emb.dim() == 3  # len x batch x embedding_dim

        output = emb.transpose(0, 1).contiguous()
        src_memory_bank = memory_bank.transpose(0, 1).contiguous()

        padding_idx = self.embeddings.word_padding_idx
        src_pad_mask = src_words.data.eq(padding_idx).unsqueeze(1) \
            .expand(src_batch, tgt_len, src_len)
        tgt_pad_mask = tgt_words.data.eq(padding_idx).unsqueeze(1) \
            .expand(tgt_batch, tgt_len, tgt_len)

        if state.cache is None:
            saved_inputs = []

        for i in range(self.num_layers):
            prev_layer_input = None
            if state.cache is None:
                if state.previous_input is not None:
                    prev_layer_input = state.previous_layer_inputs[i]
            output, attn, all_input \
                = self.transformer_layers[i](
                    output, src_memory_bank,
                    src_pad_mask, tgt_pad_mask,
                    previous_input=prev_layer_input,
                    layer_cache=state.cache["layer_{}".format(i)]
                    if state.cache is not None else None,
                    step=step)
            if state.cache is None:
                saved_inputs.append(all_input)

        if state.cache is None:
            saved_inputs = torch.stack(saved_inputs)

        output = self.layer_norm(output)

        # Process the result and update the attentions.
        outputs = output.transpose(0, 1).contiguous()
        attn = attn.transpose(0, 1).contiguous()

        attns["std"] = attn
        if self._copy:
            attns["copy"] = attn

        if state.cache is None:
            state = state.update_state(tgt, saved_inputs)

        return outputs, state, attns


def init_decoder_state(self, src, memory_bank, enc_hidden,
                           with_cache=False):
        """ Init decoder state """
        state = TransformerDecoderState(src)
        if with_cache:
            state._init_cache(memory_bank, self.num_layers,
                              self.self_attn_type)
        return state

def init_decoder_state ( self , src , memory_bank , enc_hidden , with_cache = False ) : state = TransformerDecoderState ( src ) if with_cache : state . _init_cache ( memory_bank , self . num_layers , self . self_attn_type ) return state


Args : src ( FloatTensor ) : a sequence of source words tensors with optional feature tensors of size ( len x batch ) .

def __init__(self, src):
        """
        Args:
            src (FloatTensor): a sequence of source words tensors
                    with optional feature tensors, of size (len x batch).
        """
        self.src = src
        self.previous_input = None
        self.previous_layer_inputs = None
        self.cache = None

def __init__ ( self , src ) : self . src = src self . previous_input = None self . previous_layer_inputs = None self . cache = None


Contains attributes that need to be updated in self . beam_update () .

def _all(self):
        """
        Contains attributes that need to be updated in self.beam_update().
        """
        if (self.previous_input is not None
                and self.previous_layer_inputs is not None):
            return (self.previous_input,
                    self.previous_layer_inputs,
                    self.src)
        else:
            return (self.src,)

def _all ( self ) : if ( self . previous_input is not None and self . previous_layer_inputs is not None ) : return ( self . previous_input , self . previous_layer_inputs , self . src ) else : return ( self . src , )


def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        self.src = self.src.data.repeat(1, beam_size, 1)

def repeat_beam_size_times ( self , beam_size ) : self . src = self . src . data . repeat ( 1 , beam_size , 1 )


def forward(self, tgt, memory_bank, state, memory_lengths=None, step=None):
        """ See :obj:`onmt.modules.RNNDecoderBase.forward()`"""
        # NOTE: memory_lengths is only here for compatibility reasons
        #       with onmt.modules.RNNDecoderBase.forward()
        # CHECKS
        assert isinstance(state, CNNDecoderState)
        _, tgt_batch, _ = tgt.size()
        _, contxt_batch, _ = memory_bank.size()
        aeq(tgt_batch, contxt_batch)
        # END CHECKS

        if state.previous_input is not None:
            tgt = torch.cat([state.previous_input, tgt], 0)

        # Initialize return variables.
        outputs = []
        attns = {"std": []}
        assert not self._copy, "Copy mechanism not yet tested in conv2conv"
        if self._copy:
            attns["copy"] = []

        emb = self.embeddings(tgt)
        assert emb.dim() == 3  # len x batch x embedding_dim

        tgt_emb = emb.transpose(0, 1).contiguous()
        # The output of CNNEncoder.
        src_memory_bank_t = memory_bank.transpose(0, 1).contiguous()
        # The combination of output of CNNEncoder and source embeddings.
        src_memory_bank_c = state.init_src.transpose(0, 1).contiguous()

        # Run the forward pass of the CNNDecoder.
        emb_reshape = tgt_emb.contiguous().view(
            tgt_emb.size(0) * tgt_emb.size(1), -1)
        linear_out = self.linear(emb_reshape)
        x = linear_out.view(tgt_emb.size(0), tgt_emb.size(1), -1)
        x = shape_transform(x)

        pad = torch.zeros(x.size(0), x.size(1),
                          self.cnn_kernel_width - 1, 1)

        pad = pad.type_as(x)
        base_target_emb = x

        for conv, attention in zip(self.conv_layers, self.attn_layers):
            new_target_input = torch.cat([pad, x], 2)
            out = conv(new_target_input)
            c, attn = attention(base_target_emb, out,
                                src_memory_bank_t, src_memory_bank_c)
            x = (x + (c + out) * SCALE_WEIGHT) * SCALE_WEIGHT
        output = x.squeeze(3).transpose(1, 2)

        # Process the result and update the attentions.
        outputs = output.transpose(0, 1).contiguous()
        if state.previous_input is not None:
            outputs = outputs[state.previous_input.size(0):]
            attn = attn[:, state.previous_input.size(0):].squeeze()
            attn = torch.stack([attn])
        attns["std"] = attn
        if self._copy:
            attns["copy"] = attn

        # Update the state.
        state.update_state(tgt)

        return outputs, state, attns


def init_decoder_state(self, _, memory_bank, enc_hidden, with_cache=False):
        """
        Init decoder state.
        """
        return CNNDecoderState(memory_bank, enc_hidden)

def init_decoder_state ( self , _ , memory_bank , enc_hidden , with_cache = False ) : return CNNDecoderState ( memory_bank , enc_hidden )


def _all(self):
        """
        Contains attributes that need to be updated in self.beam_update().
        """
        return (self.previous_input,)

def _all ( self ) : return ( self . previous_input , )


def update_state(self, new_input):
        """ Called for every decoder forward pass. """
        self.previous_input = new_input

def update_state ( self , new_input ) : self . previous_input = new_input


def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        self.init_src = self.init_src.data.repeat(1, beam_size, 1)

def repeat_beam_size_times ( self , beam_size ) : self . init_src = self . init_src . data . repeat ( 1 , beam_size , 1 )


def get_vocabulary(fobj, is_dict=False):
    """Read text and return dictionary that encodes vocabulary
    """
    vocab = Counter()
    for line in fobj:
        if is_dict:
            word, count = line.strip().split()
            vocab[word] = int(count)
        else:
            for word in line.split():
                vocab[word] += 1
    return vocab

def get_vocabulary ( fobj , is_dict = False ) : vocab = Counter ( ) for line in fobj : if is_dict : word , count = line . strip ( ) . split ( ) vocab [ word ] = int ( count ) else : for word in line . split ( ) : vocab [ word ] += 1 return vocab


def update_pair_statistics(pair, changed, stats, indices):
    """Minimally update the indices and frequency of symbol pairs

    if we merge a pair of symbols, only pairs that overlap with occurrences
    of this pair are affected, and need to be updated.
    """
    stats[pair] = 0
    indices[pair] = defaultdict(int)
    first, second = pair
    new_pair = first + second
    for j, word, old_word, freq in changed:

        # find all instances of pair, and update frequency/indices around it
        i = 0
        while True:
            # find first symbol
            try:
                i = old_word.index(first, i)
            except ValueError:
                break
            # if first symbol is followed by second symbol, we've found an occurrence of pair (old_word[i:i+2])
            if i < len(old_word) - 1 and old_word[i + 1] == second:
                # assuming a symbol sequence "A B C", if "B C" is merged, reduce the frequency of "A B"
                if i:
                    prev = old_word[i - 1:i + 1]
                    stats[prev] -= freq
                    indices[prev][j] -= 1
                if i < len(old_word) - 2:
                    # assuming a symbol sequence "A B C B", if "B C" is merged, reduce the frequency of "C B".
                    # however, skip this if the sequence is A B C B C, because the frequency of "C B" will be reduced by the previous code block
                    if old_word[i + 2] != first or i >= len(old_word) - 3 or old_word[i + 3] != second:
                        nex = old_word[i + 1:i + 3]
                        stats[nex] -= freq
                        indices[nex][j] -= 1
                i += 2
            else:
                i += 1

        i = 0
        while True:
            try:
                # find new pair
                i = word.index(new_pair, i)
            except ValueError:
                break
            # assuming a symbol sequence "A BC D", if "B C" is merged, increase the frequency of "A BC"
            if i:
                prev = word[i - 1:i + 1]
                stats[prev] += freq
                indices[prev][j] += 1
            # assuming a symbol sequence "A BC B", if "B C" is merged, increase the frequency of "BC B"
            # however, if the sequence is A BC BC, skip this step because the count of "BC BC" will be incremented by the previous code block
            if i < len(word) - 1 and word[i + 1] != new_pair:
                nex = word[i:i + 2]
                stats[nex] += freq
                indices[nex][j] += 1
            i += 1


Count frequency of all symbol pairs and create index

def get_pair_statistics(vocab):
    """Count frequency of all symbol pairs, and create index"""

    # data structure of pair frequencies
    stats = defaultdict(int)

    # index from pairs to words
    indices = defaultdict(lambda: defaultdict(int))

    for i, (word, freq) in enumerate(vocab):
        prev_char = word[0]
        for char in word[1:]:
            stats[prev_char, char] += freq
            indices[prev_char, char][i] += 1
            prev_char = char

    return stats, indices

def
get_pair_statistics
(
vocab
)
:
# data structure of pair frequencies
stats
=
defaultdict
(
int
)
# index from pairs to words
indices
=
defaultdict
(
lambda
:
defaultdict
(
int
)
)
for
i
,
(
word
,
freq
)
in
enumerate
(
vocab
)
:
prev_char
=
word
[
0
]
for
char
in
word
[
1
:
]
:
stats
[
prev_char
,
char
]
+=
freq
indices
[
prev_char
,
char
]
[
i
]
+=
1
prev_char
=
char
return
stats
,
indices


Replace all occurrences of a symbol pair ( A B ) with a new symbol AB

def replace_pair(pair, vocab, indices):
    """Replace all occurrences of a symbol pair ('A', 'B') with a new symbol 'AB'"""
    first, second = pair
    pair_str = ''.join(pair)
    pair_str = pair_str.replace('\\', '\\\\')
    changes = []
    pattern = re.compile(
        r'(?<!\S)' + re.escape(first + ' ' + second) + r'(?!\S)')
    if sys.version_info < (3, 0):
        iterator = indices[pair].iteritems()
    else:
        iterator = indices[pair].items()
    for j, freq in iterator:
        if freq < 1:
            continue
        word, freq = vocab[j]
        new_word = ' '.join(word)
        new_word = pattern.sub(pair_str, new_word)
        new_word = tuple(new_word.split())

        vocab[j] = (new_word, freq)
        changes.append((j, new_word, word, freq))

    return changes

def
replace_pair
(
pair
,
vocab
,
indices
)
:
first
,
second
=
pair
pair_str
=
''
.
join
(
pair
)
pair_str
=
pair_str
.
replace
(
'\\'
,
'\\\\'
)
changes
=
[
]
pattern
=
re
.
compile
(
r'(?<!\S)'
+
re
.
escape
(
first
+
' '
+
second
)
+
r'(?!\S)'
)
if
sys
.
version_info
<
(
3
,
0
)
:
iterator
=
indices
[
pair
]
.
iteritems
(
)
else
:
iterator
=
indices
[
pair
]
.
items
(
)
for
j
,
freq
in
iterator
:
if
freq
<
1
:
continue
word
,
freq
=
vocab
[
j
]
new_word
=
' '
.
join
(
word
)
new_word
=
pattern
.
sub
(
pair_str
,
new_word
)
new_word
=
tuple
(
new_word
.
split
(
)
)
vocab
[
j
]
=
(
new_word
,
freq
)
changes
.
append
(
(
j
,
new_word
,
word
,
freq
)
)
return
changes


Prune statistics dict for efficiency of max ()

def prune_stats(stats, big_stats, threshold):
    """Prune statistics dict for efficiency of max()

    The frequency of a symbol pair never increases, so pruning is generally safe
    (until we the most frequent pair is less frequent than a pair we previously pruned)
    big_stats keeps full statistics for when we need to access pruned items
    """
    for item, freq in list(stats.items()):
        if freq < threshold:
            del stats[item]
            if freq < 0:
                big_stats[item] += freq
            else:
                big_stats[item] = freq

def prune_stats ( stats , big_stats , threshold ) : for item , freq in list ( stats . items ( ) ) : if freq < threshold : del stats [ item ] if freq < 0 : big_stats [ item ] += freq else : big_stats [ item ] = freq


Learn num_symbols BPE operations from vocabulary and write to outfile .

def main(infile, outfile, num_symbols, min_frequency=2, verbose=False, is_dict=False):
    """Learn num_symbols BPE operations from vocabulary, and write to outfile.
    """

    # version 0.2 changes the handling of the end-of-word token ('</w>');
    # version numbering allows bckward compatibility
    outfile.write('#version: 0.2\n')

    vocab = get_vocabulary(infile, is_dict)
    vocab = dict([(tuple(x[:-1]) + (x[-1] + '</w>',), y)
                  for (x, y) in vocab.items()])
    sorted_vocab = sorted(vocab.items(), key=lambda x: x[1], reverse=True)

    stats, indices = get_pair_statistics(sorted_vocab)
    big_stats = copy.deepcopy(stats)
    # threshold is inspired by Zipfian assumption, but should only affect speed
    threshold = max(stats.values()) / 10
    for i in range(num_symbols):
        if stats:
            most_frequent = max(stats, key=lambda x: (stats[x], x))

        # we probably missed the best pair because of pruning; go back to full statistics
        if not stats or (i and stats[most_frequent] < threshold):
            prune_stats(stats, big_stats, threshold)
            stats = copy.deepcopy(big_stats)
            most_frequent = max(stats, key=lambda x: (stats[x], x))
            # threshold is inspired by Zipfian assumption, but should only affect speed
            threshold = stats[most_frequent] * i / (i + 10000.0)
            prune_stats(stats, big_stats, threshold)

        if stats[most_frequent] < min_frequency:
            sys.stderr.write(
                'no pair has frequency >= {0}. Stopping\n'.format(min_frequency))
            break

        if verbose:
            sys.stderr.write('pair {0}: {1} {2} -> {1}{2} (frequency {3})\n'.format(
                i, most_frequent[0], most_frequent[1], stats[most_frequent]))
        outfile.write('{0} {1}\n'.format(*most_frequent))
        changes = replace_pair(most_frequent, sorted_vocab, indices)
        update_pair_statistics(most_frequent, changes, stats, indices)
        stats[most_frequent] = 0
        if not i % 100:
            prune_stats(stats, big_stats, threshold)


def get_pairs(word):
    """Return set of symbol pairs in a word.

    word is represented as tuple of symbols (symbols being variable-length strings)
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

def get_pairs ( word ) : pairs = set ( ) prev_char = word [ 0 ] for char in word [ 1 : ] : pairs . add ( ( prev_char , char ) ) prev_char = char return pairs


Encode word based on list of BPE merge operations which are applied consecutively

def encode(orig, bpe_codes, bpe_codes_reverse, vocab, separator, version, cache, glossaries=None):
    """Encode word based on list of BPE merge operations, which are applied consecutively
    """

    if orig in cache:
        return cache[orig]

    if orig in glossaries:
        cache[orig] = (orig,)
        return (orig,)

    if version == (0, 1):
        word = tuple(orig) + ('</w>',)
    elif version == (0, 2):  # more consistent handling of word-final segments
        word = tuple(orig[:-1]) + (orig[-1] + '</w>',)
    else:
        raise NotImplementedError

    pairs = get_pairs(word)

    if not pairs:
        return orig

    while True:
        bigram = min(pairs, key=lambda pair: bpe_codes.get(pair, float('inf')))
        if bigram not in bpe_codes:
            break
        first, second = bigram
        new_word = []
        i = 0
        while i < len(word):
            try:
                j = word.index(first, i)
                new_word.extend(word[i:j])
                i = j
            except:
                new_word.extend(word[i:])
                break

            if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                new_word.append(first + second)
                i += 2
            else:
                new_word.append(word[i])
                i += 1
        new_word = tuple(new_word)
        word = new_word
        if len(word) == 1:
            break
        else:
            pairs = get_pairs(word)

    # don't print end-of-word symbols
    if word[-1] == '</w>':
        word = word[:-1]
    elif word[-1].endswith('</w>'):
        word = word[:-1] + (word[-1].replace('</w>', ''),)

    if vocab:
        word = check_vocab_and_split(word, bpe_codes_reverse, vocab, separator)

    cache[orig] = word
    return word


Recursively split segment into smaller units ( by reversing BPE merges ) until all units are either in - vocabulary or cannot be split futher .

def recursive_split(segment, bpe_codes, vocab, separator, final=False):
    """Recursively split segment into smaller units (by reversing BPE merges)
    until all units are either in-vocabulary, or cannot be split futher."""

    try:
        if final:
            left, right = bpe_codes[segment + '</w>']
            right = right[:-4]
        else:
            left, right = bpe_codes[segment]
    except:
        #sys.stderr.write('cannot split {0} further.\n'.format(segment))
        yield segment
        return

    if left + separator in vocab:
        yield left
    else:
        for item in recursive_split(left, bpe_codes, vocab, separator, False):
            yield item

    if (final and right in vocab) or (not final and right + separator in vocab):
        yield right
    else:
        for item in recursive_split(right, bpe_codes, vocab, separator, final):
            yield item


Check for each segment in word if it is in - vocabulary and segment OOV segments into smaller units by reversing the BPE merge operations

def check_vocab_and_split(orig, bpe_codes, vocab, separator):
    """Check for each segment in word if it is in-vocabulary,
    and segment OOV segments into smaller units by reversing the BPE merge operations"""

    out = []

    for segment in orig[:-1]:
        if segment + separator in vocab:
            out.append(segment)
        else:
            #sys.stderr.write('OOV: {0}\n'.format(segment))
            for item in recursive_split(segment, bpe_codes, vocab, separator, False):
                out.append(item)

    segment = orig[-1]
    if segment in vocab:
        out.append(segment)
    else:
        #sys.stderr.write('OOV: {0}\n'.format(segment))
        for item in recursive_split(segment, bpe_codes, vocab, separator, True):
            out.append(item)

    return out


read vocabulary file produced by get_vocab . py and filter according to frequency threshold .

def read_vocabulary(vocab_file, threshold):
    """read vocabulary file produced by get_vocab.py, and filter according to frequency threshold.
    """

    vocabulary = set()

    for line in vocab_file:
        word, freq = line.split()
        freq = int(freq)
        if threshold == None or freq >= threshold:
            vocabulary.add(word)

    return vocabulary

def read_vocabulary ( vocab_file , threshold ) : vocabulary = set ( ) for line in vocab_file : word , freq = line . split ( ) freq = int ( freq ) if threshold == None or freq >= threshold : vocabulary . add ( word ) return vocabulary


Isolate a glossary present inside a word .

def isolate_glossary(word, glossary):
    """
    Isolate a glossary present inside a word.

    Returns a list of subwords. In which all 'glossary' glossaries are isolated

    For example, if 'USA' is the glossary and '1934USABUSA' the word, the return value is:
        ['1934', 'USA', 'B', 'USA']
    """
    if word == glossary or glossary not in word:
        return [word]
    else:
        splits = word.split(glossary)
        segments = [segment.strip() for split in splits[:-1]
                    for segment in [split, glossary] if segment != '']
        return segments + [splits[-1].strip()] if splits[-1] != '' else segments

def isolate_glossary ( word , glossary ) : if word == glossary or glossary not in word : return [ word ] else : splits = word . split ( glossary ) segments = [ segment . strip ( ) for split in splits [ : - 1 ] for segment in [ split , glossary ] if segment != '' ] return segments + [ splits [ - 1 ] . strip ( ) ] if splits [ - 1 ] != '' else segments


segment single sentence ( whitespace - tokenized string ) with BPE encoding

def segment(self, sentence):
        """segment single sentence (whitespace-tokenized string) with BPE encoding"""
        output = []
        for word in sentence.split():
            new_word = [out for segment in self._isolate_glossaries(word)
                        for out in encode(segment,
                                          self.bpe_codes,
                                          self.bpe_codes_reverse,
                                          self.vocab,
                                          self.separator,
                                          self.version,
                                          self.cache,
                                          self.glossaries)]

            for item in new_word[:-1]:
                output.append(item + self.separator)
            output.append(new_word[-1])

        return ' '.join(output)

def
segment
(
self
,
sentence
)
:
output
=
[
]
for
word
in
sentence
.
split
(
)
:
new_word
=
[
out
for
segment
in
self
.
_isolate_glossaries
(
word
)
for
out
in
encode
(
segment
,
self
.
bpe_codes
,
self
.
bpe_codes_reverse
,
self
.
vocab
,
self
.
separator
,
self
.
version
,
self
.
cache
,
self
.
glossaries
)
]
for
item
in
new_word
[
:
1
]
:
output
.
append
(
item
+
self
.
separator
)
output
.
append
(
new_word
[
1
]
)
return
' '
.
join
(
output
)