Checking if there are existing . pt files to avoid tampering

def check_existing_pt_files(opt):
    """ Checking if there are existing .pt files to avoid tampering """
    # We will use glob.glob() to find sharded {train|valid}.[0-9]*.pt
    # when training, so check to avoid tampering with existing pt files
    # or mixing them up.
    for t in ['train', 'valid', 'vocab']:
        pattern = opt.save_data + '.' + t + '*.pt'
        if glob.glob(pattern):
            sys.stderr.write("Please backup existing pt file: %s, "
                             "to avoid tampering!\n" % pattern)
            sys.exit(1)


def parse_args():
    """ Parsing arguments """
    parser = argparse.ArgumentParser(
        description='preprocess.py',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    opts.add_md_help_argument(parser)
    opts.preprocess_opts(parser)

    opt = parser.parse_args()
    torch.manual_seed(opt.seed)

    check_existing_pt_files(opt)

    return opt

def parse_args ( ) : parser = argparse . ArgumentParser ( description = 'preprocess.py' , formatter_class = argparse . ArgumentDefaultsHelpFormatter ) opts . add_md_help_argument ( parser ) opts . preprocess_opts ( parser ) opt = parser . parse_args ( ) torch . manual_seed ( opt . seed ) check_existing_pt_files ( opt ) return opt


Divide the big corpus into shards and build dataset separately . This is currently only for data_type == text .

def build_save_in_shards(src_corpus, tgt_corpus, fields,
                         corpus_type, opt):
    """
    Divide the big corpus into shards, and build dataset separately.
    This is currently only for data_type=='text'.

    The reason we do this is to avoid taking up too much memory due
    to sucking in a huge corpus file.

    To tackle this, we only read in part of the corpus file of size
    `max_shard_size`(actually it is multiples of 64 bytes that equals
    or is slightly larger than this size), and process it into dataset,
    then write it to disk along the way. By doing this, we only focus on
    part of the corpus at any moment, thus effectively reducing memory use.
    According to test, this method can reduce memory footprint by ~50%.

    Note! As we process along the shards, previous shards might still
    stay in memory, but since we are done with them, and no more
    reference to them, if there is memory tight situation, the OS could
    easily reclaim these memory.

    If `max_shard_size` is 0 or is larger than the corpus size, it is
    effectively preprocessed into one dataset, i.e. no sharding.

    NOTE! `max_shard_size` is measuring the input corpus size, not the
    output pt file size. So a shard pt file consists of examples of size
    2 * `max_shard_size`(source + target).
    """

    corpus_size = os.path.getsize(src_corpus)
    if corpus_size > 10 * (1024 ** 2) and opt.max_shard_size == 0:
        logger.info("Warning. The corpus %s is larger than 10M bytes, "
                    "you can set '-max_shard_size' to process it by "
                    "small shards to use less memory." % src_corpus)

    if opt.max_shard_size != 0:
        logger.info(' * divide corpus into shards and build dataset '
                    'separately (shard_size = %d bytes).'
                    % opt.max_shard_size)

    ret_list = []
    src_iter = inputters.ShardedTextCorpusIterator(
        src_corpus, opt.src_seq_length_trunc,
        "src", opt.max_shard_size)
    tgt_iter = inputters.ShardedTextCorpusIterator(
        tgt_corpus, opt.tgt_seq_length_trunc,
        "tgt", opt.max_shard_size,
        assoc_iter=src_iter)

    index = 0
    while not src_iter.hit_end():
        index += 1
        dataset = inputters.TextDataset(
            fields, src_iter, tgt_iter,
            src_iter.num_feats, tgt_iter.num_feats,
            src_seq_length=opt.src_seq_length,
            tgt_seq_length=opt.tgt_seq_length,
            dynamic_dict=opt.dynamic_dict)

        # We save fields in vocab.pt separately, so make it empty.
        dataset.fields = []

        pt_file = "{:s}.{:s}.{:d}.pt".format(
            opt.save_data, corpus_type, index)
        logger.info(" * saving %s data shard to %s."
                    % (corpus_type, pt_file))
        torch.save(dataset, pt_file)

        ret_list.append(pt_file)

    return ret_list


Divide src_corpus and tgt_corpus into smaller multiples src_copus and tgt corpus files then build shards each shard will have opt . shard_size samples except last shard .

def build_save_in_shards_using_shards_size(src_corpus, tgt_corpus, fields,
                                           corpus_type, opt):
    """
    Divide src_corpus and tgt_corpus into smaller multiples
    src_copus and tgt corpus files, then build shards, each
    shard will have opt.shard_size samples except last shard.

    The reason we do this is to avoid taking up too much memory due
    to sucking in a huge corpus file.
    """

    src_data = open(src_corpus, "r", encoding="utf-8").readlines()
    tgt_data = open(tgt_corpus, "r", encoding="utf-8").readlines()

    src_corpus = "".join(src_corpus.split(".")[:-1])
    tgt_corpus = "".join(tgt_corpus.split(".")[:-1])

    for x in range(int(len(src_data) / opt.shard_size)):
        open(src_corpus + ".{0}.txt".format(x), "w",
             encoding="utf-8").writelines(
            src_data[x * opt.shard_size: (x + 1) * opt.shard_size])
        open(tgt_corpus + ".{0}.txt".format(x), "w",
             encoding="utf-8").writelines(
            tgt_data[x * opt.shard_size: (x + 1) * opt.shard_size])

    src_list = sorted(glob.glob(src_corpus + '.*.txt'))
    tgt_list = sorted(glob.glob(tgt_corpus + '.*.txt'))

    ret_list = []

    for index, src in enumerate(src_list):
        dataset = inputters.build_dataset(
            fields, opt.data_type,
            src_path=src,
            tgt_path=tgt_list[index],
            src_dir=opt.src_dir,
            src_seq_length=opt.src_seq_length,
            tgt_seq_length=opt.tgt_seq_length,
            src_seq_length_trunc=opt.src_seq_length_trunc,
            tgt_seq_length_trunc=opt.tgt_seq_length_trunc,
            dynamic_dict=opt.dynamic_dict,
            sample_rate=opt.sample_rate,
            window_size=opt.window_size,
            window_stride=opt.window_stride,
            window=opt.window,
            image_channel_size=opt.image_channel_size
        )

        pt_file = "{:s}.{:s}.{:d}.pt".format(
            opt.save_data, corpus_type, index)

        # We save fields in vocab.pt seperately, so make it empty.
        dataset.fields = []

        logger.info(" * saving %sth %s data image shard to %s."
                    % (index, corpus_type, pt_file))
        torch.save(dataset, pt_file)

        ret_list.append(pt_file)

        del dataset.examples
        gc.collect()
        del dataset
        gc.collect()

    return ret_list


def build_save_dataset(corpus_type, fields, opt):
    """ Building and saving the dataset """
    assert corpus_type in ['train', 'valid']

    if corpus_type == 'train':
        src_corpus = opt.train_src
        tgt_corpus = opt.train_tgt
    else:
        src_corpus = opt.valid_src
        tgt_corpus = opt.valid_tgt

    # Currently we only do preprocess sharding for corpus: data_type=='text'.
    if opt.data_type == 'text':
        return build_save_in_shards(
            src_corpus, tgt_corpus, fields,
            corpus_type, opt)

    if (opt.shard_size > 0):
        return build_save_in_shards_using_shards_size(src_corpus,
                                                      tgt_corpus,
                                                      fields,
                                                      corpus_type,
                                                      opt)

    # For data_type == 'img' or 'audio', currently we don't do
    # preprocess sharding. We only build a monolithic dataset.
    # But since the interfaces are uniform, it would be not hard
    # to do this should users need this feature.
    dataset = inputters.build_dataset(
        fields, opt.data_type,
        src_path=src_corpus,
        tgt_path=tgt_corpus,
        src_dir=opt.src_dir,
        src_seq_length=opt.src_seq_length,
        tgt_seq_length=opt.tgt_seq_length,
        src_seq_length_trunc=opt.src_seq_length_trunc,
        tgt_seq_length_trunc=opt.tgt_seq_length_trunc,
        dynamic_dict=opt.dynamic_dict,
        sample_rate=opt.sample_rate,
        window_size=opt.window_size,
        window_stride=opt.window_stride,
        window=opt.window,
        image_channel_size=opt.image_channel_size)

    # We save fields in vocab.pt seperately, so make it empty.
    dataset.fields = []

    pt_file = "{:s}.{:s}.pt".format(opt.save_data, corpus_type)
    logger.info(" * saving %s dataset to %s." % (corpus_type, pt_file))
    torch.save(dataset, pt_file)

    return [pt_file]


def build_save_vocab(train_dataset, fields, opt):
    """ Building and saving the vocab """
    fields = inputters.build_vocab(train_dataset, fields, opt.data_type,
                                   opt.share_vocab,
                                   opt.src_vocab,
                                   opt.src_vocab_size,
                                   opt.src_words_min_frequency,
                                   opt.tgt_vocab,
                                   opt.tgt_vocab_size,
                                   opt.tgt_words_min_frequency)

    # Can't save fields, so remove/reconstruct at training time.
    vocab_file = opt.save_data + '.vocab.pt'
    torch.save(inputters.save_fields_to_vocab(fields), vocab_file)


def run(opt, device_id, error_queue):
    """ run process """
    try:
        gpu_rank = onmt.utils.distributed.multi_init(opt, device_id)
        if gpu_rank != opt.gpu_ranks[device_id]:
            raise AssertionError("An error occurred in \
                  Distributed initialization")
        single_main(opt, device_id)
    except KeyboardInterrupt:
        pass  # killed by parent, do nothing
    except Exception:
        # propagate exception to parent process, keeping original traceback
        import traceback
        error_queue.put((opt.gpu_ranks[device_id], traceback.format_exc()))


def __init__(self, error_queue):
        """ init error handler """
        import signal
        import threading
        self.error_queue = error_queue
        self.children_pids = []
        self.error_thread = threading.Thread(
            target=self.error_listener, daemon=True)
        self.error_thread.start()
        signal.signal(signal.SIGUSR1, self.signal_handler)

def __init__ ( self , error_queue ) : import signal import threading self . error_queue = error_queue self . children_pids = [ ] self . error_thread = threading . Thread ( target = self . error_listener , daemon = True ) self . error_thread . start ( ) signal . signal ( signal . SIGUSR1 , self . signal_handler )


def add_child(self, pid):
        """ error handler """
        self.children_pids.append(pid)

def add_child ( self , pid ) : self . children_pids . append ( pid )


def error_listener(self):
        """ error listener """
        (rank, original_trace) = self.error_queue.get()
        self.error_queue.put((rank, original_trace))
        os.kill(os.getpid(), signal.SIGUSR1)

def error_listener ( self ) : ( rank , original_trace ) = self . error_queue . get ( ) self . error_queue . put ( ( rank , original_trace ) ) os . kill ( os . getpid ( ) , signal . SIGUSR1 )


def signal_handler(self, signalnum, stackframe):
        """ signal handler """
        for pid in self.children_pids:
            os.kill(pid, signal.SIGINT)  # kill children processes
        (rank, original_trace) = self.error_queue.get()
        msg = """\n\n-- Tracebacks above this line can probably
                 be ignored --\n\n"""
        msg += original_trace
        raise Exception(msg)


Build an Embeddings instance . Args : opt : the option in current environment . word_dict ( Vocab ) : words dictionary . feature_dicts ( [ Vocab ] optional ) : a list of feature dictionary . for_encoder ( bool ) : build Embeddings for encoder or decoder?

def build_embeddings(opt, word_dict, feature_dicts, for_encoder=True):
    """
    Build an Embeddings instance.
    Args:
        opt: the option in current environment.
        word_dict(Vocab): words dictionary.
        feature_dicts([Vocab], optional): a list of feature dictionary.
        for_encoder(bool): build Embeddings for encoder or decoder?
    """
    if for_encoder:
        embedding_dim = opt.src_word_vec_size
    else:
        embedding_dim = opt.tgt_word_vec_size

    word_padding_idx = word_dict.stoi[inputters.PAD_WORD]
    num_word_embeddings = len(word_dict)

    feats_padding_idx = [feat_dict.stoi[inputters.PAD_WORD]
                         for feat_dict in feature_dicts]
    num_feat_embeddings = [len(feat_dict) for feat_dict in
                           feature_dicts]

    return Embeddings(word_vec_size=embedding_dim,
                      position_encoding=opt.position_encoding,
                      feat_merge=opt.feat_merge,
                      feat_vec_exponent=opt.feat_vec_exponent,
                      feat_vec_size=opt.feat_vec_size,
                      dropout=opt.dropout,
                      word_padding_idx=word_padding_idx,
                      feat_padding_idx=feats_padding_idx,
                      word_vocab_size=num_word_embeddings,
                      feat_vocab_sizes=num_feat_embeddings,
                      sparse=opt.optim == "sparseadam")

def build_embeddings ( opt , word_dict , feature_dicts , for_encoder = True ) : if for_encoder : embedding_dim = opt . src_word_vec_size else : embedding_dim = opt . tgt_word_vec_size word_padding_idx = word_dict . stoi [ inputters . PAD_WORD ] num_word_embeddings = len ( word_dict ) feats_padding_idx = [ feat_dict . stoi [ inputters . PAD_WORD ] for feat_dict in feature_dicts ] num_feat_embeddings = [ len ( feat_dict ) for feat_dict in feature_dicts ] return Embeddings ( word_vec_size = embedding_dim , position_encoding = opt . position_encoding , feat_merge = opt . feat_merge , feat_vec_exponent = opt . feat_vec_exponent , feat_vec_size = opt . feat_vec_size , dropout = opt . dropout , word_padding_idx = word_padding_idx , feat_padding_idx = feats_padding_idx , word_vocab_size = num_word_embeddings , feat_vocab_sizes = num_feat_embeddings , sparse = opt . optim == "sparseadam" )


Various encoder dispatcher function . Args : opt : the option in current environment . embeddings ( Embeddings ) : vocab embeddings for this encoder .

def build_encoder(opt, embeddings):
    """
    Various encoder dispatcher function.
    Args:
        opt: the option in current environment.
        embeddings (Embeddings): vocab embeddings for this encoder.
    """
    if opt.encoder_type == "transformer":
        # return TransformerEncoder(opt.enc_layers, opt.rnn_size,
        #                           opt.heads, opt.transformer_ff,
        #                           opt.dropout, embeddings)
        return None
    elif opt.encoder_type == "cnn":
        return CNNEncoder(opt.enc_layers, opt.rnn_size,
                          opt.cnn_kernel_width,
                          opt.dropout, embeddings)
    elif opt.encoder_type == "mean":
        return MeanEncoder(opt.enc_layers, embeddings)
    else:
        # "rnn" or "brnn"
        return RNNEncoder(opt.rnn_type, opt.brnn, opt.enc_layers,
                          opt.rnn_size, opt.dropout, embeddings,
                          opt.bridge)

#                           opt.heads, opt.transformer_ff,

#                           opt.dropout, embeddings)


Various decoder dispatcher function . Args : opt : the option in current environment . embeddings ( Embeddings ) : vocab embeddings for this decoder .

def build_decoder(opt, embeddings):
    """
    Various decoder dispatcher function.
    Args:
        opt: the option in current environment.
        embeddings (Embeddings): vocab embeddings for this decoder.
    """
    if opt.decoder_type == "transformer":
        return TransformerDecoder(opt.dec_layers, opt.rnn_size,
                                  opt.heads, opt.transformer_ff,
                                  opt.global_attention, opt.copy_attn,
                                  opt.self_attn_type,
                                  opt.dropout, embeddings)
    elif opt.decoder_type == "cnn":
        return CNNDecoder(opt.dec_layers, opt.rnn_size,
                          opt.global_attention, opt.copy_attn,
                          opt.cnn_kernel_width, opt.dropout,
                          embeddings)
    elif opt.input_feed:
        # others
        return InputFeedRNNDecoder(opt.rnn_type, opt.brnn,
                                   opt.dec_layers, opt.rnn_size,
                                   opt.global_attention,
                                   opt.global_attention_function,
                                   opt.coverage_attn,
                                   opt.context_gate,
                                   opt.copy_attn,
                                   opt.dropout,
                                   embeddings,
                                   opt.reuse_copy_attn)
    else:
        return StdRNNDecoder(opt.rnn_type, opt.brnn,
                             opt.dec_layers, opt.rnn_size,
                             opt.global_attention,
                             opt.global_attention_function,
                             opt.coverage_attn,
                             opt.context_gate,
                             opt.copy_attn,
                             opt.dropout,
                             embeddings,
                             opt.reuse_copy_attn)

def
build_decoder
(
opt
,
embeddings
)
:
if
opt
.
decoder_type
==
"transformer"
:
return
TransformerDecoder
(
opt
.
dec_layers
,
opt
.
rnn_size
,
opt
.
heads
,
opt
.
transformer_ff
,
opt
.
global_attention
,
opt
.
copy_attn
,
opt
.
self_attn_type
,
opt
.
dropout
,
embeddings
)
elif
opt
.
decoder_type
==
"cnn"
:
return
CNNDecoder
(
opt
.
dec_layers
,
opt
.
rnn_size
,
opt
.
global_attention
,
opt
.
copy_attn
,
opt
.
cnn_kernel_width
,
opt
.
dropout
,
embeddings
)
elif
opt
.
input_feed
:
# others
return
InputFeedRNNDecoder
(
opt
.
rnn_type
,
opt
.
brnn
,
opt
.
dec_layers
,
opt
.
rnn_size
,
opt
.
global_attention
,
opt
.
global_attention_function
,
opt
.
coverage_attn
,
opt
.
context_gate
,
opt
.
copy_attn
,
opt
.
dropout
,
embeddings
,
opt
.
reuse_copy_attn
)
else
:
return
StdRNNDecoder
(
opt
.
rnn_type
,
opt
.
brnn
,
opt
.
dec_layers
,
opt
.
rnn_size
,
opt
.
global_attention
,
opt
.
global_attention_function
,
opt
.
coverage_attn
,
opt
.
context_gate
,
opt
.
copy_attn
,
opt
.
dropout
,
embeddings
,
opt
.
reuse_copy_attn
)


Args : model_opt : the option loaded from checkpoint . fields : Field objects for the model . gpu ( bool ) : whether to use gpu . checkpoint : the model gnerated by train phase or a resumed snapshot model from a stopped training . Returns : the NMTModel .

def build_base_model(model_opt, fields, gpu, checkpoint=None):
    """
    Args:
        model_opt: the option loaded from checkpoint.
        fields: `Field` objects for the model.
        gpu(bool): whether to use gpu.
        checkpoint: the model gnerated by train phase, or a resumed snapshot
                    model from a stopped training.
    Returns:
        the NMTModel.
    """
    assert model_opt.model_type in ["text", "img", "audio"], \
        ("Unsupported model type %s" % (model_opt.model_type))

    # Build encoder.
    if model_opt.model_type == "text":
        src_dict = fields["src"].vocab

        feature_dicts = inputters.collect_feature_vocabs(fields, 'src')
        src_embeddings = build_embeddings(model_opt, src_dict, feature_dicts)
        encoder = build_encoder(model_opt, src_embeddings)

    elif model_opt.model_type == "img":
        if ("image_channel_size" not in model_opt.__dict__):
            image_channel_size = 3
        else:
            image_channel_size = model_opt.image_channel_size

        encoder = ImageEncoder(model_opt.enc_layers,
                               model_opt.brnn,
                               model_opt.rnn_size,
                               model_opt.dropout,
                               image_channel_size)
    elif model_opt.model_type == "audio":
        encoder = AudioEncoder(model_opt.enc_layers,
                               model_opt.brnn,
                               model_opt.rnn_size,
                               model_opt.dropout,
                               model_opt.sample_rate,
                               model_opt.window_size)

    model_parameters = filter(lambda p: p.requires_grad, encoder.parameters())
    params = sum([np.prod(p.size()) for p in model_parameters])
    # Build decoder.
    tgt_dict = fields["tgt"].vocab
    feature_dicts = inputters.collect_feature_vocabs(fields, 'tgt')
    tgt_embeddings = build_embeddings(model_opt, tgt_dict,
                                      feature_dicts, for_encoder=False)

    # Share the embedding matrix - preprocess with share_vocab required.
    if model_opt.share_embeddings:
        # src/tgt vocab should be the same if `-share_vocab` is specified.
        if src_dict != tgt_dict:
            raise AssertionError('The `-share_vocab` should be set during '
                                 'preprocess if you use share_embeddings!')

        tgt_embeddings.word_lut.weight = src_embeddings.word_lut.weight

    decoder = build_decoder(model_opt, tgt_embeddings)

    # Build NMTModel(= encoder + decoder).
    device = torch.device("cuda" if gpu else "cpu")

    model = onmt.models.NMTModel(encoder, decoder)
    model.model_type = model_opt.model_type

    # Build Generator.
    if not model_opt.copy_attn:
        if model_opt.generator_function == "sparsemax":
            gen_func = onmt.modules.sparse_activations.LogSparsemax(dim=-1)
        else:
            gen_func = nn.LogSoftmax(dim=-1)
        generator = nn.Sequential(
            nn.Linear(model_opt.rnn_size, len(fields["tgt"].vocab)), gen_func
        )
        if model_opt.share_decoder_embeddings:
            generator[0].weight = decoder.embeddings.word_lut.weight
    else:
        generator = CopyGenerator(model_opt.rnn_size,
                                  fields["tgt"].vocab)

    # Load the model states from checkpoint or initialize them.
    if checkpoint is not None:
        model.load_state_dict(checkpoint['model'], strict=False)
        generator.load_state_dict(checkpoint['generator'])
    else:
        if model_opt.param_init != 0.0:
            for p in model.parameters():
                p.data.uniform_(-model_opt.param_init, model_opt.param_init)
            for p in generator.parameters():
                p.data.uniform_(-model_opt.param_init, model_opt.param_init)
        if model_opt.param_init_glorot:
            for p in model.parameters():
                if p.dim() > 1:
                    xavier_uniform_(p)
            for p in generator.parameters():
                if p.dim() > 1:
                    xavier_uniform_(p)

        if hasattr(model.encoder, 'embeddings'):
            model.encoder.embeddings.load_pretrained_vectors(
                model_opt.pre_word_vecs_enc, model_opt.fix_word_vecs_enc)
        if hasattr(model.decoder, 'embeddings'):
            model.decoder.embeddings.load_pretrained_vectors(
                model_opt.pre_word_vecs_dec, model_opt.fix_word_vecs_dec)

    # Add generator to model (this registers it as parameter of model).
    model.generator = generator
    model.to(device)

    return model


def build_model(model_opt, opt, fields, checkpoint):
    """ Build the Model """
    logger.info('Building model...')
    model = build_base_model(model_opt, fields,
                             use_gpu(opt), checkpoint)
    logger.info(model)
    return model

def
build_model
(
model_opt
,
opt
,
fields
,
checkpoint
)
:
logger
.
info
(
'Building model...'
)
model
=
build_base_model
(
model_opt
,
fields
,
use_gpu
(
opt
)
,
checkpoint
)
logger
.
info
(
model
)
return
model


Simplify Trainer creation based on user opt s *

def build_trainer(opt, device_id, model, fields,
                  optim, data_type, model_saver=None):
    """
    Simplify `Trainer` creation based on user `opt`s*

    Args:
        opt (:obj:`Namespace`): user options (usually from argument parsing)
        model (:obj:`onmt.models.NMTModel`): the model to train
        fields (dict): dict of fields
        optim (:obj:`onmt.utils.Optimizer`): optimizer used during training
        data_type (str): string describing the type of data
            e.g. "text", "img", "audio"
        model_saver(:obj:`onmt.models.ModelSaverBase`): the utility object
            used to save the model
    """
    # fine

    train_loss = onmt.utils.loss.build_loss_compute(
        model, fields["tgt"].vocab, opt)
    valid_loss = onmt.utils.loss.build_loss_compute(
        model, fields["tgt"].vocab, opt, train=False)

    trunc_size = opt.truncated_decoder  # Badly named...
    shard_size = opt.max_generator_batches
    norm_method = opt.normalization
    grad_accum_count = opt.accum_count
    n_gpu = opt.world_size
    if device_id >= 0:
        gpu_rank = opt.gpu_ranks[device_id]
    else:
        gpu_rank = 0
        n_gpu = 0
    gpu_verbose_level = opt.gpu_verbose_level

    report_manager = onmt.utils.build_report_manager(opt)
    trainer = onmt.Trainer(model, train_loss, valid_loss, optim, trunc_size,
                           shard_size, data_type, norm_method,
                           grad_accum_count, n_gpu, gpu_rank,
                           gpu_verbose_level, report_manager,
                           model_saver=model_saver)
    return trainer

def
build_trainer
(
opt
,
device_id
,
model
,
fields
,
optim
,
data_type
,
model_saver
=
)
:
# fine
train_loss
=
onmt
.
utils
.
loss
.
build_loss_compute
(
model
,
fields
[
"tgt"
]
.
vocab
,
opt
)
valid_loss
=
onmt
.
utils
.
loss
.
build_loss_compute
(
model
,
fields
[
"tgt"
]
.
vocab
,
opt
,
train
=
False
)
trunc_size
=
opt
.
truncated_decoder
# Badly named...
shard_size
=
opt
.
max_generator_batches
norm_method
=
opt
.
normalization
grad_accum_count
=
opt
.
accum_count
n_gpu
=
opt
.
world_size
if
device_id
>=
0
:
gpu_rank
=
opt
.
gpu_ranks
[
device_id
]
else
:
gpu_rank
=
0
n_gpu
=
0
gpu_verbose_level
=
opt
.
gpu_verbose_level
report_manager
=
onmt
.
utils
.
build_report_manager
(
opt
)
trainer
=
onmt
.
Trainer
(
model
,
train_loss
,
valid_loss
,
optim
,
trunc_size
,
shard_size
,
data_type
,
norm_method
,
grad_accum_count
,
n_gpu
,
gpu_rank
,
gpu_verbose_level
,
report_manager
,
model_saver
=
model_saver
)
return
trainer


The main training loops . by iterating over training data ( i . e . train_iter_fct ) and running validation ( i . e . iterating over valid_iter_fct

def train(self, train_iter_fct, valid_iter_fct, train_steps, valid_steps):
        """
        The main training loops.
        by iterating over training data (i.e. `train_iter_fct`)
        and running validation (i.e. iterating over `valid_iter_fct`

        Args:
            train_iter_fct(function): a function that returns the train
                iterator. e.g. something like
                train_iter_fct = lambda: generator(*args, **kwargs)
            valid_iter_fct(function): same as train_iter_fct, for valid data
            train_steps(int):
            valid_steps(int):
            save_checkpoint_steps(int):

        Return:
            None
        """
        logger.info('Start training...')

        step = self.optim._step + 1
        true_batchs = []
        accum = 0
        normalization = 0
        train_iter = train_iter_fct()

        total_stats = onmt.utils.Statistics()
        report_stats = onmt.utils.Statistics()
        self._start_report_manager(start_time=total_stats.start_time)

        while step <= train_steps:
            reduce_counter = 0
            for i, batch in enumerate(train_iter):

                # import pdb; pdb.set_trace()

                if self.n_gpu == 0 or (i % self.n_gpu == self.gpu_rank):
                    if self.gpu_verbose_level > 1:
                        logger.info("GpuRank %d: index: %d accum: %d"
                                    % (self.gpu_rank, i, accum))

                    true_batchs.append(batch)

                    if self.norm_method == "tokens":
                        num_tokens = batch.tgt[1:].ne(
                            self.train_loss.padding_idx).sum()
                        normalization += num_tokens.item()
                    else:
                        normalization += batch.batch_size
                    accum += 1
                    if accum == self.grad_accum_count:
                        reduce_counter += 1
                        if self.gpu_verbose_level > 0:
                            logger.info("GpuRank %d: reduce_counter: %d \
                                        n_minibatch %d"
                                        % (self.gpu_rank, reduce_counter,
                                           len(true_batchs)))
                        if self.n_gpu > 1:
                            normalization = sum(onmt.utils.distributed
                                                .all_gather_list
                                                (normalization))

                        self._gradient_accumulation(
                            true_batchs, normalization, total_stats,
                            report_stats)

                        report_stats = self._maybe_report_training(
                            step, train_steps,
                            self.optim.learning_rate,
                            report_stats)

                        true_batchs = []
                        accum = 0
                        normalization = 0
                        if (step % valid_steps == 0):
                            if self.gpu_verbose_level > 0:
                                logger.info('GpuRank %d: validate step %d'
                                            % (self.gpu_rank, step))
                            valid_iter = valid_iter_fct()
                            with torch.no_grad():
                                valid_stats = self.validate(valid_iter)
                            if self.gpu_verbose_level > 0:
                                logger.info('GpuRank %d: gather valid stat \
                                            step %d' % (self.gpu_rank, step))
                            valid_stats = self._maybe_gather_stats(valid_stats)
                            if self.gpu_verbose_level > 0:
                                logger.info('GpuRank %d: report stat step %d'
                                            % (self.gpu_rank, step))
                            self._report_step(self.optim.learning_rate,
                                              step, valid_stats=valid_stats)

                        if self.gpu_rank == 0:
                            self._maybe_save(step)
                        step += 1
                        if step > train_steps:
                            break
            if self.gpu_verbose_level > 0:
                logger.info('GpuRank %d: we completed an epoch \
                            at step %d' % (self.gpu_rank, step))
            train_iter = train_iter_fct()

        return total_stats


Validate model . valid_iter : validate data iterator Returns : : obj : nmt . Statistics : validation loss statistics

def validate(self, valid_iter):
        """ Validate model.
            valid_iter: validate data iterator
        Returns:
            :obj:`nmt.Statistics`: validation loss statistics
        """
        # Set model in validating mode.
        self.model.eval()

        stats = onmt.utils.Statistics()

        for batch in valid_iter:
            src = inputters.make_features(batch, 'src', self.data_type)
            if self.data_type == 'text':
                _, src_lengths = batch.src
            else:
                src_lengths = None

            tgt = inputters.make_features(batch, 'tgt')

            # F-prop through the model.
            src_sents = batch.src_sents
            outputs, attns, _ = self.model(src, tgt, src_sents,src_lengths)

            # Compute loss.
            batch_stats = self.valid_loss.monolithic_compute_loss(
                batch, outputs, attns)

            # Update statistics.
            stats.update(batch_stats)

        # Set model back to training mode.
        self.model.train()

        return stats

def
validate
(
self
,
valid_iter
)
:
# Set model in validating mode.
self
.
model
.
eval
(
)
stats
=
onmt
.
utils
.
Statistics
(
)
for
batch
in
valid_iter
:
src
=
inputters
.
make_features
(
batch
,
'src'
,
self
.
data_type
)
if
self
.
data_type
==
'text'
:
_
,
src_lengths
=
batch
.
src
else
:
src_lengths
=
tgt
=
inputters
.
make_features
(
batch
,
'tgt'
)
# F-prop through the model.
src_sents
=
batch
.
src_sents
outputs
,
attns
,
_
=
self
.
model
(
src
,
tgt
,
src_sents
,
src_lengths
)
# Compute loss.
batch_stats
=
self
.
valid_loss
.
monolithic_compute_loss
(
batch
,
outputs
,
attns
)
# Update statistics.
stats
.
update
(
batch_stats
)
# Set model back to training mode.
self
.
model
.
train
(
)
return
stats


Simple function to start report manager ( if any )

def _start_report_manager(self, start_time=None):
        """
        Simple function to start report manager (if any)
        """
        if self.report_manager is not None:
            if start_time is None:
                self.report_manager.start()
            else:
                self.report_manager.start_time = start_time

def _start_report_manager ( self , start_time = None ) : if self . report_manager is not None : if start_time is None : self . report_manager . start ( ) else : self . report_manager . start_time = start_time


Gather statistics in multi - processes cases

def _maybe_gather_stats(self, stat):
        """
        Gather statistics in multi-processes cases

        Args:
            stat(:obj:onmt.utils.Statistics): a Statistics object to gather
                or None (it returns None in this case)

        Returns:
            stat: the updated (or unchanged) stat object
        """
        if stat is not None and self.n_gpu > 1:
            return onmt.utils.Statistics.all_gather_stats(stat)
        return stat

def _maybe_gather_stats ( self , stat ) : if stat is not None and self . n_gpu > 1 : return onmt . utils . Statistics . all_gather_stats ( stat ) return stat


Simple function to report training stats ( if report_manager is set ) see onmt . utils . ReportManagerBase . report_training for doc

def _maybe_report_training(self, step, num_steps, learning_rate,
                               report_stats):
        """
        Simple function to report training stats (if report_manager is set)
        see `onmt.utils.ReportManagerBase.report_training` for doc
        """
        if self.report_manager is not None:
            return self.report_manager.report_training(
                step, num_steps, learning_rate, report_stats,
                multigpu=self.n_gpu > 1)

def _maybe_report_training ( self , step , num_steps , learning_rate , report_stats ) : if self . report_manager is not None : return self . report_manager . report_training ( step , num_steps , learning_rate , report_stats , multigpu = self . n_gpu > 1 )


Simple function to report stats ( if report_manager is set ) see onmt . utils . ReportManagerBase . report_step for doc

def _report_step(self, learning_rate, step, train_stats=None,
                     valid_stats=None):
        """
        Simple function to report stats (if report_manager is set)
        see `onmt.utils.ReportManagerBase.report_step` for doc
        """
        if self.report_manager is not None:
            return self.report_manager.report_step(
                learning_rate, step, train_stats=train_stats,
                valid_stats=valid_stats)

def _report_step ( self , learning_rate , step , train_stats = None , valid_stats = None ) : if self . report_manager is not None : return self . report_manager . report_step ( learning_rate , step , train_stats = train_stats , valid_stats = valid_stats )


def _maybe_save(self, step):
        """
        Save the model if a model saver is set
        """
        if self.model_saver is not None:
            self.model_saver.maybe_save(step)

def _maybe_save ( self , step ) : if self . model_saver is not None : self . model_saver . maybe_save ( step )


These options are passed to the construction of the model . Be careful with these as they will be used during translation .

def model_opts(parser):
    """
    These options are passed to the construction of the model.
    Be careful with these as they will be used during translation.
    """

    # Embedding Options
    group = parser.add_argument_group('Model-Embeddings')
    group.add_argument('-src_word_vec_size', type=int, default=500,
                       help='Word embedding size for src.')
    group.add_argument('-tgt_word_vec_size', type=int, default=500,
                       help='Word embedding size for tgt.')
    group.add_argument('-word_vec_size', type=int, default=-1,
                       help='Word embedding size for src and tgt.')

    group.add_argument('-share_decoder_embeddings', action='store_true',
                       help="""Use a shared weight matrix for the input and
                       output word  embeddings in the decoder.""")
    group.add_argument('-share_embeddings', action='store_true',
                       help="""Share the word embeddings between encoder
                       and decoder. Need to use shared dictionary for this
                       option.""")
    group.add_argument('-position_encoding', action='store_true',
                       help="""Use a sin to mark relative words positions.
                       Necessary for non-RNN style models.
                       """)

    group = parser.add_argument_group('Model-Embedding Features')
    group.add_argument('-feat_merge', type=str, default='concat',
                       choices=['concat', 'sum', 'mlp'],
                       help="""Merge action for incorporating features embeddings.
                       Options [concat|sum|mlp].""")
    group.add_argument('-feat_vec_size', type=int, default=-1,
                       help="""If specified, feature embedding sizes
                       will be set to this. Otherwise, feat_vec_exponent
                       will be used.""")
    group.add_argument('-feat_vec_exponent', type=float, default=0.7,
                       help="""If -feat_merge_size is not set, feature
                       embedding sizes will be set to N^feat_vec_exponent
                       where N is the number of values the feature takes.""")

    # Encoder-Decoder Options
    group = parser.add_argument_group('Model- Encoder-Decoder')
    group.add_argument('-model_type', default='text',
                       help="""Type of source model to use. Allows
                       the system to incorporate non-text inputs.
                       Options are [text|img|audio].""")

    group.add_argument('-encoder_type', type=str, default='rnn',
                       choices=['rnn', 'brnn', 'mean', 'transformer', 'cnn'],
                       help="""Type of encoder layer to use. Non-RNN layers
                       are experimental. Options are
                       [rnn|brnn|mean|transformer|cnn].""")
    group.add_argument('-decoder_type', type=str, default='rnn',
                       choices=['rnn', 'transformer', 'cnn'],
                       help="""Type of decoder layer to use. Non-RNN layers
                       are experimental. Options are
                       [rnn|transformer|cnn].""")

    group.add_argument('-layers', type=int, default=-1,
                       help='Number of layers in enc/dec.')
    group.add_argument('-enc_layers', type=int, default=2,
                       help='Number of layers in the encoder')
    group.add_argument('-dec_layers', type=int, default=2,
                       help='Number of layers in the decoder')
    group.add_argument('-rnn_size', type=int, default=500,
                       help='Size of rnn hidden states')
    group.add_argument('-cnn_kernel_width', type=int, default=3,
                       help="""Size of windows in the cnn, the kernel_size is
                       (cnn_kernel_width, 1) in conv layer""")

    group.add_argument('-input_feed', type=int, default=1,
                       help="""Feed the context vector at each time step as
                       additional input (via concatenation with the word
                       embeddings) to the decoder.""")
    group.add_argument('-bridge', action="store_true",
                       help="""Have an additional layer between the last encoder
                       state and the first decoder state""")
    group.add_argument('-rnn_type', type=str, default='LSTM',
                       choices=['LSTM', 'GRU', 'SRU'],
                       action=CheckSRU,
                       help="""The gate type to use in the RNNs""")
    # group.add_argument('-residual',   action="store_true",
    #                     help="Add residual connections between RNN layers.")

    group.add_argument('-brnn', action=DeprecateAction,
                       help="Deprecated, use `encoder_type`.")

    group.add_argument('-context_gate', type=str, default=None,
                       choices=['source', 'target', 'both'],
                       help="""Type of context gate to use.
                       Do not select for no context gate.""")

    # Attention options
    group = parser.add_argument_group('Model- Attention')
    group.add_argument('-global_attention', type=str, default='general',
                       choices=['dot', 'general', 'mlp'],
                       help="""The attention type to use:
                       dotprod or general (Luong) or MLP (Bahdanau)""")
    group.add_argument('-global_attention_function', type=str,
                       default="softmax", choices=["softmax", "sparsemax"])
    group.add_argument('-self_attn_type', type=str, default="scaled-dot",
                       help="""Self attention type in Transformer decoder
                       layer -- currently "scaled-dot" or "average" """)
    group.add_argument('-heads', type=int, default=8,
                       help='Number of heads for transformer self-attention')
    group.add_argument('-transformer_ff', type=int, default=2048,
                       help='Size of hidden transformer feed-forward')

    # Generator and loss options.
    group.add_argument('-copy_attn', action="store_true",
                       help='Train copy attention layer.')
    group.add_argument('-generator_function', default="log_softmax",
                       choices=["log_softmax", "sparsemax"],
                       help="""Which function to use for generating
                       probabilities over the target vocabulary (choices:
                       log_softmax, sparsemax)""")
    group.add_argument('-copy_attn_force', action="store_true",
                       help='When available, train to copy.')
    group.add_argument('-reuse_copy_attn', action="store_true",
                       help="Reuse standard attention for copy")
    group.add_argument('-copy_loss_by_seqlength', action="store_true",
                       help="Divide copy loss by length of sequence")
    group.add_argument('-coverage_attn', action="store_true",
                       help='Train a coverage attention layer.')
    group.add_argument('-lambda_coverage', type=float, default=1,
                       help='Lambda value for coverage.')


def preprocess_opts(parser):
    """ Pre-procesing options """
    # Data options
    group = parser.add_argument_group('Data')
    group.add_argument('-data_type', default="text",
                       help="""Type of the source input.
                       Options are [text|img].""")

    group.add_argument('-train_src', required=True,
                       help="Path to the training source data")
    group.add_argument('-train_tgt', required=True,
                       help="Path to the training target data")
    group.add_argument('-valid_src', required=True,
                       help="Path to the validation source data")
    group.add_argument('-valid_tgt', required=True,
                       help="Path to the validation target data")

    group.add_argument('-src_dir', default="",
                       help="Source directory for image or audio files.")

    group.add_argument('-save_data', required=True,
                       help="Output file for the prepared data")

    group.add_argument('-max_shard_size', type=int, default=0,
                       help="""For text corpus of large volume, it will
                       be divided into shards of this size to preprocess.
                       If 0, the data will be handled as a whole. The unit
                       is in bytes. Optimal value should be multiples of
                       64 bytes. A commonly used sharding value is 131072000.
                       It is recommended to ensure the corpus is shuffled
                       before sharding.""")

    group.add_argument('-shard_size', type=int, default=0,
                       help="""Divide src_corpus and tgt_corpus into
                       smaller multiple src_copus and tgt corpus files, then
                       build shards, each shard will have
                       opt.shard_size samples except last shard.
                       shard_size=0 means no segmentation
                       shard_size>0 means segment dataset into multiple shards,
                       each shard has shard_size samples""")

    # Dictionary options, for text corpus

    group = parser.add_argument_group('Vocab')
    group.add_argument('-src_vocab', default="",
                       help="""Path to an existing source vocabulary. Format:
                       one word per line.""")
    group.add_argument('-tgt_vocab', default="",
                       help="""Path to an existing target vocabulary. Format:
                       one word per line.""")
    group.add_argument('-features_vocabs_prefix', type=str, default='',
                       help="Path prefix to existing features vocabularies")
    group.add_argument('-src_vocab_size', type=int, default=50000,
                       help="Size of the source vocabulary")
    group.add_argument('-tgt_vocab_size', type=int, default=50000,
                       help="Size of the target vocabulary")

    group.add_argument('-src_words_min_frequency', type=int, default=0)
    group.add_argument('-tgt_words_min_frequency', type=int, default=0)

    group.add_argument('-dynamic_dict', action='store_true',
                       help="Create dynamic dictionaries")
    group.add_argument('-share_vocab', action='store_true',
                       help="Share source and target vocabulary")

    # Truncation options, for text corpus
    group = parser.add_argument_group('Pruning')
    group.add_argument('-src_seq_length', type=int, default=50,
                       help="Maximum source sequence length")
    group.add_argument('-src_seq_length_trunc', type=int, default=0,
                       help="Truncate source sequence length.")
    group.add_argument('-tgt_seq_length', type=int, default=50,
                       help="Maximum target sequence length to keep.")
    group.add_argument('-tgt_seq_length_trunc', type=int, default=0,
                       help="Truncate target sequence length.")
    group.add_argument('-lower', action='store_true', help='lowercase data')

    # Data processing options
    group = parser.add_argument_group('Random')
    group.add_argument('-shuffle', type=int, default=1,
                       help="Shuffle data")
    group.add_argument('-seed', type=int, default=3435,
                       help="Random seed")

    group = parser.add_argument_group('Logging')
    group.add_argument('-report_every', type=int, default=100000,
                       help="Report status every this many sentences")
    group.add_argument('-log_file', type=str, default="",
                       help="Output logs to a file under this path.")

    # Options most relevant to speech
    group = parser.add_argument_group('Speech')
    group.add_argument('-sample_rate', type=int, default=16000,
                       help="Sample rate.")
    group.add_argument('-window_size', type=float, default=.02,
                       help="Window size for spectrogram in seconds.")
    group.add_argument('-window_stride', type=float, default=.01,
                       help="Window stride for spectrogram in seconds.")
    group.add_argument('-window', default='hamming',
                       help="Window type for spectrogram generation.")

    # Option most relevant to image input
    group.add_argument('-image_channel_size', type=int, default=3,
                       choices=[3, 1],
                       help="""Using grayscale image can training
                       model faster and smaller""")


def train_opts(parser):
    """ Training and saving options """

    group = parser.add_argument_group('General')
    group.add_argument('-data', required=True,
                       help="""Path prefix to the ".train.pt" and
                       ".valid.pt" file path from preprocess.py""")

    group.add_argument('-save_model', default='model',
                       help="""Model filename (the model will be saved as
                       <save_model>_N.pt where N is the number
                       of steps""")

    group.add_argument('-save_checkpoint_steps', type=int, default=5000,
                       help="""Save a checkpoint every X steps""")
    group.add_argument('-keep_checkpoint', type=int, default=-1,
                       help="""Keep X checkpoints (negative: keep all)""")

    # GPU
    group.add_argument('-gpuid', default=[], nargs='+', type=int,
                       help="Deprecated see world_size and gpu_ranks.")
    group.add_argument('-gpu_ranks', default=[], nargs='+', type=int,
                       help="list of ranks of each process.")
    group.add_argument('-world_size', default=1, type=int,
                       help="total number of distributed processes.")
    group.add_argument('-gpu_backend', default='nccl', nargs='+', type=str,
                       help="Type of torch distributed backend")
    group.add_argument('-gpu_verbose_level', default=0, type=int,
                       help="Gives more info on each process per GPU.")
    group.add_argument('-master_ip', default="localhost", type=str,
                       help="IP of master for torch.distributed training.")
    group.add_argument('-master_port', default=10000, type=int,
                       help="Port of master for torch.distributed training.")

    group.add_argument('-seed', type=int, default=-1,
                       help="""Random seed used for the experiments
                       reproducibility.""")

    # Init options
    group = parser.add_argument_group('Initialization')
    group.add_argument('-param_init', type=float, default=0.1,
                       help="""Parameters are initialized over uniform distribution
                       with support (-param_init, param_init).
                       Use 0 to not use initialization""")
    group.add_argument('-param_init_glorot', action='store_true',
                       help="""Init parameters with xavier_uniform.
                       Required for transfomer.""")

    group.add_argument('-train_from', default='', type=str,
                       help="""If training from a checkpoint then this is the
                       path to the pretrained model's state_dict.""")

    # Pretrained word vectors
    group.add_argument('-pre_word_vecs_enc',
                       help="""If a valid path is specified, then this will load
                       pretrained word embeddings on the encoder side.
                       See README for specific formatting instructions.""")
    group.add_argument('-pre_word_vecs_dec',
                       help="""If a valid path is specified, then this will load
                       pretrained word embeddings on the decoder side.
                       See README for specific formatting instructions.""")
    # Fixed word vectors
    group.add_argument('-fix_word_vecs_enc',
                       action='store_true',
                       help="Fix word embeddings on the encoder side.")
    group.add_argument('-fix_word_vecs_dec',
                       action='store_true',
                       help="Fix word embeddings on the decoder side.")

    # Optimization options
    group = parser.add_argument_group('Optimization- Type')
    group.add_argument('-batch_size', type=int, default=64,
                       help='Maximum batch size for training')
    group.add_argument('-batch_type', default='sents',
                       choices=["sents", "tokens"],
                       help="""Batch grouping for batch_size. Standard
                               is sents. Tokens will do dynamic batching""")
    group.add_argument('-normalization', default='sents',
                       choices=["sents", "tokens"],
                       help='Normalization method of the gradient.')
    group.add_argument('-accum_count', type=int, default=1,
                       help="""Accumulate gradient this many times.
                       Approximately equivalent to updating
                       batch_size * accum_count batches at once.
                       Recommended for Transformer.""")
    group.add_argument('-valid_steps', type=int, default=10000,
                       help='Perfom validation every X steps')
    group.add_argument('-valid_batch_size', type=int, default=32,
                       help='Maximum batch size for validation')
    group.add_argument('-max_generator_batches', type=int, default=32,
                       help="""Maximum batches of words in a sequence to run
                        the generator on in parallel. Higher is faster, but
                        uses more memory.""")
    group.add_argument('-train_steps', type=int, default=100000,
                       help='Number of training steps')
    group.add_argument('-epochs', type=int, default=0,
                       help='Deprecated epochs see train_steps')
    group.add_argument('-optim', default='sgd',
                       choices=['sgd', 'adagrad', 'adadelta', 'adam',
                                'sparseadam'],
                       help="""Optimization method.""")
    group.add_argument('-adagrad_accumulator_init', type=float, default=0,
                       help="""Initializes the accumulator values in adagrad.
                       Mirrors the initial_accumulator_value option
                       in the tensorflow adagrad (use 0.1 for their default).
                       """)
    group.add_argument('-max_grad_norm', type=float, default=5,
                       help="""If the norm of the gradient vector exceeds this,
                       renormalize it to have the norm equal to
                       max_grad_norm""")
    group.add_argument('-dropout', type=float, default=0.3,
                       help="Dropout probability; applied in LSTM stacks.")
    group.add_argument('-truncated_decoder', type=int, default=0,
                       help="""Truncated bptt.""")
    group.add_argument('-adam_beta1', type=float, default=0.9,
                       help="""The beta1 parameter used by Adam.
                       Almost without exception a value of 0.9 is used in
                       the literature, seemingly giving good results,
                       so we would discourage changing this value from
                       the default without due consideration.""")
    group.add_argument('-adam_beta2', type=float, default=0.999,
                       help="""The beta2 parameter used by Adam.
                       Typically a value of 0.999 is recommended, as this is
                       the value suggested by the original paper describing
                       Adam, and is also the value adopted in other frameworks
                       such as Tensorflow and Kerras, i.e. see:
                       https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer
                       https://keras.io/optimizers/ .
                       Whereas recently the paper "Attention is All You Need"
                       suggested a value of 0.98 for beta2, this parameter may
                       not work well for normal models / default
                       baselines.""")
    group.add_argument('-label_smoothing', type=float, default=0.0,
                       help="""Label smoothing value epsilon.
                       Probabilities of all non-true labels
                       will be smoothed by epsilon / (vocab_size - 1).
                       Set to zero to turn off label smoothing.
                       For more detailed information, see:
                       https://arxiv.org/abs/1512.00567""")
    # learning rate
    group = parser.add_argument_group('Optimization- Rate')
    group.add_argument('-learning_rate', type=float, default=1.0,
                       help="""Starting learning rate.
                       Recommended settings: sgd = 1, adagrad = 0.1,
                       adadelta = 1, adam = 0.001""")
    group.add_argument('-learning_rate_decay', type=float, default=0.5,
                       help="""If update_learning_rate, decay learning rate by
                       this much if (i) perplexity does not decrease on the
                       validation set or (ii) steps have gone past
                       start_decay_steps""")
    group.add_argument('-start_decay_steps', type=int, default=50000,
                       help="""Start decaying every decay_steps after
                       start_decay_steps""")
    group.add_argument('-decay_steps', type=int, default=10000,
                       help="""Decay every decay_steps""")

    group.add_argument('-decay_method', type=str, default="",
                       choices=['noam'], help="Use a custom decay rate.")
    group.add_argument('-warmup_steps', type=int, default=4000,
                       help="""Number of warmup steps for custom decay.""")

    group = parser.add_argument_group('Logging')
    group.add_argument('-report_every', type=int, default=50,
                       help="Print stats at this interval.")
    group.add_argument('-log_file', type=str, default="",
                       help="Output logs to a file under this path.")
    group.add_argument('-exp_host', type=str, default="",
                       help="Send logs to this crayon server.")
    group.add_argument('-exp', type=str, default="",
                       help="Name of the experiment for logging.")
    # Use TensorboardX for visualization during training
    group.add_argument('-tensorboard', action="store_true",
                       help="""Use tensorboardX for visualization during training.
                       Must have the library tensorboardX.""")
    group.add_argument("-tensorboard_log_dir", type=str,
                       default="runs/onmt",
                       help="""Log directory for Tensorboard.
                       This is also the name of the run.
                       """)

    group = parser.add_argument_group('Speech')
    # Options most relevant to speech
    group.add_argument('-sample_rate', type=int, default=16000,
                       help="Sample rate.")
    group.add_argument('-window_size', type=float, default=.02,
                       help="Window size for spectrogram in seconds.")

    # Option most relevant to image input
    group.add_argument('-image_channel_size', type=int, default=3,
                       choices=[3, 1],
                       help="""Using grayscale image can training
                       model faster and smaller""")


def translate_opts(parser):
    """ Translation / inference options """
    group = parser.add_argument_group('Model')
    group.add_argument('-model', dest='models', metavar='MODEL',
                       nargs='+', type=str, default=[], required=True,
                       help='Path to model .pt file(s). '
                            'Multiple models can be specified, '
                            'for ensemble decoding.')

    group = parser.add_argument_group('Data')
    group.add_argument('-data_type', default="text",
                       help="Type of the source input. Options: [text|img].")

    group.add_argument('-src', required=True,
                       help="""Source sequence to decode (one line per
                       sequence)""")
    group.add_argument('-src_dir', default="",
                       help='Source directory for image or audio files')
    group.add_argument('-tgt',
                       help='True target sequence (optional)')
    group.add_argument('-output', default='pred.txt',
                       help="""Path to output the predictions (each line will
                       be the decoded sequence""")
    group.add_argument('-report_bleu', action='store_true',
                       help="""Report bleu score after translation,
                       call tools/multi-bleu.perl on command line""")
    group.add_argument('-report_rouge', action='store_true',
                       help="""Report rouge 1/2/3/L/SU4 score after translation
                       call tools/test_rouge.py on command line""")

    # Options most relevant to summarization.
    group.add_argument('-dynamic_dict', action='store_true',
                       help="Create dynamic dictionaries")
    group.add_argument('-share_vocab', action='store_true',
                       help="Share source and target vocabulary")

    group = parser.add_argument_group('Beam')
    group.add_argument('-fast', action="store_true",
                       help="""Use fast beam search (some features may not be
                       supported!)""")
    group.add_argument('-beam_size', type=int, default=5,
                       help='Beam size')
    group.add_argument('-min_length', type=int, default=0,
                       help='Minimum prediction length')
    group.add_argument('-max_length', type=int, default=100,
                       help='Maximum prediction length.')
    group.add_argument('-max_sent_length', action=DeprecateAction,
                       help="Deprecated, use `-max_length` instead")

    # Alpha and Beta values for Google Length + Coverage penalty
    # Described here: https://arxiv.org/pdf/1609.08144.pdf, Section 7
    group.add_argument('-stepwise_penalty', action='store_true',
                       help="""Apply penalty at every decoding step.
                       Helpful for summary penalty.""")
    group.add_argument('-length_penalty', default='none',
                       choices=['none', 'wu', 'avg'],
                       help="""Length Penalty to use.""")
    group.add_argument('-coverage_penalty', default='none',
                       choices=['none', 'wu', 'summary'],
                       help="""Coverage Penalty to use.""")
    group.add_argument('-alpha', type=float, default=0.,
                       help="""Google NMT length penalty parameter
                        (higher = longer generation)""")
    group.add_argument('-beta', type=float, default=-0.,
                       help="""Coverage penalty parameter""")
    group.add_argument('-block_ngram_repeat', type=int, default=0,
                       help='Block repetition of ngrams during decoding.')
    group.add_argument('-ignore_when_blocking', nargs='+', type=str,
                       default=[],
                       help="""Ignore these strings when blocking repeats.
                       You want to block sentence delimiters.""")
    group.add_argument('-replace_unk', action="store_true",
                       help="""Replace the generated UNK tokens with the
                       source token that had highest attention weight. If
                       phrase_table is provided, it will lookup the
                       identified source token and give the corresponding
                       target token. If it is not provided(or the identified
                       source token does not exist in the table) then it
                       will copy the source token""")

    group = parser.add_argument_group('Logging')
    group.add_argument('-verbose', action="store_true",
                       help='Print scores and predictions for each sentence')
    group.add_argument('-log_file', type=str, default="",
                       help="Output logs to a file under this path.")
    group.add_argument('-attn_debug', action="store_true",
                       help='Print best attn for each word')
    group.add_argument('-dump_beam', type=str, default="",
                       help='File to dump beam information to.')
    group.add_argument('-n_best', type=int, default=1,
                       help="""If verbose is set, will output the n_best
                       decoded sentences""")

    group = parser.add_argument_group('Efficiency')
    group.add_argument('-batch_size', type=int, default=30,
                       help='Batch size')
    group.add_argument('-gpu', type=int, default=-1,
                       help="Device to run on")

    # Options most relevant to speech.
    group = parser.add_argument_group('Speech')
    group.add_argument('-sample_rate', type=int, default=16000,
                       help="Sample rate.")
    group.add_argument('-window_size', type=float, default=.02,
                       help='Window size for spectrogram in seconds')
    group.add_argument('-window_stride', type=float, default=.01,
                       help='Window stride for spectrogram in seconds')
    group.add_argument('-window', default='hamming',
                       help='Window type for spectrogram generation')

    # Option most relevant to image input
    group.add_argument('-image_channel_size', type=int, default=3,
                       choices=[3, 1],
                       help="""Using grayscale image can training
                       model faster and smaller""")


def add_md_help_argument(parser):
    """ md help parser """
    parser.add_argument('-md', action=MarkdownHelpAction,
                        help='print Markdown-formatted help text and exit.')


Args : tgt ( LongTensor ) : sequences of padded tokens [ tgt_len x batch x nfeats ] . memory_bank ( FloatTensor ) : vectors from the encoder [ src_len x batch x hidden ] . state ( : obj : onmt . models . DecoderState ) : decoder state object to initialize the decoder memory_lengths ( LongTensor ) : the padded source lengths [ batch ] . Returns : ( FloatTensor : obj : onmt . Models . DecoderState FloatTensor ) : * decoder_outputs : output from the decoder ( after attn ) [ tgt_len x batch x hidden ] . * decoder_state : final hidden state from the decoder * attns : distribution over src at each tgt [ tgt_len x batch x src_len ] .

def forward(self, tgt, memory_bank, state, memory_lengths=None,
                step=None,sent_encoder=None,src_sents=None):
        """
        Args:
            tgt (`LongTensor`): sequences of padded tokens
                 `[tgt_len x batch x nfeats]`.
            memory_bank (`FloatTensor`): vectors from the encoder
                 `[src_len x batch x hidden]`.
            state (:obj:`onmt.models.DecoderState`):
                 decoder state object to initialize the decoder
            memory_lengths (`LongTensor`): the padded source lengths
                `[batch]`.
        Returns:
            (`FloatTensor`,:obj:`onmt.Models.DecoderState`,`FloatTensor`):
                * decoder_outputs: output from the decoder (after attn)
                         `[tgt_len x batch x hidden]`.
                * decoder_state: final hidden state from the decoder
                * attns: distribution over src at each tgt
                        `[tgt_len x batch x src_len]`.
        """
        # Check
        assert isinstance(state, RNNDecoderState)
        # tgt.size() returns tgt length and batch
        _, tgt_batch, _ = tgt.size()
        _, memory_batch, _ = memory_bank.size()
        aeq(tgt_batch, memory_batch)
        # END

        # 23333: TODO I changed this return value 'sent_decoder'

        # Run the forward pass of the RNN.
        decoder_final, decoder_outputs, attns = self._run_forward_pass(
            tgt, memory_bank, state, memory_lengths=memory_lengths,sent_encoder=sent_encoder,src_sents=src_sents)

        # Update the state with the result.
        final_output = decoder_outputs[-1]
        coverage = None
        if "coverage" in attns:
            coverage = attns["coverage"][-1].unsqueeze(0)
        state.update_state(decoder_final, final_output.unsqueeze(0), coverage)

        # Concatenates sequence of tensors along a new dimension.
        # NOTE: v0.3 to 0.4: decoder_outputs / attns[*] may not be list
        #       (in particular in case of SRU) it was not raising error in 0.3
        #       since stack(Variable) was allowed.
        #       In 0.4, SRU returns a tensor that shouldn't be stacke
        if type(decoder_outputs) == list:
            decoder_outputs = torch.stack(decoder_outputs)

            for k in attns:
                if type(attns[k]) == list:
                    attns[k] = torch.stack(attns[k])

        return decoder_outputs, state, attns

#       since stack(Variable) was allowed.


def init_decoder_state(self, src, memory_bank, encoder_final,
                           with_cache=False):
        """ Init decoder state with last state of the encoder """
        def _fix_enc_hidden(hidden):
            # The encoder hidden is  (layers*directions) x batch x dim.
            # We need to convert it to layers x batch x (directions*dim).
            if self.bidirectional_encoder:
                hidden = torch.cat([hidden[0:hidden.size(0):2],
                                    hidden[1:hidden.size(0):2]], 2)
            return hidden

        if isinstance(encoder_final, tuple):  # LSTM
            return RNNDecoderState(self.hidden_size,
                                   tuple([_fix_enc_hidden(enc_hid)
                                          for enc_hid in encoder_final]))
        else:  # GRU
            return RNNDecoderState(self.hidden_size,
                                   _fix_enc_hidden(encoder_final))


Private helper for running the specific RNN forward pass . Must be overriden by all subclasses . Args : tgt ( LongTensor ) : a sequence of input tokens tensors [ len x batch x nfeats ] . memory_bank ( FloatTensor ) : output ( tensor sequence ) from the encoder RNN of size ( src_len x batch x hidden_size ) . state ( FloatTensor ) : hidden state from the encoder RNN for initializing the decoder . memory_lengths ( LongTensor ) : the source memory_bank lengths . Returns : decoder_final ( Tensor ) : final hidden state from the decoder . decoder_outputs ( [ FloatTensor ] ) : an array of output of every time step from the decoder . attns ( dict of ( str [ FloatTensor ] ) : a dictionary of different type of attention Tensor array of every time step from the decoder .

def _run_forward_pass(self, tgt, memory_bank, state, memory_lengths=None):
        """
        Private helper for running the specific RNN forward pass.
        Must be overriden by all subclasses.
        Args:
            tgt (LongTensor): a sequence of input tokens tensors
                                 [len x batch x nfeats].
            memory_bank (FloatTensor): output(tensor sequence) from the encoder
                        RNN of size (src_len x batch x hidden_size).
            state (FloatTensor): hidden state from the encoder RNN for
                                 initializing the decoder.
            memory_lengths (LongTensor): the source memory_bank lengths.
        Returns:
            decoder_final (Tensor): final hidden state from the decoder.
            decoder_outputs ([FloatTensor]): an array of output of every time
                                     step from the decoder.
            attns (dict of (str, [FloatTensor]): a dictionary of different
                            type of attention Tensor array of every time
                            step from the decoder.
        """
        assert not self._copy  # TODO, no support yet.
        assert not self._coverage  # TODO, no support yet.

        # Initialize local and return variables.
        attns = {}
        emb = self.embeddings(tgt)

        # Run the forward pass of the RNN.
        if isinstance(self.rnn, nn.GRU):
            rnn_output, decoder_final = self.rnn(emb, state.hidden[0])
        else:
            rnn_output, decoder_final = self.rnn(emb, state.hidden)

        # Check
        tgt_len, tgt_batch, _ = tgt.size()
        output_len, output_batch, _ = rnn_output.size()
        aeq(tgt_len, output_len)
        aeq(tgt_batch, output_batch)
        # END

        # Calculate the attention.
        decoder_outputs, p_attn = self.attn(
            rnn_output.transpose(0, 1).contiguous(),
            memory_bank.transpose(0, 1),
            memory_lengths=memory_lengths
        )
        attns["std"] = p_attn

        # Calculate the context gate.
        if self.context_gate is not None:
            decoder_outputs = self.context_gate(
                emb.view(-1, emb.size(2)),
                rnn_output.view(-1, rnn_output.size(2)),
                decoder_outputs.view(-1, decoder_outputs.size(2))
            )
            decoder_outputs = \
                decoder_outputs.view(tgt_len, tgt_batch, self.hidden_size)

        decoder_outputs = self.dropout(decoder_outputs)

        return decoder_final, decoder_outputs, attns


Private helper returning the number of expected features .

def _input_size(self):
        """
        Private helper returning the number of expected features.
        """
        return self.embeddings.embedding_size

def _input_size ( self ) : return self . embeddings . embedding_size


This is the attention version where in the encoding part we use self - attention the score is the max value of the attention weight # sent_encoder : size ( sent_len = 9 batch = 2 dim = 512 ) # sent_decoder : size ( sent_len = 1 batch = 2 dim = 512 ) # src_sents : size ( batch = 2 sent_len = 9 ) function to calculate mmr : param sent_encoder : : param sent_decoder : : param src_sents : : return :

def _run_mmr_attention(self,sent_encoder,sent_decoder,src_sents,input_step):
        '''
        This is the attention version, where in the encoding part we use self-attention,
        the score is the max value of the attention weight
        # sent_encoder: size (sent_len=9,batch=2,dim=512)
        # sent_decoder: size (sent_len=1,batch=2,dim=512)
        # src_sents: size (batch=2,sent_len=9)
        function to calculate mmr
        :param sent_encoder:
        :param sent_decoder:
        :param src_sents:
        :return:
        '''

        pdist = nn.PairwiseDistance(p=2)
        sent_decoder=sent_decoder.permute(1,0,2) # (2,1,512)

        scores =[]
        # define sent matrix and current vector distance as the Euclidean distance
        for sent in sent_encoder:
            # distance: https://pytorch.org/docs/stable/_modules/torch/nn/modules/distance.html
            sim2 = 0.5 * torch.sum(pdist(sent_encoder.permute(1,0,2),sent.unsqueeze(1)),1).unsqueeze(1) # this is also similarity func, can be another for-loop

            sim1 = torch.bmm(self.mmr_W(sent_decoder), sent.unsqueeze(2)).squeeze(2)  # (2,1)

            scores.append(0.5*(sim1 - sim2))

        sent_ranking_att = torch.t(torch.cat(scores,1)) #(sent_len=9,batch_size)
        sent_ranking_att = torch.softmax(sent_ranking_att, dim=0).permute(1,0)  #(sent_len=9,batch_size)
        # scores is a list of score (sent_len=9, tensor shape (batch_size, 1))
        mmr_among_words = [] # should be (batch=2,input_step=200)
        for batch_id in range(sent_ranking_att.size()[0]):
            # iterate each batch, create zero weight on the input steps
            # mmr= torch.zeros([input_step], dtype=torch.float32).cuda()

            tmp = []
            for id,position in enumerate(src_sents[batch_id]):

                for x in range(position):
                    tmp.append(sent_ranking_att[batch_id][id])

            mmr = torch.stack(tmp) # make to 1-d

            if len(mmr) < input_step:
                tmp = torch.zeros(input_step - len(mmr)).float().cuda()
                # for x in range(input_step-len(mmr)):
                mmr = torch.cat((mmr, tmp), 0)
            else:
                mmr = mmr[:input_step]

            mmr_among_words.append(mmr.unsqueeze(0))

        mmr_among_words = torch.cat(mmr_among_words,0)

        # shape: (batch=2, input_step=200)

        return mmr_among_words


See StdRNNDecoder . _run_forward_pass () for description of arguments and return values . TODO : added a new param : sent_encoder from model . py this is the sentence matrix ; add attns [ mmr ] = [] .

def _run_forward_pass(self, tgt, memory_bank, state, memory_lengths=None,sent_encoder=None,src_sents=None):
        """
        See StdRNNDecoder._run_forward_pass() for description
        of arguments and return values.
        TODO: added a new param: sent_encoder, from model.py, this is the sentence matrix; add attns["mmr"] = [].

        """

        # Additional args check.
        input_feed = state.input_feed.squeeze(0)
        #print("input feed size: {}\n".format(input_feed.size()))
        input_feed_batch, _ = input_feed.size()
        _, tgt_batch, _ = tgt.size()
        aeq(tgt_batch, input_feed_batch)
        # END Additional args check.

        # Initialize local and return variables.
        decoder_outputs = []
        attns = {"std": []}
        attns["mmr"] = []
        if self._copy:
            attns["copy"] = []
        if self._coverage:
            attns["coverage"] = []

        emb = self.embeddings(tgt)
        assert emb.dim() == 3  # len x batch x embedding_dim

        hidden = state.hidden
        coverage = state.coverage.squeeze(0) \
            if state.coverage is not None else None

        # Input feed concatenates hidden state with
        # input at every time step.

        #print("emb size: {}\n".format(emb.size()));exit()
        for _, emb_t in enumerate(emb.split(1)):
            # for each output time step in the loop

            emb_t = emb_t.squeeze(0)
            decoder_input = torch.cat([emb_t, input_feed], 1)

            # TODO: the following is where we get attention!
            rnn_output, hidden = self.rnn(decoder_input, hidden)
            decoder_output, p_attn = self.attn(
                rnn_output,
                memory_bank.transpose(0, 1),
                memory_lengths=memory_lengths)
            # p_attn: size (batch=2,input_step=200)

            if self.context_gate is not None:
                # TODO: context gate should be employed
                # instead of second RNN transform.
                decoder_output = self.context_gate(
                    decoder_input, rnn_output, decoder_output
                )
            decoder_output = self.dropout(decoder_output)
            input_feed = decoder_output

            decoder_outputs += [decoder_output]
            attns["std"] += [p_attn]

            # Update the coverage attention.
            if self._coverage:
                coverage = coverage + p_attn \
                    if coverage is not None else p_attn
                attns["coverage"] += [coverage]

            # Run the forward pass of the copy attention layer.
            #

            if self._copy and not self._reuse_copy_attn:

                _, copy_attn = self.copy_attn(decoder_output, memory_bank.transpose(0, 1))
                attns["copy"] += [copy_attn]
            elif self._copy:
                attns["copy"] = attns["std"] # attns["copy"] is a list of tensor for each output step=51, each size: [batch_size=2, input_step=200]

        # 2333: TODO : the sentence representation for decoder
        sent_decoder = decoder_outputs[-1].unsqueeze(0)  # shape: (1, batch_size=2,dim=512)

        # Return result.
        # 2333: TODO: attns['std'] is a list of tensors, length is output_step, each tensor shape is (batch=2,input_step=200)

        # 2333: TODO: compute mmr attention here:
        print ('Now..')
        mmr_among_words = self._run_mmr_attention(sent_encoder, sent_decoder, src_sents,attns["std"][0].size()[-1])

        #  2333: TODO: bring mmr to attention...

        for output_step in attns["std"]:
            attention_weight = output_step
            # pairwise multiplication
            attention_weight = torch.mul(mmr_among_words,attention_weight)
            attns["mmr"].append(attention_weight.cuda())
        # pdb.set_trace()

        attns["std"] = attns["mmr"]

        # decoder_outputs is a list of tensors for each output step=51, each tensor: (batch_size=2,dim=512)
        return hidden, decoder_outputs, attns

#  2333: TODO: bring mmr to attention...


Using input feed by concatenating input with attention vectors .

def _input_size(self):
        """
        Using input feed by concatenating input with attention vectors.
        """
        return self.embeddings.embedding_size + self.hidden_size

def _input_size ( self ) : return self . embeddings . embedding_size + self . hidden_size


def detach(self):
        """ Need to document this """
        self.hidden = tuple([_.detach() for _ in self.hidden])
        self.input_feed = self.input_feed.detach()

def detach ( self ) : self . hidden = tuple ( [ _ . detach ( ) for _ in self . hidden ] ) self . input_feed = self . input_feed . detach ( )


def beam_update(self, idx, positions, beam_size):
        """ Need to document this """
        for e in self._all:
            sizes = e.size()
            br = sizes[1]
            if len(sizes) == 3:
                sent_states = e.view(sizes[0], beam_size, br // beam_size,
                                     sizes[2])[:, :, idx]
            else:
                sent_states = e.view(sizes[0], beam_size,
                                     br // beam_size,
                                     sizes[2],
                                     sizes[3])[:, :, idx]

            sent_states.data.copy_(
                sent_states.data.index_select(1, positions))

def beam_update ( self , idx , positions , beam_size ) : for e in self . _all : sizes = e . size ( ) br = sizes [ 1 ] if len ( sizes ) == 3 : sent_states = e . view ( sizes [ 0 ] , beam_size , br // beam_size , sizes [ 2 ] ) [ : , : , idx ] else : sent_states = e . view ( sizes [ 0 ] , beam_size , br // beam_size , sizes [ 2 ] , sizes [ 3 ] ) [ : , : , idx ] sent_states . data . copy_ ( sent_states . data . index_select ( 1 , positions ) )


Args : hidden_size ( int ) : the size of hidden layer of the decoder . rnnstate : final hidden state from the encoder . transformed to shape : layers x batch x ( directions * dim ) .

def __init__(self, hidden_size, rnnstate):
        """
        Args:
            hidden_size (int): the size of hidden layer of the decoder.
            rnnstate: final hidden state from the encoder.
                transformed to shape: layers x batch x (directions*dim).
        """
        if not isinstance(rnnstate, tuple):
            self.hidden = (rnnstate,)
        else:
            self.hidden = rnnstate
        self.coverage = None

        # Init the input feed.
        batch_size = self.hidden[0].size(1)
        h_size = (batch_size, hidden_size)
        self.input_feed = self.hidden[0].data.new(*h_size).zero_() \
                              .unsqueeze(0)

def
__init__
(
self
,
hidden_size
,
rnnstate
)
:
if
not
isinstance
(
rnnstate
,
tuple
)
:
self
.
hidden
=
(
rnnstate
,
)
else
:
self
.
hidden
=
rnnstate
self
.
coverage
=
# Init the input feed.
batch_size
=
self
.
hidden
[
0
]
.
size
(
1
)
h_size
=
(
batch_size
,
hidden_size
)
self
.
input_feed
=
self
.
hidden
[
0
]
.
data
.
new
(
*
h_size
)
.
zero_
(
)
.
unsqueeze
(
0
)


def update_state(self, rnnstate, input_feed, coverage):
        """ Update decoder state """
        if not isinstance(rnnstate, tuple):
            self.hidden = (rnnstate,)
        else:
            self.hidden = rnnstate
        self.input_feed = input_feed
        self.coverage = coverage

def update_state ( self , rnnstate , input_feed , coverage ) : if not isinstance ( rnnstate , tuple ) : self . hidden = ( rnnstate , ) else : self . hidden = rnnstate self . input_feed = input_feed self . coverage = coverage


Repeat beam_size times along batch dimension .

def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        vars = [e.data.repeat(1, beam_size, 1)
                for e in self._all]
        self.hidden = tuple(vars[:-1])
        self.input_feed = vars[-1]

def repeat_beam_size_times ( self , beam_size ) : vars = [ e . data . repeat ( 1 , beam_size , 1 ) for e in self . _all ] self . hidden = tuple ( vars [ : - 1 ] ) self . input_feed = vars [ - 1 ]


def load_test_model(opt, dummy_opt):
    """ Read in multiple models for ensemble """
    shared_fields = None
    shared_model_opt = None
    models = []
    for model_path in opt.models:
        fields, model, model_opt = \
            onmt.model_builder.load_test_model(opt,
                                               dummy_opt,
                                               model_path=model_path)
        import pdb;pdb.set_trace()
        if shared_fields is None:
            shared_fields = fields
        else:
            for key, field in fields.items():
                if field is not None and 'vocab' in field.__dict__:
                    assert field.vocab.stoi == shared_fields[key].vocab.stoi, \
                        'Ensemble models must use the same preprocessed data'
        models.append(model)
        if shared_model_opt is None:
            shared_model_opt = model_opt
    ensemble_model = EnsembleModel(models)
    return shared_fields, ensemble_model, shared_model_opt


def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        for model_state in self.model_decoder_states:
            model_state.repeat_beam_size_times(beam_size)

def repeat_beam_size_times ( self , beam_size ) : for model_state in self . model_decoder_states : model_state . repeat_beam_size_times ( beam_size )


Delegate squeeze to avoid modifying : obj : Translator . translate_batch ()

def squeeze(self, dim=None):
        """
        Delegate squeeze to avoid modifying
        :obj:`Translator.translate_batch()`
        """
        return EnsembleDecoderOutput([
            x.squeeze(dim) for x in self.model_outputs])

def squeeze ( self , dim = None ) : return EnsembleDecoderOutput ( [ x . squeeze ( dim ) for x in self . model_outputs ] )


def forward(self, tgt, memory_bank, state, memory_lengths=None,
                step=None):
        """ See :obj:`RNNDecoderBase.forward()` """
        # Memory_lengths is a single tensor shared between all models.
        # This assumption will not hold if Translator is modified
        # to calculate memory_lengths as something other than the length
        # of the input.
        outputs, states, attns = zip(*[
            model_decoder.forward(
                tgt, memory_bank[i], state[i], memory_lengths, step=step)
            for (i, model_decoder)
            in enumerate(self.model_decoders)])
        mean_attns = self.combine_attns(attns)
        return (EnsembleDecoderOutput(outputs),
                EnsembleDecoderState(states),
                mean_attns)


See : obj : RNNDecoderBase . init_decoder_state ()

def init_decoder_state(self, src, memory_bank, enc_hidden):
        """ See :obj:`RNNDecoderBase.init_decoder_state()` """
        return EnsembleDecoderState(
            [model_decoder.init_decoder_state(src,
                                              memory_bank[i],
                                              enc_hidden[i])
             for (i, model_decoder) in enumerate(self.model_decoders)])

def init_decoder_state ( self , src , memory_bank , enc_hidden ) : return EnsembleDecoderState ( [ model_decoder . init_decoder_state ( src , memory_bank [ i ] , enc_hidden [ i ] ) for ( i , model_decoder ) in enumerate ( self . model_decoders ) ] )


Compute a distribution over the target dictionary by averaging distributions from models in the ensemble . All models in the ensemble must share a target vocabulary .

def forward(self, hidden):
        """
        Compute a distribution over the target dictionary
        by averaging distributions from models in the ensemble.
        All models in the ensemble must share a target vocabulary.
        """
        distributions = [model_generator.forward(hidden[i])
                         for (i, model_generator)
                         in enumerate(self.model_generators)]
        return torch.stack(distributions).mean(0)

def forward ( self , hidden ) : distributions = [ model_generator . forward ( hidden [ i ] ) for ( i , model_generator ) in enumerate ( self . model_generators ) ] return torch . stack ( distributions ) . mean ( 0 )


Pass in needed options only when modify function definition .

def load_pretrained_vectors(self, opt):
        """ Pass in needed options only when modify function definition."""
        pass

def load_pretrained_vectors ( self , opt ) : pass


See : obj : onmt . encoders . encoder . EncoderBase . forward ()

def forward(self, src, lengths=None):
        "See :obj:`onmt.encoders.encoder.EncoderBase.forward()`"
        # (batch_size, 1, nfft, t)
        # layer 1
        src = self.batch_norm1(self.layer1(src[:, :, :, :]))

        # (batch_size, 32, nfft/2, t/2)
        src = F.hardtanh(src, 0, 20, inplace=True)

        # (batch_size, 32, nfft/2/2, t/2)
        # layer 2
        src = self.batch_norm2(self.layer2(src))

        # (batch_size, 32, nfft/2/2, t/2)
        src = F.hardtanh(src, 0, 20, inplace=True)

        batch_size = src.size(0)
        length = src.size(3)
        src = src.view(batch_size, -1, length)
        src = src.transpose(0, 2).transpose(1, 2)

        output, hidden = self.rnn(src)

        return hidden, output

def
forward
(
self
,
src
,
lengths
=
)
:
# (batch_size, 1, nfft, t)
# layer 1
src
=
self
.
batch_norm1
(
self
.
layer1
(
src
[
:
,
:
,
:
,
:
]
)
)
# (batch_size, 32, nfft/2, t/2)
src
=
F
.
hardtanh
(
src
,
0
,
20
,
inplace
=
True
)
# (batch_size, 32, nfft/2/2, t/2)
# layer 2
src
=
self
.
batch_norm2
(
self
.
layer2
(
src
)
)
# (batch_size, 32, nfft/2/2, t/2)
src
=
F
.
hardtanh
(
src
,
0
,
20
,
inplace
=
True
)
batch_size
=
src
.
size
(
0
)
length
=
src
.
size
(
3
)
src
=
src
.
view
(
batch_size
,
1
,
length
)
src
=
src
.
transpose
(
0
,
2
)
.
transpose
(
1
,
2
)
output
,
hidden
=
self
.
rnn
(
src
)
return
hidden
,
output


def forward(self, src, lengths=None):
        "See :obj:`EncoderBase.forward()`"
        self._check_args(src, lengths)

        emb = self.embeddings(src)
        _, batch, emb_dim = emb.size()
        mean = emb.mean(0).expand(self.num_layers, batch, emb_dim)
        memory_bank = emb
        encoder_final = (mean, mean)
        return encoder_final, memory_bank

def forward ( self , src , lengths = None ) : self . _check_args ( src , lengths ) emb = self . embeddings ( src ) _ , batch , emb_dim = emb . size ( ) mean = emb . mean ( 0 ) . expand ( self . num_layers , batch , emb_dim ) memory_bank = emb encoder_final = ( mean , mean ) return encoder_final , memory_bank


def forward(self, src, lengths=None):
        "See :obj:`EncoderBase.forward()`"
        self._check_args(src, lengths)

        emb = self.embeddings(src)

        s_len, batch, emb_dim = emb.size() # (185 16 128), s_len is changeable.

        packed_emb = emb
        if lengths is not None and not self.no_pack_padded_seq:
            # Lengths data is wrapped inside a Tensor.
            lengths = lengths.view(-1).tolist()
            packed_emb = pack(emb, lengths)

        memory_bank, encoder_final = self.rnn(packed_emb) # output, (hidden, cell), unpack using pad_packed_sequence()
        # memory_bank is the output
        # self.rnn is a LSTM(128, 256, bidirectional=True) # input dim; output dim;

        # print('Hidden..', encoder_final[0].size(), encoder_final[1].size()) # both torch.Size([2, 16, 256]), 2 directions.

        if lengths is not None and not self.no_pack_padded_seq:
            memory_bank = unpack(memory_bank)[0]

        if self.use_bridge:
            encoder_final = self._bridge(encoder_final)

        print('Out..', memory_bank.size())
        #Out.. torch.Size([16, 512]) torch.Size([16, 512]) : two dir?

        return encoder_final, memory_bank


def _bridge(self, hidden):
        """
        Forward hidden state through bridge
        """
        def bottle_hidden(linear, states):
            """
            Transform from 3D to 2D, apply linear and return initial size
            """
            size = states.size()
            result = linear(states.view(-1, self.total_hidden_dim))
            return F.relu(result).view(size)

        if isinstance(hidden, tuple):  # LSTM
            outs = tuple([bottle_hidden(layer, hidden[ix])
                          for ix, layer in enumerate(self.bridge)])
        else:
            outs = bottle_hidden(self.bridge[0], hidden)
        return outs


def forward(self, inputs, mask):
        """
        Transformer Encoder Layer definition.

        Args:
            inputs (`FloatTensor`): `[batch_size x src_len x model_dim]`
            mask (`LongTensor`): `[batch_size x src_len x src_len]`

        Returns:
            (`FloatTensor`):

            * outputs `[batch_size x src_len x model_dim]`
        """
        input_norm = self.layer_norm(inputs)
        context, _ = self.self_attn(input_norm, input_norm, input_norm,
                                    mask=mask)
        out = self.dropout(context) + inputs
        return self.feed_forward(out)

def forward ( self , inputs , mask ) : input_norm = self . layer_norm ( inputs ) context , _ = self . self_attn ( input_norm , input_norm , input_norm , mask = mask ) out = self . dropout ( context ) + inputs return self . feed_forward ( out )


def forward(self, src, lengths=None):
        """ See :obj:`EncoderBase.forward()`"""
        self._check_args(src, lengths)

        emb = self.embeddings(src)

        out = emb.transpose(0, 1).contiguous()
        words = src[:, :, 0].transpose(0, 1)
        w_batch, w_len = words.size()
        padding_idx = self.embeddings.word_padding_idx
        mask = words.data.eq(padding_idx).unsqueeze(1) \
            .expand(w_batch, w_len, w_len)
        # Run the forward pass of every layer of the tranformer.
        for i in range(self.num_layers):
            out = self.transformer[i](out, mask)
        out = self.layer_norm(out)

        return emb, out.transpose(0, 1).contiguous()


See : obj : onmt . modules . RNNDecoderBase . forward ()

def forward(self, tgt, memory_bank, state, memory_lengths=None, step=None):
        """ See :obj:`onmt.modules.RNNDecoderBase.forward()`"""
        # NOTE: memory_lengths is only here for compatibility reasons
        #       with onmt.modules.RNNDecoderBase.forward()
        # CHECKS
        assert isinstance(state, CNNDecoderState)
        _, tgt_batch, _ = tgt.size()
        _, contxt_batch, _ = memory_bank.size()
        aeq(tgt_batch, contxt_batch)
        # END CHECKS

        if state.previous_input is not None:
            tgt = torch.cat([state.previous_input, tgt], 0)

        # Initialize return variables.
        outputs = []
        attns = {"std": []}
        assert not self._copy, "Copy mechanism not yet tested in conv2conv"
        if self._copy:
            attns["copy"] = []

        emb = self.embeddings(tgt)
        assert emb.dim() == 3  # len x batch x embedding_dim

        tgt_emb = emb.transpose(0, 1).contiguous()
        # The output of CNNEncoder.
        src_memory_bank_t = memory_bank.transpose(0, 1).contiguous()
        # The combination of output of CNNEncoder and source embeddings.
        src_memory_bank_c = state.init_src.transpose(0, 1).contiguous()

        # Run the forward pass of the CNNDecoder.
        emb_reshape = tgt_emb.contiguous().view(
            tgt_emb.size(0) * tgt_emb.size(1), -1)
        linear_out = self.linear(emb_reshape)
        x = linear_out.view(tgt_emb.size(0), tgt_emb.size(1), -1)
        x = shape_transform(x)

        pad = torch.zeros(x.size(0), x.size(1),
                          self.cnn_kernel_width - 1, 1)

        pad = pad.type_as(x)
        base_target_emb = x

        for conv, attention in zip(self.conv_layers, self.attn_layers):
            new_target_input = torch.cat([pad, x], 2)
            out = conv(new_target_input)
            c, attn = attention(base_target_emb, out,
                                src_memory_bank_t, src_memory_bank_c)
            x = (x + (c + out) * SCALE_WEIGHT) * SCALE_WEIGHT
        output = x.squeeze(3).transpose(1, 2)

        # Process the result and update the attentions.
        outputs = output.transpose(0, 1).contiguous()
        if state.previous_input is not None:
            outputs = outputs[state.previous_input.size(0):]
            attn = attn[:, state.previous_input.size(0):].squeeze()
            attn = torch.stack([attn])
        attns["std"] = attn
        if self._copy:
            attns["copy"] = attn

        # Update the state.
        state.update_state(tgt)

        return outputs, state, attns


def init_decoder_state(self, _, memory_bank, enc_hidden, with_cache=False):
        """
        Init decoder state.
        """
        return CNNDecoderState(memory_bank, enc_hidden)

def init_decoder_state ( self , _ , memory_bank , enc_hidden , with_cache = False ) : return CNNDecoderState ( memory_bank , enc_hidden )


Contains attributes that need to be updated in self . beam_update () .

def _all(self):
        """
        Contains attributes that need to be updated in self.beam_update().
        """
        return (self.previous_input,)

def _all ( self ) : return ( self . previous_input , )


def update_state(self, new_input):
        """ Called for every decoder forward pass. """
        self.previous_input = new_input

def update_state ( self , new_input ) : self . previous_input = new_input


def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        self.init_src = self.init_src.data.repeat(1, beam_size, 1)

def repeat_beam_size_times ( self , beam_size ) : self . init_src = self . init_src . data . repeat ( 1 , beam_size , 1 )