This returns user - defined LossCompute object which is used to compute loss in train / validate process . You can implement your own * LossCompute class by subclassing LossComputeBase .

def build_loss_compute(model, tgt_vocab, opt, train=True):
    """
    This returns user-defined LossCompute object, which is used to
    compute loss in train/validate process. You can implement your
    own *LossCompute class, by subclassing LossComputeBase.
    """
    device = torch.device("cuda" if onmt.utils.misc.use_gpu(opt) else "cpu")

    if opt.copy_attn:
        compute = onmt.modules.CopyGeneratorLossCompute(
            model.generator, tgt_vocab, opt.copy_attn_force,
            opt.copy_loss_by_seqlength)
    else:
        compute = NMTLossCompute(
            model.generator, tgt_vocab,
            label_smoothing=opt.label_smoothing if train else 0.0)
    compute.to(device)

    return compute

def build_loss_compute ( model , tgt_vocab , opt , train = True ) : device = torch . device ( "cuda" if onmt . utils . misc . use_gpu ( opt ) else "cpu" ) if opt . copy_attn : compute = onmt . modules . CopyGeneratorLossCompute ( model . generator , tgt_vocab , opt . copy_attn_force , opt . copy_loss_by_seqlength ) else : compute = NMTLossCompute ( model . generator , tgt_vocab , label_smoothing = opt . label_smoothing if train else 0.0 ) compute . to ( device ) return compute


def filter_shard_state(state, shard_size=None):
    """ ? """
    for k, v in state.items():
        if shard_size is None:
            yield k, v

        if v is not None:
            v_split = []
            if isinstance(v, torch.Tensor):
                for v_chunk in torch.split(v, shard_size):
                    v_chunk = v_chunk.data.clone()
                    v_chunk.requires_grad = v.requires_grad
                    v_split.append(v_chunk)
            yield k, (v, v_split)

def filter_shard_state ( state , shard_size = None ) : for k , v in state . items ( ) : if shard_size is None : yield k , v if v is not None : v_split = [ ] if isinstance ( v , torch . Tensor ) : for v_chunk in torch . split ( v , shard_size ) : v_chunk = v_chunk . data . clone ( ) v_chunk . requires_grad = v . requires_grad v_split . append ( v_chunk ) yield k , ( v , v_split )


Args : state : A dictionary which corresponds to the output of * LossCompute . _make_shard_state () . The values for those keys are Tensor - like or None . shard_size : The maximum size of the shards yielded by the model . eval_only : If True only yield the state nothing else . Otherwise yield shards .

def shards(state, shard_size, eval_only=False):
    """
    Args:
        state: A dictionary which corresponds to the output of
               *LossCompute._make_shard_state(). The values for
               those keys are Tensor-like or None.
        shard_size: The maximum size of the shards yielded by the model.
        eval_only: If True, only yield the state, nothing else.
              Otherwise, yield shards.

    Yields:
        Each yielded shard is a dict.

    Side effect:
        After the last shard, this function does back-propagation.
    """
    if eval_only:
        yield filter_shard_state(state)
    else:
        # non_none: the subdict of the state dictionary where the values
        # are not None.
        non_none = dict(filter_shard_state(state, shard_size))

        # Now, the iteration:
        # state is a dictionary of sequences of tensor-like but we
        # want a sequence of dictionaries of tensors.
        # First, unzip the dictionary into a sequence of keys and a
        # sequence of tensor-like sequences.
        keys, values = zip(*((k, [v_chunk for v_chunk in v_split])
                             for k, (_, v_split) in non_none.items()))

        # Now, yield a dictionary for each shard. The keys are always
        # the same. values is a sequence of length #keys where each
        # element is a sequence of length #shards. We want to iterate
        # over the shards, not over the keys: therefore, the values need
        # to be re-zipped by shard and then each shard can be paired
        # with the keys.
        for shard_tensors in zip(*values):
            yield dict(zip(keys, shard_tensors))

        # Assumed backprop'd
        variables = []
        for k, (v, v_split) in non_none.items():
            if isinstance(v, torch.Tensor) and state[k].requires_grad:
                variables.extend(zip(torch.split(state[k], shard_size),
                                     [v_chunk.grad for v_chunk in v_split]))
        inputs, grads = zip(*variables)
        torch.autograd.backward(inputs, grads)


Make shard state dictionary for shards () to return iterable shards for efficient loss computation . Subclass must define this method to match its own _compute_loss () interface . Args : batch : the current batch . output : the predict output from the model . range_ : the range of examples for computing the whole batch or a trunc of it? attns : the attns dictionary returned from the model .

def _make_shard_state(self, batch, output, range_, attns=None):
        """
        Make shard state dictionary for shards() to return iterable
        shards for efficient loss computation. Subclass must define
        this method to match its own _compute_loss() interface.
        Args:
            batch: the current batch.
            output: the predict output from the model.
            range_: the range of examples for computing, the whole
                    batch or a trunc of it?
            attns: the attns dictionary returned from the model.
        """
        return NotImplementedError

def _make_shard_state ( self , batch , output , range_ , attns = None ) : return NotImplementedError


Compute the loss . Subclass must define this method .

def _compute_loss(self, batch, output, target, **kwargs):
        """
        Compute the loss. Subclass must define this method.

        Args:

            batch: the current batch.
            output: the predict output from the model.
            target: the validate target to compare output with.
            **kwargs(optional): additional info for computing loss.
        """
        return NotImplementedError

def _compute_loss ( self , batch , output , target , * * kwargs ) : return NotImplementedError


Compute the forward loss for the batch .

def monolithic_compute_loss(self, batch, output, attns):
        """
        Compute the forward loss for the batch.

        Args:
          batch (batch): batch of labeled examples
          output (:obj:`FloatTensor`):
              output of decoder model `[tgt_len x batch x hidden]`
          attns (dict of :obj:`FloatTensor`) :
              dictionary of attention distributions
              `[tgt_len x batch x src_len]`
        Returns:
            :obj:`onmt.utils.Statistics`: loss statistics
        """
        range_ = (0, batch.tgt.size(0))
        shard_state = self._make_shard_state(batch, output, range_, attns)
        _, batch_stats = self._compute_loss(batch, **shard_state)

        return batch_stats

def monolithic_compute_loss ( self , batch , output , attns ) : range_ = ( 0 , batch . tgt . size ( 0 ) ) shard_state = self . _make_shard_state ( batch , output , range_ , attns ) _ , batch_stats = self . _compute_loss ( batch , * * shard_state ) return batch_stats


Compute the forward loss and backpropagate . Computation is done with shards and optionally truncation for memory efficiency .

def sharded_compute_loss(self, batch, output, attns,
                             cur_trunc, trunc_size, shard_size,
                             normalization):
        """Compute the forward loss and backpropagate.  Computation is done
        with shards and optionally truncation for memory efficiency.

        Also supports truncated BPTT for long sequences by taking a
        range in the decoder output sequence to back propagate in.
        Range is from `(cur_trunc, cur_trunc + trunc_size)`.

        Note sharding is an exact efficiency trick to relieve memory
        required for the generation buffers. Truncation is an
        approximate efficiency trick to relieve the memory required
        in the RNN buffers.

        Args:
          batch (batch) : batch of labeled examples
          output (:obj:`FloatTensor`) :
              output of decoder model `[tgt_len x batch x hidden]`
          attns (dict) : dictionary of attention distributions
              `[tgt_len x batch x src_len]`
          cur_trunc (int) : starting position of truncation window
          trunc_size (int) : length of truncation window
          shard_size (int) : maximum number of examples in a shard
          normalization (int) : Loss is divided by this number

        Returns:
            :obj:`onmt.utils.Statistics`: validation loss statistics

        """
        batch_stats = onmt.utils.Statistics()
        range_ = (cur_trunc, cur_trunc + trunc_size)
        shard_state = self._make_shard_state(batch, output, range_, attns)
        for shard in shards(shard_state, shard_size):
            loss, stats = self._compute_loss(batch, **shard)
            loss.div(float(normalization)).backward()
            batch_stats.update(stats)

        return batch_stats

def sharded_compute_loss ( self , batch , output , attns , cur_trunc , trunc_size , shard_size , normalization ) : batch_stats = onmt . utils . Statistics ( ) range_ = ( cur_trunc , cur_trunc + trunc_size ) shard_state = self . _make_shard_state ( batch , output , range_ , attns ) for shard in shards ( shard_state , shard_size ) : loss , stats = self . _compute_loss ( batch , * * shard ) loss . div ( float ( normalization ) ) . backward ( ) batch_stats . update ( stats ) return batch_stats


Args : loss ( : obj : FloatTensor ) : the loss computed by the loss criterion . scores ( : obj : FloatTensor ) : a score for each possible output target ( : obj : FloatTensor ) : true targets

def _stats(self, loss, scores, target):
        """
        Args:
            loss (:obj:`FloatTensor`): the loss computed by the loss criterion.
            scores (:obj:`FloatTensor`): a score for each possible output
            target (:obj:`FloatTensor`): true targets

        Returns:
            :obj:`onmt.utils.Statistics` : statistics for this batch.
        """
        pred = scores.max(1)[1]
        non_padding = target.ne(self.padding_idx)
        num_correct = pred.eq(target) \
                          .masked_select(non_padding) \
                          .sum() \
                          .item()
        num_non_padding = non_padding.sum().item()
        return onmt.utils.Statistics(loss.item(), num_non_padding, num_correct)

def _stats ( self , loss , scores , target ) : pred = scores . max ( 1 ) [ 1 ] non_padding = target . ne ( self . padding_idx ) num_correct = pred . eq ( target ) . masked_select ( non_padding ) . sum ( ) . item ( ) num_non_padding = non_padding . sum ( ) . item ( ) return onmt . utils . Statistics ( loss . item ( ) , num_non_padding , num_correct )


output ( FloatTensor ) : batch_size x n_classes target ( LongTensor ) : batch_size

def forward(self, output, target):
        """
        output (FloatTensor): batch_size x n_classes
        target (LongTensor): batch_size
        """
        model_prob = self.one_hot.repeat(target.size(0), 1)
        model_prob.scatter_(1, target.unsqueeze(1), self.confidence)
        model_prob.masked_fill_((target == self.padding_idx).unsqueeze(1), 0)

        return F.kl_div(output, model_prob, reduction='sum')

def forward ( self , output , target ) : model_prob = self . one_hot . repeat ( target . size ( 0 ) , 1 ) model_prob . scatter_ ( 1 , target . unsqueeze ( 1 ) , self . confidence ) model_prob . masked_fill_ ( ( target == self . padding_idx ) . unsqueeze ( 1 ) , 0 ) return F . kl_div ( output , model_prob , reduction = 'sum' )


def aeq(*args):
    """
    Assert all arguments have the same value
    """
    arguments = (arg for arg in args)
    first = next(arguments)
    assert all(arg == first for arg in arguments), \
        "Not all arguments have the same value: " + str(args)


Creates a boolean mask from sequence lengths .

def sequence_mask(lengths, max_len=None):
    """
    Creates a boolean mask from sequence lengths.
    """
    batch_size = lengths.numel()
    max_len = max_len or lengths.max()
    return (torch.arange(0, max_len)
            .type_as(lengths)
            .repeat(batch_size, 1)
            .lt(lengths.unsqueeze(1)))

def sequence_mask ( lengths , max_len = None ) : batch_size = lengths . numel ( ) max_len = max_len or lengths . max ( ) return ( torch . arange ( 0 , max_len ) . type_as ( lengths ) . repeat ( batch_size , 1 ) . lt ( lengths . unsqueeze ( 1 ) ) )


def tile(x, count, dim=0):
    """
    Tiles x on dimension dim count times.
    """
    perm = list(range(len(x.size())))
    if dim != 0:
        perm[0], perm[dim] = perm[dim], perm[0]
        x = x.permute(perm).contiguous()
    out_size = list(x.size())
    out_size[0] *= count
    batch = x.size(0)
    x = x.view(batch, -1) \
         .transpose(0, 1) \
         .repeat(count, 1) \
         .transpose(0, 1) \
         .contiguous() \
         .view(*out_size)
    if dim != 0:
        x = x.permute(perm).contiguous()
    return x

def tile ( x , count , dim = 0 ) : perm = list ( range ( len ( x . size ( ) ) ) ) if dim != 0 : perm [ 0 ] , perm [ dim ] = perm [ dim ] , perm [ 0 ] x = x . permute ( perm ) . contiguous ( ) out_size = list ( x . size ( ) ) out_size [ 0 ] *= count batch = x . size ( 0 ) x = x . view ( batch , - 1 ) . transpose ( 0 , 1 ) . repeat ( count , 1 ) . transpose ( 0 , 1 ) . contiguous ( ) . view ( * out_size ) if dim != 0 : x = x . permute ( perm ) . contiguous ( ) return x


def use_gpu(opt):
    """
    Creates a boolean if gpu used
    """
    return (hasattr(opt, 'gpu_ranks') and len(opt.gpu_ranks) > 0) or \
        (hasattr(opt, 'gpu') and opt.gpu > -1)

def use_gpu ( opt ) : return ( hasattr ( opt , 'gpu_ranks' ) and len ( opt . gpu_ranks ) > 0 ) or ( hasattr ( opt , 'gpu' ) and opt . gpu > - 1 )


Args : report_every ( int ) : Report status every this many sentences start_time ( float ) : manually set report start time . Negative values means that you will need to set it later or use start ()

def __init__(self, report_every, start_time=-1.):
        """
        Args:
            report_every(int): Report status every this many sentences
            start_time(float): manually set report start time. Negative values
                means that you will need to set it later or use `start()`
        """
        self.report_every = report_every
        self.progress_step = 0
        self.start_time = start_time


This is the user - defined batch - level traing progress report function .

def report_training(self, step, num_steps, learning_rate,
                        report_stats, multigpu=False):
        """
        This is the user-defined batch-level traing progress
        report function.

        Args:
            step(int): current step count.
            num_steps(int): total number of batches.
            learning_rate(float): current learning rate.
            report_stats(Statistics): old Statistics instance.
        Returns:
            report_stats(Statistics): updated Statistics instance.
        """
        if self.start_time < 0:
            raise ValueError("""ReportMgr needs to be started
                                (set 'start_time' or use 'start()'""")

        if multigpu:
            report_stats = onmt.utils.Statistics.all_gather_stats(report_stats)

        if step % self.report_every == 0:
            self._report_training(
                step, num_steps, learning_rate, report_stats)
            self.progress_step += 1
        return onmt.utils.Statistics()


def _report_training(self, *args, **kwargs):
        """ To be overridden """
        raise NotImplementedError()

def _report_training ( self , * args , * * kwargs ) : raise NotImplementedError ( )


def report_step(self, lr, step, train_stats=None, valid_stats=None):
        """
        Report stats of a step

        Args:
            train_stats(Statistics): training stats
            valid_stats(Statistics): validation stats
            lr(float): current learning rate
        """
        self._report_step(
            lr, step, train_stats=train_stats, valid_stats=valid_stats)

def report_step ( self , lr , step , train_stats = None , valid_stats = None ) : self . _report_step ( lr , step , train_stats = train_stats , valid_stats = valid_stats )


A report manager that writes statistics on standard output as well as ( optionally ) TensorBoard

def __init__(self, report_every, start_time=-1., tensorboard_writer=None):
        """
        A report manager that writes statistics on standard output as well as
        (optionally) TensorBoard

        Args:
            report_every(int): Report status every this many sentences
            tensorboard_writer(:obj:`tensorboard.SummaryWriter`):
                The TensorBoard Summary writer to use or None
        """
        super(ReportMgr, self).__init__(report_every, start_time)
        self.tensorboard_writer = tensorboard_writer

def __init__ ( self , report_every , start_time = - 1. , tensorboard_writer = None ) : super ( ReportMgr , self ) . __init__ ( report_every , start_time ) self . tensorboard_writer = tensorboard_writer


See base class method ReportMgrBase . report_training .

def _report_training(self, step, num_steps, learning_rate,
                         report_stats):
        """
        See base class method `ReportMgrBase.report_training`.
        """
        report_stats.output(step, num_steps,
                            learning_rate, self.start_time)

        # Log the progress using the number of batches on the x-axis.
        self.maybe_log_tensorboard(report_stats,
                                   "progress",
                                   learning_rate,
                                   self.progress_step)
        report_stats = onmt.utils.Statistics()

        return report_stats


See base class method ReportMgrBase . report_step .

def _report_step(self, lr, step, train_stats=None, valid_stats=None):
        """
        See base class method `ReportMgrBase.report_step`.
        """
        if train_stats is not None:
            self.log('Train perplexity: %g' % train_stats.ppl())
            self.log('Train accuracy: %g' % train_stats.accuracy())

            self.maybe_log_tensorboard(train_stats,
                                       "train",
                                       lr,
                                       step)

        if valid_stats is not None:
            self.log('Validation perplexity: %g' % valid_stats.ppl())
            self.log('Validation accuracy: %g' % valid_stats.accuracy())

            self.maybe_log_tensorboard(valid_stats,
                                       "valid",
                                       lr,
                                       step)

def
_report_step
(
self
,
lr
,
step
,
train_stats
=
,
valid_stats
=
)
:
if
train_stats
is
not
:
self
.
log
(
'Train perplexity: %g'
%
train_stats
.
ppl
(
)
)
self
.
log
(
'Train accuracy: %g'
%
train_stats
.
accuracy
(
)
)
self
.
maybe_log_tensorboard
(
train_stats
,
"train"
,
lr
,
step
)
if
valid_stats
is
not
:
self
.
log
(
'Validation perplexity: %g'
%
valid_stats
.
ppl
(
)
)
self
.
log
(
'Validation accuracy: %g'
%
valid_stats
.
accuracy
(
)
)
self
.
maybe_log_tensorboard
(
valid_stats
,
"valid"
,
lr
,
step
)


Compute the context vector and the attention vectors .

def forward(self, key, value, query, mask=None,
                layer_cache=None, type=None):
        """
        Compute the context vector and the attention vectors.

        Args:
           key (`FloatTensor`): set of `key_len`
                key vectors `[batch, key_len, dim]`
           value (`FloatTensor`): set of `key_len`
                value vectors `[batch, key_len, dim]`
           query (`FloatTensor`): set of `query_len`
                 query vectors  `[batch, query_len, dim]`
           mask: binary mask indicating which keys have
                 non-zero attention `[batch, query_len, key_len]`
        Returns:
           (`FloatTensor`, `FloatTensor`) :

           * output context vectors `[batch, query_len, dim]`
           * one of the attention vectors `[batch, query_len, key_len]`
        """

        # CHECKS
        # batch, k_len, d = key.size()
        # batch_, k_len_, d_ = value.size()
        # aeq(batch, batch_)
        # aeq(k_len, k_len_)
        # aeq(d, d_)
        # batch_, q_len, d_ = query.size()
        # aeq(batch, batch_)
        # aeq(d, d_)
        # aeq(self.model_dim % 8, 0)
        # if mask is not None:
        #    batch_, q_len_, k_len_ = mask.size()
        #    aeq(batch_, batch)
        #    aeq(k_len_, k_len)
        #    aeq(q_len_ == q_len)
        # END CHECKS

        batch_size = key.size(0)
        dim_per_head = self.dim_per_head
        head_count = self.head_count
        key_len = key.size(1)
        query_len = query.size(1)

        def shape(x):
            """  projection """
            return x.view(batch_size, -1, head_count, dim_per_head) \
                .transpose(1, 2)

        def unshape(x):
            """  compute context """
            return x.transpose(1, 2).contiguous() \
                    .view(batch_size, -1, head_count * dim_per_head)

        # 1) Project key, value, and query.
        if layer_cache is not None:
            if type == "self":
                query, key, value = self.linear_query(query),\
                                    self.linear_keys(query),\
                                    self.linear_values(query)

                key = shape(key)
                value = shape(value)

                if layer_cache is not None:
                    device = key.device
                    if layer_cache["self_keys"] is not None:
                        key = torch.cat(
                            (layer_cache["self_keys"].to(device), key),
                            dim=2)
                    if layer_cache["self_values"] is not None:
                        value = torch.cat(
                            (layer_cache["self_values"].to(device), value),
                            dim=2)
                    layer_cache["self_keys"] = key
                    layer_cache["self_values"] = value
            elif type == "context":
                query = self.linear_query(query)
                if layer_cache is not None:
                    if layer_cache["memory_keys"] is None:
                        key, value = self.linear_keys(key),\
                                     self.linear_values(value)
                        key = shape(key)
                        value = shape(value)
                    else:
                        key, value = layer_cache["memory_keys"],\
                                   layer_cache["memory_values"]
                    layer_cache["memory_keys"] = key
                    layer_cache["memory_values"] = value
                else:
                    key, value = self.linear_keys(key),\
                                 self.linear_values(value)
                    key = shape(key)
                    value = shape(value)
        else:
            key = self.linear_keys(key)
            value = self.linear_values(value)
            query = self.linear_query(query)
            key = shape(key)
            value = shape(value)

        query = shape(query)

        key_len = key.size(2)
        query_len = query.size(2)

        # 2) Calculate and scale scores.
        query = query / math.sqrt(dim_per_head)
        scores = torch.matmul(query, key.transpose(2, 3))

        if mask is not None:
            mask = mask.unsqueeze(1).expand_as(scores)
            scores = scores.masked_fill(mask, -1e18)

        # 3) Apply attention dropout and compute context vectors.
        attn = self.softmax(scores)
        drop_attn = self.dropout(attn)
        context = unshape(torch.matmul(drop_attn, value))

        output = self.final_linear(context)
        # CHECK
        # batch_, q_len_, d_ = output.size()
        # aeq(q_len, q_len_)
        # aeq(batch, batch_)
        # aeq(d, d_)

        # Return one attn
        top_attn = attn \
            .view(batch_size, head_count,
                  query_len, key_len)[:, 0, :, :] \
            .contiguous()

        return output, top_attn

#    batch_, q_len_, k_len_ = mask.size()


def forward(self, x):
        """
        Layer definition.

        Args:
            input: [ batch_size, input_len, model_dim ]

        Returns:
            output: [ batch_size, input_len, model_dim ]
        """
        inter = self.dropout_1(self.relu(self.w_1(self.layer_norm(x))))
        output = self.dropout_2(self.w_2(inter))
        return output + x

def forward ( self , x ) : inter = self . dropout_1 ( self . relu ( self . w_1 ( self . layer_norm ( x ) ) ) ) output = self . dropout_2 ( self . w_2 ( inter ) ) return output + x


z : any dimension dim : dimension along which to apply the sparsemax

def threshold_and_support(z, dim=0):
    """
    z: any dimension
    dim: dimension along which to apply the sparsemax
    """
    sorted_z, _ = torch.sort(z, descending=True, dim=dim)
    z_sum = sorted_z.cumsum(dim) - 1  # sort of a misnomer
    k = torch.arange(1, sorted_z.size(dim) + 1, device=z.device).float().view(
        torch.Size([-1] + [1] * (z.dim() - 1))
    ).transpose(0, dim)
    support = k * sorted_z > z_sum

    k_z_indices = support.sum(dim=dim).unsqueeze(dim)
    k_z = k_z_indices.float()
    tau_z = z_sum.gather(dim, k_z_indices - 1) / k_z
    return tau_z, k_z

def
threshold_and_support
(
z
,
dim
=
0
)
:
sorted_z
,
_
=
torch
.
sort
(
z
,
descending
=
True
,
dim
=
dim
)
z_sum
=
sorted_z
.
cumsum
(
dim
)
1
# sort of a misnomer
k
=
torch
.
arange
(
1
,
sorted_z
.
size
(
dim
)
+
1
,
device
=
z
.
device
)
.
float
(
)
.
view
(
torch
.
Size
(
[
1
]
+
[
1
]
*
(
z
.
dim
(
)
1
)
)
)
.
transpose
(
0
,
dim
)
support
=
k
*
sorted_z
>
z_sum
k_z_indices
=
support
.
sum
(
dim
=
dim
)
.
unsqueeze
(
dim
)
k_z
=
k_z_indices
.
float
(
)
tau_z
=
z_sum
.
gather
(
dim
,
k_z_indices
1
)
/
k_z
return
tau_z
,
k_z


input ( FloatTensor ) : any shape returns ( FloatTensor ) : same shape with sparsemax computed on given dim

def forward(ctx, input, dim=0):
        """
        input (FloatTensor): any shape
        returns (FloatTensor): same shape with sparsemax computed on given dim
        """
        ctx.dim = dim
        tau_z, k_z = threshold_and_support(input, dim=dim)
        output = torch.clamp(input - tau_z, min=0)
        ctx.save_for_backward(k_z, output)
        return output

def forward ( ctx , input , dim = 0 ) : ctx . dim = dim tau_z , k_z = threshold_and_support ( input , dim = dim ) output = torch . clamp ( input - tau_z , min = 0 ) ctx . save_for_backward ( k_z , output ) return output


Args : h_t ( FloatTensor ) : sequence of queries [ batch x tgt_len x dim ] h_s ( FloatTensor ) : sequence of sources [ batch x src_len x dim ]

def score(self, h_t, h_s):
        """
        Args:
          h_t (`FloatTensor`): sequence of queries `[batch x tgt_len x dim]`
          h_s (`FloatTensor`): sequence of sources `[batch x src_len x dim]`

        Returns:
          :obj:`FloatTensor`:
           raw attention scores (unnormalized) for each src index
          `[batch x tgt_len x src_len]`

        """

        # Check input sizes
        src_batch, src_len, src_dim = h_s.size()
        tgt_batch, tgt_len, tgt_dim = h_t.size()
        aeq(src_batch, tgt_batch)
        aeq(src_dim, tgt_dim)
        aeq(self.dim, src_dim)

        if self.attn_type in ["general", "dot"]:
            if self.attn_type == "general":
                h_t_ = h_t.view(tgt_batch * tgt_len, tgt_dim)
                h_t_ = self.linear_in(h_t_)
                h_t = h_t_.view(tgt_batch, tgt_len, tgt_dim)
            h_s_ = h_s.transpose(1, 2)
            # (batch, t_len, d) x (batch, d, s_len) --> (batch, t_len, s_len)
            return torch.bmm(h_t, h_s_)
        else:
            dim = self.dim
            wq = self.linear_query(h_t.view(-1, dim))
            wq = wq.view(tgt_batch, tgt_len, 1, dim)
            wq = wq.expand(tgt_batch, tgt_len, src_len, dim)

            uh = self.linear_context(h_s.contiguous().view(-1, dim))
            uh = uh.view(src_batch, 1, src_len, dim)
            uh = uh.expand(src_batch, tgt_len, src_len, dim)

            # (batch, t_len, s_len, d)
            wquh = torch.tanh(wq + uh)

            return self.v(wquh.view(-1, dim)).view(tgt_batch, tgt_len, src_len)


def forward(self, source, memory_bank, memory_lengths=None, coverage=None):
        """

        Args:
          source (`FloatTensor`): query vectors `[batch x tgt_len x dim]`
          memory_bank (`FloatTensor`): source vectors `[batch x src_len x dim]`
          memory_lengths (`LongTensor`): the source context lengths `[batch]`
          coverage (`FloatTensor`): None (not supported yet)

        Returns:
          (`FloatTensor`, `FloatTensor`):

          * Computed vector `[tgt_len x batch x dim]`
          * Attention distribtutions for each query
             `[tgt_len x batch x src_len]`
        """

        # one step input
        if source.dim() == 2:
            one_step = True
            source = source.unsqueeze(1)
        else:
            one_step = False

        batch, source_l, dim = memory_bank.size()
        batch_, target_l, dim_ = source.size()
        aeq(batch, batch_)
        aeq(dim, dim_)
        aeq(self.dim, dim)
        if coverage is not None:
            batch_, source_l_ = coverage.size()
            aeq(batch, batch_)
            aeq(source_l, source_l_)

        if coverage is not None:
            cover = coverage.view(-1).unsqueeze(1)
            memory_bank += self.linear_cover(cover).view_as(memory_bank)
            memory_bank = torch.tanh(memory_bank)

        # compute attention scores, as in Luong et al.
        align = self.score(source, memory_bank)

        if memory_lengths is not None:
            mask = sequence_mask(memory_lengths, max_len=align.size(-1))
            mask = mask.unsqueeze(1)  # Make it broadcastable.
            align.masked_fill_(1 - mask, -float('inf'))

        # Softmax or sparsemax to normalize attention weights
        if self.attn_func == "softmax":
            align_vectors = F.softmax(align.view(batch*target_l, source_l), -1)
        else:
            align_vectors = sparsemax(align.view(batch*target_l, source_l), -1)
        align_vectors = align_vectors.view(batch, target_l, source_l)

        # each context vector c_t is the weighted average
        # over all the source hidden states
        c = torch.bmm(align_vectors, memory_bank)

        # concatenate
        concat_c = torch.cat([c, source], 2).view(batch*target_l, dim*2)
        attn_h = self.linear_out(concat_c).view(batch, target_l, dim)
        if self.attn_type in ["general", "dot"]:
            attn_h = torch.tanh(attn_h)

        if one_step:
            attn_h = attn_h.squeeze(1)
            align_vectors = align_vectors.squeeze(1)

            # Check output sizes
            batch_, dim_ = attn_h.size()
            aeq(batch, batch_)
            aeq(dim, dim_)
            batch_, source_l_ = align_vectors.size()
            aeq(batch, batch_)
            aeq(source_l, source_l_)

        else:
            attn_h = attn_h.transpose(0, 1).contiguous()
            align_vectors = align_vectors.transpose(0, 1).contiguous()
            # Check output sizes
            target_l_, batch_, dim_ = attn_h.size()
            aeq(target_l, target_l_)
            aeq(batch, batch_)
            aeq(dim, dim_)
            target_l_, batch_, source_l_ = align_vectors.size()
            aeq(target_l, target_l_)
            aeq(batch, batch_)
            aeq(source_l, source_l_)

        return attn_h, align_vectors


def context_gate_factory(gate_type, embeddings_size, decoder_size,
                         attention_size, output_size):
    """Returns the correct ContextGate class"""

    gate_types = {'source': SourceContextGate,
                  'target': TargetContextGate,
                  'both': BothContextGate}

    assert gate_type in gate_types, "Not valid ContextGate type: {0}".format(
        gate_type)
    return gate_types[gate_type](embeddings_size, decoder_size, attention_size,
                                 output_size)

def
context_gate_factory
(
gate_type
,
embeddings_size
,
decoder_size
,
attention_size
,
output_size
)
:
gate_types
=
{
'source'
:
SourceContextGate
,
'target'
:
TargetContextGate
,
'both'
:
BothContextGate
}
assert
gate_type
in
gate_types
,
"Not valid ContextGate type: {0}"
.
format
(
gate_type
)
return
gate_types
[
gate_type
]
(
embeddings_size
,
decoder_size
,
attention_size
,
output_size
)


Builds the mask to compute the cumulative average as described in https : // arxiv . org / abs / 1805 . 00631 -- Figure 3

def cumulative_average_mask(self, batch_size, inputs_len):
        """
        Builds the mask to compute the cumulative average as described in
        https://arxiv.org/abs/1805.00631 -- Figure 3

        Args:
            batch_size (int): batch size
            inputs_len (int): length of the inputs

        Returns:
            (`FloatTensor`):

            * A Tensor of shape `[batch_size x input_len x input_len]`
        """

        triangle = torch.tril(torch.ones(inputs_len, inputs_len))
        weights = torch.ones(1, inputs_len) / torch.arange(
            1, inputs_len + 1, dtype=torch.float)
        mask = triangle * weights.transpose(0, 1)

        return mask.unsqueeze(0).expand(batch_size, inputs_len, inputs_len)

def cumulative_average_mask ( self , batch_size , inputs_len ) : triangle = torch . tril ( torch . ones ( inputs_len , inputs_len ) ) weights = torch . ones ( 1 , inputs_len ) / torch . arange ( 1 , inputs_len + 1 , dtype = torch . float ) mask = triangle * weights . transpose ( 0 , 1 ) return mask . unsqueeze ( 0 ) . expand ( batch_size , inputs_len , inputs_len )


Computes the cumulative average as described in https : // arxiv . org / abs / 1805 . 00631 -- Equations ( 1 ) ( 5 ) ( 6 )

def cumulative_average(self, inputs, mask_or_step,
                           layer_cache=None, step=None):
        """
        Computes the cumulative average as described in
        https://arxiv.org/abs/1805.00631 -- Equations (1) (5) (6)

        Args:
            inputs (`FloatTensor`): sequence to average
                `[batch_size x input_len x dimension]`
            mask_or_step: if cache is set, this is assumed
                to be the current step of the
                dynamic decoding. Otherwise, it is the mask matrix
                used to compute the cumulative average.
            cache: a dictionary containing the cumulative average
                of the previous step.
        """
        if layer_cache is not None:
            step = mask_or_step
            device = inputs.device
            average_attention = (inputs + step *
                                 layer_cache["prev_g"].to(device)) / (step + 1)
            layer_cache["prev_g"] = average_attention
            return average_attention
        else:
            mask = mask_or_step
            return torch.matmul(mask, inputs)

def cumulative_average ( self , inputs , mask_or_step , layer_cache = None , step = None ) : if layer_cache is not None : step = mask_or_step device = inputs . device average_attention = ( inputs + step * layer_cache [ "prev_g" ] . to ( device ) ) / ( step + 1 ) layer_cache [ "prev_g" ] = average_attention return average_attention else : mask = mask_or_step return torch . matmul ( mask , inputs )


Args : inputs ( FloatTensor ) : [ batch_size x input_len x model_dim ]

def forward(self, inputs, mask=None, layer_cache=None, step=None):
        """
        Args:
            inputs (`FloatTensor`): `[batch_size x input_len x model_dim]`

        Returns:
            (`FloatTensor`, `FloatTensor`):

            * gating_outputs `[batch_size x 1 x model_dim]`
            * average_outputs average attention `[batch_size x 1 x model_dim]`
        """
        batch_size = inputs.size(0)
        inputs_len = inputs.size(1)

        device = inputs.device
        average_outputs = self.cumulative_average(
          inputs, self.cumulative_average_mask(batch_size,
                                               inputs_len).to(device).float()
          if layer_cache is None else step, layer_cache=layer_cache)
        average_outputs = self.average_layer(average_outputs)
        gating_outputs = self.gating_layer(torch.cat((inputs,
                                                      average_outputs), -1))
        input_gate, forget_gate = torch.chunk(gating_outputs, 2, dim=2)
        gating_outputs = torch.sigmoid(input_gate) * inputs + \
            torch.sigmoid(forget_gate) * average_outputs

        return gating_outputs, average_outputs

def forward ( self , inputs , mask = None , layer_cache = None , step = None ) : batch_size = inputs . size ( 0 ) inputs_len = inputs . size ( 1 ) device = inputs . device average_outputs = self . cumulative_average ( inputs , self . cumulative_average_mask ( batch_size , inputs_len ) . to ( device ) . float ( ) if layer_cache is None else step , layer_cache = layer_cache ) average_outputs = self . average_layer ( average_outputs ) gating_outputs = self . gating_layer ( torch . cat ( ( inputs , average_outputs ) , - 1 ) ) input_gate , forget_gate = torch . chunk ( gating_outputs , 2 , dim = 2 ) gating_outputs = torch . sigmoid ( input_gate ) * inputs + torch . sigmoid ( forget_gate ) * average_outputs return gating_outputs , average_outputs


Compute a distribution over the target dictionary extended by the dynamic dictionary implied by compying source words .

def forward(self, hidden, attn, src_map):
        """
        Compute a distribution over the target dictionary
        extended by the dynamic dictionary implied by compying
        source words.

        Args:
           hidden (`FloatTensor`): hidden outputs `[batch*tlen, input_size]`
           attn (`FloatTensor`): attn for each `[batch*tlen, input_size]`
           src_map (`FloatTensor`):
             A sparse indicator matrix mapping each source word to
             its index in the "extended" vocab containing.
             `[src_len, batch, extra_words]`
        """
        # CHECKS
        batch_by_tlen, _ = hidden.size()
        batch_by_tlen_, slen = attn.size()
        slen_, batch, cvocab = src_map.size()
        aeq(batch_by_tlen, batch_by_tlen_)
        aeq(slen, slen_)

        # Original probabilities.
        logits = self.linear(hidden)
        logits[:, self.tgt_dict.stoi[inputters.PAD_WORD]] = -float('inf')
        prob = self.softmax(logits)

        # Probability of copying p(z=1) batch.
        p_copy = self.sigmoid(self.linear_copy(hidden))
        # Probibility of not copying: p_{word}(w) * (1 - p(z))
        out_prob = torch.mul(prob, 1 - p_copy.expand_as(prob))
        mul_attn = torch.mul(attn, p_copy.expand_as(attn))
        copy_prob = torch.bmm(mul_attn.view(-1, batch, slen)
                              .transpose(0, 1),
                              src_map.transpose(0, 1)).transpose(0, 1)
        copy_prob = copy_prob.contiguous().view(-1, cvocab)
        return torch.cat([out_prob, copy_prob], 1)


def _make_shard_state(self, batch, output, range_, attns):
        """ See base class for args description. """
        if getattr(batch, "alignment", None) is None:
            raise AssertionError("using -copy_attn you need to pass in "
                                 "-dynamic_dict during preprocess stage.")

        return {
            "output": output,
            "target": batch.tgt[range_[0] + 1: range_[1]],
            "copy_attn": attns.get("copy"),
            "align": batch.alignment[range_[0] + 1: range_[1]]
        }


Compute the loss . The args must match self . _make_shard_state () . Args : batch : the current batch . output : the predict output from the model . target : the validate target to compare output with . copy_attn : the copy attention value . align : the align info .

def _compute_loss(self, batch, output, target, copy_attn, align):
        """
        Compute the loss. The args must match self._make_shard_state().
        Args:
            batch: the current batch.
            output: the predict output from the model.
            target: the validate target to compare output with.
            copy_attn: the copy attention value.
            align: the align info.
        """
        target = target.view(-1)
        align = align.view(-1)
        scores = self.generator(self._bottle(output),
                                self._bottle(copy_attn),
                                batch.src_map)
        loss = self.criterion(scores, align, target)
        scores_data = scores.data.clone()
        scores_data = inputters.TextDataset.collapse_copy_scores(
            self._unbottle(scores_data, batch.batch_size),
            batch, self.tgt_vocab, batch.dataset.src_vocabs)
        scores_data = self._bottle(scores_data)

        # Correct target copy token instead of <unk>
        # tgt[i] = align[i] + len(tgt_vocab)
        # for i such that tgt[i] == 0 and align[i] != 0
        target_data = target.data.clone()
        correct_mask = target_data.eq(0) * align.data.ne(0)
        correct_copy = (align.data + len(self.tgt_vocab)) * correct_mask.long()
        target_data = target_data + correct_copy

        # Compute sum of perplexities for stats
        loss_data = loss.sum().data.clone()
        stats = self._stats(loss_data, scores_data, target_data)

        if self.normalize_by_length:
            # Compute Loss as NLL divided by seq length
            # Compute Sequence Lengths
            pad_ix = batch.dataset.fields['tgt'].vocab.stoi[inputters.PAD_WORD]
            tgt_lens = batch.tgt.ne(pad_ix).float().sum(0)
            # Compute Total Loss per sequence in batch
            loss = loss.view(-1, batch.batch_size).sum(0)
            # Divide by length of each sequence and sum
            loss = torch.div(loss, tgt_lens).sum()
        else:
            loss = loss.sum()

        return loss, stats


def word_lut(self):
        """ word look-up table """
        return self.make_embedding[0][0]

def word_lut ( self ) : return self . make_embedding [ 0 ] [ 0 ]


def emb_luts(self):
        """ embedding look-up table """
        return self.make_embedding[0]

def emb_luts ( self ) : return self . make_embedding [ 0 ]


def load_pretrained_vectors(self, emb_file, fixed):
        """Load in pretrained embeddings.

        Args:
          emb_file (str) : path to torch serialized embeddings
          fixed (bool) : if true, embeddings are not updated
        """
        if emb_file:
            pretrained = torch.load(emb_file)
            pretrained_vec_size = pretrained.size(1)
            if self.word_vec_size > pretrained_vec_size:
                self.word_lut.weight.data[:, :pretrained_vec_size] = pretrained
            elif self.word_vec_size < pretrained_vec_size:
                self.word_lut.weight.data \
                    .copy_(pretrained[:, :self.word_vec_size])
            else:
                self.word_lut.weight.data.copy_(pretrained)
            if fixed:
                self.word_lut.weight.requires_grad = False

def load_pretrained_vectors ( self , emb_file , fixed ) : if emb_file : pretrained = torch . load ( emb_file ) pretrained_vec_size = pretrained . size ( 1 ) if self . word_vec_size > pretrained_vec_size : self . word_lut . weight . data [ : , : pretrained_vec_size ] = pretrained elif self . word_vec_size < pretrained_vec_size : self . word_lut . weight . data . copy_ ( pretrained [ : , : self . word_vec_size ] ) else : self . word_lut . weight . data . copy_ ( pretrained ) if fixed : self . word_lut . weight . requires_grad = False


Computes the embeddings for words and features .

def forward(self, source, step=None):
        """
        Computes the embeddings for words and features.

        Args:
            source (`LongTensor`): index tensor `[len x batch x nfeat]`
        Return:
            `FloatTensor`: word embeddings `[len x batch x embedding_size]`
        """
        if self.position_encoding:
            for i, module in enumerate(self.make_embedding._modules.values()):
                if i == len(self.make_embedding._modules.values()) - 1:
                    source = module(source, step=step)
                else:
                    source = module(source)
        else:
            source = self.make_embedding(source)

        return source

def forward ( self , source , step = None ) : if self . position_encoding : for i , module in enumerate ( self . make_embedding . _modules . values ( ) ) : if i == len ( self . make_embedding . _modules . values ( ) ) - 1 : source = module ( source , step = step ) else : source = module ( source ) else : source = self . make_embedding ( source ) return source


input ( FloatTensor ) : n x num_classes target ( LongTensor ) : n the indices of the target classes

def forward(ctx, input, target):
        """
        input (FloatTensor): n x num_classes
        target (LongTensor): n, the indices of the target classes
        """
        input_batch, classes = input.size()
        target_batch = target.size(0)
        aeq(input_batch, target_batch)

        z_k = input.gather(1, target.unsqueeze(1)).squeeze()
        tau_z, support_size = threshold_and_support(input, dim=1)
        support = input > tau_z
        x = torch.where(
            support, input**2 - tau_z**2,
            torch.tensor(0.0, device=input.device)
        ).sum(dim=1)
        ctx.save_for_backward(input, target, tau_z)
        # clamping necessary because of numerical errors: loss should be lower
        # bounded by zero, but negative values near zero are possible without
        # the clamp
        return torch.clamp(x / 2 - z_k + 0.5, min=0.0)


def seq_linear(linear, x):
    """ linear transform for 3-d tensor """
    batch, hidden_size, length, _ = x.size()
    h = linear(torch.transpose(x, 1, 2).contiguous().view(
        batch * length, hidden_size))
    return torch.transpose(h.view(batch, length, hidden_size, 1), 1, 2)

def seq_linear ( linear , x ) : batch , hidden_size , length , _ = x . size ( ) h = linear ( torch . transpose ( x , 1 , 2 ) . contiguous ( ) . view ( batch * length , hidden_size ) ) return torch . transpose ( h . view ( batch , length , hidden_size , 1 ) , 1 , 2 )


def apply_mask(self, mask):
        """ Apply mask """
        self.mask = mask

def apply_mask ( self , mask ) : self . mask = mask


Args : base_target_emb : target emb tensor input : output of decode conv encoder_out_t : the key matrix for calculation of attetion weight which is the top output of encode conv encoder_out_combine : the value matrix for the attention - weighted sum which is the combination of base emb and top output of encode

def forward(self, base_target_emb, input_from_dec, encoder_out_top,
                encoder_out_combine):
        """
        Args:
            base_target_emb: target emb tensor
            input: output of decode conv
            encoder_out_t: the key matrix for calculation of attetion weight,
                which is the top output of encode conv
            encoder_out_combine:
                the value matrix for the attention-weighted sum,
                which is the combination of base emb and top output of encode

        """
        # checks
        # batch, channel, height, width = base_target_emb.size()
        batch, _, height, _ = base_target_emb.size()
        # batch_, channel_, height_, width_ = input_from_dec.size()
        batch_, _, height_, _ = input_from_dec.size()
        aeq(batch, batch_)
        aeq(height, height_)

        # enc_batch, enc_channel, enc_height = encoder_out_top.size()
        enc_batch, _, enc_height = encoder_out_top.size()
        # enc_batch_, enc_channel_, enc_height_ = encoder_out_combine.size()
        enc_batch_, _, enc_height_ = encoder_out_combine.size()

        aeq(enc_batch, enc_batch_)
        aeq(enc_height, enc_height_)

        preatt = seq_linear(self.linear_in, input_from_dec)
        target = (base_target_emb + preatt) * SCALE_WEIGHT
        target = torch.squeeze(target, 3)
        target = torch.transpose(target, 1, 2)
        pre_attn = torch.bmm(target, encoder_out_top)

        if self.mask is not None:
            pre_attn.data.masked_fill_(self.mask, -float('inf'))

        pre_attn = pre_attn.transpose(0, 2)
        attn = F.softmax(pre_attn, dim=-1)
        attn = attn.transpose(0, 2).contiguous()
        context_output = torch.bmm(
            attn, torch.transpose(encoder_out_combine, 1, 2))
        context_output = torch.transpose(
            torch.unsqueeze(context_output, 3), 1, 2)
        return context_output, attn


def get_var_maybe_avg(namespace, var_name, training, polyak_decay):
    """ utility for retrieving polyak averaged params
        Update average
    """
    v = getattr(namespace, var_name)
    v_avg = getattr(namespace, var_name + '_avg')
    v_avg -= (1 - polyak_decay) * (v_avg - v.data)

    if training:
        return v
    else:
        return v_avg

def get_var_maybe_avg ( namespace , var_name , training , polyak_decay ) : v = getattr ( namespace , var_name ) v_avg = getattr ( namespace , var_name + '_avg' ) v_avg -= ( 1 - polyak_decay ) * ( v_avg - v . data ) if training : return v else : return v_avg


def get_vars_maybe_avg(namespace, var_names, training, polyak_decay):
    """ utility for retrieving polyak averaged params """
    vars = []
    for vn in var_names:
        vars.append(get_var_maybe_avg(
            namespace, vn, training, polyak_decay))
    return vars

def get_vars_maybe_avg ( namespace , var_names , training , polyak_decay ) : vars = [ ] for vn in var_names : vars . append ( get_var_maybe_avg ( namespace , vn , training , polyak_decay ) ) return vars


This is a hack . Something is broken with torch pickle .

def __reduce_ex__(self, proto):
        "This is a hack. Something is broken with torch pickle."
        return super(DatasetBase, self).__reduce_ex__()

def __reduce_ex__ ( self , proto ) : return super ( DatasetBase , self ) . __reduce_ex__ ( )


Load fields from vocab . pt and set the fields attribute .

def load_fields(self, vocab_dict):
        """ Load fields from vocab.pt, and set the `fields` attribute.

        Args:
            vocab_dict (dict): a dict of loaded vocab from vocab.pt file.
        """
        fields = onmt.inputters.inputter.load_fields_from_vocab(
            vocab_dict.items(), self.data_type)
        self.fields = dict([(k, f) for (k, f) in fields.items()
                            if k in self.examples[0].__dict__])

def load_fields ( self , vocab_dict ) : fields = onmt . inputters . inputter . load_fields_from_vocab ( vocab_dict . items ( ) , self . data_type ) self . fields = dict ( [ ( k , f ) for ( k , f ) in fields . items ( ) if k in self . examples [ 0 ] . __dict__ ] )


Args : tokens : A list of tokens where each token consists of a word optionally followed by u ￨ - delimited features . Returns : A sequence of words a sequence of features and num of features .

def extract_text_features(tokens):
        """
        Args:
            tokens: A list of tokens, where each token consists of a word,
                optionally followed by u"￨"-delimited features.
        Returns:
            A sequence of words, a sequence of features, and num of features.
        """
        if not tokens:
            return [], [], -1

        specials = [PAD_WORD, UNK_WORD, BOS_WORD, EOS_WORD]
        words = []
        features = []
        n_feats = None
        for token in tokens:
            split_token = token.split(u"￨")
            assert all([special != split_token[0] for special in specials]), \
                "Dataset cannot contain Special Tokens"

            if split_token[0]:
                words += [split_token[0]]
                features += [split_token[1:]]

                if n_feats is None:
                    n_feats = len(split_token)
                else:
                    assert len(split_token) == n_feats, \
                        "all words must have the same number of features"
        features = list(zip(*features))
        return tuple(words), features, n_feats - 1


Args:
            dictionaries with disjoint keys.

Args : dictionaries with disjoint keys .

def _join_dicts(self, *args):
        """
        Args:
            dictionaries with disjoint keys.

        Returns:
            a single dictionary that has the union of these keys.
        """
        return dict(chain(*[d.items() for d in args]))

def _join_dicts ( self , * args ) : return dict ( chain ( * [ d . items ( ) for d in args ] ) )


def _peek(self, seq):
        """
        Args:
            seq: an iterator.

        Returns:
            the first thing returned by calling next() on the iterator
            and an iterator created by re-chaining that value to the beginning
            of the iterator.
        """
        first = next(seq)
        return first, chain([first], seq)

def _peek ( self , seq ) : first = next ( seq ) return first , chain ( [ first ] , seq )


Args : data : the data to be set as the value of the attributes of the to - be - created Example associating with respective Field objects with same key . fields : a dict of torchtext . data . Field objects . The keys are attributes of the to - be - created Example .

def _construct_example_fromlist(self, data, fields):
        """
        Args:
            data: the data to be set as the value of the attributes of
                the to-be-created `Example`, associating with respective
                `Field` objects with same key.
            fields: a dict of `torchtext.data.Field` objects. The keys
                are attributes of the to-be-created `Example`.

        Returns:
            the created `Example` object.
        """
        ex = torchtext.data.Example()
        for (name, field), val in zip(fields, data):
            if field is not None:
                setattr(ex, name, field.preprocess(val))
            else:
                setattr(ex, name, val)
        return ex

def _construct_example_fromlist ( self , data , fields ) : ex = torchtext . data . Example ( ) for ( name , field ) , val in zip ( fields , data ) : if field is not None : setattr ( ex , name , field . preprocess ( val ) ) else : setattr ( ex , name , val ) return ex


def sort_key(self, ex):
        """ Sort using length of source sentences. """
        # Default to a balanced sort, prioritizing tgt len match.
        # TODO: make this configurable.
        if hasattr(ex, "tgt"):
            return len(ex.src), len(ex.tgt)
        return len(ex.src)


Given scores from an expanded dictionary corresponeding to a batch sums together copies with a dictionary word when it is ambigious .

def collapse_copy_scores(scores, batch, tgt_vocab, src_vocabs):
        """
        Given scores from an expanded dictionary
        corresponeding to a batch, sums together copies,
        with a dictionary word when it is ambigious.
        """
        offset = len(tgt_vocab)
        for b in range(batch.batch_size):
            blank = []
            fill = []
            index = batch.indices.data[b]
            src_vocab = src_vocabs[index]
            for i in range(1, len(src_vocab)):
                sw = src_vocab.itos[i]
                ti = tgt_vocab.stoi[sw]
                if ti != 0:
                    blank.append(offset + i)
                    fill.append(ti)
            if blank:
                blank = torch.Tensor(blank).type_as(batch.indices.data)
                fill = torch.Tensor(fill).type_as(batch.indices.data)
                scores[:, b].index_add_(1, fill,
                                        scores[:, b].index_select(1, blank))
                scores[:, b].index_fill_(1, blank, 1e-10)
        return scores

def collapse_copy_scores ( scores , batch , tgt_vocab , src_vocabs ) : offset = len ( tgt_vocab ) for b in range ( batch . batch_size ) : blank = [ ] fill = [ ] index = batch . indices . data [ b ] src_vocab = src_vocabs [ index ] for i in range ( 1 , len ( src_vocab ) ) : sw = src_vocab . itos [ i ] ti = tgt_vocab . stoi [ sw ] if ti != 0 : blank . append ( offset + i ) fill . append ( ti ) if blank : blank = torch . Tensor ( blank ) . type_as ( batch . indices . data ) fill = torch . Tensor ( fill ) . type_as ( batch . indices . data ) scores [ : , b ] . index_add_ ( 1 , fill , scores [ : , b ] . index_select ( 1 , blank ) ) scores [ : , b ] . index_fill_ ( 1 , blank , 1e-10 ) return scores


Args : text_iter ( iterator ) : an iterator ( or None ) that we can loop over to read examples . It may be an openned file a string list etc ... text_path ( str ) : path to file or None path ( str ) : location of a src or tgt file . truncate ( int ) : maximum sequence length ( 0 for unlimited ) . side ( str ) : src or tgt .

def make_text_examples_nfeats_tpl(text_iter, text_path, truncate, side):
        """
        Args:
            text_iter(iterator): an iterator (or None) that we can loop over
                to read examples.
                It may be an openned file, a string list etc...
            text_path(str): path to file or None
            path (str): location of a src or tgt file.
            truncate (int): maximum sequence length (0 for unlimited).
            side (str): "src" or "tgt".

        Returns:
            (example_dict iterator, num_feats) tuple.
        """
        assert side in ['src', 'tgt']

        if text_iter is None:
            if text_path is not None:
                text_iter = TextDataset.make_text_iterator_from_file(text_path)
            else:
                return (None, 0)

        # All examples have same number of features, so we peek first one
        # to get the num_feats.
        examples_nfeats_iter = \
            TextDataset.make_examples(text_iter, truncate, side)

        first_ex = next(examples_nfeats_iter)
        num_feats = first_ex[1]

        # Chain back the first element - we only want to peek it.
        examples_nfeats_iter = chain([first_ex], examples_nfeats_iter)
        examples_iter = (ex for ex, nfeats in examples_nfeats_iter)

        return (examples_iter, num_feats)


Args : text_iter ( iterator ) : iterator of text sequences truncate ( int ) : maximum sequence length ( 0 for unlimited ) . side ( str ) : src or tgt .

def make_examples(text_iter, truncate, side):
        """
        Args:
            text_iter (iterator): iterator of text sequences
            truncate (int): maximum sequence length (0 for unlimited).
            side (str): "src" or "tgt".

        Yields:
            (word, features, nfeat) triples for each line.
        """
        for i, line in enumerate(text_iter):
            line = line.strip().split()
            if truncate:
                line = line[:truncate]

            words, feats, n_feats = \
                TextDataset.extract_text_features(line)

            example_dict = {side: words, "indices": i}
            if feats:
                prefix = side + "_feat_"
                example_dict.update((prefix + str(j), f)
                                    for j, f in enumerate(feats))
            yield example_dict, n_feats

def make_examples ( text_iter , truncate , side ) : for i , line in enumerate ( text_iter ) : line = line . strip ( ) . split ( ) if truncate : line = line [ : truncate ] words , feats , n_feats = TextDataset . extract_text_features ( line ) example_dict = { side : words , "indices" : i } if feats : prefix = side + "_feat_" example_dict . update ( ( prefix + str ( j ) , f ) for j , f in enumerate ( feats ) ) yield example_dict , n_feats


Args : n_src_features ( int ) : the number of source features to create torchtext . data . Field for . n_tgt_features ( int ) : the number of target features to create torchtext . data . Field for .

def get_fields(n_src_features, n_tgt_features):
        """
        Args:
            n_src_features (int): the number of source features to
                create `torchtext.data.Field` for.
            n_tgt_features (int): the number of target features to
                create `torchtext.data.Field` for.

        Returns:
            A dictionary whose keys are strings and whose values
            are the corresponding Field objects.
        """
        fields = {}

        fields["src"] = torchtext.data.Field(
            pad_token=PAD_WORD,
            include_lengths=True)

        for j in range(n_src_features):
            fields["src_feat_" + str(j)] = \
                torchtext.data.Field(pad_token=PAD_WORD)

        fields["tgt"] = torchtext.data.Field(
            init_token=BOS_WORD, eos_token=EOS_WORD,
            pad_token=PAD_WORD)

        for j in range(n_tgt_features):
            fields["tgt_feat_" + str(j)] = \
                torchtext.data.Field(init_token=BOS_WORD, eos_token=EOS_WORD,
                                     pad_token=PAD_WORD)

        def make_src(data, vocab):
            """ ? """
            #pdb.set_trace()
            src_size = max([t.size(0) for t in data])

            src_vocab_size = int(max([t.max() for t in data])) + 1

            try:
                alignment = torch.zeros(src_size, len(data), src_vocab_size)
            except:

                print(src_size)
                print(len(data))
                print(src_vocab_size)

            for i, sent in enumerate(data):
                for j, t in enumerate(sent):
                    alignment[j, i, t] = 1
            return alignment

        fields["src_map"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.float,
            postprocessing=make_src, sequential=False)

        def make_tgt(data, vocab):
            """ ? """
            #pdb.set_trace()
            tgt_size = max([t.size(0) for t in data])
            alignment = torch.zeros(tgt_size, len(data)).long()
            for i, sent in enumerate(data):
                alignment[:sent.size(0), i] = sent
            return alignment

        fields["alignment"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            postprocessing=make_tgt, sequential=False)

        fields["indices"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            sequential=False)

        return fields

def
get_fields
(
n_src_features
,
n_tgt_features
)
:
fields
=
{
}
fields
[
"src"
]
=
torchtext
.
data
.
Field
(
pad_token
=
PAD_WORD
,
include_lengths
=
True
)
for
j
in
range
(
n_src_features
)
:
fields
[
"src_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
pad_token
=
PAD_WORD
)
fields
[
"tgt"
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
for
j
in
range
(
n_tgt_features
)
:
fields
[
"tgt_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
def
make_src
(
data
,
vocab
)
:
""" ? """
#pdb.set_trace()
src_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
src_vocab_size
=
int
(
max
(
[
t
.
max
(
)
for
t
in
data
]
)
)
+
1
try
:
alignment
=
torch
.
zeros
(
src_size
,
len
(
data
)
,
src_vocab_size
)
except
:
print
(
src_size
)
print
(
len
(
data
)
)
print
(
src_vocab_size
)
for
i
,
sent
in
enumerate
(
data
)
:
for
j
,
t
in
enumerate
(
sent
)
:
alignment
[
j
,
i
,
t
]
=
1
return
alignment
fields
[
"src_map"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
float
,
postprocessing
=
make_src
,
sequential
=
False
)
def
make_tgt
(
data
,
vocab
)
:
""" ? """
#pdb.set_trace()
tgt_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
alignment
=
torch
.
zeros
(
tgt_size
,
len
(
data
)
)
.
long
(
)
for
i
,
sent
in
enumerate
(
data
)
:
alignment
[
:
sent
.
size
(
0
)
,
i
]
=
sent
return
alignment
fields
[
"alignment"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
postprocessing
=
make_tgt
,
sequential
=
False
)
fields
[
"indices"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
sequential
=
False
)
return
fields


Peek one line and get number of features of it . ( All lines must have same number of features ) . For text corpus both sides are in text form thus it works the same .

def get_num_features(corpus_file, side):
        """
        Peek one line and get number of features of it.
        (All lines must have same number of features).
        For text corpus, both sides are in text form, thus
        it works the same.

        Args:
            corpus_file (str): file path to get the features.
            side (str): 'src' or 'tgt'.

        Returns:
            number of features on `side`.
        """
        with codecs.open(corpus_file, "r", "utf-8") as cf:
            f_line = cf.readline().strip().split()
            _, _, num_feats = TextDataset.extract_text_features(f_line)

        return num_feats

def get_num_features ( corpus_file , side ) : with codecs . open ( corpus_file , "r" , "utf-8" ) as cf : f_line = cf . readline ( ) . strip ( ) . split ( ) _ , _ , num_feats = TextDataset . extract_text_features ( f_line ) return num_feats


Args : corpus_path : the corpus file path . line_truncate : the maximum length of a line to read . 0 for unlimited . side : src or tgt . shard_size : the shard size 0 means not sharding the file . assoc_iter : if not None it is the associate iterator that this iterator should align its step with .

def __init__(self, corpus_path, line_truncate, side, shard_size,
                 assoc_iter=None):
        """
        Args:
            corpus_path: the corpus file path.
            line_truncate: the maximum length of a line to read.
                            0 for unlimited.
            side: "src" or "tgt".
            shard_size: the shard size, 0 means not sharding the file.
            assoc_iter: if not None, it is the associate iterator that
                        this iterator should align its step with.
        """
        try:
            # The codecs module seems to have bugs with seek()/tell(),
            # so we use io.open().
            self.corpus = io.open(corpus_path, "r", encoding="utf-8")
        except IOError:
            sys.stderr.write("Failed to open corpus file: %s" % corpus_path)
            sys.exit(1)

        self.line_truncate = line_truncate
        self.side = side
        self.shard_size = shard_size
        self.assoc_iter = assoc_iter
        self.last_pos = 0
        self.line_index = -1
        self.eof = False


Iterator of ( example_dict nfeats ) . On each call it iterates over as many ( example_dict nfeats ) tuples until this shard s size equals to or approximates self . shard_size .

def __iter__(self):
        """
        Iterator of (example_dict, nfeats).
        On each call, it iterates over as many (example_dict, nfeats) tuples
        until this shard's size equals to or approximates `self.shard_size`.
        """
        iteration_index = -1
        if self.assoc_iter is not None:
            # We have associate iterator, just yields tuples
            # util we run parallel with it.
            while self.line_index < self.assoc_iter.line_index:
                line = self.corpus.readline()
                if line == '':
                    raise AssertionError(
                        "Two corpuses must have same number of lines!")

                self.line_index += 1
                iteration_index += 1
                yield self._example_dict_iter(line, iteration_index)

            if self.assoc_iter.eof:
                self.eof = True
                self.corpus.close()
        else:
            # Yield tuples util this shard's size reaches the threshold.
            self.corpus.seek(self.last_pos)
            while True:
                if self.shard_size != 0 and self.line_index % 64 == 0:
                    # This part of check is time consuming on Py2 (but
                    # it is quite fast on Py3, weird!). So we don't bother
                    # to check for very line. Instead we chekc every 64
                    # lines. Thus we are not dividing exactly per
                    # `shard_size`, but it is not too much difference.
                    cur_pos = self.corpus.tell()
                    if cur_pos >= self.last_pos + self.shard_size:
                        self.last_pos = cur_pos
                        return

                line = self.corpus.readline()
                if line == '':
                    self.eof = True
                    self.corpus.close()
                    return

                self.line_index += 1
                iteration_index += 1
                yield self._example_dict_iter(line, iteration_index)


def hit_end(self):
        """ ? """
        return self.eof

def hit_end ( self ) : return self . eof


We peek the first line and seek back to the beginning of the file .

def num_feats(self):
        """
        We peek the first line and seek back to
        the beginning of the file.
        """
        saved_pos = self.corpus.tell()

        line = self.corpus.readline().split()
        if self.line_truncate:
            line = line[:self.line_truncate]
        _, _, self.n_feats = TextDataset.extract_text_features(line)

        self.corpus.seek(saved_pos)

        return self.n_feats

def num_feats ( self ) : saved_pos = self . corpus . tell ( ) line = self . corpus . readline ( ) . split ( ) if self . line_truncate : line = line [ : self . line_truncate ] _ , _ , self . n_feats = TextDataset . extract_text_features ( line ) self . corpus . seek ( saved_pos ) return self . n_feats


Sort using the size of the image : ( width height ) .

def sort_key(self, ex):
        """ Sort using the size of the image: (width, height)."""
        return (ex.src.size(2), ex.src.size(1))

def sort_key ( self , ex ) : return ( ex . src . size ( 2 ) , ex . src . size ( 1 ) )


Note : one of img_iter and img_path must be not None Args : img_iter ( iterator ) : an iterator that yields pairs ( img filename ) ( or None ) img_path ( str ) : location of a src file containing image paths ( or None ) src_dir ( str ) : location of source images

def make_image_examples_nfeats_tpl(img_iter, img_path, img_dir,
                                       image_channel_size=3):
        """
        Note: one of img_iter and img_path must be not None
        Args:
            img_iter(iterator): an iterator that yields pairs (img, filename)
                (or None)
            img_path(str): location of a src file containing image paths
                (or None)
            src_dir (str): location of source images

        Returns:
            (example_dict iterator, num_feats) tuple
        """
        if img_iter is None:
            if img_path is not None:
                img_iter = ImageDataset. \
                    make_img_iterator_from_file(img_path,
                                                img_dir,
                                                image_channel_size)
            else:
                raise ValueError("""One of 'img_iter' and 'img_path'
                                    must be not None""")
        examples_iter = ImageDataset.make_examples(img_iter, img_dir, 'src')
        num_feats = 0  # Source side(img) has no features.

        return (examples_iter, num_feats)


Args : path ( str ) : location of a src file containing image paths src_dir ( str ) : location of source images side ( str ) : src or tgt truncate : maximum img size (( 0 0 ) or None for unlimited )

def make_examples(img_iter, src_dir, side, truncate=None):
        """
        Args:
            path (str): location of a src file containing image paths
            src_dir (str): location of source images
            side (str): 'src' or 'tgt'
            truncate: maximum img size ((0,0) or None for unlimited)

        Yields:
            a dictionary containing image data, path and index for each line.
        """
        assert (src_dir is not None) and os.path.exists(src_dir), \
            'src_dir must be a valid directory if data_type is img'

        for index, (img, filename) in enumerate(img_iter):
            if truncate and truncate != (0, 0):
                if not (img.size(1) <= truncate[0]
                        and img.size(2) <= truncate[1]):
                    continue

            example_dict = {side: img,
                            side + '_path': filename,
                            'indices': index}
            yield example_dict


Args:
            path(str):
            src_dir(str):

def make_img_iterator_from_file(path, src_dir, image_channel_size=3):
        """
        Args:
            path(str):
            src_dir(str):

        Yields:
            img: and image tensor
            filename(str): the image filename
        """
        from PIL import Image
        from torchvision import transforms

        with codecs.open(path, "r", "utf-8") as corpus_file:
            for line in corpus_file:
                filename = line.strip()
                img_path = os.path.join(src_dir, filename)
                if not os.path.exists(img_path):
                    img_path = line

                assert os.path.exists(img_path), \
                    'img path %s not found' % (line.strip())

                if (image_channel_size == 1):
                    img = transforms.ToTensor()(
                        Image.fromarray(cv2.imread(img_path, 0)))
                else:
                    img = transforms.ToTensor()(Image.open(img_path))

                yield img, filename

def
make_img_iterator_from_file
(
path
,
src_dir
,
image_channel_size
=
3
)
:
from
PIL
import
Image
from
torchvision
import
transforms
with
codecs
.
open
(
path
,
"r"
,
"utf-8"
)
as
corpus_file
:
for
line
in
corpus_file
:
filename
=
line
.
strip
(
)
img_path
=
os
.
path
.
join
(
src_dir
,
filename
)
if
not
os
.
path
.
exists
(
img_path
)
:
img_path
=
line
assert
os
.
path
.
exists
(
img_path
)
,
'img path %s not found'
%
(
line
.
strip
(
)
)
if
(
image_channel_size
==
1
)
:
img
=
transforms
.
ToTensor
(
)
(
Image
.
fromarray
(
cv2
.
imread
(
img_path
,
0
)
)
)
else
:
img
=
transforms
.
ToTensor
(
)
(
Image
.
open
(
img_path
)
)
yield
img
,
filename


Args : n_src_features : the number of source features to create torchtext . data . Field for . n_tgt_features : the number of target features to create torchtext . data . Field for .

def get_fields(n_src_features, n_tgt_features):
        """
        Args:
            n_src_features: the number of source features to
                create `torchtext.data.Field` for.
            n_tgt_features: the number of target features to
                create `torchtext.data.Field` for.

        Returns:
            A dictionary whose keys are strings and whose values
            are the corresponding Field objects.
        """
        fields = {}

        def make_img(data, vocab):
            """ ? """
            c = data[0].size(0)
            h = max([t.size(1) for t in data])
            w = max([t.size(2) for t in data])
            imgs = torch.zeros(len(data), c, h, w).fill_(1)
            for i, img in enumerate(data):
                imgs[i, :, 0:img.size(1), 0:img.size(2)] = img
            return imgs

        fields["src"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.float,
            postprocessing=make_img, sequential=False)

        for j in range(n_src_features):
            fields["src_feat_" + str(j)] = \
                torchtext.data.Field(pad_token=PAD_WORD)

        fields["tgt"] = torchtext.data.Field(
            init_token=BOS_WORD, eos_token=EOS_WORD,
            pad_token=PAD_WORD)

        for j in range(n_tgt_features):
            fields["tgt_feat_" + str(j)] = \
                torchtext.data.Field(init_token=BOS_WORD, eos_token=EOS_WORD,
                                     pad_token=PAD_WORD)

        def make_src(data, vocab):
            """ ? """
            src_size = max([t.size(0) for t in data])
            src_vocab_size = max([t.max() for t in data]) + 1
            alignment = torch.zeros(src_size, len(data), src_vocab_size)
            for i, sent in enumerate(data):
                for j, t in enumerate(sent):
                    alignment[j, i, t] = 1
            return alignment

        fields["src_map"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.float,
            postprocessing=make_src, sequential=False)

        def make_tgt(data, vocab):
            """ ? """
            tgt_size = max([t.size(0) for t in data])
            alignment = torch.zeros(tgt_size, len(data)).long()
            for i, sent in enumerate(data):
                alignment[:sent.size(0), i] = sent
            return alignment

        fields["alignment"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            postprocessing=make_tgt, sequential=False)

        fields["indices"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            sequential=False)

        return fields

def
get_fields
(
n_src_features
,
n_tgt_features
)
:
fields
=
{
}
def
make_img
(
data
,
vocab
)
:
""" ? """
c
=
data
[
0
]
.
size
(
0
)
h
=
max
(
[
t
.
size
(
1
)
for
t
in
data
]
)
w
=
max
(
[
t
.
size
(
2
)
for
t
in
data
]
)
imgs
=
torch
.
zeros
(
len
(
data
)
,
c
,
h
,
w
)
.
fill_
(
1
)
for
i
,
img
in
enumerate
(
data
)
:
imgs
[
i
,
:
,
0
:
img
.
size
(
1
)
,
0
:
img
.
size
(
2
)
]
=
img
return
imgs
fields
[
"src"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
float
,
postprocessing
=
make_img
,
sequential
=
False
)
for
j
in
range
(
n_src_features
)
:
fields
[
"src_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
pad_token
=
PAD_WORD
)
fields
[
"tgt"
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
for
j
in
range
(
n_tgt_features
)
:
fields
[
"tgt_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
def
make_src
(
data
,
vocab
)
:
""" ? """
src_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
src_vocab_size
=
max
(
[
t
.
max
(
)
for
t
in
data
]
)
+
1
alignment
=
torch
.
zeros
(
src_size
,
len
(
data
)
,
src_vocab_size
)
for
i
,
sent
in
enumerate
(
data
)
:
for
j
,
t
in
enumerate
(
sent
)
:
alignment
[
j
,
i
,
t
]
=
1
return
alignment
fields
[
"src_map"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
float
,
postprocessing
=
make_src
,
sequential
=
False
)
def
make_tgt
(
data
,
vocab
)
:
""" ? """
tgt_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
alignment
=
torch
.
zeros
(
tgt_size
,
len
(
data
)
)
.
long
(
)
for
i
,
sent
in
enumerate
(
data
)
:
alignment
[
:
sent
.
size
(
0
)
,
i
]
=
sent
return
alignment
fields
[
"alignment"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
postprocessing
=
make_tgt
,
sequential
=
False
)
fields
[
"indices"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
sequential
=
False
)
return
fields


For image corpus source side is in form of image thus no feature ; while target side is in form of text thus we can extract its text features .

def get_num_features(corpus_file, side):
        """
        For image corpus, source side is in form of image, thus
        no feature; while target side is in form of text, thus
        we can extract its text features.

        Args:
            corpus_file (str): file path to get the features.
            side (str): 'src' or 'tgt'.

        Returns:
            number of features on `side`.
        """
        if side == 'src':
            num_feats = 0
        else:
            with codecs.open(corpus_file, "r", "utf-8") as cf:
                f_line = cf.readline().strip().split()
                _, _, num_feats = ImageDataset.extract_text_features(f_line)

        return num_feats

def get_num_features ( corpus_file , side ) : if side == 'src' : num_feats = 0 else : with codecs . open ( corpus_file , "r" , "utf-8" ) as cf : f_line = cf . readline ( ) . strip ( ) . split ( ) _ , _ , num_feats = ImageDataset . extract_text_features ( f_line ) return num_feats


Sort using duration time of the sound spectrogram .

def sort_key(self, ex):
        """ Sort using duration time of the sound spectrogram. """
        return ex.src.size(1)

def sort_key ( self , ex ) : return ex . src . size ( 1 )


Args : path ( str ) : location of a src file containing audio paths . audio_dir ( str ) : location of source audio files . sample_rate ( int ) : sample_rate . window_size ( float ) : window size for spectrogram in seconds . window_stride ( float ) : window stride for spectrogram in seconds . window ( str ) : window type for spectrogram generation . normalize_audio ( bool ) : subtract spectrogram by mean and divide by std or not . truncate ( int ) : maximum audio length ( 0 or None for unlimited ) .

def make_audio_examples_nfeats_tpl(path, audio_dir,
                                       sample_rate, window_size,
                                       window_stride, window,
                                       normalize_audio, truncate=None):
        """
        Args:
            path (str): location of a src file containing audio paths.
            audio_dir (str): location of source audio files.
            sample_rate (int): sample_rate.
            window_size (float) : window size for spectrogram in seconds.
            window_stride (float): window stride for spectrogram in seconds.
            window (str): window type for spectrogram generation.
            normalize_audio (bool): subtract spectrogram by mean and divide
                by std or not.
            truncate (int): maximum audio length (0 or None for unlimited).

        Returns:
            (example_dict iterator, num_feats) tuple
        """
        examples_iter = AudioDataset.read_audio_file(
            path, audio_dir, "src", sample_rate,
            window_size, window_stride, window,
            normalize_audio, truncate)
        num_feats = 0  # Source side(audio) has no features.

        return (examples_iter, num_feats)

def
make_audio_examples_nfeats_tpl
(
path
,
audio_dir
,
sample_rate
,
window_size
,
window_stride
,
window
,
normalize_audio
,
truncate
=
)
:
examples_iter
=
AudioDataset
.
read_audio_file
(
path
,
audio_dir
,
"src"
,
sample_rate
,
window_size
,
window_stride
,
window
,
normalize_audio
,
truncate
)
num_feats
=
0
# Source side(audio) has no features.
return
(
examples_iter
,
num_feats
)


Args : path ( str ) : location of a src file containing audio paths . src_dir ( str ) : location of source audio files . side ( str ) : src or tgt . sample_rate ( int ) : sample_rate . window_size ( float ) : window size for spectrogram in seconds . window_stride ( float ) : window stride for spectrogram in seconds . window ( str ) : window type for spectrogram generation . normalize_audio ( bool ) : subtract spectrogram by mean and divide by std or not . truncate ( int ) : maximum audio length ( 0 or None for unlimited ) .

def read_audio_file(path, src_dir, side, sample_rate, window_size,
                        window_stride, window, normalize_audio,
                        truncate=None):
        """
        Args:
            path (str): location of a src file containing audio paths.
            src_dir (str): location of source audio files.
            side (str): 'src' or 'tgt'.
            sample_rate (int): sample_rate.
            window_size (float) : window size for spectrogram in seconds.
            window_stride (float): window stride for spectrogram in seconds.
            window (str): window type for spectrogram generation.
            normalize_audio (bool): subtract spectrogram by mean and divide
                by std or not.
            truncate (int): maximum audio length (0 or None for unlimited).

        Yields:
            a dictionary containing audio data for each line.
        """
        assert (src_dir is not None) and os.path.exists(src_dir),\
            "src_dir must be a valid directory if data_type is audio"

        import torchaudio
        import librosa
        import numpy as np

        with codecs.open(path, "r", "utf-8") as corpus_file:
            index = 0
            for line in corpus_file:
                audio_path = os.path.join(src_dir, line.strip())
                if not os.path.exists(audio_path):
                    audio_path = line

                assert os.path.exists(audio_path), \
                    'audio path %s not found' % (line.strip())

                sound, sample_rate = torchaudio.load(audio_path)
                if truncate and truncate > 0:
                    if sound.size(0) > truncate:
                        continue

                assert sample_rate == sample_rate, \
                    'Sample rate of %s != -sample_rate (%d vs %d)' \
                    % (audio_path, sample_rate, sample_rate)

                sound = sound.numpy()
                if len(sound.shape) > 1:
                    if sound.shape[1] == 1:
                        sound = sound.squeeze()
                    else:
                        sound = sound.mean(axis=1)  # average multiple channels

                n_fft = int(sample_rate * window_size)
                win_length = n_fft
                hop_length = int(sample_rate * window_stride)
                # STFT
                d = librosa.stft(sound, n_fft=n_fft, hop_length=hop_length,
                                 win_length=win_length, window=window)
                spect, _ = librosa.magphase(d)
                spect = np.log1p(spect)
                spect = torch.FloatTensor(spect)
                if normalize_audio:
                    mean = spect.mean()
                    std = spect.std()
                    spect.add_(-mean)
                    spect.div_(std)

                example_dict = {side: spect,
                                side + '_path': line.strip(),
                                'indices': index}
                index += 1

                yield example_dict


def get_fields(n_src_features, n_tgt_features):
        """
        Args:
            n_src_features: the number of source features to
                create `torchtext.data.Field` for.
            n_tgt_features: the number of target features to
                create `torchtext.data.Field` for.

        Returns:
            A dictionary whose keys are strings and whose values
            are the corresponding Field objects.
        """
        fields = {}

        def make_audio(data, vocab):
            """ ? """
            nfft = data[0].size(0)
            t = max([t.size(1) for t in data])
            sounds = torch.zeros(len(data), 1, nfft, t)
            for i, spect in enumerate(data):
                sounds[i, :, :, 0:spect.size(1)] = spect
            return sounds

        fields["src"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.float,
            postprocessing=make_audio, sequential=False)

        for j in range(n_src_features):
            fields["src_feat_" + str(j)] = \
                torchtext.data.Field(pad_token=PAD_WORD)

        fields["tgt"] = torchtext.data.Field(
            init_token=BOS_WORD, eos_token=EOS_WORD,
            pad_token=PAD_WORD)

        for j in range(n_tgt_features):
            fields["tgt_feat_" + str(j)] = \
                torchtext.data.Field(init_token=BOS_WORD, eos_token=EOS_WORD,
                                     pad_token=PAD_WORD)

        def make_src(data, vocab):
            """ ? """
            src_size = max([t.size(0) for t in data])
            src_vocab_size = max([t.max() for t in data]) + 1
            alignment = torch.zeros(src_size, len(data), src_vocab_size)
            for i, sent in enumerate(data):
                for j, t in enumerate(sent):
                    alignment[j, i, t] = 1
            return alignment

        fields["src_map"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.float,
            postprocessing=make_src, sequential=False)

        def make_tgt(data, vocab):
            """ ? """
            tgt_size = max([t.size(0) for t in data])
            alignment = torch.zeros(tgt_size, len(data)).long()
            for i, sent in enumerate(data):
                alignment[:sent.size(0), i] = sent
            return alignment

        fields["alignment"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            postprocessing=make_tgt, sequential=False)

        fields["indices"] = torchtext.data.Field(
            use_vocab=False, dtype=torch.long,
            sequential=False)

        return fields

def
get_fields
(
n_src_features
,
n_tgt_features
)
:
fields
=
{
}
def
make_audio
(
data
,
vocab
)
:
""" ? """
nfft
=
data
[
0
]
.
size
(
0
)
t
=
max
(
[
t
.
size
(
1
)
for
t
in
data
]
)
sounds
=
torch
.
zeros
(
len
(
data
)
,
1
,
nfft
,
t
)
for
i
,
spect
in
enumerate
(
data
)
:
sounds
[
i
,
:
,
:
,
0
:
spect
.
size
(
1
)
]
=
spect
return
sounds
fields
[
"src"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
float
,
postprocessing
=
make_audio
,
sequential
=
False
)
for
j
in
range
(
n_src_features
)
:
fields
[
"src_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
pad_token
=
PAD_WORD
)
fields
[
"tgt"
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
for
j
in
range
(
n_tgt_features
)
:
fields
[
"tgt_feat_"
+
str
(
j
)
]
=
torchtext
.
data
.
Field
(
init_token
=
BOS_WORD
,
eos_token
=
EOS_WORD
,
pad_token
=
PAD_WORD
)
def
make_src
(
data
,
vocab
)
:
""" ? """
src_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
src_vocab_size
=
max
(
[
t
.
max
(
)
for
t
in
data
]
)
+
1
alignment
=
torch
.
zeros
(
src_size
,
len
(
data
)
,
src_vocab_size
)
for
i
,
sent
in
enumerate
(
data
)
:
for
j
,
t
in
enumerate
(
sent
)
:
alignment
[
j
,
i
,
t
]
=
1
return
alignment
fields
[
"src_map"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
float
,
postprocessing
=
make_src
,
sequential
=
False
)
def
make_tgt
(
data
,
vocab
)
:
""" ? """
tgt_size
=
max
(
[
t
.
size
(
0
)
for
t
in
data
]
)
alignment
=
torch
.
zeros
(
tgt_size
,
len
(
data
)
)
.
long
(
)
for
i
,
sent
in
enumerate
(
data
)
:
alignment
[
:
sent
.
size
(
0
)
,
i
]
=
sent
return
alignment
fields
[
"alignment"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
postprocessing
=
make_tgt
,
sequential
=
False
)
fields
[
"indices"
]
=
torchtext
.
data
.
Field
(
use_vocab
=
False
,
dtype
=
torch
.
long
,
sequential
=
False
)
return
fields


For audio corpus source side is in form of audio thus no feature ; while target side is in form of text thus we can extract its text features .

def get_num_features(corpus_file, side):
        """
        For audio corpus, source side is in form of audio, thus
        no feature; while target side is in form of text, thus
        we can extract its text features.

        Args:
            corpus_file (str): file path to get the features.
            side (str): 'src' or 'tgt'.

        Returns:
            number of features on `side`.
        """
        if side == 'src':
            num_feats = 0
        else:
            with codecs.open(corpus_file, "r", "utf-8") as cf:
                f_line = cf.readline().strip().split()
                _, _, num_feats = AudioDataset.extract_text_features(f_line)

        return num_feats

def get_num_features ( corpus_file , side ) : if side == 'src' : num_feats = 0 else : with codecs . open ( corpus_file , "r" , "utf-8" ) as cf : f_line = cf . readline ( ) . strip ( ) . split ( ) _ , _ , num_feats = AudioDataset . extract_text_features ( f_line ) return num_feats


Args : data_type : type of the source input . Options are [ text|img|audio ] . n_src_features : the number of source features to create torchtext . data . Field for . n_tgt_features : the number of target features to create torchtext . data . Field for .

def get_fields(data_type, n_src_features, n_tgt_features):
    """
    Args:
        data_type: type of the source input. Options are [text|img|audio].
        n_src_features: the number of source features to
            create `torchtext.data.Field` for.
        n_tgt_features: the number of target features to
            create `torchtext.data.Field` for.

    Returns:
        A dictionary whose keys are strings and whose values are the
        corresponding Field objects.
    """
    if data_type == 'text':
        return TextDataset.get_fields(n_src_features, n_tgt_features)
    elif data_type == 'img':
        return ImageDataset.get_fields(n_src_features, n_tgt_features)
    elif data_type == 'audio':
        return AudioDataset.get_fields(n_src_features, n_tgt_features)
    else:
        raise ValueError("Data type not implemented")

def
get_fields
(
data_type
,
n_src_features
,
n_tgt_features
)
:
if
data_type
==
'text'
:
return
TextDataset
.
get_fields
(
n_src_features
,
n_tgt_features
)
elif
data_type
==
'img'
:
return
ImageDataset
.
get_fields
(
n_src_features
,
n_tgt_features
)
elif
data_type
==
'audio'
:
return
AudioDataset
.
get_fields
(
n_src_features
,
n_tgt_features
)
else
:
raise
ValueError
(
"Data type not implemented"
)


Load Field objects from vocab . pt file .

def load_fields_from_vocab(vocab, data_type="text"):
    """
    Load Field objects from `vocab.pt` file.
    """
    vocab = dict(vocab)
    n_src_features = len(collect_features(vocab, 'src'))
    n_tgt_features = len(collect_features(vocab, 'tgt'))
    fields = get_fields(data_type, n_src_features, n_tgt_features)
    for k, v in vocab.items():
        # Hack. Can't pickle defaultdict :(
        v.stoi = defaultdict(lambda: 0, v.stoi)
        fields[k].vocab = v
    return fields

def
load_fields_from_vocab
(
vocab
,
data_type
=
"text"
)
:
vocab
=
dict
(
vocab
)
n_src_features
=
len
(
collect_features
(
vocab
,
'src'
)
)
n_tgt_features
=
len
(
collect_features
(
vocab
,
'tgt'
)
)
fields
=
get_fields
(
data_type
,
n_src_features
,
n_tgt_features
)
for
k
,
v
in
vocab
.
items
(
)
:
# Hack. Can't pickle defaultdict :(
v
.
stoi
=
defaultdict
(
lambda
:
0
,
v
.
stoi
)
fields
[
k
]
.
vocab
=
v
return
fields


Save Vocab objects in Field objects to vocab . pt file .

def save_fields_to_vocab(fields):
    """
    Save Vocab objects in Field objects to `vocab.pt` file.
    """
    vocab = []
    for k, f in fields.items():
        if f is not None and 'vocab' in f.__dict__:
            f.vocab.stoi = f.vocab.stoi
            vocab.append((k, f.vocab))
    return vocab

def save_fields_to_vocab ( fields ) : vocab = [ ] for k , f in fields . items ( ) : if f is not None and 'vocab' in f . __dict__ : f . vocab . stoi = f . vocab . stoi vocab . append ( ( k , f . vocab ) ) return vocab


Merge individual vocabularies ( assumed to be generated from disjoint documents ) into a larger vocabulary .

def merge_vocabs(vocabs, vocab_size=None):
    """
    Merge individual vocabularies (assumed to be generated from disjoint
    documents) into a larger vocabulary.

    Args:
        vocabs: `torchtext.vocab.Vocab` vocabularies to be merged
        vocab_size: `int` the final vocabulary size. `None` for no limit.
    Return:
        `torchtext.vocab.Vocab`
    """
    merged = sum([vocab.freqs for vocab in vocabs], Counter())
    return torchtext.vocab.Vocab(merged,
                                 specials=[UNK_WORD, PAD_WORD,
                                           BOS_WORD, EOS_WORD],
                                 max_size=vocab_size)

def merge_vocabs ( vocabs , vocab_size = None ) : merged = sum ( [ vocab . freqs for vocab in vocabs ] , Counter ( ) ) return torchtext . vocab . Vocab ( merged , specials = [ UNK_WORD , PAD_WORD , BOS_WORD , EOS_WORD ] , max_size = vocab_size )


Args : data_type ( str ) : type of the source input . Options are [ text|img|audio ] . corpus_file ( str ) : file path to get the features . side ( str ) : for source or for target .

def get_num_features(data_type, corpus_file, side):
    """
    Args:
        data_type (str): type of the source input.
            Options are [text|img|audio].
        corpus_file (str): file path to get the features.
        side (str): for source or for target.

    Returns:
        number of features on `side`.
    """
    assert side in ["src", "tgt"]

    if data_type == 'text':
        return TextDataset.get_num_features(corpus_file, side)
    elif data_type == 'img':
        return ImageDataset.get_num_features(corpus_file, side)
    elif data_type == 'audio':
        return AudioDataset.get_num_features(corpus_file, side)
    else:
        raise ValueError("Data type not implemented")

def
get_num_features
(
data_type
,
corpus_file
,
side
)
:
assert
side
in
[
"src"
,
"tgt"
]
if
data_type
==
'text'
:
return
TextDataset
.
get_num_features
(
corpus_file
,
side
)
elif
data_type
==
'img'
:
return
ImageDataset
.
get_num_features
(
corpus_file
,
side
)
elif
data_type
==
'audio'
:
return
AudioDataset
.
get_num_features
(
corpus_file
,
side
)
else
:
raise
ValueError
(
"Data type not implemented"
)


Args : batch ( Tensor ) : a batch of source or target data . side ( str ) : for source or for target . data_type ( str ) : type of the source input . Options are [ text|img|audio ] . Returns : A sequence of src / tgt tensors with optional feature tensors of size ( len x batch ) .

def make_features(batch, side, data_type='text'):
    """
    Args:
        batch (Tensor): a batch of source or target data.
        side (str): for source or for target.
        data_type (str): type of the source input.
            Options are [text|img|audio].
    Returns:
        A sequence of src/tgt tensors with optional feature tensors
        of size (len x batch).
    """
    assert side in ['src', 'tgt']
    if isinstance(batch.__dict__[side], tuple):
        data = batch.__dict__[side][0]
    else:
        data = batch.__dict__[side]

    feat_start = side + "_feat_"
    keys = sorted([k for k in batch.__dict__ if feat_start in k])
    features = [batch.__dict__[k] for k in keys]
    levels = [data] + features

    if data_type == 'text':
        return torch.cat([level.unsqueeze(2) for level in levels], 2)
    else:
        return levels[0]

def make_features ( batch , side , data_type = 'text' ) : assert side in [ 'src' , 'tgt' ] if isinstance ( batch . __dict__ [ side ] , tuple ) : data = batch . __dict__ [ side ] [ 0 ] else : data = batch . __dict__ [ side ] feat_start = side + "_feat_" keys = sorted ( [ k for k in batch . __dict__ if feat_start in k ] ) features = [ batch . __dict__ [ k ] for k in keys ] levels = [ data ] + features if data_type == 'text' : return torch . cat ( [ level . unsqueeze ( 2 ) for level in levels ] , 2 ) else : return levels [ 0 ]


def collect_features(fields, side="src"):
    """
    Collect features from Field object.
    """
    assert side in ["src", "tgt"]
    feats = []
    for j in count():
        key = side + "_feat_" + str(j)
        if key not in fields:
            break
        feats.append(key)
    return feats

def collect_features ( fields , side = "src" ) : assert side in [ "src" , "tgt" ] feats = [ ] for j in count ( ) : key = side + "_feat_" + str ( j ) if key not in fields : break feats . append ( key ) return feats