Private helper for running the specific RNN forward pass . Must be overriden by all subclasses . Args : tgt ( LongTensor ) : a sequence of input tokens tensors [ len x batch x nfeats ] . memory_bank ( FloatTensor ) : output ( tensor sequence ) from the encoder RNN of size ( src_len x batch x hidden_size ) . state ( FloatTensor ) : hidden state from the encoder RNN for initializing the decoder . memory_lengths ( LongTensor ) : the source memory_bank lengths . Returns : decoder_final ( Tensor ) : final hidden state from the decoder . decoder_outputs ( [ FloatTensor ] ) : an array of output of every time step from the decoder . attns ( dict of ( str [ FloatTensor ] ) : a dictionary of different type of attention Tensor array of every time step from the decoder .

def _run_forward_pass(self, tgt, memory_bank, state, memory_lengths=None, dec=False):
        """
        Private helper for running the specific RNN forward pass.
        Must be overriden by all subclasses.
        Args:
            tgt (LongTensor): a sequence of input tokens tensors
                                 [len x batch x nfeats].
            memory_bank (FloatTensor): output(tensor sequence) from the encoder
                        RNN of size (src_len x batch x hidden_size).
            state (FloatTensor): hidden state from the encoder RNN for
                                 initializing the decoder.
            memory_lengths (LongTensor): the source memory_bank lengths.
        Returns:
            decoder_final (Tensor): final hidden state from the decoder.
            decoder_outputs ([FloatTensor]): an array of output of every time
                                     step from the decoder.
            attns (dict of (str, [FloatTensor]): a dictionary of different
                            type of attention Tensor array of every time
                            step from the decoder.
        """
        assert not self._copy  # TODO, no support yet.
        assert not self._coverage  # TODO, no support yet.

        # Initialize local and return variables.
        attns = {}
        emb = self.embeddings(tgt)

        # Run the forward pass of the RNN.
        if isinstance(self.rnn, nn.GRU):
            rnn_output, decoder_final = self.rnn(emb, state.hidden[0])
        else:
            rnn_output, decoder_final = self.rnn(emb, state.hidden)

        # Check
        tgt_len, tgt_batch, _ = tgt.size()
        output_len, output_batch, _ = rnn_output.size()
        aeq(tgt_len, output_len)
        aeq(tgt_batch, output_batch)
        # END

        # Calculate the attention.
        decoder_outputs, p_attn = self.attn(
            rnn_output.transpose(0, 1).contiguous(),
            memory_bank.transpose(0, 1),
            memory_lengths=memory_lengths
        )
        attns["std"] = p_attn

        # Calculate the context gate.
        if self.context_gate is not None:
            decoder_outputs = self.context_gate(
                emb.view(-1, emb.size(2)),
                rnn_output.view(-1, rnn_output.size(2)),
                decoder_outputs.view(-1, decoder_outputs.size(2))
            )
            decoder_outputs = \
                decoder_outputs.view(tgt_len, tgt_batch, self.hidden_size)

        decoder_outputs = self.dropout(decoder_outputs)

        return decoder_final, decoder_outputs, attns


Private helper returning the number of expected features .

def _input_size(self):
        """
        Private helper returning the number of expected features.
        """
        return self.embeddings.embedding_size

def _input_size ( self ) : return self . embeddings . embedding_size


# sent_encoder : size ( sent_len = 9 batch = 2 dim = 512 ) # sent_decoder : size ( sent_len = 1 batch = 2 dim = 512 ) # src_sents : size ( batch = 2 sent_len = 9 ) function to calculate mmr : param sent_encoder : : param sent_decoder : : param src_sents : : return :

def _run_mmr(self,sent_encoder,sent_decoder,src_sents, input_step):
        '''
        # sent_encoder: size (sent_len=9,batch=2,dim=512)
        # sent_decoder: size (sent_len=1,batch=2,dim=512)
        # src_sents: size (batch=2,sent_len=9)
        function to calculate mmr
        :param sent_encoder:
        :param sent_decoder:
        :param src_sents:
        :return:
        '''
        pdist = nn.PairwiseDistance(p=2)
        sent_decoder=sent_decoder.permute(1,0,2) # (2,1,512)

        scores =[]
        # define sent matrix and current vector distance as the Euclidean distance
        for sent in sent_encoder: # iterate over each batch sample
            # distance: https://pytorch.org/docs/stable/_modules/torch/nn/modules/distance.html

            # import pdb;
            # pdb.set_trace()

            # sim1=torch.sum(pdist(sent_encoder.permute(1,0,2),sent.unsqueeze(1)),1).unsqueeze(1)  # -> this is sim2 on my equation, note this is distance!

            sim1 = 1 - torch.mean(pdist(sent_encoder.permute(1, 0, 2), sent.unsqueeze(1)), 1).unsqueeze(1) # this is a similarity function
            # sim1 shape: (batch_size,1)

            sim2=torch.bmm(self.mmr_W(sent_decoder),sent.unsqueeze(2)).squeeze(2) # (2,1) -> this is sim1 on my equation

            # scores.append(sim1-sim2)
            scores.append(sim2 - sim1)

        sent_ranking_att = torch.t(torch.cat(scores,1)) #(sent_len=9,batch_size)
        sent_ranking_att = torch.softmax(sent_ranking_att, dim=0).permute(1,0)  #(sent_len=9,batch_size)
        # scores is a list of score (sent_len=9, tensor shape (batch_size, 1))
        mmr_among_words = [] # should be (batch=2,input_step=200)
        for batch_id in range(sent_ranking_att.size()[0]):
            # iterate each batch, create zero weight on the input steps
            # mmr= torch.zeros([input_step], dtype=torch.float32).cuda()

            tmp = []
            for id,position in enumerate(src_sents[batch_id]):

                for x in range(position):
                    tmp.append(sent_ranking_att[batch_id][id])

            mmr = torch.stack(tmp) # make to 1-d

            if len(mmr) < input_step: # pad with 0
                tmp = torch.zeros(input_step - len(mmr)).float().cuda()
                # for x in range(input_step-len(mmr)):
                mmr = torch.cat((mmr, tmp), 0)
            else:
                mmr = mmr[:input_step]

            mmr_among_words.append(mmr.unsqueeze(0))

        mmr_among_words = torch.cat(mmr_among_words,0)

        # shape: (batch=2, input_step=200)

        return mmr_among_words


See StdRNNDecoder . _run_forward_pass () for description of arguments and return values . TODO : added a new param : sent_encoder from model . py this is the sentence matrix ; add attns [ mmr ] = [] .

def _run_forward_pass(self, tgt, memory_bank, state, memory_lengths=None,sent_encoder=None,src_sents=None,dec=None):
        """
        See StdRNNDecoder._run_forward_pass() for description
        of arguments and return values.
        TODO: added a new param: sent_encoder, from model.py, this is the sentence matrix; add attns["mmr"] = [].

        """

        # Additional args check.
        input_feed = state.input_feed.squeeze(0)
        #print("input feed size: {}\n".format(input_feed.size()))
        input_feed_batch, _ = input_feed.size()
        _, tgt_batch, _ = tgt.size()
        aeq(tgt_batch, input_feed_batch)
        # END Additional args check.

        # Initialize local and return variables.
        decoder_outputs = []
        attns = {"std": []}

        if self._copy:
            attns["copy"] = []
        if self._coverage:
            attns["coverage"] = []

        emb = self.embeddings(tgt)
        assert emb.dim() == 3  # len x batch x embedding_dim

        hidden = state.hidden
        coverage = state.coverage.squeeze(0) \
            if state.coverage is not None else None

        # Input feed concatenates hidden state with
        # input at every time step.

        #print("emb size: {}\n".format(emb.size()));exit()
        for _, emb_t in enumerate(emb.split(1)):
            # for each output time step in the loop

            emb_t = emb_t.squeeze(0)
            decoder_input = torch.cat([emb_t, input_feed], 1)

            # TODO: the following is where we get attention!
            rnn_output, hidden = self.rnn(decoder_input, hidden)
            decoder_output, p_attn = self.attn(
                rnn_output,
                memory_bank.transpose(0, 1),
                memory_lengths=memory_lengths)
            # p_attn: size (batch=2,input_step=200)

            if self.context_gate is not None:
                # TODO: context gate should be employed (not me)
                # instead of second RNN transform.
                decoder_output = self.context_gate(
                    decoder_input, rnn_output, decoder_output
                )
            decoder_output = self.dropout(decoder_output)
            input_feed = decoder_output

            decoder_outputs += [decoder_output]
            attns["std"] += [p_attn]

            # Update the coverage attention.
            if self._coverage:
                coverage = coverage + p_attn \
                    if coverage is not None else p_attn
                attns["coverage"] += [coverage]

            # Run the forward pass of the copy attention layer.
            #

            if self._copy and not self._reuse_copy_attn:

                _, copy_attn = self.copy_attn(decoder_output, memory_bank.transpose(0, 1))
                attns["copy"] += [copy_attn]
            elif self._copy:
                attns["copy"] = attns["std"] # attns["copy"] is a list of tensor for each output step=51, each size: [batch_size=2, input_step=200]

        if not dec: #if this is not dec?
            attns["mmr"] = []
            # 2333: TODO : the sentence representation for decoder
            sent_decoder = decoder_outputs[-1].unsqueeze(0)  # shape: (1, batch_size=2,dim=512)

            # Return result.
            # 2333: TODO: attns['std'] is a list of tensors, length is output_step, each tensor shape is (batch=2,input_step=200)

            # 2333: TODO: compute mmr attention here:

            mmr_among_words = self._run_mmr(sent_encoder, sent_decoder, src_sents,attns["std"][0].size()[-1])

            #  2333: TODO: bring mmr to attention...

            for output_step in attns["std"]:
                attention_weight = output_step
                # pairwise multiplication
                attention_weight = torch.mul(mmr_among_words,attention_weight)
                attns["mmr"].append(attention_weight.cuda())
            # pdb.set_trace()

            attns["std"] = attns["mmr"]

        # decoder_outputs is a list of tensors for each output step=51, each tensor: (batch_size=2,dim=512)
        return hidden, decoder_outputs, attns

#  2333: TODO: bring mmr to attention...


Using input feed by concatenating input with attention vectors .

def _input_size(self):
        """
        Using input feed by concatenating input with attention vectors.
        """
        return self.embeddings.embedding_size + self.hidden_size

def _input_size ( self ) : return self . embeddings . embedding_size + self . hidden_size


def detach(self):
        """ Need to document this """
        self.hidden = tuple([_.detach() for _ in self.hidden])
        self.input_feed = self.input_feed.detach()

def detach ( self ) : self . hidden = tuple ( [ _ . detach ( ) for _ in self . hidden ] ) self . input_feed = self . input_feed . detach ( )


def beam_update(self, idx, positions, beam_size):
        """ Need to document this """
        for e in self._all:
            sizes = e.size()
            br = sizes[1]
            if len(sizes) == 3:
                sent_states = e.view(sizes[0], beam_size, br // beam_size,
                                     sizes[2])[:, :, idx]
            else:
                sent_states = e.view(sizes[0], beam_size,
                                     br // beam_size,
                                     sizes[2],
                                     sizes[3])[:, :, idx]

            sent_states.data.copy_(
                sent_states.data.index_select(1, positions))

def beam_update ( self , idx , positions , beam_size ) : for e in self . _all : sizes = e . size ( ) br = sizes [ 1 ] if len ( sizes ) == 3 : sent_states = e . view ( sizes [ 0 ] , beam_size , br // beam_size , sizes [ 2 ] ) [ : , : , idx ] else : sent_states = e . view ( sizes [ 0 ] , beam_size , br // beam_size , sizes [ 2 ] , sizes [ 3 ] ) [ : , : , idx ] sent_states . data . copy_ ( sent_states . data . index_select ( 1 , positions ) )


Args : hidden_size ( int ) : the size of hidden layer of the decoder . rnnstate : final hidden state from the encoder . transformed to shape : layers x batch x ( directions * dim ) .

def __init__(self, hidden_size, rnnstate):
        """
        Args:
            hidden_size (int): the size of hidden layer of the decoder.
            rnnstate: final hidden state from the encoder.
                transformed to shape: layers x batch x (directions*dim).
        """
        if not isinstance(rnnstate, tuple):
            self.hidden = (rnnstate,)
        else:
            self.hidden = rnnstate
        self.coverage = None

        # Init the input feed.
        batch_size = self.hidden[0].size(1)
        h_size = (batch_size, hidden_size)
        self.input_feed = self.hidden[0].data.new(*h_size).zero_() \
                              .unsqueeze(0)

def
__init__
(
self
,
hidden_size
,
rnnstate
)
:
if
not
isinstance
(
rnnstate
,
tuple
)
:
self
.
hidden
=
(
rnnstate
,
)
else
:
self
.
hidden
=
rnnstate
self
.
coverage
=
# Init the input feed.
batch_size
=
self
.
hidden
[
0
]
.
size
(
1
)
h_size
=
(
batch_size
,
hidden_size
)
self
.
input_feed
=
self
.
hidden
[
0
]
.
data
.
new
(
*
h_size
)
.
zero_
(
)
.
unsqueeze
(
0
)


def update_state(self, rnnstate, input_feed, coverage):
        """ Update decoder state """
        if not isinstance(rnnstate, tuple):
            self.hidden = (rnnstate,)
        else:
            self.hidden = rnnstate
        self.input_feed = input_feed
        self.coverage = coverage

def update_state ( self , rnnstate , input_feed , coverage ) : if not isinstance ( rnnstate , tuple ) : self . hidden = ( rnnstate , ) else : self . hidden = rnnstate self . input_feed = input_feed self . coverage = coverage


Repeat beam_size times along batch dimension .

def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        vars = [e.data.repeat(1, beam_size, 1)
                for e in self._all]
        self.hidden = tuple(vars[:-1])
        self.input_feed = vars[-1]

def repeat_beam_size_times ( self , beam_size ) : vars = [ e . data . repeat ( 1 , beam_size , 1 ) for e in self . _all ] self . hidden = tuple ( vars [ : - 1 ] ) self . input_feed = vars [ - 1 ]


def load_test_model(opt, dummy_opt):
    """ Read in multiple models for ensemble """
    shared_fields = None
    shared_model_opt = None
    models = []
    for model_path in opt.models:
        fields, model, model_opt = \
            onmt.model_builder.load_test_model(opt,
                                               dummy_opt,
                                               model_path=model_path)
        import pdb;pdb.set_trace()
        if shared_fields is None:
            shared_fields = fields
        else:
            for key, field in fields.items():
                if field is not None and 'vocab' in field.__dict__:
                    assert field.vocab.stoi == shared_fields[key].vocab.stoi, \
                        'Ensemble models must use the same preprocessed data'
        models.append(model)
        if shared_model_opt is None:
            shared_model_opt = model_opt
    ensemble_model = EnsembleModel(models)
    return shared_fields, ensemble_model, shared_model_opt


def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        for model_state in self.model_decoder_states:
            model_state.repeat_beam_size_times(beam_size)

def repeat_beam_size_times ( self , beam_size ) : for model_state in self . model_decoder_states : model_state . repeat_beam_size_times ( beam_size )


Delegate squeeze to avoid modifying : obj : Translator . translate_batch ()

def squeeze(self, dim=None):
        """
        Delegate squeeze to avoid modifying
        :obj:`Translator.translate_batch()`
        """
        return EnsembleDecoderOutput([
            x.squeeze(dim) for x in self.model_outputs])

def squeeze ( self , dim = None ) : return EnsembleDecoderOutput ( [ x . squeeze ( dim ) for x in self . model_outputs ] )


def forward(self, tgt, memory_bank, state, memory_lengths=None,
                step=None):
        """ See :obj:`RNNDecoderBase.forward()` """
        # Memory_lengths is a single tensor shared between all models.
        # This assumption will not hold if Translator is modified
        # to calculate memory_lengths as something other than the length
        # of the input.
        outputs, states, attns = zip(*[
            model_decoder.forward(
                tgt, memory_bank[i], state[i], memory_lengths, step=step)
            for (i, model_decoder)
            in enumerate(self.model_decoders)])
        mean_attns = self.combine_attns(attns)
        return (EnsembleDecoderOutput(outputs),
                EnsembleDecoderState(states),
                mean_attns)


See : obj : RNNDecoderBase . init_decoder_state ()

def init_decoder_state(self, src, memory_bank, enc_hidden):
        """ See :obj:`RNNDecoderBase.init_decoder_state()` """
        return EnsembleDecoderState(
            [model_decoder.init_decoder_state(src,
                                              memory_bank[i],
                                              enc_hidden[i])
             for (i, model_decoder) in enumerate(self.model_decoders)])

def init_decoder_state ( self , src , memory_bank , enc_hidden ) : return EnsembleDecoderState ( [ model_decoder . init_decoder_state ( src , memory_bank [ i ] , enc_hidden [ i ] ) for ( i , model_decoder ) in enumerate ( self . model_decoders ) ] )


Compute a distribution over the target dictionary by averaging distributions from models in the ensemble . All models in the ensemble must share a target vocabulary .

def forward(self, hidden):
        """
        Compute a distribution over the target dictionary
        by averaging distributions from models in the ensemble.
        All models in the ensemble must share a target vocabulary.
        """
        distributions = [model_generator.forward(hidden[i])
                         for (i, model_generator)
                         in enumerate(self.model_generators)]
        return torch.stack(distributions).mean(0)

def forward ( self , hidden ) : distributions = [ model_generator . forward ( hidden [ i ] ) for ( i , model_generator ) in enumerate ( self . model_generators ) ] return torch . stack ( distributions ) . mean ( 0 )


Args : inputs ( FloatTensor ) : [ batch_size x 1 x model_dim ] memory_bank ( FloatTensor ) : [ batch_size x src_len x model_dim ] src_pad_mask ( LongTensor ) : [ batch_size x 1 x src_len ] tgt_pad_mask ( LongTensor ) : [ batch_size x 1 x 1 ]

def forward(self, inputs, memory_bank, src_pad_mask, tgt_pad_mask,
                previous_input=None, layer_cache=None, step=None):
        """
        Args:
            inputs (`FloatTensor`): `[batch_size x 1 x model_dim]`
            memory_bank (`FloatTensor`): `[batch_size x src_len x model_dim]`
            src_pad_mask (`LongTensor`): `[batch_size x 1 x src_len]`
            tgt_pad_mask (`LongTensor`): `[batch_size x 1 x 1]`

        Returns:
            (`FloatTensor`, `FloatTensor`, `FloatTensor`):

            * output `[batch_size x 1 x model_dim]`
            * attn `[batch_size x 1 x src_len]`
            * all_input `[batch_size x current_step x model_dim]`

        """
        dec_mask = torch.gt(tgt_pad_mask +
                            self.mask[:, :tgt_pad_mask.size(1),
                                      :tgt_pad_mask.size(1)], 0)
        input_norm = self.layer_norm_1(inputs)
        all_input = input_norm
        if previous_input is not None:
            all_input = torch.cat((previous_input, input_norm), dim=1)
            dec_mask = None

        if self.self_attn_type == "scaled-dot":
            query, attn = self.self_attn(all_input, all_input, input_norm,
                                         mask=dec_mask,
                                         layer_cache=layer_cache,
                                         type="self")
        elif self.self_attn_type == "average":
            query, attn = self.self_attn(input_norm, mask=dec_mask,
                                         layer_cache=layer_cache, step=step)

        query = self.drop(query) + inputs

        query_norm = self.layer_norm_2(query)
        mid, attn = self.context_attn(memory_bank, memory_bank, query_norm,
                                      mask=src_pad_mask,
                                      layer_cache=layer_cache,
                                      type="context")
        output = self.feed_forward(self.drop(mid) + query)

        return output, attn, all_input

def forward ( self , inputs , memory_bank , src_pad_mask , tgt_pad_mask , previous_input = None , layer_cache = None , step = None ) : dec_mask = torch . gt ( tgt_pad_mask + self . mask [ : , : tgt_pad_mask . size ( 1 ) , : tgt_pad_mask . size ( 1 ) ] , 0 ) input_norm = self . layer_norm_1 ( inputs ) all_input = input_norm if previous_input is not None : all_input = torch . cat ( ( previous_input , input_norm ) , dim = 1 ) dec_mask = None if self . self_attn_type == "scaled-dot" : query , attn = self . self_attn ( all_input , all_input , input_norm , mask = dec_mask , layer_cache = layer_cache , type = "self" ) elif self . self_attn_type == "average" : query , attn = self . self_attn ( input_norm , mask = dec_mask , layer_cache = layer_cache , step = step ) query = self . drop ( query ) + inputs query_norm = self . layer_norm_2 ( query ) mid , attn = self . context_attn ( memory_bank , memory_bank , query_norm , mask = src_pad_mask , layer_cache = layer_cache , type = "context" ) output = self . feed_forward ( self . drop ( mid ) + query ) return output , attn , all_input


Get an attention mask to avoid using the subsequent info .

def _get_attn_subsequent_mask(self, size):
        """
        Get an attention mask to avoid using the subsequent info.

        Args:
            size: int

        Returns:
            (`LongTensor`):

            * subsequent_mask `[1 x size x size]`
        """
        attn_shape = (1, size, size)
        subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')
        subsequent_mask = torch.from_numpy(subsequent_mask)
        return subsequent_mask

def _get_attn_subsequent_mask ( self , size ) : attn_shape = ( 1 , size , size ) subsequent_mask = np . triu ( np . ones ( attn_shape ) , k = 1 ) . astype ( 'uint8' ) subsequent_mask = torch . from_numpy ( subsequent_mask ) return subsequent_mask


See : obj : onmt . modules . RNNDecoderBase . forward ()

def forward(self, tgt, memory_bank, state, memory_lengths=None,
                step=None, cache=None):
        """
        See :obj:`onmt.modules.RNNDecoderBase.forward()`
        """
        src = state.src
        src_words = src[:, :, 0].transpose(0, 1)
        tgt_words = tgt[:, :, 0].transpose(0, 1)
        src_batch, src_len = src_words.size()
        tgt_batch, tgt_len = tgt_words.size()

        # Initialize return variables.
        outputs = []
        attns = {"std": []}
        if self._copy:
            attns["copy"] = []

        # Run the forward pass of the TransformerDecoder.
        emb = self.embeddings(tgt, step=step)
        assert emb.dim() == 3  # len x batch x embedding_dim

        output = emb.transpose(0, 1).contiguous()
        src_memory_bank = memory_bank.transpose(0, 1).contiguous()

        padding_idx = self.embeddings.word_padding_idx
        src_pad_mask = src_words.data.eq(padding_idx).unsqueeze(1) \
            .expand(src_batch, tgt_len, src_len)
        tgt_pad_mask = tgt_words.data.eq(padding_idx).unsqueeze(1) \
            .expand(tgt_batch, tgt_len, tgt_len)

        if state.cache is None:
            saved_inputs = []

        for i in range(self.num_layers):
            prev_layer_input = None
            if state.cache is None:
                if state.previous_input is not None:
                    prev_layer_input = state.previous_layer_inputs[i]
            output, attn, all_input \
                = self.transformer_layers[i](
                    output, src_memory_bank,
                    src_pad_mask, tgt_pad_mask,
                    previous_input=prev_layer_input,
                    layer_cache=state.cache["layer_{}".format(i)]
                    if state.cache is not None else None,
                    step=step)
            if state.cache is None:
                saved_inputs.append(all_input)

        if state.cache is None:
            saved_inputs = torch.stack(saved_inputs)

        output = self.layer_norm(output)

        # Process the result and update the attentions.
        outputs = output.transpose(0, 1).contiguous()
        attn = attn.transpose(0, 1).contiguous()

        attns["std"] = attn
        if self._copy:
            attns["copy"] = attn

        if state.cache is None:
            state = state.update_state(tgt, saved_inputs)

        return outputs, state, attns


def init_decoder_state(self, src, memory_bank, enc_hidden,
                           with_cache=False):
        """ Init decoder state """
        state = TransformerDecoderState(src)
        if with_cache:
            state._init_cache(memory_bank, self.num_layers,
                              self.self_attn_type)
        return state

def init_decoder_state ( self , src , memory_bank , enc_hidden , with_cache = False ) : state = TransformerDecoderState ( src ) if with_cache : state . _init_cache ( memory_bank , self . num_layers , self . self_attn_type ) return state


Args : src ( FloatTensor ) : a sequence of source words tensors with optional feature tensors of size ( len x batch ) .

def __init__(self, src):
        """
        Args:
            src (FloatTensor): a sequence of source words tensors
                    with optional feature tensors, of size (len x batch).
        """
        self.src = src
        self.previous_input = None
        self.previous_layer_inputs = None
        self.cache = None

def __init__ ( self , src ) : self . src = src self . previous_input = None self . previous_layer_inputs = None self . cache = None


Contains attributes that need to be updated in self . beam_update () .

def _all(self):
        """
        Contains attributes that need to be updated in self.beam_update().
        """
        if (self.previous_input is not None
                and self.previous_layer_inputs is not None):
            return (self.previous_input,
                    self.previous_layer_inputs,
                    self.src)
        else:
            return (self.src,)

def _all ( self ) : if ( self . previous_input is not None and self . previous_layer_inputs is not None ) : return ( self . previous_input , self . previous_layer_inputs , self . src ) else : return ( self . src , )


def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        self.src = self.src.data.repeat(1, beam_size, 1)

def repeat_beam_size_times ( self , beam_size ) : self . src = self . src . data . repeat ( 1 , beam_size , 1 )


def forward(self, tgt, memory_bank, state, memory_lengths=None, step=None):
        """ See :obj:`onmt.modules.RNNDecoderBase.forward()`"""
        # NOTE: memory_lengths is only here for compatibility reasons
        #       with onmt.modules.RNNDecoderBase.forward()
        # CHECKS
        assert isinstance(state, CNNDecoderState)
        _, tgt_batch, _ = tgt.size()
        _, contxt_batch, _ = memory_bank.size()
        aeq(tgt_batch, contxt_batch)
        # END CHECKS

        if state.previous_input is not None:
            tgt = torch.cat([state.previous_input, tgt], 0)

        # Initialize return variables.
        outputs = []
        attns = {"std": []}
        assert not self._copy, "Copy mechanism not yet tested in conv2conv"
        if self._copy:
            attns["copy"] = []

        emb = self.embeddings(tgt)
        assert emb.dim() == 3  # len x batch x embedding_dim

        tgt_emb = emb.transpose(0, 1).contiguous()
        # The output of CNNEncoder.
        src_memory_bank_t = memory_bank.transpose(0, 1).contiguous()
        # The combination of output of CNNEncoder and source embeddings.
        src_memory_bank_c = state.init_src.transpose(0, 1).contiguous()

        # Run the forward pass of the CNNDecoder.
        emb_reshape = tgt_emb.contiguous().view(
            tgt_emb.size(0) * tgt_emb.size(1), -1)
        linear_out = self.linear(emb_reshape)
        x = linear_out.view(tgt_emb.size(0), tgt_emb.size(1), -1)
        x = shape_transform(x)

        pad = torch.zeros(x.size(0), x.size(1),
                          self.cnn_kernel_width - 1, 1)

        pad = pad.type_as(x)
        base_target_emb = x

        for conv, attention in zip(self.conv_layers, self.attn_layers):
            new_target_input = torch.cat([pad, x], 2)
            out = conv(new_target_input)
            c, attn = attention(base_target_emb, out,
                                src_memory_bank_t, src_memory_bank_c)
            x = (x + (c + out) * SCALE_WEIGHT) * SCALE_WEIGHT
        output = x.squeeze(3).transpose(1, 2)

        # Process the result and update the attentions.
        outputs = output.transpose(0, 1).contiguous()
        if state.previous_input is not None:
            outputs = outputs[state.previous_input.size(0):]
            attn = attn[:, state.previous_input.size(0):].squeeze()
            attn = torch.stack([attn])
        attns["std"] = attn
        if self._copy:
            attns["copy"] = attn

        # Update the state.
        state.update_state(tgt)

        return outputs, state, attns


def init_decoder_state(self, _, memory_bank, enc_hidden, with_cache=False):
        """
        Init decoder state.
        """
        return CNNDecoderState(memory_bank, enc_hidden)

def init_decoder_state ( self , _ , memory_bank , enc_hidden , with_cache = False ) : return CNNDecoderState ( memory_bank , enc_hidden )


def _all(self):
        """
        Contains attributes that need to be updated in self.beam_update().
        """
        return (self.previous_input,)

def _all ( self ) : return ( self . previous_input , )


def update_state(self, new_input):
        """ Called for every decoder forward pass. """
        self.previous_input = new_input

def update_state ( self , new_input ) : self . previous_input = new_input


def repeat_beam_size_times(self, beam_size):
        """ Repeat beam_size times along batch dimension. """
        self.init_src = self.init_src.data.repeat(1, beam_size, 1)

def repeat_beam_size_times ( self , beam_size ) : self . init_src = self . init_src . data . repeat ( 1 , beam_size , 1 )


def _tokenize(self, text):

        """

        Tokenizes input using the fastest possible SpaCy configuration.
        This is optional, can be disabled in constructor.

        """

        return self._en(text, disable = ["tagger", "parser", "ner", "textcat"])

def _tokenize ( self , text ) : return self . _en ( text , disable = [ "tagger" , "parser" , "ner" , "textcat" ] )


def _normalize(self, tokens, case = False):

        """

        Lowercases and turns tokens into distinct words.

        """

        return [
            str(t).lower()
            if not case
            else str(t)
            for t in tokens
        ]

def _normalize ( self , tokens , case = False ) : return [ str ( t ) . lower ( ) if not case else str ( t ) for t in tokens ]


def overlaps(self):

        """

        Return a list of Fragments.Match objects between summary and text.
        This is a list of named tuples of the form (summary, text, length):

            - summary (int): the start index of the match in the summary
            - text (int): the start index of the match in the reference
            - length (int): the length of the extractive fragment

        """

        return self._matches

def overlaps ( self ) : return self . _matches


def strings(self, min_length = 0, raw = None, summary_base = True):

        """

        Return a list of explicit match strings between the summary and reference.
        Note that this will be in the same format as the strings are input. This is
        important to remember if tokenization is done manually. If tokenization is
        specified automatically on the raw strings, raw strings will automatically
        be returned rather than SpaCy tokenized sequences.

        Arguments:

            - min_length (int): filter out overlaps shorter than this (default = 0)
            - raw (bool): return raw input rather than stringified
                - (default = False if automatic tokenization, True otherwise)
            - summary_base (true): strings are based of summary text (default = True)

        Returns:

            - list of overlaps, where overlaps are strings or token sequences

        """

        # Compute the strings against the summary or the text?

        base = self.summary if summary_base else self.text

        # Generate strings, filtering out strings below the minimum length.

        strings = [
            base[i : i + length]
            for i, j, length
            in self.overlaps()
            if length > min_length
        ]

        # By default, we just return the tokenization being used.
        # But if they user wants a raw string, then we convert.
        # Mostly, this will be used along with spacy.

        if self._tokens and raw:

            for i, s in enumerate(strings):
                strings[i] = str(s)

        # Return the list of strings.

        return strings


def coverage(self, summary_base = True):

        """

        Return the COVERAGE score of the summary and text.

        Arguments:

            - summary_base (bool): use summary as numerator (default = True)

        Returns:

            - decimal COVERAGE score within [0, 1]

        """

        numerator = sum(o.length for o in self.overlaps())

        if summary_base: denominator = len(self.summary)
        else:            denominator = len(self.reference)

        if denominator == 0: return 0
        else:                return numerator / denominator

def coverage ( self , summary_base = True ) : numerator = sum ( o . length for o in self . overlaps ( ) ) if summary_base : denominator = len ( self . summary ) else : denominator = len ( self . reference ) if denominator == 0 : return 0 else : return numerator / denominator


def density(self, summary_base = True):

        """

        Return the DENSITY score of summary and text.

        Arguments:

            - summary_base (bool): use summary as numerator (default = True)

        Returns:

            - decimal DENSITY score within [0, ...]

        """

        numerator = sum(o.length ** 2 for o in self.overlaps())

        if summary_base: denominator = len(self.summary)
        else:            denominator = len(self.reference)

        if denominator == 0: return 0
        else:                return numerator / denominator

def density ( self , summary_base = True ) : numerator = sum ( o . length ** 2 for o in self . overlaps ( ) ) if summary_base : denominator = len ( self . summary ) else : denominator = len ( self . reference ) if denominator == 0 : return 0 else : return numerator / denominator


def compression(self, text_to_summary = True):

        """

        Return compression ratio between summary and text.

        Arguments:

            - text_to_summary (bool): compute text/summary ratio (default = True)

        Returns:

            - decimal compression score within [0, ...]

        """

        ratio = [len(self.text), len(self.summary)]

        try:

            if text_to_summary: return ratio[0] / ratio[1]
            else:               return ratio[1] / ratio[0]

        except ZeroDivisionError:

            return 0

def compression ( self , text_to_summary = True ) : ratio = [ len ( self . text ) , len ( self . summary ) ] try : if text_to_summary : return ratio [ 0 ] / ratio [ 1 ] else : return ratio [ 1 ] / ratio [ 0 ] except ZeroDivisionError : return 0


def _match(self, a, b):

        """

        Raw procedure for matching summary in text, described in paper.

        """

        self._matches = []

        a_start = b_start = 0

        while a_start < len(a):

            best_match = None
            best_match_length = 0

            while b_start < len(b):

                if a[a_start] == b[b_start]:

                    a_end = a_start
                    b_end = b_start

                    while a_end < len(a) and b_end < len(b) \
                            and b[b_end] == a[a_end]:

                        b_end += 1
                        a_end += 1

                    length = a_end - a_start

                    if length > best_match_length:

                        best_match = Fragments.Match(a_start, b_start, length)
                        best_match_length = length

                    b_start = b_end

                else:

                    b_start += 1

            b_start = 0

            if best_match:

                if best_match_length > 0:
                    self._matches.append(best_match)

                a_start += best_match_length

            else:

                a_start += 1

def _match ( self , a , b ) : self . _matches = [ ] a_start = b_start = 0 while a_start < len ( a ) : best_match = None best_match_length = 0 while b_start < len ( b ) : if a [ a_start ] == b [ b_start ] : a_end = a_start b_end = b_start while a_end < len ( a ) and b_end < len ( b ) and b [ b_end ] == a [ a_end ] : b_end += 1 a_end += 1 length = a_end - a_start if length > best_match_length : best_match = Fragments . Match ( a_start , b_start , length ) best_match_length = length b_start = b_end else : b_start += 1 b_start = 0 if best_match : if best_match_length > 0 : self . _matches . append ( best_match ) a_start += best_match_length else : a_start += 1


def _htmltokens(self, tokens):

        """

        Carefully process tokens to handle whitespace and HTML characters.

        """

        return [
            [
                _html.escape(t.text).replace("\n", "
"),
                _html.escape(t.whitespace_).replace("\n", "
")
            ]

            for t in tokens
        ]

def _htmltokens ( self , tokens ) : return [ [ _html . escape ( t . text ) . replace ( "\n" , "
" ) , _html . escape ( t . whitespace_ ) . replace ( "\n" , "
" ) ] for t in tokens ]


def annotate(self, min_length = 0, text_truncation = None, novel_italics = False):

        """

        Used to annotate fragments for website visualization.

        Arguments:

            - min_length (int): minimum length overlap to count (default = 0)
            - text_truncation (int): tuncated text length (default = None)
            - novel_italics (bool): italicize novel words (default = True)

        Returns:

            - a tuple of strings: (summary HTML, text HTML)

        """

        start = """
            <u
            style="color: {color}; border-color: {color};"
            data-ref="{ref}" title="Length: {length}"
            >
        """.strip()

        end = """
            </u>
        """.strip()

        # Here we tokenize carefully to preserve sane-looking whitespace.
        # (This part does require text to use a SpaCy tokenization.)

        summary = self._htmltokens(self.summary)
        text = self._htmltokens(self.text)

        # Compute novel word set, if requested.

        if novel_italics:

            novel = set(self._norm_summary) - set(self._norm_text)

            for word_whitespace in summary:

                if word_whitespace[0].lower() in novel:
                    word_whitespace[0] = "<em>" + word_whitespace[0] + "</em>"

        # Truncate text, if requested.
        # Must be careful later on with this.

        if text_truncation is not None:
            text = text[:text_truncation]

        # March through overlaps, replacing tokens with HTML-tagged strings.

        colors = self._itercolors()

        for overlap in self.overlaps():

            # Skip overlaps that are too short.

            if overlap.length < min_length:
                continue

            # Reference ID for JavaScript highlighting.
            # This is random, but shared between corresponding fragments.

            ref = _random.randint(0, 1e10)
            color = next(colors)

            # Summary starting tag.

            summary[overlap.summary][0] = start.format(
                color = color,
                ref = ref,
                length = overlap.length,
            ) + summary[overlap.summary][0]

            # Text starting tag.

            text[overlap.text][0] = start.format(
                color = color,
                ref = ref,
                length = overlap.length,
            ) + text[overlap.text][0]

            # Summary ending tag.

            summary[overlap.summary + overlap.length - 1][0] += end

            # Text ending tag.

            text[overlap.text + overlap.length - 1][0] += end

        # Carefully join tokens and whitespace to reconstruct the string.

        summary = " ".join("".join("".join(tw) for tw in summary).split())
        text = " ".join("".join("".join(tw) for tw in text).split())

        # Return the tuple.

        return summary, text