RuntimeError: shape '[-1, 1280, 1]' is invalid for input of size 2776

username: asifdadra
cooked: <p>Hello Everyone,</p>
<p>I am implementing Stack BiLSTM upon the bert sequential output for text classification of dialogue state tracking dataset in encoder. Afterwards the output from Stacked BiLSTM and Bert overall features are concatenating to pass the decoder for mask attention. I am getting the error RuntimeError: shape ‘[-1, 1280, 1]’ is invalid for input of size 2776. The part of code is following where the error is occurring:<br>
class UtteranceAttention(nn.Module):<br>
def <strong>init</strong>(self, attn_head, model_output_dim, dropout=0.):<br>
super(UtteranceAttention, self).<strong>init</strong>()</p>
<pre><code>    self.attn_head = attn_head
    self.model_output_dim = model_output_dim
    self.dropout = dropout
    self.attn_fun = MultiHeadAttention(attn_head, model_output_dim, dropout=0.)

def forward(self, query, value, attention_mask=None):
    num_query = query.size(0)
    batch_size = value.size(0)
    seq_length = value.size(1)

    expanded_query = query.unsqueeze(0).expand(batch_size, *query.shape)
    if attention_mask is not None:
        expanded_attention_mask = attention_mask.view(-1, seq_length, 1).expand(value.size()).float()
        new_value = torch.mul(value, expanded_attention_mask)
        attn_mask = attention_mask.unsqueeze(1).expand(batch_size, num_query, seq_length)
    else:
        new_value = value
        attn_mask = None

    attended_embedding = self.attn_fun(expanded_query, new_value, new_value, mask=attn_mask)

    return attended_embedding
</code></pre>
<p>class Decoder(nn.Module):<br>
def <strong>init</strong>(self, attn_head, bert_output_dim, dropout_prob):<br>
super(Decoder, self).<strong>init</strong>()<br>
# slot utterance attention<br>
self.slot_utter_attn = UtteranceAttention(attn_head, bert_output_dim, dropout=0.)</p>
<pre><code>    # prediction
    self.pred = nn.Sequential(nn.Dropout(p=dropout_prob),
                              nn.Linear(bert_output_dim, bert_output_dim),
                              nn.LayerNorm(bert_output_dim))

def forward(self, sequence_output, attention_mask, slot_embedding):
    # slot utterance attention
    slot_utter_emb = self.slot_utter_attn(slot_embedding, sequence_output, attention_mask)

    # prediction
    hidden = self.pred(slot_utter_emb)

    return hidden  # [batch_size, num_slots, dim]
</code></pre>
<p>class BeliefTracker(nn.Module):<br>
def <strong>init</strong>(self, pretrained_model_type, lstm_hidden_dim, n_layers, lstm_drop, attn_head, dropout = 0.25, num_labels = None, dropout_prob=0.):<br>
super(BeliefTracker, self).<strong>init</strong>()<br>
self.encoder = UtteranceEncoding.from_pretrained(pretrained_model_type)<br>
self.hidden_size = self.encoder.config.hidden_size<br>
self.rnn = nn.LSTM(self.hidden_size, lstm_hidden_dim, num_layers=n_layers, bidirectional=True,<br>
batch_first=True, dropout=lstm_drop)<br>
<span class="hashtag">#self</span>.pool_dropout = nn.Dropout(self.encoder.config.hidden_dropout_prob)<br>
<span class="hashtag">#self</span>.classifier = nn.Linear(self.encoder.config.hidden_size + 2 * lstm_hidden_dim, num_labels)<br>
<span class="hashtag">#self</span>.dropout = nn.Dropout(dropout)<br>
self.decoder = Decoder(attn_head, self.hidden_size, dropout_prob)</p>
<pre><code>def forward(self, input_ids, attention_mask, token_type_ids, slot_emb):
    # encoder, a pretrained model, output is a tuple
    sequence_output, pooled_output = self.encoder(input_ids, attention_mask, token_type_ids)[0: 2]  # [batch_size, seq_length, dim]
    print(attention_mask.size())
    self.rnn.flatten_parameters()
    _, (hn, cn) = self.rnn(sequence_output)
    hidden = torch.cat((hn[-2, :, :], hn[-1, :, :]), dim=1)
    print(hidden.size())
    #pooled_output = self.pool_dropout(pooled_output)
    pooled_hidden = torch.cat((pooled_output, hidden), dim=1)
    print(pooled_hidden.size())
</code></pre>
<h1>
<a name="attention_mask-none-1" class="anchor" href="#attention_mask-none-1"></a>attention_mask = None</h1>
<p><span class="hashtag">#logits</span> = self.classifier(pooled_hidden)</p>
<pre><code>    # decoder, slot utterance attention, followed by a linear layer        
    slot_output = self.decoder(pooled_hidden, attention_mask, slot_emb)
    return slot_output
</code></pre>
<p>Here is the error</p>
<p>Epoch:   0% 0/1 [00:00&lt;?, ?it/s]<br>
0% 0/922 [00:00&lt;?, ?it/s]torch.Size([8, 347])<br>
torch.Size([8, 512])<br>
torch.Size([8, 1280])<br>
0% 0/922 [00:12&lt;?, ?it/s]<br>
Epoch:   0% 0/1 [00:12&lt;?, ?it/s]<br>
Traceback (most recent call last):<br>
File “/content/drive/My Drive/AUX-DST/train-aux.py”, line 264, in <br>
main(args)<br>
File “/content/drive/My Drive/AUX-DST/train-aux.py”, line 150, in main<br>
slot_emb=slot_lookup)  # [batch_size, num_slots, dim]<br>
File “/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py”, line 1130, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “/content/drive/My Drive/AUX-DST/models/DST.py”, line 155, in forward<br>
slot_output = self.decoder(pooled_hidden, attention_mask, slot_emb)<br>
File “/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py”, line 1130, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “/content/drive/My Drive/AUX-DST/models/DST.py”, line 120, in forward<br>
slot_utter_emb = self.slot_utter_attn(slot_embedding, sequence_output, attention_mask)<br>
File “/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py”, line 1130, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “/content/drive/My Drive/AUX-DST/models/DST.py”, line 95, in forward<br>
expanded_attention_mask = attention_mask.view(-1, 1280, 1).expand(value.size()).float()<br>
RuntimeError: shape ‘[-1, 1280, 1]’ is invalid for input of size 2776</p>