Loss is same in each batch and gradients are also same

username: culms
cooked: <p>Hi, I am new to deeplearning and pytorch ,<br>
My Seq2seq attention model is not training and my code is</p>
<p>class Encoder(nn.Module):<br>
def <strong>init</strong>(self,vocab_size,embedding_dim,enc_units,batch_sz):<br>
super(Encoder,self).<strong>init</strong>()<br>
self.batch_sz = batch_sz<br>
self.enc_units = enc_units<br>
self.embedding=nn.Embedding(vocab_size, embedding_dim)<br>
self.gru = nn.GRU(input_size =embedding_dim,hidden_size = enc_units,batch_first =True)</p>
<pre><code>def forward(self, x, hidden):
    x=x.type(torch.LongTensor)
    x = self.embedding(x)
    output, hidden = self.gru(x, hidden)
    return output, hidden

def initialize_hidden_state(self):
    return torch.zeros((1,self.batch_sz,self.enc_units))
</code></pre>
<p>class Attention(nn.Module):<br>
def <strong>init</strong>(self,units,max_length_inp):<br>
super(Attention,self).<strong>init</strong>()<br>
self.units = units<br>
self.max_length_inp=max_length_inp<br>
self.densor1 =nn.Linear(in_features=units*2,out_features=max_length_inp)<br>
self.act1 =nn.Tanh()<br>
self.densor2 =nn.Linear(max_length_inp,max_length_inp)<br>
self.act2=nn.Softmax(dim=-1)</p>
<pre><code>def forward(self,hidden,enc_output):
    hidden=hidden.permute(1,0,2)
    hidden = hidden.repeat(1,max_length_inp,1)
    concat =torch.cat((hidden, enc_output), -1)
    score =self.densor1(concat)
    score=self.act1(score)
    score =self.densor2(score)
    score=self.act2(score)

    mix =torch.matmul(score,enc_output)
    mix=mix.permute(0,2,1)
    mix =torch.sum(mix,dim=2)
    return mix
</code></pre>
<p>class Decoder(nn.Module):<br>
def <strong>init</strong>(self, vocab_size, embedding_dim, dec_units, batch_sz,attention):<br>
super(Decoder, self).<strong>init</strong>()<br>
self.batch_sz = batch_sz<br>
self.dec_units = dec_units<br>
self.embedding = nn.Embedding(vocab_size, embedding_dim)<br>
self.gru = nn.GRU(embedding_dim+dec_units, dec_units,batch_first =True)<br>
self.fc = nn.Linear(dec_units,vocab_size)<br>
self.attention =attention<br>
self.act2=nn.Softmax(dim=-1)</p>
<pre><code>def forward(self, x, hidden, enc_output):
     context_vector =self.attention(hidden,enc_output)
    x=x.type(torch.LongTensor)
    x = self.embedding(x)
    x=torch.cat([torch.unsqueeze(context_vector,1),x],axis=-1)    
    output, state = self.gru(x,hidden)
    output =torch.reshape(output,(-1,output.shape[2]))
    x = self.fc(output)
    x = self.act2(x)
    return x, state
    
def initialize_hidden_state(self):
    return torch.zeros((1,self.batch_sz,self.enc_units))
</code></pre>
<p>class Seq2Seq(nn.Module):<br>
def <strong>init</strong>(self, encoder, decoder,batch_size,max_length,trg_vocab_size):<br>
super().<strong>init</strong>()<br>
self.batch_size=batch_size<br>
self.max_length=max_length<br>
self.trg_vocab_size=trg_vocab_size<br>
self.encoder = encoder<br>
self.decoder = decoder</p>
<pre><code>def forward(self,src,targ,hidden):
    
    outputs = torch.zeros(self.max_length,self.batch_size,self.trg_vocab_size)
    enc_output, enc_hidden = self.encoder(src, hidden)
    a=torch.tensor(targ_lang.word2idx['&lt;start&gt;'])
    dec_hidden = enc_hidden
    dec_input  =a.expand(BATCH_SIZE,1)
    for t in range(1,targ.shape[1]):
        predictions, dec_hidden =self.decoder(dec_input, dec_hidden, enc_output)
        outputs[t]=predictions
        dec_input =torch.unsqueeze(targ[:,t],1)
   return outputs
</code></pre>
<p>model =Seq2Seq(encoder,decoder,BATCH_SIZE,max_length_targ,vocab_tar_size)<br>
optimizer =torch.optim.SGD(model.parameters(),lr=0.01, momentum=0.9)<br>
criterion = nn.CrossEntropyLoss()</p>
<p>def train(model, optimizer, criterion, clip):<br>
model.train()<br>
epoch_loss = 0<br>
for (batch, (inp, targ)) in enumerate(dataloader_train):<br>
hidden=encoder.initialize_hidden_state()<br>
src = inp<br>
trg = targ<br>
optimizer.zero_grad()<br>
output = model(src, trg,hidden)<br>
trg = trg.transpose(0,1)<br>
trg=trg.view(trg.shape[0],trg.shape[1],1)<br>
input=output.reshape(output.shape[0]*output.shape[1],output.shape[2])<br>
trg=trg.reshape(trg.shape[0]*trg.shape[1],trg.shape[2])<br>
trg = trg.type(torch.long)<br>
trg=trg.squeeze_()<br>
loss = criterion(input,trg)<br>
a = list(model.parameters())[0]<br>
loss.backward()<br>
torch.nn.utils.clip_grad_norm_(model.parameters(), clip)<br>
optimizer.step()<br>
b = list(model.parameters())[0]<br>
print(torch.equal(a.data, b.data))<br>
print(loss)<br>
epoch_loss += loss.item()<br>
return epoch_loss / len(dataloader_train)</p>
<p>CLIP =1</p>
<p>train_loss = train(model, optimizer, criterion, CLIP)</p>

username: vdw
cooked: <p>No offense, but don’t just post badly formatted and not documented code, expecting others to help.</p>
<p>Having said that, you first want to get a simplified version training properly. You don’t immediately need attention, and I’ve never seen just an complicated code for just the decoder.</p>
<p>Start with something working – there are good Seq2Seq PyTorch tutorials – and extend them as needed.</p>