Model not Learning anything

username: namdev
cooked: <p>Before posting a query, check the <a href="https://discuss.pytorch.org/c/faq/30">FAQs</a> - it might already be answered!<br>
Hi I am trying to create a Encoder -Decoder Model with No attention involved and I have trained the data on 15 epochs the Loss doesn’t converge right from the 1st epoch to the last epoch What mistake I might be making For Ref. I am attaching my code</p>
<p>class EncoderLSTM(nn.Module):<br>
def <strong>init</strong>(self,input_size,hidden_size,dropout_p=0.5):<br>
super(EncoderLSTM,self).<strong>init</strong>()<br>
self.hidden_size=hidden_size<br>
self.embedding=nn.Embedding(input_size,hidden_size,device=device)<br>
self.LSTM=nn.LSTM(hidden_size,hidden_size,num_layers=2,batch_first=True)<br>
self.dropout=nn.Dropout(dropout_p)<br>
def forward(self,input):<br>
embedding=self.dropout(self.embedding(input))<br>
<span class="hashtag-raw">#print</span>(embedding.shape)<br>
out,(hide,cs)=self.LSTM(embedding)<br>
return out,(hide,cs)<br>
def predict(self,input):<br>
z1=vocab.lookup_indices(input.split(’ ‘))<br>
z1.insert(0,vocab[’‘])<br>
z1.append(vocab[’'])<br>
out,(hide,cs)=EncoderLSTM.forward(self,input=torch.tensor(z1).view(1,len(z1)).to(device))<br>
return out,(hide,cs)</p>
<p>class DecoderLSTM(nn.Module):<br>
def <strong>init</strong>(self,hidden_size,output_size,encoder):<br>
super(DecoderLSTM,self).<strong>init</strong>()<br>
self.Embedding=nn.Embedding(output_size,hidden_size)<br>
self.LSTM=nn.LSTM(hidden_size,hidden_size,num_layers=2,batch_first=True)<br>
self.out1=nn.Linear(hidden_size,output_size<em>3)<br>
self.out2=nn.Linear(output_size</em>3,output_size)<br>
self.grelu=nn.GELU()<br>
self.relu=nn.ReLU()<br>
self.encoder=encoder<br>
def forward(self,encoder_output,encoder_hidden,target_tensor=None):<br>
batch_size=encoder_output.size(0)<br>
decoder_input=torch.empty(batch_size,1,dtype=torch.long,device=device).fill_(sos_tensor)<br>
decoder_hidden=encoder_hidden<br>
decoder_outputs=<span class="chcklst-box fa fa-square-o fa-fw"></span><br>
for i in range(8):<br>
decoder_output,decoder_hidden=self.forward_step(decoder_input,decoder_hidden)<br>
decoder_outputs.append(decoder_output)<br>
if target_tensor is not None:<br>
<span class="hashtag-raw">#print</span>(target_tensor[:,i].shape)<br>
decoder_input=target_tensor[:,i].unsqueeze(1)<br>
<span class="hashtag-raw">#print</span>(decoder_input.shape)<br>
else:<br>
<em>,topi=decoder_output.topk(1)<br>
decoder_input=topi.squeeze(-1).detach()<br>
decoder_outputs=torch.cat(decoder_outputs,dim=1)<br>
return decoder_outputs,decoder_hidden,None<br>
def forward_step(self,input,hidden):<br>
output=self.Embedding(input)<br>
output,(hidden_new,cell_state_new)=self.LSTM(output,hidden)<br>
output=self.grelu(self.out1(output))<br>
output=self.relu(self.out2(output))<br>
return output,(self.grelu(hidden_new),self.grelu(cell_state_new))<br>
def predict(self,encoderpredict):<br>
encoder_output,encoder_hidden=encoderpredict<br>
d_o,d_h,</em>=DecoderLSTM.forward(self,encoder_output,encoder_hidden,None)<br>
z=torch.argmax(d_o,dim=2)<br>
z=z.view(-1,z.shape[1])<br>
z=z.tolist()<br>
z1=vocab.lookup_tokens(z[0])<br>
return z1</p>
<p>encoder=EncoderLSTM(256,128)<br>
encoder_optimizer=torch.optim.Adam(encoder.parameters(),1e-3,weight_decay=0.09)<br>
encoder.to(device)<br>
decoder=DecoderLSTM(128,len(vocab),encoder)<br>
loss_fn_decoder=torch.nn.CrossEntropyLoss()<br>
decoder_optimizer=torch.optim.Adam(decoder.parameters(),learning_rate,weight_decay=0.09)<br>
decoder.to(device)</p>
<p>for i in range(epochs):<br>
encoder.train()<br>
decoder.train()<br>
train_loss=0<br>
for batch in tqdm(train_dataloader,desc=‘Training’):<br>
data,labels=batch<br>
decoder_optimizer.zero_grad()<br>
encoder_optimizer.zero_grad()                <a class="hashtag-cooked" href="/c/data/37" data-type="category" data-slug="data" data-id="37"><span class="hashtag-icon-placeholder"><svg class="fa d-icon d-icon-square-full svg-icon svg-node"><use href="#square-full"></use></svg></span><span>data</span></a>.to(device)<br>
output_encoder,hidden_encoder=encoder(data.to(device))<br>
output_decoder,hidden,none=decoder(encoder_output=output_encoder.to(device),encoder_hidden=hidden_encoder,target_tensor=labels.to(device))<br>
loss=loss_fn_decoder(output_decoder.view(-1,output_decoder.shape[-1]).to(device),labels.view(-1).to(device))<br>
loss.backward()<br>
encoder_optimizer.step()<br>
decoder_optimizer.step()<br>
train_loss+=loss<br>
print(train_loss)<br>
!</p>
<p>the next Image is for the Loss<br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/4/7/47dd3c5ddc2257aa8fc0ee60c1430b3e675a6b76.png" data-download-href="https://discuss.pytorch.org/uploads/default/47dd3c5ddc2257aa8fc0ee60c1430b3e675a6b76" title="image"><img src="https://discuss.pytorch.org/uploads/default/original/3X/4/7/47dd3c5ddc2257aa8fc0ee60c1430b3e675a6b76.png" alt="image" data-base62-sha1="afJTxXRclTK0pMclGL3c549KwFE" width="452" height="500" data-dominant-color="F4F4F4"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">605×668 26.3 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>

username: vdw
cooked: <p>You may want to format your post so that your code is easier to read. Try rapping your code in ```…```.</p>
<p>Since you’re using an LSTM encoder and decoder, why are you passing only the hidden state but no the cell state from the encoder to the decoder? In fact, in your <code>forward_step</code> method of the decoder, you call <code>self.LSTM</code> with only the hidden state, so the cell state will always be randomly initialized.</p>
<p>Here’s a working implementation of a basic <a href="https://github.com/chrisvdweth/nus-cs4246x/blob/master/3-neural-nlp/src/rnn.py" rel="noopener nofollow ugc">RNN-based Seq2Seq model</a>, incl. a <a href="https://github.com/chrisvdweth/nus-cs4246x/blob/master/3-neural-nlp/Section%203.2%20-%20RNN%20Machine%20Translation.ipynb" rel="noopener nofollow ugc">notebook</a>. Note that the method <code>_create_final_hidden()</code> which combines the hidden and cell state in case the encoder and decoder are LSTMs.</p>