Dont know what to do next

dada_daa: <p>Hello,<br>
i am creating a encoder-decoder neural translation with a GRU unit. I am trying to code this for good 3 months but cant seem to make it better. The loss keeps being around 11 and accuracy is 17%, it was 75 before but after i put encoder.train() and encoder.eval() in the loops it dropped.</p>
<p><strong>encoder</strong>:<br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/e/9/e969908c44f15eca9ee06470a9c14c2d31a4ef73.png" data-download-href="https://discuss.pytorch.org/uploads/default/e969908c44f15eca9ee06470a9c14c2d31a4ef73" title="image"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/e/9/e969908c44f15eca9ee06470a9c14c2d31a4ef73_2_690x481.png" alt="image" data-base62-sha1="xiRq9rXw1W3hhquLAWX5vSWRbfJ" width="690" height="481" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/e/9/e969908c44f15eca9ee06470a9c14c2d31a4ef73_2_690x481.png, https://discuss.pytorch.org/uploads/default/optimized/3X/e/9/e969908c44f15eca9ee06470a9c14c2d31a4ef73_2_1035x721.png 1.5x, https://discuss.pytorch.org/uploads/default/original/3X/e/9/e969908c44f15eca9ee06470a9c14c2d31a4ef73.png 2x" data-dominant-color="2E313A"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">1150×803 93.5 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>
<p><strong>hyperparams</strong>:<br>
batch_size = 16<br>
embedding_dim = 256<br>
enc_units = 256<br>
epochs = 10<br>
lr_rate = 0.05</p>
<p>optimizer = torch.optim.Adam(list(decoder.parameters()) + list(encoder.parameters()), lr_rate)<br>
loss_fn = nn.CrossEntropyLoss()</p>
<p><strong>decoder:</strong><br>
<strong>class Decoder(nn.Module):</strong><br>
def <strong>init</strong>(self, embedding_dim, enc_units, vocab_size, vocab):<br>
super(Decoder, self).<strong>init</strong>()<br>
self.embedding_dim = embedding_dim  # TODO<br>
self.vocab_size = vocab_size<br>
self.enc_units = enc_units<br>
self.vocab = vocab<br>
self.embedding = nn.Embedding(self.vocab_size, self.embedding_dim)<br>
self.embedding.weight.data = torch.nn.functional.normalize(self.embedding.weight.data)<br>
self.gru = nn.GRU(self.embedding_dim, self.enc_units)  <span class="hashtag-raw">#gru</span> has dropout in itself<br>
<span class="hashtag-raw">#self</span>.dropout = nn.Dropout(0.2)<br>
self.linear_layer = nn.Linear(self.enc_units, len(self.vocab))<br>
self.linear_layer_inference = nn.Linear(self.enc_units, self.embedding_dim)<br>
self.softmax_layer = nn.Softmax(2)<br>
self.activation_layer = nn.ReLU()</p>
<pre><code>def forward(self, y, initial_state, loss_fn, previous_output, max_generated_length, training=True):
    prediction = torch.randn(2, 2)
    count = 0
    vocab_range = torch.arange(0, len(self.vocab), device='cuda')
    loss = 0
    correct = 0
    false = 0
    embedding_table = self.embedding(vocab_range)

    if training:
        for step_t in range(y.T.size()[0]):
            mask = torch.where(y.T[step_t] == 0, 0, 1)
            debug = y.T[step_t]
            output, initial_state = self.gru(torch.unsqueeze(embedding_table[y.T[step_t]], 0), initial_state)
            #output = self.dropout(output)
            linear_layer_output = self.linear_layer(output)
            linear_layer_output = self.activation_layer(linear_layer_output)
            mask = torch.unsqueeze(mask, 0).T
            prob = self.softmax_layer(linear_layer_output)
            prediction = torch.squeeze(torch.argmax(prob, dim=2))
            linear_layer_output = linear_layer_output * mask
            loss_step = loss_fn(torch.squeeze(linear_layer_output), y.T[step_t])
            loss += loss_step / y.T.size()[0]
            correct += (prediction == y.T[step_t]).type(torch.float).sum().item()
            false += (prediction != y.T[step_t]).type(torch.float).sum().item()

        return loss, correct, false

    else:
        end_number = self.vocab['&lt;end&gt;']
        for i in range(max_generated_length):
            previous_output = self.linear_layer_inference(previous_output)
            previous_output, initial_state = self.gru(previous_output, initial_state)
            logits = self.linear_layer(previous_output[-1:, :, :])
            logits = self.activation_layer(logits)
            prob = self.softmax_layer(logits)
            prediction = torch.squeeze(torch.argmax(prob, dim=2))
            count = count + (prediction == end_number).sum().item()
            if y.T.size()[0] &lt; i+1:
                false += 1
            else:
                correct += (prediction == y.T[i]).type(torch.float).sum().item()
                false += (prediction != y.T[i]).type(torch.float).sum().item()
            if count == prediction.size()[0]:
                return correct, false
        return correct, false
</code></pre>
<p>def sort_batch(X, y, lengths):<br>
lengths, indx = lengths.sort(dim=0, descending=True)<br>
X = X[indx]<br>
y = y[indx]<br>
return X.transpose(0, 1), y, lengths</p>
<p><strong>train loops:</strong><br>
def train_loop(dataloader, model, encoder, optimizer, loss_fn, device):<br>
size = len(dataloader.dataset)<br>
model.train()<br>
encoder.train()<br>
total_correct, total_false = 0, 0<br>
for batch, (X_, y, len_) in enumerate(iter(dataloader)):<br>
X, y, len__ = sort_batch(X_, y, len_)<br>
encoder_output, hidden_state = encoder.forward(X.to(device), len__, device)<br>
<span class="hashtag-raw">#print</span>(‘encoding done’)<br>
loss, correct, false = model.forward(y.to(device), hidden_state, loss_fn, encoder_output, max(len__))<br>
<span class="hashtag-raw">#print</span>(‘decoding done’)<br>
total_correct += correct<br>
total_false += false<br>
loss.backward()<br>
optimizer.step()<br>
optimizer.zero_grad()<br>
if batch % 100 == 0:<br>
loss, current = loss.item(), (batch + 1) * len(X_)<br>
print(f"loss: {loss:&gt;7f} [{current:&gt;5d}/{size:&gt;5d}]“)<br>
print(f"Train Error: \n”<br>
f"Accuracy: {total_correct/(total_false+total_correct)*100:&gt;0.3f}% "<br>
f"Correct: {total_correct} out of {total_correct+total_false}")</p>
<p>def test_loop(dataloader, model, encoder, loss_fn, device):<br>
model.eval()<br>
encoder.eval()<br>
size = len(dataloader.dataset)<br>
total_correct, total_false, total_loss = 0, 0, 0<br>
with torch.no_grad():<br>
for batch, (X_, y, len_) in enumerate(dataloader):<br>
X, y, len__ = sort_batch(X_, y, len_)<br>
encoder_output, hidden_state = encoder.forward(X.to(device), len__, device)<br>
correct, false = model.forward(y.to(device), hidden_state, loss_fn, encoder_output, max(len__)*3, training=False)<br>
<span class="hashtag-raw">#total_loss</span> += loss<br>
total_correct += correct<br>
total_false += false<br>
if batch % 100 == 0:<br>
current = (batch + 1) * len(X_)<br>
print(f"[{current:&gt;5d}/{size:&gt;5d}] "<br>
f"Correct so far: {(100 * total_correct/(total_false+total_correct)):&gt;0.1f} "<br>
f"Correct: {total_correct}, false: {total_false} ")<br>
print(f"Test Error: \n Accuracy: {(100 * total_correct/(total_false+total_correct)):&gt;0.1f}%, ")</p>