LSTM generating same word over and over

theo-oriol: <p>I am developping a LSTM to generate new sequence of word, but the outputs i get are the same everytime.<br>
It looks like this :<br>
i am "–said "come "edwin "–said "edwin "edwin "'tis "'tis "–said "come "–said "'tis "come "coming "'tis "edwin "'tis "come "coming "come "come "'tis "–said "coming "coming "edwin "coming "come "coming "edwin "–said "–said "edwin "come "'tis "–said "'tis "'tis "edwin "'tis "edwin "'tis "–said "coming "coming "come "–said "'tis "–said "–said "'tis "'tis "come "'tis "come "edwin "come "coming "'tis "coming "coming "coming "'tis "'tis "coming "edwin</p>
<p>I don’t know what i’ve done wrong i don’t find any clues online.<br>
Here’s my code<br>
model.py</p>
<pre><code class="lang-auto">import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np 

class LSTM(nn.Module) :
    def __init__(self,vocab_size, seq_size, embedding_dim, hidden_dim):
        super(LSTM, self).__init__()
        self.vocab = vocab_size
        self.hidden_size = hidden_dim
        self.seq_size = seq_size

        self.encoder = nn.Embedding(vocab_size, embedding_dim)

        self.lstm = nn.LSTM(embedding_dim, hidden_dim , batch_first=True )
        self.dropout_1 = nn.Dropout(p=0.2, inplace=False)
        self.lstm2 = nn.LSTM(embedding_dim, hidden_dim , batch_first=True )
        self.dropout_2 = nn.Dropout(p=0.2, inplace=False)
        self.decoder = nn.Linear(in_features=hidden_dim,out_features=vocab_size)
        self.activation = nn.Softmax(dim=0)
    def zero_state(self, batch_size):
        return (
                (torch.zeros(1, batch_size, self.hidden_size),
                torch.zeros(1, batch_size, self.hidden_size)), 

                (torch.zeros(1, batch_size, self.hidden_size),
                torch.zeros(1, batch_size, self.hidden_size)),                
                )

    def forward(self,sentence,prev_state_1,prev_state_2):
        #print("sentence shape ",np.shape(sentence))
        #print("prev shape ",np.shape(prev_state_1))
        x = self.encoder(sentence)
        x , hidden_cell_1 = self.lstm(x,prev_state_1)
        x = self.dropout_1(x)
        x , hidden_cell_2 = self.lstm2(x,prev_state_2)
        x = self.dropout_2(x)
        x = self.decoder(x)
        x = F.log_softmax(x, dim=1)
        return x , hidden_cell_1,hidden_cell_2

</code></pre>
<p>train.py</p>
<pre><code class="lang-auto">from model import LSTM
import numpy as np 
from dataset import *
import torch 
import torch.nn as nn
from argparse import Namespace

flags = Namespace(
    seq_size=100,
    batch_size=64,
    embedding_size=256,
    lstm_size=256,
    gradients_norm=5,
    checkpoint_path='checkpoint',
    num_epochs = 200
)

def train():
    int_to_vocab, vocab_to_int, n_vocab, in_text = get_data_from_file( flags.batch_size, flags.seq_size)
    x_batch,y_batch = create_batch(in_text,flags.batch_size,flags.seq_size)
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    model = LSTM(n_vocab, flags.seq_size,flags.embedding_size, flags.lstm_size).to(device)

    #optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.7)
    optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
    loss_function = nn.CrossEntropyLoss()

    for e in range(flags.num_epochs):
        print(f'epoch #{e}: ',end="")
        batches = get_batches(x_batch,y_batch,flags.batch_size, flags.seq_size)
        (state_h_1, state_c_1),(state_h_2, state_c_2) = model.zero_state(flags.batch_size)
        state_h_1 = state_h_1.to(device)
        state_c_1 = state_c_1.to(device)
        state_h_2 = state_h_2.to(device)
        state_c_2 = state_c_2.to(device)
        
        for i,(x, y) in enumerate(batches):
            model.train()
            optimizer.zero_grad()
 

            x = torch.tensor(x , dtype=torch.int64).to(device)
            #print("x shape {} ".format(np.shape(x)))
            
            tmp = []
            for index,el in enumerate(y) :
                tmp.append(np.zeros(n_vocab))
                tmp[index][y[index]] = 1
            #print(y)
            y = tmp 
            y = torch.tensor(y , dtype=torch.int64).to(device)
            logits, (state_h_1, state_c_1),(state_h_2, state_c_2) = model(x, (state_h_1, state_c_1),(state_h_2, state_c_2))
            #print("logits shape {} , y shape {}".format(np.shape(logits),np.shape(y)))
            loss = loss_function(logits, y)

            state_h_1 = state_h_1.detach()
            state_c_1 = state_c_1.detach()
            state_h_2 = state_h_2.detach()
            state_c_2 = state_c_2.detach()

            loss_value = loss.item()

            loss.backward()
            _ = torch.nn.utils.clip_grad_norm_(model.parameters(), flags.gradients_norm)
            optimizer.step()
        print(f'batch #{i}:\tloss={loss.item():.10f}')
    return model 

if __name__ == "__main__":
    model = train()
    torch.save(model,'save/model')

</code></pre>
<p>dataset.py</p>
<pre><code class="lang-auto">import numpy as np 
import random
import torch 
from collections import Counter
import string

def get_data_from_file(batch_size, seq_size) :
    filename = "corpus.txt"
    raw_text = open(filename, 'r', encoding='utf-8').read()
    text = raw_text.lower()
    #text = text.translate(str.maketrans('', '', string.punctuation))
    text = text.split()
    
    chars = sorted(list(set(text)))


    #word_counts = Counter(text)
    #sorted_vocab = dict((c, i) for i, c in enumerate(chars))
    #int_to_vocab = {k: w for k, w in enumerate(sorted_vocab)}
    #vocab_to_int = {w: k for k, w in int_to_vocab.items()}
    vocab_to_int = dict((c, i) for i, c in enumerate(chars))
    int_to_vocab = dict((i, c) for i, c in enumerate(chars))
    n_vocab = len(int_to_vocab)
    np.save("save/int_to_vocab",int_to_vocab)
    np.save("save/vocab_to_int",vocab_to_int)
    print('Vocabulary size', n_vocab)
    print("total charact : ", len(text))

    int_text = np.array([vocab_to_int[w] for w in text])
    #encod_text = one_hot_encod(int_text)
    return int_to_vocab, vocab_to_int, n_vocab,int_text


def create_batch(array,batch_size,seq_size):
    num_batches = int(len(array) / (seq_size * batch_size))
    array = array[:num_batches * batch_size * seq_size]
    #x_batch = np.reshape(array, (num_batches, -1))
    x_batch = np.array(np.split(array, int(len(array) / seq_size)      ))
    y_batch = x_batch[:,-1]
    #print(y_batch)
    x_batch = np.delete(x_batch, np.s_[::2], 1)
    print("x shape {} , y shape {} ".format(np.shape(x_batch),np.shape(y_batch)))
    return x_batch,y_batch
    

def get_batches(x,y ,batch_size, seq_size):
    num_batches = len(x)
    for i in range(0, num_batches,batch_size):
        yield x[i:i+batch_size], y[i:i+batch_size]
</code></pre>
<p>pred.py</p>
<pre><code class="lang-auto">import torch
import numpy as np 

model = torch.load("save/model")
vocab_to_int = np.load("save/vocab_to_int.npy",allow_pickle='TRUE').item()
int_to_vocab = np.load("save/int_to_vocab.npy",allow_pickle='TRUE').item()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.eval()

top_k = 5
words = ['i','am']

(state_h_1, state_c_1),(state_h_2, state_c_2) = model.zero_state(1)
state_h_1 = state_h_1.to(device)
state_c_1 = state_c_1.to(device)
state_h_2 = state_h_2.to(device)
state_c_2 = state_c_2.to(device)

for w in words:
        ix = torch.tensor([[vocab_to_int[w]]]).to(device)
        output, (state_h_1, state_c_1),(state_h_2, state_c_2) = model(ix, (state_h_1, state_c_1),(state_h_2, state_c_2))
    
_, top_ix = torch.topk(output[0], k=top_k)
choices = top_ix.tolist()
choice = np.random.choice(choices[0])

words.append(int_to_vocab[choice])

for _ in range(100):
    ix = torch.tensor([[choice]],dtype=torch.int64).to(device)
    output, (state_h_1, state_c_1),(state_h_2, state_c_2) = model(ix, (state_h_1, state_c_1),(state_h_2, state_c_2))

    _, top_ix = torch.topk(output[0], k=top_k)
    choices = top_ix.tolist()
    choice = np.random.choice(choices[0])
    words.append(int_to_vocab[choice])

print(' '.join(words))
</code></pre>