Why transformer model is behaving like this?

Ansh_Dadhich: <p>loss at epoch 10 is 5.1940178871154785</p>
<p>[1614, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136, 136]<br>
sort a a a a a a a a a a a a a a<br>
loss at epoch 11 is 5.237536907196045</p>
<p>[2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160]<br>
i i i i i i i i i i i i i i i i i<br>
loss at epoch 12 is 5.147286415100098</p>
<p>[913, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 199, 393, 393, 393]<br>
are to to to to to to to to to to to you to to to<br>
loss at epoch 13 is 4.996100902557373</p>
<p>[1033, 740, 393, 740, 740, 740, 740, 740, 740, 740, 393, 740, 740, 136, 740, 740]<br>
now the to the the the the the the the to the the a the the<br>
loss at epoch 14 is 5.0865559577941895</p>
<p>[2212, 2212, 2212, 2212, 2212, 2212, 2212, 2212, 2212, 2212, 2212, 2212, 2212, 2212, 2212, 2212]<br>
maybe maybe maybe maybe maybe maybe maybe maybe maybe maybe maybe maybe maybe maybe maybe maybe<br>
loss at epoch 15 is 4.944348335266113</p>
<p>[1858, 393, 199, 393, 199, 393, 199, 393, 199, 199, 393, 199, 199, 199, 199]<br>
he to you to you to you to you you to you you you you<br>
loss at epoch 16 is 4.972095012664795</p>
<p>[1858, 1858, 1858, 1858, 1858, 1858, 1858, 1858, 1858, 1858, 1858, 1858, 1858, 1858, 1858, 1858, 1858]<br>
he he he he he he he he he he he he he he he he he<br>
loss at epoch 17 is 4.938692569732666</p>
<p>[2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 136]<br>
okay okay okay okay okay okay okay okay okay okay okay okay okay okay okay a<br>
loss at epoch 18 is 4.899395942687988</p>
<p>model is</p>
<p>import torch<br>
import torch.nn as nn<br>
from torch.utils.data import Dataset,DataLoader<br>
import re<br>
import pandas<br>
import math<br>
import random<br>
import matplotlib.pyplot as mat</p>
<p>device = torch.device(“cuda” if torch.cuda.is_available() else “cpu”)<br>
print(device)</p>
<p>dataset = pandas.read_csv(r"Conversation.csv", usecols = [“question”,“answer”])</p>
<p>question = dataset[“question”].values<br>
answer = dataset[“answer”].values</p>
<p>def remove_punctuation(sentence):<br>
sentence = re.sub(r"[^\w\s]", “”, str(sentence)).lower()<br>
sentence = sentence.split()<br>
return sentence</p>
<p>question = list(map(remove_punctuation, question))<br>
answer = list(map(remove_punctuation, answer))</p>
<p>vocab = [“”, “”, “”, “”] + list(set([word for sentence in question for word in sentence] + [word for sentence in answer for word in sentence]))</p>
<p>word_to_index = {word : index for index,word in enumerate(vocab)}<br>
index_to_word = {index : word for word,index in word_to_index.items()}</p>
<p>pad_index = word_to_index.get(“”,0)<br>
unknown_index = word_to_index.get(“”,3)<br>
end_index = word_to_index.get(“”,2)</p>
<p>class custom_dataset(Dataset):<br>
def <strong>init</strong>(self,question,answer):<br>
self.question = question<br>
self.answer = answer</p>
<pre><code>def __len__(self):
    return len(self.question)

def __getitem__(self,index):
    question =  [word_to_index.get(word) for word in self.question[index]] 
    
    answer = [word_to_index.get(word) for word in self.answer[index]] 
    
    return torch.tensor(question, dtype = torch.long).to(device),torch.tensor(answer, dtype = torch.long).to(device)
</code></pre>
<p>def add_padding(batch):<br>
questions,answers = zip(*batch)</p>
<pre><code>max_question_length = max(len(question) for question in questions) 
max_answer_length = max(len(answer) for answer in answers)

max_len = max(max_question_length,max_answer_length) 

padded_questions = []
padded_answers = []

for question in questions:
    extend_till = max_len - len(question)
    padded_question = torch.cat([question, torch.tensor([pad_index] * extend_till).to(device)])
    padded_questions.append(padded_question)
    
for answer in answers:
    extend_till = max_len - len(answer)
    padded_answer = torch.cat([answer, torch.tensor([pad_index] * extend_till).to(device)])
    padded_answers.append(padded_answer)
    
questions = torch.stack(padded_questions).long().to(device)
answers = torch.stack(padded_answers).long().to(device)

return questions,answers
</code></pre>
<p>def create_key_padding_mask(tensor,pad_index):<br>
mask = (tensor == pad_index).to(device)<br>
return mask</p>
<p>def look_ahead_mask(sentence_length):<br>
mask = torch.triu(torch.ones(sentence_length,sentence_length) * float(“-inf”), diagonal = 1).bool().to(device)<br>
return mask</p>
<p>max_len = max(dataset[“question”].apply(len).max(), dataset[“answer”].apply(len).max())</p>
<p>class positional_encoding(nn.Module):<br>
def <strong>init</strong>(self,embedding_dimension,max_len):<br>
super().<strong>init</strong>()</p>
<pre><code>    position = torch.arange(0,max_len).unsqueeze(1).to(device)
    
    self.positional_encodings = torch.empty(max_len,embedding_dimension).to(device)
    
    div_term = torch.exp(torch.arange(0,embedding_dimension,2) * (-math.log(10000)/embedding_dimension)).to(device)
    div_term = div_term.unsqueeze(0)
    
    self.positional_encodings[:,0::2] = torch.sin(position * div_term).to(device)
    self.positional_encodings[:,1::2] = torch.cos(position * div_term).to(device)
    
    self.register_buffer("positional_encoding", self.positional_encodings)
    
def forward(self,x):
    x = x + self.positional_encodings[:x.size(1),:].to(device)
    return x
</code></pre>
<p>class transformer(nn.Module):<br>
def <strong>init</strong>(self,embedding_dimension,number_of_heads,num_encoder_layers, num_decoder_layers,ff_dimesnion,vocab):<br>
super().<strong>init</strong>()</p>
<pre><code>    self.embedding_dimension = embedding_dimension
    
    self.embedding = nn.Embedding(len(vocab),embedding_dimension)
    self.positional_encodings = positional_encoding(self.embedding_dimension,max_len)
    
    self.transformer = nn.Transformer(d_model = embedding_dimension, nhead = number_of_heads,num_encoder_layers = num_encoder_layers, num_decoder_layers = num_decoder_layers, dim_feedforward = ff_dimesnion, batch_first = True)
    
    self.output = nn.Linear(self.embedding_dimension,len(vocab))
            
def forward(self,src,target,src_key_padding_mask = None, tgt_key_padding_mask = None, tgt_mask = None, memory_mask = None):
    
    src = self.embedding(src) * math.sqrt(self.embedding_dimension)
    src = self.positional_encodings(src)
    
    target = self.embedding(target) * math.sqrt(self.embedding_dimension)
    target = self.positional_encodings(target)
    
    output = self.transformer(src,target, src_key_padding_mask = src_key_padding_mask, tgt_key_padding_mask = tgt_key_padding_mask, tgt_mask = tgt_mask, memory_mask = memory_mask)
    
    output = self.output(output)

    return output
    
def encoder(self,src,src_key_padding_mask = None):
    src = self.embedding(src) * math.sqrt(self.embedding_dimension)
    src = self.positional_encodings(src)
    
    encoder_output = self.transformer.encoder(src,src_key_padding_mask = src_key_padding_mask)
    return encoder_output

def decoder(self,encoder_output,target,memory_mask = None,target_key_padding_mask = None,target_mask = None):
    target = self.embedding(target) * math.sqrt(self.embedding_dimension)
    target = self.positional_encodings(target)
    
    decoder_output = self.transformer.decoder(target,encoder_output, memory_mask = memory_mask, tgt_key_padding_mask = target_key_padding_mask, tgt_mask = target_mask)
    
    output = self.output(decoder_output)
    
    return output
</code></pre>
<p>datasett = custom_dataset(question,answer)</p>
<p>training_data = DataLoader(datasett, batch_size = 64, shuffle = True, collate_fn = add_padding)</p>
<p>model = transformer(128,8,3,3,512,vocab).to(device)</p>
<p>optimizer = torch.optim.Adam(model.parameters(), lr = 0.001)</p>
<p>loss = nn.CrossEntropyLoss(ignore_index = pad_index)</p>
<p>losses = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<p>for epoch in range(200):<br>
for batch,(question,answer) in enumerate(training_data):</p>
<pre><code>    optimizer.zero_grad()
    
    inputt  = question.to(device)
    target = answer.to(device)
    
    src_key_padding_mask = create_key_padding_mask(inputt,pad_index).to(device)
    target_key_padding_mask = create_key_padding_mask(target,pad_index).to(device)
    
    no_peek_mask = look_ahead_mask(target.size(1)).to(device)

    teacher_forcing_ratio = max(0.2, 1 - (epoch * 0.1))
    
    if random.random() + 1 &lt; teacher_forcing_ratio:
        
        predicted = model.forward(inputt,target, src_key_padding_mask = src_key_padding_mask, tgt_key_padding_mask = target_key_padding_mask,tgt_mask = no_peek_mask)

        predictedd = torch.argmax(predicted, dim = -1)

        if (batch == 25):
            for sentence in predictedd:
                sentence = sentence.cpu().tolist()
                print(sentence)
                for word in sentence:
                    print(index_to_word.get(word), end = " ")
                break
            print()
            break

    else:
        
        decoder_input = target[:,0].unsqueeze(1)
        
        encoder_output = model.encoder(inputt,src_key_padding_mask = src_key_padding_mask)
        
        predicted = []
        
        for predict in range(target.size(1)):
            decoder_output = model.decoder(encoder_output,decoder_input,target_key_padding_mask = create_key_padding_mask(decoder_input,pad_index).to(device),target_mask = look_ahead_mask(decoder_input.size(1))).to(device)
            
            predicted_word = torch.argmax(decoder_output[:,-1:],dim = -1)
            predicted.append(decoder_output[:,-1:])
            decoder_input = torch.cat([decoder_input,predicted_word],dim = 1).to(device)
            
        predicted = torch.stack(predicted,dim = 1).to(device)

        if (batch == 25):
            for sentence in decoder_input:
                sentence = sentence.cpu().tolist()
                print(sentence)
                for word in sentence:
                    print(index_to_word.get(word),end = " ")
                break
            print()
            break
        
    predicted = predicted.view(-1,predicted.size(-1))
    target = target.view(-1)
        
                        
    losss = loss(predicted,target)
    losss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(),max_norm = 1.0)
    optimizer.step()
    
if(epoch % 4 == 0):
   losses.append(losss.detach().item())
    
print(f"loss at epoch {epoch} is {losss.detach().item()}")
print()
</code></pre>
<p>mat.plot(range(len(losses)),losses)<br>
mat.xlabel(“epochs”)<br>
mat.ylabel(“loss”)<br>
mat.savefig(“loss.png”)</p>
<p>torch.save({“model_state_dict”: model.state_dict(),<br>
“word_to_index” : word_to_index,<br>
“index_to_word” : index_to_word},“transformer.pt”)</p>
<p>with torch.no_grad():<br>
model.eval()<br>
loaded = torch.load(“transformer.pt”)</p>
<pre><code>model.load_state_dict(loaded["model_state_dict"])
word_to_index = loaded["word_to_index"]
index_to_word = loaded["index_to_word"]

question = "hello, may i speak to alice please?"
answer = "&lt;START&gt;"

question = remove_punctuation(question)
answer = remove_punctuation(answer)

inputt = torch.tensor([word_to_index.get(word,unknown_index) for word in question]).unsqueeze(0).to(device)
target = torch.tensor([word_to_index.get(word,unknown_index) for word in answer]).unsqueeze(0).to(device)
    
src_key_padding_mask = create_key_padding_mask(inputt,pad_index).to(device)
target_key_padding_mask = create_key_padding_mask(target,pad_index).to(device)

encoder_output = model.encoder(inputt,src_key_padding_mask)

for predict in range(10):
    target_key_padding_mask = create_key_padding_mask(target,pad_index).to(device)
    no_peek_mask = look_ahead_mask(target.size(1)).to(device)
    
    output = model.decoder(inputt,target,target_key_padding_mask = target_key_padding_mask,target_mask = no_peek_mask)
    
    predicted = output[:,-1].to(device)
    predicted_word = torch.argmax(predicted,dim = 1)
    
    target = torch.cat([target,predicted_word], dim = 1).to(device)
    
    print(predicted_word,end = " ")
    
    if(predicted_word == end_index):
        break
</code></pre>
<p>on using  and  token it only predicts  and  token</p>

vdw: <p>I have a <a href="https://github.com/chrisvdweth/nus-cs4248x/blob/master/3-neural-nlp/Section%204.2%20-%20Transformer%20Machine%20Translation.ipynb" rel="noopener nofollow ugc">working example</a> for a machine translation use case. Maybe this helps.</p>