Update PrateritumGPT.py

Changed some values and added <BOS> token

PrateritumGPT.py

@@ -5,8 +5,11 @@ from torch.utils.data import Dataset, DataLoader
 from torch.nn.utils.rnn import pad_sequence
 import math
 import progressbar
+import os
 
-device="cpu"
+Path=os.path.dirname(os.path.abspath(__file__))+"\\"
+
+device="cuda"
 
 def CreateBar():
     global bar
@@ -21,7 +24,7 @@ for i in range(len(tokens)):
     tokensdict.update({tokens[i]: [0] * i + [0] * (len(tokens) - (i + 1))})
 
 # Ouvrir le fichier CSV
-with open("C:\\Users\\marc2\\Downloads\\7eaaf0e22461b505c749e268c0b72bc4-12ebe211a929f039791dfeaa1a019b64cadddaf1\\7eaaf0e22461b505c749e268c0b72bc4-12ebe211a929f039791dfeaa1a019b64cadddaf1\\top-german-verbs.csv", 'r', encoding="utf-8") as file:
+with open(Path+"top-german-verbs.csv", 'r', encoding="utf-8") as file:
     # Créer un objet lecteur CSV
     reader = [i for i in csv.reader(file)][1:]
 
@@ -37,7 +40,6 @@ class CSVDataset(Dataset):
         sample = self.features[idx], self.labels[idx]
         return sample
 
-# Supposons que vous ayez vos données sous forme de listes
 features = []
 labels = []
 padding=len(tokens)
@@ -48,7 +50,7 @@ for i in reader:
         k += [tokens.index(j)]
     #k += [-1] * (25 - len(k))
     features += [torch.Tensor(k)]
-    k = []
+    k = [len(tokens)+1]
     for j in i[8]:
         k += [tokens.index(j)]
     #k += [-1] * (25 - len(k))
@@ -109,12 +111,41 @@ def collate_fn(batch):
 
 train_loader = DataLoader(MyDataset, batch_size=32, shuffle=True, collate_fn=collate_fn)
 
-model = TransformerModel(vocab_size=len(tokens)+1, emb_dim=16, nhead=4, num_encoder_layers=2, num_decoder_layers=2, dim_feedforward=256)
+#Embedding Dimension on epoch 10
+#32:10.49
+#64:6.55
+#128:6.44
+#256:9.63
+
+#Head Number on epoch 15
+#32:6.44
+#64:5.17
+#16:5.9402
+
+#Feed Forward Dimension on epoch 15+ (minimum)
+#128:5.17
+#256:3.49
+#512:3.44
+#1024:3.23
+
+#Num Encoder Layers on epochs 25 (minimum)
+#1:3.15
+#2:4.01
+
+#Num Decoder Layers on epochs 25 (minimum)
+#1:3.15
+#2:2.14
+#3:1.75
+#4:1.60
+
+#New model:
+#Dropout: 0
+#Forward Dim: 1024
+
+model = TransformerModel(vocab_size=len(tokens)+2, emb_dim=128, nhead=32, num_encoder_layers=1, num_decoder_layers=1, dim_feedforward=1024,dropout=0)
 loss_fn = nn.CrossEntropyLoss()
 optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
 
-epochs = 100
-
 try:
     model.load_state_dict(torch.load("data/PrateritumGPT.pth"))
     print("Sucessfully loaded model.")
@@ -122,35 +153,42 @@ except:
     pass
 
 #print(model(torch.zeros((1,25)).to(device),torch.zeros((1,25)).to(device)))
-inp=input("Which verb? ")
-src=[[]]
-tgt=[[tokens.index(inp[0])]]
-for i in inp:
-    src[0]+=[tokens.index(i)]
-str_=inp[0]
-for i in range(100):
-    out=model(torch.Tensor(src).to(device),torch.Tensor(tgt).to(device)).tolist()[0]
-    Best=0
-    Best_=tokens.index(" ")
-    for k,f in enumerate(out):
-        if f>Best:
-            Best=f
-            Best_=k
-    if Best_==len(tokens):
-        break
-    str_+=tokens[Best_]
-    tgt[0]+=[Best_]
-
-print(str_)
-
+def Prompt():
+    global tokens
+    global model
+    inp=input("Give me a verb: ")
+    src=[[]]
+    tgt=[[len(tokens)+1]]
+    for i in inp:
+        src[0]+=[tokens.index(i)]
+    str_=""
+    for i in range(100):
+        tgt_=torch.Tensor(tgt)
+        out=model(torch.Tensor(src).to(device),tgt_.to(device)).tolist()[0]
+        Best=0
+        Best_=tokens.index(" ")
+        for k,f in enumerate(out):
+            if f>Best:
+                Best=f
+                Best_=k
+        if Best_==len(tokens):
+            break
+        str_+=tokens[Best_]
+        tgt[0]+=[Best_]
+
+    print(str_)
+
+if eval(input('Train? ')):
+    epochs=eval(input("epochs "))
+else:
+    while True:
+        Prompt()
 
 for epoch in range(epochs):
     total_loss = 0.0
 
     CreateBar()
 
-    bar.start()
-
     for batch_idx, (inputs, targets) in enumerate(train_loader):
 
         #print("",inputs,targets)
@@ -168,7 +206,7 @@ for epoch in range(epochs):
 
             total_loss += loss.item()
 
-            mask = targets[:, i] != len(tokens)
+            mask = targets[:, i] != padding
             targets = targets[mask]
             inputs = inputs[mask]