Update PrateritumGPT.py
Browse files- PrateritumGPT.py +6 -4
PrateritumGPT.py
CHANGED
@@ -4,7 +4,6 @@ import torch.nn as nn
|
|
4 |
from torch.utils.data import Dataset, DataLoader
|
5 |
from torch.nn.utils.rnn import pad_sequence
|
6 |
import math
|
7 |
-
import os
|
8 |
|
9 |
tokens = list("azertyuiopqsdfghjklmwxcvbnäüöß—– ")
|
10 |
tokensdict = {}
|
@@ -13,7 +12,7 @@ for i in range(len(tokens)):
|
|
13 |
tokensdict.update({tokens[i]: [0] * i + [0] * (len(tokens) - (i + 1))})
|
14 |
|
15 |
# Ouvrir le fichier CSV
|
16 |
-
with open(
|
17 |
# Créer un objet lecteur CSV
|
18 |
reader = [i for i in csv.reader(file)][1:]
|
19 |
|
@@ -38,12 +37,12 @@ for i in reader:
|
|
38 |
for j in i[2]:
|
39 |
k += [tokens.index(j)]
|
40 |
k += [len(tokens) + 1] * (25 - len(k))
|
41 |
-
features += [torch.Tensor(
|
42 |
k = []
|
43 |
for j in i[8]:
|
44 |
k += [tokens.index(j)]
|
45 |
k += [len(tokens) + 1] * (25 - len(k))
|
46 |
-
labels += [torch.Tensor(
|
47 |
|
48 |
MyDataset = CSVDataset(features=features, labels=labels)
|
49 |
|
@@ -105,6 +104,9 @@ for epoch in range(epochs):
|
|
105 |
total_loss = 0.0
|
106 |
|
107 |
for batch_idx, (inputs, targets) in enumerate(train_loader):
|
|
|
|
|
|
|
108 |
optimizer.zero_grad()
|
109 |
output = model(inputs, targets[:, :-1]) # Shifted targets
|
110 |
output = output.transpose(1, 2) # Adjust shape for loss function
|
|
|
4 |
from torch.utils.data import Dataset, DataLoader
|
5 |
from torch.nn.utils.rnn import pad_sequence
|
6 |
import math
|
|
|
7 |
|
8 |
tokens = list("azertyuiopqsdfghjklmwxcvbnäüöß—– ")
|
9 |
tokensdict = {}
|
|
|
12 |
tokensdict.update({tokens[i]: [0] * i + [0] * (len(tokens) - (i + 1))})
|
13 |
|
14 |
# Ouvrir le fichier CSV
|
15 |
+
with open("C:\\Users\\marc2\\Downloads\\7eaaf0e22461b505c749e268c0b72bc4-12ebe211a929f039791dfeaa1a019b64cadddaf1\\7eaaf0e22461b505c749e268c0b72bc4-12ebe211a929f039791dfeaa1a019b64cadddaf1\\top-german-verbs.csv", 'r', encoding="utf-8") as file:
|
16 |
# Créer un objet lecteur CSV
|
17 |
reader = [i for i in csv.reader(file)][1:]
|
18 |
|
|
|
37 |
for j in i[2]:
|
38 |
k += [tokens.index(j)]
|
39 |
k += [len(tokens) + 1] * (25 - len(k))
|
40 |
+
features += [torch.Tensor(k)]
|
41 |
k = []
|
42 |
for j in i[8]:
|
43 |
k += [tokens.index(j)]
|
44 |
k += [len(tokens) + 1] * (25 - len(k))
|
45 |
+
labels += [torch.Tensor(k)]
|
46 |
|
47 |
MyDataset = CSVDataset(features=features, labels=labels)
|
48 |
|
|
|
104 |
total_loss = 0.0
|
105 |
|
106 |
for batch_idx, (inputs, targets) in enumerate(train_loader):
|
107 |
+
|
108 |
+
print(inputs.shape,targets.shape)
|
109 |
+
|
110 |
optimizer.zero_grad()
|
111 |
output = model(inputs, targets[:, :-1]) # Shifted targets
|
112 |
output = output.transpose(1, 2) # Adjust shape for loss function
|