Update PrateritumGPT.py
Browse files- PrateritumGPT.py +31 -25
PrateritumGPT.py
CHANGED
@@ -35,13 +35,13 @@ labels = []
|
|
35 |
for i in reader:
|
36 |
k = []
|
37 |
for j in i[2]:
|
38 |
-
k += [tokens.index(j)]
|
39 |
-
k += [
|
40 |
features += [torch.Tensor(k)]
|
41 |
k = []
|
42 |
for j in i[8]:
|
43 |
-
k += [tokens.index(j)]
|
44 |
-
k += [
|
45 |
labels += [torch.Tensor(k)]
|
46 |
|
47 |
MyDataset = CSVDataset(features=features, labels=labels)
|
@@ -51,21 +51,26 @@ class TransformerModel(nn.Module):
|
|
51 |
super().__init__()
|
52 |
self.custom_embedding = nn.Embedding(vocab_size, emb_dim)
|
53 |
self.pos_encoder = PositionalEncoding(emb_dim, dropout)
|
54 |
-
encoder_layer = nn.TransformerEncoderLayer(emb_dim, nhead, dim_feedforward, dropout)
|
55 |
self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_encoder_layers)
|
56 |
-
decoder_layer = nn.TransformerDecoderLayer(emb_dim, nhead, dim_feedforward, dropout)
|
57 |
self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_decoder_layers)
|
58 |
self.output_layer = nn.Linear(emb_dim, vocab_size)
|
59 |
|
60 |
def forward(self, src, tgt, src_mask=None, tgt_mask=None, memory_mask=None, src_key_padding_mask=None, tgt_key_padding_mask=None, memory_key_padding_mask=None):
|
61 |
-
src_emb = self.
|
62 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
63 |
encoder_output = self.transformer_encoder(src_emb, src_mask, src_key_padding_mask)
|
64 |
decoder_output = self.transformer_decoder(tgt_emb, encoder_output, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask)
|
65 |
output = self.output_layer(decoder_output)
|
66 |
return output
|
67 |
|
68 |
-
# Définition de la classe PositionalEncoding (identique à l'exemple précédent)
|
69 |
class PositionalEncoding(nn.Module):
|
70 |
def __init__(self, d_model, dropout=0.1, max_len=5000):
|
71 |
super(PositionalEncoding, self).__init__()
|
@@ -76,14 +81,13 @@ class PositionalEncoding(nn.Module):
|
|
76 |
div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
|
77 |
pe[:, 0::2] = torch.sin(position * div_term)
|
78 |
pe[:, 1::2] = torch.cos(position * div_term)
|
79 |
-
pe = pe.unsqueeze(0)
|
80 |
self.register_buffer('pe', pe)
|
81 |
|
82 |
def forward(self, x):
|
83 |
-
x = x + self.pe[:x.size(
|
84 |
return self.dropout(x)
|
85 |
-
|
86 |
-
# Préparation des données
|
87 |
def collate_fn(batch):
|
88 |
inputs = [item[0] for item in batch]
|
89 |
targets = [item[1] for item in batch]
|
@@ -93,8 +97,7 @@ def collate_fn(batch):
|
|
93 |
|
94 |
train_loader = DataLoader(MyDataset, batch_size=32, shuffle=True, collate_fn=collate_fn)
|
95 |
|
96 |
-
|
97 |
-
model = TransformerModel(vocab_size=len(tokens) + 2, emb_dim=512, nhead=8, num_encoder_layers=6, num_decoder_layers=6, dim_feedforward=2048)
|
98 |
loss_fn = nn.CrossEntropyLoss()
|
99 |
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
|
100 |
|
@@ -105,18 +108,21 @@ for epoch in range(epochs):
|
|
105 |
|
106 |
for batch_idx, (inputs, targets) in enumerate(train_loader):
|
107 |
|
108 |
-
|
109 |
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
|
|
|
|
|
|
116 |
|
117 |
-
|
118 |
|
119 |
-
|
120 |
-
|
121 |
|
122 |
print(f"Epoch {epoch + 1}/{epochs}, Loss: {total_loss / len(train_loader)}")
|
|
|
35 |
for i in reader:
|
36 |
k = []
|
37 |
for j in i[2]:
|
38 |
+
k += [tokens.index(j)+1]
|
39 |
+
k += [0] * (25 - len(k))
|
40 |
features += [torch.Tensor(k)]
|
41 |
k = []
|
42 |
for j in i[8]:
|
43 |
+
k += [tokens.index(j)+1]
|
44 |
+
k += [0] * (25 - len(k))
|
45 |
labels += [torch.Tensor(k)]
|
46 |
|
47 |
MyDataset = CSVDataset(features=features, labels=labels)
|
|
|
51 |
super().__init__()
|
52 |
self.custom_embedding = nn.Embedding(vocab_size, emb_dim)
|
53 |
self.pos_encoder = PositionalEncoding(emb_dim, dropout)
|
54 |
+
encoder_layer = nn.TransformerEncoderLayer(emb_dim, nhead, dim_feedforward, dropout, batch_first=True)
|
55 |
self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_encoder_layers)
|
56 |
+
decoder_layer = nn.TransformerDecoderLayer(emb_dim, nhead, dim_feedforward, dropout, batch_first=True)
|
57 |
self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_decoder_layers)
|
58 |
self.output_layer = nn.Linear(emb_dim, vocab_size)
|
59 |
|
60 |
def forward(self, src, tgt, src_mask=None, tgt_mask=None, memory_mask=None, src_key_padding_mask=None, tgt_key_padding_mask=None, memory_key_padding_mask=None):
|
61 |
+
src_emb = self.custom_embedding(src.long())
|
62 |
+
print("Source Embedding:", src_emb.shape)
|
63 |
+
src_emb = self.pos_encoder(src_emb)
|
64 |
+
print("Source Embedding:", src_emb.shape)
|
65 |
+
tgt_emb = self.custom_embedding(tgt.long())
|
66 |
+
print("Target Embedding:", tgt_emb.shape)
|
67 |
+
tgt_emb = self.pos_encoder(tgt_emb)
|
68 |
+
print("Target Embedding:", tgt_emb.shape)
|
69 |
encoder_output = self.transformer_encoder(src_emb, src_mask, src_key_padding_mask)
|
70 |
decoder_output = self.transformer_decoder(tgt_emb, encoder_output, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask)
|
71 |
output = self.output_layer(decoder_output)
|
72 |
return output
|
73 |
|
|
|
74 |
class PositionalEncoding(nn.Module):
|
75 |
def __init__(self, d_model, dropout=0.1, max_len=5000):
|
76 |
super(PositionalEncoding, self).__init__()
|
|
|
81 |
div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
|
82 |
pe[:, 0::2] = torch.sin(position * div_term)
|
83 |
pe[:, 1::2] = torch.cos(position * div_term)
|
84 |
+
pe = pe.unsqueeze(0)
|
85 |
self.register_buffer('pe', pe)
|
86 |
|
87 |
def forward(self, x):
|
88 |
+
x = x + self.pe[:, :x.size(1), :]
|
89 |
return self.dropout(x)
|
90 |
+
|
|
|
91 |
def collate_fn(batch):
|
92 |
inputs = [item[0] for item in batch]
|
93 |
targets = [item[1] for item in batch]
|
|
|
97 |
|
98 |
train_loader = DataLoader(MyDataset, batch_size=32, shuffle=True, collate_fn=collate_fn)
|
99 |
|
100 |
+
model = TransformerModel(vocab_size=len(tokens)+1, emb_dim=32, nhead=8, num_encoder_layers=6, num_decoder_layers=6, dim_feedforward=2048)
|
|
|
101 |
loss_fn = nn.CrossEntropyLoss()
|
102 |
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
|
103 |
|
|
|
108 |
|
109 |
for batch_idx, (inputs, targets) in enumerate(train_loader):
|
110 |
|
111 |
+
for i in range(1,targets.shape[1]):
|
112 |
|
113 |
+
optimizer.zero_grad()
|
114 |
+
output = model(inputs, targets[:, :i]) # Shifted targets
|
115 |
+
output = output.transpose(1, 2) # Adjust shape for loss function
|
116 |
+
#loss = loss_fn(output, targets[:, i].unsqueeze(1).long()) # Shifted targets
|
117 |
+
print(output.shape)
|
118 |
+
print(targets[:, i].unsqueeze(1).long().shape)
|
119 |
+
loss = loss_fn(output, targets[:, i].unsqueeze(1).long())
|
120 |
+
loss.backward()
|
121 |
+
optimizer.step()
|
122 |
|
123 |
+
total_loss += loss.item()
|
124 |
|
125 |
+
if batch_idx % 100 == 0:
|
126 |
+
print(f"Epoch {epoch + 1}/{epochs}, Batch {batch_idx}/{len(train_loader)}, Loss: {total_loss / (batch_idx + 1)}")
|
127 |
|
128 |
print(f"Epoch {epoch + 1}/{epochs}, Loss: {total_loss / len(train_loader)}")
|