Upload PrateritumGPT.py

PrateritumGPT.py

@@ -0,0 +1,119 @@
+import csv
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader
+from torch.nn.utils.rnn import pad_sequence
+import math
+
+tokens = list("azertyuiopqsdfghjklmwxcvbnäüöß—– ")
+tokensdict = {}
+
+for i in range(len(tokens)):
+    tokensdict.update({tokens[i]: [0] * i + [0] * (len(tokens) - (i + 1))})
+
+# Ouvrir le fichier CSV
+with open("C:\\Users\\marc2\\Downloads\\7eaaf0e22461b505c749e268c0b72bc4-12ebe211a929f039791dfeaa1a019b64cadddaf1\\7eaaf0e22461b505c749e268c0b72bc4-12ebe211a929f039791dfeaa1a019b64cadddaf1\\top-german-verbs.csv", 'r', encoding="utf-8") as file:
+    # Créer un objet lecteur CSV
+    reader = [i for i in csv.reader(file)][1:]
+
+class CSVDataset(Dataset):
+    def __init__(self, features, labels):
+        self.features = features
+        self.labels = labels
+
+    def __len__(self):
+        return len(self.features)
+
+    def __getitem__(self, idx):
+        sample = self.features[idx], self.labels[idx]
+        return sample
+
+# Supposons que vous ayez vos données sous forme de listes
+features = []
+labels = []
+
+for i in reader:
+    k = []
+    for j in i[2]:
+        k += [tokens.index(j)]
+    k += [len(tokens) + 1] * (25 - len(k))
+    features += [torch.Tensor(k)]
+    k = []
+    for j in i[8]:
+        k += [tokens.index(j)]
+    k += [len(tokens) + 1] * (25 - len(k))
+    labels += [torch.Tensor(k)]
+
+MyDataset = CSVDataset(features=features, labels=labels)
+
+class TransformerModel(nn.Module):
+    def __init__(self, vocab_size, emb_dim, nhead, num_encoder_layers, num_decoder_layers, dim_feedforward, dropout=0.1):
+        super().__init__()
+        self.custom_embedding = nn.Embedding(vocab_size, emb_dim)
+        self.pos_encoder = PositionalEncoding(emb_dim, dropout)
+        encoder_layer = nn.TransformerEncoderLayer(emb_dim, nhead, dim_feedforward, dropout)
+        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_encoder_layers)
+        decoder_layer = nn.TransformerDecoderLayer(emb_dim, nhead, dim_feedforward, dropout)
+        self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_decoder_layers)
+        self.output_layer = nn.Linear(emb_dim, vocab_size)
+
+    def forward(self, src, tgt, src_mask=None, tgt_mask=None, memory_mask=None, src_key_padding_mask=None, tgt_key_padding_mask=None, memory_key_padding_mask=None):
+        src_emb = self.pos_encoder(self.custom_embedding(src.long()))
+        tgt_emb = self.pos_encoder(self.custom_embedding(tgt.long()))
+        encoder_output = self.transformer_encoder(src_emb, src_mask, src_key_padding_mask)
+        decoder_output = self.transformer_decoder(tgt_emb, encoder_output, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask)
+        output = self.output_layer(decoder_output)
+        return output
+
+# Définition de la classe PositionalEncoding (identique à l'exemple précédent)
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model, dropout=0.1, max_len=5000):
+        super(PositionalEncoding, self).__init__()
+        self.dropout = nn.Dropout(p=dropout)
+
+        pe = torch.zeros(max_len, d_model)
+        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
+        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(0).transpose(0, 1)
+        self.register_buffer('pe', pe)
+
+    def forward(self, x):
+        x = x + self.pe[:x.size(0), :]
+        return self.dropout(x)
+
+# Préparation des données
+def collate_fn(batch):
+    inputs = [item[0] for item in batch]
+    targets = [item[1] for item in batch]
+    inputs = pad_sequence(inputs, batch_first=True, padding_value=len(tokens) + 1)
+    targets = pad_sequence(targets, batch_first=True, padding_value=len(tokens) + 1)
+    return inputs, targets
+
+train_loader = DataLoader(MyDataset, batch_size=32, shuffle=True, collate_fn=collate_fn)
+
+# Définition du modèle, de la fonction de perte et de l'optimiseur
+model = TransformerModel(vocab_size=len(tokens) + 2, emb_dim=512, nhead=8, num_encoder_layers=6, num_decoder_layers=6, dim_feedforward=2048)
+loss_fn = nn.CrossEntropyLoss()
+optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
+
+epochs = 10
+
+for epoch in range(epochs):
+    total_loss = 0.0
+
+    for batch_idx, (inputs, targets) in enumerate(train_loader):
+        optimizer.zero_grad()
+        output = model(inputs, targets[:, :-1])  # Shifted targets
+        output = output.transpose(1, 2)  # Adjust shape for loss function
+        loss = loss_fn(output, targets[:, 1:].long())  # Shifted targets
+        loss.backward()
+        optimizer.step()
+
+        total_loss += loss.item()
+
+        if batch_idx % 100 == 0:
+            print(f"Epoch {epoch + 1}/{epochs}, Batch {batch_idx}/{len(train_loader)}, Loss: {total_loss / (batch_idx + 1)}")
+
+    print(f"Epoch {epoch + 1}/{epochs}, Loss: {total_loss / len(train_loader)}")