Please use this code: import json
import numpy as np
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Input, LSTM, Dense, Embedding
from keras.models import Model
import pickle

NUM_TOKENS = 5000  # You mentioned 20,000 instructions, assuming a vocab to cover all
MAX_INSTRUCTION_LENGTH = 75  # Adjust according to your dataset
MAX_OUTPUT_LENGTH = 75       # Adjust according to your dataset
EMBEDDING_DIM = 96
LSTM_UNITS = 192
EPOCHS = 10  # Set this based on your need
BATCH_SIZE = 100  # Depending on your machine’s memory, adjust this accordingly

def load_data(json_file_path):
    with open(json_file_path, "r") as f:
        return json.load(f)

json_file_path = "C:/Users/Dell-PC/Desktop/Projets/Datasets/code_alpaca_5k.json"
data = load_data(json_file_path)

instructions = [item["instruction"] for item in data]
outputs = ["<start> " + item["output"] + " <end>" for item in data]  # Add start and end tokens

tokenizer = Tokenizer(num_words=NUM_TOKENS, filters="", lower=False)
tokenizer.fit_on_texts(instructions + outputs)

instr_sequences = tokenizer.texts_to_sequences(instructions)
out_sequences = tokenizer.texts_to_sequences(outputs)

instr_padded = pad_sequences(instr_sequences, maxlen=MAX_INSTRUCTION_LENGTH, padding="post")
output_padded = pad_sequences(out_sequences, maxlen=MAX_OUTPUT_LENGTH, padding="post")

encoder_inputs = Input(shape=(MAX_INSTRUCTION_LENGTH,))
encoder_embedding = Embedding(NUM_TOKENS, EMBEDDING_DIM)(encoder_inputs)
encoder_outputs, state_h, state_c = LSTM(LSTM_UNITS, return_state=True)(encoder_embedding)
encoder_states = [state_h, state_c]

decoder_inputs = Input(shape=(MAX_OUTPUT_LENGTH,))
decoder_embedding = Embedding(NUM_TOKENS, EMBEDDING_DIM)(decoder_inputs)
decoder_lstm = LSTM(LSTM_UNITS, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)
decoder_dense = Dense(NUM_TOKENS, activation="softmax")
decoder_outputs = decoder_dense(decoder_outputs)

model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy")

decoder_input_data = np.zeros_like(output_padded)
decoder_input_data[:, 1:] = output_padded[:, :-1]
decoder_input_data[:, 0] = tokenizer.word_index["<start>"]

decoder_target_data = np.expand_dims(output_padded, -1)

model.fit([instr_padded, decoder_input_data], decoder_target_data,
          batch_size=BATCH_SIZE,
          epochs=EPOCHS,
          validation_split=0.2)

model.save("T-2PY_mini.h5")

with open("instruction_output_tokenizer_mini.pickle", "wb") as handle:
    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)   to create the dataloader for this code: import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, Dataset
from torch.nn import TransformerDecoder, TransformerDecoderLayer
from torch.nn.functional import cross_entropy



EPOCHS = 100



class CustomTransformerDecoder(nn.Module):
    def init(self, vocab_size, d_model, nhead, num_decoder_layers, dim_feedforward, max_seq_length):
        super(CustomTransformerDecoder, self).init()
        self.embeddings = nn.Embedding(vocab_size, d_model)
        self.positional_encodings = nn.Parameter(torch.zeros(max_seq_length, d_model))
        decoder_layer = TransformerDecoderLayer(d_model, nhead, dim_feedforward)
        self.transformer_decoder = TransformerDecoder(decoder_layer, num_decoder_layers)
        self.output_layer = nn.Linear(d_model, vocab_size)

    def forward(self, src, tgt, tgt_mask):
        src_embeddings = self.embeddings(src) + self.positional_encodings[:src.size(1)]
        tgt_embeddings = self.embeddings(tgt) + self.positional_encodings[:tgt.size(1)]
        memory = self.transformer_decoder(src_embeddings, tgt_embeddings, tgt_mask)
        out = self.output_layer(memory)
        return out

def create_target_mask(size):
    # Mask out subsequent positions (to prevent lookahead)
    mask = torch.triu(torch.ones(size, size) * float("-inf"), diagonal=1)
    return mask

# Model parameters - these would be more carefully chosen in a real application
vocab_size = 20000
d_model = 512
nhead = 4
num_decoder_layers = 6
dim_feedforward = 1024
max_seq_length = 20

# Create the model
model = CustomTransformerDecoder(vocab_size, d_model, nhead, num_decoder_layers, dim_feedforward, max_seq_length)

# Prepare data
# Define your custom dataset, data loaders, etc.

# Define loss function and optimizer
loss_function = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0)
scheduler = optim.lr_scheduler.OneCycleLR(optimizer, max_lr=2.5e-4, total_steps=2000)

# Training loop
for epoch in range(EPOCHS):
    model.train()
    total_loss = 0
    
    for input_seqs, target_seqs in data_loader: # Define your own DataLoader
        # This is a simplified loop. You will need to prepare your data accordingly.
        optimizer.zero_grad()
        input_seqs = input_seqs.to(device)
        target_seqs = target_seqs.to(device)
        
        tgt_input = target_seqs[:, :-1]
        targets = target_seqs[:, 1:].contiguous().view(-1)
        
        tgt_mask = create_target_mask(tgt_input.size(1)).to(device)
        
        outputs = model(input_seqs, tgt_input, tgt_mask)
        outputs = outputs.view(-1, vocab_size)
        
        loss = loss_function(outputs, targets)
        loss.backward()
        optimizer.step()
        scheduler.step()
        
        total_loss += loss.item()
    print("Epoch: {}, Loss: {:.4f}".format(epoch, total_loss))