Don't reply: import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.utils.rnn import pad_sequence
from torch.utils.data import DataLoader, Dataset
from collections import Counter
import json
from tqdm import tqdm
import math




def positional_encoding(seq_len, d_model, device):
    pos = torch.arange(seq_len, dtype=torch.float, device=device).unsqueeze(1)
    div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)).to(device)
    pe = torch.zeros(seq_len, d_model, device=device)
    pe[:, 0::2] = torch.sin(pos * div_term)
    pe[:, 1::2] = torch.cos(pos * div_term)
    return pe.unsqueeze(0)

# Expert Transformer Model
class TransformerExpert(nn.Module):
    def __init__(self, input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers=1):
        super(TransformerExpert, self).__init__()
        self.d_model = d_model
        self.input_fc = nn.Linear(input_size, d_model)
        encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward, batch_first=True)
        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_encoder_layers)
        self.output_fc = nn.Linear(d_model, output_size)

    def forward(self, x):
        x = self.input_fc(x) + positional_encoding(x.size(1), self.d_model, x.device)
        transformer_output = self.transformer_encoder(x)
        output = self.output_fc(transformer_output[:, -1, :])
        return output

# Gating Network
class GatingNetwork(nn.Module):
    def __init__(self, input_feature_dim, num_experts):
        super(GatingNetwork, self).__init__()
        self.fc = nn.Linear(input_feature_dim, num_experts)
        self.softmax = nn.Softmax(dim=1)
    
    def forward(self, x):
        # Assuming x is of shape [batch_size, seq_len, d_model] you may want to aggregate
        # across the sequence length dimension before the gating network
        x = x.mean(dim=1)  # Take mean or another operation that reduces seq_len dimension
        x = self.fc(x)  # Now the shape of x should match the weight matrix of the fc layer
        return self.softmax(x)

# Mixture of Experts Model
class MixtureOfTransformerExperts(nn.Module):
    def __init__(self, input_size, d_model, output_size, nhead, dim_feedforward, num_experts, num_encoder_layers=1):
        super(MixtureOfTransformerExperts, self).__init__()
        self.num_experts = num_experts
        self.experts = nn.ModuleList([TransformerExpert(input_size, d_model, output_size, nhead, dim_feedforward, num_encoder_layers) for _ in range(num_experts)])
        self.gating_network = GatingNetwork(d_model, num_experts)

    def forward(self, x):
        gating_scores = self.gating_network(x)
        expert_outputs = [expert(x).unsqueeze(1) for expert in self.experts]
        expert_outputs = torch.cat(expert_outputs, dim=1) # shape: [batch, num_experts, output_size]
        mixed_output = torch.bmm(expert_outputs.permute(0, 2, 1), gating_scores.unsqueeze(-1)).squeeze(-1)
        return mixed_output

class QAJsonlDataset(Dataset):
    def __init__(self, path, seq_len):
        self.seq_len = seq_len
        self.pairs = self.load_data(path)
        # Flatten the pairs completely before passing them to build_vocab
        self.vocab, self.idx2token = self.build_vocab([word for pair in self.pairs for sublist in pair for word in sublist])
        self.tokenized_pairs = [(self.tokenize(q), self.tokenize(a)) for q, a in self.pairs]

    def load_data(self, path):
        pairs = []
        with open(path, "r", encoding="utf-8") as f:
            for line in f:
                data = json.loads(line.strip())
                question, answer = data.get("question", ""), data.get("answer", "")
                pairs.append((question.split(), answer.split()))
        return pairs

    def tokenize(self, words):
    # Tokenize a sentence and pad if necessary
    # Add <eos> token at the end if there’s room
        tokens = [self.vocab.get(w, self.vocab["<unk>"]) for w in words]
        if len(tokens) < self.seq_len:
            tokens.append(self.vocab["<eos>"])  # Add <eos> token
            tokens.extend([self.vocab["<pad>"]] * (self.seq_len - len(tokens)))  # Pad the rest
        else:
            tokens = tokens[:self.seq_len - 1] + [self.vocab["<eos>"]]
        return tokens

    def build_vocab(self, words):
    # Start with special tokens with fixed indices
        vocab = {"<unk>": 0, "<pad>": 1, "<eos>": 2}
        start_index = len(vocab)

    # Use Counter to count word frequencies in the corpus
        counts = Counter(words)
    
    # Create the vocab dictionary with all words, starting indices after the special tokens
        for word, _ in counts.most_common():
            if word not in vocab:  # Skip special tokens
                vocab[word] = len(vocab)

    # Create the reverse mapping from indices to words
        idx2token = {idx: token for token, idx in vocab.items()}
        return vocab, idx2token
        
    def __len__(self):
        return len(self.tokenized_pairs)

    def __getitem__(self, idx):
        tokenized_question, tokenized_answer = self.tokenized_pairs[idx]
        return torch.tensor(tokenized_question, dtype=torch.long), torch.tensor(tokenized_answer, dtype=torch.long)

class MoETransformerModel(nn.Module):
    def __init__(self, vocab_size, d_model, moe):
        super(MoETransformerModel, self).__init__()
        self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=d_model)
        self.moe = moe
        self.dropout = nn.Dropout(p=0.1)  # Dropout added for regularization

    def forward(self, x):
        embedded = self.dropout(self.embedding(x))
        embedded_with_position = embedded + positional_encoding(x.size(1), d_model, x.device)
        return self.moe(embedded_with_position)

def collate_fn(batch):
    questions, answers = zip(*batch)
    questions = pad_sequence(questions, batch_first=True, padding_value=0)
    answers = pad_sequence(answers, batch_first=True, padding_value=0)
    return questions, answers

# Set the path to your text file and define sequence length
path_to_text = 'GSM2K.jsonl'  # replace with the path to your text file
seq_len = 64  # sequence length

# Create a dataset and data loader
dataset = QAJsonlDataset(path_to_text, seq_len)
data_loader = DataLoader(dataset, batch_size=128, shuffle=True, collate_fn=collate_fn)
    
# Training loop - added gradient clipping to avoid exploding gradients
def train_model(model, criterion, optimizer, num_epochs, data_loader):
    model.train()
    for epoch in range(num_epochs):
        total_loss = 0
        progress_bar = tqdm(enumerate(data_loader), total=len(data_loader), desc=f"Epoch {epoch+1}", leave=False)
        for i, (inputs, targets) in progress_bar:
            optimizer.zero_grad()
            predictions = model(inputs)
            loss = criterion(predictions.view(-1, predictions.size(-1)), targets.view(-1))
            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)  # Gradient clipping
            optimizer.step()
            total_loss += loss.item()
            progress_bar.set_postfix({"Loss": loss.item()})
        average_loss = total_loss / len(data_loader.dataset)
        print(f"Epoch {epoch+1}, Average Loss: {average_loss}")

def generate_text(model, dataset, seed_text, num_generate, temperature=1.0):
    model.eval()  # Put the model in evaluation mode
    
    # List to store the generated tokens
    generated_tokens = []

    # Initial sequence (prefix) to start the generation process
    input_sequence = [dataset.vocab.get(word, dataset.vocab["<pad>"]) for word in seed_text.split()]  # Convert to token IDs
    current_sequence = torch.tensor(input_sequence, dtype=torch.long).unsqueeze(0)

    # Generate num_generate tokens
    for _ in range(num_generate):
        # Forward pass through the model
        with torch.no_grad():
            output = model(current_sequence)
        
        # Get probabilities, apply temperature scaling, and sample from the distribution
        probabilities = F.softmax(output[-1] / temperature, dim=0).detach()
        next_token_idx = torch.multinomial(probabilities, 1).item()

        # Append token to the current sequence and to the generated tokens
        generated_tokens.append(next_token_idx)
        current_sequence = torch.cat((current_sequence, torch.tensor([[next_token_idx]])), 1)
    
    # Convert tokens to words
    generated_text = " ".join([dataset.idx2token.get(token, "<unk>") for token in generated_tokens])  # Use .get() to provide a default value for missing keys
    return generated_text

# Define hyperparameters specific to the transformer
d_model = 128
nhead = 8
dim_feedforward = 256
num_encoder_layers = 2
num_experts = 2
vocab_size = len(dataset.vocab)  # Assume dataset.vocab is defined in the QAJsonlDataset class

# Instantiate resulting MoE transformer model
moe = MixtureOfTransformerExperts(
    input_size=d_model,  # Assuming each word is represented by d_model features
    d_model=d_model,
    output_size=vocab_size,  # Output size is the vocab size for token generation
    nhead=nhead,
    dim_feedforward=dim_feedforward,
    num_experts=num_experts,
    num_encoder_layers=num_encoder_layers
)

# Instantiate the MoE transformer model
moe_transformer_model = MoETransformerModel(vocab_size, d_model, moe)

def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

# Example usage with your model:
total_params = count_parameters(moe_transformer_model)
print(f"Total trainable parameters: {total_params}")

# Training parameters
num_epochs = 4
learning_rate = 1e-4

# Define Loss Function and Optimizer for MoE model - using Label Smoothing for better generalization
criterion = nn.CrossEntropyLoss(label_smoothing=0.1)
optimizer = torch.optim.AdamW(moe_transformer_model.parameters(), lr=learning_rate, weight_decay=0.01)  # Using AdamW with weight decay
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, "min")

# Replace references to single_lstm_txt_model with moe_model
# Train the model with the text data
train_model(moe_transformer_model, criterion, optimizer, num_epochs, data_loader)

# Start a loop for the interactive chat-like text generation
while True:
    try:
        # Get user input
        seed_text = input("Enter seed text (type 'quit' to stop): ")
        
        # Check if user wants to quit the interaction
        if seed_text.lower() == "quit":
            print("Exiting text generation chat.")
            break
        
        # User input is not empty and not “quit”, generate text
        if seed_text.strip():
            num_generate = 16  # Number of words to generate
            temperature = 1.0  # Sampling temperature, higher will increase diversity
            
            # Use the trained model to generate text
            generated_text = generate_text(moe_transformer_model, dataset, seed_text, num_generate, temperature)
            print("Generated Text:", generated_text)
        else:
            print("Seed text cannot be empty.")
    except KeyboardInterrupt:
        # Handle KeyboardInterrupt (Ctrl+C) to gracefully exit
        print("\nExiting text generation chat.")
        break

torch.save(moe_transformer_model.state_dict(), "MoE_Transformer-X2-9M-QA.pth")