Context constant: here is the code we're working with : import torch
import torch.nn as nn
import torch.nn.functional as F
from torchtext.data.utils import get_tokenizer
from torchtext.vocab import build_vocab_from_iterator
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm

# Define the Expert module
class Expert(nn.Module):
    def __init__(self, input_dim, hidden_dim):
        super(Expert, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.activation = nn.ReLU()
        self.fc2 = nn.Linear(hidden_dim, input_dim)

    def forward(self, x):
        x = self.activation(self.fc1(x))
        x = self.fc2(x)
        return x

# Define the MoELayer module
class MoELayer(nn.Module):
    def __init__(self, num_experts, input_dim, hidden_dim):
        super(MoELayer, self).__init__()
        self.experts = nn.ModuleList([Expert(input_dim, hidden_dim) for _ in range(num_experts)])
        self.gate = nn.Linear(input_dim, num_experts)

    def forward(self, x):
        gating_distribution = F.softmax(self.gate(x), dim=1)
        expert_outputs = torch.stack([expert(x) for expert in self.experts], dim=1)
        output = torch.einsum('bi,bij->bj', gating_distribution, expert_outputs)  # This efficiently computes the weighted sum of expert outputs
        return output

# Define the SelectionMechanism module
class SelectionMechanism(nn.Module):
    def __init__(self, input_dim, state_dim):
        super(SelectionMechanism, self).__init__()
        self.selection_layer = nn.Linear(input_dim + state_dim, state_dim)

    def forward(self, state, u):
        # Ensure both state and u have a batch dimension (state already has it)
        combined = torch.cat((state, u), dim=1)
        new_state = self.selection_layer(combined)
        return new_state

# Define the StateSpaceMamba module
class StateSpaceMamba(nn.Module):
    def __init__(self, input_dim, state_dim, num_experts, hidden_dim):
        super(StateSpaceMamba, self).__init__()
        self.state_dim = state_dim
        self.state = nn.Parameter(torch.zeros(1, state_dim))
        self.input_layer = nn.Linear(input_dim, state_dim)
        self.selection_mechanism = SelectionMechanism(input_dim, state_dim)
        self.moe_layer = MoELayer(num_experts, state_dim, hidden_dim)

    def forward(self, input_sequence):
        outputs = []
        state = self.state

        for u in input_sequence.split(1, dim=0):  # Assuming input_sequence is shape (sequence_length, input_dim)
            u = u.squeeze(0)  # Remove the sequence dimension
            state = self.selection_mechanism(state, u.unsqueeze(0))  # Add batch dimension to u
            u_projected = self.input_layer(u)
            output = self.moe_layer(u_projected + state)
            outputs.append(output)

        return torch.stack(outputs, dim=0)  # Reshape to (sequence_length, input_dim)

# Load the dataset
print("Loading dataset...")
with open("C:/Users/Dell-PC/Desktop/The-Physics-Dataset.txt", "r") as f:
    text = f.read()

# Tokenize the text
print("Tokenizing text...")
tokenizer = get_tokenizer("basic_english")
tokens = tokenizer(text.lower())

# Build the vocabulary
print("Building vocabulary...")
vocab = build_vocab_from_iterator([tokens], specials=["<unk>", "<pad>", "<sos>", "<eos>"])
vocab.set_default_index(vocab["<unk>"])
numericalized_tokens = [vocab["<sos>"]] + [vocab[token] for token in tokens] + [vocab["<eos>"]]

# Prepare the input sequence
batch_size = 24
seq_length = 12
vocab_size = len(vocab)
num_batches = len(numericalized_tokens) // (batch_size * seq_length)

if len(numericalized_tokens) < (batch_size * seq_length):
    raise ValueError(f"Dataset is too small for the chosen batch size and sequence length. Tokens available: {len(numericalized_tokens)}")

# Calculate the number of batches and ensure it’s not zero
if num_batches == 0:
    raise ValueError(f"Number of batches is zero. Check if your dataset is large enough or consider reducing batch size/sequence length.")

numericalized_tokens = numericalized_tokens[:num_batches * batch_size * seq_length]
input_sequence = torch.tensor(numericalized_tokens).view(batch_size, -1)

# Define the model
num_experts = 5
state_dim = 256
hidden_dim = 256
input_dim = vocab_size
model = StateSpaceMamba(input_dim, state_dim, num_experts, hidden_dim)
learning_rate = 0.001

# Training loop
print("Training model...")
def train(model, data, num_batches, batch_size, seq_length, vocab_size, epochs):
    model.train()
    loss_fn = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
    loss_history = []

    for epoch in tqdm(range(epochs)):
        total_loss = 0.0
        for i in range(0, data.size(1) - seq_length, seq_length):
            # Corrected to batch inputs and targets
            inputs = data[:, i:i+seq_length]
            targets = data[:, i+1:i+1+seq_length].reshape(-1)

            optimizer.zero_grad()
            outputs = model(inputs)
            loss = loss_fn(outputs.reshape(-1, vocab_size), targets)
            loss.backward()
            optimizer.step()

            total_loss += loss.item()
        epoch_loss = total_loss / num_batches
        loss_history.append(epoch_loss)
        print(f"Epoch {epoch}, Loss: {epoch_loss}")
        
    return loss_history

epochs = 10
loss_history = train(model, input_sequence, num_batches, batch_size, seq_length, vocab_size, epochs)

# Plot loss history
plt.plot(loss_history)
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.title("Training Loss History")
plt.show()

# Generate function
print("Generating text...")
def generate(model, start_seq, length, temperature=0.8):
    model.eval()
    state = model.state
    numericalized_start = torch.tensor([vocab[token] for token in tokenizer(start_seq.lower())], dtype=torch.long).unsqueeze(0)
    input_seq = numericalized_start

    generated_sequence = []
    for i in range(length):
        output = model(input_seq)
        
        # Sample from the network as a multinomial distribution
        output_prob = F.softmax(output[:, -1] / temperature, dim=-1).detach()
        predicted_index = torch.multinomial(output_prob, 1).item()
        
        generated_sequence.append(predicted_index)
        
        input_seq = torch.cat((input_seq, torch.tensor([[predicted_index]])), dim=1)
    return " ".join(vocab.lookup_tokens(generated_sequence))

# Generating a sample
start_seq = "The fundamental concepts of physics"  # Start of sequence to begin text generation
generated_text = generate(model, start_seq, length=100)  # Length of generated sequence
print(generated_text)