improve import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import DataLoader, Dataset

class DecoherenceMemoryDecay(nn.Module):
    def __init__(self, embed_dim, decay_rate=0.99):
        super(DecoherenceMemoryDecay, self).__init__()
        self.decay_rate = decay_rate
        self.register_buffer('decay_factor', torch.tensor(decay_rate))
    
    def forward(self, memory):
        # Apply exponential decay to the memory representations
        return memory * self.decay_factor

class NoiseResilientAttention(nn.Module):
    def __init__(self, embed_dim, num_heads, dropout=0.1):
        super(NoiseResilientAttention, self).__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.dropout = nn.Dropout(dropout)
        
        self.head_dim = embed_dim // num_heads
        assert self.head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads"
        
        self.query = nn.Linear(embed_dim, embed_dim)
        self.key = nn.Linear(embed_dim, embed_dim)
        self.value = nn.Linear(embed_dim, embed_dim)
        self.out = nn.Linear(embed_dim, embed_dim)
        
    def forward(self, query, key, value, memory):
        batch_size, seq_length, _ = query.size()
        mem_length = memory.size(2)  # Corrected from size(1) to size(2)
        
        Q = self.query(query).view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2)  # (batch, heads, seq, head_dim)
        K = self.key(key).view(batch_size, mem_length, self.num_heads, self.head_dim).transpose(1, 2)      # (batch, heads, mem_length, head_dim)
        V = self.value(value).view(batch_size, mem_length, self.num_heads, self.head_dim).transpose(1, 2)  # (batch, heads, mem_length, head_dim)
        
        # Combine keys and values with memory
        K = torch.cat([K, memory], dim=2)  # (batch, heads, mem_length + mem_length, head_dim)
        V = torch.cat([V, memory], dim=2)  # (batch, heads, mem_length + mem_length, head_dim)
        
        # Calculate attention scores
        attn_scores = torch.matmul(Q, K.transpose(-2, -1)) / (self.head_dim ** 0.5)
        
        # Introduce stochasticity to attention weights
        attn_scores = attn_scores + torch.randn_like(attn_scores) * 0.01  # Small Gaussian noise
        
        attn_weights = F.softmax(attn_scores, dim=-1)
        attn_weights = self.dropout(attn_weights)
        
        attn_output = torch.matmul(attn_weights, V)  # (batch, heads, seq, head_dim)
        attn_output = attn_output.transpose(1, 2).contiguous().view(batch_size, seq_length, self.embed_dim)
        return self.out(attn_output)

class OptimizedTransformerLayerQD(nn.Module):
    def __init__(self, embed_dim, num_heads, decay_rate=0.99, dropout=0.1):
        super(OptimizedTransformerLayerQD, self).__init__()
        self.self_attn = NoiseResilientAttention(embed_dim, num_heads, dropout)
        self.memory_decay = DecoherenceMemoryDecay(embed_dim, decay_rate)
        self.feed_forward = nn.Sequential(
            nn.Linear(embed_dim, embed_dim * 4),
            nn.ReLU(),
            nn.Linear(embed_dim * 4, embed_dim)
        )
        self.norm1 = nn.LayerNorm(embed_dim)
        self.norm2 = nn.LayerNorm(embed_dim)
    
    def forward(self, x, memory):
        # Self-attention with noise resilience
        attn_output = self.self_attn(x, x, x, memory)
        x = self.norm1(x + attn_output)
        
        # Apply memory decay
        memory = self.memory_decay(memory)
        
        # Feed-forward network
        ff_output = self.feed_forward(x)
        x = self.norm2(x + ff_output)
        return x, memory

class Velocity2QDModel(nn.Module):
    def __init__(self, vocab_size, embed_dim=512, num_heads=8, num_layers=12, decay_rate=0.99, dropout=0.1):
        super(Velocity2QDModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim)
        self.transformer_layers = nn.ModuleList([
            OptimizedTransformerLayerQD(embed_dim, num_heads, decay_rate, dropout) for _ in range(num_layers)
        ])
        self.output_layer = nn.Linear(embed_dim, vocab_size)
    
    def forward(self, input_ids, memory=None):
        # input_ids shape: (batch_size, seq_length)
        x = self.embedding(input_ids)  # (batch_size, seq_length, embed_dim)
        
        batch_size = input_ids.size(0)
        
        if memory is None:
            # Initialize memory for each layer based on the batch size
            memory = [torch.zeros(batch_size, layer.self_attn.num_heads, 1, layer.self_attn.head_dim, device=x.device) for layer in self.transformer_layers]
        
        updated_memory = []
        for i, layer in enumerate(self.transformer_layers):
            x, mem = layer(x, memory[i])
            updated_memory.append(mem)
        
        logits = self.output_layer(x)  # (batch_size, seq_length, vocab_size)
        return logits, updated_memory

# Dummy Dataset
class DummyDatasetQD(Dataset):
    def __init__(self, vocab_size, seq_length, num_samples):
        self.vocab_size = vocab_size
        self.seq_length = seq_length
        self.num_samples = num_samples
    
    def __len__(self):
        return self.num_samples
    
    def __getitem__(self, idx):
        # Random sequence of tokens
        input_ids = torch.randint(0, self.vocab_size, (self.seq_length,))
        target_ids = torch.randint(0, self.vocab_size, (self.seq_length,))
        return input_ids, target_ids

# Hyperparameters
vocab_size = 10000
embed_dim = 512
num_heads = 8
num_layers = 12
decay_rate = 0.98
dropout = 0.1
batch_size = 16
seq_length = 1024
num_epochs = 1

# Initialize model, loss, optimizer
model_qd = Velocity2QDModel(vocab_size, embed_dim, num_heads, num_layers, decay_rate, dropout)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model_qd.parameters(), lr=1e-4)

# DataLoader
dataset_qd = DummyDatasetQD(vocab_size, seq_length, num_samples=1000)
dataloader_qd = DataLoader(dataset_qd, batch_size=batch_size, shuffle=True)

# Training Loop
model_qd.train()
for epoch in range(num_epochs):
    for batch_idx, (input_ids, target_ids) in enumerate(dataloader_qd):
        optimizer.zero_grad()
        logits, _ = model_qd(input_ids)  # (batch_size, seq_length, vocab_size)
        loss = criterion(logits.view(-1, vocab_size), target_ids.view(-1))
        loss.backward()
        optimizer.step()
        
        if batch_idx % 10 == 0:
            print(f"Epoch {epoch+1}, Batch {batch_idx}, Loss: {loss.item():.4f}")

# Inference Example
def generate_text_qd(model, input_ids, max_length=50):
    model.eval()
    generated = input_ids
    memory = None
    with torch.no_grad():
        for _ in range(max_length):
            logits, memory = model(generated, memory)
            next_token = torch.argmax(logits[:, -1, :], dim=-1).unsqueeze(-1)
            generated = torch.cat((generated, next_token), dim=-1)
    return generated

# Example Usage
input_ids_qd = torch.tensor([[1, 2, 3, 4, 5]])  # Example input
generated_ids_qd, _ = model_qd(input_ids_qd)
print("Generated Token IDs:", generated_ids_qd)