What can we improve on this code ? suggestions ? thoughts ? code: import torch
import torch.nn as nn
import torch.optim as optim
from torch.nn import TransformerEncoder, TransformerEncoderLayer, Conv1d
import torch.optim as optim
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import pandas as pd
import shap
import json
import joblib
import numpy as np

# --- Load Data from CSV ---
def load_data(csv_file):
    return pd.read_csv(csv_file)

# --- Data Preprocessing ---
def preprocess_data(df):
    numeric_features = [
        "Clock Speed (GHz)",
        "ROPs",
        "TMUs",
        "TFlops",
        "L2 Cache (MB)",
        "VRAM (GB)",
        "Bandwidth (GB/s)",
        "Num Shaders",
    ]
    categorical_features = ["Architecture"]
    ARCHITECTURE_MAPPING = {
        "PASCAL": 0,
        "TURING": 1,
        "AMPERE": 2,
        "ADA": 3,
        "GCN5": 4,
        "RDNA": 5,
        "RDNA2": 6,
        "RDNA3": 7,
    }

    df["Architecture"] = df["Architecture"].map(ARCHITECTURE_MAPPING)
    target = "Relative Performance"

    # Create a transformer for numerical features
    numeric_transformer = StandardScaler()

    # Create a transformer for categorical features
    categorical_transformer = OneHotEncoder(handle_unknown="ignore")

    # Combine both transformers using ColumnTransformer
    preprocessor = ColumnTransformer(
        transformers=[
            ("num", numeric_transformer, numeric_features),
            ("cat", categorical_transformer, categorical_features),
        ]
    )

    # Fit and transform the data
    X = preprocessor.fit_transform(df[numeric_features + categorical_features])
    y = torch.tensor(df[target].values, dtype=torch.float32).unsqueeze(
        1
    )  # Unsqueeze for single target

    # Reshape for Transformer: [batch_size, sequence_length, features]
    X = torch.tensor(X, dtype=torch.float32).unsqueeze(1)

    return X, y, preprocessor

# --- Model Architecture ---
class GPUPerformanceModel(nn.Module):
    def __init__(self, n_features, nhead, nhid, nlayers, dropout, kernel_size=3):
        super(GPUPerformanceModel, self).__init__()
        self.encoder = nn.Linear(n_features, nhid)
        encoder_layers = TransformerEncoderLayer(
            nhid, nhead, nhid * 2, dropout, batch_first=True
        )
        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
        self.dropout = nn.Dropout(dropout)
        self.conv1d = Conv1d(nhid, nhid, kernel_size=kernel_size, padding='same') # Convolutional layer
        self.decoder = nn.Sequential(
            nn.Linear(nhid, 64), nn.ReLU(), nn.Linear(64, 1)
        )

    def forward(self, src):
        src = self.encoder(src)
        output = self.transformer_encoder(src)
        output = self.dropout(output) 
        
        # Reshape for Conv1d: [batch_size, channels, sequence_length]
        output = output.permute(0, 2, 1)
        output = self.conv1d(output)

        # Reshape back for decoder: [batch_size, sequence_length, features]
        output = output.permute(0, 2, 1)
        output = self.dropout(output[:, 0, :])
        output = self.decoder(output)
        return output

# --- Model Training ---
def train_model(model, X_train, y_train, X_val, y_val, epochs, learning_rate):
    criterion = nn.MSELoss()
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)

    train_losses = []
    val_losses = []
    train_mae = []
    val_mae = []
    train_rmse = []
    val_rmse = []
    gradient_norms = []

    for epoch in range(epochs):
        # Training
        model.train()
        optimizer.zero_grad()
        outputs = model(X_train)
        loss = criterion(outputs, y_train)
        loss.backward()
        optimizer.step()
        train_losses.append(loss.item())
        train_mae.append(nn.L1Loss()(outputs, y_train).item())
        train_rmse.append(torch.sqrt(nn.MSELoss()(outputs, y_train)).item())
        gradient_norms.append(torch.norm(torch.stack([torch.norm(p.grad.detach(), 2) for p in model.parameters() if p.grad is not None]).float(), 2).item())

        # Validation
        model.eval()
        with torch.no_grad():
            val_outputs = model(X_val)
            val_loss = criterion(val_outputs, y_val)
            val_losses.append(val_loss.item())
            val_mae.append(nn.L1Loss()(val_outputs, y_val).item())
            val_rmse.append(torch.sqrt(nn.MSELoss()(val_outputs, y_val)).item())

        # Print epoch statistics
        for param_group in optimizer.param_groups:
            current_lr = param_group["lr"]
        if (epoch + 1) % 250 == 0:
            print(
                f"Epoch [{epoch+1}/{epochs}], Train Loss: {loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Train MAE: {nn.L1Loss()(outputs, y_train).item():.4f}, Val MAE: {nn.L1Loss()(val_outputs, y_val).item():.4f}, Train RMSE: {torch.sqrt(nn.MSELoss()(outputs, y_train)).item():.4f}, Val RMSE: {torch.sqrt(nn.MSELoss()(val_outputs, y_val)).item():.4f}, LR: {current_lr:.6f}, Gradient Norm: {gradient_norms[-1]:.4f}"
            )

    return train_losses, val_losses, train_mae, val_mae, train_rmse, val_rmse, gradient_norms

# --- Model Evaluation ---
def evaluate_model(model, X, y):
    model.eval()
    with torch.no_grad():
        outputs = model(X)
        loss = nn.MSELoss()(outputs, y)
        mae = nn.L1Loss()(outputs, y)
        rmse = torch.sqrt(nn.MSELoss()(outputs, y))
        return loss.item(), mae.item(), rmse.item()

# --- SHAP Values Calculation ---
#def calculate_shap_values(model, X):
#    explainer = shap.Explainer(model)
#    shap_values = explainer(X)
#    return shap_values

# --- Main Function ---
def main():
    # Load data
    df = load_data("gpu_data.csv")

    # Preprocess data
    X, y, preprocessor = preprocess_data(df)

    # Split data into training and validation sets
    X_train, X_val, y_train, y_val = train_test_split(
        X, y, test_size=0.1, random_state=42
    )

    # Model Parameters
    nhead = 4
    nhid = 128
    nlayers = 4
    dropout = 0.125

    # Training Parameters
    epochs = 3000
    learning_rate = 0.00011

    # Initialize model
    model = GPUPerformanceModel(
        n_features=X.shape[2],
        nhead=nhead,
        nhid=nhid,
        nlayers=nlayers,
        dropout=dropout,
    )

    # Train model
    train_losses, val_losses, train_mae, val_mae, train_rmse, val_rmse, gradient_norms = train_model(
        model, X_train, y_train, X_val, y_val, epochs=epochs, learning_rate=learning_rate
    )

    # Evaluate model
    loss, mae, rmse = evaluate_model(model, X_val, y_val)
    print(f"Test Loss: {loss:.4f}, Test MAE: {mae:.4f}, Test RMSE: {rmse:.4f}")

    # Calculate SHAP values
    #shap_values = calculate_shap_values(model, X_val)

    # Save model and preprocessor
    model_name = "GPU-PERF-3.2"
    torch.save(model.state_dict(), f"{model_name}.pth")
    joblib.dump(preprocessor, f"{model_name}_preprocessor.pkl")

    # Save model configuration
    model_config = {
        "model_params": {
            "n_features": X.shape[2],
            "nhead": nhead,
            "nhid": nhid,
            "nlayers": nlayers,
            "dropout": dropout,
        },
        "training_params": {
            "epochs": epochs,
            "optimizer": "Adam",
            "learning_rate": learning_rate,
            "loss_function": "MSELoss",
        },
        "test_loss": loss,
        "test_mae": mae,
        "test_rmse": rmse,
    }
    with open(f"{model_name}.json", "w") as f:
        json.dump(model_config, f)

    # Plot training and validation loss curves
    plt.figure(figsize=(10, 5))
    plt.plot(train_losses, label="Training Loss")
    plt.plot(val_losses, label="Validation Loss")
    plt.xlabel("Epochs")
    plt.ylabel("Loss")
    plt.legend()
    plt.title("Training and Validation Loss Curves")
    plt.show()

    # Plot training and validation MAE curves
    plt.figure(figsize=(10, 5))
    plt.plot(train_mae, label="Training MAE")
    plt.plot(val_mae, label="Validation MAE")
    plt.xlabel("Epochs")
    plt.ylabel("MAE")
    plt.legend()
    plt.title("Training and Validation MAE Curves")
    plt.show()

    # Plot training and validation RMSE curves
    plt.figure(figsize=(10, 5))
    plt.plot(train_rmse, label="Training RMSE")
    plt.plot(val_rmse, label="Validation RMSE")
    plt.xlabel("Epochs")
    plt.ylabel("RMSE")
    plt.legend()
    plt.title("Training and Validation RMSE Curves")
    plt.show()

    # Plot gradient norms
    plt.figure(figsize=(10, 5))
    plt.plot(gradient_norms, label="Gradient Norms")
    plt.xlabel("Epochs")
    plt.ylabel("Gradient Norms")
    plt.legend()
    plt.title("Gradient Norms")
    plt.show()

    # Plot predicted vs actual performance
    plt.figure(figsize=(10, 5))
    plt.scatter(y_val.numpy(), model(X_val).numpy(), color="blue", label="Predictions", marker='x')
    plt.xlabel("Actual Performance")
    plt.ylabel("Predicted Performance")
    plt.legend()
    plt.title("Predicted vs Actual GPU Performance")
    plt.show()

if __name__ == "__main__":
    main()