Please fix the theta = []: import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.nn.init as init

def initialize_random_params(model, m):
    omega_params = []
    for _ in range(m):
        param_set = []
        for param in model.parameters():
            # Create a new Parameter with the same shape and random initialization
            rand_param = torch.empty_like(param)
            if param.ndimension() > 1:  # Weights
                init.xavier_uniform_(rand_param)
            else:  # Biases
                init.zeros_(rand_param)
            param_set.append(nn.Parameter(rand_param.requires_grad_(True)))  # Wrap in Parameter
        omega_params.append(param_set)
    return omega_params

def interpolate_parameters(alpha, w1, w2):
    return (1 - alpha) * w1 + alpha * w2

# Cosine similarity regularization (cos^2(ωj, ωk))
def cosine_similarity_regularization(wj, wk):
    # Ensure that wj and wk are tensors and concatenate if they are lists
    wj_tensor = torch.cat([param.view(-1) for param in wj])
    wk_tensor = torch.cat([param.view(-1) for param in wk])
    
    # Calculate cosine similarity
    cos_sim = F.cosine_similarity(wj_tensor.view(1, -1), wk_tensor.view(1, -1))
    
    return cos_sim.pow(2)

# Validation function
def validate_ensemble(model, val_loader, omega_params, loss_fn, beta, alpha=0.2):
    model.eval()  # Set model to evaluation mode
    val_loss = 0
    correct_single = 0
    correct_ensemble = 0
    total = 0

    with torch.no_grad():
        for x, y in val_loader:
            x, y = x.to(config.device), y.to(config.device)
            
            # Sample j, k from {1, ..., m} without replacement
            j, k = torch.randint(0, len(omega_params), (2,)).tolist()

            # Compute θ ← P(α; {ωi}m i=1) and θ_1 ← P(1 − α; {ωi}m i=1)
            theta_alpha = [interpolate_parameters(alpha, omega_params[j][i], omega_params[k][i])
                           for i in range(len(omega_params[j]))]
            theta_1_minus_alpha = [interpolate_parameters(1 - alpha, omega_params[j][i], omega_params[k][i])
                                   for i in range(len(omega_params[j]))]

            # Load mixed parameters θ_alpha into the model
            with torch.no_grad():
                for param, mixed_param in zip(model.parameters(), theta_alpha):
                    param.copy_(mixed_param)

            # Forward pass ˆy ← f (x, θ_alpha)
            y_pred_alpha = model(x)

            # Load mixed parameters θ_1_minus_alpha into the model
            with torch.no_grad():
                for param, mixed_param in zip(model.parameters(), theta_1_minus_alpha):
                    param.copy_(mixed_param)

            # Forward pass ˆy ← f (x, θ_1_minus_alpha)
            y_pred_1_minus_alpha = model(x)

            # Compute supervised loss on θ_alpha model
            loss = loss_fn(y_pred_alpha, y)
            val_loss += loss.item()

            # Compute accuracy for f(x, P(α))
            _, predicted_single = torch.max(y_pred_alpha, 1)
            correct_single += (predicted_single == y).sum().item()

            # Ensemble the outputs by averaging logits (before applying softmax)
            ensemble_logits = (y_pred_alpha + y_pred_1_minus_alpha) / 2

            # Compute accuracy for the ensemble model
            _, predicted_ensemble = torch.max(ensemble_logits, 1)
            correct_ensemble += (predicted_ensemble == y).sum().item()

            total += y.size(0)

    val_loss /= len(val_loader)  # Average loss
    accuracy_single = correct_single / total
    accuracy_ensemble = correct_ensemble / total

    return val_loss, accuracy_single, accuracy_ensemble

def create_optimizers(omega_params, lr, momentum):
    optimizers = []
    for params in omega_params:
        # Create an SGD optimizer for each set of parameters
        optimizer = AdEMAMix(params,lr=lr) #optim.SGD(params, lr=lr, momentum=momentum)
        optimizers.append(optimizer)
    return optimizers

# Training function implementing TrainSubspace algorithm with validation and ensemble
def train_subspace(model, train_loader, val_loader, loss_fn, omega_params, beta, lr, momentum, epochs=10):
    optimizers = create_optimizers(omega_params, lr, momentum)

    for epoch in range(epochs):
        model.train()  # Set model to training mode
        train_loss = 0
        correct = 0
        for batch_idx, (x, y) in enumerate(train_loader):
            x, y = x.to(config.device), y.to(config.device)

            # Sample α uniformly from Λ (here, we assume Λ = [0, 1])
            alpha = torch.rand(1).item()
            #params = interpolate_parameters(alpha, omega_params)

            # Sample j, k from {1, ..., m} without replacement
            j, k = torch.randint(0, len(omega_params), (2,)).tolist()

            # Compute θ ← P(α; {ωi}m i=1)
            #theta = [interpolate_parameters(alpha, omega_params[j][i], omega_params[k][i])
            #         for i in range(len(omega_params[j]))]
            
            theta = [] #BASED ON THE PARAMETERS IN THE OPTIMIZER, WE NEED TO INTERPOLATE THETA LINEARLY USING THE INTERPOLATE FUNCTION
            
            with torch.no_grad():
                for param, mixed_param in zip(model.parameters(), theta):
                        param.copy_(mixed_param)

            # Forward pass ˆy ← f (x, θ)
            y_pred = model(x)
            pred = y_pred.argmax(dim=1, keepdim=True)
            correct += pred.eq(y.view_as(pred)).sum().item()


            # Compute supervised loss
            loss_supervised = loss_fn(y_pred, y)

            # Compute regularization term β cos²(ωj , ωk)
            loss_regularization = beta * cosine_similarity_regularization(omega_params[j], omega_params[k])

            # Total loss
            loss = loss_supervised + loss_regularization
            train_loss += loss.item()
            
            for i in range(len(optimizers)):
                optimizer = optimizers[i]
                
                if i != len(optimizers)-1:
                    retain_graph = True
                else:
                    retain_graph = False
                optimizer.zero_grad()
                loss.backward(retain_graph=retain_graph)
                optimizer.step()

            #for param_idx, param_group in enumerate(optimizer.param_groups):
            #    for param in param_group['params']:
            #        # Directly assign optimizer parameters to omega_params
            #        omega_params[i][param_idx].data = param.data.detach() 
        # Validate after each epoch, with ensembling
        val_loss, accuracy_single, accuracy_ensemble = validate_ensemble(model, val_loader, omega_params, loss_fn, beta)
        train_accuracy = 100. * correct / len(train_loader.dataset)
        train_loss /= len(train_loader.dataset)
        print(f'\nEpoch: {epoch+1}\tTraining Loss: {train_loss:.6f}\tTraining Accuracy: {train_accuracy:.2f}%\t'
              f'Validation Loss: {val_loss:.6f}\tValidation Accuracy: {accuracy_ensemble:.2f}%\n')

# Configuration setup
class Config:
    def __init__(self):
        self.batch_size = 256
        self.learning_rate = 0.001
        self.epochs = 100
        self.model_name = 'SOPCNN'
        self.num_classes = 10
        self.data_path = './data'
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        self.loss = nn.CrossEntropyLoss()

config = Config()
dataset = "MNIST"

model = SOPCNN(num_classes=10).to(config.device)

# Define ω1, ω2, ..., ωm (initialize randomly, copying model parameters)
m = 2
omega_params = initialize_random_params(model, m)
loss_fn = nn.CrossEntropyLoss()
beta = 0 #0.1
lr = 0.001
momentum = 0.9

# Example data loaders (you should replace `get_train_loader` and `get_test_loader` with actual implementations)
train_loader = get_train_loader(config.batch_size, config.data_path, dataset=dataset)
val_loader = get_test_loader(config.batch_size, config.data_path, dataset=dataset)

# Train the model and validate
train_subspace(model, train_loader, val_loader, loss_fn, omega_params, beta, lr, momentum, epochs=25)