main.py

# %matplotlib inline
# optuna-dashboard sqlite:///db.sqlite3
# https://github.com/optuna/optuna-dashboard

import optuna   # Used for the hyperparameter tuning, because cba anymore to do in any other way.
import gc
import torch
from torch.utils import data
import torchvision.datasets as datasets
from torchvision import transforms
from torch import nn
import random
import my_utils as mu
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')



train_trans =  transforms.Compose([
    # add transformations, and data augmentation, to increase batch size and increase generalization.
    # transforms.FiveCrop(size=(32,32)),  # might remove this.
    transforms.RandomPerspective(distortion_scale=0.6, p=0.4),
    transforms.GaussianBlur(kernel_size=(5, 11), sigma=(0.1, 0.2)),
    transforms.RandomRotation(degrees=(-8, 8)),
    transforms.ToTensor(),
    # transforms.Normalize((0.49139968, 0.48215827 ,0.44653124), (0.24703233, 0.24348505, 0.26158768))
])

test_trans = transforms.Compose([
    # This is all we need for the normalization of the model.
    transforms.ToTensor(),
    # transforms.Normalize((0.49139968, 0.48215827, 0.44653124), (0.24703233, 0.24348505, 0.26158768))
])


# Required for data to be in Tensor form not PIL Image
trans = [transforms.ToTensor()]
trans = transforms.Compose(trans)

cifar_trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=train_trans)
cifar_testset = datasets.CIFAR10(root='./data', train=False, download=True, transform=test_trans)
# 60,000 32x32 color images in 10 different classes

# batch_size = 15
# data_iter =  data.DataLoader(cifar_trainset, batch_size, shuffle=True)
# test_iter = data.DataLoader(cifar_testset, batch_size, shuffle=True)

print("Read dataset and create dataloaders - 5%")


def SpatialAveragePool(X):
    return torch.mean(X, dim=[2,3])


def init_weights(m):
    if type(m) == nn.Linear or type(m) == nn.Conv2d:
        torch.nn.init.xavier_uniform_(m.weight)         


loss = nn.CrossEntropyLoss()


class MakiNet(nn.Module):
    def __init__(self, conv_arch, num_classes, dropout_rate=0.0001):  # conv_arch: 
        super(MakiNet, self).__init__()
        self.out_classes = num_classes
        self.conv_arch = conv_arch
        k= 0
        for i, (num_conv, in_channels, out_channels) in enumerate(conv_arch):
            self.add_module(f"maki_block{i}", MakiBlock(num_conv, in_channels, out_channels, dropout_rate))
            # input_channels = out_channels
            # k = out_channels * (32-(2*len(conv_arch))) * (32-(2*len(conv_arch)))
            k = out_channels
        print(str(k) + " parameters")

        self.classifier = nn.Sequential(
            nn.Flatten(),
            nn.Linear(k, 75),
            nn.Dropout(p=dropout_rate),
            nn.ReLU(),
            nn.Linear(75, 25),
            nn.Dropout(p=dropout_rate),
            nn.ReLU(),
            nn.Linear(25, num_classes)
        )
        
    def forward(self, x):
        out = x
        # print(f"number of blocks: {len(self.conv_arch)}")
        for i in range(len(self.conv_arch)):
            out = self._modules[f"maki_block{i}"](out)
        out = SpatialAveragePool(out)
        out = self.classifier(out)
        return out
    

class MakiBlock(nn.Module):
    def __init__(self, num_conv, input_channels, output_channels, dropout_rate):
        super(MakiBlock, self).__init__()
        self.num_convs = num_conv
        self.linear = nn.Linear(input_channels, num_conv, bias=False)
        self.relu = nn.ReLU()
        #self.max = nn.MaxPool2d(kernel_size=3, padding=1, stride=1)
        #self.avg = nn.AvgPool2d(kernel_size=3, padding=1, stride=1)
        self.dropout = nn.Dropout(p=dropout_rate)
        
        for i in range(num_conv):
            # add convolution layer
            self.add_module(f"conv{i}", nn.Conv2d(input_channels, output_channels, kernel_size=5, padding=1, stride=1))
            # add batch norm layer
            self.add_module(f"batch_norm{i}", nn.BatchNorm2d(output_channels))


    def forward(self, x):
        # apply the linear model x, to a number of outputs.
        x = x.to(device)
        # Initial MLP part. 
        avg_out = SpatialAveragePool(x)
        avg_out = avg_out.to(device)
        
        lin_out = self.linear(avg_out)
        lin_out - self.dropout(lin_out)
        
        a = self.relu(lin_out)  # a vector.     
        
        
        total_output = []
        for j in range(self.num_convs):

            out = self._modules[f"conv{j}"](x)
            out = self._modules[f"batch_norm{j}"](out)
            out = self.dropout(out)
            out = self.relu(out)
            #  out = self.max(out)  # removing it, as shown ineffective.
            s = a[:, j].unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) * out
            total_output.append(s)
        
        total_output = torch.stack(total_output, dim=0)  
        out = torch.sum(total_output, dim=0) 
        return torch.Tensor(out)  
    
print("Create the model - 40%")


# def train_model(net, train_iter, test_iter, num_epochs, lr, wd=1e-9, device=device, param_dict=None):
def train_model(trail, study, device=device):
    gc.collect()
    torch.cuda.empty_cache()
    gpu_memory = torch.cuda.memory_allocated(device='cuda:0')
    print(f"GPU memory allocated: {gpu_memory}")
    # To be completed.
    try:
        batch_size = trail.suggest_int("batch_size", 32, 256)
        train_iter = data.DataLoader(cifar_trainset, batch_size, shuffle=True)
        test_iter = data.DataLoader(cifar_testset, batch_size, shuffle=True)

        # dropout_rate = trail.suggest_float("dropout_rate", 1e-5, 1e-1, log=True)
        dropout_rate = 0.15

        #number_of_layers = trail.suggest_int("number_of_layers", 3, 8)
        #number_of_channels = trail.suggest_int("number_of_channels", 50, 200)
        # number_of_layers = 4
        # number_of_channels = 10

        # num_conv = trail.suggest_int(3, 5)  # 3, try 12 next after this.
        num_conv = 3
        #model_arch = [
        #    [num_conv, 3, number_of_channels],  # num_conv, in_channels, out_channels
        #]
        #for i in range(number_of_layers):
        #    model_arch.append([num_conv, number_of_channels, number_of_channels])

        model_arch = [
            [3, 3, 120],  # num_conv, in_channels, out_channels
            [3, 120, 100],
            [3, 100, 80],
        ]
        net = MakiNet(model_arch, 10, dropout_rate=dropout_rate)
        net.to(device)
        #state_dict = torch.load(f"optuna_coursework_multi_arch_hyper_maki_net_4_10_0.713.params")
        #net.load_state_dict(state_dict)
        net.apply(init_weights)

        lr = trail.suggest_float("lr", 1e-5, 9e-1, log=True)
        # wd = trail.suggest_float("wd", 1e-9, 1e-1, log=True)
        wd=0
        optimizer = torch.optim.SGD(net.parameters(), lr=lr, weight_decay=wd)
        loss = nn.CrossEntropyLoss()
        timer = mu.Timer()

        num_epochs = trail.suggest_int("num_epochs", 20, 50)  # 10, 40

        metric = mu.Accumulator(3)  # train_loss, train_acc, num_examples
        train_loss = 0
        train_acc = 0
        for epoch in range(num_epochs):
            print(f"Epoch: {epoch}/ {num_epochs}")
            for i, (X, y) in enumerate(train_iter):
                timer.start()
                net.train()
                optimizer.zero_grad()
                X, y = X.to(device), y.to(device)
                y_hat = net(X)
                l = loss(y_hat, y)
                l.backward()
                optimizer.step()
                with torch.no_grad():
                    metric.add(l*X.shape[0], mu.accuracy(y_hat, y), X.shape[0])
                timer.stop()
                train_loss, train_acc = metric[0]/metric[2], metric[1]/metric[2]
                if (i+1) % 50 == 0:
                    print(f'batch {i+1}, train loss {train_loss:.3f}, train acc {train_acc:.3f}')
            test_acc = mu.evaluate_accuracy_gpu(net, test_iter)
            print(f'Test Accuracy for epoch {epoch+1} is {test_acc:.3f}')

            if epoch == 5 and test_acc <= 0.1:
            #     # Stop the trial if the test accuracy is less than 0.1 after 10 epochs. To save time.
                raise optuna.exceptions.TrialPruned()

        test_acc = mu.evaluate_accuracy_gpu(net, test_iter)
        test_acc_delta = test_acc - train_acc
        if test_acc_delta < -0.25:
            # overfitting of more than 25%, prune.
            raise optuna.exceptions.TrialPruned()

        try:
            if test_acc > study.best_trials[0].values[0]:
                torch.save(net.state_dict(), f"attempt6{test_acc:.3f}.params")
        except IndexError:
            print("No best trial yet.")
            torch.save(net.state_dict(), f"attempt6{test_acc:.3f}.params")
    except Exception as e:
        print("Exception occurred")
        return optuna.exceptions.OptunaError(f"Exception occurred during training. {e}")
    return test_acc  # , train_loss, test_acc_delta


if __name__ == "__main__":



    # model_arch = [
    #     [6, 3, 12],  # num_conv, in_channels, out_channels  max 73% no aug
    #     [4, 12, 15],
    #     [4, 15, 12],
    #     [5, 12, 9],
    # ]
    #
    # model_arch2 = [
    #     [3, 3, 5],  # num_conv, in_channels, out_channels
    #     [3, 5, 6],
    #     [3, 6, 3],
    #     # [2, 6, 3]
    # ]

    #study = optuna.create_study(study_name="attempt6", storage="sqlite:///db.sqlite3", directions=["maximize"])  # maximise the test accuracy.
    study = optuna.load_study(study_name="attempt6", storage="sqlite:///db.sqlite3")
    # print(f"Study Attributes: {study.user_attrs}")
    #print(f"{study.best_trials[0].values[0]} - {study.best_trials[0].params}")

    study.optimize(lambda trial: train_model(trial, study, device), n_trials=1000)  # 2 hours.

    # All task runs up to run 25, are with transformer simple model, not with data augmentation.

    # optuna-dashboard sqlite:///db.sqlite3


    # test_acc, train_loss, test_acc_delta