Hi, I've implemented the following AlexNet and train_model() function to train the model based on a images dataset. I believe that there is still room for some improvements and also Tuning and fixing few lines of code. Reduce the redundancy and modifications for better memory efficient.                                                             
# Define AlexNet and ImageDataset classes

from torch.optim.lr_scheduler import ReduceLROnPlateau

class AlexNet(nn.Module):
    def __init__(self):
        super(AlexNet, self).__init__()

        self.features = nn.Sequential(
            nn.Conv2d(3, 96, kernel_size=11, stride=4),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(96, 256, kernel_size=5, padding=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(256, 384, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(384, 384, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(384, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
        )

        self.classifier = nn.Sequential(
            nn.Dropout(p=0.5),
            nn.Linear(256 * 6 * 6, 4096),
            nn.ReLU(inplace=True),
            nn.Dropout(p=0.5),
            nn.Linear(4096, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, len(folder_names)),
        )

    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        return x                                                                                                                                                                                                                                                                                                                
 from torchvision import transforms
# Preprocessing transform
transform = transforms.Compose([
    transforms.Resize((227, 227)),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
batch_size = 32
image_dataset = ImageDataset(folder_names, transform) 
all_indices = np.arange(len(image_dataset))
train_indices, test_indices = train_test_split(all_indices, test_size=0.2, random_state=42, stratify=image_dataset.y)                                                                                           
 class ImageDataset(Dataset):
    def __init__(self, folder_names, transform=None):
        self.folder_names = folder_names
        self.transform = transform
        self.y = self.get_labels()
        
    def get_labels(self):
        labels = []
        for i, folder in enumerate(self.folder_names):
            labels.extend([i] * len(os.listdir(folder)))
        return labels
        
    def __len__(self):
        return sum(len(os.listdir(folder)) for folder in self.folder_names)
    
    def __getitem__(self, idx):
        for i, folder_name in enumerate(self.folder_names):
            if idx < len(os.listdir(folder_name)):
                file = os.listdir(folder_name)[idx]
                img = Image.open(os.path.join(folder_name, file))
                if self.transform:
                    img = self.transform(img)
                label = i
                return img, label
            idx -= len(os.listdir(folder_name))def train_model(model, train_loader, test_loader, epochs = None, learning_rate = None, optimization_technique = None,  patience=None, scheduler_patience=None,num_batches = None, **kwargs):
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(model.parameters(), lr=learning_rate)
   
    if optimization_technique == 'learning_rate_scheduler' and scheduler_patience:
        scheduler = ReduceLROnPlateau(optimizer, 'min', patience=scheduler_patience, verbose=True)

    train_losses = []
    train_accuracies = []
    test_losses = []
    test_accuracies = []

    best_loss = float('inf')
    stopping_counter = 0

    with_dropout = model.classifier[6]
    without_dropout = nn.Sequential(*[layer for layer in model.classifier if layer != with_dropout])

    for epoch in range(epochs):
        running_train_loss = 0.0
        running_train_acc = 0
        num_batches_train = 0
        
        for X_batch, y_batch in itertools.islice(train_loader, 0, num_batches):
        # for X_batch, y_batch in train_loader:
            optimizer.zero_grad()
            y_pred = model(X_batch)
            loss = criterion(y_pred, y_batch)
            loss.backward()
            optimizer.step()

            running_train_loss += loss.item()
            running_train_acc += accuracy_score(y_batch.numpy(), y_pred.argmax(dim=1).numpy())
            num_batches_train += 1

        train_losses.append(running_train_loss / num_batches_train)
        train_accuracies.append(running_train_acc / num_batches_train)

        # Testing segment
        running_test_loss = 0.0
        running_test_acc = 0
        num_batches_test = 0
        

        with torch.no_grad():
            for X_batch, y_batch in itertools.islice(test_loader, 0, num_batches):
    
        # with torch.no_grad():
        #     for X_batch, y_batch in test_loader:
                y_pred = model(X_batch)
                #y_pred_without_dropout = y_pred.view(y_pred.size(0), 256 * 6 * 6)
                #y_pred_without_dropout = without_dropout(y_pred)  # Use the classifier without dropout for testing
                # y_pred = y_pred.view(y_pred.size(0), 256 * 6 * 6)
                # y_pred = without_dropout(y_pred)  # Use the classifier without dropout for testing
                loss = criterion(y_pred, y_batch)

                running_test_loss += loss.item()
                running_test_acc += accuracy_score(y_batch.numpy(), y_pred.argmax(dim=1).numpy())
                num_batches_test += 1

        test_losses.append(running_test_loss / num_batches_test)
        test_accuracies.append(running_test_acc / num_batches_test)

        # Early stopping
        if optimization_technique == 'early_stopping' and patience:
            if test_losses[-1] < best_loss:
                best_loss = test_losses[-1]
                stopping_counter = 0
            else:
                stopping_counter += 1
            if stopping_counter > patience:
                print(f"Early stopping at epoch {epoch+1}/{epochs}")
                break

        # Learning rate scheduler
        if optimization_technique == 'learning_rate_scheduler' and scheduler_patience and scheduler:
            scheduler.step(test_losses[-1])

        if epoch % 10 == 0:
            print(f"Epoch: {epoch+1}/{epochs}, Training Loss: {train_losses[-1]}, Test Loss: {test_losses[-1]}, Training Accuracy: {train_accuracies[-1]}, Test Accuracy: {test_accuracies[-1]}")

    return train_losses, train_accuracies, test_losses, test_accuracies

# Train the model
train_loader = DataLoader(image_dataset, batch_size=batch_size, sampler=torch.utils.data.SubsetRandomSampler(train_indices))
test_loader = DataLoader(image_dataset, batch_size=batch_size, sampler=torch.utils.data.SubsetRandomSampler(test_indices))

alexnet = AlexNet()
train_losses, train_accuracies, test_losses, test_accuracies = train_model(
    alexnet, train_loader, test_loader, epochs=5, learning_rate=0.01, optimization_technique=None, patience=10, num_batches=50)