Fix augmentation settings sample-wise for ImageNet

AhmedHussKhalifa: <p>Hi everyone,</p>
<p>I’m working on calculating the logits for my entire training set to identify specific training parameters. To avoid mismatches between the logits used in training and those used for hyperparameter calculation, I need to keep the augmentation settings per sample fixed during both processes. If the augmentation settings vary, it could lead to inaccuracies in my results.</p>
<p>I have written this simple solution and I would like to know if it will affect any other parts of the training except the augmentation part.  Any tips or insights would be greatly appreciated!</p>
<pre><code class="lang-auto">import torch
import torchvision.transforms as transforms
from torchvision.datasets import CIFAR10
from torch.utils.data import DataLoader
from PIL import Image
import random
from torchvision import datasets
# Define criterion and optimizer
import torch.optim as optim
# Define the model
import torch.nn as nn
import torch.nn.functional as F

def get_deterministic_transforms(seed):
  # Set the random seed for reproducibility
  torch.manual_seed(seed)
  random.seed(seed)
  # Define your augmentation transforms here
  transform = transforms.Compose([
                                    transforms.RandomHorizontalFlip(),
                                    transforms.RandomRotation(10),
                                    transforms.ToTensor(),
                                ])
  return transform


class CIFAR10Instance(datasets.CIFAR10):
    """CIFAR100Instance Dataset.
    """
    def __init__(self, root, train=True,
                 transform=None, target_transform=None,
                 download=False):
        super().__init__(root=root, train=train, download=download,
                         transform=transform, target_transform=target_transform)
        self.seed = 10

    def __getitem__(self, index):
        img, target = self.data[index], self.targets[index]
        img = Image.fromarray(img)
        
        # Store current random state
        random_state = torch.get_rng_state()
        img = get_deterministic_transforms(index+self.seed)(img) 
        # Restore previous random state
        torch.set_rng_state(random_state)

        if self.transform is not None:
            img = self.transform(img)

        if self.target_transform is not None:
            target = self.target_transform(target)

        return img, target, index


# Create dataset and dataloader
train_dataset = CIFAR10Instance(root='./data', train=True, download=True)
train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)



class SimpleCNN(nn.Module):
    def __init__(self):
        super(SimpleCNN, self).__init__()
        self.conv1 = nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, padding=1)
        self.conv2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, padding=1)
        self.fc1 = nn.Linear(in_features=32*8*8, out_features=128)
        self.fc2 = nn.Linear(in_features=128, out_features=10)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2)
        x = x.view(-1, 32*8*8)  # Flatten the tensor
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return x

# Check if GPU is available
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model = SimpleCNN().to(device)

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

# Training loop
num_epochs = 5

idx = 1000

for epoch in range(num_epochs):
    train_loader.dataset.seed += 1
    for batch_idx, (data, target, index) in enumerate(train_loader):
        # Move data and target to GPU
        data, target = data.to(device), target.to(device)
        if idx in index:
            print(data[torch.nonzero(index==idx)[0]][0,0,0])
        
        
        # Forward pass
        output = model(data)
        loss = criterion(output, target)
        
        # Backward pass and optimization
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        if batch_idx % 100 == 0:
            print(f'Epoch [{epoch+1}/{num_epochs}], Step [{batch_idx+1}/{len(train_loader)}], Loss: {loss.item():.4f}')

</code></pre>