Understanding the effect of various normalization techniques

username: renken
cooked: <p>Hi, I am new to machine learning and i use a unet as a toy model in order to deepen my knowledge. I woud like to perform various normalization techniques in order to understand practically their effects on my grayscale dataset.</p>
<ol>
<li>
<p>Please could you tell me how many data normalization techniques for image processing are available on Pytorch ? Do we have any practical documentation on them for beginners ?</p>
</li>
<li>
<p>How can i implement them on my simple unet architecture ? I heard about Batch normalisation, layer normalisation, spectral normalisation, instance normalisation and weight normalisation. Where in my code is it more interesting to apply them please ?</p>
</li>
</ol>
<p>Thank you</p>
<pre><code class="lang-auto"># architecture +++++++++++++++++++++++++++++++++++++
def double_conv(in_channels, out_channels):                  
    return nn.Sequential(
        nn.Conv2d(in_channels, out_channels, 5, padding=2),
        nn.ReLU(inplace=True),
        nn.Conv2d(out_channels, out_channels, 5, padding=2),
        nn.ReLU(inplace=True)
    )   

class UNet(nn.Module):                  

    def __init__(self, n_class):        
        super().__init__()
        dim_0,dim_1,dim_2,dim_3,dim_4 = 1,64,128,256,512
        self.db_conv_1 = double_conv(dim_0, dim_1)
        self.db_conv_2 = double_conv(dim_1, dim_2)
        self.db_conv_3 = double_conv(dim_2, dim_3)
        self.db_conv_4 = double_conv(dim_3, dim_4)   
        self.maxpool = nn.MaxPool2d(2)                        
        self.upsample = nn.Upsample(scale_factor=2, mode='nearest')       
        self.db_transpose_conv_3 = double_conv(dim_3 + dim_4, dim_3)
        self.db_transpose_conv_2 = double_conv(dim_2 + dim_3, dim_2)
        self.db_transpose_conv_1 = double_conv(dim_2 + dim_1, dim_1)           
        self.conv_last = nn.Conv2d(dim_1, n_class, 1)      
        
        
    def forward(self, x):                             #
        conv1 = self.db_conv_1(x)
        x = self.maxpool(conv1)

        conv2 = self.db_conv_2(x)
        x = self.maxpool(conv2)
     
        conv3 = self.db_conv_3(x)
        x = self.maxpool(conv3)   

        conv4 = self.db_conv_4(x)                                                                              
        x = self.upsample(x) 

        x = torch.cat([x, conv3], dim=1)          # skips connections betweent layers 
        x = self.db_transpose_conv_3(x)
        x = self.upsample(x)  

        x = torch.cat([x, conv2], dim=1)     
        x = self.db_transpose_conv_2(x)
        x = self.upsample(x)  

        x = torch.cat([x, conv1], dim=1)   
        x = self.db_transpose_conv_1(x)
    
        x = self.conv_last(x)

        result = torch.sigmoid(x)
        
        return result

# training function ++++++++++++++++++++++++++++++++++
def train_model(model, optimizer, scheduler, num_epochs=25):
    best_model_wts = copy.deepcopy(model.state_dict())
    best_loss = 1e10
    train_loss,val_loss = [],[]
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch+1,num_epochs))
        print('-' * 10)
        since = time.time()
        # Each epoch has a training and validation phase
        for phase in ['train','val']:
            if phase == 'train':
                # print value of learning rate
                for param_group in optimizer.param_groups:
                    print("LR", param_group['lr'])   
                # Set model to training mode           
                model.train()  
            else:
                # Set model to evaluate mode
                model.eval()   

            metrics, epoch_samples = defaultdict(float), 0
            for inputs, labels in dataloaders[phase]:
                inputs, labels = (inputs.type(torch.FloatTensor)).to(device), labels.to(device)         
                # zero the parameter gradients before each weight update and avoid loss accumulation
                optimizer.zero_grad()
                # forward: make a prediction
                with torch.set_grad_enabled(phase == 'train'):
                    # compute outputs
                    outputs = model(inputs)
                    # track losses for each phase
                    if phase=='train':
                      loss = calc_loss(outputs, labels, metrics)
                    else:
                      loss = calc_loss(outputs, labels, metrics)

                    # backward: backpropagation to compute loss
                    # then optimize: tune hyper parameters of the model
                    # and call scheduler for l_r (only during training phase)
                    if phase == 'train':
                        loss.backward(); optimizer.step(); scheduler.step()
                # count the number of samples used in this training phase
                epoch_samples += inputs.size(0)

            # print and save loss for model actualisation 
            print_metrics(metrics, epoch_samples, phase)
            epoch_loss = metrics['loss'] / epoch_samples
            if phase == 'train':
              train_loss.append(metrics['loss'] / epoch_samples)
            else:
              val_loss.append(metrics['loss'] / epoch_samples)
            # deep copy the model
            if phase == 'val' and epoch_loss &lt; best_loss:
                print("saving best model")
                best_loss = epoch_loss
                best_model_wts = copy.deepcopy(model.state_dict())

        time_elapsed = time.time() - since
        print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
        r_t = (time_elapsed % 60)*num_epochs-(time_elapsed % 60)*(epoch+1)
        r_t = r_t/60
        print("Remaining time: %i minute(s)" %r_t )

    print('Best val loss: {:4f}'.format(best_loss))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model, np.array(train_loss), np.array(val_loss)
</code></pre>

username: suraj.pt
cooked: <p>Hi, welcome! Take a look at <a href="https://pytorch.org/docs/stable/nn.html#normalization-layers" class="inline-onebox">torch.nn — PyTorch 1.11.0 documentation</a> and click into any of the layers to understand more. Clicking on <code>[SOURCE]</code> near the API definition will show you the implementation in the codebase.</p>