Normalisation question

ljeonjko: <p>Hi all!</p>
<p>I’m using a generator:</p>
<pre><code class="lang-auto">class Downscale(nn.Module):
    def __init__(self, in_size, out_size, normalize = True, dropout = 0.0):
        super(Downscale, self).__init__()
        
        model = [nn.Conv2d(
            in_size,
            out_size,
            kernel_size = 4,
            stride = 2,
            padding = 1,
            bias = False
            )]
        
        if normalize:
            model.append(
                nn.BatchNorm2d(out_size, 0.8)
                )
            
        model.append(
            nn.LeakyReLU(0.2)
            )
        
        if dropout:
            
            model.append(
                nn.Dropout(dropout)
                )
        
        self.model = nn.Sequential(*model)
        
    def forward(self, x):
        return self.model(x)
##############################################################################       
# #############################################################################   
class Upscale(nn.Module):
    def __init__(self, in_size, out_size, dropout = 0.0):
        super(Upscale, self).__init__()
        
        model =[
            nn.ConvTranspose2d(
                in_size,
                out_size,
                kernel_size = 4,
                stride = 2,
                padding = 1,
                bias = False
                ),
            nn.BatchNorm2d(out_size, 0.8),
            nn.ReLU(inplace = True)
            ]
        
        if dropout:
            model.append(
                nn.Dropout(dropout)
                )
            
        self.model = nn.Sequential(*model)
        
    def forward(self, x, skip_input):
        x = self.model(x)
        out = torch.cat((x, skip_input), dim = 1)
        return out
##############################################################################       
# #############################################################################   

class Generator(nn.Module):
    def __init__(self, features_g, num_channels):
        super(Generator, self).__init__()
        self.features_g = features_g
        self.num_channels = num_channels
        self.optimizer = None
        
    def build(self):
        #  input: channels X 64 X 64
        self.down1 = Downscale(
            in_size = self.num_channels, 
            out_size = self.features_g, 
            normalize = False)
        
        # input: features_g X 32 X 32
        self.down2 = Downscale(
            in_size = self.features_g, 
            out_size = self.features_g * 2)
        
        
        # input: (features_g * 2) X 16 x 16
        self.down3 = Downscale(
            in_size = (self.features_g * 2 + self.num_channels),
            out_size = self.features_g * 4,
            dropout = 0.5
            )
        
        # input: (features_g * 4) X 8 X 8
        self.down4 = Downscale(
            in_size = self.features_g * 4,
            out_size = self.features_g * 8,
            dropout = 0.5
            )
        
        # input: (features_g * 8) X 4 X 4
        self.down5 = Downscale(
            in_size = self.features_g * 8,
            out_size = self.features_g * 8,
            dropout = 0.5
            )
        
        # input: (features_g * 8) X 2 X 2
        self.down6 = Downscale(
            in_size = self.features_g * 8,
            out_size = self.features_g * 8,
            dropout = 0.5
            )
        ## state: (features_g * 8) X 1 X 1 ##
        
        # input: (features_g * 8) X 1 X 1 
        self.up1 = Upscale(
            in_size = self.features_g * 8,
            out_size = self.features_g * 8,
            dropout = 0.5
            )

        # input: (features_g * 8) X 2 X 2
        self.up2 = Upscale(
            in_size = self.features_g * 16,
            out_size = self.features_g * 8, 
            dropout = 0.5
            )
        
        # input: (features_g * 8) X 4 X 4
        self.up3 = Upscale(
            in_size = self.features_g * 16,
            out_size = self.features_g * 4,
            dropout = 0.5
            )
    
        # input: (features_g * 4) X 8 X 8
        self.up4 = Upscale(
            in_size = self.features_g * 8,
            out_size = self.features_g * 2
            )
        
        # input: (features_g * 2) X 16 X 16
        self.up5 = Upscale(
            in_size = (self.features_g * 4 + self.num_channels),
            out_size = self.features_g
            )

        ## state: features_g X 32 X 32 ##
        
        final = [
            nn.Upsample(scale_factor = 2),
            
            # input: features_g X 64 X 64
            
            nn.Conv2d(
                in_channels = self.features_g * 2, 
                out_channels = self.num_channels,
                kernel_size = 3,
                stride = 1,
                padding = 1
                ),
            
            # input: num_channels X 64 X 64
            
            nn.Tanh()
            ]
         
        self.final = nn.Sequential(*final)
            
    def forward(self, input, constraint_map):
        
        d1 = self.down1(input)
        d2 = self.down2(d1)
        d2 = torch.cat((d2, constraint_map), dim = 1)
        d3 = self.down3(d2)
        d4 = self.down4(d3)
        d5 = self.down5(d4)
        d6 = self.down6(d5)
        u1 = self.up1(d6, d5)
        u2 = self.up2(u1, d4)
        u3 = self.up3(u2, d3)
        u4 = self.up4(u3, d2)
        u5 = self.up5(u4, d1)
        
        return self.final(u5)

    def define_optim(self, learning_rate, beta1):
        self.optimizer = optim.Adam(self.parameters(), lr = learning_rate, betas = (beta1, 0.999))
     
    @staticmethod    
    def init_weights(layers):
        classname = layers.__class__.__name__
        
        if classname.find('Conv') != -1:
            nn.init.normal_(layers.weight.data, 0.0, 0.02)
            
        elif classname.find('BatchNorm') != -1:
            nn.init.normal_(layers.weight.data, 1.0, 0.02)
            nn.init.constant_(layers.bias.data, 0)

</code></pre>
<p>Initially, it was trained with unormalised data, ranging from <code>[-1, ~10]</code>.  However, upon inspection, all the outputs were within <code>[-1, 1]</code> range, regardless that the data was  <code>[-1, ~10]</code>. Now, I’m feeding it normalised data <code>[0,1]</code>.</p>
<p>Question is, since the input data for traning is already normalised, the generator (regardless of <code>BatchNorm2d</code> or any other normalising function) should not alter the input data range/distribution/variance/mean/deviation in any way, shape or form?</p>
<p>Sorry for the long post, but this oversight of normalisation caused me a lot of additional work, and I’d like to avoid that in future trainings.</p>
<p>Thanks!</p>

ptrblck: <aside class="quote no-group" data-username="ljeonjko" data-post="1" data-topic="161292">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/l/b5ac83/48.png" class="avatar"> ljeonjko:</div>
<blockquote>
<p>Initially, it was trained with unormalised data, ranging from <code>[-1, ~10]</code>. However, upon inspection, all the outputs were within <code>[-1, 1]</code> range, regardless that the data was <code>[-1, ~10]</code>. Now, I’m feeding it normalised data <code>[0,1]</code>.</p>
</blockquote>
</aside>
<p>That’s expected since your last layer is an <code>nn.Tanh</code> which will create outputs in <code>[-1, 1]</code>.</p>
<aside class="quote no-group" data-username="ljeonjko" data-post="1" data-topic="161292">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/l/b5ac83/48.png" class="avatar"> ljeonjko:</div>
<blockquote>
<p>Question is, since the input data for traning is already normalised, the generator (regardless of <code>BatchNorm2d</code> or any other normalising function) should not alter the input data range/distribution/variance/mean/deviation in any way, shape or form?</p>
</blockquote>
</aside>
<p>Yes, the model should not change the input data stats and I’m unsure what the question exactly is.<br>
Did you see that the model changes the actual input data or are you wondering why the output is in <code>[-1 ,1]</code> (see previous point about the <code>tanh</code>)?</p>

ljeonjko: <p>Thanks!<br>
So the question would be (it’s a bit complicated to ask a singular question):</p>
<p>I have a real dataset, which I’ve normalised to <code>[0,1]</code> by doing</p>
<pre><code class="lang-auto">real_data_normalised = (real_data - real_data.min())/(real_data.max() - real_data.min())
</code></pre>
<p>I’m traning the generator by comparing real images to generated images:</p>
<pre><code class="lang-auto">generator_loss = loss(real_data_normalised, generated_data)
</code></pre>
<p>So, knowing that the real_dataset_nromalised is between  <code>[0,1]</code>, is it reasonable to expect that a well trained generator will produce values between <code>[0,1]</code>? If not, then I should normalisse the generated_data to <code>[0,1]</code> before the <code>generator_loss</code>, which is fine, but can I than retrieve the “true” values of the generated data by doing</p>
<pre><code class="lang-auto">"true"_generated_data = generated_data * (real_data.max() - real_data.min()) + real_data.min()
</code></pre>
<p>or will the distribution/variance/deviation be shifted?</p>
<p>Also, if I just remove the tanh, I should get output in thr same range as input once trained?</p>
<p>If you need any further clarification, please ask. You’ve assisted me a few times already, and I appreciate your tips and help very much. <img src="https://discuss.pytorch.org/images/emoji/apple/slight_smile.png?v=12" title=":slight_smile:" class="emoji" alt=":slight_smile:" loading="lazy" width="20" height="20"></p>