Negative loss for BC learning

username: NickOTeen
cooked: <p>I am pretty new to PyTorch and I am trying to implement BC learning for images as in this paper <a href="https://arxiv.org/pdf/1711.10284.pdf" rel="nofollow noopener">https://arxiv.org/pdf/1711.10284.pdf</a></p>
<p>What I’m basically trying to is to mix 2 images and their one-hot encoded labels with a random ratio and then learn the mixing ratio.</p>
<p>My model looks as follows:</p>
<pre><code class="lang-auto">class ConvNet(nn.Module):
    def __init__(self, n_classes):
        super(ConvNet, self).__init__()
        self.model = nn.Sequential(OrderedDict([
            ('conv1', ConvBNReLu(3, 32, 3, padding=1)),
            ('conv2', ConvBNReLu(32, 32, 3, padding=1)),
            ('max_pool1', nn.MaxPool2d(2, ceil_mode=True)),
            ('conv3', ConvBNReLu(32, 64, 3, padding=1)),
            ('conv4', ConvBNReLu(64, 64, 3, padding=1)),
            ('max_pool2', nn.MaxPool2d(2, ceil_mode=True)),
            ('conv5', ConvBNReLu(64, 128, 3, padding=1)),
            ('conv6', ConvBNReLu(128, 128, 3, padding=1)),
            ('conv7', ConvBNReLu(128, 128, 3, padding=1)),
            ('conv8', ConvBNReLu(128, 128, 3, padding=1)),
            ('max_pool3', nn.MaxPool2d(2, ceil_mode=True)),
            ('flatten', Flatten()),
            ('fc4', nn.Linear(in_features=128 * 4 * 4, out_features=512, bias=True)),
            ('relu5', nn.ReLU()),
            ('dropout5', nn.Dropout()),
            ('fc5', nn.Linear(512, 512)),
            ('relu6', nn.ReLU()),
            ('dropout6', nn.Dropout()),
            ('fc6', nn.Linear(512, n_classes)),
            ('softmax', nn.Softmax(dim=-1)) 
        ]))

    def forward(self, inp):
        return self.model(inp)

</code></pre>
<p>Then my learner like this:</p>
<pre><code class="lang-auto">criterion = nn.KLDivLoss()
learning_rate = 0.01
optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9, nesterov=True, weight_decay=5e-4)
scheduler = MultiStepLR(optimizer, milestones=[30, 60, 90], gamma=0.1)
loss_values = []
PATH = './BCplus_epoch10.pth'

for epoch in range(10):  # Number of epochs (loops over dataset)
  epoch_loss = 0.0
  running_loss = 0.0
  input = []
  labels = []
  for i, data in enumerate(trainloader, 0):
      # get the inputs; data is a list of [inputs, labels]
      
      if i % 3 == 2: # batch size of 3 images to ConvNet
        input = torch.stack(input) # stack tensors
        labels = torch.stack(labels)

        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(input)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics every 200 loops
        epoch_loss += loss.item() * images_cifar[0][0].size(0)
        running_loss += loss.item()
        input = [] # clear input
        labels = [] # clear labels

      images_cifar, labels_cifar = data

      images_mix, labels_mix = mix(images_cifar[0], images_cifar[1], labels_cifar[0], labels_cifar[1], training,
                                    True)
      input.append(images_mix)
      labels.append(labels_mix)



      if i % 200 == 199:
          print('[%d, %5d] loss: %.3f' %
                (epoch + 1, i + 1, running_loss / 200))
          running_loss = 0.0
  loss_values.append(epoch_loss / 25000) # 25000 images downloaded
  print(epoch_loss)
  scheduler.step()

</code></pre>
<p>And lastly the mixing like this:</p>
<pre><code class="lang-auto">def preprocess(image, optplus, train):
    if optplus:
        normalizer = zero_mean
        mean = np.array([4.60, 2.24, -6.84])
        std = np.array([55.9, 53.7, 56.5])
    else:
        normalizer = normalize
        mean = np.array([125.3, 123.0, 113.9])
        std = np.array([63.0, 62.1, 66.7])
    if train:
        image = normalize(image, mean, std)
        image = horizontal_flip(image)
        image = padding(image, 4)
        image = random_crop(image, 32)

    else:
        image = normalize(image, mean, std)

    return image


def mix(image1, image2, label1, label2, optplus, train):
    image1 = tensor_to_numpy(image1)
    image2 = tensor_to_numpy(image2)
    image1 = preprocess(image1, optplus, train)
    image2 = preprocess(image2, optplus, train)
    image1 = torch.from_numpy(image1).float().to(device)
    image2 = torch.from_numpy(image2).float().to(device)
    label1 = label1.to(device)
    label2 = label1.to(device)

    # Mix two images
    r = torch.rand(1).to(device)
    if optplus:
        g1 = torch.std(image1).to(device)
        g2 = torch.std(image2).to(device)
        p = (1.0 / (1 + g1 / g2 * (1 - r) / r)).to(device)
        image = ((image1 * p + image2 * (1 - p)) / torch.sqrt(p ** 2 + (1 - p) ** 2)).to(device)
    else:
        image = (image1 * r + image2 * (1 - r)).to(device)

    # Mix two labels
    eye = torch.eye(nClasses).to(device)
    label = (eye[label1] * r + eye[label2] * (1 - r)).to(device)

    return image, label

</code></pre>
<p>I am trying to stack 3 images together and then pass them through the model. However when doing this I get a negative loss using KL divergence. I am using this loss function since it is what they use in the paper. Any idea what I am doing incorrectly here?</p>

username: ptrblck
cooked: <p>Based on the <a href="https://pytorch.org/docs/stable/nn.html?highlight=kldiv#torch.nn.KLDivLoss">docs</a> your output and target might be wrong:</p>
<blockquote>
<p>As with <a href="https://pytorch.org/docs/stable/nn.html?highlight=kldiv#torch.nn.NLLLoss"> <code>NLLLoss</code> </a>, the input given is expected to contain  <em>log-probabilities</em>  and is not restricted to a 2D Tensor. The targets are given as  <em>probabilities</em>  (i.e. without taking the logarithm).</p>
</blockquote>
<p>You should probably change the last <code>nn.Softmax</code> activation to <code>nn.LogSoftmax</code> and make sure the target values are a probability distribution.</p>

username: NickOTeen
cooked: <p>Thank you for your answer.<br>
I tried setting the Softmax to a LogSoftmax and the KL divergence error is indeed positive now. The target values look something like this</p>
<pre><code class="lang-auto">tensor([[0.000 0.000 0.149 0.000 0.000 0.000 0.851 0.000 0.000 0.000] ... (and then 2 more)]])
</code></pre>
<p>Which should be correct since they add up to 1, right? After chancing the Softmax to LogSoftmax however, the output values look something like this</p>
<pre><code class="lang-auto">tensor([[-2.129 -2.201 -2.1317 ....
</code></pre>
<p>Which is not a normal probability distribution anymore. Should anything be changed to the target values as well?</p>
<p>Another problem I had is in the network architecture, in the paper they use</p>
<pre><code class="lang-auto">('conv1', ConvBNReLu(3, 64, 3, padding=1)),('conv2', ConvBNReLu(64, 64, 3, padding=1)) ...
</code></pre>
<p>which gave me the error:</p>
<pre><code class="lang-auto">RuntimeError: Given groups=1, weight of size 32 32 3 3, expected input[3, 64, 32, 32] to have 32 channels, but got 64 channels instead

</code></pre>
<p>Which was solved by halving all out_channel values, however the authors of the paper do use a higher number of filters, looking at Table 7 in the paper. What am I doing wrong there?</p>

username: ptrblck
cooked: <aside class="quote no-group" data-username="NickOTeen" data-post="3" data-topic="73657">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/nickoteen/48/21839_2.png" class="avatar"> NickOTeen:</div>
<blockquote>
<p>Which is not a normal probability distribution anymore. Should anything be changed to the target values as well?</p>
</blockquote>
</aside>
<p>These values are representing now log probabilities, so it should be fine.</p>
<aside class="quote no-group" data-username="NickOTeen" data-post="3" data-topic="73657">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/nickoteen/48/21839_2.png" class="avatar"> NickOTeen:</div>
<blockquote>
<p>Another problem I had is in the network architecture</p>
</blockquote>
</aside>
<p>Could you post the definition of <code>ConvBNReLU</code>, please?</p>

username: NickOTeen
cooked: <p>I think the criterion and model seems to work now using a standard method now (so without using the mixing class), so thanks a lot for that! However when I do use the mixing function, the network doesn’t seem to learn anything at all.<br>
I’m trying to follow the paper more closely so I took a batch size of 128, and mix them together to 64 images. This does not decrease the loss during training at all however.<br>
The train function looks like this:</p>
<pre><code class="lang-auto">criterion = nn.KLDivLoss()
learning_rate = 0.1
optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9, nesterov=True, weight_decay=5e-4)
scheduler = MultiStepLR(optimizer, milestones=[30, 60, 90], gamma=0.1)
loss_values = []
PATH = '/content/drive/My Drive/CS4240 - Deep Learning/Reproducibility project/Results/BCplus_epoch20.pth'

for epoch in range(20):  # Number of epochs (loops over dataset)
  epoch_loss = 0.0
  running_loss = 0.0
  inputs = []
  labels = []
  for i, data in enumerate(trainloader, 0):      

      images_cifar, labels_cifar = data
      inputs = [] # clear input
      labels = [] # clear labels
      for j in range(0,len(data),2):

        images_mix, labels_mix = mix(images_cifar[j], images_cifar[j+1], labels_cifar[j], labels_cifar[j+1], False,
                                    True)
        inputs.append(images_mix)
        labels.append(labels_mix)

      inputs = torch.stack(inputs) # stack tensors
      labels = torch.stack(labels)

      optimizer.zero_grad()

      # forward + backward + optimize
      outputs = net(inputs)
      loss = criterion(outputs, labels) # calculate loss
      optimizer.step() # update weights

      # print statistics every 200 loops
      running_loss += loss.item()

      if i % 200 == 199:
          print('[%d, %5d] loss: %.3f' %
                (epoch + 1, i + 1, running_loss / 200))
          running_loss = 0.0

  scheduler.step()

</code></pre>
<p>With mixing function (again)</p>
<pre><code class="lang-auto">def preprocess(image, optplus, train):
    if optplus:
        normalizer = zero_mean
        mean = np.array([4.60, 2.24, -6.84])
        std = np.array([55.9, 53.7, 56.5])
    else:
        normalizer = normalize
        mean = np.array([125.3, 123.0, 113.9])
        std = np.array([63.0, 62.1, 66.7])
    if train:
        image = normalize(image, mean, std)
        image = horizontal_flip(image)
        image = padding(image, 4)
        image = random_crop(image, 32)

    else:
        image = normalize(image, mean, std)

    return image


def mix(image1, image2, label1, label2, optplus, train):
    image1 = tensor_to_numpy(image1)
    image2 = tensor_to_numpy(image2)
    image1 = preprocess(image1, optplus, train)
    image2 = preprocess(image2, optplus, train)
    image1 = torch.from_numpy(image1).float()
    image2 = torch.from_numpy(image2).float()
    label1 = label1.to(device)
    label2 = label2.to(device)
    image1 = image1.to(device)
    image2 = image2.to(device)
    # Mix two images
    r = torch.rand(1).to(device)
    if optplus:
        g1 = torch.std(image1).to(device)
        g2 = torch.std(image2).to(device)
        p = (1.0 / (1 + g1 / g2 * (1 - r) / r)).to(device)
        image = ((image1 * p + image2 * (1 - p)) / torch.sqrt(p ** 2 + (1 - p) ** 2)).to(device)
    else:
        image = (image1 * r + image2 * (1 - r)).to(device)

    # Mix two labels
    eye = torch.eye(nClasses).to(device)
    label = (eye[label1] * r + eye[label2] * (1 - r)).to(device)

    return image, label
</code></pre>
<p>When training the network like this it does not reduce the loss at all, however when removing the mixing function from the trainer it does learn something, so something must be going wrong when mixing the images and labels. Is there something I am doing wrong in the mixing?</p>

username: ptrblck
cooked: <p>I’m not familiar with the paper, but I’m not sure how your <code>preprocess</code> method works exactly.</p>
<p>Assuming <code>optplus</code> uses the “mixing” function, the mean seems to differ a lot.<br>
Also in the <code>if train</code> block, <code>normalize</code> is used, while <code>normalizer</code> doesn’t seem to be used at all.</p>

username: NickOTeen
cooked: <p>Hmm you could be right that the preprocess method is not working correctly, but even when I remove the preprocessing and set r = 1 (mixing ratio), so that mixing and preprocessing does not occur. When I test images_cifar == images_mix it always return true, but then still my loss remains the same over each epoch. The mixing class now looks like this (with optplus=False):</p>
<pre><code class="lang-auto">def mix(image1, image2, label1, label2, optplus, train):
    image1 = image1.numpy()
    image2 = image2.numpy()
    label1 = label1.numpy()
    label2 = label2.numpy()
    # image1 = preprocess(image1, optplus, train)
    # image2 = preprocess(image2, optplus, train)
    # Mix two images
    #r = np.array(random.random())
    r = np.array([1])
    if optplus:
        g1 = np.std(image1)
        g2 = np.std(image2)
        p = 1.0 / (1 + g1 / g2 * (1 - r) / r)
        image = ((image1 * p + image2 * (1 - p)) / np.sqrt(p ** 2 + (1 - p) ** 2)).astype(np.float32)
    else:
        image = (image1 * r + image2 * (1 - r)).astype(np.float32)

        # Mix two labels
    eye = np.eye(nClasses)
    label = (eye[label1] * r + eye[label2] * (1 - r))

    image = torch.from_numpy(image).float().to(device)
    label = torch.from_numpy(label).float().to(device)

    return image, label
</code></pre>
<p>However when I do remove the whole mixing function the loss does seem to decrease, while to me it seems that I am still doing the same then (except that I’m skipping every second image)</p>
<p>EDIT: seems like loss.backward() was not in my loop anymore and len(data) should have been len(images_cifar). The learner does seem to learn something now without using the preprocess function. I will be trying to fix that now</p>

username: NickOTeen
cooked: <p>My test error is still much higher than in the paper, I get 14% where they get 5%. I think there might be something wrong in my model. I am trying to implement this model:<br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/4/8/484b8d2a10750e8d185f5504bd9c3ec9e73e0328.png" data-download-href="https://discuss.pytorch.org/uploads/default/484b8d2a10750e8d185f5504bd9c3ec9e73e0328" title="Schermafbeelding 2020-03-25 om 15.57.52"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/4/8/484b8d2a10750e8d185f5504bd9c3ec9e73e0328_2_331x499.png" alt="Schermafbeelding 2020-03-25 om 15.57.52" data-base62-sha1="ajyffOys86WWUK73t4naXx9yjfq" width="331" height="499" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/4/8/484b8d2a10750e8d185f5504bd9c3ec9e73e0328_2_331x499.png, https://discuss.pytorch.org/uploads/default/optimized/3X/4/8/484b8d2a10750e8d185f5504bd9c3ec9e73e0328_2_496x748.png 1.5x, https://discuss.pytorch.org/uploads/default/optimized/3X/4/8/484b8d2a10750e8d185f5504bd9c3ec9e73e0328_2_662x998.png 2x" data-dominant-color="EBEBEB"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">Schermafbeelding 2020-03-25 om 15.57.52</span><span class="informations">882×1332 150 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div><br>
As such:</p>
<pre><code class="lang-auto">class ConvBNReLu(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, bias=False):
        super(ConvBNReLu, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(in_channels=in_channels,
                      out_channels=out_channels,
                      kernel_size=kernel_size,
                      stride=stride,
                      padding=padding,
                      bias=bias),
            nn.BatchNorm2d(out_channels),
        )

    def forward(self, input):
        return self.main(input)

class Flatten(nn.Module):
    def forward(self, x):
        x = x.view(x.size()[0], -1)
        return x

class ConvNet(nn.Module):
    def __init__(self, n_classes):
        super(ConvNet, self).__init__()
        self.model = nn.Sequential(OrderedDict([
            ('conv1', ConvBNReLu(3, 64, 3, padding=1)),
            ('conv2', ConvBNReLu(64, 64, 3, padding=1)),
            ('max_pool1', nn.MaxPool2d(2, stride=2, ceil_mode=True)),
            ('conv3', ConvBNReLu(64, 128, 3, padding=1)),
            ('conv4', ConvBNReLu(128, 128, 3, padding=1)),
            ('max_pool2', nn.MaxPool2d(2, stride=2, ceil_mode=True)),
            ('conv5', ConvBNReLu(128, 256, 3, padding=1)),
            ('conv6', ConvBNReLu(256, 256, 3, padding=1)),
            ('conv7', ConvBNReLu(256, 256, 3, padding=1)),
            ('conv8', ConvBNReLu(256, 256, 3, padding=1)),
            ('max_pool3', nn.MaxPool2d(2, stride=2, ceil_mode=True)),
            ('flatten', Flatten()),
            ('fc4', nn.Linear(in_features=256 * 4 * 4, out_features=1024, bias=True)),
            ('relu5', nn.ReLU()),
            ('dropout5', nn.Dropout()),
            ('fc5', nn.Linear(1024, 1024)),
            ('relu6', nn.ReLU()),
            ('dropout6', nn.Dropout()),
            ('fc6', nn.Linear(1024, n_classes)),
            ('softmax', nn.LogSoftmax(dim=-1)) # Changed to LogSoftmax from Softmax, negative KL divergence otherwise
        ]))

    def forward(self, inp):
        return self.model(inp)

</code></pre>
<p>I am quite new to making model however this works, but not as good as I had hoped. Did I implement the model correctly?</p>
<p>I’m testing the model each epoch like this (no mixing):</p>
<pre><code class="lang-auto">criterion = nn.KLDivLoss()
learning_rate = 0.1 # Initial learning rate
optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9, nesterov=True, weight_decay=5e-4)
scheduler = MultiStepLR(optimizer, milestones=[100,150,200], gamma=0.1) # Decrease learning rate at milestones

test_error_standard = []
log_test_error = True
nEpoch = 250
for epoch in range(nEpoch):  # Number of epochs (loops over dataset)
  epoch_loss = 0.0
  running_loss = 0.0
  for i, data in enumerate(trainloader):
      # get the inputs; data is a list of [inputs, labels]
      
      label = []
      images_cifar, labels_cifar = data

      for j in range(len(labels_cifar)):
        label.append((torch.eye(10)[int(labels_cifar[j])]))
      labels = torch.stack(label).to(device)
      optimizer.zero_grad()
      # forward + backward + optimize
      outputs = net(images_cifar.to(device))
      loss = criterion(outputs, labels)
      loss.backward()
      optimizer.step()

      # print statistics every 200 loops

      running_loss += loss.item()

  
  print('[%d, %5d] loss: %.3f' %
        (epoch + 1, i, running_loss / i))
  running_loss = 0.0

  scheduler.step()
  if log_test_error: # Test network at each epoch and save test error
    correct = 0
    total = 0
    with torch.no_grad():
        for data in testloader:
            images, labels = data
            images = images.to(device)
            labels = labels.to(device)
            outputs = net(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
    test_error = 100 - 100 * correct / total
    test_error_standard.append(test_error)
    print('Test error: %.2f %%' % (
        test_error))


print('Finished Training')

torch.save(net.state_dict(), PATH)
</code></pre>
<p>EDIT: The error was not having nn.ReLU() at the end of ConvBNReLu(nn.Module), getting similar validation errors now!</p>