Trying to adapt a 2 class classifier into a 3 class one

username: Julio
cooked: <p>Hi,</p>
<p>I’m working on a 3-class classifier, and a s a base i started with a simple 2 class one. I’ve been having strange performances when trying to use Sigmoid (i think that is used for binary classification), and errors while trying nn.LogSoftmax(dim=-1).</p>
<p>Anyways, i’d like to know what exactly are the things i’ll have to change and how to adapt the 2 class classifier into a 3 class one.</p>
<p>Here’s my model</p>
<pre><code class="lang-auto">model = nn.Sequential(
        nn.Conv2d(3, 64, kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2),
        nn.Conv2d(64, 128, kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2),
        nn.Conv2d(128, 256, kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2),
        nn.Conv2d(256, 256, kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2),
        nn.Flatten(),
        nn.Linear(7 * 7 * 256, 512),
        nn.ReLU(),
        nn.Linear(512, 2),
        nn.LogSoftmax
)

model.to(device)

optimizer = optim.Adam(model.parameters(), lr=0.001)
loss_fn = nn.BCELoss()

transform = transforms.Compose([
                                transforms.Resize(150), # Resize the short side of the image to 150 keeping aspect ratio
                                transforms.CenterCrop(150), # Crop a square in the center of the image
                                transforms.ToTensor(), # Convert the image to a tensor with pixels in the range [0, 1]
                                ])

batch_size = 64
</code></pre>
<pre><code class="lang-auto">class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count


def train_model(model, optimizer, loss_fn, train_loader, val_loader, epochs):
    train_accuracies, train_losses, val_accuracies, val_losses = [], [], [], []
    val_loss = AverageMeter()
    val_accuracy = AverageMeter()
    train_loss = AverageMeter()
    train_accuracy = AverageMeter()

    for epoch in range(epochs):
        # train
        model.train()
        train_loss.reset()
        train_accuracy.reset()
        train_loop = tqdm(train_loader, unit=" batches")  # For printing the progress bar
        for data, target in train_loop:
            train_loop.set_description('[TRAIN] Epoch {}/{}'.format(epoch + 1, epochs))
            data, target = data.float().to(device), target.float().to(device)
            target = target.unsqueeze(-1)
            optimizer.zero_grad()
            output = model(data)
            loss = loss_fn(output, target)
            loss.backward()
            optimizer.step()

            train_loss.update(loss.item(), n=len(target))
            pred = output.round()  # get the prediction
            acc = pred.eq(target.view_as(pred)).sum().item() / len(target)
            train_accuracy.update(acc, n=len(target))
            train_loop.set_postfix(loss=train_loss.avg, accuracy=train_accuracy.avg)

        train_losses.append(train_loss.avg)
        train_accuracies.append(train_accuracy.avg)

        # validation
        model.eval()
        val_loss.reset()
        val_accuracy.reset()
        val_loop = tqdm(val_loader, unit=" batches")  # For printing the progress bar
        with torch.no_grad():
            for data, target in val_loop:
                val_loop.set_description('[VAL] Epoch {}/{}'.format(epoch + 1, epochs))
                data, target = data.float().to(device), target.float().to(device)
                target = target.unsqueeze(-1)
                output = model(data)
                loss = loss_fn(output, target)
                val_loss.update(loss.item(), n=len(target))
                pred = output.round()  # get the prediction
                acc = pred.eq(target.view_as(pred)).sum().item() / len(target)
                val_accuracy.update(acc, n=len(target))
                val_loop.set_postfix(loss=val_loss.avg, accuracy=val_accuracy.avg)

        val_losses.append(val_loss.avg)
        val_accuracies.append(val_accuracy.avg)

    return train_accuracies, train_losses, val_accuracies, val_losses


epochs = 60
train_accuracies, train_losses, val_accuracies, val_losses = train_model(model, optimizer, loss_fn, train_loader, val_loader, epochs)

torch.save(model.state_dict(), 'Types_of_lane_small_3_class.pt')
np.savez('history_Types_of_lane_small_3_class.npz', train_accuracies=train_accuracies, train_losses=train_losses, val_accuracies=val_accuracies, val_losses=val_losses)

epochs = range(len(train_accuracies))

plt.plot(epochs, train_accuracies, 'b', label='Training acc')
plt.plot(epochs, val_accuracies, 'r', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs, train_losses, 'b', label='Training loss')
plt.plot(epochs, val_losses, 'r', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()
</code></pre>
<p>I’m getting this error:</p>
<pre><code class="lang-auto">/home/env2/lib/python3.6/site-packages/torch/nn/modules/container.py:100: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.
  input = module(input)
/home/env2/lib/python3.6/site-packages/torch/nn/modules/loss.py:498: UserWarning: Using a target size (torch.Size([64, 1])) that is different to the input size (torch.Size([64, 2])) is deprecated. Please ensure they have the same size.
  return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)
Traceback (most recent call last):
  File "Model3class.py", line 264, in &lt;module&gt;
    train_accuracies, train_losses, val_accuracies, val_losses = train_model(model, optimizer, loss_fn, train_loader, val_loader, epochs)
  File "Model3class.py", line 226, in train_model
    loss = loss_fn(output, target)
  File "/home/env2/lib/python3.6/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/env2/lib/python3.6/site-packages/torch/nn/modules/loss.py", line 498, in forward
    return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)
  File "/home/env2/lib/python3.6/site-packages/torch/nn/functional.py", line 2070, in binary_cross_entropy
    "!= input nelement ({})".format(target.numel(), input.numel()))
ValueError: Target and input must have the same number of elements. target nelement (64) != input nelement (128)
</code></pre>

username: Julio
cooked: <p>Actually, my final objective is to adapt it to a 4 class classifier, but i suppose it will be similar to a 3 class one</p>

username: ptrblck
cooked: <p>The <code>LogSoftmax</code> is missing the parentheses, so no instance is created. Use this:</p>
<pre><code class="lang-python">...
nn.Linear(512, 2).
nn.LogSoftmax()
)
</code></pre>
<p>Since you are dealing with a multi-class classification, use <code>nn.NLLLoss</code> as your criterion.</p>

username: Julio
cooked: <p>I made thise changes:</p>
<pre><code class="lang-auto">model = nn.Sequential(
        nn.Conv2d(3, 64, kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2),
        nn.Conv2d(64, 128, kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2),
        nn.Conv2d(128, 256, kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2),
        nn.Conv2d(256, 256, kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2),
        nn.Flatten(),
        nn.Linear(7 * 7 * 256, 512),
        nn.ReLU(),
        nn.ReLU(),
        nn.Linear(512, 2),
        nn.LogSoftmax(dim=0)
)

model = model.to(device)

optimizer = optim.Adam(model.parameters(), lr=0.001)
loss_fn = nn.NLLLoss()

transform = transforms.Compose([
                                transforms.Resize(150), # Resize the short side of the image to 150 keeping aspect ratio
                                transforms.CenterCrop(150), # Crop a square in the center of the image
                                transforms.ToTensor(), # Convert the image to a tensor with pixels in the range [0, 1]
                                ])

batch_size = 64
</code></pre>
<p>(also, I’m not sure about the dimension = 0 and the 2 on the last nn.Linear. Should it be like that?)</p>
<p>and i got this error:</p>
<pre><code class="lang-auto">
Traceback (most recent call last):
  File "Model3class.py", line 264, in &lt;module&gt;
    train_accuracies, train_losses, val_accuracies, val_losses = train_model(model, optimizer, loss_fn, train_loader, val_loader, epochs)
  File "Model3class.py", line 227, in train_model
    loss = loss_fn(output, target.unsqueeze(-1))
  File "/home/env2/lib/python3.6/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/env2/lib/python3.6/site-packages/torch/nn/modules/loss.py", line 204, in forward
    return F.nll_loss(input, target, weight=self.weight, ignore_index=self.ignore_index, reduction=self.reduction)
  File "/home/env2/lib/python3.6/site-packages/torch/nn/functional.py", line 1838, in nll_loss
    ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
RuntimeError: Expected object of scalar type Long but got scalar type Float for argument #2 'target' in call to _thnn_nll_loss_forward

</code></pre>
<p>I can’t figure out how to change the object to float, or generate it as float previously, or how to expect a float instead of a Long</p>
<p>Any ideas of what the solution could be?</p>
<p>Thank You very much for all the help</p>

username: jubick
cooked: <aside class="quote no-group quote-modified" data-username="Julio" data-post="4" data-topic="75825">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/julio/48/12132_2.png" class="avatar"> Julio:</div>
<blockquote>
<p>RuntimeError: Expected object of scalar type Long but got scalar type Float for argument <span class="hashtag-raw">#2</span> ‘target’ in call to _thnn_nll_loss_forward</p>
</blockquote>
</aside>
<p>As it says you should cast target tensor to type long.<br>
It can be done like that</p>
<pre><code class="lang-auto">target = target.long()
</code></pre>

username: Julio
cooked: <p>I’ve casted as long, and i get the following error now:</p>
<pre><code class="lang-auto">
Traceback (most recent call last):
  File "Model3class.py", line 264, in &lt;module&gt;
    train_accuracies, train_losses, val_accuracies, val_losses = train_model(model, optimizer, loss_fn, train_loader, val_loader, epochs)
  File "Model3class.py", line 232, in train_model
    acc = pred.eq(target.view_as(pred)).sum().item() / len(target.long())
RuntimeError: shape '[64, 2]' is invalid for input of size 64

</code></pre>
<p>I’m not really sure “where” is my input of size 64 or how to change it</p>

username: jubick
cooked: <p>Since you have logsoftmax as last layer your output is log of softmax which gives you a tensor [64, 2]. And you’re trying to cast target tensor to shape of pred and it’s not clear how to do it.<br>
And another thing that seems strange is that you have</p>
<aside class="quote no-group" data-username="Julio" data-post="1" data-topic="75825">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/julio/48/12132_2.png" class="avatar"> Julio:</div>
<blockquote>
<p>nn.Linear(512, 2),</p>
</blockquote>
</aside>
<p>but you want to have 3 class classification. So i think you should change it to</p>
<pre><code class="lang-auto">nn.Linear(512, 3)
</code></pre>
<p>And another thing is last layer.<br>
Are you sure you want to use logsoftmax instead of softmax? As softmax would just give you probability distribution over yout classes like [0.01, 0.5, 0.49].</p>

username: ptrblck
cooked: <p><code>nn.LogSoftmax</code> should use <code>dim=1</code> for a multi-class classification use case and, as <a class="mention" href="/u/jubick">@jubick</a> explained, you would also have to increase the number of output features to 3.</p>
<p>Since you are using <code>nn.LogSoftmax</code> as the last activation, you should use <code>nn.NLLLoss</code> as the criterion, which expects the target in the shape <code>[batch_size]</code> containing target indices in the range <code>[0, nb_classes-1]</code>.</p>

username: Julio
cooked: <p>First of all, thank you both for all of your help, you’ve been really helpful and i understand so much more now.</p>
<p>I think the problem i’m facing now is about the way the script i based the model on calculates and represents the accuracy.<br>
Here’s the code that gets the accuracy and loss:</p>
<pre><code class="lang-auto">def train_model(model, optimizer, loss_fn, train_loader, val_loader, epochs):
    train_accuracies, train_losses, val_accuracies, val_losses = [], [], [], []
    val_loss = AverageMeter()
    val_accuracy = AverageMeter()
    train_loss = AverageMeter()
    train_accuracy = AverageMeter()

    for epoch in range(epochs):
        # train
        model.train()
        train_loss.reset()
        train_accuracy.reset()
        train_loop = tqdm(train_loader, unit=" batches")  # For printing the progress bar
        for data, target in train_loop:
            train_loop.set_description('[TRAIN] Epoch {}/{}'.format(epoch + 1, epochs))
            data, target = data.float().to(device), target.long().to(device)
            target = target.unsqueeze(-1).long()
            optimizer.zero_grad()
            output = model(data)
            loss = loss_fn(output, torch.max(target, 1)[1])
            loss.backward()
            optimizer.step()
            train_loss.update(loss.item(), n=len(target))
            pred = output.round()  # get the prediction
            acc = pred.eq(target.view_as(pred)).sum().item() / len(target.long())
            train_accuracy.update(acc, n=len(target))
            train_loop.set_postfix(loss=train_loss.avg, accuracy=train_accuracy.avg)

        train_losses.append(train_loss.avg)
        train_accuracies.append(train_accuracy.avg)

        # validation
        model.eval()
        val_loss.reset()
        val_accuracy.reset()
        val_loop = tqdm(val_loader, unit=" batches")  # For printing the progress bar
        with torch.no_grad():
            for data, target in val_loop:
                val_loop.set_description('[VAL] Epoch {}/{}'.format(epoch + 1, epochs))
                data, target = data.float().to(device), target.long().to(device)
                target = target.unsqueeze(-1).long()
                output = model(data)
                loss = loss_fn(output, torch.max(target, 1)[1])
                val_loss.update(loss.item(), n=len(target))
                pred = output.round()  # get the prediction
                acc = pred.eq(target.view_as(pred)).sum().item() / len(target.long())
                val_accuracy.update(acc, n=len(target))
                val_loop.set_postfix(loss=val_loss.avg, accuracy=val_accuracy.avg)

        val_losses.append(val_loss.avg)
        val_accuracies.append(val_accuracy.avg)

    return train_accuracies, train_losses, val_accuracies, val_losses


epochs = 30
train_accuracies, train_losses, val_accuracies, val_losses = train_model(model, optimizer, loss_fn, train_loader, val_loader, epochs)

torch.save(model.state_dict(), 'Types_of_lane_small_3_class.pt')
np.savez('history_Types_of_lane_small_3_class.npz', train_accuracies=train_accuracies, train_losses=train_losses, val_accuracies=val_accuracies, val_losses=val_losses)

epochs = range(len(train_accuracies))

plt.plot(epochs, train_accuracies, 'b', label='Training acc')
plt.plot(epochs, val_accuracies, 'r', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs, train_losses, 'b', label='Training loss')
plt.plot(epochs, val_losses, 'r', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()
</code></pre>
<p>And the error i’m getting now, with <code>nn.LogSoftmax</code> with dim=1 and the number of output features set to 3</p>
<pre><code class="lang-auto">
Traceback (most recent call last):
  File "Model3class.py", line 265, in &lt;module&gt;
    train_accuracies, train_losses, val_accuracies, val_losses = train_model(model, optimizer, loss_fn, train_loader, val_loader, epochs)
  File "Model3class.py", line 233, in train_model
    acc = pred.eq(target.view_as(pred)).sum().item() / len(target.long())
RuntimeError: shape '[32, 3]' is invalid for input of size 32

</code></pre>
<p>I don’t really know if this way of getting the accuracy is valid for a 3 or 4 class classification model</p>