Input doesn't same dimension as tensor for result?

pipibo: <p>Hi, I’m new to pytorch, I have read the pytorch tutorials and doc document, So I try to program some code to test. But there is a error that my “input” doesn’t have the same dimension as tensor for “result”, I’ve tried to change the size  of input and reduce the kernel_size but none of them work.<br>
here is my code, I use cifar10 dataset and make a AlexNet model.</p>
<pre><code class="lang-auto">import torch.nn as nn
import torch.nn.functional as F 
import argparse
import torch
import torch.optim as optim
from torchvision import datasets, transforms

class alexnet(nn.Module):

    def __init__(self):
        super(alexnet,self).__init__()
    
        self.conv_block1 = nn.Sequential(
            nn.Conv2d(3,96,kernel_size=11,stride=4),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(3,2,ceil_mode=True),
            nn.BatchNorm2d(96,affine=False),
        )

        self.conv_block2 = nn.Sequential(
            nn.Conv2d(96,256,kernel_size=11,padding=2,groups=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(3,2,ceil_mode=True),
            nn.BatchNorm2d(256,affine=False),
        )

        self.conv_block3 = nn.Sequential(
            nn.Conv2d(256,384,kernel_size=3,padding=1),
            nn.ReLU(inplace=True),
        )

        self.conv_block4 = nn.Sequential(
            nn.Conv2d(384,384,kernel_size=3,padding=1,groups=2),
            nn.ReLU(inplace=True),
        )

        self.conv_block5 = nn.Sequential(
            nn.Conv2d(384,256,kernel_size=3,padding=1,groups=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(3,2),
        )

        self.classifier = nn.Sequential(
            nn.Linear(6*6*256,4096),
            nn.ReLU(True),
            nn.Linear(4096,4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096,10),
        )

    def forward(self,x):

        conv1 = self.conv_block1(x)
        conv2 = self.conv_block2(conv1)
        conv3 = self.conv_block3(conv2)
        conv4 = self.conv_block4(conv3)
        conv5 = self.conv_block5(conv4)
        score = self.classifier(conv5)

        return score

def train(args, model, device, train_loader, optimizer, epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        if batch_idx % args.log_interbal == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item()
            ))

def test(args, model, device, test_loader):
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += F.nll_loss(output, target, size_average=False).item()
            pred = output.max(1, keepdim=True)[1]
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)
    print ('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} (:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)
    ))

def main():
    parser = argparse.ArgumentParser(description='PyTorch AlexNet Try')
    parser.add_argument('--batch-size', type=int, default=64, metavar='N',
                        help='input batch size for training (default: 64)')
    parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',
                        help='input batch size for testing (default: 1000)')
    parser.add_argument('--epochs', type=int, default=10, metavar='N',
                        help='number of epochs to train (default: 10)')
    parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
                        help='learning rate (default: 0.01)')
    parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
                        help='SGD momentum (default: 0.5)')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')
    parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                        help='how many batches to wait before logging training status')
    args = parser.parse_args()

    if torch.cuda.is_available():    
        device = torch.device("cuda")
    else:
        device = torch.device("cpu")
    
    torch.manual_seed(args.seed)

    kwargs = {'num_workers': 1, 'pin_memory': True}

    transform = transforms.Compose(
        [
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
        ])

    train_loader = torch.utils.data.DataLoader(
        datasets.CIFAR10('./alextry', train=True, download=True,
                        transform=transform), 
        batch_size = args.batch_size, shuffle=True, **kwargs
    )
    test_loader = torch.utils.data.DataLoader(
        datasets.CIFAR10('./alextry', train=False, download=True,
                        transform=transform),
        batch_size=args.test_batch_size, shuffle=True, **kwargs
    )

    model = alexnet().to(device)
    optimizer = optim.SGD(model.parameters(), lr = args.lr, momentum=args.momentum)

    for epoch in range(1, args.epochs+1):
        train(args, model, device, train_loader, optimizer, epoch)
        test(args, model, device, test_loader)


if __name__ == '__main__':
    main()
</code></pre>
<p>thanks for your help</p>

ptrblck: <p>Could you post the complete error message with the line it’s thrown from?</p>

pipibo: <p>this is  all the error message:</p>
<pre><code class="lang-auto">Traceback (most recent call last):
  File "alexnet_try.py", line 166, in &lt;module&gt;
    main()
  File "alexnet_try.py", line 162, in main
    train(args, model, device, train_loader, optimizer, epoch)
  File "alexnet_try.py", line 84, in train
    output = model(data)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "alexnet_try.py", line 71, in forward
    conv2 = self.conv_block2(conv1)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/modules/container.py", line 91, in forward
    input = module(input)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/modules/conv.py", line 301, in forward
    self.padding, self.dilation, self.groups)
RuntimeError: Expected tensor for argument #1 'input' to have the same dimension as tensor for 'result'; but 4 does not equal 2 (while checking arguments for cudnn_convolution)
</code></pre>
<p>the error comes from</p>
<blockquote>
<p>output = model(data)</p>
</blockquote>
<blockquote>
<p>conv2 = self.conv_block2(conv1)</p>
</blockquote>
<p>thus I believe there is something wrong in the forward process</p>

ptrblck: <p>I get another error running your model with CIFAR10:</p>
<blockquote>
<p>RuntimeError: Calculated padded input size per channel: (7 x 7). Kernel size: (11 x 11). Kernel size can’t be greater than actual input size at /opt/conda/conda-bld/pytorch_1532581333611/work/aten/src/THNN/generic/SpatialConvolutionMM.c:48</p>
</blockquote>
<p>If you print the shape of <code>conv1</code> in the forward pass, you will see, that it’s just <code>[batch_size, 96, 3, 3]</code>, which is too small for <code>conv2</code> using <code>kernel_size=11</code>.</p>

pipibo: <p>oh, I understand , it’s because that the kernel size of conv_layers is too large and the input size is too small.<br>
thanks for your help!</p>

pipibo: <p>hi again, I just reduce some layer and the size of kernel, after that , the model looks  like this:</p>
<pre><code class="lang-auto">        self.conv_block1 = nn.Sequential(
            nn.Conv2d(3,96,kernel_size=3,stride=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(3,1,ceil_mode=True),
            nn.BatchNorm2d(96,affine=False),
        )

        self.conv_block2 = nn.Sequential(
            nn.Conv2d(96,256,kernel_size=3,groups=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(3,1,ceil_mode=True),
            nn.BatchNorm2d(256,affine=False),
        )

        self.conv_block3_ch = nn.Sequential(
            nn.Conv2d(256,256,kernel_size=3,stride=1,padding=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(3,3),
        )

        self.classifier = nn.Sequential(
            nn.Linear(8*8*256,4096),
            nn.ReLU(True),
            nn.Linear(4096,4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096,10),
        )
</code></pre>
<pre><code class="lang-auto">    def forward(self,x):

        conv1 = self.conv_block1(x)
        conv2 = self.conv_block2(conv1)
        conv3 = self.conv_block3_ch(conv2)
        score = self.classifier(conv3)

        return score
</code></pre>
<p>but after I run this code, there is still some error:</p>
<pre><code class="lang-auto">Traceback (most recent call last):
  File "alexnet_try.py", line 165, in &lt;module&gt;
    main()
  File "alexnet_try.py", line 161, in main
    train(args, model, device, train_loader, optimizer, epoch)
  File "alexnet_try.py", line 83, in train
    output = model(data)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "alexnet_try.py", line 74, in forward
    score = self.classifier(conv3)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/modules/container.py", line 91, in forward
    input = module(input)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/modules/linear.py", line 55, in forward
    return F.linear(input, self.weight, self.bias)
  File "/home/zqk/anaconda2/lib/python2.7/site-packages/torch/nn/functional.py", line 994, in linear
    output = input.matmul(weight.t())
RuntimeError: size mismatch, m1: [131072 x 8], m2: [16384 x 4096] at /opt/conda/conda-bld/pytorch_1524577177097/work/aten/src/THC/generic/THCTensorMathBlas.cu:249
</code></pre>
<p>I have computed the size after every conv_layer, the size of cifar10 dataset is 32x32, after conv1 the size of data is 28x28x96, the size reduce to 24x24x256 after conv2, and after the conv3 it becomes 8x8x256,what confuses me is that where is the <em><strong>131072</strong></em> and <em><strong>8</strong></em> in the error message come from?</p>

ptrblck: <p>You have to flatten your activation to pass it into a <code>Linear</code> layer.<br>
Add this line before calling <code>self.classifier</code>:</p>
<pre><code class="lang-python">conv3 = conv3.view(conv3.size(0), -1)
</code></pre>
<p>and run it again.</p>

pipibo: <p>It works ! thank you very much ! but I still have some question：<br>
1.what do the <strong>131072</strong> and <strong>8</strong> mean?<br>
2.Is it means that I need to use <code>view</code> function to flatten activation before passing to <code>Linear</code> layer every time ?</p>

ptrblck: <p>These number represent the input shape you are passing to <code>Linear</code>.<br>
Your activation shape right before the <code>classifier</code> is <code>[batch_size, 256, 8, 8]</code>.<br>
From the <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Linear">docs</a> of <code>Linear</code> we know, that the input shape to this layer should be <code>[batch_size, *, input_features]</code>, where <code>*</code> means any number of additional dimensions.<br>
I assume you used <code>batch_size=64</code>. Your input was therefore handled as <code>[64*256*8, 8]</code>, which is a mismatch to the linear layer expecting <code>256*8*8=16384</code> features.</p>
<p>You would have to handle the shape depending on the layer. Usually if you change the layer type, e.g. from conv to linear, you want to flatten the activation. You can also reshape the activation so that you can pass it to a recurrent layer.</p>

pipibo: <p>thank you, I think I understand what you mean.</p>

mei96: <p>hi, I’m new to pytorch too, I try to use the alexnet network to train my dataset, and the pretrainedmodels is adopted, but there is an error occured. It looks like this:</p>
<p>RuntimeError: Calculated padded input size per channel: (10 x 10). Kernel size: (11 x 11). Kernel size can’t be greater than actual input size at /opt/conda/conda-</p>
<p>can you help me? Thanks!</p>

ptrblck: <p>If your inputs are smaller than 224 in their spatial size, some intermediate activations might get too small for the kernel size of that layer.<br>
How large is your current input?</p>

mei96: <p>my current input is 224 x 224, so should I enlarge my inputs?</p>

mei96: <p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/2X/0/01fe87d2158e8ffc56578d02653cfa40528a78e3.png" data-download-href="https://discuss.pytorch.org/uploads/default/01fe87d2158e8ffc56578d02653cfa40528a78e3" title="QQ%E5%9B%BE%E7%89%8720190915185722"><img src="https://discuss.pytorch.org/uploads/default/optimized/2X/0/01fe87d2158e8ffc56578d02653cfa40528a78e3_2_690x438.png" alt="QQ%E5%9B%BE%E7%89%8720190915185722" data-base62-sha1="hDO0N3KysSzXAtltrAQgpxIDUD" width="690" height="438" srcset="https://discuss.pytorch.org/uploads/default/optimized/2X/0/01fe87d2158e8ffc56578d02653cfa40528a78e3_2_690x438.png, https://discuss.pytorch.org/uploads/default/original/2X/0/01fe87d2158e8ffc56578d02653cfa40528a78e3.png 1.5x, https://discuss.pytorch.org/uploads/default/original/2X/0/01fe87d2158e8ffc56578d02653cfa40528a78e3.png 2x" data-dominant-color="292929"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">QQ%E5%9B%BE%E7%89%8720190915185722</span><span class="informations">883×561 136 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div> !</p>
<p>this is the critical code,  and the img_model is alexnet</p>

mei96: <p><img src="https://discuss.pytorch.org/uploads/default/original/2X/b/b8fc72cda18004494d542ef3e11b858d4c0467f8.png" alt="QQ%E5%9B%BE%E7%89%8720190915185754" data-base62-sha1="qosIOINw1VYAFSbU5cqddr0f00g" width="542" height="427"><br>
Looking forward to your reply</p>

ptrblck: <p>Thanks for the code. Which layer throws the size error? Also, how did you define <code>FCViewer</code>?</p>
<p>PS: it’s a bit easier to debug your code, if you post a code snippet by wrapping it into three backticks ``` <img src="https://discuss.pytorch.org/images/emoji/apple/wink.png?v=9" title=":wink:" class="emoji" alt=":wink:"></p>

mei96: <p>Thanks for your reply. I guess is the first layer encountered the error，becouse the kernel size of alexnet‘s first layer is 11. I have gone back to my dormitory now，It’s evening here because of the time difference. So I will tell you the details of my code tomorrow morning. Thanks for your understanding.</p>

ptrblck: <p>Yeah, you are right.<br>
In that case, it looks like the spatial size of your input tensor is <code>10x10</code>.<br>
Make sure you pass the input as <code>[batch_size, channels, height, width]</code>.</p>

mei96: <p>Hi,  I just debugged my code and found an odd thing,  the image is pooled globally before passing through the network, as you can see, the output size is 6 x 6<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/2X/2/2ec0e3f3949b86ffcf3af99493455c314896df65.png" data-download-href="https://discuss.pytorch.org/uploads/default/2ec0e3f3949b86ffcf3af99493455c314896df65" title="QQ%E5%9B%BE%E7%89%8720190916090332"><img src="https://discuss.pytorch.org/uploads/default/optimized/2X/2/2ec0e3f3949b86ffcf3af99493455c314896df65_2_690x404.png" alt="QQ%E5%9B%BE%E7%89%8720190916090332" data-base62-sha1="6FBeeaxPkVu0dNz25kh3QWeE8qp" width="690" height="404" srcset="https://discuss.pytorch.org/uploads/default/optimized/2X/2/2ec0e3f3949b86ffcf3af99493455c314896df65_2_690x404.png, https://discuss.pytorch.org/uploads/default/original/2X/2/2ec0e3f3949b86ffcf3af99493455c314896df65.png 1.5x, https://discuss.pytorch.org/uploads/default/original/2X/2/2ec0e3f3949b86ffcf3af99493455c314896df65.png 2x" data-dominant-color="4C4D5C"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">QQ%E5%9B%BE%E7%89%8720190916090332</span><span class="informations">698×409 126 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>

mei96: <p>My FCViewer is defined as this<img src="https://discuss.pytorch.org/uploads/default/original/2X/2/2f3f492202bc980ce45107be90a9c65953cac854.png" alt="QQ%E5%9B%BE%E7%89%8720190916090733" data-base62-sha1="6JY1RuxNtxd3f3Azz3FfpfVaSi0" width="509" height="112"></p>