Temporal CNN Dimension error

Jam: <p>I am working with sequences of images for a binary classification. With batch size 1 my sequence has dimension [1, 9, 3, 256, 256]. Since I want to use a Temporal CN (from this GitHub: <a href="https://github.com/locuslab/TCN/tree/master/TCN/mnist_pixel" class="inline-onebox" rel="noopener nofollow ugc">TCN/TCN/mnist_pixel at master · locuslab/TCN · GitHub</a>), which requires input size [N, C_in, L_in], I am reshaping the the images like this: images = images.view(-1, 3, 9) where 3 is the channel size and 9 the sequence length. However, I get the following error when calling loss = criterion(outputs, labels):</p>
<pre><code class="lang-auto">The size of tensor a (9) must match the size of tensor b (65536) at non-singleton dimension 0
</code></pre>
<p>The 65536 results from 256*256 but I don’t know how to adjust my dimensions accordingly so that they match my labels’ dimensions.</p>
<p>The model looks like this</p>
<pre><code class="lang-auto">class Chomp1d(nn.Module):
    def __init__(self, chomp_size):
        super(Chomp1d, self).__init__()
        self.chomp_size = chomp_size

    def forward(self, x):
        return x[:, :, :-self.chomp_size].contiguous()


class TemporalBlock(nn.Module):
    def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2):
        super(TemporalBlock, self).__init__()
        self.conv1 = weight_norm(nn.Conv1d(n_inputs, n_outputs, kernel_size,
                                           stride=stride, padding=padding, dilation=dilation))
        self.chomp1 = Chomp1d(padding)
        self.relu1 = nn.ReLU()
        self.dropout1 = nn.Dropout(dropout)

        self.conv2 = weight_norm(nn.Conv1d(n_outputs, n_outputs, kernel_size,
                                           stride=stride, padding=padding, dilation=dilation))
        self.chomp2 = Chomp1d(padding)
        self.relu2 = nn.ReLU()
        self.dropout2 = nn.Dropout(dropout)

        self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1,
                                 self.conv2, self.chomp2, self.relu2, self.dropout2)
        self.downsample = nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None
        self.relu = nn.ReLU()
        self.init_weights()

    def init_weights(self):
        self.conv1.weight.data.normal_(0, 0.01)
        self.conv2.weight.data.normal_(0, 0.01)
        if self.downsample is not None:
            self.downsample.weight.data.normal_(0, 0.01)

    def forward(self, x):
        out = self.net(x)
        res = x if self.downsample is None else self.downsample(x)
        return self.relu(out + res)


class TemporalConvNet(nn.Module):
    def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2):
        super(TemporalConvNet, self).__init__()
        layers = []
        num_levels = len(num_channels)
        for i in range(num_levels):
            dilation_size = 2 ** i
            in_channels = num_inputs if i == 0 else num_channels[i-1]
            out_channels = num_channels[i]
            layers += [TemporalBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,
                                     padding=(kernel_size-1) * dilation_size, dropout=dropout)]

        self.network = nn.Sequential(*layers)

    def forward(self, x):
        return self.network(x)

class TCN(nn.Module):
    def __init__(self, input_size, output_size, num_channels, kernel_size, dropout):
        super(TCN, self).__init__()
        self.tcn = TemporalConvNet(input_size, num_channels, kernel_size=kernel_size, dropout=dropout)
        self.linear = nn.Linear(num_channels[-1], output_size)

    def forward(self, inputs):
        y1 = self.tcn(inputs)  # input should have dimension (N, C, L)
        o = self.linear(y1[:, :, -1])
        return F.log_softmax(o, dim=1)
</code></pre>

ptrblck: <p>It seems you are permuting the spatial dimensions into the batch dimension of the input tensor, while the target tensor seems to use the original batch dimension, which will thus break.<br>
Both, the model output and target, are expected to have the same batch dimension, which represents the number of samples in the current batch.<br>
I’m not familiar with your use case, so don’t know why the reshaping is needed, so could you explain it a bit more?</p>

Jam: <p>Thanks for your reply! In fact, I realized that my reshaping was incorrect. I used the following dimensions now and do not get an error anymore: [1<em>9, 3, 256</em>256]. I realized that I obviously do not need the log_softmax at the end, since I am performing binary classification. So the model now looks likes this:</p>
<pre><code class="lang-auto">class Chomp1d(nn.Module):
    def __init__(self, chomp_size):
        super(Chomp1d, self).__init__()
        self.chomp_size = chomp_size

    def forward(self, x):
        return x[:, :, :-self.chomp_size].contiguous()
    

# Cell
class GAP1d(nn.Module):
    "Global Adaptive Pooling + Flatten"
    def __init__(self, output_size=1):
        super(GAP1d, self).__init__()
        self.gap = nn.AdaptiveAvgPool1d(output_size)
        self.flatten = nn.Flatten()
    def forward(self, x):
        return self.flatten(self.gap(x))

class TemporalBlock(nn.Module):
    def __init__(self, ni, nf, ks, stride, dilation, padding, dropout=0.):
        super(TemporalBlock, self).__init__()
        self.conv1 = weight_norm(nn.Conv1d(ni,nf,ks,stride=stride,padding=padding,dilation=dilation))
        self.chomp1 = Chomp1d(padding)
        self.relu1 = nn.ReLU()
        self.dropout1 = nn.Dropout(dropout)
        self.conv2 = weight_norm(nn.Conv1d(nf,nf,ks,stride=stride,padding=padding,dilation=dilation))
        self.chomp2 = Chomp1d(padding)
        self.relu2 = nn.ReLU()
        self.dropout2 = nn.Dropout(dropout)
        self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1, 
                                 self.conv2, self.chomp2, self.relu2, self.dropout2)
        self.downsample = nn.Conv1d(ni,nf,1) if ni != nf else None
        self.relu = nn.ReLU()
        self.init_weights()

    def init_weights(self):
        self.conv1.weight.data.normal_(0, 0.01)
        self.conv2.weight.data.normal_(0, 0.01)
        if self.downsample is not None: self.downsample.weight.data.normal_(0, 0.01)

    def forward(self, x):
        out = self.net(x)
        res = x if self.downsample is None else self.downsample(x)
        return self.relu(out + res)

def TemporalConvNet(c_in, layers, ks=2, dropout=0.):
    temp_layers = []
    for i in range(len(layers)):
        dilation_size = 2 ** i
        ni = c_in if i == 0 else layers[i-1]
        nf = layers[i]
        temp_layers += [TemporalBlock(ni, nf, ks, stride=1, dilation=dilation_size, padding=(ks-1) * dilation_size, dropout=dropout)]
    return nn.Sequential(*temp_layers)

class TCN(nn.Module):
    def __init__(self, c_in, c_out, layers=8*[25], ks=7, conv_dropout=0., fc_dropout=0.):
        super(TCN1, self).__init__()
        self.tcn = TemporalConvNet(c_in, layers, ks=ks, dropout=conv_dropout)
        self.gap = GAP1d()
        self.dropout = nn.Dropout(fc_dropout) if fc_dropout else None
        self.linear = nn.Linear(layers[-1],c_out)
        self.init_weights()

    def init_weights(self):
        self.linear.weight.data.normal_(0, 0.01)

    def forward(self, x):
        x = self.tcn(x)
        x = self.gap(x)
        if self.dropout is not None: x = self.dropout(x)
        x = self.linear(x)
        return x
</code></pre>
<p>And I am defining the model e.g. like this:</p>
<pre><code class="lang-auto">input_channels = 3
output_dim=1

model = TCN(input_channels, output_dim, layers = 2*[5], ks=3,  conv_dropout=0.2, fc_dropout=0.4)
</code></pre>
<p>I am using BCELossWithLogits but it seems that my model isn’t learning. Both AUCs fluctuate a lot and the training AUC is never bigger then 59%, Validation AUC increases up to 80%. But for my LSTM model, the data works perfectly fine. The code I used (<a href="https://github.com/locuslab/TCN/tree/master/TCN/mnist_pixel" class="inline-onebox" rel="noopener nofollow ugc">TCN/TCN/mnist_pixel at master · locuslab/TCN · GitHub</a>) is from a use case where they perform a multiclass classification on sequences of the MNIST dataset. I am not sure whether I incorrectly implemented something when using the github code but if so, I just can’t figure out what.</p>