nn.Sequential multiple arguments in Dynamic numbers of layers

Erica_Zheng: <p>Is there a way to enable <code>nn.Sequential</code> to handle multiple inputs with dynamic number of blocks?<br>
My code follows <a href="https://github.com/pytorch/pytorch/issues/19808#" rel="nofollow noopener">#19808</a>, but it doesn’t work from the 2nd blocks.</p>
<pre><code class="lang-auto">Traceback (most recent call last):
  File "/home/Workspace/one-implementation/models/cifar/resnet_stochastic_depth.py", line 380, in &lt;module&gt;
    out = model(data)
  File "/home/Workspace/one-implementation/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/Workspace/one-implementation/models/cifar/resnet_stochastic_depth.py", line 350, in forward
    x = self.layer1(x, self.actives, self.probabilities)
  File "/home/Workspace/one-implementation/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/Workspace/one-implementation/models/cifar/resnet_stochastic_depth.py", line 282, in forward
    input = module(*input)
  File "/home/Workspace/one-implementation/venv/lib/python3.6/site-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
TypeError: forward() takes 4 positional arguments but 513 were given
cnt = 1 module = BasicBlock_stochastic_depth(
  (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (relu): ReLU(inplace=True)
  (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)

Process finished with exit code 1
</code></pre>
<pre><code class="lang-auto">class mySequential(nn.Sequential):
    def forward(self, *input):
        cnt = 0
        for module in self._modules.values():
            input = module(*input)
            cnt = cnt + 1
            print('cnt = {} module = {}'.format(cnt, module))
        return input

class BasicBlock_stochastic_depth(nn.Module):
    expansion = 1

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(BasicBlock_stochastic_depth, self).__init__()
        self.conv1 = conv3x3(inplanes, planes, stride)
        self.bn1 = nn.BatchNorm2d(planes)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x3(planes, planes)
        self.bn2 = nn.BatchNorm2d(planes)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x, active, prob):
        if self.training:
            if active == 1:
                print('active')
                residual = x
                out = self.conv1(x)
                out = self.bn1(out)
                out = self.relu(out)

                out = self.conv2(out)
                out = self.bn2(out)

                if self.downsample is not None:
                    residual = self.downsample(x)

                out += residual
                out = self.relu(out)
            else:
                print('inactive')
                out = x
                if self.downsample is not None:
                    out = self.downsample(out)
                out = self.relu(out)

        else:
            residual = x

            out = self.conv1(x)
            out = self.bn1(out)
            out = self.relu(out)

            out = self.conv2(out)
            out = self.bn2(out)

            if self.downsample is not None:
                residual = self.downsample(x)

            out = prob * out + residual
            out = self.relu(out)

        return (out)

class ResNetSD(nn.Module):

    def __init__(self, depth, num_classes=1000, block_name='BasicBlock'):
        super(ResNetSD, self).__init__()
        # Model type specifies number of layers for CIFAR-10 model
        if block_name.lower() == 'basicblock':
            assert(depth - 2) % 6 == 0, 'When use basicblock, depth should be 6n+2, e.g. 20, 32, 44, 56, 110, 1202'
            n = (depth - 2) // 6
            block = BasicBlock_stochastic_depth
        elif block_name.lower() == 'bottleneck':
            assert (depth == 2) % 9 == 0, 'When use bottleneck, depth should be 9n + 2, e.g. 20, 29, 47, 56, 110, 1199'
            n = (depth - 2) // 9
            block = Bottleneck_stochastic_depth
        else:
            raise ValueError('block_name should be Basicblock or Bottleneck')

        self.probabilities = torch.tensor(0.5)
        self.actives = torch.bernoulli(self.probabilities)

        self.inplanes = 16
        self.conv1 = nn.Conv2d(3, 16, kernel_size=3, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(16)
        self.relu = nn.ReLU(inplace=True)
        self.layer1 = self._make_layer(block, 16, n)
        self.layer2 = self._make_layer(block, 32, n, stride=2)
        self.layer3 = self._make_layer(block, 64, n, stride=2)
        self.avgpool = nn.AvgPool2d(8)
        self.fc = nn.Linear(64 * block.expansion, num_classes)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2./n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion)
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        group_layers = mySequential(*layers)
        # group_layers = mySequential(nn.Sequential(*layers))
        # return nn.Sequential(*layers)
        return group_layers

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)

        x = self.layer1(x, self.actives, self.probabilities)
        x = self.layer2(x, self.actives, self.probabilities)
        x = self.layer3(x, self.actives, self.probabilities)

        x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        x = self.fc(x)

        return x
</code></pre>

Dexter: <p>Hi Erica,</p>
<p>As per your post did you mean that everytime nn.Sequential is called it will change the number of layers it has?</p>

Erica_Zheng: <p>Hi, Dexter</p>
<p>Yes. Please advice. Thank you.</p>