Gradients are always zero

mojzaar: <p>I have designed a DNN which has the following structure:</p>
<pre><code class="lang-auto">class my_Net(nn.Module):
    '''
    '''
    def __init__(self, device='cuda:0'):
        super(my_Net, self).__init__()

        self.device = device
        
        
        self.tdmp = TimeDistributed(step = 81)
        
        self.bncnn3drelu = BnCNN3DReLU(input_channels=1, output_channels=32)
        
        self.cnn3drelu_1 = CNN3DReLU(input_channels=32, output_channels=64)
        
        self.cnn3drelu_2 = CNN3DReLU(input_channels=64, output_channels=128)
        
        self.cnn3drelu_4  = CNN3DReLU(input_channels=128, output_channels=256)
                
        self.convlstm = ConvLSTM(input_size=(3, 3), input_channels = 256, hidden_dim=[16, 16, 16],
                 kernel_size=(3, 3),
                 num_layers=3,
                 target=[[80, 2]],
                 bias=True,
                 return_all_layers=False, device = self.device)
#        
        self.cnn2dup_1 = UPCNN2DReLU(size=(3 ,3), input_channels=16, output_channels=1)


    def forward(self, x):
        _ = self.bncnn3drelu(x)
        _ = self.cnn3drelu_1(_)
        _ = self.tdmp(_)
        _ = self.cnn3drelu_2(_)
        _ = self.tdmp(_)
        _ = self.cnn3drelu_3(_)
        _ = self.tdmp(_)
        _ = self.cnn3drelu_4(_)
        _ = self.tdmp(_)
        _ = self.convlstm(_)
        _ = self.cnn2dup_1(_)
        _ = F.interpolate(_, (61,61))
        return _
</code></pre>
<p>where TimeDistributed, BnCNN3DReLU, CNN3DReLU, ConvLSTM, and UPCNN2DReLU are"Apply maxpool2d to each fram of time", “<em>batch-normalization+3D CNN + ReLU</em>”, “<em>3D CNN + ReLU</em>”, “<em>convolutional LSTM</em>” and "<em>2D CNN + ReLU +Upsampling</em> ", respectively.</p>
<p>My data is a tensor with <em>batch_size, num_channel, time, heights, width = data.size()</em>. No matter which loss function I have, the gradients are always zero. I’m training the network using the following codes:</p>
<pre><code class="lang-auto">out = model(data)

loss = criterion_b(out, label) 
optimizer = optim.Adam(model.parameters(), lr=opt.lr)
optimizer.zero_grad()
loss.backward()
optimizer.step()
</code></pre>

IapCaL: <p>Hi,</p>
<p>I think optimizer should be defined before the forward function.</p>
<pre><code class="lang-auto">optimizer = optim.Adam(model.parameters(), lr=opt.lr)
optimizer.zero_grad()

out = model(data)
loss = criterion_b(out, label) 
loss.backward()
optimizer.step()
</code></pre>
<p>Hope this helps.</p>

mojzaar: <p>Hi,<br>
Thanks for your reply. Unfortunately it does not work yet.</p>

mojzaar: <p>I actually thick that the convLSTM causes the problem. Even, when I have a single convLSTM as my model, in the first iteration, only the outer layers have gradient and the rest are zero. See figure1.</p>
<p><img src="https://discuss.pytorch.org/uploads/default/original/3X/1/0/100273a1fa3b694c1dd1ac4b2704f52ea8c01c2d.png" alt="plot" data-base62-sha1="2hCTjO2hDd2hXLAkZCqrxJ7TWXz" width="411" height="428"></p>
<p>If I keep doing the calculation, again, the gradients all go to zero! See figure 2.</p>
<p><img src="https://discuss.pytorch.org/uploads/default/original/3X/2/0/209afd6422a9959dd451f40a07d5cd848040639b.png" alt="plot2" data-base62-sha1="4Erlhdzy3UesTiClk1iGvVyBzXJ" width="400" height="428"></p>
<p>I did not find any built-in function for convLSTM and  I’m using the following code to have them. I do appreciate your comments if you have better code, or tell me if there is any problem in the code.</p>
<pre><code class="lang-auto">class ConvLSTMCell(nn.Module):

    def __init__(self, input_size, input_dim, hidden_dim, kernel_size, bias, device):
        """
        Initialize ConvLSTM cell.
        
        Parameters
        ----------
        input_size: (int, int)
            Height and width of input tensor as (height, width).
        input_dim: int
            Number of channels of input tensor.
        hidden_dim: int
            Number of channels of hidden state.
        kernel_size: (int, int)
            Size of the convolutional kernel.
        bias: bool
            Whether or not to add the bias.
        """

        super(ConvLSTMCell, self).__init__()

        self.height, self.width = input_size
        self.input_dim  = input_dim
        self.hidden_dim = hidden_dim

        self.kernel_size = kernel_size
        self.padding     = kernel_size[0] // 2, kernel_size[1] // 2
        self.bias        = bias
        
        self.conv = nn.Conv2d(in_channels=self.input_dim + self.hidden_dim,
                              out_channels=4 * self.hidden_dim,
                              kernel_size=self.kernel_size,
                              padding=self.padding,
                              bias=self.bias)
        self.device = device

    def forward(self, input_tensor, cur_state):
        
        
        
        h_cur, c_cur = cur_state
        
        
        combined = torch.cat([input_tensor, h_cur], dim=1).to(self.device) # concatenate along channel axis
        

        combined_conv = self.conv(combined).to(self.device)
        cc_i, cc_f, cc_o, cc_g = torch.split(combined_conv, self.hidden_dim, dim=1) 
        i = torch.sigmoid(cc_i)
        f = torch.sigmoid(cc_f)
        o = torch.sigmoid(cc_o)
        g = torch.tanh(cc_g)

        c_next = f * c_cur + i * g
        h_next = o * torch.tanh(c_next)
        
        return h_next, c_next

    def init_hidden(self, batch_size):
        return (Variable(torch.zeros(batch_size, self.hidden_dim, self.height, self.width), requires_grad = True).to(self.device),
                Variable(torch.zeros(batch_size, self.hidden_dim, self.height, self.width), requires_grad = True).to(self.device))


class ConvLSTM(nn.Module):

    def __init__(self, input_size, input_channels, hidden_dim, kernel_size, num_layers,
                 target=[[0,0]], bias=True, return_all_layers=False, device='cuda:0'):
        super(ConvLSTM, self).__init__()

        self._check_kernel_size_consistency(kernel_size)
        self.device = device
        self.target = target

        # Make sure that both `kernel_size` and `hidden_dim` are lists having len == num_layers
        kernel_size = self._extend_for_multilayer(kernel_size, num_layers)
        hidden_dim  = self._extend_for_multilayer(hidden_dim, num_layers)
        if not len(kernel_size) == len(hidden_dim) == num_layers:
            raise ValueError('Inconsistent list length.')

        self.height, self.width = input_size

        self.input_dim  = input_channels
        self.hidden_dim = hidden_dim
        self.kernel_size = kernel_size
        self.num_layers = num_layers
        self.bias = bias
        self.return_all_layers = return_all_layers

        cell_list = []
        for i in range(0, self.num_layers):
            cur_input_dim = self.input_dim if i == 0 else self.hidden_dim[i-1]

            cell_list.append(ConvLSTMCell(input_size=(self.height, self.width),
                                          input_dim=cur_input_dim,
                                          hidden_dim=self.hidden_dim[i],
                                          kernel_size=self.kernel_size[i],
                                          bias=self.bias,
                                          device = self.device))

        self.cell_list = nn.ModuleList(cell_list)

    def forward(self, input_tensor, hidden_state=None):

        # Implement stateful ConvLSTM
        if hidden_state is not None:
            raise NotImplementedError()
        else:
            hidden_state = self._init_hidden(batch_size=input_tensor.size(0))

        layer_output_list = []
        last_state_list   = []
        out = torch.tensor([]).to(self.device)

        seq_len = input_tensor.size(2)
        cur_layer_input = input_tensor

        for layer_idx in range(self.num_layers):

            h, c = hidden_state[layer_idx]
            output_inner = []
            for t in range(seq_len):

                h, c = self.cell_list[layer_idx](input_tensor=cur_layer_input[:, :, t, :, :],
                                                 cur_state=[h, c])
                output_inner.append(h)
                
            layer_output = torch.stack(output_inner, dim=2)
            cur_layer_input = layer_output

            layer_output_list.append(layer_output)
            last_state_list.append([h, c])
            
            if [t, layer_idx] in self.target:
                    out = torch.cat([out , h], 1)

        return out

    def _init_hidden(self, batch_size):
        init_states = []
        for i in range(self.num_layers):
            init_states.append(self.cell_list[i].init_hidden(batch_size))
        return init_states

    @staticmethod
    def _check_kernel_size_consistency(kernel_size):
        if not (isinstance(kernel_size, tuple) or
                    (isinstance(kernel_size, list) and all([isinstance(elem, tuple) for elem in kernel_size]))):
            raise ValueError('`kernel_size` must be tuple or list of tuples')

    @staticmethod
    def _extend_for_multilayer(param, num_layers):
        if not isinstance(param, list):
            param = [param] * num_layers
        return param
</code></pre>