DDPG in pytorch

Ibrahim_Shaer: <p>Hello everyone. I am using the DDPG implementation provided by <a href="https://github.com/ghliu/pytorch-ddpg" class="inline-onebox" rel="noopener nofollow ugc">GitHub - ghliu/pytorch-ddpg: Implementation of the Deep Deterministic Policy Gradient (DDPG) using PyTorch</a> and adapting it to my purposes. However, I am facing some issues with the propagation of the actor’s loss as I am getting <code>None</code> for its gradients. I will attach a code snippet of the function and all the helper functions that it uses.</p>
<pre><code class="lang-auto">def optimize(self):
        
        if self.rm.len &lt; (self.size_buffer):
            return

        self.state_encoder.eval()
        state, idx, action, set_actions, reward, next_state, curr_perf, curr_acc, done = self.rm.sample(self.batch_size)

        state = torch.from_numpy(state)
        next_state = torch.from_numpy(next_state)
        set_actions = torch.from_numpy(set_actions)
        action = torch.from_numpy(action)
        reward = [r[-1] for r in reward]
        reward = np.expand_dims(np.array(reward), axis = 1)
        reward = torch.from_numpy(np.array(reward))
        reward = reward.cuda()
        done = np.expand_dims(done, axis = 1)
        terminal = torch.from_numpy(done)
        terminal = terminal.cuda()

        # ------- optimize critic ----- #
        state = state.cuda()
        next_state = next_state.cuda()
        a_pred = self.target_actor(next_state)
        pred_perf = self.train_actions(set_actions, a_pred.data, idx, terminal)

        pred_perf = torch.from_numpy(pred_perf)

        new_set_states = torch.Tensor()
        for idx_s, single_state in enumerate(next_state):
            new_state = single_state
            if done[idx_s]:
                next_indx = int(idx[idx_s])
            else:
                if idx[idx_s] &lt; 5:
                    next_indx = int(idx[idx_s] + 1)
                else:
                    next_indx = int(idx[idx_s])
                    new_state[next_indx, :] = self.state_encoder(a_pred[idx_s].data.cpu().float(), pred_perf[idx_s].cpu().float())
            new_state = new_state[None, :]
            new_set_states = torch.cat((new_set_states, new_state.cpu()), dim = 0)
        new_set_states = torch.from_numpy(np.array(new_set_states))
        new_set_states = new_set_states.cuda()
        target_values = torch.add(reward, torch.mul(~terminal, self.target_critic(new_set_states)))


        val_expected = self.critic(next_state)

        criterion = nn.MSELoss()
        loss_critic = criterion(target_values, val_expected)
        print(val_expected, target_values, loss_critic)
        self.critic_optimizer.zero_grad()

        loss_critic.backward()
        self.critic_optimizer.step()
        for name, param in self.critic.named_parameters():
            print('here', name, param.grad, param.requires_grad, param.is_leaf)

        # ----- optimize actor ----- #
        pred_a1 = self.actor(state)
        # pred_perf = self.train_actions(idx, pred_a1, curr_arch, inputs)
        pred_perf = self.train_actions(set_actions, pred_a1.data, idx, terminal)

        pred_perf = torch.from_numpy(pred_perf)
        new_set_states = torch.Tensor()
        for idx_s, single_state in enumerate(state):
            new_state = single_state
            if done[idx_s]:
                next_indx = int(idx[idx_s])
            else:
                if idx[idx_s] &lt; 5:
                    next_indx = int(idx[idx_s] + 1)
                else:
                    next_indx = int(idx[idx_s])
                    new_state[next_indx, :] = self.state_encoder(pred_a1[idx_s].data.cpu().float(), pred_perf[idx_s].cpu().float())
            new_state = new_state[None, :]
            new_set_states = torch.cat((new_set_states, new_state.cpu()), dim = 0)
        new_set_states = torch.from_numpy(np.array(new_set_states))
        new_set_states = new_set_states.cuda()

        # self.actor_optimizer.zero_grad()


        # print(a_pred, pred_perf)
        # print(self.critic(new_set_states), -1*self.critic(new_set_states), (-1*self.critic(new_set_states)).mean())
        # loss_actor = (-1*self.critic(new_set_states)).mean()
        loss_fn = CustomLoss(self.actor, self.critic)

        loss_actor = loss_fn(new_set_states)
        # print('loss_actor', loss_actor)
        self.actor_optimizer.zero_grad()

        loss_actor.backward()
        self.actor_optimizer.step()
        for name, param in self.actor.named_parameters():
            print('here', name, param.grad, param.requires_grad, param.is_leaf)
        # self.actor_optimizer.step()

        # self.losses['actor_loss'].append(loss_actor.item())
        self.losses['critic_loss'].append(loss_critic.item())
        
        TAU = 0.001

        self.utils.soft_update(self.target_actor, self.actor, TAU)
        self.utils.soft_update(self.target_critic, self.critic, TAU)
</code></pre>
<p>The code for the CustomLoss is as follows:</p>
<pre><code class="lang-auto">class CustomLoss(nn.Module):

    def __init__(self, actor, critic):
        super(CustomLoss, self).__init__()
        self.actor = actor
        self.critic = critic

    def forward(self, state):
        loss = torch.mul(-1, self.critic(state))
        loss = loss.mean()

        return loss
</code></pre>
<p>Any help is really appreciated.</p>