Pytorch lightening: RuntimeError: Require grad

Jayden_W: <p>Kindly assist to find out why I am still this error message: “RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn”. This code works for another test system, but after I changed test system, it has run time error.</p>
<p>I first post the error:</p>
<pre><code class="lang-auto">File "c:\users\0716w\neural_clbf\neural_clbf\training\train_microgrid.py", line 219, in &lt;module&gt;
    main (args)

  File "c:\users\0716w\neural_clbf\neural_clbf\training\train_microgrid.py", line 211, in main
    trainer.fit(clbf_controller)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 458, in fit
    self._run(model)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 756, in _run
    self.dispatch()

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 797, in dispatch
    self.accelerator.start_training(self)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\accelerators\accelerator.py", line 96, in start_training
    self.training_type_plugin.start_training(trainer)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\plugins\training_type\training_type_plugin.py", line 144, in start_training
    self._results = trainer.run_stage()

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 807, in run_stage
    return self.run_train()

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\trainer.py", line 869, in run_train
    self.train_loop.run_training_epoch()

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\training_loop.py", line 489, in run_training_epoch
    batch_output = self.run_training_batch(batch, batch_idx, dataloader_idx)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\training_loop.py", line 728, in run_training_batch
    self.optimizer_step(optimizer, opt_idx, batch_idx, train_step_and_backward_closure)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\training_loop.py", line 424, in optimizer_step
    model_ref.optimizer_step(

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\core\lightning.py", line 1403, in optimizer_step
    optimizer.step(closure=optimizer_closure)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\core\optimizer.py", line 214, in step
    self.__optimizer_step(*args, closure=closure, profiler_name=profiler_name, **kwargs)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\core\optimizer.py", line 134, in __optimizer_step
    trainer.accelerator.optimizer_step(optimizer, self._optimizer_idx, lambda_closure=closure, **kwargs)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\accelerators\accelerator.py", line 329, in optimizer_step
    self.run_optimizer_step(optimizer, opt_idx, lambda_closure, **kwargs)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\accelerators\accelerator.py", line 336, in run_optimizer_step
    self.training_type_plugin.optimizer_step(optimizer, lambda_closure=lambda_closure, **kwargs)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\plugins\training_type\training_type_plugin.py", line 193, in optimizer_step
    optimizer.step(closure=lambda_closure, **kwargs)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\torch\optim\optimizer.py", line 88, in wrapper
    return func(*args, **kwargs)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\torch\autograd\grad_mode.py", line 28, in decorate_context
    return func(*args, **kwargs)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\torch\optim\adam.py", line 66, in step
    loss = closure()

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\training_loop.py", line 722, in train_step_and_backward_closure
    result = self.training_step_and_backward(

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\training_loop.py", line 826, in training_step_and_backward
    self.backward(result, optimizer, opt_idx)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\trainer\training_loop.py", line 859, in backward
    result.closure_loss = self.trainer.accelerator.backward(

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\accelerators\accelerator.py", line 308, in backward
    output = self.precision_plugin.backward(

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\plugins\precision\precision_plugin.py", line 79, in backward
    model.backward(closure_loss, optimizer, opt_idx)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\pytorch_lightning\core\lightning.py", line 1275, in backward
    loss.backward(*args, **kwargs)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\torch\_tensor.py", line 255, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)

  File "C:\Users\0716w\anaconda3\envs\neural_gpu0422\lib\site-packages\torch\autograd\__init__.py", line 147, in backward
    Variable._execution_engine.run_backward(

RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn
````Preformatted text`




 





 
             
# The code that may produce the error is listed:
</code></pre>
<pre><code>    self.clbf_hidden_layers = clbf_hidden_layers
    self.clbf_hidden_size = clbf_hidden_size
    self.hidden_layers = [clbf_hidden_layers, clbf_hidden_size]
    # We're going to build the network up layer by layer, starting with the input
    self.f = MLP(
        self.in_dim, self.in_dim, self.hidden_layers, hidden_activation, None, init, gain
    )
    self.Q = nn.Parameter(torch.eye(self.in_dim) + 0.1 * torch.ones((self.in_dim, self.in_dim)))
    self.g = MLP(
        self.in_dim, 1, self.hidden_layers, hidden_activation, nn.ReLU(), init, gain
    )
    # Define the Controller network, which we denote unn
    self.ctrl_hidden_layers = clbf_hidden_layers
    self.ctrl_hidden_size = clbf_hidden_size
    # We're going to build the network up layer by layer, starting with the input
    self.ctrl_layers: OrderedDict[str, nn.Module] = OrderedDict()
    self.ctrl_layers["input_linear"] = nn.Linear(
        self.in_dim, self.ctrl_hidden_size
    )
    self.ctrl_layers["input_activation"] = nn.Tanh()
    for i in range(self.ctrl_hidden_layers):
        self.ctrl_layers[f"layer_{i}_linear"] = nn.Linear(
            self.ctrl_hidden_size, self.ctrl_hidden_size
        )
        if i &lt; self.ctrl_hidden_layers - 1:
            self.ctrl_layers[f"layer_{i}_activation"] = nn.Tanh()
    self.ctrl_layers["output_linear"] = nn.Linear(self.ctrl_hidden_size, self.dynamics_model.n_controls)
    self.ctrl_nn = nn.Sequential(self.ctrl_layers)



def forward(self, x):

    x_norm = (x - self.x_center.type_as(x)) / self.x_range.type_as(x)
    return self.ctrl_nn(x_norm)

def boundary_loss(
    self,
    x: torch.Tensor,
    goal_mask: torch.Tensor,
    safe_mask: torch.Tensor,
    unsafe_mask: torch.Tensor,
    accuracy: bool = False,
) -&gt; List[Tuple[str, torch.Tensor]]:

    eps = 1e-2
    # Compute loss to encourage satisfaction of the following conditions...
    loss = []

    V = self.V(x)

    #   1.) CLBF should be minimized on the goal point
    V_goal_pt = self.V(self.dynamics_model.goal_point.type_as(x))
    goal_term = 1e1 * V_goal_pt.mean()
    loss.append(("CLBF goal term", goal_term))

    # Only train these terms if we have a barrier requirement
    if self.barrier:
        #   2.) 0 &lt; V &lt;= safe_level in the safe region
        V_safe = V[safe_mask]
        safe_violation = F.relu(eps + V_safe - self.safe_level)
        safe_V_term = 1e2 * safe_violation.mean()
        loss.append(("CLBF safe region term", safe_V_term))
        if accuracy:
            safe_V_acc = (safe_violation &lt;= eps).sum() / safe_violation.nelement()
            loss.append(("CLBF safe region accuracy", safe_V_acc))

        #   3.) V &gt;= unsafe_level in the unsafe region
        V_unsafe = V[unsafe_mask]
        unsafe_violation = F.relu(eps + self.unsafe_level - V_unsafe)
        unsafe_V_term = 1e2 * unsafe_violation.mean()
        loss.append(("CLBF unsafe region term", unsafe_V_term))
        if accuracy:
            unsafe_V_acc = (
                unsafe_violation &lt;= eps
            ).sum() / unsafe_violation.nelement()
            loss.append(("CLBF unsafe region accuracy", unsafe_V_acc))

    return loss

def descent_loss(
    self,
    x: torch.Tensor,
    goal_mask: torch.Tensor,
    safe_mask: torch.Tensor,
    unsafe_mask: torch.Tensor,
    accuracy: bool = False,
    requires_grad: bool = False,
) -&gt; List[Tuple[str, torch.Tensor]]:

    # Compute loss to encourage satisfaction of the following conditions...
    loss = []


    # First figure out where this condition needs to hold
    eps = 0.1
    V = self.V(x)
    if self.barrier:
        condition_active = torch.sigmoid(10 * (self.safe_level + eps - V))
    else:
        condition_active = 1.0

    # Get the control input from neural network
    u_nn = self.ctrl_nn(x)
    u_nn_loss = (1e-3)*(torch.mean((u_nn-self.u_reference(x))**2) * condition_active).mean()
    loss.append(("u_nn loss", u_nn_loss))

    # Now compute the decrease using linearization
    eps = 1.0
    clbf_descent_term_lin = torch.tensor(0.0).type_as(x)
    clbf_descent_acc_lin = torch.tensor(0.0).type_as(x)
    # Get the current value of the CLBF and its Lie derivatives
    Lf_V, Lg_V = self.V_lie_derivatives(x)
    for i, s in enumerate(self.scenarios):
        # Use the dynamics to compute the derivative of V
        Vdot = Lf_V[:, i, :].unsqueeze(1) + torch.bmm(
            Lg_V[:, i, :].unsqueeze(1),
            u_nn.reshape(-1, self.dynamics_model.n_controls, 1),
        )
        Vdot = Vdot.reshape(V.shape)
        violation = F.relu(eps + Vdot + self.clf_lambda * V)
        violation *= condition_active
        clbf_descent_term_lin += violation.mean()
        clbf_descent_acc_lin += (violation &lt;= eps).sum() / (
            violation.nelement() * self.n_scenarios
        )

    loss.append(("CLBF descent term (linearized)", clbf_descent_term_lin))
    if accuracy:
        loss.append(("CLBF descent accuracy (linearized)", clbf_descent_acc_lin))

    # Now compute the decrease using simulation
    eps = 1.0
    clbf_descent_term_sim = torch.tensor(0.0).type_as(x)
    clbf_descent_acc_sim = torch.tensor(0.0).type_as(x)
    for s in self.scenarios:
        s_pre = PreData(s)
        xdot = self.dynamics_model.closed_loop_dynamics(x, u_nn, params=s_pre)
        x_hat = x +self.dynamics_model.dt*xdot
        x_hatdot = self.dynamics_model.closed_loop_dynamics(x_hat, u_nn, params=s_pre)
        x_next  = x + self.dynamics_model.dt * 0.5*(xdot+x_hatdot)
        V_next = self.V(x_next)
        violation = F.relu(
            eps + (V_next - V) / self.controller_period + self.clf_lambda * V
        )
        violation *= condition_active

        clbf_descent_term_sim += violation.mean()
        clbf_descent_acc_sim += (violation &lt;= eps).sum() / (
            violation.nelement() * self.n_scenarios
        )
    loss.append(("CLBF descent term (simulated)", clbf_descent_term_sim))
    if accuracy:
        loss.append(("CLBF descent accuracy (simulated)", clbf_descent_acc_sim))

    return loss


def configure_optimizers(self):
    clbf_params = list(self.parameters())

    clbf_opt = torch.optim.Adam(
        clbf_params,
        lr=self.primal_learning_rate,
        weight_decay=1e-6,
    )
    self.opt_idx_dict = {0: "clbf"}

    return [clbf_opt]
</code></pre>
<pre><code class="lang-auto"></code></pre>

srishti-git1110: <p>Could you please try to post some minimal code that reproduces the error?</p>

Jayden_W: <p>Hi Srishti, I have revised the post. Thanks for your help.</p>