Variables needed for gradient computation has been modified

giuseppedipoce: <p>Hello!<br>
I’m trying to implement the following optimization problem minimizing a loss functions built as follow:<br>
<img src="https://discuss.pytorch.org/uploads/default/original/3X/4/2/42be25a0b7abacd666c33d4046e297e67952459f.png" alt="image" data-base62-sha1="9wqTRLlsXQgmvnRt0Bdro87NIiH" width="477" height="137"></p>
<p>During the implementation i’ve developed the surrogate of agent i (f tilde) and the other term with respect to all the j other agents of a multi agent system. Trying to solve this using loss.backward() arises issued related to "variables needed for gradient computation has been modified ". This is my code:</p>
<pre><code>class Agents:
def __init__(self, train_loaders, val_loaders, test_loader, num_agents, pie = [], tau = 10, epochs=5):
    self.train_loaders = train_loaders
    self.val_loaders = val_loaders
    self.test_loader = test_loader
    self.num_agents = num_agents
    self.epochs = epochs
    self.gamma_values = [ 0.5 ]
    self.models = [Net() for _ in range(num_agents)]
    self.optimizers = [optim.SGD(model.parameters(), lr= 1/tau) for model in self.models]
    self.criterion = nn.CrossEntropyLoss()
    self.validation_accuracies = {i: 0 for i in range(num_agents)}  # Dictionary to store validation accuracies
    self.losses = {i: [] for i in range(num_agents)}  # Dictionary to store losses


def gamma_update(self, epsilon=2): #updating step size rule
    gamma_next = self.gamma_values[-1]*(1- (epsilon*self.gamma_values[-1])) #gamma[0] = 1/eps
    self.gamma_values.append(gamma_next)


def full_linearization(self, y_k, agent_idx):
      model_i = self.models[agent_idx]
      x_i = [param.clone() for param in model_i.parameters()]  # Store the current parameters (x_i)
      model_i.load_state_dict(y_k)
      model_i.train()
      # Initialize pi_i and difference with zeros (same shape as model_i state_dict)
      pi_i = {k: torch.zeros_like(v) for k, v in model_i.state_dict().items()}
      difference = {k: torch.zeros_like(v) for k, v in model_i.state_dict().items()}
      total_surrogate_loss = 0.0

      for inputs, labels in self.train_loaders[agent_idx]:
          optimizer = self.optimizers[agent_idx]
          optimizer.zero_grad()

          # Forward pass for agent i
          outputs_i = model_i(inputs)
          CE_surrogate = self.criterion(outputs_i, labels)
          total_surrogate_loss += CE_surrogate

          # Compute pi_i(x_i[n]) gradients
          for j in range(self.num_agents):
              if j != agent_idx:
                  model_j = self.models[j]
                  model_j.load_state_dict(y_k)  # Load y_k for agent j

                  for inputs_j, labels_j in self.train_loaders[j]:
                      outputs_j = model_j(inputs_j)
                      loss_j = self.criterion(outputs_j, labels_j)
                      grads = torch.autograd.grad(loss_j, model_j.parameters(), retain_graph=True)

                      # Accumulate gradients in pi_i (note the use of `clone` to avoid in-place operations)
                      for (name, param), grad in zip(model_j.named_parameters(), grads):
                          pi_i[name] = pi_i[name] + grad.clone()

          # Flatten pi_i
          pi_flatten = torch.cat([v.view(-1) for v in pi_i.values()])

          # Compute difference vector x_i - y_k
          for (name, y_k_param), x_i_param in zip(y_k.items(), x_i):
              difference[name] = difference[name] + (x_i_param - y_k_param).clone()

          diff_flat = torch.cat([v.view(-1) for v in difference.values()])

          # Add pi_flatten^T * diff_flat to the surrogate loss
          total_surrogate_loss += torch.dot(pi_flatten, diff_flat)

          # Backward pass and optimization
          total_surrogate_loss.backward(retain_graph=True)  # Using retain_graph=True since we might need it for further computations
          optimizer.step()

      post_opt = model_i.state_dict()

      avg_loss = total_surrogate_loss.item() / len(self.train_loaders[agent_idx])
      self.losses[agent_idx].append(avg_loss)
      print(f"Surrogate agent {agent_idx}: {avg_loss}")

      return pre_opt, post_opt
</code></pre>
<p>Can anyone see the issue related to this code? There are apparently no inplace operation s that modify the tensors required for the optimization procedure. This is the error:</p>
<hr>
<p>RuntimeError                              Traceback (most recent call last)<br>
 in &lt;cell line: 4&gt;()<br>
2 s = Agents(train_loaders, val_loaders, test_loader, num_agents)<br>
3 y_k = Net().state_dict()<br>
----&gt; 4 s.full_linearization(y_k, 0)</p>
<p>3 frames<br>
/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py in _engine_run_backward(t_outputs, *args, **kwargs)<br>
766         unregister_hooks = _register_logging_hooks_on_whole_graph(t_outputs)<br>
767     try:<br>
 → 768         return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass<br>
769             t_outputs, *args, **kwargs<br>
770         )  # Calls into the C++ engine to run the backward pass</p>
<p>RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [32, 10]], which is output 0 of AsStridedBackward0, is at version 4; expected version 3 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!</p>

KFrank: <p>Hi Giuseppe!</p>
<aside class="quote no-group quote-modified" data-username="giuseppedipoce" data-post="1" data-topic="208733" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/giuseppedipoce/48/72388_2.png" class="avatar"> giuseppedipoce:</div>
<blockquote>
<pre><code>          total_surrogate_loss.backward(retain_graph=True)  # Using retain_graph=True since we might need it for further computations
          optimizer.step()
</code></pre>
<p>There are apparently no inplace operation s that modify the tensors required for the optimization procedure.</p>
</blockquote>
</aside>
<p>Note that <code>optimizer.step()</code> performs inplace modifications on<br>
the <code>Parameter</code>s it is optimizing.  This, together with your use of<br>
<code>retain_graph = True</code> strongly suggests that <code>optimizer.step()</code><br>
is the cause of your issue.</p>
<p>You should think through with care how you are performing your<br>
optimization.  Do you really want to be optimizing an “agent” with<br>
a stale computation graph?  Or is your use of <code>retain_graph = True</code><br>
just an expedient to avoid doing the necessary bookkeeping?</p>
<p>In your case, I think you should track down the root cause of your<br>
error and fix it because it likely indicates a flaw in the design of your<br>
optimization loop.</p>
<p>However, you could also try fixing your issue “automatically” by<br>
using pytorch’s <a href="https://pytorch.org/docs/2.4/autograd.html#torch.autograd.graph.allow_mutation_on_saved_tensors" rel="noopener nofollow ugc">sweep-inplace-modification-errors-under-the-rug</a><br>
context manager.</p>
<p>Best.</p>
<p>K. Frank</p>

giuseppedipoce: <p>Dear KFrank, thanks a lot for your answer.<br>
Retain graph is just an expedit to be alble to let work the code. The main issue with i’m struggling is tha fact that I wanto to optimize a strongly convex optimization problem like this:<br>
<img src="https://discuss.pytorch.org/uploads/default/original/3X/5/c/5c3c53e0766f6bbaea27318c6cdd65686ee51592.png" alt="image" data-base62-sha1="d9Xb4M9PQeX8AiRxGi7NELQYNnY" width="612" height="85"><br>
Where f is the loss function (f_tilde is a surrogate loss function) of i^th/j^th agent of the system.<br>
This problem is the sum of basically two terms (f_tilde and the other one that is like a first order taylor expansion). <em><strong>The issue by a code point of view is that I’m not able to optimize wrt the whole optimization problem, how can I try to implement this in torch? Can I do it using the simple .backward() procedure?</strong></em></p>
<p>Thanks in advice for your time!</p>

KFrank: <p>Hi Giuseppe!</p>
<aside class="quote no-group" data-username="giuseppedipoce" data-post="3" data-topic="208733" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/giuseppedipoce/48/72388_2.png" class="avatar"> giuseppedipoce:</div>
<blockquote>
<p><em><strong>The issue by a code point of view is that I’m not able to optimize wrt the whole optimization problem, how can I try to implement this in torch?</strong></em></p>
</blockquote>
</aside>
<p>I’m not sure what you’re asking here.</p>
<p>Do you believe that the code you originally posted would perform<br>
your optimization procedure correctly if you could resolve your<br>
inplace-modification error?</p>
<p>Or is the code you posted just a red herring and your real question<br>
is how to use pytorch to solve your optimization problem?</p>
<p>Best.</p>
<p>K. Frank</p>

giuseppedipoce: <p>Hi Frank!<br>
No, this code posted above is not a red herring.<br>
I’ve changed the way how I’m updating the variable that keep track of the loss function (total_surrogate_loss)  and it seems that the inplace-modification issue is solved.<br>
By the way there is something strange in the optimization procedure, sometimes it works and in other run times arises a different error:<br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/8/3/837ec9ed2a95d7f71fd85c1cad6ff82a73de2a28.png" data-download-href="https://discuss.pytorch.org/uploads/default/837ec9ed2a95d7f71fd85c1cad6ff82a73de2a28" title="image"><img src="https://discuss.pytorch.org/uploads/default/original/3X/8/3/837ec9ed2a95d7f71fd85c1cad6ff82a73de2a28.png" alt="image" data-base62-sha1="iLgbViQaZX4y0lkbk4TrWF6ClrW" width="690" height="234" data-dominant-color="3E3D3E"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">1350×458 29 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div><br>
I believe that the implemented optimization procedure should be correct but I have been stuck for days on this point and I’m asking if someone can see some issue that i’m not able to see. I attach here the last implementation code :</p>
<pre><code>class Agents:
def __init__(self, train_loaders, val_loaders, test_loader, num_agents, pie = [], tau = 100, epochs=5):
    self.train_loaders = train_loaders
    self.val_loaders = val_loaders
    self.test_loader = test_loader
    self.num_agents = num_agents
    self.epochs = epochs
    self.gamma_values = [np.random.uniform(0.2,1/0.5)]
    self.models = [Net() for _ in range(num_agents)]
    self.optimizers = [optim.SGD(model.parameters(), lr= 1/tau) for model in self.models]
    self.criterion = nn.CrossEntropyLoss()
    self.validation_accuracies = {i: 0 for i in range(num_agents)}  # Dictionary to store validation accuracies
    self.losses = {i: [] for i in range(num_agents)}  # Dictionary to store losses


def gamma_update(self, epsilon=0.5): #updating step size rule
    gamma_next = self.gamma_values[-1]*(1- (epsilon*self.gamma_values[-1])) #gamma[0] &lt; 1/eps and eps in (0,1])
    self.gamma_values.append(gamma_next)


def centralized_fullL(self, y_k, agent_idx): #best response map implementation
  model_i = self.models[agent_idx]
  x_i = [param.clone() for param in model_i.parameters()]  # Store the current parameters (x_i)
  model_i.load_state_dict(y_k)
  model_i.train()

  # Initialize pi_i and difference with zeros (same shape as model_i state_dict)
  pi_i = {k: torch.zeros_like(v) for k, v in model_i.state_dict().items()}
  difference = {k: torch.zeros_like(v) for k, v in model_i.state_dict().items()}

  for inputs, labels in self.train_loaders[agent_idx]:
      optimizer = self.optimizers[agent_idx]
      optimizer.zero_grad()

      # Forward pass for agent i
      outputs_i = model_i(inputs)
      surrogate_loss = self.criterion(outputs_i, labels)  # Surrogate loss for agent i

      # Accumulate gradients from other agents
      for j in range(self.num_agents):
          if j != agent_idx:
              model_j = self.models[j]
              model_j.load_state_dict(y_k)  # Load y_k for agent j

              for inputs_j, labels_j in self.train_loaders[j]:
                  outputs_j = model_j(inputs_j)
                  loss_j = self.criterion(outputs_j, labels_j)
                  grads = torch.autograd.grad(loss_j, model_j.parameters(), retain_graph=True)

                  for (name, param), grad in zip(model_j.named_parameters(), grads):
                      pi_i[name] += grad.clone()  # Accumulate gradients

      # Flatten pi_i
      pi_flatten = torch.cat([v.view(-1) for v in pi_i.values()])

      # Compute difference vector x_i - y_k
      for (name, y_k_param), x_i_param in zip(y_k.items(), x_i):
          difference[name] += (x_i_param - y_k_param).clone()

      diff_flat = torch.cat([v.view(-1) for v in difference.values()])

      # Add pi_flatten^T * diff_flat to the surrogate loss
      surrogate_loss += torch.dot(pi_flatten, diff_flat)

      # Backward pass and optimization
      surrogate_loss.backward()
      optimizer.step()

  avg_loss = surrogate_loss.item() / len(self.train_loaders[agent_idx])
  self.losses[agent_idx].append(avg_loss)
  print(f"Surrogate agent {agent_idx}: {avg_loss}")
  return x_i, model_i.state_dict()
</code></pre>
<p>Thanks,</p>
<p>Giuseppe</p>