RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation. could you please solve it

username: vaishu-99
cooked: <p>This is my code<br>
"<br>
import warnings<br>
warnings.filterwarnings(‘ignore’)<br>
warnings.simplefilter(‘ignore’)<br>
import torch, yaml, cv2, os, shutil<br>
import torch.nn as nn<br>
import numpy as np<br>
np.random.seed(0)<br>
import matplotlib.pyplot as plt<br>
import time<br>
import os.path<br>
from tqdm import trange<br>
from PIL import Image<br>
from models.yolo import Model<br>
from utils.torch_utils import intersect_dicts<br>
from utils.datasets import letterbox<br>
from utils.general import xywh2xyxy<br>
from utils.general import non_max_suppression, scale_coords<br>
import torch</p>
<p><span class="hashtag-raw">#from</span> pytorch_grad_cam import GradCAMPlusPlus, GradCAM, XGradCAM<br>
from pytorch_grad_cam import (<br>
GradCAM, HiResCAM, ScoreCAM, GradCAMPlusPlus,<br>
AblationCAM, XGradCAM, EigenCAM, EigenGradCAM,<br>
LayerCAM, FullGrad, GradCAMElementWise<br>
)<br>
from pytorch_grad_cam.utils.image import show_cam_on_image<br>
from pytorch_grad_cam.activations_and_gradients import ActivationsAndGradients</p>
<p>class yolov7_heatmap:<br>
def <strong>init</strong>(self, weight, cfg, device, method, layer, backward_type, conf_threshold, ratio):<br>
device = torch.device(device)<br>
<span class="hashtag-raw">#ckpt</span> = torch.load(weight,map_location=torch.device(‘cpu’))<br>
ckpt = torch.load(weight)<br>
model_names = ckpt[‘model’].names<br>
csd = ckpt[‘model’].float().state_dict()  # checkpoint state_dict as FP32<br>
model = Model(cfg, ch=3, nc=len(model_names)).to(device)<br>
csd = intersect_dicts(csd, model.state_dict(), exclude=[‘anchor’])  # intersect<br>
model.load_state_dict(csd, strict=False)  # load<br>
model.eval()</p>
<pre><code>    target_layers = [eval(layer)]
    method = eval(method)

    colors = np.random.uniform(0, 255, size=(len(model_names), 3)).astype(np.int_)
    self.__dict__.update(locals())

def post_process(self, result):
    boxes_ = result[0][..., :4]
    logits_ = []
    #print(logits_)
    for data in result[1]:
        bs, n, w, h, _ = data.size()
        logits_.append(data.reshape((bs, n * w * h, _)))
    logits_ = torch.cat(logits_, dim=1)[..., 4:]
    sorted, indices = torch.sort(logits_[..., 0], descending=True)
    logits_ = logits_[0][indices[0]]
    logits_[:, 0] = torch.sigmoid(logits_[:, 0])
    print("post result size------------ : ",logits_.size())
    print("logits_--------------: " , logits_)
    print("boxes_",boxes_)
    print("boxes_[0][indices[0]]----: ",boxes_[0][indices[0]])
    print("xywh2xyxy(boxes_[0][indices[0]]).cpu().detach().numpy()-------------:",  xywh2xyxy(boxes_[0][indices[0]]).cpu().detach().numpy())
    return logits_, xywh2xyxy(boxes_[0][indices[0]]).cpu().detach().numpy()
    
def post_process_2(self,result):
    """
    Function to extract coordinates (boxes) and confidence scores from the tensor.
    Args:
        tensor (torch.Tensor): Input tensor containing detection data.
    Returns:
        torch.Tensor: Extracted coordinates.
        list of str: Extracted and formatted confidence scores.
    """
    # Extract the coordinates
    boxes_ = result[..., :4]
    # Extract the confidence scores
    confidence = result[..., 4]

    # Format confidence values
    confidence =torch.sigmoid(confidence)
    print("boxes _ : ", boxes_)
    print("xywh2xyxy(boxes_).cpu().detach().numpy() --- :", boxes_.cpu().detach().numpy())
    return confidence, xywh2xyxy(boxes_).cpu().detach().numpy()


def draw_detections(self, box, color, name, img):
    xmin, ymin, xmax, ymax = list(map(int, list(box)))
    f=open("Traffic_2560x1600_30_0118_060.csv","a")
    f.write(name)
    f.write(" : ")
    f.write(str(xmin))
    f.write(",")
    f.write(str(ymin))
    f.write(",")
    f.write(str(xmax))
    f.write(",")
    f.write(str(ymax))
    f.write("\n")
    f.close()
    
    cv2.rectangle(img, (xmin, ymin), (xmax, ymax), tuple(int(x) for x in color), 2)
    cv2.putText(img, str(name), (xmin, ymin - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.8, tuple(int(x) for x in color), 2, lineType=cv2.LINE_AA)
    
    return img
    
def combine_images(self, folder_path, output_path):
    images = []
    
    # Iterate through all files in the folder
    for filename in os.listdir(folder_path):
        if filename.endswith(".jpg") or filename.endswith(".png"):
            img_path = os.path.join(folder_path, filename)
            img = Image.open(img_path)
            images.append(img)
    
    # Get the dimensions of the first image
    width, height = images[0].size
    
    # Create a new image with a height that can accommodate all images
    combined_img = Image.new("RGB", (width, len(images) * height))
    
    # Paste each image into the combined image
    y_offset = 0
    for img in images:
        combined_img.paste(img, (0, y_offset))
        y_offset += height
    
    # Save the combined image
    combined_img.save(f'{output_path}/{filename}.png')
    
def __call__(self, img_path, save_path):
    # remove dir if exist
    if os.path.exists(save_path):
        shutil.rmtree(save_path)
    # make dir if not exist
    os.makedirs(save_path, exist_ok=True)
    

    # img process
    img = cv2.imread(img_path)
    img = letterbox(img)[0]
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img = np.float32(img) / 255.0
    tensor = torch.from_numpy(np.transpose(img, axes=[2, 0, 1])).unsqueeze(0).to(self.device)
    
    count=1
    #torch.autograd.set_detect_anomaly(True)
    # init ActivationsAndGradients
    print(f'The count is : {count}  start................')
    grads = ActivationsAndGradients(self.model, self.target_layers, reshape_transform=None)
    # get ActivationsAndResult
    result = grads(tensor)
    activations = grads.activations[0].cpu().detach().numpy()
    #print("results for the images after going through the model........................... \n", result )
    
    conf_thres = 0.25
    iou_thres = 0.5 
    predictions = non_max_suppression(result[0], conf_thres, iou_thres)
    
    """Runs Non-Maximum Suppression (NMS) on inference results

    Returns:
     list of detections, on (n,6) tensor per image [xyxy, conf, cls]
    """
    
    #print("activations are ..................\n", activations)
    
    print("Here is the result ........................\n")
    print(result)
    print("\n Here is the predictions ---------------------------\n")
    #print(torch.eq(result,predictions))

    # postprocess to yolo output
    post_result, post_boxes = self.post_process(result)
    print("\n result size------------ : ")
    #.......................................................................................................................................
    print(" type of all ------------ : \n")
    def check_tensor_type(tensor):
        if isinstance(tensor, tuple):
            print("This is a tuple-type tensor")
        elif isinstance(tensor, list):
            print("This is a list-type tensor")
        elif isinstance(tensor, torch.Tensor):
            print("This is a regular PyTorch tensor")
        else:
            print("This is not a recognized tensor type")
    check_tensor_type(result)
    check_tensor_type(predictions)
    check_tensor_type(post_boxes)
    check_tensor_type(post_result)
    
    
    post_result, post_boxes = self.post_process_2(predictions[0])
    print("\n post_result[0]------------------: ",post_result[0])
    print("post_result[0].requires_grad----------:",post_result[0].requires_grad)
    self.model.zero_grad()
    if self.backward_type == 'conf':
            #confidence = confidence.type(torch.cuda.FloatTensor)
            post_result[0].backward(retain_graph=True, create_graph=True)
    else:
        # get max probability for this prediction
        class_id.backward(retain_graph=True)
        
    gradients = grads.gradients[0]
    b, k, u, v = gradients.size()
    weights = self.method.get_cam_weights(self.method, None, None, None, activations, gradients.detach().numpy())
    weights = weights.reshape((b, k, 1, 1))
    saliency_map = np.sum(weights * activations, axis=1)
    saliency_map = np.squeeze(np.maximum(saliency_map, 0))
    saliency_map = cv2.resize(saliency_map, (tensor.size(3), tensor.size(2)))
    saliency_map_min, saliency_map_max = saliency_map.min(), saliency_map.max()
    #if (saliency_map_max - saliency_map_min) == 0:
     #   continue
    saliency_map = (saliency_map - saliency_map_min) / (saliency_map_max - saliency_map_min)
    # add heatmap and box to image
    cam_image = show_cam_on_image(img.copy(), saliency_map, use_rgb=True)
    cam_image = self.draw_detections(post_boxes[i], self.colors[int(post_result[i, 1:].argmax())], f'{self.model_names[int(post_result[i, 1:].argmax())]} {post_result[i][0]:.2f}', cam_image)
    total_frames=total_frames+1
    cam_image = Image.fromarray(cam_image)
    #cv2.imwrite("tokkalo_reuslt.png",cam_image)
    cam_image.save(f'{save_path}/{i}.png')
    
#&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;&amp;7    
</code></pre>
<p>def get_params():<br>
params = {<br>
‘weight’: ‘yolov7.pt’,<br>
‘cfg’: ‘cfg/training/yolov7.yaml’,<br>
‘device’: ‘cuda:0’,<br>
‘method’: ‘GradCAM’, # GradCAMPlusPlus, GradCAM, XGradCAM<br>
‘layer’: ‘model.model[-2]’,<br>
‘backward_type’: ‘conf’, # class or conf<br>
‘conf_threshold’: 0.6, # 0.6<br>
‘ratio’: 0.02 # 0.02-0.1<br>
}<br>
return params</p>
<p>if <strong>name</strong> == ‘<strong>main</strong>’:<br>
model = yolov7_heatmap(**get_params())<br>
f=open(“Traffic_2560x1600_30_0118_060.csv”,“w”)<br>
f.write(“name : xmin, ymin, xmax, ymax \n”)<br>
f.close()<br>
model(‘Input/image3.jpg’, ‘output/output/image3_2’)</p>
<pre><code>'''
input_folder='Input/Traffic_2560x1600_30_0118_10frames'
k=0
for image in os.listdir(input_folder):
    print(image)
    #print(image.type)
    image_path=os.path.join(input_folder,image)
    output_path=os.path.join('/mnt/beegfs/home/vdhulipudi2023/venv_example/pytorch-grad-cam/tutorials/yolov7/output/10frames',image)
    print(output_path)
    model(image_path,output_path)
    ''' 
</code></pre>
<p>and I am getting this error<br>
“Traceback (most recent call last):<br>
File “/mnt/beegfs/home/vdhulipudi2023/venv_example/pytorch-grad-cam/tutorials/yolov7/z.py”, line 251, in <br>
model(‘Input/image3.jpg’, ‘output/output/image3_2’)<br>
File “/mnt/beegfs/home/vdhulipudi2023/venv_example/pytorch-grad-cam/tutorials/yolov7/z.py”, line 202, in <strong>call</strong><br>
post_result[0].backward(retain_graph=True, create_graph=True)<br>
File “/mnt/beegfs/home/vdhulipudi2023/venv_example/new_env/lib/python3.12/site-packages/torch/_tensor.py”, line 525, in backward<br>
torch.autograd.backward(<br>
File “/mnt/beegfs/home/vdhulipudi2023/venv_example/new_env/lib/python3.12/site-packages/torch/autograd/<strong>init</strong>.py”, line 267, in backward<br>
_engine_run_backward(<br>
File “/mnt/beegfs/home/vdhulipudi2023/venv_example/new_env/lib/python3.12/site-packages/torch/autograd/graph.py”, line 744, in _engine_run_backward<br>
return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass<br>
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^<br>
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [15, 1]], which is output 0 of AsStridedBackward0, is at version 2; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).” how can i resolve this ? actually after doing NMS I got this error</p>

username: soulitzer
cooked: <p>Would the following work for you? <a href="https://pytorch.org/docs/stable/autograd.html#torch.autograd.graph.allow_mutation_on_saved_tensors" class="inline-onebox" rel="noopener nofollow ugc">Automatic differentiation package - torch.autograd — PyTorch 2.3 documentation</a></p>

username: KFrank
cooked: <p>Hi Vaishnavi!</p>
<aside class="quote no-group" data-username="vaishu-99" data-post="1" data-topic="205648" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/v/f14d63/48.png" class="avatar"> vaishu-99:</div>
<blockquote>
<p>RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [15, 1]], which is output 0 of AsStridedBackward0, is at version 2; expected version 0 instead.</p>
</blockquote>
</aside>
<p>Soulitzer’s suggestion of using <code>allow_mutation_on_saved_tensors():</code><br>
may be appropriate for your use case.</p>
<p>But if you don’t need to modify tensors inplace (maybe you’re doing it<br>
inadvertently somewhere), it may makes sense to fix the underlying issue.</p>
<p>To track things down, start with the information in your error message.<br>
You have a tensor of shape <code>[15, 1]</code> that is being modified inplace.<br>
Where in your forward pass do you have such a tensor (or tensors)? Its<br>
<code>._version</code> property is changing from <code>0</code> to <code>2</code>.  Try printing out <code>t._version</code><br>
at various place in your code to see where <code>t._version</code> increases from <code>0</code><br>
to <code>1</code> (one inplace modification) and then from <code>1</code> to <code>2</code> (a second inplace<br>
modification).</p>
<p>Does your suspect tensor have <code>._version = 2</code> just before your call to<br>
<code>post_result[0].backward(retain_graph=True, create_graph=True)</code>?</p>
<p>(Note, if you find and fix any inplace-modification errors for a given tensor,<br>
errors for additional tensors may show up – autograd aborts the backward<br>
pass by raising the <code>RuntimeError</code> you saw, so it only flags the first error<br>
it encounters.)</p>
<aside class="quote no-group">
<blockquote>
<p>Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).</p>
</blockquote>
</aside>
<p>You should use anomaly detection – it provides additional information that<br>
can help you track down your error.</p>
<aside class="quote no-group">
<blockquote>
<p>This is my code</p>
</blockquote>
</aside>
<p>Some comments that may or may not be relevant to your error:</p>
<aside class="quote no-group">
<blockquote>
<p>class yolov7_heatmap:<br>
def <strong>init</strong>(self, weight, cfg, device, method, layer, backward_type, conf_threshold, ratio):<br>
…<br>
model = Model(cfg, ch=3, nc=len(model_names)).to(device)</p>
</blockquote>
</aside>
<p>You do define <code>model</code> in your <code>__init__()</code> function, but, in the code as<br>
posted, it is a local variable to <code>__init__()</code>.  You do not define <code>self.model</code>.</p>
<aside class="quote no-group">
<blockquote>
<pre><code>def __call__(self, img_path, save_path):
    ...
    grads = ActivationsAndGradients(self.model, self.target_layers, reshape_transform=None)
</code></pre>
</blockquote>
</aside>
<p>Your <code>__call__()</code> function uses <code>self.model</code>, but it is not defined in your<br>
code, as posted.</p>
<aside class="quote no-group">
<blockquote>
<pre><code>def post_process(self, result):
    ...
    logits_[:, 0] = torch.sigmoid(logits_[:, 0])
</code></pre>
</blockquote>
</aside>
<p>Assigning into a tensor by indexing the tensor is an inplace modification<br>
and is sometimes the cause of inplace-modification errors.  But it doesn’t<br>
appear that you are using the result of <code>post_process()</code> in your forward<br>
pass.</p>
<p>The following post discusses how to debug inplace-modification errors<br>
in some detail:</p>
<aside class="quote quote-modified" data-post="7" data-topic="171826">
  <div class="title">
    <div class="quote-controls"></div>
    <img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/k/ecb155/48.png" class="avatar">
    <a href="https://discuss.pytorch.org/t/runtimeerror-one-of-the-variables-needed-for-gradient-computation-has-been-modified-by-an-inplace-operation-torch-floattensor-64-1-which-is-output-0-of-asstridedbackward0-is-at-version-3-expected-version-2-instead-hint-the-backtrace-further-a/171826/7">"RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [64, 1]], which is output 0 of AsStridedBackward0, is at version 3; expected version 2 instead. Hint: the backtrace further a</a> <a class="badge-category__wrapper " href="/c/autograd/7"><span data-category-id="7" style="--category-badge-color: #AB9364; --category-badge-text-color: #FFFFFF;" data-drop-close="true" class="badge-category " title="A category of posts relating to the autograd engine itself."><span class="badge-category__name">autograd</span></span></a>
  </div>
  <blockquote>
    Hi Fahmyadan and Sangyoon! 

Here are some suggestions about how to track down (and maybe fix) 
inplace-modification errors.  Note that an inplace modification in the forward 
pass is not necessarily* an error – it depends on whether and how the tensor 
that was modified is used in the backward pass.  Note that inplace operations 
can be useful for saving memory – if you replace an innocent inplace operation 
with an out-of-place equivalent, your training will use more memory (and, to a 
minor e…
  </blockquote>
</aside>

<p>Good luck!</p>
<p>K. Frank</p>