RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [64, 64, 1, 1]], which is output 0 of ReluBackward0, is at version 1; expected version 0 instead

username: Chao_Fan
cooked: <p>hi, i am facing this problem and i don’t know how to solve it.</p>
<p>import pickle<br>
import torch<br>
import torch.nn as nn<br>
import torch.optim as optim<br>
from pandas.plotting._matplotlib import hist<br>
from torchvision.transforms import transforms<br>
from torch.utils.data import DataLoader<br>
import torchvision.datasets as datasets<br>
import numpy as np<br>
import pandas as pd<br>
import sys<br>
import json<br>
import os<br>
import sklearn.metrics as metrics<br>
from watermark_regularizer import WatermarkRegularizer<br>
from watermark_regularizer import get_wmark_regularizer<br>
from resnet import ResNet</p>
<h1>
<a name="set-device-1" class="anchor" href="#set-device-1"></a>set device</h1>
<p>device = torch.device(‘cuda’ if torch.cuda.is_available() else ‘cpu’)</p>
<p>RESULT_PATH = ‘./result’<br>
MODEL_CHKPOINT_FNAME = os.path.join(RESULT_PATH, ‘WRN-Weights.pth’)</p>
<p>def update_hdf5(fname, path, data):<br>
store = pd.HDFStore(fname)<br>
if path in store.keys():<br>
store.remove(path)<br>
store.append(path, data)<br>
store.close()</p>
<p>def save_wmark_signatures(prefix, model):<br>
for layer_id, wmark_regularizer in get_wmark_regularizer(model):<br>
fname_w = prefix + ‘_layer{}_w.npy’.format(layer_id)<br>
fname_b = prefix + ‘_layer{}_b.npy’.format(layer_id)<br>
np.save(fname_w, wmark_regularizer.get_matrix())<br>
np.save(fname_b, wmark_regularizer.get_signature())</p>
<p>lr_schedule = [60, 120, 160]</p>
<p>def schedule(epoch_idx):<br>
if(epoch_idx + 1) &lt; lr_schedule[0]:<br>
return 0.1<br>
elif(epoch_idx + 1) &lt; lr_schedule[1]:<br>
return 0.02<br>
elif(epoch_idx + 1) &lt; lr_schedule[2]:<br>
return 0.004<br>
return 0.0008</p>
<p>if <strong>name</strong> == ‘<strong>main</strong>’:<br>
if len(sys.argv) &lt; 2:<br>
print(“Fatal: You forgot to include the directory name on the command line.”)<br>
print(“Usage:  python %s ” % sys.argv[0])<br>
sys.exit(1)<br>
settings_json_fname = sys.argv[1]<br>
train_settings = json.load(open(settings_json_fname))</p>
<pre><code>if not os.path.isdir(RESULT_PATH):
    os.makedirs(RESULT_PATH)

torch.autograd.set_detect_anomaly(True)  # enable anomaly detection

# load dataset and fitting data for learning
if train_settings['dataset'] == 'cifar10':
    transform_train = transforms.Compose([transforms.RandomCrop(32, padding=4),
                                          transforms.RandomHorizontalFlip(),
                                          transforms.ToTensor(),
                                          transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
                                          ])
    transform_test = transforms.Compose([transforms.ToTensor(),
                                         transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
                                         ])

    train_set = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)
    test_set = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_test)

    train_loader = DataLoader(train_set, batch_size=train_settings['batch_size'], shuffle=True)
    test_loader = DataLoader(test_set, batch_size=train_settings['batch_size'], shuffle=False)
    nb_classes = 10
else:
    print('not supported dataset"{}"'.format(train_settings['dataset']))
    exit(1)

if 'replace_train_y' in train_settings and len(train_settings['replace_train_y']) &gt; 0:
    print('train_loader was replaced from"{}"'.format(train_settings['replace_train_y']))
    # train_loader = np.load(train_settings['replace_train_y'])
    train_loader = torch.from_numpy(np.load(train_settings['replace_train_y']))

# read parameters
batch_size = train_settings['batch_size']
nb_epoch = train_settings['epoch']
scale = train_settings['scale']
embed_dim = train_settings['embed_dim']
N = train_settings['N']
k = train_settings['k']

target_blk_id = train_settings['target_blk_id']
base_modelw_fname = train_settings['base_modelw_fname']
wtype = train_settings['wmark_wtype']
randseed = train_settings['randseed'] if 'randseed' in train_settings else 'none'
ohist_fname = train_settings['history']
hist_hdf_path = 'WTYPE_{}/DIM{}/SCALE{}/N{}K{}B{}EPOCH{}/TBLK{}'.format(wtype, embed_dim, scale, N, k, batch_size, nb_epoch, target_blk_id)
modelname_prefix = os.path.join(RESULT_PATH, 'wrn_' + hist_hdf_path.replace('/', '_'))

# initialize process for watermark
b = np.ones((1, embed_dim))
wmark_regularizer = WatermarkRegularizer(scale, b, wtype=wtype, randseed=randseed)
init_shape = (3, 32, 32)
# model = ResNet(init_shape, nb_classes)
# model = ResNet(init_shape, image_channels=3, num_classes=10, layers=3)
model = ResNet(image_channels=3, num_classes=10, layers=[2, 3, 4])

print(model)
print('watermark matrix: \n{}'.format(wmark_regularizer.get_matrix()))

# training process
optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9, nesterov=True)
criterion = nn.CrossEntropyLoss()
scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda x: schedule(x))

if len(base_modelw_fname) &gt; 0:
    model.load_state_dict(torch.load(base_modelw_fname))
print("finished compiling")

# train network
for epoch in range(nb_epoch):
    losses = []

    for batch_idx, (data, targets) in enumerate(train_loader):
        # get data to cuda if possible
        data = data.to(device=device)
        targets = targets.to(device=device)

        # forward
        scores = model(data)
        loss = criterion(scores, targets)

        losses.append(loss.item())

        # backward
        optimizer.zero_grad()
        loss.backward()

        # gradient descent or adam step
        optimizer.step()

    print(f'Cost at epoch {epoch} is {sum(losses) / len(losses)}')

# check accuracy
def check_accuracy(loader, model):
    if loader.dataset:
        print("checking accuracy on training data")
    else:
        print("checking accuracy on test data")

    num_correct = 0
    num_samples = 0
    model.eval()

    with torch.no_grad():
        for x, y in loader:
            x = x.to(device=device)
            y = y.to(device=device)

            scores = model(x)
            _, predictions = scores.max(1)
            num_correct += (predictions == y).sum()
            num_samples += predictions.size(0)

    print(f'got {num_correct}/{num_samples} with accuracy {float(num_correct)/float(num_samples)*100:2f}')
    model.train()

check_accuracy(train_loader, model)
check_accuracy(test_loader, model)

####
torch.save(model.state_dict(), modelname_prefix + '.pt')
with open(ohist_fname, 'ab') as f:
    pickle.dump(hist.history, f)

if target_blk_id &gt; 0:
    save_wmark_signatures(model, modelname_prefix)
</code></pre>

username: ptrblck
cooked: <p>I don’t see any obvious issues in your code, but also don’t know how e.g. <code>WatermarkRegularizer</code> works and how its used.<br>
Do you see the error in during the training of the standalone model? If so, how is <code>ResNet</code> defined?</p>

username: KFrank
cooked: <p>Hi Chao!</p>
<aside class="quote no-group" data-username="Chao_Fan" data-post="1" data-topic="176938" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/chao_fan/48/58651_2.png" class="avatar"> Chao_Fan:</div>
<blockquote>
<p>i am facing this problem and i don’t know how to solve it.</p>
</blockquote>
</aside>
<p>Here is a post that discusses how to diagnose and fix inplace-modification<br>
errors:</p>
<aside class="quote quote-modified" data-post="7" data-topic="171826">
  <div class="title">
    <div class="quote-controls"></div>
    <img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/k/ecb155/48.png" class="avatar">
    <a href="https://discuss.pytorch.org/t/runtimeerror-one-of-the-variables-needed-for-gradient-computation-has-been-modified-by-an-inplace-operation-torch-floattensor-64-1-which-is-output-0-of-asstridedbackward0-is-at-version-3-expected-version-2-instead-hint-the-backtrace-further-a/171826/7">"RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [64, 1]], which is output 0 of AsStridedBackward0, is at version 3; expected version 2 instead. Hint: the backtrace further a</a> <a class="badge-category__wrapper " href="/c/autograd/7"><span data-category-id="7" data-drop-close="true" class="badge-category " title="A category of posts relating to the autograd engine itself."><span class="badge-category__name">autograd</span></span></a>
  </div>
  <blockquote>
    Hi Fahmyadan and Sangyoon! 

Here are some suggestions about how to track down (and maybe fix) 
inplace-modification errors.  Note that an inplace modification in the forward 
pass is not necessarily* an error – it depends on whether and how the tensor 
that was modified is used in the backward pass.  Note that inplace operations 
can be useful for saving memory – if you replace an innocent inplace operation 
with an out-of-place equivalent, your training will use more memory (and, to a 
minor e…
  </blockquote>
</aside>

<p>Please feel free to post follow-up questions if you’re still having problems<br>
with this issue.</p>
<p>Good luck!</p>
<p>K. Frank</p>

username: Chao_Fan
cooked: <p>hi , <a class="mention" href="/u/ptrblck">@ptrblck</a> . thank you for responding.<br>
this is my resnet.py</p>
<p>import torch<br>
import torch.nn as nn</p>
<p><span class="hashtag">#set</span> device<br>
device = torch.device(‘cuda’ if torch.cuda.is_available() else ‘cpu’)</p>
<p>class block(nn.Module):<br>
def <strong>init</strong>(self, in_channels, num_filters, stride=1):<br>
super(block, self).<strong>init</strong>()</p>
<pre><code>    self.conv1 = nn.Conv2d(in_channels, num_filters, kernel_size=1, stride=1, padding=0)
    self.bn1 = nn.BatchNorm2d(num_filters)
    self.conv2 = nn.Conv2d(num_filters, num_filters, kernel_size=3, stride=stride, padding=1)
    self.bn2 = nn.BatchNorm2d(num_filters)
    self.relu = nn.ReLU()

    # Add this 1x1 convolutional layer to match the number of channels in output tensor with input tensor
    self.conv_identity = nn.Conv2d(in_channels, num_filters, kernel_size=1, stride=stride, padding=0)

def forward(self, x):
    identity = x

    x = self.conv1(x)
    x = self.bn1(x)
    x = self.relu(x)
    x = self.conv2(x)
    x = self.bn2(x)
    x = self.relu(x)

    # Pass identity tensor through the 1x1 convolutional layer to match the number of channels
    identity = self.conv_identity(identity)

    x += identity
    x = self.relu(x)
    return x
</code></pre>
<p>class ResNet(nn.Module):<br>
def <strong>init</strong>(self, image_channels, num_classes, layers):<br>
super(ResNet, self).<strong>init</strong>()<br>
self.in_channels = 16<br>
self.conv1 = nn.Conv2d(image_channels, 16, kernel_size=3, stride=2, padding=1)<br>
self.bn1 = nn.BatchNorm2d(16)<br>
self.relu = nn.ReLU()<br>
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)</p>
<pre><code>    # resnet layer
    self.layer1 = self._make_layer(block, 16, layers[0])
    self.layer2 = self._make_layer(block, 32, layers[1])
    self.layer3 = self._make_layer(block, 64, layers[2])
    self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
    self.fc = nn.Linear(64, num_classes)

def forward(self, x):
    x = self.conv1(x)
    x = self.bn1(x)
    x = self.relu(x)
    x = self.maxpool(x)

    x = self.layer1(x)
    x = self.layer2(x)
    x = self.layer3(x)
    x = self.avgpool(x)
    x = x.view(x.size(0), -1)
    x = self.fc(x)
    return x

def _make_layer(self, block, num_filters, stride=1):
    strides = [stride] + [1] * (num_filters - 1)
    layers = []
    for stride in strides:
        layers.append(block(self.in_channels, num_filters, stride))
        self.in_channels = num_filters
    return nn.Sequential(*layers)
</code></pre>
<p>def test():<br>
net = ResNet(image_channels=3, num_classes=100, layers=[2, 3, 4])<br>
x = torch.randn(3, 3, 32, 32)<br>
y = net(x).to(device)<br>
print(y.shape)</p>
<p>test()</p>
<p>and watermark regularizer</p>
<p>import torch<br>
import numpy as np<br>
import torch.nn as nn<br>
import torch.nn.functional as f</p>
<p>def random_index_generator(count):<br>
indices = np.arrange(0, count)<br>
np.random.shuffle(indices)</p>
<pre><code>for idx in indices:
    yield idx
</code></pre>
<p>class WatermarkRegularizer(nn.Module):<br>
def <strong>init</strong>(self, k, b, wtype=‘random’, randseed=‘none’):<br>
super(WatermarkRegularizer, self).<strong>init</strong>()<br>
self.k = torch.tensor(k, dtype=torch.float)<br>
self.uses_learning_phase = True<br>
self.wtype = wtype<br>
self.w = None<br>
self.p = None<br>
self.b = torch.tensor(b)</p>
<pre><code>    if randseed == 'time':
        import time
        np.random.seed(int(time.time()))

def set_param(self, p):
    if self.p is not None:
        raise Exception('Regularizer cannot be reused.''Instantiate one regularizer per layer.')

    self.p = p

    #make matrix
    p_shape = p.shape
    w_rows = np.prod(p_shape[0:3])
    w_cols = self.b.shape[1]

    if self.wtype == 'random':
        self.w = np.random.randn(w_rows, w_cols)
    elif self.wtype == 'direct':
        self.w = np.zeros((w_rows, w_cols), dtype=None)
        rand_idx_gen = random_index_generator(w_rows)

        for col in range(w_cols):
            self.w[next(rand_idx_gen)][col] = 1.
    elif self.wtype == 'diff':
        self.w = np.zeros((w_rows, w_cols), dtype=None)
        rand_idx_gen = random_index_generator(w_rows)

        for col in range(w_cols):
            self.w[next(rand_idx_gen)][col] = 1.
            self.w[next(rand_idx_gen)][col] = -1.
        else:
            raise Exception('wtype="{}" is not supported'.format(self.wtype))

def forward(self, loss):
    if self.p is None:
        raise Exception('Need to call `set_param` on'
                        'WeightRegularizer instance'
                        'before calling the instance.')

    regularized_loss = loss
    x = torch.mean(self.p, dim=3)
    y = torch.reshape(x, (1, torch.numel(x)))
    z = torch.tensor(self.w, dtype=torch.float)
    regularized_loss += self.k * torch.sum(f.binary_cross_entropy_with_logits(torch.matmul(y, z),
                                                                              self.b.float()))
    return regularized_loss

def set_layer(self, layer):
    print('called WatermarkRegularizer.set_layer()')
    super(WatermarkRegularizer, self).set_layer(layer)

def get_matrix(self):
    return self.w

def get_signature(self):
    return self.b

def get_encoded_code(self):
# this function will not work if set_layer was not called
    layer = self.layer
    weights = layer.get_weights()
    weight = (np.array(weights[0])).mean(axis=3)
    print(torch.sigmoid(torch.matmul(torch.tensor(weight.reshape(1, weight.size)), self.w)))
    return None
</code></pre>
<p>def get_wmark_regularizer(model):<br>
ret = []</p>
<pre><code>for i, layer in enumerate(model.layers):
    for regularizer in layer.regularizers:
        if str(regularizer.__class__).find('WatermarkRegularizer') &gt;= 0:
            ret.append((i, regularizer))
return ret
</code></pre>
<p>def show_encoded_wmark(model):<br>
for i, layer in enumerate(model.layers):<br>
for regularizer in layer.regularizers:<br>
if str(regularizer.<strong>class</strong>).find(‘WatermarkRegularizer’) &gt;= 0:<br>
print(‘’.format(i, layer.<strong>class</strong>))<br>
weights = layer.get_weights()<br>
weight = (np.array(weights[0])).mean(axis=3)<br>
print(torch.sigmoid(torch.matmul(torch.tensor(weight.reshape(1, weight.size)),<br>
regularizer.w)))<br>
print(torch.sigmoid(torch.matmul(torch.tensor(weight.reshape(1, weight.size)),<br>
regularizer.w))&gt;0.5)</p>