Why does the performance of the model get worse after continuous training？

wangtz: <p>IMA_SIZE,BATCH_SIZE,EPOCH=299,1,5<br>
class MeninDataset(Dataset):<br>
def <strong>init</strong>(self, root_dir, train):<br>
self.root_dir = root_dir<br>
self.images = os.listdir(self.root_dir)<br>
self.train = train<br>
self.transforms_train=transforms.Compose([<br>
transforms.Resize(IMA_SIZE),<span class="hashtag">#inception</span> 为299 其他为 224<br>
transforms.RandomHorizontalFlip(p=0.5),<br>
transforms.RandomVerticalFlip(p=0.5),<br>
transforms.RandomApply([<br>
transforms.RandomRotation(degrees=(0,360))],p=0.5),<br>
transforms.ToTensor()<br>
])<br>
self.transforms_test=transforms.Compose([<br>
transforms.Resize(IMA_SIZE),<span class="hashtag">#inception</span> 为299 其他为 224<br>
transforms.ToTensor()<br>
])</p>
<pre><code>def __getitem__(self, index):  # 根据索引返回图像与标签

    if self.train==True:
        list = os.listdir(self.root_dir)
        img = Image.open(self.root_dir + list[index])
        img = self.transforms_train(img)
    else:
        list = os.listdir(self.root_dir)
        img = Image.open(self.root_dir + list[index])
        img = self.transforms_test(img)

    img = img.expand(3,IMA_SIZE,IMA_SIZE)

    if list[index].split('.')[0][0] == 'l':
        lab = 0
    else:
        lab = 1
    return img, lab

def __len__(self):
    if self.train==True:
        list = os.listdir(self.root_dir)
    else:
        list = os.listdir(self.root_dir)

    len_data = len(list)
    return len_data
</code></pre>
<p>class Net(nn.Module):<br>
def <strong>init</strong>(self, model):<br>
super(Net, self).<strong>init</strong>()<br>
self.resnet = nn.Sequential(*list(model.children())[:-1])<br>
self.flatten = nn.Flatten()<br>
self.linear = nn.Linear(2048,2, bias=True)<br>
def forward(self, x):<br>
x = self.resnet(x)<br>
x = self.flatten(x)<br>
x = self.linear(x)<br>
return x</p>
<p>train_data = MeninDataset(root_dir=’./train_fig/’, train=True)<br>
test_data = MeninDataset(root_dir=’./test_fig/’, train=False)<br>
ld_train = DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True)<br>
ld_test = DataLoader(dataset=test_data, batch_size=BATCH_SIZE, shuffle=True)<br>
cuda = torch.device(“cuda:0” if torch.cuda.is_available() else “cpu”)<br>
model = models.resnet34(pretrained=True).to(cuda)<br>
<span class="hashtag">#model</span> = Net(model).to(cuda)<br>
scaler = torch.cuda.amp.GradScaler()<br>
optimizer = optim.Adam(model.parameters(), weight_decay=1e-4, lr=1e-3)<br>
loss_fn = nn.CrossEntropyLoss()<br>
#<span class="hashtag">#训练</span><br>
tape_trainAcc, tape_testAcc  = [], []<br>
for epoch in range(EPOCH):<br>
train_correct, test_correct, train_total, test_total = 0, 0, 0, 0<br>
model.train()<br>
for _, (train_x, train_y) in enumerate(ld_train):<br>
train_x, train_y = train_x.to(cuda), train_y.to(cuda)<br>
if True:  # 半精度<br>
optimizer.zero_grad()<br>
with amp.autocast():<br>
train_pred_y = model(train_x)<br>
loss = loss_fn(train_pred_y, train_y)<br>
scaler.scale(loss).backward()<br>
scaler.step(optimizer)<br>
scaler.update()<br>
if False:  # 全精度<br>
optimizer.zero_grad()<br>
train_pred_y = model(train_x)<br>
loss = loss_function(train_pred_y, train_y)<br>
loss.backward()<br>
optimizer.step()</p>
<pre><code>    _, train_pred = torch.max(train_pred_y.data, 1)
    train_total += train_y.size(0)
    train_correct += (train_pred == train_y).sum().item()

model.eval()
with torch.no_grad():
    for _, (test_x, test_y) in enumerate(ld_test):
        test_x, test_y = test_x.to(cuda), test_y.to(cuda)
        test_pred_y = model(test_x)
        _, test_pred = torch.max(test_pred_y.data, dim=1)  # 预测结果
        test_correct += (test_pred == test_y).sum().item()
        test_total += test_y.size(0)

tape_trainAcc.append(100 * train_correct / train_total)
tape_testAcc.append(100 * test_correct / test_total)
print('epoch:{}'.format(epoch), ';',
      'train Acc:%.2f' % (100 * train_correct / train_total), ';',
      'test Acc:%.2f' % (100 * test_correct / test_total))
#记录
checkpoint = {
    "net": model.state_dict(),
    'optimizer': optimizer.state_dict(),
    "epoch": epoch,
    "tape_trainAcc":tape_trainAcc,
    "tape_testAcc": tape_testAcc,
}
if epoch % 10 == 0 or epoch==(EPOCH-1):
    torch.save(checkpoint, './checkpoint/checkpoint.pth')
    if tape_trainAcc[epoch] &gt; 90:
        torch.save(model.state_dict(), './checkpoint/model_{}.pth'.format(epoch))
</code></pre>
<p><span class="hashtag">#torch</span>.cuda.empty_cache()<br>
#<span class="hashtag">#断点</span><br>
checkpoint = torch.load(’./checkpoint/checkpoint.pth’)<span class="hashtag">#断点模型</span><br>
model.load_state_dict(checkpoint[‘net’])<br>
model.to(cuda)<br>
optimizer.load_state_dict(checkpoint[‘optimizer’])<br>
tape_trainAcc, tape_testAcc= checkpoint[‘tape_trainAcc’],checkpoint[‘tape_testAcc’]<br>
start_epoch=checkpoint[‘epoch’]+1<br>
end_epoch =start_epoch+8 <span class="hashtag">#训练次数</span><br>
for epoch in range(start_epoch, end_epoch):<br>
train_correct, test_correct, train_total, test_total = 0, 0, 0, 0<br>
model.train()<br>
for _, (train_x, train_y) in enumerate(ld_train):<br>
train_x, train_y = train_x.to(cuda), train_y.to(cuda)<br>
if True:<span class="hashtag">#半精度</span><br>
optimizer.zero_grad()<br>
with amp.autocast():<br>
train_pred_y = model(train_x)<br>
loss = loss_fn(train_pred_y, train_y)<br>
scaler.scale(loss).backward()<br>
scaler.step(optimizer)<br>
scaler.update()<br>
if False:<span class="hashtag">#全精度</span><br>
optimizer.zero_grad()<br>
train_pred_y = model(train_x)<br>
loss = loss_function(train_pred_y, train_y)<br>
loss.backward()<br>
optimizer.step()<br>
_, train_pred = torch.max(train_pred_y.data, 1)<br>
train_total += train_y.size(0)<br>
train_correct += (train_pred == train_y).sum().item()</p>
<pre><code>model.eval()
with torch.no_grad():
    for _, (test_x, test_y) in enumerate(ld_test):
        test_x, test_y = test_x.to(cuda), test_y.to(cuda)
        test_pred_y = model(test_x)
        _, test_pred = torch.max(test_pred_y.data, dim=1) #预测结果
        test_correct += (test_pred == test_y).sum().item()
        test_total += test_y.size(0)

tape_trainAcc.append(100 * train_correct / train_total)
tape_testAcc.append(100 * test_correct / test_total)
print('epoch:{}'.format(epoch),';',
      'train Acc:%.2f'%(100 * train_correct / train_total),';',
      'test Acc:%.2f'%(100 * test_correct / test_total))
#记录
checkpoint = {
    "net": model.state_dict(),
    'optimizer': optimizer.state_dict(),
    "epoch": epoch,
    "tape_trainAcc": tape_trainAcc,
    "tape_testAcc": tape_testAcc,
}
if epoch % 10 == 0:
    torch.save(checkpoint, './checkpoint/checkpoint.pth')
    if tape_trainAcc[epoch] &gt; 90:
        torch.save(model.state_dict(), './checkpoint/Spark_{}.pth'.format(epoch))</code></pre>

ptrblck: <p>If you are seeing a slowdown during the training, you might (accidentally) store some tensors, which are still attached to the computation graph e.g. in a <code>list</code>. This would however also increase the memory usage. If that’s not the case, your system might reduce its clock due to e.g. overheating, so you could also check the sensors in your system.</p>
<p>PS: you can post code snippets by wrapping them into three backticks ```, which would make debugging easier, as your code is currently hard to read <img src="https://discuss.pytorch.org/images/emoji/apple/wink.png?v=9" title=":wink:" class="emoji" alt=":wink:"></p>