Calculate the average model for kfold cross validation models

Abdelrahman_Mohamed: <p>Hi, I am trying to calculate the average model for  five models generated by k fold cross validation (five folds ) .</p>
<p>I tried the code below but it doesn’t work  .</p>
<p>Also,if I run each model separately only the last model is working in our case will be the fifth model (if we have 3 folds will be the third  model).</p>
<pre><code class="lang-auto">from torch.autograd import Variable
k_folds =5
num_epochs = 5
 
# For fold results
results = {}
 
# Set fixed random number seed
#torch.manual_seed(0)
  
dataset = torch.utils.data.ConcatDataset([image_datasets['train'],])

print(f'&lt;&gt;------------ Dataset length : {len(dataset)} ------------&lt;&gt;')
# Define the K-fold Cross Validator

kfold = StratifiedKFold(n_splits=k_folds, shuffle=True)

# kfold = Kfold(n_splits=k_folds, shuffle=True)
since = time.time()

# K-fold Cross Validation model evaluation
for fold, (train_ids, validate_ids) in enumerate(kfold.split(dataset,image_datasets['train'].targets)):
    
    # Print
    print(f'&lt;&gt;-----------------FOLD {fold+1}-----------------&lt;&gt;')
    print('train: %s, test: %s' % (len(train_ids), len(validate_ids)))
    
    # Sample elements randomly from a given list of ids, no replacement.
    train_subsampler = torch.utils.data.SubsetRandomSampler(train_ids)
    validate_subsampler = torch.utils.data.SubsetRandomSampler(validate_ids)
    
    
    # Define data loaders for training and validation data in this fold
    
    trainloader = torch.utils.data.DataLoader(
                      dataset, 
                      batch_size=10,  num_workers= 0,sampler=train_subsampler)
    validatloader = torch.utils.data.DataLoader(
                      dataset,
                      batch_size=10, num_workers= 0, sampler=validate_subsampler)
    
    # Init the neural network
    model = MyEnsemble( modelA, modelB, modelC , modelD).to(device)
    #reset layer's weights from previous training
    model.apply(reset_weights)
    
#     # Initialize optimizer
    optimizer = torch.optim.SGD(model.parameters(), lr=.001)
    
    # Run the training loop for defined number of epochs
    Training_loss=[]
    Validation_loss=[]
  
    valid_loss_min = np.Inf # track change in validation loss
    
    for epoch in range(1, num_epochs+1):
        print('Epoch: {}  '.format(epoch))
        #print('--------')
        
        # keep track of training and validation loss
        train_loss = 0.0
        valid_loss = 0.0
        #train
        model.train()
        for data, target in trainloader:
        
            if torch.cuda.is_available():
                   data, target = data.cuda(), target.cuda()
            
        # clear the gradients of all optimized variables
            optimizer.zero_grad()
        # forward pass: compute predicted outputs by passing inputs to the model
            output = model(Variable(data))
        
        #print(target.shape)
        # calculate the batch loss
            loss = criterion(output, Variable(target))
        # backward pass: compute gradient of the loss with respect to model parameters
            loss.backward()
        # perform a single optimization step (parameter update)
            optimizer.step()
        # update training loss
            train_loss += loss.item()*data.size(0)
    #validate
        model.eval()
        accuracy=0.0
        correct, total = 0, 0
        with torch.no_grad():
            for data, target in validatloader:
                 data, target = Variable(data), Variable(target)
        
        #data, target = Variable(data), Variable(target)
                 if torch.cuda.is_available():
                         data, target = data.cuda(), target.cuda()
        # forward pass: compute predicted outputs by passing inputs to the model
                 output = model(data)
        
        # calculate the batch loss
                 loss = criterion(output, target)
        # update average validation loss 
                 valid_loss += loss.item()*data.size(0)
        # Set total and correct
                 _, predicted = torch.max(output.data, 1)
                 total += target.size(0)
                 correct += (predicted == target).sum().item()
                    
            results[fold] = 100.0 * (correct / total)
    # calculate average losses
        train_loss = train_loss/len(trainloader.dataset)
        valid_loss = valid_loss/len(validatloader.dataset)
        
        Training_loss.append(train_loss/len(trainloader))
        Validation_loss.append(valid_loss/len(validatloader))
        
        # print training/validation statistics 
        print('Training Loss: {:.6f} \tValidation Loss: {:.6f} \t '.format(
         train_loss, valid_loss))
        
    # save model if validation loss has decreased
        if valid_loss &lt;= valid_loss_min:
            print("==============================================================================================")
            print('Validation loss decreased ({:.6f} --&gt; {:.6f}).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...'.format(
                   valid_loss_min,
                     valid_loss)) 
            print("==============================================================================================")  
            save_path = f'./EnsembleModelfold-{fold+1}.pt'
            #torch.save(model.state_dict(), save_path)
#             save_path = f'./EnsembleModelfold-{fold+1}.pt'
            torch.save({
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
            }, save_path)
            valid_loss_min = valid_loss
    print('Accuracy for fold %d: %d %%' % (fold+1, 100.0 * correct / total))
#     plt.figure(figsize = (25,10))                
#     plt.plot(Training_loss, label='Training loss')
#     plt.plot(Validation_loss, label='Validation loss')
#     plt.legend(frameon=False)
#     results[fold] = 100.0 * (correct / total)
    
#Print fold results
print(f'K-FOLD CROSS VALIDATION RESULTS FOR {k_folds} FOLDS')
print('--------------------------------')
sum = 0.0
for key, value in results.items():
    print(f'Fold {key}: {value} %')
    sum += value
print(f'Average: {sum/len(results.items())} %')

time_elapsed = time.time() - since
print('Training completed in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
</code></pre>
<pre><code class="lang-auto">
AverageModel = MyEnsemble( modelA, modelB, modelC , modelD).to(device)
for key in AverageModel.state_dict():
    AverageModel.state_dict()[key]=AverageModel.state_dict()[key].zero_()

</code></pre>
<pre><code class="lang-auto">checkpoint_names=['EnsembleModelfold-1.pt','EnsembleModelfold-2.pt','EnsembleModelfold-3.pt','EnsembleModelfold-4.pt','EnsembleModelfold-5.pt']
for name in checkpoint_names:
    checkpoint = torch.load(name)
    fold_model = MyEnsemble( modelA, modelB, modelC , modelD).to(device)
    fold_model.load_state_dict(checkpoint['model_state_dict'])
    for key in AverageModel.state_dict():
        AverageModel.state_dict()[key]+=fold_model.state_dict()[key]
for key in AverageModel.state_dict():
    AverageModel.state_dict()[key]=AverageModel.state_dict()[key]/len(checkpoint_names)
        
AverageModel.state_dict()
</code></pre>

eqy: <p>What is the actual error that you are getting here?</p>

Abdelrahman_Mohamed: <p>Thanks for replying,…</p>
<p>When I run the model on the test data set one class gives me 100% Accuracy and the other class gives 0% Accuracy<br>
This happens for first four models , only the last model gives me an acceptable result</p>

eqy: <p>Do you notice anything strange about the output of the models? Are they stuck to a single prediction value?<br>
What do the training curves look like for the models?</p>

Abdelrahman_Mohamed: <p>Here is the output of the training</p>
<pre><code class="lang-auto">&lt;&gt;------------ Dataset length : 8583 ------------&lt;&gt;
&lt;&gt;-----------------FOLD 1-----------------&lt;&gt;
train: 6866, test: 1717
Epoch: 1  
Training Loss: 0.486925 	Validation Loss: 0.112415 	 
==============================================================================================
Validation loss decreased (inf --&gt; 0.112415).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 2  
Training Loss: 0.443527 	Validation Loss: 0.106931 	 
==============================================================================================
Validation loss decreased (0.112415 --&gt; 0.106931).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 3  
Training Loss: 0.426966 	Validation Loss: 0.105515 	 
==============================================================================================
Validation loss decreased (0.106931 --&gt; 0.105515).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 4  
Training Loss: 0.415489 	Validation Loss: 0.095385 	 
==============================================================================================
Validation loss decreased (0.105515 --&gt; 0.095385).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 5  
Training Loss: 0.409037 	Validation Loss: 0.097692 	 
Accuracy for fold 1: 76 %
&lt;&gt;-----------------FOLD 2-----------------&lt;&gt;
train: 6866, test: 1717
Epoch: 1  
Training Loss: 0.485727 	Validation Loss: 0.120089 	 
==============================================================================================
Validation loss decreased (inf --&gt; 0.120089).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 2  
Training Loss: 0.445043 	Validation Loss: 0.104871 	 
==============================================================================================
Validation loss decreased (0.120089 --&gt; 0.104871).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 3  
Training Loss: 0.428421 	Validation Loss: 0.120718 	 
Epoch: 4  
Training Loss: 0.416499 	Validation Loss: 0.098932 	 
==============================================================================================
Validation loss decreased (0.104871 --&gt; 0.098932).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 5  
Training Loss: 0.410152 	Validation Loss: 0.098041 	 
==============================================================================================
Validation loss decreased (0.098932 --&gt; 0.098041).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Accuracy for fold 2: 77 %
&lt;&gt;-----------------FOLD 3-----------------&lt;&gt;
train: 6866, test: 1717
Epoch: 1  
Training Loss: 0.485534 	Validation Loss: 0.110139 	 
==============================================================================================
Validation loss decreased (inf --&gt; 0.110139).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 2  
Training Loss: 0.448009 	Validation Loss: 0.101339 	 
==============================================================================================
Validation loss decreased (0.110139 --&gt; 0.101339).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 3  
Training Loss: 0.424282 	Validation Loss: 0.098663 	 
==============================================================================================
Validation loss decreased (0.101339 --&gt; 0.098663).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 4  
Training Loss: 0.408519 	Validation Loss: 0.097198 	 
==============================================================================================
Validation loss decreased (0.098663 --&gt; 0.097198).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 5  
Training Loss: 0.407020 	Validation Loss: 0.103193 	 
Accuracy for fold 3: 76 %
&lt;&gt;-----------------FOLD 4-----------------&lt;&gt;
train: 6867, test: 1716
Epoch: 1  
Training Loss: 0.484172 	Validation Loss: 0.113273 	 
==============================================================================================
Validation loss decreased (inf --&gt; 0.113273).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 2  
Training Loss: 0.445774 	Validation Loss: 0.103834 	 
==============================================================================================
Validation loss decreased (0.113273 --&gt; 0.103834).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 3  
Training Loss: 0.427883 	Validation Loss: 0.103864 	 
Epoch: 4  
Training Loss: 0.413481 	Validation Loss: 0.101294 	 
==============================================================================================
Validation loss decreased (0.103834 --&gt; 0.101294).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 5  
Training Loss: 0.404326 	Validation Loss: 0.099928 	 
==============================================================================================
Validation loss decreased (0.101294 --&gt; 0.099928).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Accuracy for fold 4: 76 %
&lt;&gt;-----------------FOLD 5-----------------&lt;&gt;
train: 6867, test: 1716
Epoch: 1  
Training Loss: 0.472580 	Validation Loss: 0.105664 	 
==============================================================================================
Validation loss decreased (inf --&gt; 0.105664).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 2  
Training Loss: 0.427865 	Validation Loss: 0.097842 	 
==============================================================================================
Validation loss decreased (0.105664 --&gt; 0.097842).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 3  
Training Loss: 0.408531 	Validation Loss: 0.094644 	 
==============================================================================================
Validation loss decreased (0.097842 --&gt; 0.094644).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Epoch: 4  
Training Loss: 0.401408 	Validation Loss: 0.103782 	 
Epoch: 5  
Training Loss: 0.395811 	Validation Loss: 0.091990 	 
==============================================================================================
Validation loss decreased (0.094644 --&gt; 0.091990).  &gt;&gt;&gt;&gt;&gt;&gt;&gt;  Saving model ...
==============================================================================================
Accuracy for fold 5: 78 %
K-FOLD CROSS VALIDATION RESULTS FOR 5 FOLDS
--------------------------------
Fold 0: 76.93651718112989 %
Fold 1: 77.34420500873617 %
Fold 2: 76.41234711706466 %
Fold 3: 76.3986013986014 %
Fold 4: 78.55477855477857 %
Average: 77.12928985206214 %
Training completed in 130m 3s
</code></pre>

eqy: <p>Ok, so the actual training looks fine but why are you averaging all of the parameters across the models? Do you mean to average the predictions instead?</p>

Abdelrahman_Mohamed: <p>I want to create the <strong>averageModel.pt</strong>  to use it later</p>

Abdelrahman_Mohamed: <p>I solved the problem but there is something I can’t understand , in this block of code the problem is solved and the generated average model work as expected</p>
<pre><code class="lang-auto">model_1=torch.load('Ensemble_Modelfold-1.pt')
model_2=torch.load('Ensemble_Modelfold-2.pt')
model_3=torch.load('Ensemble_Modelfold-3.pt')
model_4=torch.load('Ensemble_Modelfold-4.pt')
model_5=torch.load('Ensemble_Modelfold-5.pt')

for key in model_1.state_dict():
    model_1.state_dict()[key] = (model_1.state_dict()[key] + model_2.state_dict()[key]+  model_3.state_dict()[key]+model_4.state_dict()[key] + model_5.state_dict()[key] ) / 5
    
Average_Model = MyEnsemble( modelA, modelB, modelC , modelD).to(device)
Average_Model.load_state_dict(model_1.state_dict())
</code></pre>
<p>but If we use this block which almost similar to the first one, the generated average model gives me the same issue</p>
<pre><code class="lang-auto">model_1=torch.load('Ensemble_Modelfold-1.pt')
model_2=torch.load('Ensemble_Modelfold-2.pt')
model_3=torch.load('Ensemble_Modelfold-3.pt')
model_4=torch.load('Ensemble_Modelfold-4.pt')
model_5=torch.load('Ensemble_Modelfold-5.pt')

sd1=model_1.state_dict()
sd2=model_2.state_dict()
sd3=model_3.state_dict()
sd4=model_4.state_dict()
sd5=model_5.state_dict()

for key in sd1:
    sd1[key] = (sd1[key] + sd2[key] + sd3[key]+sd4[key] + sd5[key] ) / 5
    
Average_Model = MyEnsemble( modelA, modelB, modelC , modelD).to(device)
Average_Model.load_state_dict(sd1)
</code></pre>
<p><strong>So, Why this is happening ?</strong></p>