I'm trying to perform transfer learning on my Mask R_CNN model but receiving this error "AssertionError: targets should not be none when in training model"

LowKeyStud: <p>This is my code:</p>
<blockquote>
<p>import torch<br>
import torch.nn as nn<br>
import torch.optim as optim<br>
from torch.optim import lr_scheduler<br>
import torch.backends.cudnn as cudnn<br>
import numpy as np<br>
import torchvision<br>
from torchvision import datasets, models, transforms<br>
import matplotlib.pyplot as plt<br>
import time<br>
import os<br>
from PIL import Image<br>
from tempfile import TemporaryDirectory</p>
<h1>Define device</h1>
<p>device = torch.device(“cuda:0” if torch.cuda.is_available() else “cpu”)</p>
<h1>Mount Google Drive if needed</h1>
<p>from google.colab import drive<br>
drive.mount(‘/content/drive’)</p>
<h1>License: BSD</h1>
<h1>Author: Sasank Chilamkurthy</h1>
<p>cudnn.benchmark = True<br>
plt.ion()   # interactive mode</p>
<h1>Define your custom dataset class</h1>
<p>class CustomDataset(torch.utils.data.Dataset):<br>
def <strong>init</strong>(self, data_dir, transform=None):<br>
self.data_dir = data_dir<br>
self.transform = transform<br>
self.images = <span class="chcklst-box fa fa-square-o fa-fw"></span>  # List to store image paths<br>
self.labels = <span class="chcklst-box fa fa-square-o fa-fw"></span>  # List to store corresponding labels</p>
<pre><code>    # Iterate over your dataset directory to collect image paths and labels
    for class_name in os.listdir(data_dir):
        class_dir = os.path.join(data_dir, class_name)
        if os.path.isdir(class_dir):
            for image_name in os.listdir(class_dir):
                image_path = os.path.join(class_dir, image_name)
                self.images.append(image_path)
                self.labels.append(class_name)

def __len__(self):
    return len(self.images)

def __getitem__(self, idx):
    image_path = self.images[idx]
    label = self.labels[idx]

    # Open image using PIL
    image = Image.open(image_path).convert('RGB')

    # Apply transformations if specified
    if self.transform:
        image = self.transform(image)

    # Convert label to integer index
    label_idx = class_names.index(label)

    return image, label_idx
</code></pre>
<h1>Define data transformations</h1>
<p>data_transforms = {<br>
‘train’: transforms.Compose([<br>
transforms.RandomResizedCrop(224),<br>
transforms.RandomHorizontalFlip(),<br>
transforms.ToTensor(),<br>
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])<br>
]),<br>
‘val’: transforms.Compose([<br>
transforms.Resize(256),<br>
transforms.CenterCrop(224),<br>
transforms.ToTensor(),<br>
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])<br>
]),<br>
}</p>
<h1>Specify the path to your custom dataset directory</h1>
<p>data_dir = ‘/content/drive/MyDrive/OBJDataset.v1i.tensorflow’</p>
<h1>Create custom dataset instances for training and validation</h1>
<p>custom_datasets = {x: CustomDataset(os.path.join(data_dir, x), transform=data_transforms<span class="chcklst-box checked fa fa-check-square-o fa-fw"></span>)<br>
for x in [‘train’, ‘val’]}</p>
<h1>Create data loaders for training and validation</h1>
<p>dataloaders = {x: torch.utils.data.DataLoader(custom_datasets<span class="chcklst-box checked fa fa-check-square-o fa-fw"></span>, batch_size=4, shuffle=True, num_workers=4)<br>
for x in [‘train’, ‘val’]}</p>
<h1>Get the sizes of the datasets</h1>
<p>dataset_sizes = {x: len(custom_datasets<span class="chcklst-box checked fa fa-check-square-o fa-fw"></span>) for x in [‘train’, ‘val’]}</p>
<h1>Get the class names</h1>
<p>class_names = sorted(custom_datasets[‘train’].labels)</p>
<h1>Define the function to display images</h1>
<p>def imshow(inp, title=None):<br>
“”“Display image for Tensor.”“”<br>
inp = inp.numpy().transpose((1, 2, 0))<br>
mean = np.array([0.485, 0.456, 0.406])<br>
std = np.array([0.229, 0.224, 0.225])<br>
inp = std * inp + mean<br>
inp = np.clip(inp, 0, 1)<br>
plt.imshow(inp)<br>
if title is not None:<br>
plt.title(title)<br>
plt.pause(0.001)  # pause a bit so that plots are updated</p>
<h1>Get a batch of training data</h1>
<p>inputs, labels = next(iter(dataloaders[‘train’]))</p>
<h1>Make a grid from the batch</h1>
<p>out = torchvision.utils.make_grid(inputs)</p>
<h1>Convert class indices to class names</h1>
<p>class_names_list = [class_names<span class="chcklst-box checked fa fa-check-square-o fa-fw"></span> for x in labels]</p>
<h1>Display the images with their corresponding class names</h1>
<p>imshow(out, title=class_names_list)</p>
<p>def train_model(model, criterion, optimizer, scheduler, num_epochs=25):<br>
since = time.time()</p>
<pre><code>best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0

for epoch in range(num_epochs):
    print(f'Epoch {epoch}/{num_epochs - 1}')
    print('-' * 10)

    for phase in ['train', 'val']:
        if phase == 'train':
            model.train()
        else:
            model.eval()

        running_loss = 0.0
        running_corrects = 0

        for batch_idx, (inputs, targets) in enumerate(dataloaders[phase]):
            inputs = inputs.to(device)
            targets = targets.to(device)

            optimizer.zero_grad()

            with torch.set_grad_enabled(phase == 'train'):
                outputs = model(inputs)
                loss = criterion(outputs, targets)

                if phase == 'train':
                    loss.backward()
                    optimizer.step()

            running_loss += loss.item() * inputs.size(0)
            _, preds = torch.max(outputs, 1)
            running_corrects += torch.sum(preds == targets.data)

        epoch_loss = running_loss / dataset_sizes[phase]
        epoch_acc = running_corrects.double() / dataset_sizes[phase]

        print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')

        if phase == 'val' and epoch_acc &gt; best_acc:
            best_acc = epoch_acc
            best_model_wts = copy.deepcopy(model.state_dict())

    print()

time_elapsed = time.time() - since
print(f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s')
print(f'Best val Acc: {best_acc:4f}')

model.load_state_dict(best_model_wts)
return model
</code></pre>
<p>def visualize_model(model, num_images=6):<br>
was_training = model.training<br>
model.eval()<br>
images_so_far = 0<br>
fig = plt.figure()</p>
<pre><code>with torch.no_grad():
    for i, (inputs, labels) in enumerate(dataloaders['val']):
        inputs = inputs.to(device)
        labels = labels.to(device)

        outputs = model(inputs)
        _, preds = torch.max(outputs, 1)

        for j in range(inputs.size()[0]):
            images_so_far += 1
            ax = plt.subplot(num_images//2, 2, images_so_far)
            ax.axis('off')
            ax.set_title(f'predicted: {class_names[preds[j]]}')
            imshow(inputs.cpu().data[j])

            if images_so_far == num_images:
                model.train(mode=was_training)
                return
    model.train(mode=was_training)
</code></pre>
<h1>Initialize the Mask R-CNN model</h1>
<p>model_ft = torchvision.models.detection.maskrcnn_resnet50_fpn_v2(pretrained=True)</p>
<h1>Modify the classification and box regression heads</h1>
<p>in_features = model_ft.roi_heads.box_predictor.cls_score.in_features<br>
model_ft.roi_heads.box_predictor = nn.Sequential(<br>
nn.Linear(in_features, 7),  # Modify output features to match your task<br>
nn.Softmax(dim=1)  # Apply softmax if necessary<br>
)</p>
<h1>Optionally, modify the mask predictor for instance segmentation</h1>
<h1>in_features_mask = model_ft.roi_heads.mask_predictor.conv5_mask.in_channels</h1>
<h1>hidden_layer = 256  # Example</h1>
<h1>model_ft.roi_heads.mask_predictor = nn.Sequential(</h1>
<h1>nn.ConvTranspose2d(in_features_mask, hidden_layer, kernel_size=2, stride=2),</h1>
<h1>nn.ReLU(inplace=True),</h1>
<h1>nn.Conv2d(hidden_layer, 7, kernel_size=1, stride=1)  # Modify output features to match your task</h1>
<h1>)</h1>
<h1>Move the model to the device</h1>
<p>model_ft = model_ft.to(device)</p>
<p>from torchvision.models.detection import MaskRCNN_ResNet50_FPN_Weights, MaskRCNN_ResNet50_FPN_V2_Weights<br>
weights = MaskRCNN_ResNet50_FPN_V2_Weights.DEFAULT<br>
model_ft = torchvision.models.detection.maskrcnn_resnet50_fpn_v2(weights=weights)</p>
<h1>Define loss function (criterion)</h1>
<p>criterion = nn.CrossEntropyLoss()</p>
<h1>Define optimizer</h1>
<p>optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)</p>
<h1>Define learning rate scheduler</h1>
<p>exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)</p>
<h1>Train the model</h1>
<p>model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, num_epochs=25)</p>
</blockquote>

ptrblck: <aside class="quote no-group" data-username="LowKeyStud" data-post="1" data-topic="201066">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/lowkeystud/48/63325_2.png" class="avatar"> LowKeyStud:</div>
<blockquote>
<pre><code class="lang-auto">            with torch.set_grad_enabled(phase == 'train'):
                outputs = model(inputs)
</code></pre>
</blockquote>
</aside>
<p>will most likely fail since the targets are not passed to the forward method as the error message points out.</p>

LowKeyStud: <p>New code :</p>
<blockquote>
<h1>forward</h1>
<pre><code>            with torch.set_grad_enabled(phase == 'train'):
              # Pass both inputs and targets to the model
              outputs = model(inputs, targets)
              # For Mask R-CNN, the model returns a dictionary containing the loss
              loss = outputs['loss']
              
              # Perform backward pass and optimization only during training phase
              if phase == 'train':
                loss.backward()
                optimizer.step()

            # statistic
            running_loss += loss.item() * inputs.size(0)
</code></pre>
</blockquote>
<p>I changed the code but I am not sure if this is the correct approach. After the changes, I am receiving this new error "</p>
<blockquote>
<p>IndexError                                Traceback (most recent call last)<br>
 in &lt;cell line: 252&gt;()<br>
250 model_ft = torchvision.models.detection.maskrcnn_resnet50_fpn_v2(weights=weights)<br>
251<br>
 → 252 model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,<br>
253                        num_epochs=25)</p>
<p>3 frames<br>
 in train_model(model, criterion, optimizer, scheduler, num_epochs)<br>
155                 with torch.set_grad_enabled(phase == ‘train’):<br>
156                   # Pass both inputs and targets to the model<br>
 → 157                   outputs = model(inputs, targets)<br>
158                   # For Mask R-CNN, the model returns a dictionary containing the loss<br>
159                   loss = outputs[‘loss’]</p>
<p>/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)<br>
1509             return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]<br>
1510         else:<br>
 → 1511             return self._call_impl(*args, **kwargs)<br>
1512<br>
1513     def _call_impl(self, *args, **kwargs):</p>
<p>/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)<br>
1518                 or _global_backward_pre_hooks or _global_backward_hooks<br>
1519                 or _global_forward_hooks or _global_forward_pre_hooks):<br>
 → 1520             return forward_call(*args, **kwargs)<br>
1521<br>
1522         try:</p>
<p>/usr/local/lib/python3.10/dist-packages/torchvision/models/detection/generalized_rcnn.py in forward(self, images, targets)<br>
63             else:<br>
64                 for target in targets:<br>
—&gt; 65                     boxes = target[“boxes”]<br>
66                     if isinstance(boxes, torch.Tensor):<br>
67                         torch._assert(</p>
<p>IndexError: too many indices for tensor of dimension 0</p>
</blockquote>