Trying to run the official tutorial on object detection gives me PicklingError: Can't pickle at 0x000002A41479A1E0>: attribute lookup on __main__ failed

username: Shisho_Sama
cooked: <p>Hello everyone.<br>
I’m trying to run the official tutorial explained <a href="https://colab.research.google.com/github/pytorch/vision/blob/temp-tutorial/tutorials/torchvision_finetuning_instance_segmentation.ipynb#scrollTo=at-h4OWK0aoc" rel="nofollow noopener">here</a> The code runs ok on google colab, however, in my system I’m facing this error :</p>
<pre><code class="lang-auto">---------------------------------------------------------------------------
PicklingError                             Traceback (most recent call last)
e:\DeepLearning\Object detection and instance segmentation\ob_detection_img_captioning_seq_labeling.py in 
    128 epochs = 10
    129 for e in range(epochs):
--&gt; 130     engine.train_one_epoch(model, optimizer,dataloader_train, device, e, 10)
    131     lr_scheduler.step()
    132     engine.evaluate(model, dataloader_test,device)

e:\DeepLearning\Object detection and instance segmentation\engine.py in train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq)
     24         lr_scheduler = utils.warmup_lr_scheduler(optimizer, warmup_iters, warmup_factor)
     25 
---&gt; 26     for images, targets in metric_logger.log_every(data_loader, print_freq, header):
     27         images = list(image.to(device) for image in images)
     28         targets = [{k: v.to(device) for k, v in t.items()} for t in targets]

e:\DeepLearning\Object detection and instance segmentation\utils.py in log_every(self, iterable, print_freq, header)
    209             ])
    210         MB = 1024.0 * 1024.0
--&gt; 211         for obj in iterable:
    212             data_time.update(time.time() - end)
    213             yield obj

~\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py in __iter__(self)
    276             return _SingleProcessDataLoaderIter(self)
    277         else:
--&gt; 278             return _MultiProcessingDataLoaderIter(self)
    279 
    280     @property

~\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py in __init__(self, loader)
    680             #     before it starts, and __del__ tries to join but will get:
    681             #     AssertionError: can only join a started process.
--&gt; 682             w.start()
    683             self.index_queues.append(index_queue)
    684             self.workers.append(w)

~\Anaconda3\lib\multiprocessing\process.py in start(self)
    103                'daemonic processes are not allowed to have children'
    104         _cleanup()
--&gt; 105         self._popen = self._Popen(self)
    106         self._sentinel = self._popen.sentinel
    107         # Avoid a refcycle if the target function holds an indirect

~\Anaconda3\lib\multiprocessing\context.py in _Popen(process_obj)
    221     @staticmethod
    222     def _Popen(process_obj):
--&gt; 223         return _default_context.get_context().Process._Popen(process_obj)
    224 
    225 class DefaultContext(BaseContext):

~\Anaconda3\lib\multiprocessing\context.py in _Popen(process_obj)
    320         def _Popen(process_obj):
    321             from .popen_spawn_win32 import Popen
--&gt; 322             return Popen(process_obj)
    323 
    324     class SpawnContext(BaseContext):

~\Anaconda3\lib\multiprocessing\popen_spawn_win32.py in __init__(self, process_obj)
     63             try:
     64                 reduction.dump(prep_data, to_child)
---&gt; 65                 reduction.dump(process_obj, to_child)
     66             finally:
     67                 set_spawning_popen(None)

~\Anaconda3\lib\multiprocessing\reduction.py in dump(obj, file, protocol)
     58 def dump(obj, file, protocol=None):
     59     '''Replacement for pickle.dump() using ForkingPickler.'''
---&gt; 60     ForkingPickler(file, protocol).dump(obj)
     61 
     62 #

PicklingError: Can't pickle  at 0x000002A41479A1E0&gt;: attribute lookup  on __main__ failed
</code></pre>
<p>And this is my code</p>
<pre><code class="lang-auto">#%% Instance segmentation using MASK RCNN . 2017 from facebook he etal. 
from PIL import Image
import numpy as np 
import torch, os, sys, path
import torchvision 
from torchvision import datasets, transforms, models 

import PIL  
import matplotlib.pyplot as plt

class mydataset(torch.utils.data.Dataset):
    def __init__(self, root, transformations=None):
        super().__init__()

        self.root = root
        self.image_folder = 'PNGImages'
        self.mask_folder = 'PedMasks'
        self.annotation_folder = 'Annotation'
        self.transformations = transformations

        self.imgs_filename_list = list(sorted(os.listdir(os.path.join(self.root, self.image_folder))))
        self.mask_filename_list = list(sorted(os.listdir(os.path.join(self.root, self.mask_folder))))
        self.annotation_list = list(sorted(os.listdir(os.path.join(self.root, self.annotation_folder))))

    def __getitem__(self, idx):
        img_path = os.path.join(self.root, self.image_folder, self.imgs_filename_list[idx])
        mask_path = os.path.join(self.root, self.mask_folder,self.mask_filename_list[idx])

        img = PIL.Image.open(img_path).convert('RGB')
        mask = PIL.Image.open(mask_path)
        mask_np = np.array(mask)
        obj_ids = np.unique(mask_np)[1:]
        masks = mask_np == obj_ids[:, None, None]

        num_objects = len(obj_ids)
        boxes = []
        for i in range(num_objects):
            pos = np.where(masks[i])
            xmin = np.min(pos[1])
            xmax = np.max(pos[1])
            ymin = np.min(pos[0])
            ymax = np.max(pos[0])
            boxes.append([xmin, xmax, ymin, ymax]) 

        boxes = torch.as_tensor(boxes, dtype=torch.float32)

        labels = torch.ones((num_objects,), dtype=torch.int64)
        masks = torch.as_tensor(masks, dtype=torch.uint8)
        image_id = torch.tensor(idx)
        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
        is_crowd = torch.zeros(size=(num_objects,), dtype=torch.int64)

        info_dic={}
        info_dic['boxes'] = boxes
        info_dic['labels'] = labels
        info_dic['masks'] = masks
        info_dic['image_id'] = image_id
        info_dic['area'] = area
        info_dic['iscrowd'] = is_crowd

        if self.transformations is not None:
            img, info_dic = self.transformations(img, info_dic)
        
        return img, info_dic


    def __len__(self):
        return len(self.imgs_filename_list)


ds = mydataset(r'data\PennFudanPed')
print(ds.imgs_filename_list[:5])
print(ds.mask_filename_list[:5])

ds[0]
realimg_path = os.path.join(ds.root, ds.image_folder,
                         ds.imgs_filename_list[0])

mask_path = os.path.join(ds.root, ds.mask_folder,
                         ds.mask_filename_list[0])

from PIL import Image  
                    
mask = Image.open(mask_path)
realimg = Image.open(realimg_path).convert('RGB')
mask1= mask.convert('RGB')
# print(mask)
# plt.imshow(mask)
# we can also view the image from PIL usin putpalette 

# Attaches a palette to this image. The image must be a “P” or “L”
# image, and the palette sequence must contain 768 integer values, 
# where each group of three values represent the red, green, and 
# blue values for the corresponding pixel index. Instead of an 
# integer sequence, you can use an 8-bit string.
mask.putpalette([0,0,0,# the first entry is for background
                 255,0,0,# lets color our first instance red
                 0,255,0 # lets color our second instance green
                 ])
plt.imshow(mask)                
fig, axes = plt.subplots(nrows=1,ncols=3,sharex=False)
for ax,img,title in zip(axes.flatten(),[realimg,mask,mask1],['real image','mask','mask rgb']):
    ax.imshow(img)
    ax.set_title(title+str(np.array(img).shape))

#%%
# models are in torchvision datasets 
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
import torchvision.models as models
# load a pretrained model 
model = models.detection.fasterrcnn_resnet50_fpn(pretrained=True)
print(model)
# lets replace the existing classifier with a new one. 
# our classifier is FastRCNNPredictor ! 
# number of classes is alwyas +1, that 1 is for background 
num_classes = 2 # person and background 
# lets getthe input features count from the existing classifier 
in_features = model.roi_heads.box_predictor.cls_score.in_features  
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)

# Thats it. 
#%%
backend = models.mobilenet_v2(pretrained=True).features
print(backend)
backend.out_channels = 1280

from torchvision.models.detection.faster_rcnn import FasterRCNN
from torchvision.models.detection.rpn import AnchorGenerator

anchors = AnchorGenerator(sizes=(32,64,128,256,512), 
                          aspect_ratios=(0.5,1.0,2.0))

roi_pooler = torchvision.ops.MultiScaleRoIAlign(featmap_names=[],
                                             output_size=7,
                                             sampling_ratio=2)

model = FasterRCNN(backend,2, rpn_anchor_generator=anchors,box_roi_pool=roi_pooler)


def get_model_for_segmentation_detection(num_classes=2):
    model = models.detection.maskrcnn_resnet50_fpn(pretrained=True)
    from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
    from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor

    in_features_detection = model.roi_heads.box_predictor.cls_score.in_features
    in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels 
    hiddensize = 256
    model.roi_heads.box_predictor = FastRCNNPredictor(in_features_detection, num_classes)

    model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask,                                                              hiddensize, 2)
    return model

model = get_model_for_segmentation_detection(num_classes=2)


# lets train it 
# Note that we do not need to add a mean/std normalization nor image rescaling in
# the data transforms, as those are handled internally by the Mask R-CNN model.
def get_transform(train):

    transform = []
    transform.append(transforms.ToTensor()) 
    if train:

        transform.append(transforms.RandomHorizontalFlip(0.5))
    return transforms.Compose(transform)


# train 
# before going any further make sure to install pycocotools 
# git clone https: // github.com/cocodataset/cocoapi.git
# cd cocoapi/PythonAPI
# python setup.py build_ext install
import engine
import utils 

# create datasets
dataset_train = mydataset(r'data\PennFudanPed',get_transform(True))

dataset_test = mydataset(r'data\PennFudanPed', get_transform(False))

indices = torch.randperm(len(dataset_train)).tolist()

# now lets create train and test datasets 
dataset = torch.utils.data.Subset(dataset_train, indices[:-50])
dataset_test=torch.utils.data.Subset(dataset_test, indices[-50:])

n_workers=2
dataloader_train = torch.utils.data.DataLoader(
    dataset, batch_size=2, shuffle=True, num_workers=n_workers, collate_fn=lambda batch: tuple(zip(*batch)))


dataloader_test = torch.utils.data.DataLoader(
    dataset_test, batch_size=2, shuffle=False, num_workers=n_workers, collate_fn=lambda batch: tuple(zip(*batch)))

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)

optimizer = torch.optim.SGD(model.parameters(), lr=0.005,
                            momentum=0.9, weight_decay=0.0005)

lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,step_size=3)

epochs = 10 
for e in range(epochs):
    engine.train_one_epoch(model, optimizer,dataloader_train, device, e, 10)
    lr_scheduler.step()
    engine.evaluate(model, dataloader_test,device)
</code></pre>
<p>As you can see all the code is the same yet I’m failing!<br>
I have Pytorch 1.2 on windows 10 x64<br>
I’d appreciate any kind of help in this regard<br>
Thank you all in advance</p>

username: peterjc123
cooked: <p>Well, the multiprocessing library with Jupyter Notebook is hard to use under Windows. You’ll need to refactor your code a little bit to make it work. See <a href="https://medium.com/@grvsinghal/speed-up-your-python-code-using-multiprocessing-on-windows-and-jupyter-or-ipython-2714b49d6fac" rel="nofollow noopener">https://medium.com/@grvsinghal/speed-up-your-python-code-using-multiprocessing-on-windows-and-jupyter-or-ipython-2714b49d6fac</a> for more details.</p>