Multiprocessing Error pytorch

doctum: <p>Hi,</p>
<p>I am running into the following error when running:</p>
<pre><code class="lang-auto">&gt; import os 
&gt; os.chdir("/Users/Wu/Desktop/Research/DL_train/GradCam_classific/DL_train")
&gt; 
&gt; 
&gt; import argparse
&gt; from pathlib import Path
&gt; 
&gt; import torch
&gt; import numpy as np
&gt; from pytorch_lightning.callbacks import ModelCheckpoint
&gt; from pytorch_lightning.loggers import TensorBoardLogger
&gt; from sklearn.model_selection import train_test_split
&gt; from torch.utils.data import DataLoader
&gt; 
&gt; from dataloader import MRTDataset, get_patients
&gt; from networks.attention_unet import AttentionUNet
&gt; import pytorch_lightning as pl
&gt; import matplotlib.pyplot as plt
&gt; 
&gt; from polyaxon_client.tracking import Experiment
&gt; 
&gt; from densenet import DenseNet
&gt; from test_densenet import test_densenet
&gt; parser = argparse.ArgumentParser()
&gt; 
&gt; parser.add_argument('--in_channels', type=int, default=1)
&gt; parser.add_argument('--out_channels', type=int, default=1)
&gt; parser.add_argument('--out_classes', type=int, default=3)
&gt; 
&gt; parser.add_argument('--learning_rate', type=float, default=1e-3)
&gt; parser.add_argument('--weight_decay', type=str, default=1e-3)
&gt; 
&gt; parser.add_argument('--train_batch_size', type=int, default=8)
&gt; parser.add_argument('--test_batch_size', type=int, default=8)
&gt; 
&gt; parser.add_argument('--num_epochs', type=int, default=32)
&gt; parser.add_argument('--num_workers', type=int, default=4)
&gt; 
&gt; parser.add_argument('--validation_size', type=float, default=0.2)
&gt; parser.add_argument('--test_size', type=float, default=0.2)
&gt; 
&gt; parser.add_argument('--split_seed', type=int, default=42)
&gt; 
&gt; 
&gt; def compute_mean_and_std(train_loader, key): # mean and std for mri intensity
&gt;     mean = 0.
&gt;     std = 0.
&gt;     min = 1000
&gt;     max = 0
&gt;     for data in train_loader:
&gt;         images = data[key]
&gt;         mean += images.mean(dim=[2, 3, 4]).sum(0)
&gt;         std += images.std(dim=[2, 3, 4]).sum(0)
&gt;         if data['supervised'].sum()==0:
&gt;             continue
&gt;         if min &gt; images.min():
&gt;             min = images.min()
&gt;         if max &lt; images.max():
&gt;             max = images.max()
&gt; 
&gt;     mean /= len(train_loader.dataset)
&gt;     std /= len(train_loader.dataset)
&gt;     print(min)
&gt;     print(max)
&gt;     return mean, std
&gt; 
&gt; if __name__ == '__main__':
&gt;     hparams = parser.parse_args()
&gt;     #experiment = Experiment()
&gt;     data_path = '//Volumes/Extreme Pro/DL_train/'
&gt;     csv_path = '/pain.csv'
&gt;     hparams.out_classes = 2
&gt;     hparams.learning_rate = 1e-3
&gt;     hparams.weight_decay = 1e-3
&gt;     hparams.train_batch_size = 4
&gt;     hparams.test_batch_size = 4
&gt;     hparams.num_epochs = 32
&gt;     hparams.split_seed = 42
&gt;     output_path = '/Volumes/Extreme Pro/DL_train/'
&gt;     
&gt;    
&gt;     sizes, spacings = [112, 128, 112], [1.5, 1.5, 1.5]
&gt;     strides, kernels = [], []
&gt;     
&gt;     csv_path = '/Volumes/Extreme Pro/DL_train/pain.csv'
&gt;     
&gt;     patients, patient_idx, labels = get_patients(data_path, csv_path)
&gt;     patient_idx, labels = torch.tensor(patient_idx), torch.tensor(labels)
&gt;     print("Splitting the data...")
&gt;     train_idx, test_idx, train_patients, test_patients, train_labels, test_labels = train_test_split(
&gt;         patient_idx,
&gt;         patients,
&gt;         labels,
&gt;         test_size=hparams.test_size,
&gt;         random_state=hparams.split_seed,
&gt;         stratify=labels,
&gt;         shuffle=True)
&gt; 
&gt;     train_idx, valid_idx, train_patients, valid_patients, train_labels, valid_labels = train_test_split(
&gt;         train_idx,
&gt;         train_patients,
&gt;         train_labels,
&gt;         test_size=hparams.validation_size,
&gt;         random_state=hparams.split_seed,
&gt;         stratify=train_labels,
&gt;         shuffle=True)
&gt;     print("Train: ")
&gt;     print("Number of patients: ", len(train_patients))
&gt;     train_set = MRTDataset(data_path, csv_path, train_patients)#, transforms=train_transform)
&gt;     print("Validation: ")
&gt;     print("Number of patients: ", len(valid_patients))
&gt;     valid_set = MRTDataset(data_path, csv_path, valid_patients)
&gt;     print("Test: ")
&gt;     print("Number of patients: ", len(test_patients))
&gt;     test_set = MRTDataset(data_path, csv_path, test_patients)
&gt;     train_loader = DataLoader(train_set, batch_size=4, num_workers=8, shuffle=True)
&gt;     valid_loader = DataLoader(valid_set, batch_size=1, num_workers=8)
&gt;     test_loader = DataLoader(test_set, batch_size=1, num_workers=0)
&gt;     print("Out Channels: ", hparams.out_channels)
&gt;    
&gt;     densenet = DenseNet(learning_rate=hparams.learning_rate)
&gt; 
&gt;     trainer = pl.Trainer(max_epochs=hparams.num_epochs, progress_bar_refresh_rate=200)
&gt; 
&gt; 
&gt;     trainer.fit(densenet, train_dataloader=train_loader, val_dataloaders=[valid_loader])
</code></pre>
<p>when executing the trainer.fit the following error is thrown:</p>
<pre><code class="lang-auto">11.2 M    Trainable params
0         Non-trainable params
11.2 M    Total params
Validation sanity check: 0it [00:00, ?it/s]
Traceback (most recent call last):
  File "&lt;string&gt;", line 1, in &lt;module&gt;
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
ModuleNotFoundError: No module named 'dataloader'
Traceback (most recent call last):
  File "&lt;string&gt;", line 1, in &lt;module&gt;
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
ModuleNotFoundError: No module named 'dataloader'
Traceback (most recent call last):
  File "&lt;string&gt;", line 1, in &lt;module&gt;
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
ModuleNotFoundError: No module named 'dataloader'
Traceback (most recent call last):
  File "&lt;string&gt;", line 1, in &lt;module&gt;
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
ModuleNotFoundError: No module named 'dataloader'
Traceback (most recent call last):
  File "&lt;string&gt;", line 1, in &lt;module&gt;
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
ModuleNotFoundError: No module named 'dataloader'
Traceback (most recent call last):
  File "&lt;string&gt;", line 1, in &lt;module&gt;
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
ModuleNotFoundError: No module named 'dataloader'
Traceback (most recent call last):
  File "&lt;string&gt;", line 1, in &lt;module&gt;
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/spawn.py", line 126, in _main
    self = reduction.pickle.load(from_parent)
ModuleNotFoundError: No module named 'dataloader'
Traceback (most recent call last):

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 990, in _try_get_data
    data = self._data_queue.get(timeout=timeout)

  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/queues.py", line 113, in get
    if not self._poll(timeout):

  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/connection.py", line 262, in poll
    return self._poll(timeout)

  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/connection.py", line 429, in _poll
    r = wait([self], timeout)

  File "/Users/Wu/opt/anaconda3/lib/python3.9/multiprocessing/connection.py", line 936, in wait
    ready = selector.select(timeout)

  File "/Users/Wu/opt/anaconda3/lib/python3.9/selectors.py", line 416, in select
    fd_event_list = self._selector.poll(timeout)

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/_utils/signal_handling.py", line 66, in handler
    _error_if_any_worker_fails()

RuntimeError: DataLoader worker (pid 66043) exited unexpectedly with exit code 1. Details are lost due to multiprocessing. Rerunning with num_workers=0 may give better error trace.


The above exception was the direct cause of the following exception:

Traceback (most recent call last):

  File "/var/folders/79/z7g43_0x08g2yj7w6lb_j5280000gn/T/ipykernel_62506/90799514.py", line 1, in &lt;module&gt;
    trainer.fit(densenet, train_dataloader=train_loader, val_dataloaders=[valid_loader])

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 470, in fit
    results = self.accelerator_backend.train()

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/pytorch_lightning/accelerators/cpu_accelerator.py", line 62, in train
    results = self.train_or_test()

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/pytorch_lightning/accelerators/accelerator.py", line 69, in train_or_test
    results = self.trainer.train()

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 492, in train
    self.run_sanity_check(self.get_model())

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 690, in run_sanity_check
    _, eval_results = self.run_evaluation(test_mode=False, max_batches=self.num_sanity_val_batches)

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 593, in run_evaluation
    for batch_idx, batch in enumerate(dataloader):

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 521, in __next__
    data = self._next_data()

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 1186, in _next_data
    idx, data = self._get_data()

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 1152, in _get_data
    success, data = self._try_get_data()

  File "/Users/Wu/opt/anaconda3/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 1003, in _try_get_data
    raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str)) from e

RuntimeError: DataLoader worker (pid(s) 66043) exited unexpectedly
</code></pre>

mizoru: <p>I’m having a very similar issue. Have you found a solution?</p>