How to construct `Dataset` with iterator for multi-process `DataLoader`?

phaypyt: <p>I want to do something like this:</p>
<pre><code class="lang-python">class myDataset():
    def __init__():
        self.iter = some_iterator_on_my_dataset
    def __len__():
        return len_of_my_dataset
    def __getitem__():
        return self.iter.next()

loader = DataLoader(myDataset, num_workers=16)
</code></pre>
<p>which works well when <code>num_workers=1</code>, failed when <code>num_workers&gt;1</code> however.<br>
I guess that <code>self.iter</code>, the iterator failed in multi-process loader.<br>
How shuold I do that?</p>

saransh: <p>It usually works good for more number of workers, have you tried with 2 or 4 workers?<br>
Also, how many cores does your computer have?</p>

phaypyt: <p>I got more than 40 cores in my computer, I tried some <code>num_workers</code> like 20, 25, it failed anyway.</p>

RicCu: <p>Hi,<br>
You might want to try out the new <a href="https://pytorch.org/docs/master/data.html#torch.utils.data.IterableDataset" rel="nofollow noopener">IterableDataset</a> on PyTorch master or nightly release. However note that to correctly use <code>num_workers&gt;0</code> you will have to configure your dataset based on the <a href="https://pytorch.org/docs/master/data.html#torch.utils.data.get_worker_info" rel="nofollow noopener">worker info</a> to avoid generating duplicate data.</p>

DivyanshJha: <p>It doesn’t seem to work for me. I was trying to benchmark the old dataset with the new IterableDataset.<br>
Can you look out what’s wrong here?</p>
<p>My Cifar100 images are in folders.</p>
<pre><code class="lang-auto">def read_files(files):
    images = [torch.tensor(np.array(Image.open(i))) for i in files]
    yield images

class Cifar100Dataset(IterableDataset):
    def __init__(self, path, folder, start, end):
        self.files = [file for directory in (path / folder).ls() for file in directory.ls()]
        self.start = start
        self.end = end
    
    def __iter__(self):
        worker_info = torch.utils.data.get_worker_info()
        if worker_info is None:
            return read_files(self.files[self.start:self.end])
        else:
            per_worker = int(math.ceil((self.end - self.start) / float(worker_info.num_workers)))
            worker_id = worker_info.id
            iter_start = self.start + worker_id * per_worker
            iter_end = min(iter_start + per_worker, self.end)
            return iter(read_files(self.files[iter_start:iter_end]))
</code></pre>
<p>It works well with num_workers = 0, but does not work with num_worker&gt;1</p>
<p>ds = Cifar100Dataset(path, train_folder, start=0, end=1024)<br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/2X/2/2015cb1a3a542f33c74c14375980ed4ed161e057.png" data-download-href="https://discuss.pytorch.org/uploads/default/2015cb1a3a542f33c74c14375980ed4ed161e057" title="image"><img src="https://discuss.pytorch.org/uploads/default/original/2X/2/2015cb1a3a542f33c74c14375980ed4ed161e057.png" alt="image" data-base62-sha1="4zPYd1WWNUywRvgKFd003oqDv71" width="690" height="290" data-dominant-color="F4F4F4"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">706×297 7.01 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/2X/1/180c0b19cc0c1a415510943a685571f2eb5c2032.png" data-download-href="https://discuss.pytorch.org/uploads/default/180c0b19cc0c1a415510943a685571f2eb5c2032" title="image"><img src="https://discuss.pytorch.org/uploads/default/original/2X/1/180c0b19cc0c1a415510943a685571f2eb5c2032.png" alt="image" data-base62-sha1="3qJgfpYihngV1vjdu4wyCUuXBFE" width="690" height="198" data-dominant-color="F5E2E2"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">1273×366 14.1 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>
<pre><code class="lang-auto">Exception ignored in: &lt;function _MultiProcessingDataLoaderIter.__del__ at 0x000001F99F5CB9D8&gt;
Traceback (most recent call last):
  File "C:\Users\Divyansh J\Anaconda3\envs\pytorch\lib\site-packages\torch\utils\data\dataloader.py", line 883, in __del__
    self._shutdown_workers()
  File "C:\Users\Divyansh J\Anaconda3\envs\pytorch\lib\site-packages\torch\utils\data\dataloader.py", line 860, in _shutdown_workers
    if self.workers_status[worker_id]:
IndexError: list index out of range

---------------------------------------------------------------------------
BrokenPipeError                           Traceback (most recent call last)
&lt;timed exec&gt; in &lt;module&gt;

~\Anaconda3\envs\pytorch\lib\site-packages\torch\utils\data\dataloader.py in __iter__(self)
    240             return _SingleProcessDataLoaderIter(self)
    241         else:
--&gt; 242             return _MultiProcessingDataLoaderIter(self)
    243 
    244     @property

~\Anaconda3\envs\pytorch\lib\site-packages\torch\utils\data\dataloader.py in __init__(self, loader)
    639             #     before it starts, and __del__ tries to join but will get:
    640             #     AssertionError: can only join a started process.
--&gt; 641             w.start()
    642             self.index_queues.append(index_queue)
    643             self.workers.append(w)

~\Anaconda3\envs\pytorch\lib\multiprocessing\process.py in start(self)
    110                'daemonic processes are not allowed to have children'
    111         _cleanup()
--&gt; 112         self._popen = self._Popen(self)
    113         self._sentinel = self._popen.sentinel
    114         # Avoid a refcycle if the target function holds an indirect

~\Anaconda3\envs\pytorch\lib\multiprocessing\context.py in _Popen(process_obj)
    221     @staticmethod
    222     def _Popen(process_obj):
--&gt; 223         return _default_context.get_context().Process._Popen(process_obj)
    224 
    225 class DefaultContext(BaseContext):

~\Anaconda3\envs\pytorch\lib\multiprocessing\context.py in _Popen(process_obj)
    320         def _Popen(process_obj):
    321             from .popen_spawn_win32 import Popen
--&gt; 322             return Popen(process_obj)
    323 
    324     class SpawnContext(BaseContext):

~\Anaconda3\envs\pytorch\lib\multiprocessing\popen_spawn_win32.py in __init__(self, process_obj)
     87             try:
     88                 reduction.dump(prep_data, to_child)
---&gt; 89                 reduction.dump(process_obj, to_child)
     90             finally:
     91                 set_spawning_popen(None)

~\Anaconda3\envs\pytorch\lib\multiprocessing\reduction.py in dump(obj, file, protocol)
     58 def dump(obj, file, protocol=None):
     59     '''Replacement for pickle.dump() using ForkingPickler.'''
---&gt; 60     ForkingPickler(file, protocol).dump(obj)
     61 
     62 #

BrokenPipeError: [Errno 32] Broken pipe


</code></pre>
<p>Also I want to know the usage of <code>start</code>, and <code>end</code> and what should be the batch_size I  should pass in the <code>Dataloader</code>, because I am already specifying the start and end in Dataloader.</p>