Repository: pytorch
Issue: mmap error - no such device (19)
Labels: module: dataloader,triaged

### 🐛 Describe the bug

I created my own Dataset and got a very strange mmap error, which is impossible to debug at the moment. Maybe you have an idea. 

I use Lightning, but the error seems to come from pytorch. My training runs through with small fractions of the train dataset (10-20%), but it fails when I use a big split for the training/validation part. So it works when I adjust `test_size` in `_load_data()` but fails with big training data.

Strangely enough this only occurs on some of my machines but not all. Would be thankful for any hints where to look for the error.

Stacktrace:
```
_Epoch 0:  21%|██        | 10/48 [00:03<00:14,  2.55it/s, v_num=95]Traceback (most recent call last):
  File "/raid/.../classifier_lightning/main.py", line 33, in <module>
    trainer.fit(model, mydata)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 544, in fit
    call._call_and_handle_interrupt(
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 44, in _call_and_handle_interrupt
    return trainer_fn(*args, **kwargs)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 580, in _fit_impl
    self._run(model, ckpt_path=ckpt_path)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 987, in _run
    results = self._run_stage()
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 1033, in _run_stage
    self.fit_loop.run()
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 205, in run
    self.advance()
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 363, in advance
    self.epoch_loop.run(self._data_fetcher)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 140, in run
    self.advance(data_fetcher)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 212, in advance
    batch, _, __ = next(data_fetcher)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/loops/fetchers.py", line 133, in __next__
    batch = super().__next__()
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/loops/fetchers.py", line 60, in __next__
    batch = next(self.iterator)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/utilities/combined_loader.py", line 341, in __next__
    out = next(self._iterator)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/lightning/pytorch/utilities/combined_loader.py", line 78, in __next__
    out[i] = next(self.iterators[i])
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 631, in __next__
    data = self._next_data()
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1346, in _next_data
    return self._process_data(data)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1372, in _process_data
    data.reraise()
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torch/_utils.py", line 722, in reraise
    raise exception
RuntimeError: Caught RuntimeError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 308, in _worker_loop
    data = fetcher.fetch(index)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 49, in fetch
    data = self.dataset.__getitems__(possibly_batched_index)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torch/utils/data/dataset.py", line 399, in __getitems__
    return [self.dataset[self.indices[idx]] for idx in indices]
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torch/utils/data/dataset.py", line 399, in <listcomp>
    return [self.dataset[self.indices[idx]] for idx in indices]
  File "/raid/.../classifier_lightning/dataset.py", line 40, in __getitem__
    image = read_image(img_path)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torchvision/io/image.py", line 258, in read_image
    data = read_file(path)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torchvision/io/image.py", line 52, in read_file
    data = torch.ops.image.read_file(path)
  File "/raid/.../classifier_lightning/venv/lib/python3.10/site-packages/torch/_ops.py", line 755, in __call__
    return self._op(*args, **(kwargs or {}))
RuntimeError: unable to mmap 671744 bytes from file </raid/nils/datasets/Complete_16k_unbalanced/>: No such device (19)_
```

This is my Dataset and loaders as implemented:
```

from config import Settings
from torch.utils.data import Dataset
from torchvision.io import read_image
from torch.utils.data import random_split
from torch.utils import data
import lightning as L
import csv


class CustomImageDataset(Dataset):
    def __init__(self, data_dir, train=True, transform=None, target_transform=None):
        self.data_dir = data_dir
        self.transform = transform
        self.target_transform = target_transform
        self.train = train
        self.data_paths = self._load_data()

    def __len__(self):
        return len(self.data_paths)

    def _load_data(self):
        with open(self.data_dir + "/anno.csv", newline='') as file:
            reader = csv.reader(file)
            rows = list(reader)

        test_size = int(len(rows) * 0.2)
        if self.train:
            rows = rows[test_size:]
        else:
            rows = rows[:test_size]

        return rows

    def __getitem__(self, idx):
        img_path = self.data_paths[idx][0].split(",")[0]
        image = read_image(img_path)
        label = int(self.data_paths[idx][0].split(",")[1])
        if self.transform:
            image = self.transform(image)
            image = image.to(eval("torch.float"+f"{Settings.PRECISION.value}"))
        if self.target_transform:
            label = self.target_transform(label)
        return image, label


class LightningDataSet(L.LightningDataModule):
    def __init__(self, data_dir: str, batch_size, num_workers, seed, transform, target_transform):
        super().__init__()
        self.val_set = None
        self.test_set = None
        self.train_set = None
        self.predict_set = None
        self.transform = transform
        self.target_transform = target_transform

        self.data_dir = data_dir
        self.batch_size = batch_size
        self.num_workers = num_workers
       # self.gen = Generator().manual_seed(seed)

    def setup(self, stage):

        if stage == "fit":
            full = CustomImageDataset(self.data_dir,
                                      train=True,
                                      transform=self.transform,
                                      target_transform=self.target_transform)
            train_size = int(0.9 * len(full))
            val_size = int(len(full))-train_size
            self.train_set, self.val_set = random_split(full, [train_size, val_size])

        elif stage == "test":
            self.test_set = CustomImageDataset(self.data_dir,
                                               train=False,
                                               transform=self.transform,
                                               target_transform=self.target_transform)

        elif stage == "predict":
            self.predict_set = CustomImageDataset(self.data_dir,
                                                  train=False,
                                                  transform=self.transform,
                                                  target_transform=self.target_transform)

    def train_dataloader(self):
        return data.DataLoader(self.train_set, num_workers=self.num_workers, batch_size=self.batch_size)

    def val_dataloader(self):
        return data.DataLoader(self.val_set, num_workers=self.num_workers, batch_size=self.batch_size)

    def test_dataloader(self):
        return data.DataLoader(self.test_set, num_workers=self.num_workers, batch_size=self.batch_size)

    def predict_dataloader(self):
        return data.DataLoader(self.predict_set, num_workers=self.num_workers, batch_size=self.batch_size)
```


### Versions

Ubuntu 22.04
Newest torch & lightning

cc @SsnL @VitalyFedyunin @ejguan @dzhulgakov