Repository: pytorch
Issue: DataLoader leaking resources?
Labels: module: dataloader,triaged

### 🐛 Describe the bug

There seems to be some sort of resource leakage happening when using `DataLoader` with >0 `num_workers`. I have discovered this with LMDB not sure if it will apply to other similar resources.

Here is a minimum repro
```python
import lmdb
import torch
from torch.utils.data import DataLoader
from itertools import count
from pathlib import Path

path = Path("~/data/mp_repro").expanduser().absolute()

class Dataset():

    def __init__(self):

        self._cache = None


    def __getitem__(self, idx):

        if self._cache is None:
            self._cache = lmdb.open(
                path.as_posix(),
                readonly=False,
                # lock=False,
                readahead=False,
                meminit=False,
                map_size=int(1e10),  # 10gb
        )


        with self._cache.begin() as tx:
            results = tx.get(f"{idx}".encode('utf-8'))

        return results

    def __len__(self):
        return 1000


cache = lmdb.open(
                path.as_posix(),
                readonly=False,
                # lock=False,
                readahead=False,
                meminit=False,
                map_size=int(1e10),  # 10gb
        )
for idx in range(1000):
    encoded = f"{idx}".encode('utf-8')
    with cache.begin(write=True) as tx:
        tx.put(encoded, encoded)


del cache

if __name__=="__main__":

    loader = DataLoader(
        Dataset(),
        batch_size=100,
        num_workers = 8
    )

    print(f"Running with 8 workers")

    for i in count():
        print(f"Running epoch {i}")
        list(loader)
```

Running this I get 

```
Running with 8 workers
Running epoch 0
Running epoch 1
Running epoch 2
Running epoch 3
Running epoch 4
Running epoch 5
Running epoch 6
Running epoch 7
Running epoch 8
Running epoch 9
Running epoch 10
Running epoch 11
Running epoch 12
Running epoch 13
Running epoch 14
Running epoch 15
Traceback (most recent call last):
  File "mp_repro_torch.py", line 66, in <module>
    list(loader)
  File "/home/oliver/miniconda3/envs/nvflare/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 530, in __next__
    data = self._next_data()
  File "/home/oliver/miniconda3/envs/nvflare/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1224, in _next_data
    return self._process_data(data)
  File "/home/oliver/miniconda3/envs/nvflare/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1250, in _process_data
    data.reraise()
  File "/home/oliver/miniconda3/envs/nvflare/lib/python3.8/site-packages/torch/_utils.py", line 457, in reraise
    raise exception
lmdb.ReadersFullError: Caught ReadersFullError in DataLoader worker process 2.
Original Traceback (most recent call last):
  File "/home/oliver/miniconda3/envs/nvflare/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py", line 287, in _worker_loop
    data = fetcher.fetch(index)
  File "/home/oliver/miniconda3/envs/nvflare/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 49, in fetch
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/home/oliver/miniconda3/envs/nvflare/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 49, in <listcomp>
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "mp_repro_torch.py", line 19, in __getitem__
    self._cache = lmdb.open(
lmdb.ReadersFullError: mdb_txn_begin: MDB_READERS_FULL: Environment maxreaders limit reached
```

At first I believed this to be an LMDB issue but was unable to reproduce without pytorch. Here is the pytorch-less code that does **not** exhibit this behavior

```python
import lmdb
import torch
from itertools import count, repeat
from pathlib import Path
import multiprocessing

path = Path("~/data/mp_repro").expanduser().absolute()

class Dataset():

    def __init__(self):

        self._cache = None


    def __getitem__(self, idx):

        if self._cache is None:
            self._cache = lmdb.open(
                path.as_posix(),
                readonly=False,
                # lock=False,
                readahead=False,
                meminit=False,
                map_size=int(1e10),  # 10gb
        )


        with self._cache.begin() as tx:
            results = tx.get(f"{idx}".encode('utf-8'))

        return results

    def __len__(self):
        return 1000


cache = lmdb.open(
                path.as_posix(),
                readonly=False,
                # lock=False,
                readahead=False,
                meminit=False,
                map_size=int(1e10),  # 10gb
        )
for idx in range(1000):
    encoded = f"{idx}".encode('utf-8')
    with cache.begin(write=True) as tx:
        tx.put(encoded, encoded)


del cache

def batch(dataset, start, batch_size=100):
    return [dataset[i] for i in range(start, start+batch_size)]

def batch_dataset(dataset, batch_size=100):
    pool = multiprocessing.Pool(8)

    for i in pool.starmap(batch, zip(repeat(dataset, 10), range(0, len(dataset), batch_size))):
        pass
    

if __name__=="__main__":

    dataset = Dataset()

    
    print(f"Running with {torch.multiprocessing.cpu_count()} workers")

    for i in count():
        print(f"Running epoch {i}")
        batch_dataset(dataset)
```

I let this run to >epoch 1000 with no issues

### Versions

```
[pip3] mypy==0.950
[pip3] mypy-extensions==0.4.3
[pip3] numpy==1.20.0
[pip3] pytorch-lightning==1.5.10
[pip3] torch==1.11.0+cpu
[pip3] torchaudio==0.11.0+cpu
[pip3] torchmetrics==0.7.3
[pip3] torchvision==0.12.0+cpu
[conda] numpy                     1.20.0                   pypi_0    pypi
[conda] pytorch-lightning         1.5.10                   pypi_0    pypi
[conda] torch                     1.11.0+cpu               pypi_0    pypi
[conda] torchaudio                0.11.0+cpu               pypi_0    pypi
[conda] torchmetrics              0.7.3                    pypi_0    pypi
[conda] torchvision               0.12.0+cpu               pypi_0    pypi
```

cc @SsnL @VitalyFedyunin @ejguan @NivekT