Repository: pytorch
Issue: PyTorch 2.1.0 High memory utilization compared to PyTorch 2.0.1
Labels: high priority,module: dataloader,triaged

### 🐛 Describe the bug

Hi, we have found that PyTorch 2.1.2 utilizes significantly higher CPU memory utilization from PyTorch 2.0.1 on AWS g5.xlarge instance type. Below are the results we observed from running an example training scripts.

Training script
```python
from torch.utils.data import DataLoader, Dataset
import numpy as np
import os
import psutil
import warnings
from torchvision.transforms import ToTensor, Resize, InterpolationMode
import cv2
import torch
import logging


use_cv2_resize_for_labels = False


class MyTransforms:
    def __init__(self):
        self.to_tensor = ToTensor()
        self.resize_image = Resize(size=(272, 320), interpolation=InterpolationMode.BILINEAR)
        self.resize_labels = Resize(size=(272, 320), interpolation=InterpolationMode.NEAREST)

    def __call__(self, sample):
        image = self.to_tensor(sample["image"])
        image = self.resize_image(image)

        labels = self.transforms_labels(sample["label"])

        return {"semantic_segmentation": labels,
                "image": image}

    def transforms_labels(self, labels):
        if use_cv2_resize_for_labels:
            resized_labels = cv2.resize(labels, (320, 272), interpolation=cv2.INTER_NEAREST)
            segmentation_value = torch.from_numpy(resized_labels[..., 0])
        else:
            resized_labels = self.resize_labels(self.to_tensor(labels))
            segmentation_value = resized_labels[0, ...]

        # required for manipulation like one hot encoding
        return segmentation_value.to(dtype=torch.long)


def create_dataloaders(num_workers, batch_size):
    dataloaders = dict()
    for mode in ["train", "test"]:
        dataloaders[mode] = DataLoader(
            dataset=FakeDataset(),
            num_workers=num_workers,
            persistent_workers=True,  # so we keep queue and procs in mem
            batch_size=batch_size
        )
    return dataloaders


class FakeDataset(Dataset):
    def __init__(self):
        self.transforms = MyTransforms()

    def __getitem__(self, item):
        sample = {
            'label': np.random.randint(size=(1080, 1920, 3), low=0, high=255).astype(np.uint8),
            'image': np.random.randint(size=(1080, 1920, 3), low=0, high=255).astype(np.uint8)
        }
        return self.transforms(sample)

    def __len__(self):
        return 1000000

def log_mem_legacy(shift='MB', desc=None):
    SHIFTS = {
        "KB": 10,
        "MB": 20,
        "GB": 30
    }
    used_percent = psutil.virtual_memory().percent
    mem_used = psutil.virtual_memory().used >> SHIFTS[shift]
    msg = f"mem usage {mem_used}{shift} {used_percent}%"
    if desc:
        msg = f"{msg} - {desc}"
    print(msg)
    return mem_used, used_percent


def get_most_memory_consuming_processes(num_processes=10):
    processes = []

    # Iterate over all running processes
    for pid in psutil.pids():
        try:
            process = psutil.Process(pid)
            if process.name().startswith("python"):
                processes.append(process)
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
            pass
    # Sort processes by memory usage
    processes.sort(key=lambda x: x.memory_info().rss, reverse=True)

    # Take the top N processes
    top_processes = processes[:num_processes]

    return top_processes


def log_most_memory_consuming_processes():
    # Configure logging
    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')

    # Get the most memory-consuming processes
    try:
        most_memory_consuming_processes = get_most_memory_consuming_processes()
    except psutil.NoSuchProcess as e:
        logging.error(f"Error retrieving process information: {e}")
        return

    # Log the information
    logging.info("Most memory-consuming processes:")
    for process in most_memory_consuming_processes:
        try:
            memory_info = process.memory_info()
            logging.info(f"PID: {process.pid}, Name: {process.name()}, RSS: {memory_info.rss / (1024 * 1024):.2f} MB, VMS: {memory_info.vms / (1024 * 1024):.2f} MB")
        except psutil.NoSuchProcess:
            logging.warning(f"Process with PID {process.pid} no longer exists.")


snaps = []


def log_mem(shift='MB', desc=None):
    if desc:
        print(f"----------------------{desc}----------------------")
    log_most_memory_consuming_processes()


def display_stats():
    stats = snaps[0].statistics('filename')
    print("\n*** top 5 stats grouped by filename ***")
    for s in stats[:5]:
        print(s)


def compare():
    first = snaps[0]
    for snapshot in snaps[1:]:
        stats = snapshot.compare_to(first, 'lineno')
        print("\n*** top 10 stats ***")
        for s in stats[:10]:
            print(s)


def forward_dataloader(mode, dataloader):
    log_mem(desc=f'{mode} start')
    for idx, _ in enumerate(dataloader, start=1):
        if idx == 1:
            break
        else:
            continue
    log_mem(desc=f'{mode} end')


def run():
    num_workers = os.cpu_count()
    batch_size = 128

    dataloaders = create_dataloaders(
        num_workers=num_workers,
        batch_size=batch_size
    )
    for mode, dataloader in dataloaders.items():
        forward_dataloader(mode, dataloader)


if __name__ == "__main__":
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        run()

```

Training logs for PyTorch 2.1.2
```
[1,mpirank:0,algo-1]<stdout>:----------------------train start----------------------
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:03,205 - Most memory-consuming processes:
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:03,205 - PID: 75, Name: python3.10, RSS: 407.04 MB, VMS: 3136.24 MB
[1,mpirank:0,algo-1]<stdout>:----------------------train end----------------------[1,mpirank:0,algo-1]<stdout>:
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,080 - Most memory-consuming processes:
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,080 - PID: 82, Name: python3.10, RSS: 3144.31 MB, VMS: 6080.99 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,080 - PID: 80, Name: python3.10, RSS: 3014.76 MB, VMS: 5963.30 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,080 - PID: 81, Name: python3.10, RSS: 3007.06 MB, VMS: 6055.72 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,080 - PID: 76, Name: python3.10, RSS: 2961.25 MB, VMS: 6009.89 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,081 - PID: 75, Name: python3.10, RSS: 410.59 MB, VMS: 3858.34 MB
[1,mpirank:0,algo-1]<stdout>:----------------------test start----------------------
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,127 - Most memory-consuming processes:
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,127 - PID: 82, Name: python3.10, RSS: 3138.67 MB, VMS: 6063.18 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,127 - PID: 80, Name: python3.10, RSS: 3030.76 MB, VMS: 5963.30 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,128 - PID: 81, Name: python3.10, RSS: 3007.06 MB, VMS: 6055.72 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,128 - PID: 76, Name: python3.10, RSS: 2961.25 MB, VMS: 6009.89 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 22:53:22,128 - PID: 75, Name: python3.10, RSS: 410.59 MB, VMS: 3645.84 MB
[1,mpirank:0,algo-1]<stderr>:Traceback (most recent call last):
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/runpy.py", line 196, in _run_module_as_main
[1,mpirank:0,algo-1]<stderr>:    return _run_code(code, main_globals, None,
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/runpy.py", line 86, in _run_code
[1,mpirank:0,algo-1]<stderr>:    exec(code, run_globals)
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/site-packages/mpi4py/__main__.py", line 7, in <module>
[1,mpirank:0,algo-1]<stderr>:    main()
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/site-packages/mpi4py/run.py", line 230, in main
[1,mpirank:0,algo-1]<stderr>:    run_command_line(args)
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/site-packages/mpi4py/run.py", line 47, in run_command_line
[1,mpirank:0,algo-1]<stderr>:    run_path(sys.argv[0], run_name='__main__')
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/runpy.py", line 289, in run_path
[1,mpirank:0,algo-1]<stderr>:    return _run_module_code(code, init_globals, run_name,
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/runpy.py", line 96, in _run_module_code
[1,mpirank:0,algo-1]<stderr>:    _run_code(code, mod_globals, init_globals,
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/runpy.py", line 86, in _run_code
[1,mpirank:0,algo-1]<stderr>:    exec(code, run_globals)
[1,mpirank:0,algo-1]<stderr>:  File "train.py", line 174, in <module>
[1,mpirank:0,algo-1]<stderr>:    run()
[1,mpirank:0,algo-1]<stderr>:  File "train.py", line 168, in run
[1,mpirank:0,algo-1]<stderr>:    forward_dataloader(mode, dataloader)
[1,mpirank:0,algo-1]<stderr>:  File "train.py", line 151, in forward_dataloader
[1,mpirank:0,algo-1]<stderr>:    [1,mpirank:0,algo-1]<stderr>:for idx, _ in enumerate(dataloader, start=1):
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__
[1,mpirank:0,algo-1]<stderr>:    data = self._next_data()
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1328, in _next_data
[1,mpirank:0,algo-1]<stderr>:    idx, data = self._get_data()
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1294, in _get_data
[1,mpirank:0,algo-1]<stderr>:    success, data = self._try_get_data()
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1132, in _try_get_data
[1,mpirank:0,algo-1]<stderr>:    data = self._data_queue.get(timeout=timeout)
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/multiprocessing/queues.py", line 113, in get
[1,mpirank:0,algo-1]<stderr>:    if not self._poll(timeout):
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/multiprocessing/connection.py", line 262, in poll
[1,mpirank:0,algo-1]<stderr>:    return self._poll(timeout)
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/multiprocessing/connection.py", line 429, in _poll
[1,mpirank:0,algo-1]<stderr>:    r = wait([self], timeout)
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/multiprocessing/connection.py", line 936, in wait
[1,mpirank:0,algo-1]<stderr>:    ready = selector.select(timeout)
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/selectors.py", line 416, in select
[1,mpirank:0,algo-1]<stderr>:    fd_event_list = self._selector.poll(timeout)
[1,mpirank:0,algo-1]<stderr>:  File "/opt/conda/lib/python3.10/site-packages/torch/utils/data/_utils/signal_handling.py", line 66, in handler
[1,mpirank:0,algo-1]<stderr>:    _error_if_any_worker_fails()
[1,mpirank:0,algo-1]<stderr>:RuntimeError: DataLoader worker (pid 80) is killed by signal: Killed.
```

Training logs for PyTorch 2.0.1
```
[1,mpirank:0,algo-1]<stdout>:----------------------train start----------------------
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:10:54,884 - Most memory-consuming processes:
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:10:54,884 - PID: 69, Name: python3.10, RSS: 397.57 MB, VMS: 3366.55 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:10:54,884 - PID: 71, Name: python3, RSS: 36.92 MB, VMS: 44.57 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:10:54,885 - PID: 73, Name: python3, RSS: 26.58 MB, VMS: 44.55 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:10:54,885 - PID: 72, Name: python3, RSS: 0.00 MB, VMS: 0.00 MB
[1,mpirank:0,algo-1]<stdout>:----------------------train end----------------------
[1,mpirank:0,algo-1]<stdout>:
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,844 - Most memory-consuming processes:
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,845 - PID: 84, Name: python3.10, RSS: 609.24 MB, VMS: 3905.44 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,845 - PID: 75, Name: python3.10, RSS: 566.61 MB, VMS: 3862.88 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,845 - PID: 76, Name: python3.10, RSS: 544.78 MB, VMS: 3841.10 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,845 - PID: 80, Name: python3.10, RSS: 519.24 MB, VMS: 3815.44 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,845 - PID: 69, Name: python3.10, RSS: 400.96 MB, VMS: 4512.64 MB
[1,mpirank:0,algo-1]<stdout>:----------------------test start----------------------
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,864 - Most memory-consuming processes:
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,864 - PID: 84, Name: python3.10, RSS: 609.24 MB, VMS: 3905.44 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,864 - PID: 75, Name: python3.10, RSS: 566.61 MB, VMS: 3862.88 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,864 - PID: 76, Name: python3.10, RSS: 544.78 MB, VMS: 3841.10 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,865 - PID: 80, Name: python3.10, RSS: 519.24 MB, VMS: 3815.44 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:11:29,865 - PID: 69, Name: python3.10, RSS: 400.96 MB, VMS: 4300.14 MB
[1,mpirank:0,algo-1]<stdout>:----------------------test end----------------------
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:12:36,362 - Most memory-consuming processes:
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:12:36,363 - PID: 103, Name: python3.10, RSS: 633.68 MB, VMS: 4697.07 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:12:36,363 - PID: 84, Name: python3.10, RSS: 609.24 MB, VMS: 3905.44 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:12:36,363 - PID: 75, Name: python3.10, RSS: 566.68 MB, VMS: 3862.88 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:12:36,364 - PID: 76, Name: python3.10, RSS: 503.47 MB, VMS: 3799.62 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:12:36,364 - PID: 108, Name: python3.10, RSS: 492.91 MB, VMS: 4555.33 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:12:36,365 - PID: 112, Name: python3.10, RSS: 488.10 MB, VMS: 4551.40 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:12:36,365 - PID: 80, Name: python3.10, RSS: 485.38 MB, VMS: 3781.55 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:12:36,365 - PID: 107, Name: python3.10, RSS: 484.03 MB, VMS: 4546.44 MB
[1,mpirank:0,algo-1]<stderr>:2024-01-16 23:12:36,366 - PID: 69, Name: python3.10, RSS: 401.12 MB, VMS: 5447.24 MB
```

### Versions

```Collecting environment information...
PyTorch version: 2.1.2+cu118
Is debug build: False
CUDA used to build PyTorch: 11.8
ROCM used to build PyTorch: N/A

OS: Ubuntu 20.04.6 LTS (x86_64)
GCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0
Clang version: Could not collect
CMake version: version 3.16.3
Libc version: glibc-2.31

Python version: 3.10.8 | packaged by conda-forge | (main, Nov 22 2022, 08:26:04) [GCC 10.4.0] (64-bit runtime)
Python platform: Linux-5.10.201-191.748.amzn2.x86_64-x86_64-with-glibc2.31
Is CUDA available: True
CUDA runtime version: 11.8.89
CUDA_MODULE_LOADING set to: LAZY
GPU models and configuration: GPU 0: NVIDIA A10G
Nvidia driver version: 535.104.12
cuDNN version: Could not collect
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

CPU:
Architecture:                       x86_64
CPU op-mode(s):                     32-bit, 64-bit
Byte Order:                         Little Endian
Address sizes:                      48 bits physical, 48 bits virtual
CPU(s):                             4
On-line CPU(s) list:                0-3
Thread(s) per core:                 2
Core(s) per socket:                 2
Socket(s):                          1
NUMA node(s):                       1
Vendor ID:                          AuthenticAMD
CPU family:                         23
Model:                              49
Model name:                         AMD EPYC 7R32
Stepping:                           0
CPU MHz:                            3294.269
BogoMIPS:                           5599.00
Hypervisor vendor:                  KVM
Virtualization type:                full
L1d cache:                          64 KiB
L1i cache:                          64 KiB
L2 cache:                           1 MiB
L3 cache:                           8 MiB
NUMA node0 CPU(s):                  0-3
Vulnerability Gather data sampling: Not affected
Vulnerability Itlb multihit:        Not affected
Vulnerability L1tf:                 Not affected
Vulnerability Mds:                  Not affected
Vulnerability Meltdown:             Not affected
Vulnerability Mmio stale data:      Not affected
Vulnerability Retbleed:             Mitigation; untrained return thunk; SMT enabled with STIBP protection
Vulnerability Spec rstack overflow: Mitigation; safe RET
Vulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp
Vulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization
Vulnerability Spectre v2:           Mitigation; Retpolines, IBPB conditional, STIBP always-on, RSB filling
Vulnerability Srbds:                Not affected
Vulnerability Tsx async abort:      Not affected
Flags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch topoext ssbd ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru wbnoinvd arat npt nrip_save rdpid

Versions of relevant libraries:
[pip3] numpy==1.26.2
[pip3] pytorch-lightning==2.1.2
[pip3] sagemaker-pytorch-training==2.8.1
[pip3] torch==2.1.2+cu118
[pip3] torchaudio==2.1.2+cu118
[pip3] torchdata==0.7.1+cpu
[pip3] torchmetrics==1.2.0
[pip3] torchnet==0.0.4
[pip3] torchtext==0.16.2+cpu
[pip3] torchvision==0.16.2+cu118
[pip3] triton==2.1.0
[conda] blas                      2.120                       mkl    conda-forge
[conda] blas-devel                3.9.0            20_linux64_mkl    conda-forge
[conda] cudatoolkit               11.8.0              h4ba93d1_12    conda-forge
[conda] libblas                   3.9.0            20_linux64_mkl    conda-forge
[conda] libcblas                  3.9.0            20_linux64_mkl    conda-forge
[conda] liblapack                 3.9.0            20_linux64_mkl    conda-forge
[conda] liblapacke                3.9.0            20_linux64_mkl    conda-forge
[conda] mkl                       2023.2.0         h84fe81f_50496    conda-forge
[conda] mkl-devel                 2023.2.0         ha770c72_50496    conda-forge
[conda] mkl-include               2023.2.0         h84fe81f_50496    conda-forge
[conda] numpy                     1.26.2          py310hb13e2d6_0    conda-forge
[conda] pytorch-cuda              11.8                 h7e8668a_3    https://aws-ml-conda-ec2.s3.us-west-2.amazonaws.com
[conda] pytorch-lightning         2.1.2                    pypi_0    pypi
[conda] pytorch-mutex             1.0                        cuda    https://aws-ml-conda-ec2.s3.us-west-2.amazonaws.com
[conda] sagemaker-pytorch-training 2.8.1                    pypi_0    pypi
[conda] torch                     2.1.2+cu118              pypi_0    pypi
[conda] torchaudio                2.1.2+cu118              pypi_0    pypi
[conda] torchdata                 0.7.1+cpu                pypi_0    pypi
[conda] torchmetrics              1.2.0                    pypi_0    pypi
[conda] torchnet                  0.0.4                    pypi_0    pypi
[conda] torchtext                 0.16.2+cpu               pypi_0    pypi
[conda] torchvision               0.16.2+cu118             pypi_0    pypi
[conda] triton                    2.1.0                    pypi_0    pypi
```

cc @ezyang @gchanan @zou3519 @kadeng @SsnL @VitalyFedyunin @ejguan @dzhulgakov