Repository: pytorch
Issue: `init_process_group` hanging on HPC multi-node system w GPU 
Labels: oncall: distributed,triaged

### 🐛 Describe the bug

Hi! I was trying to train the model with `torch.nn.parallel.DistributedDataParallel` on an HPC instance with 4 nodes, and 1 GPU per node. But the model verification program hangs for hours when encountered with `init_process_group`.

Here is the program to reproduce the same error, I have tried to explore the previous issues(#53395) but no success till now.

```python
import argparse
import builtins
import os
import socket
from contextlib import closing

import torch
import torch.distributed as dist
from dartorch.data import DARDatasetOnVideos
from dartorch.models import DrivingActionClassifier
from torch.nn.parallel.distributed import DistributedDataParallel
from torch.utils.data import DataLoader, DistributedSampler
from torchinfo import summary


def get_open_port():
    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
        s.bind(('', 0))
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        return s.getsockname()[1]

def train_one_epoch(train_loader, model):
    for i, (data, target) in enumerate(train_loader):
        data, target = data.cuda(), target.cuda()
        cls = model(data)
        print(f'Input shape : {data.shape}, Output shape : {cls.shape}')
        break
    return

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('data_dir', type=str, help='Data root directory.')
    parser.add_argument('--batch_size', type=int, default=1, help='Model batch size.')
    parser.add_argument('--im_size', type=int, default=128, help='Model image size(assumed square in size).')
    parser.add_argument('--nproc_dl', type=int, default=5, help='Number of workers to load data.')

    # DDP configuration
    parser.add_argument('--gpu', default=None, type=int)
    parser.add_argument('--world-size', default=-1, type=int,
                        help='number of nodes for distributed training')
    parser.add_argument('--rank', default=-1, type=int,
                        help='node rank for distributed training')
    parser.add_argument('--dist-url', default='env://', type=str,
                        help='url used to set up distributed training')
    parser.add_argument('--dist-backend', default='nccl', type=str,
                        help='distributed backend')
    parser.add_argument('--local_rank', default=-1, type=int,
                        help='local rank for distributed training')

    args = parser.parse_args()

    port = get_open_port()
    os.environ['MASTER_ADDR'] = 'localhost'
    os.environ['MASTER_PORT'] = str(port)

    if "WORLD_SIZE" in os.environ:
        args.world_size = int(os.environ["WORLD_SIZE"])
    args.distributed = args.world_size > 1
    ngpus_per_node = torch.cuda.device_count()

    if args.distributed:
        if args.local_rank != -1: # for torch.distributed.launch
            args.rank = args.local_rank
            args.gpu = args.local_rank
        elif 'SLURM_PROCID' in os.environ: # for slurm scheduler
            args.rank = int(os.environ['SLURM_PROCID'])
            args.gpu = args.rank % torch.cuda.device_count()
        dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
                                world_size=args.world_size, rank=args.rank)

    # suppress printing if not on master gpu
    if args.rank!=0:
        def print_pass(*args):
            pass
        builtins.print = print_pass

    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    print(f'Target decvice : {device}')

    # Define model
    model = DrivingActionClassifier(in_size=(args.im_size, args.im_size))
    # model_without_ddp = None
    # if args.distributed:
    #     # For multiprocessing distributed, DistributedDataParallel constructor
    #     # should always set the single device scope, otherwise,
    #     # DistributedDataParallel will use all available devices.
    #     if args.gpu is not None:
    #         torch.cuda.set_device(args.gpu)
    #         model.cuda(args.gpu)
    #         model = DistributedDataParallel(model, device_ids=[args.gpu])
    #         model_without_ddp = model.module
    #     else:
    #         model.cuda()
    #         model = DistributedDataParallel(model)
    #         model_without_ddp = model.module
    # else:
    #     raise NotImplementedError("Only DistributedDataParallel is supported.")

    # train_dataset = DARDatasetOnVideos(args.data_dir, test_user_ids = ["user_id_49381"],
    #                              img_size=(args.im_size, args.im_size),
    #                              load_reader_instances = True,
    #                              mode='train', shuffle_views=True)
    # train_sampler = DistributedSampler(train_dataset, shuffle=True)

    # train_loader = DataLoader(
    #         train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None),
    #         num_workers=args.nproc_dl, pin_memory=True, sampler=train_sampler, drop_last=False)

    for epoch in range(2):
        if args.distributed:
            print(f'Training on epoch : {epoch}')
            # train_loader.sampler.set_epoch(epoch)
            # train_one_epoch(train_loader, model)

    print(f'Test completed.')

```

### Versions

Collecting environment information...
PyTorch version: 1.10.0
Is debug build: False
CUDA used to build PyTorch: 11.3
ROCM used to build PyTorch: N/A

OS: CentOS Linux 7 (Core) (x86_64)
GCC version: (GCC) 7.3.0
Clang version: Could not collect
CMake version: version 3.20.5
Libc version: glibc-2.17

Python version: 3.7.11 (default, Jul 27 2021, 14:32:16)  [GCC 7.5.0] (64-bit runtime)
Python platform: Linux-3.10.0-1062.el7.x86_64-x86_64-with-centos-7.7.1908-Core
Is CUDA available: True
CUDA runtime version: 11.2.67
GPU models and configuration: GPU 0: Tesla V100-SXM2-16GB
Nvidia driver version: 450.51.06
cuDNN version: Could not collect
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

Versions of relevant libraries:
[pip3] dartorch==0.0.0
[pip3] numpy==1.21.2
[pip3] torch==1.10.0
[pip3] torch-cluster==1.5.9
[pip3] torch-geometric==2.0.2
[pip3] torch-scatter==2.0.9
[pip3] torch-sparse==0.6.12
[pip3] torch-spline-conv==1.2.1
[pip3] torchaudio==0.10.0
[pip3] torchinfo==1.6.5
[pip3] torchsummary==1.5.1
[pip3] torchvision==0.11.1
[conda] blas                      1.0                         mkl
[conda] cudatoolkit               11.3.1               h2bc3f7f_2
[conda] dartorch                  0.0.0                     dev_0    <develop>
[conda] ffmpeg                    4.3                  hf484d3e_0    pytorch
[conda] mkl                       2021.4.0           h06a4308_640
[conda] mkl-service               2.4.0            py37h7f8727e_0
[conda] mkl_fft                   1.3.1            py37hd3c417c_0
[conda] mkl_random                1.2.2            py37h51133e4_0
[conda] numpy                     1.21.2           py37h20f2e39_0
[conda] numpy-base                1.21.2           py37h79a1101_0
[conda] pyg                       2.0.2           py37_torch_1.10.0_cu113    pyg
[conda] pytorch                   1.10.0          py3.7_cuda11.3_cudnn8.2.0_0    pytorch
[conda] pytorch-cluster           1.5.9           py37_torch_1.10.0_cu113    pyg
[conda] pytorch-mutex             1.0                        cuda    pytorch
[conda] pytorch-scatter           2.0.9           py37_torch_1.10.0_cu113    pyg
[conda] pytorch-sparse            0.6.12          py37_torch_1.10.0_cu113    pyg
[conda] pytorch-spline-conv       1.2.1           py37_torch_1.10.0_cu113    pyg
[conda] torchaudio                0.10.0               py37_cu113    pytorch
[conda] torchinfo                 1.6.5                    pypi_0    pypi
[conda] torchsummary              1.5.1                    pypi_0    pypi
[conda] torchvision               0.11.1               py37_cu113    pytorch

cc @pietern @mrshenli @pritamdamania87 @zhaojuanmao @satgera @rohan-varma @gqchen @aazzolini @osalpekar @jiayisuse @SciPioneer @H-Huang