Repository: pytorch
Issue: MultiGPU distributed training Hang
Labels: oncall: distributed

### 🐛 Describe the bug

PyTorch deadlocks when using distributed training.
### To Reproduce
```
mport argparse
import os
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from datetime import timedelta


DEFAULT_TIMEOUT = timedelta(seconds=10)


def func(args):
    print(torch.randn(1))

def run(main_func, backend, num_machines, num_gpus, machine_rank, dist_url, args=()):
    world_size = num_machines * num_gpus

    mp.spawn(
        distributed_worker,
        nprocs=num_gpus,
        args=(
            main_func,
            backend,
            world_size,
            num_gpus,
            machine_rank,
            dist_url,
            args,
        ),
        daemon=False,
    )

def distributed_worker(
    local_rank,
    main_func,
    backend,
    world_size,
    num_gpus_per_machine,
    machine_rank,
    dist_url,
    args,
    timeout=DEFAULT_TIMEOUT,
):
    LOCAL_PROCESS_GROUP = None

    if not torch.cuda.is_available():
        raise RuntimeError("CUDA is not available. Please check your installation.")

    global_rank = machine_rank * num_gpus_per_machine + local_rank
    try:
        dist.init_process_group(
            backend=backend,
            init_method=dist_url,
            world_size=world_size,
            rank=global_rank,
            timeout=timeout,
        )
    except Exception as e:
        print(f"Process group URL: {dist_url}")
        raise e


    dist.barrier()

    print(f"Global rank {global_rank}.")
    print("Synchronized GPUs.")

    if num_gpus_per_machine > torch.cuda.device_count():
        raise RuntimeError
    torch.cuda.set_device(local_rank)

    # Setup the local process group (which contains ranks within the same machine)
    if LOCAL_PROCESS_GROUP is not None:
        raise RuntimeError

    num_machines = world_size // num_gpus_per_machine

    for idx in range(num_machines):
        ranks_on_i = list(range(idx * num_gpus_per_machine, (idx + 1) * num_gpus_per_machine))
        pg = dist.new_group(ranks_on_i)
        if idx == machine_rank:
            LOCAL_PROCESS_GROUP = pg

    main_func(args)
def main():
    torch.set_num_threads(1)
    os.environ["OMP_NUM_THREADS"] = "1"
    os.environ["NCCL_DEBUG"] = "INFO"

    print(f"CUDA {torch.version.cuda} - cuDNN {torch.backends.cudnn.version()} - cudaNCCL {torch.cuda.nccl.version()}")
    parser = argparse.ArgumentParser()
    parser.add_argument("--backend", type=str, default="NCCL", help="'gloo' or 'NCCL'.")
    parser.add_argument("--num-gpus", type=int, default=1, help="# GPUs per machine.")
    parser.add_argument("--num-machines", type=int, default=1, help="# of machines.")
    parser.add_argument(
        "--machine-rank",
        type=int,
        default=0,
        help="the rank of this machine (unique per machine).",
    )

    port = 1234
    parser.add_argument(
        "--dist-url",
        type=str,
        default=f"tcp://127.0.0.1:{port}",
        help="initialization URL for pytorch distributed backend. See "
        "https://pytorch.org/docs/stable/distributed.html for details.",
    )

    args = parser.parse_args()

    run(
        main_func=func,
        backend=args.backend,
        num_machines=args.num_machines,
        num_gpus=args.num_gpus,
        machine_rank=args.machine_rank,
        dist_url=args.dist_url,
        args=()
    )


if __name__ == '__main__':
    main()
```

My H100 * 8 gpu working fine if I only applied two GPU
```
(base) root@g3-xlarge-x86-dal-1:/home/ubuntu/examples/distributed/FSDP# python test.py --num-gpus 2
CUDA 12.1 - cuDNN 8902 - cudaNCCL (2, 19, 3)
g3-xlarge-x86-dal-1:3209:3209 [0] NCCL INFO Bootstrap : Using eno1:160.202.129.119<0>
g3-xlarge-x86-dal-1:3209:3209 [0] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
g3-xlarge-x86-dal-1:3209:3209 [0] NCCL INFO cudaDriverVersion 12010
NCCL version 2.19.3+cuda12.3
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Failed to open libibverbs.so[.1]
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO NET/Socket : Using [0]eno1:160.202.129.119<0> [1]eno2:10.87.1.117<0>
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Using non-device net plugin version 0
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Using network Socket
g3-xlarge-x86-dal-1:3210:3210 [1] NCCL INFO cudaDriverVersion 12010
g3-xlarge-x86-dal-1:3210:3210 [1] NCCL INFO Bootstrap : Using eno1:160.202.129.119<0>
g3-xlarge-x86-dal-1:3210:3210 [1] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Failed to open libibverbs.so[.1]
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO NET/Socket : Using [0]eno1:160.202.129.119<0> [1]eno2:10.87.1.117<0>
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Using non-device net plugin version 0
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Using network Socket
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO comm 0x563e73944f40 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 21000 commId 0x366b80c8232eae07 - Init START
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO comm 0x555a13551440 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 1000 commId 0x366b80c8232eae07 - Init START
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Setting affinity for GPU 1 to ffffffff,00000000,ffffffff
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Setting affinity for GPU 0 to ffffffff,00000000,ffffffff
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 00/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 01/16 :    0   1
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] -1/-1/-1->1->0 [2] -1/-1/-1->1->0 [3] -1/-1/-1->1->0 [4] 0/-1/-1->1->-1 [5] 0/-1/-1->1->-1 [6] 0/-1/-1->1->-1 [7] 0/-1/-1->1->-1 [8] -1/-1/-1->1->0 [9] -1/-1/-1->1->0 [10] -1/-1/-1->1->0 [11] -1/-1/-1->1->0 [12] 0/-1/-1->1->-1 [13] 0/-1/-1->1->-1 [14] 0/-1/-1->1->-1 [15] 0/-1/-1->1->-1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 02/16 :    0   1
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO P2P Chunksize set to 524288
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 03/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 04/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 05/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 06/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 07/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 08/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 09/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 10/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 11/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 12/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 13/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 14/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 15/16 :    0   1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] -1/-1/-1->0->1 [5] -1/-1/-1->0->1 [6] -1/-1/-1->0->1 [7] -1/-1/-1->0->1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] -1/-1/-1->0->1 [13] -1/-1/-1->0->1 [14] -1/-1/-1->0->1 [15] -1/-1/-1->0->1
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO P2P Chunksize set to 524288
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 03/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 04/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 05/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 06/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 07/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 08/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 09/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 10/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 11/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 12/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 13/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 14/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Channel 15/0 : 1[1] -> 0[0] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Connected all rings
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Connected all rings
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO Connected all trees
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO Connected all trees
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO 16 coll channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO 16 coll channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer
g3-xlarge-x86-dal-1:3209:3246 [0] NCCL INFO comm 0x555a13551440 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 1000 commId 0x366b80c8232eae07 - Init COMPLETE
g3-xlarge-x86-dal-1:3210:3248 [1] NCCL INFO comm 0x563e73944f40 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 21000 commId 0x366b80c8232eae07 - Init COMPLETE
Global rank 0.
Synchronized GPUs.
Global rank 1.
Synchronized GPUs.
tensor([-0.5722])
tensor([0.6162])
g3-xlarge-x86-dal-1:3209:3252 [0] NCCL INFO [Service thread] Connection closed by localRank 1
g3-xlarge-x86-dal-1:3210:3251 [1] NCCL INFO [Service thread] Connection closed by localRank 1
g3-xlarge-x86-dal-1:3209:3252 [0] NCCL INFO [Service thread] Connection closed by localRank 0
g3-xlarge-x86-dal-1:3210:3251 [1] NCCL INFO [Service thread] Connection closed by localRank 0
g3-xlarge-x86-dal-1:3210:3210 [1] NCCL INFO comm 0x563e73944f40 rank 1 nranks 2 cudaDev 1 busId 21000 - Abort COMPLETE
g3-xlarge-x86-dal-1:3209:3209 [0] NCCL INFO comm 0x555a13551440 rank 0 nranks 2 cudaDev 0 busId 1000 - Abort COMPLETE
```

But it will Hang If I try 4 or more GPUs:
```
(base) root@g3-xlarge-x86-dal-1:/home/ubuntu/examples/distributed/FSDP# python test.py --num-gpus 4
CUDA 12.1 - cuDNN 8902 - cudaNCCL (2, 19, 3)
g3-xlarge-x86-dal-1:3419:3419 [0] NCCL INFO Bootstrap : Using eno1:160.202.129.119<0>
g3-xlarge-x86-dal-1:3419:3419 [0] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
g3-xlarge-x86-dal-1:3419:3419 [0] NCCL INFO cudaDriverVersion 12010
NCCL version 2.19.3+cuda12.3
g3-xlarge-x86-dal-1:3419:3468 [0] NCCL INFO Failed to open libibverbs.so[.1]
g3-xlarge-x86-dal-1:3419:3468 [0] NCCL INFO NET/Socket : Using [0]eno1:160.202.129.119<0> [1]eno2:10.87.1.117<0>
g3-xlarge-x86-dal-1:3419:3468 [0] NCCL INFO Using non-device net plugin version 0
g3-xlarge-x86-dal-1:3419:3468 [0] NCCL INFO Using network Socket
g3-xlarge-x86-dal-1:3422:3422 [3] NCCL INFO cudaDriverVersion 12010
g3-xlarge-x86-dal-1:3422:3422 [3] NCCL INFO Bootstrap : Using eno1:160.202.129.119<0>
g3-xlarge-x86-dal-1:3422:3422 [3] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
g3-xlarge-x86-dal-1:3421:3421 [2] NCCL INFO cudaDriverVersion 12010
g3-xlarge-x86-dal-1:3421:3421 [2] NCCL INFO Bootstrap : Using eno1:160.202.129.119<0>
g3-xlarge-x86-dal-1:3421:3421 [2] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
g3-xlarge-x86-dal-1:3422:3469 [3] NCCL INFO Failed to open libibverbs.so[.1]
g3-xlarge-x86-dal-1:3421:3470 [2] NCCL INFO Failed to open libibverbs.so[.1]
g3-xlarge-x86-dal-1:3421:3470 [2] NCCL INFO NET/Socket : Using [0]eno1:160.202.129.119<0> [1]eno2:10.87.1.117<0>
g3-xlarge-x86-dal-1:3422:3469 [3] NCCL INFO NET/Socket : Using [0]eno1:160.202.129.119<0> [1]eno2:10.87.1.117<0>
g3-xlarge-x86-dal-1:3421:3470 [2] NCCL INFO Using non-device net plugin version 0
g3-xlarge-x86-dal-1:3421:3470 [2] NCCL INFO Using network Socket
g3-xlarge-x86-dal-1:3422:3469 [3] NCCL INFO Using non-device net plugin version 0
g3-xlarge-x86-dal-1:3422:3469 [3] NCCL INFO Using network Socket

```
Because I am using AMD CPU, I also disable ACS and IOMMU
```
/home/ubuntu/examples/distributed/FSDP# sudo lspci -vvv | grep ACSCtl
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-
		ACSCtl:	SrcValid- TransBlk- ReqRedir- CmpltRedir- UpstreamFwd- EgressCtrl- DirectTrans-

```
Try to debug it for two days but no good luck, please help.




### Versions

```
HTTP request sent, awaiting response... 200 OK
Length: 22068 (22K) [text/plain]
Saving to: ‘collect_env.py’

collect_env.py                                100%[=================================================================================================>]  21.55K  --.-KB/s    in 0s      

2024-01-27 20:31:28 (54.3 MB/s) - ‘collect_env.py’ saved [22068/22068]

Collecting environment information...
PyTorch version: 2.3.0.dev20240127+cu121
Is debug build: False
CUDA used to build PyTorch: 12.1
ROCM used to build PyTorch: N/A

OS: Ubuntu 22.04.3 LTS (x86_64)
GCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
Clang version: Could not collect
CMake version: Could not collect
Libc version: glibc-2.35

Python version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)
Python platform: Linux-5.15.0-92-generic-x86_64-with-glibc2.35
Is CUDA available: True
CUDA runtime version: Could not collect
CUDA_MODULE_LOADING set to: LAZY
GPU models and configuration: 
GPU 0: NVIDIA H100 PCIe
GPU 1: NVIDIA H100 PCIe
GPU 2: NVIDIA H100 PCIe
GPU 3: NVIDIA H100 PCIe
GPU 4: NVIDIA H100 PCIe
GPU 5: NVIDIA H100 PCIe
GPU 6: NVIDIA H100 PCIe
GPU 7: NVIDIA H100 PCIe

Nvidia driver version: 530.30.02
cuDNN version: Probably one of the following:
/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.3
/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.3
/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.3
/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.3
/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.3
/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.3
/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.3
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

CPU:
Architecture:                       x86_64
CPU op-mode(s):                     32-bit, 64-bit
Address sizes:                      52 bits physical, 57 bits virtual
Byte Order:                         Little Endian
CPU(s):                             128
On-line CPU(s) list:                0-127
Vendor ID:                          AuthenticAMD
Model name:                         AMD EPYC 9354 32-Core Processor
CPU family:                         25
Model:                              17
Thread(s) per core:                 2
Core(s) per socket:                 32
Socket(s):                          2
Stepping:                           1
Frequency boost:                    enabled
CPU max MHz:                        3799.0720
CPU min MHz:                        1500.0000
BogoMIPS:                           6500.01
Flags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 invpcid_single hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq la57 rdpid overflow_recov succor smca fsrm flush_l1d
Virtualization:                     AMD-V
L1d cache:                          2 MiB (64 instances)
L1i cache:                          2 MiB (64 instances)
L2 cache:                           64 MiB (64 instances)
L3 cache:                           512 MiB (16 instances)
NUMA node(s):                       2
NUMA node0 CPU(s):                  0-31,64-95
NUMA node1 CPU(s):                  32-63,96-127
Vulnerability Gather data sampling: Not affected
Vulnerability Itlb multihit:        Not affected
Vulnerability L1tf:                 Not affected
Vulnerability Mds:                  Not affected
Vulnerability Meltdown:             Not affected
Vulnerability Mmio stale data:      Not affected
Vulnerability Retbleed:             Not affected
Vulnerability Spec rstack overflow: Mitigation; safe RET
Vulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp
Vulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization
Vulnerability Spectre v2:           Mitigation; Retpolines, IBPB conditional, IBRS_FW, STIBP always-on, RSB filling, PBRSB-eIBRS Not affected
Vulnerability Srbds:                Not affected
Vulnerability Tsx async abort:      Not affected

Versions of relevant libraries:
[pip3] msgpack-numpy-opentensor==0.5.0
[pip3] mypy-extensions==1.0.0
[pip3] numpy==1.26.2
[pip3] pytorch-lightning==2.1.3
[pip3] pytorch-triton==3.0.0+901819d2b6
[pip3] rotary-embedding-torch==0.5.3
[pip3] torch==2.3.0.dev20240127+cu121
[pip3] torchaudio==2.2.0.dev20240127+cu121
[pip3] torchmetrics==1.3.0.post0
[pip3] torchvision==0.18.0.dev20240127+cu121
[pip3] triton==2.1.0
[conda] No relevant packages
```

cc @mrshenli @pritamdamania87 @zhaojuanmao @satgera @rohan-varma @gqchen @aazzolini @osalpekar @jiayisuse @H-Huang @kwen2501 @awgu @penguinwu @fegin @XilunWu @wanchaol @fduwjj @wz337 @tianyu-l @wconstab @yf225