training hangs at line torch.cuda.synchronize()
Labels: module: cuda, triaged, module: deadlock

### 🐛 Describe the bug

Hi there,

I am using pytorch 1.12.1 to train a hugging face model on 8 V100 across two nodes and it hung after epoch 1 on the line torch.cuda.synchronize().

All the ranks are waiting for "cudaDeviceSynchronize ()" except the last one on the second node. I retrieved its stack trace from gdb which says
"""
#0  0x000014cd955ebdbc in ?? () from /lib64/libcuda.so.1
#1  0x000014cd953608b6 in ?? () from /lib64/libcuda.so.1
#2  0x000014cd956bb12f in ?? () from /lib64/libcuda.so.1
#3  0x000014cd956bda2f in ?? () from /lib64/libcuda.so.1
#4  0x000014cd9536f0df in ?? () from /lib64/libcuda.so.1
#5  0x000014cd9558ea8c in ?? () from /lib64/libcuda.so.1
#6  0x000014cd9558f351 in ?? () from /lib64/libcuda.so.1
#7  0x000014cd956b4940 in ?? () from /lib64/libcuda.so.1
#8  0x000014cd9532f833 in ?? () from /lib64/libcuda.so.1
#9  0x000014cd9532fd41 in ?? () from /lib64/libcuda.so.1
#10 0x000014cd95330ca8 in ?? () from /lib64/libcuda.so.1
#11 0x000014cd954fd9e1 in ?? () from /lib64/libcuda.so.1
#12 0x000014cdd8a4d4f9 in ?? () from /g/data/z00/yxs900/installation/cuda/11.3.0/lib64/libcudart.so.11.0
#13 0x000014cdd8a21f6d in ?? () from /g/data/z00/yxs900/installation/cuda/11.3.0/lib64/libcudart.so.11.0
#14 0x000014cdd8a6cf36 in cudaMemcpyAsync () from /g/data/z00/yxs900/installation/cuda/11.3.0/lib64/libcudart.so.11.0
#15 0x000014cdb53ae194 in at::native::_local_scalar_dense_cuda(at::Tensor const&)::{lambda()#1}::operator()() const [clone .isra.100] ()
   from /g/data/z00/yxs900/installation/pytorch/v1.12.1/lib/python3.9/site-packages/torch/lib/libtorch_cuda.so
#16 0x000014cdb53b00dc in at::native::_local_scalar_dense_cuda(at::Tensor const&) () from /g/data/z00/yxs900/installation/pytorch/v1.12.1/lib/python3.9/site-packages/torch/lib/libtorch_cuda.so
 #17 0x000014cdb681920d in at::(anonymous namespace)::(anonymous namespace)::wrapper___local_scalar_dense (self=...) at aten/src/ATen/RegisterCUDA.cpp:22556
#18 0x000014cdb681926c in c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<c10::Scalar(const at::Tensor&), at::(anonymous namespace)::(anonymous namespace)::wrapper___local_scalar_dense>, c10::Scalar, c10::guts::typelist::typelist<const at::Tensor&> >::operator() (args#0=..., this=<optimized out>) at ../aten/src/ATen/core/boxing/impl/WrapFunctionIntoFunctor.h:12
#19 c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<c10::Scalar(const at::Tensor&), at::(anonymous namespace)::(anonymous namespace)::wrapper___local_scalar_dense>, c10::Scalar, c10::guts::typelist::typelist<const at::Tensor&> >, c10::Scalar(const at::Tensor&)>::call(c10::OperatorKernel *, c10::DispatchKeySet, const at::Tensor &) (
    functor=<optimized out>, args#0=...) at ../aten/src/ATen/core/boxing/impl/make_boxed_from_unboxed_functor.h:453
#20 0x000014cdc0d6acc9 in c10::callUnboxedKernelFunction<c10::Scalar, at::Tensor const&> (dispatchKeySet=..., functor=<optimized out>, unboxed_kernel_func=<optimized out>)
    at ../aten/src/ATen/core/boxing/KernelFunction_impl.h:54
#21 c10::KernelFunction::call<c10::Scalar, at::Tensor const&> (dispatchKeySet=..., opHandle=..., this=<optimized out>) at ../aten/src/ATen/core/boxing/KernelFunction_impl.h:67
#22 c10::Dispatcher::redispatch<c10::Scalar, at::Tensor const&>(c10::TypedOperatorHandle<c10::Scalar (at::Tensor const&)> const&, c10::DispatchKeySet, at::Tensor const&) const (op=..., 
    this=<optimized out>, currentDispatchKeySet=...) at ../aten/src/ATen/core/dispatch/Dispatcher.h:545
#23 c10::TypedOperatorHandle<c10::Scalar (at::Tensor const&)>::redispatch(c10::DispatchKeySet, at::Tensor const&) const (args#0=..., currentDispatchKeySet=..., 
    this=0x14cdc7ef22b0 <at::_ops::_local_scalar_dense::redispatch(c10::DispatchKeySet, at::Tensor const&)::op>) at ../aten/src/ATen/core/dispatch/Dispatcher.h:419
#24 at::_ops::_local_scalar_dense::redispatch (dispatchKeySet=..., self=...) at aten/src/ATen/Operators_2.cpp:7573
#25 0x000014cdc1f3e470 in at::redispatch::_local_scalar_dense (self=..., dispatchKeySet=...) at aten/src/ATen/RedispatchFunctions.h:7757
#26 torch::autograd::VariableType::(anonymous namespace)::<lambda()>::operator() (__closure=<optimized out>, __closure=<optimized out>) at ../torch/csrc/autograd/generated/VariableType_2.cpp:1482
#27 torch::autograd::VariableType::(anonymous namespace)::_local_scalar_dense (ks=..., self=...) at ../torch/csrc/autograd/generated/VariableType_2.cpp:1483
#28 0x000014cdc1f3e82f in c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<c10::Scalar(c10::DispatchKeySet, const at::Tensor&), torch::autograd::VariableType::(anonymous namespace)::_local_scalar_dense>, c10::Scalar, c10::guts::typelist::typelist<c10::DispatchKeySet, const at::Tensor&> >::operator() (args#1=..., args#0=..., this=<optimized out>)
    at ../aten/src/ATen/core/boxing/impl/WrapFunctionIntoFunctor.h:12
#29 c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<c10::Scalar(c10::DispatchKeySet, const at::Tensor&), torch::autograd::VariableType::(anonymous namespace)::_local_scalar_dense>, c10::Scalar, c10::guts::typelist::typelist<c10::DispatchKeySet, const at::Tensor&> >, c10::Scalar(c10::DispatchKeySet, const at::Tensor&)>::call(c10::OperatorKernel *, c10::DispatchKeySet, const at::Tensor &) (functor=<optimized out>, dispatchKeySet=..., args#0=...) at ../aten/src/ATen/core/boxing/impl/make_boxed_from_unboxed_functor.h:470
#30 0x000014cdc0ded073 in c10::callUnboxedKernelFunction<c10::Scalar, at::Tensor const&> (dispatchKeySet=..., functor=<optimized out>, unboxed_kernel_func=<optimized out>)
    at ../aten/src/ATen/core/boxing/KernelFunction_impl.h:54
#31 c10::KernelFunction::call<c10::Scalar, at::Tensor const&> (dispatchKeySet=..., opHandle=..., this=0x1da1f28) at ../aten/src/ATen/core/boxing/KernelFunction_impl.h:67
#32 c10::Dispatcher::call<c10::Scalar, at::Tensor const&>(c10::TypedOperatorHandle<c10::Scalar (at::Tensor const&)> const&, at::Tensor const&) const (op=..., this=<optimized out>)
    at ../aten/src/ATen/core/dispatch/Dispatcher.h:536
#33 c10::TypedOperatorHandle<c10::Scalar (at::Tensor const&)>::call(at::Tensor const&) const (args#0=..., this=0x14cdc7ef22d0 <at::_ops::_local_scalar_dense::call(at::Tensor const&)::op>)
    at ../aten/src/ATen/core/dispatch/Dispatcher.h:414
#34 at::_ops::_local_scalar_dense::call (self=...) at aten/src/ATen/Operators_2.cpp:7566
#35 0x000014cdc07d9c40 in at::_local_scalar_dense (self=...) at aten/src/ATen/ops/_local_scalar_dense.h:27
#36 at::native::item (self=...) at ../aten/src/ATen/native/Scalar.cpp:17
#37 0x000014cdc1200dec in at::(anonymous namespace)::(anonymous namespace)::wrapper__item (self=...) at aten/src/ATen/RegisterCompositeImplicitAutograd.cpp:7062
#38 c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<c10::Scalar(const at::Tensor&), at::(anonymous namespace)::(anonymous namespace)::wrapper__item>, c10::Scalar, c10::guts::--Type <RET> for more, q to quit, c to continue without paging--  
typelist::typelist<const at::Tensor&> >::operator() (args#0=..., this=<optimized out>) at ../aten/src/ATen/core/boxing/impl/WrapFunctionIntoFunctor.h:13
#39 c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<c10::Scalar(const at::Tensor&), at::(anonymous namespace)::(anonymous namespace)::wrapper__item>, c10::Scalar, c10::guts::typelist::typelist<const at::Tensor&> >, c10::Scalar(const at::Tensor&)>::call(c10::OperatorKernel *, c10::DispatchKeySet, const at::Tensor &) (
    functor=<optimized out>, args#0=...) at ../aten/src/ATen/core/boxing/impl/make_boxed_from_unboxed_functor.h:453
#40 0x000014cdc0ce1453 in c10::callUnboxedKernelFunction<c10::Scalar, at::Tensor const&> (dispatchKeySet=..., functor=<optimized out>, unboxed_kernel_func=<optimized out>)
    at ../aten/src/ATen/core/boxing/KernelFunction_impl.h:54
#41 c10::KernelFunction::call<c10::Scalar, at::Tensor const&> (dispatchKeySet=..., opHandle=..., this=0x1ef8e88) at ../aten/src/ATen/core/boxing/KernelFunction_impl.h:67
#42 c10::Dispatcher::call<c10::Scalar, at::Tensor const&>(c10::TypedOperatorHandle<c10::Scalar (at::Tensor const&)> const&, at::Tensor const&) const (op=..., this=<optimized out>)
    at ../aten/src/ATen/core/dispatch/Dispatcher.h:536
#43 c10::TypedOperatorHandle<c10::Scalar (at::Tensor const&)>::call(at::Tensor const&) const (args#0=..., this=0x14cdc7eea170 <at::_ops::item::call(at::Tensor const&)::op>)
    at ../aten/src/ATen/core/dispatch/Dispatcher.h:414
#44 at::_ops::item::call (self=...) at aten/src/ATen/Operators_1.cpp:6382
#45 0x000014cdc1578d71 in at::Tensor::item (this=0x14cd800ab2d8) at aten/src/ATen/core/TensorBody.h:3927
#45 0x000014cdc1578d71 in at::Tensor::item (this=0x14cd800ab2d8) at aten/src/ATen/core/TensorBody.h:3927
#46 at::Tensor::item<double> (this=this@entry=0x14cd800ab2d8) at aten/src/ATen/core/TensorMethods.cpp:30
#47 0x000014cdc87201d8 in torch::autograd::dispatch_to_CDouble (self=...) at ../torch/csrc/autograd/generated/python_variable_methods.cpp:844
#48 0x000014cdc8720516 in torch::autograd::THPVariable_float_scalar (self=0x14cd800ab2c0, args=0x0) at ../torch/csrc/autograd/generated/python_variable_methods.cpp:881
#49 0x000014ce217d8658 in method_vectorcall_NOARGS (func=func@entry=0x14cdf8251ea0, args=args@entry=0x7ffeb4e28e38, nargsf=<optimized out>, kwnames=kwnames@entry=0x0) at ../Objects/descrobject.c:434
#50 0x000014ce21816315 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x7ffeb4e28e38, callable=0x14cdf8251ea0, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#51 vectorcall_unbound (nargs=<optimized out>, args=0x7ffeb4e28e38, func=0x14cdf8251ea0, unbound=1, tstate=0xe393f0) at ../Objects/typeobject.c:1505
#52 vectorcall_method (name=0x14ce21c08580 <id>, args=0x7ffeb4e28e38, nargs=<optimized out>) at ../Objects/typeobject.c:1537
#53 0x000014ce218a6bbf in slot_nb_float (self=<optimized out>) at ../Objects/typeobject.c:6596
#54 0x000014ce217c8e4b in PyNumber_Float (o=0x14cd800ab2c0) at ../Objects/abstract.c:1524
#55 0x000014ce218120d3 in type_call (type=type@entry=0x14ce21bfe9c0 <PyFloat_Type>, args=args@entry=0x14cd8e4490d0, kwds=kwds@entry=0x0) at ../Objects/typeobject.c:1014
#56 0x000014ce217d2a6d in _PyObject_MakeTpCall (tstate=0xe393f0, callable=0x14ce21bfe9c0 <PyFloat_Type>, args=0x14cd6f78c1d0, nargs=1, keywords=0x0) at ../Objects/call.c:191
#57 0x000014ce218450c7 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=9223372036854775809, args=0x14cd6f78c1d0, callable=0x14ce21bfe9c0 <PyFloat_Type>, tstate=<optimized out>)
    at ../Include/cpython/abstract.h:116
#58 _PyObject_VectorcallTstate (kwnames=0x0, nargsf=9223372036854775809, args=0x14cd6f78c1d0, callable=0x14ce21bfe9c0 <PyFloat_Type>, tstate=<optimized out>) at ../Include/cpython/abstract.h:103
#59 PyObject_Vectorcall (kwnames=0x0, nargsf=9223372036854775809, args=0x14cd6f78c1d0, callable=0x14ce21bfe9c0 <PyFloat_Type>) at ../Include/cpython/abstract.h:127
#60 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0xe393f0) at ../Python/ceval.c:5072
#61 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3518
#62 0x000014ce217d3282 in _PyEval_EvalFrame (throwflag=0, f=0x14cd6f78c040, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#63 function_code_fastcall (globals=<optimized out>, nargs=2, args=<optimized out>, co=<optimized out>, tstate=0xe393f0) at ../Objects/call.c:330
#64 _PyFunction_Vectorcall (func=<optimized out>, stack=0x14cd6f778920, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:367
#65 0x000014ce21844f2e in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x14cd6f778920, callable=0x14cd97e67040, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#66 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x14cd6f778920, callable=<optimized out>) at ../Include/cpython/abstract.h:127
#67 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0xe393f0) at ../Python/ceval.c:5072
#68 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3487
#69 0x000014ce217d3282 in _PyEval_EvalFrame (throwflag=0, f=0x14cd6f778780, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#70 function_code_fastcall (globals=<optimized out>, nargs=2, args=<optimized out>, co=<optimized out>, tstate=0xe393f0) at ../Objects/call.c:330
#71 _PyFunction_Vectorcall (func=<optimized out>, stack=0x14cd6f7f0b40, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:367
#72 0x000014ce21840e10 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x14cd6f7f0b40, callable=0x14cd97e61ee0, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#73 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x14cd6f7f0b40, callable=<optimized out>) at ../Include/cpython/abstract.h:127
#74 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0xe393f0) at ../Python/ceval.c:5072
#75 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3504
#76 0x000014ce2183fab9 in _PyEval_EvalFrame (throwflag=0, f=0x14cd6f7f09a0, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#77 _PyEval_EvalCode (tstate=tstate@entry=0xe393f0, _co=<optimized out>, globals=<optimized out>, locals=locals@entry=0x0, args=args@entry=0x14cd6f7f0958, argcount=argcount@entry=2, 
    kwnames=0x14cd97e4c4d8, kwargs=0x14cd6f7f0968, kwcount=<optimized out>, kwstep=1, defs=0x14cd97e4c598, defcount=1, kwdefs=0x0, closure=0x0, name=0x14cd97e4e730, qualname=0x14cd97e50440)
    at ../Python/ceval.c:4327
#78 0x000014ce217d331c in _PyFunction_Vectorcall (func=<optimized out>, stack=0x14cd6f7f0958, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:396
#79 0x000014ce217d4fb7 in _PyObject_VectorcallTstate (kwnames=<optimized out>, nargsf=<optimized out>, args=0x14cd6f7f0958, callable=0x14cd97e61f70, tstate=0xe393f0)
    at ../Include/cpython/abstract.h:118
#80 method_vectorcall (method=<optimized out>, args=0x14cd6f7f0960, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/classobject.c:53
#81 0x000014ce2184198b in _PyObject_VectorcallTstate (kwnames=0x14cd97e4c4c0, nargsf=<optimized out>, args=0x14cd6f7f0960, callable=0x14cd8e294980, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
--Type <RET> for more, q to quit, c to continue without paging--
#82 PyObject_Vectorcall (kwnames=<optimized out>, nargsf=<optimized out>, args=<optimized out>, callable=0x14cd8e294980) at ../Include/cpython/abstract.h:127
#83 call_function (kwnames=<optimized out>, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=<optimized out>) at ../Python/ceval.c:5072
#84 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3535
#85 0x000014ce2183fab9 in _PyEval_EvalFrame (throwflag=0, f=0x14cd6f7f07c0, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#86 _PyEval_EvalCode (tstate=tstate@entry=0xe393f0, _co=<optimized out>, globals=<optimized out>, locals=locals@entry=0x0, args=args@entry=0x8e3c530, argcount=argcount@entry=1, kwnames=0x0, 
    kwargs=0x8e3c538, kwcount=<optimized out>, kwstep=1, defs=0x14cd97e4c568, defcount=1, kwdefs=0x0, closure=0x0, name=0x14ce12381ef0, qualname=0x14cd97e50260) at ../Python/ceval.c:4327
#87 0x000014ce217d331c in _PyFunction_Vectorcall (func=<optimized out>, stack=0x8e3c530, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:396
#88 0x000014ce21840e10 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x8e3c530, callable=0x14cd97e61e50, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#89 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x8e3c530, callable=<optimized out>) at ../Include/cpython/abstract.h:127
#90 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0xe393f0) at ../Python/ceval.c:5072
#91 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3504
#92 0x000014ce2183fab9 in _PyEval_EvalFrame (throwflag=0, f=0x8e3c350, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#93 _PyEval_EvalCode (tstate=tstate@entry=0xe393f0, _co=<optimized out>, globals=<optimized out>, locals=locals@entry=0x0, args=args@entry=0x14cd6f7da9e8, argcount=argcount@entry=1, kwnames=0x0, 
    kwargs=0x14cd6f7da9f0, kwcount=<optimized out>, kwstep=1, defs=0x14cd96f4c598, defcount=1, kwdefs=0x0, closure=0x0, name=0x14ce21ddfd30, qualname=0x14cd96f4f490) at ../Python/ceval.c:4327
#94 0x000014ce217d331c in _PyFunction_Vectorcall (func=<optimized out>, stack=0x14cd6f7da9e8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:396
#95 0x000014ce21840e10 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x14cd6f7da9e8, callable=0x14cd96f4e820, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#96 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x14cd6f7da9e8, callable=<optimized out>) at ../Include/cpython/abstract.h:127
#97 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0xe393f0) at ../Python/ceval.c:5072
#98 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3504
#99 0x000014ce2183fab9 in _PyEval_EvalFrame (throwflag=0, f=0x14cd6f7da840, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#100 _PyEval_EvalCode (tstate=tstate@entry=0xe393f0, _co=<optimized out>, globals=<optimized out>, locals=locals@entry=0x0, args=args@entry=0x14cd6f7ea9a8, argcount=argcount@entry=2, kwnames=0x0, 
    kwargs=0x14cd6f7ea9b8, kwcount=<optimized out>, kwstep=1, defs=0x14cd96f218c8, defcount=1, kwdefs=0x0, closure=0x0, name=0x14cd96fd2f80, qualname=0x14cd96fd6390) at ../Python/ceval.c:4327
#101 0x000014ce217d331c in _PyFunction_Vectorcall (func=<optimized out>, stack=0x14cd6f7ea9a8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:396
#102 0x000014ce217d4fb7 in _PyObject_VectorcallTstate (kwnames=<optimized out>, nargsf=<optimized out>, args=0x14cd6f7ea9a8, callable=0x14cd96f42f70, tstate=0xe393f0)
#103 method_vectorcall (method=<optimized out>, args=0x14cd6f7ea9b0, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/classobject.c:53
#104 0x000014ce21844f2e in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x14cd6f7ea9b0, callable=0x14cd8e162cc0, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#105 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x14cd6f7ea9b0, callable=<optimized out>) at ../Include/cpython/abstract.h:127
#106 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0xe393f0) at ../Python/ceval.c:5072
#107 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3487
#108 0x000014ce2183fab9 in _PyEval_EvalFrame (throwflag=0, f=0x14cd6f7ea800, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#109 _PyEval_EvalCode (tstate=tstate@entry=0xe393f0, _co=<optimized out>, globals=<optimized out>, locals=locals@entry=0x0, args=args@entry=0x5888390, argcount=argcount@entry=1, kwnames=0x0, 
    kwargs=0x5888398, kwcount=<optimized out>, kwstep=1, defs=0x14cd96fe2658, defcount=1, kwdefs=0x0, closure=0x0, name=0x14ce21ddfd30, qualname=0x14cd96fd93a0) at ../Python/ceval.c:4327
#110 0x000014ce217d331c in _PyFunction_Vectorcall (func=<optimized out>, stack=0x5888390, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:396
#111 0x000014ce217d4fb7 in _PyObject_VectorcallTstate (kwnames=<optimized out>, nargsf=<optimized out>, args=0x5888390, callable=0x14cd96f43040, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#112 method_vectorcall (method=<optimized out>, args=0x5888398, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/classobject.c:53
#113 0x000014ce21844f2e in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x5888398, callable=0x14cd8e165b40, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#114 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x5888398, callable=<optimized out>) at ../Include/cpython/abstract.h:127
#115 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0xe393f0) at ../Python/ceval.c:5072
#116 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3487
#117 0x000014ce2183ffe7 in _PyEval_EvalFrame (throwflag=0, f=0x5888160, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#118 _PyEval_EvalCode (tstate=tstate@entry=0xe393f0, _co=<optimized out>, globals=<optimized out>, locals=locals@entry=0x0, args=args@entry=0x522c600, argcount=argcount@entry=7, kwnames=0x0, 
    kwargs=0x522c638, kwcount=<optimized out>, kwstep=1, defs=0x14ce133fcd48, defcount=1, kwdefs=0x0, closure=0x0, name=0x14ce13459f30, qualname=0x14ce13459f30) at ../Python/ceval.c:4327
#119 0x000014ce217d331c in _PyFunction_Vectorcall (func=<optimized out>, stack=0x522c600, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:396
#120 0x000014ce21840bb0 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x522c600, callable=0x14cd96ee4f70, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#121 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x522c600, callable=<optimized out>) at ../Include/cpython/abstract.h:127
#122 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0xe393f0) at ../Python/ceval.c:5072
#123 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3518
#124 0x000014ce217d3282 in _PyEval_EvalFrame (throwflag=0, f=0x522c410, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#125 function_code_fastcall (globals=<optimized out>, nargs=5, args=<optimized out>, co=<optimized out>, tstate=0xe393f0) at ../Objects/call.c:330
#126 _PyFunction_Vectorcall (func=<optimized out>, stack=0x14cd972159b0, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:367
#127 0x000014ce21840bb0 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x14cd972159b0, callable=0x14cd96eec5e0, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#128 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x14cd972159b0, callable=<optimized out>) at ../Include/cpython/abstract.h:127
--Type <RET> for more, q to quit, c to continue without paging--
#129 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0xe393f0) at ../Python/ceval.c:5072
#130 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3518
#131 0x000014ce217d3282 in _PyEval_EvalFrame (throwflag=0, f=0x14cd97215800, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#132 function_code_fastcall (globals=<optimized out>, nargs=0, args=<optimized out>, co=<optimized out>, tstate=0xe393f0) at ../Objects/call.c:330
#133 _PyFunction_Vectorcall (func=<optimized out>, stack=0xe96030, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:367
#134 0x000014ce21840bb0 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0xe96030, callable=0x14cd96eec8b0, tstate=0xe393f0) at ../Include/cpython/abstract.h:118
#135 PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0xe96030, callable=<optimized out>) at ../Include/cpython/abstract.h:127
#136 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0xe393f0) at ../Python/ceval.c:5072
#137 _PyEval_EvalFrameDefault (tstate=<optimized out>, f=<optimized out>, throwflag=<optimized out>) at ../Python/ceval.c:3518
#138 0x000014ce2183fab9 in _PyEval_EvalFrame (throwflag=0, f=0xe95ec0, tstate=0xe393f0) at ../Include/internal/pycore_ceval.h:40
#139 _PyEval_EvalCode (tstate=<optimized out>, _co=<optimized out>, globals=<optimized out>, locals=<optimized out>, args=<optimized out>, argcount=<optimized out>, kwnames=0x0, kwargs=0x0, 
    kwcount=<optimized out>, kwstep=2, defs=0x0, defcount=0, kwdefs=0x0, closure=0x0, name=0x0, qualname=0x0) at ../Python/ceval.c:4327
#140 0x000014ce2183f7cf in _PyEval_EvalCodeWithName (_co=<optimized out>, globals=<optimized out>, locals=<optimized out>, args=<optimized out>, argcount=<optimized out>, kwnames=<optimized out>, 
    kwargs=0x0, kwcount=0, kwstep=2, defs=0x0, defcount=0, kwdefs=0x0, closure=0x0, name=0x0, qualname=0x0) at ../Python/ceval.c:4359
#141 0x000014ce2183f779 in PyEval_EvalCodeEx (_co=<optimized out>, globals=<optimized out>, locals=<optimized out>, args=<optimized out>, argcount=<optimized out>, kws=<optimized out>, kwcount=0, 
    defs=0x0, defcount=0, kwdefs=0x0, closure=0x0) at ../Python/ceval.c:4375
#142 0x000014ce2183f73b in PyEval_EvalCode (co=co@entry=0x14ce134600e0, globals=globals@entry=0x14ce21d45dc0, locals=locals@entry=0x14ce21d45dc0) at ../Python/ceval.c:826
#143 0x000014ce218d29cd in run_eval_code_obj (tstate=0xe393f0, co=0x14ce134600e0, globals=0x14ce21d45dc0, locals=0x14ce21d45dc0) at ../Python/pythonrun.c:1218
#144 0x000014ce218d252a in run_mod (mod=<optimized out>, filename=<optimized out>, globals=0x14ce21d45dc0, locals=0x14ce21d45dc0, flags=<optimized out>, arena=<optimized out>)
    at ../Python/pythonrun.c:1239
#145 0x000014ce217748a3 in pyrun_file (fp=0xe29340, filename=0x14ce21cc7130, start=<optimized out>, globals=0x14ce21d45dc0, locals=0x14ce21d45dc0, closeit=1, flags=0x7ffeb4e2a8c8)
    at ../Python/pythonrun.c:1137
#146 0x000014ce218d2285 in pyrun_simple_file (flags=0x7ffeb4e2a8c8, closeit=1, filename=0x14ce21cc7130, fp=0xe29340) at ../Python/pythonrun.c:449
#147 PyRun_SimpleFileExFlags (fp=0xe29340, filename=<optimized out>, closeit=1, flags=0x7ffeb4e2a8c8) at ../Python/pythonrun.c:482
#148 0x000014ce218d82cb in pymain_run_file (cf=0x7ffeb4e2a8c8, config=0xe35710) at ../Modules/main.c:373
#149 pymain_run_python (exitcode=0x7ffeb4e2a8c0) at ../Modules/main.c:598
#150 Py_RunMain () at ../Modules/main.c:677
#151 0x000014ce218d7e49 in Py_BytesMain (argc=<optimized out>, argv=<optimized out>) at ../Modules/main.c:731
#152 0x000014ce2072ad85 in __libc_start_main () from /lib64/libc.so.6
#153 0x00000000004006ae in _start ()
"""
When I try to understand the problem, I cannot find "aten/src/ATen/RegisterCUDA.cpp:22556" in line 17.

Any suggestion about how to dig into the problem further is appreciated.

Cheers,
Yue

### Versions

Collecting environment information...
PyTorch version: 1.12.0a0
Is debug build: False
CUDA used to build PyTorch: 11.3
ROCM used to build PyTorch: N/A

OS: Rocky Linux release 8.7 (Green Obsidian) (x86_64)
GCC version: (GCC) 8.5.0 20210514 (Red Hat 8.5.0-15)
Clang version: 14.0.6 (Red Hat 14.0.6-1.module+el8.7.0+1080+d88dc670)
CMake version: version 3.20.2
Libc version: glibc-2.28

Python version: 3.9.2 (default, Mar 29 2021, 10:41:26)  [GCC 8.3.1 20191121 (Red Hat 8.3.1-5)] (64-bit runtime)
Python platform: Linux-4.18.0-425.3.1.el8.nci.x86_64-x86_64-with-glibc2.28
Is CUDA available: False
CUDA runtime version: 11.3.58
CUDA_MODULE_LOADING set to: N/A
GPU models and configuration: Could not collect
Nvidia driver version: Could not collect
cuDNN version: Could not collect
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

cc @ngimel