Repository: pytorch
Issue: Segmentation fault when loss.backward()
Labels: module: crash,module: cuda,triaged

Hi all, I try to train a model on AWS, the model runs normally for the first a few iterations and crashed with a Segmentation fault. It is weird as the model runs successfully on Colab, I ran `gdb where` and this is the output:

`Thread 74 "python3" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7ffee6ffd700 (LWP 30327)]
0x00007fff9371d48f in at::cuda::getCurrentCUDABlasHandle() () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
(gdb) where
#0  0x00007fff9371d48f in at::cuda::getCurrentCUDABlasHandle() () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
#1  0x00007fff9370b7dd in void at::cuda::blas::gemm<float>(char, char, long, long, long, float, float const*, long, float const*, long, float, float*, long) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
#2  0x00007fff949278f0 in at::native::(anonymous namespace)::addmm_out_cuda_impl(at::Tensor&, at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::Scalar, c10::Scalar) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
#3  0x00007fff94929092 in at::native::mm_cuda(at::Tensor const&, at::Tensor const&) () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
#4  0x00007fff93762848 in at::(anonymous namespace)::(anonymous namespace)::wrapper_mm(at::Tensor const&, at::Tensor const&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
#5  0x00007fff9376288f in c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<at::Tensor (at::Tensor const&, at::Tensor const&), &at::(anonymous namespace)::(anonymous namespace)::wrapper_mm>, at::Tensor, c10::guts::typelist::typelist<at::Tensor const&, at::Tensor const&> >, at::Tensor (at::Tensor const&, at::Tensor const&)>::call(c10::OperatorKernel*, at::Tensor const&, at::Tensor const&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
#6  0x00007fff81be01b6 in at::Tensor c10::Dispatcher::call<at::Tensor, at::Tensor const&, at::Tensor const&>(c10::TypedOperatorHandle<at::Tensor (at::Tensor const&, at::Tensor const&)> const&, at::Tensor const&, at::Tensor const&) const [clone .constprop.4808] () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#7  0x00007fff81be57df in at::mm(at::Tensor const&, at::Tensor const&) () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#8  0x00007fff834c950c in torch::autograd::VariableType::(anonymous namespace)::mm(at::Tensor const&, at::Tensor const&) () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#9  0x00007fff834c9a7f in c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<at::Tensor (at::Tensor const&, at::Tensor const&), &torch::autograd::VariableType::(anonymous namespace)::mm>, at::Tensor, c10::guts::typelist::typelist<at::Tensor const&, at::Tensor const&> >, at::Tensor (at::Tensor const&, at::Tensor const&)>::call(c10::OperatorKernel*, at::Tensor const&, at::Tensor const&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#10 0x00007fff820323c6 in at::Tensor c10::Dispatcher::call<at::Tensor, at::Tensor const&, at::Tensor const&>(c10::TypedOperatorHandle<at::Tensor (at::Tensor const&, at::Tensor const&)> const&, at::Tensor const&, at::Tensor const&) const [clone .constprop.1487] () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#11 0x00007fff820365ef in at::Tensor::mm(at::Tensor const&) const () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#12 0x00007fff83f08501 in torch::autograd::generated::details::mm_mat1_backward(at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::Scalar const&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#13 0x00007fff832ddc83 in torch::autograd::generated::AddmmBackward::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#14 0x00007fff8399b361 in torch::autograd::Node::operator()(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#15 0x00007fff8399716b in torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&, std::shared_ptr<torch::autograd::ReadyQueue> const&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#16 0x00007fff83997d8f in torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&) () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#17 0x00007fff8398f569 in torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#18 0x00007ffff41170c3 in torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_python.so
#19 0x00007ffff533c6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#20 0x00007ffff7bbb6db in start_thread (arg=0x7ffee6ffd700) at pthread_create.c:463
#21 0x00007ffff6f3771f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95`

### Config
Ubuntu 18.04.5 LTS
Python 3.7.10 | packaged by conda-forge | (default, Feb 19 2021, 16:07:37) \n[GCC 9.3.0]
GPU: four Tesla M60

Thank you for any help.



cc @ezyang @gchanan @zou3519 @bdhirsh @jbschlosser @anjali411 @ngimel