Repository: pytorch
Issue: Internal assert failed at rref_context.cpp
Labels: oncall: distributed,module: crash

### 🐛 Describe the bug

I'm reporting as written on the compile log output to do so:

```
[E request_callback_no_python.cpp:559] Received error while processing request type 260: falseINTERNAL ASSERT FAILED at "/dev/shm/strube1/juwelsbooster/PyTorch/1.11/GCCcore-11.2.0-CUDA-11.5/pytorch/torch/csrc/distributed/rpc/rref_context.cpp":385, please report a bug to PyTorch. Expected OwnerRRef with id GloballyUniqueId(created_on=0, local_id=0) to be created.
Exception raised from getOwnerRRef at /dev/shm/strube1/juwelsbooster/PyTorch/1.11/GCCcore-11.2.0-CUDA-11.5/pytorch/torch/csrc/distributed/rpc/rref_context.cpp:385 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x95 (0x153db62491a5 in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libc10.so)
frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0xe4 (0x153db6226fb1 in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libc10.so)
frame #2: c10::detail::torchInternalAssertFail(char const*, char const*, unsigned int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0x3f (0x153db62471df in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libc10.so)
frame #3: torch::distributed::rpc::RRefContext::getOwnerRRef(torch::distributed::rpc::GloballyUniqueId const&, bool) + 0x397 (0x153dc1316d47 in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)
frame #4: torch::distributed::rpc::RequestCallbackNoPython::assignOwnerRRef(torch::distributed::rpc::GloballyUniqueId const&, torch::distributed::rpc::GloballyUniqueId const&, c10::intrusive_ptr<c10::ivalue::Future, c10::detail::intrusive_target_default_null_type<c10::ivalue::Future> >) const + 0x62 (0x153dc1306e02 in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)
frame #5: torch::distributed::rpc::RequestCallbackImpl::processScriptRemoteCall(torch::distributed::rpc::RpcCommandBase&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const + 0xfc (0x153dc6d14ddc in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libtorch_python.so)
frame #6: torch::distributed::rpc::RequestCallbackNoPython::processRpc(torch::distributed::rpc::RpcCommandBase&, torch::distributed::rpc::MessageType const&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const + 0x119 (0x153dc130b1f9 in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)
frame #7: torch::distributed::rpc::RequestCallbackImpl::processRpcWithErrors(torch::distributed::rpc::RpcCommandBase&, torch::distributed::rpc::MessageType const&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const + 0x49 (0x153dc6d16e89 in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libtorch_python.so)
frame #8: <unknown function> + 0x343eeb3 (0x153dc1303eb3 in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)
frame #9: torch::distributed::rpc::RequestCallbackNoPython::processMessage(torch::distributed::rpc::Message&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const + 0xa4a (0x153dc130915a in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)
frame #10: torch::distributed::rpc::RequestCallback::operator()(torch::distributed::rpc::Message&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const + 0x3a (0x153dc130180a in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)
frame #11: <unknown function> + 0x3469731 (0x153dc132e731 in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so)
frame #12: c10::ThreadPool::main_loop(unsigned long) + 0x2a5 (0x153db623a305 in /tmp/eb-m4ioh8sj/tmp2968_rli/lib/python3.9/site-packages/torch/lib/libc10.so)
frame #13: <unknown function> + 0xee874 (0x153dc8a8b874 in /p/software/juwelsbooster/stages/2022/software/GCCcore/11.2.0/lib64/libstdc++.so.6)
frame #14: <unknown function> + 0x817a (0x153df5ca117a in /usr/lib64/libpthread.so.0)
frame #15: clone + 0x43 (0x153df504fdc3 in /usr/lib64/libc.so.6)
```

### Versions

tag v1.11.0-rc

This is the JUWELS BOOSTER supercomputer.

Rocky Linux 8.5 and these are the versions of the softwares:

```python

builddependencies = [
    ('CMake', '3.21.1'),
    ('hypothesis', '6.14.6'),
]

dependencies = [
    ('CUDA', '11.5', '', True),
    ('Ninja', '1.10.2'),  # Required for JIT compilation of C++ extensions
    ('Python', '3.9.6'),
    ('protobuf', '3.17.3'),
    ('protobuf-python', '3.17.3'),
    ('pybind11', '2.7.1'),
    ('SciPy-bundle', '2021.10'),
    ('typing-extensions', '3.10.0.0'),
    ('PyYAML', '5.4.1'),
    ('MPFR', '4.1.0'),
    ('GMP', '6.2.1'),
    ('numactl', '2.0.14', '', SYSTEM),
    ('FFmpeg', '4.4.1'),
    ('Pillow-SIMD', '8.3.1'),
    ('cuDNN', '8.3.1.22', '-CUDA-%(cudaver)s', True),
    ('magma', '2.6.1', '-CUDA-%(cudaver)s'),
    ('NCCL', '2.11.4', '-CUDA-%(cudaver)s'),
    ('expecttest', '0.1.3'),
]
```

cc @pietern @mrshenli @pritamdamania87 @zhaojuanmao @satgera @rohan-varma @gqchen @aazzolini @osalpekar @jiayisuse @SciPioneer @H-Huang