Repository: pytorch
Issue: remote failure INTERNAL ASSERT FAILED at "../torch/csrc/distributed/rpc/rref_context.cpp":389
Labels: oncall: distributed,triaged,module: rpc

### 🐛 Describe the bug

When running the following code, it crashes. The issue is due to the usage of a timeout in `rref.remote(timeout=0.1).slow_copy()`. If you remove it, it works fine.

Test code: 
```
import torch
import torch.distributed.rpc as rpc

import multiprocessing as mp
import os
import socket
from contextlib import closing
import sys
import threading
import time

SLOWNESS = 1

class MyObject:
    def slow_copy(self):
        time.sleep(SLOWNESS)
        return MyObject()

def find_free_port():
    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
        s.bind(('', 0))
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        return s.getsockname()[1]

def common_setup(idx, port):
    os.environ["MASTER_ADDR"] = "localhost"
    os.environ["MASTER_PORT"] = str(port)
    rpc.init_rpc(f'worker{idx}', rank=idx, world_size=2)

def slow_create():
    time.sleep(SLOWNESS)
    return MyObject()

def worker0(port):
    common_setup(0, port)

    try:
        rref = rpc.remote("worker0", slow_create)
        rref2 = rref.remote(timeout=0.1).slow_copy()
        rref2.to_here()
    finally:
        rpc.shutdown()


def worker1(port):
    common_setup(1, port)
    time.sleep(10)
    rpc.shutdown()

if __name__ == "__main__":
    port = find_free_port()
    w0 = mp.Process(target=worker0, args=(port,))
    w0.start()

    w1 = mp.Process(target=worker1, args=(port,))
    w1.start()

    w0.join()
    w1.join()
```

Output:
```
[E request_callback_no_python.cpp:559] Received error while processing request type 261: falseINTERNAL ASSERT FAILED at "../torch/csrc/distributed/rpc/rref_context.cpp":387, please report a bug to PyTorch. Expected OwnerRRef with id GloballyUniqueId(created_on=0, local_id=2) to be created.
Exception raised from getOwnerRRef at ../torch/csrc/distributed/rpc/rref_context.cpp:387 (most recent call first):
frame #0: <unknown function> + 0x6c5a0 (0x7f2ac16045a0 in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #1: std::function<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > ()>::operator()() const + 0x48 (0x7f2ac6712ab8 in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #2: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x65 (0x7f2ac1604815 in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #3: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0x7f (0x7f2ac1601adf in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #4: c10::detail::torchInternalAssertFail(char const*, char const*, unsigned int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) + 0x5d (0x7f2ac16022dd in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #5: torch::distributed::rpc::RRefContext::getOwnerRRef(torch::distributed::rpc::GloballyUniqueId const&, bool) + 0x119 (0x7f2ac736ee39 in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #6: torch::distributed::rpc::RequestCallbackNoPython::assignOwnerRRef(torch::distributed::rpc::GloballyUniqueId const&, torch::distributed::rpc::GloballyUniqueId const&, c10::intrusive_ptr<c10::ivalue::Future, c10::detail::intrusive_target_default_null_type<c10::ivalue::Future> >) const + 0x8e (0x7f2ac734ea8e in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #7: torch::distributed::rpc::RequestCallbackImpl::processPythonRemoteCall(torch::distributed::rpc::RpcCommandBase&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const + 0x111 (0x7f2ad3f99f51 in /home/kumpera/pytorch/torch/lib/libtorch_python.so)
frame #8: torch::distributed::rpc::RequestCallbackNoPython::processRpc(torch::distributed::rpc::RpcCommandBase&, torch::distributed::rpc::MessageType const&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const + 0x23c (0x7f2ac734da4c in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #9: torch::distributed::rpc::RequestCallbackImpl::processRpcWithErrors(torch::distributed::rpc::RpcCommandBase&, torch::distributed::rpc::MessageType const&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const + 0x9e (0x7f2ad3f9a21e in /home/kumpera/pytorch/torch/lib/libtorch_python.so)
frame #10: <unknown function> + 0x5adc349 (0x7f2ac7352349 in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #11: <unknown function> + 0x5adc079 (0x7f2ac7352079 in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #12: <unknown function> + 0x5adbe41 (0x7f2ac7351e41 in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #13: <unknown function> + 0x5adbb55 (0x7f2ac7351b55 in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #14: <unknown function> + 0x5adb8d0 (0x7f2ac73518d0 in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #15: torch::distributed::rpc::RequestCallbackNoPython::processMessage(torch::distributed::rpc::Message&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const + 0x1cb (0x7f2ac735146b in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #16: torch::distributed::rpc::RequestCallback::operator()(torch::distributed::rpc::Message&, std::vector<c10::Stream, std::allocator<c10::Stream> >) const + 0x69 (0x7f2ac734d3b9 in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #17: <unknown function> + 0x5b2dd04 (0x7f2ac73a3d04 in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #18: <unknown function> + 0x5b2d94d (0x7f2ac73a394d in /home/kumpera/pytorch/torch/lib/libtorch_cpu.so)
frame #19: std::function<void ()>::operator()() const + 0x35 (0x7f2ad3a51015 in /home/kumpera/pytorch/torch/lib/libtorch_python.so)
frame #20: c10::ThreadPool::main_loop(unsigned long) + 0x396 (0x7f2ac15e8846 in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #21: <unknown function> + 0x5113f (0x7f2ac15e913f in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #22: <unknown function> + 0x510dd (0x7f2ac15e90dd in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #23: <unknown function> + 0x5106d (0x7f2ac15e906d in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #24: <unknown function> + 0x51045 (0x7f2ac15e9045 in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #25: <unknown function> + 0x51015 (0x7f2ac15e9015 in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #26: <unknown function> + 0x50ec9 (0x7f2ac15e8ec9 in /home/kumpera/pytorch/torch/lib/libc10.so)
frame #27: <unknown function> + 0xc9039 (0x7f2ac14ec039 in /home/kumpera/miniconda3/envs/pytorch/lib/libstdc++.so.6)
frame #28: <unknown function> + 0x817a (0x7f2aeb59717a in /lib64/libpthread.so.0)
frame #29: clone + 0x43 (0x7f2aeb2c6dc3 in /lib64/libc.so.6)
```

### Versions

master bc026c05775a6504e48363941285ae381e52b931

cc @pietern @mrshenli @pritamdamania87 @zhaojuanmao @satgera @rohan-varma @gqchen @aazzolini @osalpekar @jiayisuse @SciPioneer @H-Huang @jjlilley @mrzzd