Repository: pytorch
Issue: test_backward_accumulate_grads (__main__.TensorPipeDistAutogradTest) is flaky
Labels: oncall: distributed,triaged,module: flaky-tests,module: rpc

https://app.circleci.com/pipelines/github/pytorch/pytorch/371361/workflows/a14cd549-a37b-47ce-ad6a-a8baa3d40a54/jobs/15666994/steps

```
Aug 26 22:20:15   test_backward_accumulate_grads (__main__.TensorPipeDistAutogradTest) ... terminate called after throwing an instance of 'std::runtime_error'
Aug 26 22:20:15   what():  In listenFromLoop at tensorpipe/transport/uv/uv.h:176 "rv < 0: address already in use"
Aug 26 22:20:15 SIGABRT(6), PID: 17407, Thread 17407: 
Aug 26 22:20:15  frame #0: c10::FatalSignalHandler::stacktraceSignalHandler(bool) + 0x59 (0x7f945a0d7569 in /opt/conda/lib/python3.6/site-packages/torch/lib/libc10.so)
Aug 26 22:20:15 frame #1: <unknown function> + 0x11390 (0x7f946f4c0390 in /lib/x86_64-linux-gnu/libpthread.so.0)
Aug 26 22:20:15 frame #2: syscall + 0x19 (0x7f946f1e65d9 in /lib/x86_64-linux-gnu/libc.so.6)
Aug 26 22:20:15 frame #3: std::__atomic_futex_unsigned_base::_M_futex_wait_until(unsigned int*, unsigned int, bool, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000000000l> >) + 0x29 (0x7f9459fef9e5 in /opt/conda/lib/libstdc++.so.6)
Aug 26 22:20:15 frame #4: tensorpipe::transport::ListenerBoilerplate<tensorpipe::transport::uv::ContextImpl, tensorpipe::transport::uv::ListenerImpl, tensorpipe::transport::uv::ConnectionImpl>::addr() const + 0x25a (0x7f945f8c9fda in /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
Aug 26 22:20:15 frame #5: tensorpipe::channel::mpt::ContextImpl::ContextImpl(std::vector<std::shared_ptr<tensorpipe::transport::Context>, std::allocator<std::shared_ptr<tensorpipe::transport::Context> > >, std::vector<std::shared_ptr<tensorpipe::transport::Listener>, std::allocator<std::shared_ptr<tensorpipe::transport::Listener> > >, std::unordered_map<tensorpipe::Device, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<tensorpipe::Device>, std::equal_to<tensorpipe::Device>, std::allocator<std::pair<tensorpipe::Device const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >) + 0x4f5 (0x7f945f91f5d5 in /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
Aug 26 22:20:15 frame #6: tensorpipe::channel::mpt::ContextImpl::create(std::vector<std::shared_ptr<tensorpipe::transport::Context>, std::allocator<std::shared_ptr<tensorpipe::transport::Context> > >, std::vector<std::shared_ptr<tensorpipe::transport::Listener>, std::allocator<std::shared_ptr<tensorpipe::transport::Listener> > >) + 0x503 (0x7f945f920913 in /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
Aug 26 22:20:15 frame #7: tensorpipe::channel::mpt::create(std::vector<std::shared_ptr<tensorpipe::transport::Context>, std::allocator<std::shared_ptr<tensorpipe::transport::Context> > >, std::vector<std::shared_ptr<tensorpipe::transport::Listener>, std::allocator<std::shared_ptr<tensorpipe::transport::Listener> > >) + 0xf5 (0x7f945f8bbae5 in /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
Aug 26 22:20:15 frame #8: <unknown function> + 0x4126894 (0x7f945e430894 in /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
Aug 26 22:20:15 frame #9: std::_Function_handler<std::unique_ptr<torch::distributed::rpc::ChannelRegistration, std::default_delete<torch::distributed::rpc::ChannelRegistration> > (), std::unique_ptr<torch::distributed::rpc::ChannelRegistration, std::default_delete<torch::distributed::rpc::ChannelRegistration> > (*)()>::_M_invoke(std::_Any_data const&) + 0x1a (0x7f945e43e25a in /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
Aug 26 22:20:15 frame #10: torch::distributed::rpc::TensorPipeAgent::startImpl() + 0xab1 (0x7f945e437b51 in /opt/conda/lib/python3.6/site-packages/torch/lib/libtorch
```

cc @pietern @mrshenli @pritamdamania87 @zhaojuanmao @satgera @rohan-varma @gqchen @aazzolini @osalpekar @jiayisuse @agolynski @SciPioneer @H-Huang @mrzzd @cbalioglu @gcramer23 @jjlilley