NCCL/Gloo: freeze with cyclic isend and recv

Fatvvs: <p>Hello!</p>
<p>I’ve made a very simple algorithm in two versions: on Gloo backed and on NCCL backend. And the both versions freeze. Could you please explain me why these programs hang?</p>
<p>I assume the cause is that I send data in circle:</p>
<ol>
<li>process <span class="hashtag">#1</span> sends data to process <span class="hashtag">#2</span>
</li>
<li>process <span class="hashtag">#2</span> sends data to process <span class="hashtag">#3</span>
</li>
<li>process <span class="hashtag">#3</span> sends data to process <span class="hashtag">#1</span>
</li>
</ol>
<p>But I don’t understand why it can lead to a freeze.</p>
<pre><code class="lang-auto">import os
import torch as th
import torch.distributed as dist
import torch.multiprocessing as mp
 
 
def run(rank: int, value: float, src:int, dst: int):
    tensor = th.FloatTensor([value,]).to(f"cuda:{rank}")
    print(f"[rk={rank}] tensor before send-recv: {tensor}")
    req = dist.isend(tensor=tensor, dst=dst)
    print(f"[rk={rank}] after isend")
    dist.recv(tensor=tensor, src=src)
    print(f"[rk={rank}] after recv")
    req.wait()
    print(f"[rk={rank}] after wait")
    print(f"[rk={rank}] tensor after send-recv: {tensor}")
   
def init_process(rank: int):
    dist.init_process_group(
        "nccl",
        rank=rank,
        world_size=3,
        init_method="file:///home/user/store"
    )
    if rank==0:
        run(rank=rank, value=float(rank), src=1, dst=1)
    elif rank==1:
        run(rank, value=float(rank), src=2, dst=2)
    elif rank==2:
        run(rank, value=float(rank), src=0, dst=0)
    else:
        raise Exception()
 
 
if __name__ == "__main__":
    mp.set_start_method("spawn")
    processes = []
    for rank in [0,1, 2]:
        p = mp.Process(target=init_process, args=(rank, ))
        p.start()
        processes.append(p)
    for p in processes:
        p.join()
</code></pre>
<p>And here’s a Gloo version (and without CUDA):</p>
<pre><code class="lang-auto">import os
import torch as th
import torch.distributed as dist
import torch.multiprocessing as mp
 
 
def run(rank: int, value: float, src:int, dst: int):
    tensor = th.FloatTensor([value,])
    print(f"[rk={rank}] tensor before send-recv: {tensor}")
    req = dist.isend(tensor=tensor, dst=dst)
    print(f"[rk={rank}] after isend")
    dist.recv(tensor=tensor, src=src)
    print(f"[rk={rank}] after recv")
    req.wait()
    print(f"[rk={rank}] after wait")
    print(f"[rk={rank}] tensor after send-recv: {tensor}")
   
def init_process(rank: int):
    dist.init_process_group(
        "gloo",
        rank=rank,
        world_size=3,
        init_method="file:///home/user/store"
    )
    if rank==0:
        run(rank=rank, value=float(rank), src=1, dst=1)
    elif rank==1:
        run(rank, value=float(rank), src=2, dst=2)
    elif rank==2:
        run(rank, value=float(rank), src=0, dst=0)
    else:
        raise Exception()
 
 
if __name__ == "__main__":
    mp.set_start_method("spawn")
    processes = []
    for rank in [0,1, 2]:
        p = mp.Process(target=init_process, args=(rank, ))
        p.start()
        processes.append(p)
    for p in processes:
        p.join()

</code></pre>
<p>Thank you!</p>

wanchaol: <p>Hi, looking at your code seems like your <code>src</code> and <code>dst</code> passed to the <code>run</code> function are wrong.</p>