Error in using DistributedDataParallel

username: 887574002
cooked: <p>Hi, I am trying to use DistributedDataParallel for a toy example . But I got the following error, when I passed the arguments. Here I want to have <code>4</code> nodes and <code>1</code> gpu per node.</p>
<pre><code class="lang-auto">class ConvNet(nn.Module):
    def __init__(self, num_classes=10):
        
        super(ConvNet, self).__init__()
        
        self.layer1 = nn.Sequential(
            nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(16),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))
        
        self.layer2 = nn.Sequential(
            nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))
        
        self.fc = nn.Linear(7*7*32, num_classes)

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = out.reshape(out.size(0), -1)
        out = self.fc(out)
        return(out)

if __name__=='__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-n', '--nodes', default=1,
                        type=int, metavar='N')
    parser.add_argument('-g', '--gpus', default=1, type=int,
                        help='number of gpus per node')
    parser.add_argument('-gi', '--gpui', default=4, type=int,
                        help='the index of gpu')
    
    parser.add_argument('-nr', '--nr', default=0, type=int,
                        help='ranking within the nodes')
    parser.add_argument('--epochs', default=2, type=int, 
                        metavar='N',
                        help='number of total epochs to run')
    args = parser.parse_args()
    #########################################################
    args.world_size = args.gpus * args.nodes                #  it is equal to the total number of gpus, because we use each gpu per node 
    os.environ['MASTER_ADDR'] = '10.57.23.164'              #  it tells which IP address it should look for process 0
    os.environ['MASTER_PORT'] = '8888'                      #
    mp.spawn(train,args=(args,),nprocs=args.gpus)         #
    #########################################################


def train(i,args):
    
    ########################################
    rank =args.nr * args.gpus + args.gpui
    dist.init_process_group(backed = 'nccl',
                           init_method = 'env//',
                           world_size= args.world_size,
                           rank=rank)
    ###########################################
    
    torch.manual_seed(0)
    
    model = ConvNet()
    
    torch.cuda.set_device(args.gpui)
    
    model.to(args.gpui)
    
    batch_size = 100
    
    # define loss function (criterion) and optimizer
    criterion = nn.CrossEntropyLoss().to(args.gpui)
    optimizer = torch.optim.SGD(model.parameters(), 1e-4)
    #################################################
    
    # wrap the model
    model = nn.parallel.DistributedDataParallel(model,device_ids = [args.gpui])
    
    #####################################################################
    
    # Data loading code
    train_dataset = torchvision.datasets.MNIST(root='./data',
                                               train=True,
                                               transform=transforms.ToTensor(),
                                               download=True)
    
    ##################################################
    train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset,
                                                                    num_replicas=args.word_size,
                                                                    rank = rank)
    
    ################################################ 
    
    train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                               batch_size=batch_size,
                                               shuffle=False,
                                               num_workers=0,
                                               pin_memory=True,
                                               sampler = train_sampler)
    ###################################################

    start = datetime.now()
    
    total_step = len(train_loader)
    
    for epoch in range(args.epochs):
        
        for i, (images, labels) in enumerate(train_loader):
            images = images.to(non_blocking=True)
            labels = labels.to(non_blocking=True)
            # Forward pass
            outputs = model(images)
            loss = criterion(outputs, labels)

            # Backward and optimize
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            
            if (i + 1) % 100 == 0:
                print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(
                    epoch + 1, 
                    args.epochs, 
                    i + 1, 
                    total_step,
                    loss.item())
                   )
   
   print("Training complete in: " + str(datetime.now() - start))
</code></pre>
<p>I passed the following arguments and I got the following error:</p>
<pre><code class="lang-auto">%run mnist_dit.py -n 4 -g 1 -gi 0 -nr 0 

error:

Exception                                 Traceback (most recent call last)
/mnt/home/pytorch_projects/mnist_dit.py in &lt;module&gt;
    139     os.environ['MATSER_PORT']='8890'
    140 
--&gt; 141     mp.spawn(train, nprocs=args.gpus, args=(args,))

~/anaconda3/lib/python3.7/site-packages/torch/multiprocessing/spawn.py in spawn(fn, args, nprocs, join, daemon, start_method)
    198                ' torch.multiprocessing.start_process(...)' % start_method)
    199         warnings.warn(msg)
--&gt; 200     return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')

~/anaconda3/lib/python3.7/site-packages/torch/multiprocessing/spawn.py in start_processes(fn, args, nprocs, join, daemon, start_method)
    156 
    157     # Loop on join until it returns True or raises an exception.
--&gt; 158     while not context.join():
    159         pass
    160 

~/anaconda3/lib/python3.7/site-packages/torch/multiprocessing/spawn.py in join(self, timeout)
    117         msg = "\n\n-- Process %d terminated with the following error:\n" % error_index
    118         msg += original_trace
--&gt; 119         raise Exception(msg)
    120 
    121 

Exception: 

-- Process 0 terminated with the following error:
Traceback (most recent call last):
  File "/home/anaconda3/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 20, in _wrap
    fn(i, *args)
  File "/mnt/home/pytorch_projects/mnist_dit.py", line 44, in train
    dist.init_process_group(backend='nccl',init_method='env//',world_size= args.word_size,rank = rank)
  File "/home/anaconda3/lib/python3.7/site-packages/torch/distributed/distributed_c10d.py", line 391, in init_process_group
    init_method, rank, world_size, timeout=timeout
  File "/home/anaconda3/lib/python3.7/site-packages/torch/distributed/rendezvous.py", line 79, in rendezvous
    raise RuntimeError("No rendezvous handler for {}://".format(result.scheme))
RuntimeError: No rendezvous handler for ://</code></pre>

username: mrshenli
cooked: <p>It’s <code>env://</code> instead of <code>env//</code>. But you should be able to omit <code>init_method</code> argument as you have already set <code>MASTER_ADDR</code> and <code>MASTER_PORT</code>.</p>
<p>BTW, could you please add a “distributed” tag to distributed training related questions? so that people working on the distributed package can get back to you promptly.</p>

username: 887574002
cooked: <p>Thank you for your reply. Yes, I would do that. sorry, I didn’t know that. Just, I have a question regrading <code>MASTER_ADDR</code> and <code>MASTER_PORT</code>, how should I set them while I am working on a remote server?<br>
Here I just used the one in the tutorial.</p>

username: mrshenli
cooked: <aside class="quote no-group" data-username="887574002" data-post="3" data-topic="90798">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/887574002/48/35835_2.png" class="avatar"> 887574002:</div>
<blockquote>
<p>how should I set them while I am working on a remote server?</p>
</blockquote>
</aside>
<p>The IP address should be reachable for all processes in the group. Usually you can get that IP by <code>ifconfig</code> and then pick the NIC you would like to use.</p>

username: 887574002
cooked: <p>Ok, thank you for your answer.</p>