Using torch.distributed & torch.multiprocessing for multiple gpu environment

creatives07: <p>I want to configure the Multiple gpu environment using ‘torch.multiprocessing’ and ‘torch.distributed’. However, I received the following error message.</p>
<pre><code>  RuntimeError: Default process group has not been initialized, please make sure to call init_process_group.
</code></pre>
<p>I think I did ‘init_process_group’. My code is as follows.</p>
<pre><code>  #opt
  ...

  def run(rank, size):
     torch.manual_seed(1234)

  device = torch.device("cuda:{}".format(rank))


  netG_A2B = Generator(opt.input_nc + opt.mask_nc, opt.output_nc).to(device)
  netG_B2A = Generator(opt.output_nc + opt.mask_nc, opt.input_nc).to(device)
  netD_A = Discriminator(opt.input_nc).to(device)
  netD_B = Discriminator(opt.output_nc).to(device)

  netG_A2B.apply(weights_init_normal)
  netG_B2A.apply(weights_init_normal)
  netD_A.apply(weights_init_normal)
  netD_B.apply(weights_init_normal)


  # Lossess
  ...

  # Optimizers &amp; LR schedulers
  optimizer_G = torch.optim.Adam(itertools.chain(netG_A2B.parameters(), netG_B2A.parameters()),
                            lr=opt.lr, betas=(0.5, 0.999))
  optimizer_D_A = torch.optim.Adam(netD_A.parameters(), lr=opt.lr, betas=(0.5, 0.999))
  optimizer_D_B = torch.optim.Adam(netD_B.parameters(), lr=opt.lr, betas=(0.5, 0.999))

  lr_scheduler_G = torch.optim.lr_scheduler.LambdaLR(optimizer_G, lr_lambda=LambdaLR(opt.n_epochs, opt.epoch, opt.decay_epoch).step)
  lr_scheduler_D_A = torch.optim.lr_scheduler.LambdaLR(optimizer_D_A, lr_lambda=LambdaLR(opt.n_epochs, opt.epoch, opt.decay_epoch).step)
  lr_scheduler_D_B = torch.optim.lr_scheduler.LambdaLR(optimizer_D_B, lr_lambda=LambdaLR(opt.n_epochs, opt.epoch, opt.decay_epoch).step)

  Tensor = torch.cuda.FloatTensor if opt.cuda else torch.Tensor
  input_A = Tensor(opt.batchSize, opt.input_nc, opt.size, opt.size)
  input_B = Tensor(opt.batchSize, opt.output_nc, opt.size, opt.size)
  input_M = Tensor(opt.batchSize, opt.mask_nc, opt.size, opt.size)

  target_real = Variable(Tensor(opt.batchSize).fill_(1.0), requires_grad=False)
  target_fake = Variable(Tensor(opt.batchSize).fill_(0.0), requires_grad=False)

  fake_A_buffer = ReplayBuffer()
  fake_B_buffer = ReplayBuffer()


  plt.ioff()
  curr_iter = 0
  G_losses = []
  D_A_losses = []
  D_B_losses = []
  to_pil = transforms.ToPILImage()


  # Dataset loader
  print('Preparing data...')
  transforms_image = [#transforms.Resize((opt.size, opt.size), Image.BICUBIC),
              transforms.Resize(int(opt.size * 1.12), Image.BICUBIC),
              transforms.RandomCrop(opt.size),
              transforms.RandomHorizontalFlip(),
              transforms.ToTensor(),
              transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]

  transforms_mask = [#transforms.Resize((opt.size, opt.size), Image.BICUBIC),
              transforms.Resize(int(opt.size * 1.12), Image.BICUBIC),
              transforms.RandomCrop(opt.size),
              transforms.RandomHorizontalFlip(),
              transforms.ToTensor(),
              transforms.Normalize([0.5], [0.5])]
  size = dist.get_world_size()
  bsz = 128 / float(size)
  dataloader = DataLoader(ImageNMaskDataset(opt.dataroot, transforms_image=transforms_image,    transforms_mask=transforms_mask),
                       batch_size=opt.batchSize, shuffle=True, num_workers=opt.n_cpu)

  for epoch in range(opt.epoch, opt.n_epochs):
     for i, batch in enumerate(dataloader):
        # Set model input
        real_A = Variable(input_A.copy_(batch['A']))
        real_B = Variable(input_B.copy_(batch['B']))
        real_M = Variable(input_M.copy_(batch['M']))
        ###### Generators A2B and B2A ######
        optimizer_G.zero_grad()

        # Identity loss
        # G_A2B(B) should equal B if real B is fed
        ...
        # G_B2A(A) should equal A if real A is fed
        ...
        # GAN loss
        ...

        # Cycle loss
        ...

        # Total loss
        ...

        optimizer_G.step()
        ###################################

        ###### Discriminator A ######
        optimizer_D_A.zero_grad()

        # Real loss
        ...

        # Fake loss
        ...

        # Total loss
        ...

        optimizer_D_A.step()
        ###################################

        ###### Discriminator B ######
        optimizer_D_B.zero_grad()

        # Real loss
        ...

        # Fake loss
        ...

        # Total loss
        ...

        optimizer_D_B.step()
        ###################################

        curr_iter += 1

        if i % 1 == 0:
           log = '[iter %d], [loss_G %.5f], [loss_G_identity %.5f], [loss_G_GAN %.5f],' \
              '[loss_G_cycle %.5f], [loss_D %.5f], [epoch %d]' % \
              (curr_iter, loss_G, (loss_identity_A + loss_identity_B), (loss_GAN_A2B + loss_GAN_B2A),
              (loss_cycle_ABA + loss_cycle_BAB), (loss_D_A + loss_D_B), epoch)
           print(log)

           img_fake_A = 0.5 * (fake_A.detach().data + 1.0)
           img_fake_A = (to_pil(img_fake_A.data[0].squeeze(0).cpu()))
           img_fake_A.save('output/fake_A.png')

           img_fake_B = 0.5 * (fake_B.detach().data + 1.0)
           img_fake_B = (to_pil(img_fake_B.data[0].squeeze(0).cpu()))
           img_fake_B.save('output/fake_B.png')

        # Progress report (http://137.189.90.150:8097)
        # logger.log({'loss_G': loss_G, 'loss_G_identity': (loss_identity_A + loss_identity_B), 'loss_G_GAN': (loss_GAN_A2B + loss_GAN_B2A),
        #             'loss_G_cycle': (loss_cycle_ABA + loss_cycle_BAB), 'loss_D': (loss_D_A + loss_D_B)},
        #             images={'real_A': real_A, 'real_B': real_B, 'fake_A': fake_A, 'fake_B': fake_B})

     # Update learning rates
     lr_scheduler_G.step()
     lr_scheduler_D_A.step()
     lr_scheduler_D_B.step()

     # Save models checkpoints
     ...

     if (epoch+1) % opt.snapshot_epochs == 0:
        torch.save(netG_A2B.state_dict(), ('output/netG_A2B_%d.pth' % (epoch+1)))
        torch.save(netG_B2A.state_dict(), ('output/netG_B2A_%d.pth' % (epoch+1)))

     print('Epoch:{}'.format(epoch))


  def init_process(rank, size, fn, backend='gloo'):
      """ Initialize the distributed environment. """
      os.environ['MASTER_ADDR'] = '127.0.0.1'
      os.environ['MASTER_PORT'] = '29500'
      dist.init_process_group(backend, rank=rank, world_size=size)
      fn(rank, size)


  ###################################

  if __name__ == "__main__":
      mp.set_start_method('spawn', force=True)
      size = 4
      processes = []
      for rank in range(size):
          p = Process(target=init_process, args=(rank, size, run))
          p.start()
          processes.append(p)

      for p in processes:
          p.join()</code></pre>

wayi: <p>In your main method, you can do this:</p>
<pre><code class="lang-auto">    world_size = torch.cuda.device_count()
    backend = 'gloo'
    mp.spawn(init_process, args=(world_size, backend), nprocs=world_size, join=True)
</code></pre>
<p>For more details, check out this <a href="https://pytorch.org/tutorials/intermediate/ddp_tutorial.html#basic-use-case" rel="noopener nofollow ugc">example</a> in the tutorial.</p>

creatives07: <p>I try this, and I get this error message.</p>
<p>– Process 2 terminated with the following error:<br>
Traceback (most recent call last):<br>
File “/usr/local/lib/python3.8/dist-packages/torch/multiprocessing/spawn.py”, line 59, in _wrap<br>
fn(i, *args)<br>
File “/root/USRGAN_step2/train_custom.py”, line 390, in init_process<br>
fn(rank, size)<br>
TypeError: ‘str’ object is not callable</p>

wayi: <p>You need to change the order of <code>fn</code> and <code>backend</code> in <code>init_process</code> method, now <code>backend</code> value is fed to <code>fn</code> position.</p>

creatives07: <p>Thank you. But I got same error.</p>
<p><code>RuntimeError: Default process group has not been initialized, please make sure to call init_process_group.</code></p>

wayi: <aside class="quote no-group" data-username="creatives07" data-post="1" data-topic="120915">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/c/77aa72/48.png" class="avatar"> creatives07:</div>
<blockquote>
<p><code>init_process</code></p>
</blockquote>
</aside>
<p>That’s weird. <code>init_process_group</code> should have been called by your <code>init_process</code> in your last line. Can you add a log and confirm that your <code>init_process_group</code> is really called?</p>

creatives07: <p>Traceback (most recent call last):<br>
File “”, line 1, in <br>
File “/usr/lib/python3.8/multiprocessing/spawn.py”, line 116, in spawn_main<br>
exitcode = _main(fd, parent_sentinel)<br>
File “/usr/lib/python3.8/multiprocessing/spawn.py”, line 126, in _main<br>
self = reduction.pickle.load(from_parent)<br>
_pickle.UnpicklingError: pickle data was truncated<br>
Traceback (most recent call last):<br>
File “train_custom.py”, line 398, in <br>
mp.spawn(init_process, args=(world_size, backend, run), nprocs=world_size, join=True)<br>
File “/usr/local/lib/python3.8/dist-packages/torch/multiprocessing/spawn.py”, line 230, in spawn<br>
return start_processes(fn, args, nprocs, join, daemon, start_method=‘spawn’)<br>
File “/usr/local/lib/python3.8/dist-packages/torch/multiprocessing/spawn.py”, line 188, in start_processes<br>
while not context.join():<br>
File “/usr/local/lib/python3.8/dist-packages/torch/multiprocessing/spawn.py”, line 150, in join<br>
raise ProcessRaisedException(msg, error_index, failed_process.pid)<br>
torch.multiprocessing.spawn.ProcessRaisedException:</p>
<p>– Process 0 terminated with the following error:<br>
Traceback (most recent call last):<br>
File “/usr/local/lib/python3.8/dist-packages/torch/multiprocessing/spawn.py”, line 59, in _wrap<br>
fn(i, *args)<br>
File “/root/USRGAN_step2/train_custom.py”, line 390, in init_process<br>
fn(rank, size2)<br>
File “/root/USRGAN_step2/train_custom.py”, line 226, in run<br>
for i, batch in enumerate(dataloader):<br>
File “/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py”, line 517, in <strong>next</strong><br>
data = self._next_data()<br>
File “/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py”, line 1199, in _next_data<br>
return self._process_data(data)<br>
File “/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py”, line 1225, in _process_data<br>
data.reraise()<br>
File “/usr/local/lib/python3.8/dist-packages/torch/_utils.py”, line 429, in reraise<br>
raise self.exc_type(msg)<br>
RuntimeError: Caught RuntimeError in DataLoader worker process 0.<br>
Original Traceback (most recent call last):<br>
File “/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/worker.py”, line 202, in _worker_loop<br>
data = fetcher.fetch(index)<br>
File “/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/fetch.py”, line 44, in fetch<br>
data = [self.dataset[idx] for idx in possibly_batched_index]<br>
File “/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/fetch.py”, line 44, in <br>
data = [self.dataset[idx] for idx in possibly_batched_index]<br>
File “/root/USRGAN_step2/datasets_S3_2.py”, line 57, in <strong>getitem</strong><br>
size = dist.get_world_size()<br>
File “/usr/local/lib/python3.8/dist-packages/torch/distributed/distributed_c10d.py”, line 711, in get_world_size<br>
return _get_group_size(group)<br>
File “/usr/local/lib/python3.8/dist-packages/torch/distributed/distributed_c10d.py”, line 263, in _get_group_size<br>
default_pg = _get_default_group()<br>
File “/usr/local/lib/python3.8/dist-packages/torch/distributed/distributed_c10d.py”, line 347, in _get_default_group<br>
raise RuntimeError("Default process group has not been initialized, "<br>
RuntimeError: Default process group has not been initialized, please make sure to call init_process_group.</p>
<p>That is all of Traceback. And I will review the code again according to your advice. Thank you so much.</p>