Repository: ultralytics/yolov5

Issue:
subprocess.CalledProcessError with Multi-GPU Training
## ❔Question
I want to train on my 2 machines, each machine have 4 GPUs， and I change the code:
`dist.init_process_group(backend='nccl', init_method='tcp://' + opt.ip, world_size=2, rank=opt.rank)`
then ：
`$python -m torch.distributed.launch --nproc_per_node 8 train.py --ip='125.217.41.155:8888' --rank=0`
`$python -m torch.distributed.launch --nproc_per_node 8 train.py --ip='125.217.41.155:8888' --rank=1`

But unfortunately：
Traceback (most recent call last):                                                                                                                                                             
    File "train.py", line 472, in <module>                                                                                                                                                         
        train(hyp, tb_writer, opt, device)                                                                                                                                                         
    File "train.py", line 118, in train                                                                                                                                                            
        with torch_distributed_zero_first(rank):                                                                                                                                                   
    File "/home/luguangjian/.conda/envs/yolov5/lib/python3.7/contextlib.py", line 112, in __enter__                                                                                                
        return next(self.gen)                                                                                                                                                                     
    File "/home/luguangjian/yolov5-master/utils/utils.py", line 41, in torch_distributed_zero_first                                                                                                
        torch.distributed.barrier()                                                                                                                                                               
    File "/home/luguangjian/.conda/envs/yolov5/lib/python3.7/site-packages/torch/distributed/distributed_c10d.py", line 1485, in barrier                                                           
        work = _default_pg.barrier()                                                                                                                                                             RuntimeError: NCCL error in: /pytorch/torch/lib/c10d/ProcessGroupNCCL.cpp:514, unhandled system error, NCCL version 2.4.8

Traceback (most recent call last):                                                                                                                                                             
    File "/home/luguangjian/.conda/envs/yolov5/lib/python3.7/runpy.py", line 193, in _run_module_as_main                                                                                           
        "__main__", mod_spec)                                                                                                                                                                      
    File "/home/luguangjian/.conda/envs/yolov5/lib/python3.7/runpy.py", line 85, in _run_code                                                                                                      
        exec(code, run_globals)                                                                                                                                                                    
    File "/home/luguangjian/.conda/envs/yolov5/lib/python3.7/site-packages/torch/distributed/launch.py", line 263, in <module>                                                                     
        main()                                                                                                                                                                                     
    File "/home/luguangjian/.conda/envs/yolov5/lib/python3.7/site-packages/torch/distributed/launch.py", line 259, in main                                                                         
        cmd=cmd)    
                                                                                                                                                                             subprocess.CalledProcessError: Command '['/home/luguangjian/.conda/envs/yolov5/bin/python', '-u', 'train.py', '--local_rank=3', '--ip=125.217.41.155:8878', '--rank=1']' returned non-zero exit status 1. 

Has anyone had the same problem？What should I do?



## Additional context



Discussion:
Hi @yiwangde , when writing the code, we expected it to run on only **one** `node`/machine. Although it wouldn't be too hard to update it, I do not have machines to test this on..

Edit: I am working on a PR for this.

Resolving patch:
diff --git a/train.py b/train.py
--- a/train.py
+++ b/train.py
@@ -62,9 +62,9 @@ def train(hyp, opt, device, tb_writer=None):
     best = wdir + 'best.pt'
     results_file = log_dir + os.sep + 'results.txt'
     epochs, batch_size, total_batch_size, weights, rank = \
-        opt.epochs, opt.batch_size, opt.total_batch_size, opt.weights, opt.local_rank
+        opt.epochs, opt.batch_size, opt.total_batch_size, opt.weights, opt.global_rank
+    
     # TODO: Use DDP logging. Only the first process is allowed to log.
-
     # Save run settings
     with open(Path(log_dir) / 'hyp.yaml', 'w') as f:
         yaml.dump(hyp, f, sort_keys=False)
@@ -184,7 +184,7 @@ def train(hyp, opt, device, tb_writer=None):
 
     # DDP mode
     if cuda and rank != -1:
-        model = DDP(model, device_ids=[rank], output_device=rank)
+        model = DDP(model, device_ids=[opt.local_rank], output_device=(opt.local_rank))
 
     # Trainloader
     dataloader, dataset = create_dataloader(train_path, imgsz, batch_size, gs, opt, hyp=hyp, augment=True,
@@ -441,8 +441,7 @@ def train(hyp, opt, device, tb_writer=None):
     if last and not opt.weights:
         print(f'Resuming training from {last}')
     opt.weights = last if opt.resume and not opt.weights else opt.weights
-
-    if opt.local_rank in [-1, 0]:
+    if opt.local_rank == -1 or ("RANK" in os.environ and os.environ["RANK"] == "0"):
         check_git_status()
     opt.cfg = check_file(opt.cfg)  # check file
     opt.data = check_file(opt.data)  # check file
@@ -454,7 +453,8 @@ def train(hyp, opt, device, tb_writer=None):
     device = select_device(opt.device, batch_size=opt.batch_size)
     opt.total_batch_size = opt.batch_size
     opt.world_size = 1
-
+    opt.global_rank = -1
+    
     # DDP mode
     if opt.local_rank != -1:
         assert torch.cuda.device_count() > opt.local_rank
@@ -462,6 +462,7 @@ def train(hyp, opt, device, tb_writer=None):
         device = torch.device('cuda', opt.local_rank)
         dist.init_process_group(backend='nccl', init_method='env://')  # distributed backend
         opt.world_size = dist.get_world_size()
+        opt.global_rank = dist.get_rank()
         assert opt.batch_size % opt.world_size == 0, '--batch-size must be multiple of CUDA device count'
         opt.batch_size = opt.total_batch_size // opt.world_size
 
@@ -470,7 +471,7 @@ def train(hyp, opt, device, tb_writer=None):
     # Train
     if not opt.evolve:
         tb_writer = None
-        if opt.local_rank in [-1, 0]:
+        if opt.global_rank in [-1, 0]:
             print('Start Tensorboard with "tensorboard --logdir=runs", view at http://localhost:6006/')
             tb_writer = SummaryWriter(log_dir=increment_dir('runs/exp', opt.name))
 


Test patch: