Repository: Lightning-AI/pytorch-lightning

Issue:
Horovod with native 16 precision not working
<!-- 
### Common bugs:
1. Tensorboard not showing in Jupyter-notebook see [issue 79](https://github.com/PyTorchLightning/pytorch-lightning/issues/79).    
2. PyTorch 1.1.0 vs 1.2.0 support [see FAQ](https://github.com/PyTorchLightning/pytorch-lightning#faq)    
-->

## 🐛 Bug

<!-- A clear and concise description of what the bug is. -->

### To Reproduce

Steps to reproduce the behavior:

1. using precision=16 with distributed_backend=horovod
 

```
Traceback (most recent call last):
  File "/workspace/main_lightning.py", line 500, in <module>
    main(hyperparams)
  File "/workspace/main_lightning.py", line 492, in main
    trainer.fit(model)
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/states.py", line 48, in wrapped_fn
    result = fn(self, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/trainer.py", line 1068, in fit
    results = self.horovod_train(model)
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/distrib_parts.py", line 213, in horovod_train
    model, optimizers = model.configure_apex(amp, model, self.optimizers, self.amp_level)
  File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/core/lightning.py", line 954, in configure_apex
    model, optimizers = amp.initialize(model, optimizers, opt_level=amp_level)
```

#### Code sample
```
trainer = Trainer(
        precision=16,
        gpus=1,
        distributed_backend="horovod")
```

### Environment
 - PyTorch Version: 1.6.0+cu101
 - How you installed PyTorch: pip


Discussion:
mind have look @tgaddair 🐰 
Absolutely, let me take a look today and get back to you, @mutasem-mattar.

Resolving patch:
diff --git a/pytorch_lightning/accelerators/horovod_backend.py b/pytorch_lightning/accelerators/horovod_backend.py
--- a/pytorch_lightning/accelerators/horovod_backend.py
+++ b/pytorch_lightning/accelerators/horovod_backend.py
@@ -72,11 +72,6 @@ def setup(self, model):
             if isinstance(scheduler, _LRScheduler):
                 scheduler.base_lrs = [lr * hvd.size() for lr in scheduler.base_lrs]
 
-        if self.trainer.amp_backend:
-            model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)
-            self.trainer.optimizers = optimizers
-            self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)
-
         # Horovod: broadcast parameters & optimizer state to ensure consistent initialization
         hvd.broadcast_parameters(model.state_dict(), root_rank=0)
         for optimizer in self.trainer.optimizers:
@@ -92,6 +87,11 @@ def filter_named_parameters(model, optimizer):
             for optimizer in self.trainer.optimizers
         ]
 
+        if self.trainer.amp_backend == AMPType.APEX:
+            model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)
+            self.trainer.optimizers = optimizers
+            self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)
+
         # Update logger rank info from Horovod to avoid race conditions from  different ranks
         # creating directories / writing files in the same locations.
         self.trainer.global_rank = hvd.rank()


Test patch:
diff --git a/tests/models/test_horovod.py b/tests/models/test_horovod.py
--- a/tests/models/test_horovod.py
+++ b/tests/models/test_horovod.py
@@ -42,8 +42,8 @@ def _nccl_available():
 def _run_horovod(trainer_options, on_gpu=False):
     """Execute the training script across multiple workers in parallel."""
     num_processes = trainer_options.get('gpus', 2)
-    # gpus trainer argument does not apply for horovod
-    trainer_options.update(gpus=None)
+    # for Horovod, we interpret `gpus` to be set per worker
+    trainer_options.update(gpus=1 if on_gpu else None)
     tutils.reset_seed()
     cmdline = [
         'horovodrun',
@@ -110,6 +110,27 @@ def test_horovod_multi_gpu(tmpdir):
     _run_horovod(trainer_options, on_gpu=True)
 
 
+@pytest.mark.skipif(platform.system() == "Windows", reason="Horovod is not supported on Windows")
+@pytest.mark.skipif(not _nccl_available(), reason="test requires Horovod with NCCL support")
+@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
+def test_horovod_amp(tmpdir):
+    """Test Horovod with multi-GPU support."""
+    trainer_options = dict(
+        default_root_dir=str(tmpdir),
+        weights_save_path=str(tmpdir),
+        gradient_clip_val=1.0,
+        progress_bar_refresh_rate=0,
+        max_epochs=1,
+        limit_train_batches=0.4,
+        limit_val_batches=0.2,
+        gpus=2,
+        deterministic=True,
+        distributed_backend='horovod',
+        precision=16,
+    )
+    _run_horovod(trainer_options, on_gpu=True)
+
+
 @pytest.mark.skipif(platform.system() == "Windows", reason="Horovod is not supported on Windows")
 @pytest.mark.skipif(not _nccl_available(), reason="test requires Horovod with NCCL support")
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine")