Repository: horovod/horovod

Issue:
Duplicated Synchronization with Gradient Clipping?
**Environment:**
1. Framework: pytorch
2. Framework version: 1.0
3. Horovod version: latest
4. MPI version: -
5. CUDA version: -
6. NCCL version: -
7. Python version: 3.6
8. OS and version: Ubuntu 16.04

https://github.com/horovod/horovod/blob/master/horovod/torch/__init__.py#L174

If you see this documentation, when using gradient clipping, there is duplicated(twice) synchronization. Once is when 'synchronize' is called and second one is when 'step' is called.

After I followed this documentation with gradient clipping, training speed is slower.

Any solusions?


Discussion:
@ildoonet, thanks for raising this!  This regression has been introduced by #597.

While we're thinking about a proper solution, can you specify `optimizer._requires_update = set()` after `DistributedOptimizer` wraps original optimizer, [like this](https://gist.github.com/alsrgv/0713add50fe49a409316832a31612dde)?
@alsrgv Thanks for the tip. I will try with it and wait for the proper solutions.

Resolving patch:
diff --git a/horovod/torch/__init__.py b/horovod/torch/__init__.py
--- a/horovod/torch/__init__.py
+++ b/horovod/torch/__init__.py
@@ -17,6 +17,8 @@
 from __future__ import division
 from __future__ import print_function
 
+import warnings
+
 from horovod.common.util import check_extension
 
 try:
@@ -74,6 +76,7 @@ def __init__(self, params, named_parameters, compression,
         self._handles = {}
         self._grad_accs = []
         self._requires_update = set()
+        self._synchronized = False
         if size() > 1:
             self._register_hooks()
 
@@ -146,8 +149,18 @@ def synchronize(self):
             p.grad.set_(self._compression.decompress(output, ctx))
         self._handles.clear()
 
-    def step(self, closure=None):
-        self.synchronize()
+        self._synchronized = True
+
+    def step(self, closure=None, synchronize=True):
+        if synchronize:
+            if self._synchronized:
+                warnings.warn("optimizer.step(synchronize=True) called after "
+                              "optimizer.synchronize(). This can cause training "
+                              "slowdown. You may want to consider using "
+                              "optimizer.step(synchronize=False) if you use "
+                              "optimizer.synchronize() in your code.")
+            self.synchronize()
+        self._synchronized = False
         return super(self.__class__, self).step(closure)
 
 
@@ -165,6 +178,8 @@ def DistributedOptimizer(optimizer, named_parameters=None,
     DistributedOptimizer exposes the `synchronize()` method, which forces allreduce operations
     to finish before continuing the execution. It's useful in conjunction with gradient
     clipping, or other operations that modify gradients in place before `step()` is executed.
+    Make sure to pass `synchronize=False` to `step()` method if you're calling `synchronize()`
+    in your code.
 
     Example of gradient clipping:
     ```
@@ -172,8 +187,8 @@ def DistributedOptimizer(optimizer, named_parameters=None,
     loss = F.nll_loss(output, target)
     loss.backward()
     optimizer.synchronize()
-    torch.nn.utils.clip_grad_norm(model.parameters(), args.clip)
-    optimizer.step()
+    torch.nn.utils.clip_grad_norm_(model.parameters(), args.clip)
+    optimizer.step(synchronize=False)
     ```
 
     Arguments:


Test patch:
diff --git a/test/test_torch.py b/test/test_torch.py
--- a/test/test_torch.py
+++ b/test/test_torch.py
@@ -1223,3 +1223,64 @@ def train_step(train_generator=False, train_discriminator=False):
 
             # Step 2: train discriminator.
             train_step(train_discriminator=True)
+
+    def test_gradient_clipping(self):
+        """Test gradient clipping example."""
+        hvd.init()
+        size = hvd.size()
+
+        # This test does not apply if there is only one worker.
+        if size == 1:
+            return
+
+        x = torch.ones(1, 1).requires_grad_()
+        y = torch.ones(1, 1).requires_grad_()
+
+        model = torch.nn.Linear(1, 1)
+        model.weight = torch.nn.Parameter(torch.zeros(1, 1) + 0.5)
+        model.bias = torch.nn.Parameter(torch.zeros(1))
+        hvd.broadcast_parameters(model.state_dict(), root_rank=0)
+        optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
+        optimizer = hvd.DistributedOptimizer(
+            optimizer, named_parameters=model.named_parameters())
+
+        y_pred = model(x)
+        loss = F.mse_loss(y_pred, y)
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.synchronize()
+        prior_grad = model.weight.grad.item()
+        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)
+        clipped_grad = model.weight.grad.item()
+        assert abs(prior_grad) > abs(clipped_grad)
+        optimizer.step(synchronize=False)
+
+    def test_synchronize_step_warning(self):
+        """Test that .synchronize() followed by .step(synchronize=True) will produce a warning."""
+        hvd.init()
+        size = hvd.size()
+
+        # This test does not apply if there is only one worker.
+        if size == 1:
+            return
+
+        x = torch.zeros(1, 1).requires_grad_()
+        y = torch.ones(1, 1).requires_grad_()
+
+        model = torch.nn.Linear(1, 1)
+        hvd.broadcast_parameters(model.state_dict(), root_rank=0)
+        optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
+        optimizer = hvd.DistributedOptimizer(
+            optimizer, named_parameters=model.named_parameters())
+
+        y_pred = model(x)
+        loss = F.mse_loss(y_pred, y)
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.synchronize()
+        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)
+        with warnings.catch_warnings(record=True) as ws:
+            optimizer.step(synchronize=True)
+            assert len(ws) == 1
+            assert 'optimizer.step(synchronize=True) called after optimizer.synchronize()' \
+                in str(ws[0].message)