diff --git a/docs/source/quicktour.rst b/docs/source/quicktour.rst
index 7ccdf5daf..07035a427 100644
--- a/docs/source/quicktour.rst
+++ b/docs/source/quicktour.rst
@@ -358,6 +358,20 @@ softmax). However you might want to put your loss computation inside the `accele
     with accelerator.autocast():
         loss = complex_loss_function(outputs, target):
 
+Another caveat with Mixed Precision training is that the gradient will skip a few updates at the beginning and
+sometimes during training: because of the dynamic loss scaling strategy, there are points during training where the
+gradients have overflown, and the loss scaling factor is reduced to avoid this happening again at the next step.
+
+This means that you may update your learning rate scheduler when there was no update, which is fine in general, but may
+have an impact when you have very little training data, or if the first learning rate values of your scheduler are very
+important. In this case, you can skip the learning rate scheduler updates when the optimizer step was not done like
+this:
+
+.. codeblock::
+
+    if not accelerator.optimizer_step_was_skipped:
+        lr_scheduler.step()
+
 
 Internal mechanism
 -----------------------------------------------------------------------------------------------------------------------
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index a11b7a7a5..a35f7e268 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -566,3 +566,14 @@ def autocast(self):
             autocast_context.__exit__()
         else:
             yield
+
+    @property
+    def optimizer_step_was_skipped(self):
+        """
+        Whether or not the optimizer update was skipped (because of gradient overflow in mixed precision), in which
+        case the learning rate should not be changed.
+        """
+        for optimizer in self._optimizers:
+            if optimizer.is_overflow:
+                return True
+        return False
diff --git a/src/accelerate/deepspeed_utils.py b/src/accelerate/deepspeed_utils.py
index e52c0c030..07450a87e 100644
--- a/src/accelerate/deepspeed_utils.py
+++ b/src/accelerate/deepspeed_utils.py
@@ -89,7 +89,7 @@ def step(self):
 
     @property
     def is_overflow(self):
-        """This must be called before lr_scheduler.step() when using deepspeed with fp16"""
+        """Whether or not the optimizer step was done, or skipped because of gradient overflow."""
         overflow = False
         if hasattr(self.optimizer, "overflow"):
             overflow = self.optimizer.overflow
diff --git a/src/accelerate/optimizer.py b/src/accelerate/optimizer.py
index b2909dbb5..37ca78833 100644
--- a/src/accelerate/optimizer.py
+++ b/src/accelerate/optimizer.py
@@ -53,6 +53,7 @@ def __init__(self, optimizer, device_placement=True, scaler=None):
         self.scaler = scaler
         self.state = AcceleratorState()
         self.device_placement = device_placement
+        self._is_overflow = False
 
         # Handle device placement
         if device_placement:
@@ -108,8 +109,12 @@ def step(self, closure=None):
             optimizer_args = {"closure": closure} if closure is not None else {}
             xm.optimizer_step(self.optimizer, optimizer_args=optimizer_args)
         elif self.scaler is not None:
+            scale_before = self.scaler.get_scale()
             self.scaler.step(self.optimizer, closure)
             self.scaler.update()
+            scale_after = self.scaler.get_scale()
+            # If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
+            self._is_overflow = scale_after < scale_before
         else:
             self.optimizer.step(closure)
 
@@ -119,5 +124,5 @@ def _switch_parameters(self, parameters_map):
 
     @property
     def is_overflow(self):
-        """This needs to be implemented at the end"""
-        return False  # TODO: implement it
+        """Whether or not the optimizer step was done, or skipped because of gradient overflow."""
+        return self._is_overflow


Review comments: body: ```suggestion
sometimes during training: because of the dynamic loss scaling strategy, there are points during training where the
```
diff_hunk: @@ -358,6 +358,20 @@ softmax). However you might want to put your loss computation inside the `accele
     with accelerator.autocast():
         loss = complex_loss_function(outputs, target):
 
+Another caveat with Mixed Precision training is that the gradient will skip a few updates at the beginning and
+sometimes during training: because of the dynamic loss scaling strategy, there are points at training where the
body: ```suggestion
have an impact when you have very little training data, or if the first learning rate values of your scheduler are very
```
diff_hunk: @@ -358,6 +358,20 @@ softmax). However you might want to put your loss computation inside the `accele
     with accelerator.autocast():
         loss = complex_loss_function(outputs, target):
 
+Another caveat with Mixed Precision training is that the gradient will skip a few updates at the beginning and
+sometimes during training: because of the dynamic loss scaling strategy, there are points at training where the
+gradients have overflown, and the loss scaling factor is reduced to avoid this happening again at the next step.
+
+This means that you may update your learning rate scheduler when there was no update, which is fine in general, but may
+have an impace when you have very little training data, or if the first learning rate values of your scheduler are very