Repository: pytorch/ignite

Issue:
add gradient normalization and accumulation in supervised_training_step_* functions
## 🚀 Feature

After #1589 , `grad_norm` is left to add (#419), and also [gradient accumulation](https://pytorch.org/ignite/master/faq.html#gradients-accumulation) is also common in supervised training, we should also add that to have full supervised training.



Discussion:
@ydcjeff yes, adding a grad accumulation could a be great addition ! We provide a code snippet in FAQ for that but in the code it could be also helpful.
Hey @ydcjeff can i work on this?
Hi @KickItLikeShika Sure, feel free to go ahead
Hi @ydcjeff and @vfdev-5 I would like to add gradient accumulation in the ``create_supervised_trainer``. It can be added as a seperate step like ``supervised_training_step_grad_acc`` which applies just grad acc or would it be useful to allow grad acc as a flag in the existing ``supervised_training_step*`` functions so that it can be coupled with existing ones like amp?
@Ishan-Kumar2 I think a flag to existing `supervised_training_step*` can be a good option. 

Resolving patch:
diff --git a/ignite/engine/__init__.py b/ignite/engine/__init__.py
--- a/ignite/engine/__init__.py
+++ b/ignite/engine/__init__.py
@@ -51,6 +51,7 @@ def supervised_training_step(
     non_blocking: bool = False,
     prepare_batch: Callable = _prepare_batch,
     output_transform: Callable = lambda x, y, y_pred, loss: loss.item(),
+    gradient_accumulation_steps: int = 1,
 ) -> Callable:
     """Factory function for supervised training.
 
@@ -67,7 +68,8 @@ def supervised_training_step(
             tuple of tensors `(batch_x, batch_y)`.
         output_transform: function that receives 'x', 'y', 'y_pred', 'loss' and returns value
             to be assigned to engine's state.output after each iteration. Default is returning `loss.item()`.
-
+        gradient_accumulation_steps: Number of steps the gradients should be accumulated across.
+            (default: 1 (means no gradient accumulation))
     Returns:
         Callable: update function.
 
@@ -84,16 +86,27 @@ def supervised_training_step(
             trainer = Engine(update_fn)
 
     .. versionadded:: 0.4.5
+    .. versionchanged:: 0.5.0
+        Added Gradient Accumulation.
     """
 
+    if gradient_accumulation_steps <= 0:
+        raise ValueError(
+            "Gradient_accumulation_steps must be strictly positive. "
+            "No gradient accumulation if the value set to one (default)."
+        )
+
     def update(engine: Engine, batch: Sequence[torch.Tensor]) -> Union[Any, Tuple[torch.Tensor]]:
         model.train()
-        optimizer.zero_grad()
         x, y = prepare_batch(batch, device=device, non_blocking=non_blocking)
         y_pred = model(x)
         loss = loss_fn(y_pred, y)
+        if gradient_accumulation_steps > 1:
+            loss = loss / gradient_accumulation_steps
         loss.backward()
-        optimizer.step()
+        if engine.state.iteration % gradient_accumulation_steps == 0:
+            optimizer.step()
+            optimizer.zero_grad()
         return output_transform(x, y, y_pred, loss)
 
     return update
@@ -108,6 +121,7 @@ def supervised_training_step_amp(
     prepare_batch: Callable = _prepare_batch,
     output_transform: Callable = lambda x, y, y_pred, loss: loss.item(),
     scaler: Optional["torch.cuda.amp.GradScaler"] = None,
+    gradient_accumulation_steps: int = 1,
 ) -> Callable:
     """Factory function for supervised training using ``torch.cuda.amp``.
 
@@ -125,6 +139,8 @@ def supervised_training_step_amp(
         output_transform: function that receives 'x', 'y', 'y_pred', 'loss' and returns value
             to be assigned to engine's state.output after each iteration. Default is returning `loss.item()`.
         scaler: GradScaler instance for gradient scaling. (default: None)
+        gradient_accumulation_steps: Number of steps the gradients should be accumulated across.
+            (default: 1 (means no gradient accumulation))
 
     Returns:
         Callable: update function
@@ -143,6 +159,8 @@ def supervised_training_step_amp(
             trainer = Engine(update_fn)
 
     .. versionadded:: 0.4.5
+    .. versionchanged:: 0.5.0
+        Added Gradient Accumulation.
     """
 
     try:
@@ -150,20 +168,31 @@ def supervised_training_step_amp(
     except ImportError:
         raise ImportError("Please install torch>=1.6.0 to use amp_mode='amp'.")
 
+    if gradient_accumulation_steps <= 0:
+        raise ValueError(
+            "Gradient_accumulation_steps must be strictly positive. "
+            "No gradient accumulation if the value set to one (default)."
+        )
+
     def update(engine: Engine, batch: Sequence[torch.Tensor]) -> Union[Any, Tuple[torch.Tensor]]:
         model.train()
-        optimizer.zero_grad()
         x, y = prepare_batch(batch, device=device, non_blocking=non_blocking)
         with autocast(enabled=True):
             y_pred = model(x)
             loss = loss_fn(y_pred, y)
+            if gradient_accumulation_steps > 1:
+                loss = loss / gradient_accumulation_steps
         if scaler:
             scaler.scale(loss).backward()
-            scaler.step(optimizer)
-            scaler.update()
+            if engine.state.iteration % gradient_accumulation_steps == 0:
+                scaler.step(optimizer)
+                scaler.update()
+                optimizer.zero_grad()
         else:
             loss.backward()
-            optimizer.step()
+            if engine.state.iteration % gradient_accumulation_steps == 0:
+                optimizer.step()
+                optimizer.zero_grad()
         return output_transform(x, y, y_pred, loss)
 
     return update
@@ -177,6 +206,7 @@ def supervised_training_step_apex(
     non_blocking: bool = False,
     prepare_batch: Callable = _prepare_batch,
     output_transform: Callable = lambda x, y, y_pred, loss: loss.item(),
+    gradient_accumulation_steps: int = 1,
 ) -> Callable:
     """Factory function for supervised training using apex.
 
@@ -193,6 +223,8 @@ def supervised_training_step_apex(
             tuple of tensors `(batch_x, batch_y)`.
         output_transform: function that receives 'x', 'y', 'y_pred', 'loss' and returns value
             to be assigned to engine's state.output after each iteration. Default is returning `loss.item()`.
+        gradient_accumulation_steps: Number of steps the gradients should be accumulated across.
+            (default: 1 (means no gradient accumulation))
 
     Returns:
         Callable: update function.
@@ -210,6 +242,8 @@ def supervised_training_step_apex(
             trainer = Engine(update_fn)
 
     .. versionadded:: 0.4.5
+    .. versionchanged:: 0.5.0
+        Added Gradient Accumulation.
     """
 
     try:
@@ -217,15 +251,24 @@ def supervised_training_step_apex(
     except ModuleNotFoundError:
         raise ModuleNotFoundError("Please install apex from https://github.com/nvidia/apex to use amp_mode='apex'.")
 
+    if gradient_accumulation_steps <= 0:
+        raise ValueError(
+            "Gradient_accumulation_steps must be strictly positive. "
+            "No gradient accumulation if the value set to one (default)."
+        )
+
     def update(engine: Engine, batch: Sequence[torch.Tensor]) -> Union[Any, Tuple[torch.Tensor]]:
         model.train()
-        optimizer.zero_grad()
         x, y = prepare_batch(batch, device=device, non_blocking=non_blocking)
         y_pred = model(x)
         loss = loss_fn(y_pred, y)
+        if gradient_accumulation_steps > 1:
+            loss = loss / gradient_accumulation_steps
         with apex_amp.scale_loss(loss, optimizer) as scaled_loss:
             scaled_loss.backward()
-        optimizer.step()
+        if engine.state.iteration % gradient_accumulation_steps == 0:
+            optimizer.step()
+            optimizer.zero_grad()
         return output_transform(x, y, y_pred, loss)
 
     return update
@@ -239,6 +282,7 @@ def supervised_training_step_tpu(
     non_blocking: bool = False,
     prepare_batch: Callable = _prepare_batch,
     output_transform: Callable = lambda x, y, y_pred, loss: loss.item(),
+    gradient_accumulation_steps: int = 1,
 ) -> Callable:
     """Factory function for supervised training using ``torch_xla``.
 
@@ -255,6 +299,8 @@ def supervised_training_step_tpu(
             tuple of tensors `(batch_x, batch_y)`.
         output_transform: function that receives 'x', 'y', 'y_pred', 'loss' and returns value
             to be assigned to engine's state.output after each iteration. Default is returning `loss.item()`.
+        gradient_accumulation_steps: Number of steps the gradients should be accumulated across.
+            (default: 1 (means no gradient accumulation))
 
     Returns:
         Callable: update function.
@@ -272,20 +318,31 @@ def supervised_training_step_tpu(
             trainer = Engine(update_fn)
 
     .. versionadded:: 0.4.5
+    .. versionchanged:: 0.5.0
+       Added Gradient Accumulation argument for all supervised training methods.
     """
     try:
         import torch_xla.core.xla_model as xm
     except ModuleNotFoundError:
         raise ModuleNotFoundError("torch_xla cannot be imported, please install PyTorch XLA.")
 
+    if gradient_accumulation_steps <= 0:
+        raise ValueError(
+            "Gradient_accumulation_steps must be strictly positive. "
+            "No gradient accumulation if the value set to one (default)."
+        )
+
     def update(engine: Engine, batch: Sequence[torch.Tensor]) -> Union[Any, Tuple[torch.Tensor]]:
         model.train()
-        optimizer.zero_grad()
         x, y = prepare_batch(batch, device=device, non_blocking=non_blocking)
         y_pred = model(x)
         loss = loss_fn(y_pred, y)
+        if gradient_accumulation_steps > 1:
+            loss = loss / gradient_accumulation_steps
         loss.backward()
-        xm.optimizer_step(optimizer, barrier=True)
+        if engine.state.iteration % gradient_accumulation_steps == 0:
+            xm.optimizer_step(optimizer, barrier=True)
+            optimizer.zero_grad()
         return output_transform(x, y, y_pred, loss)
 
     return update
@@ -330,6 +387,7 @@ def create_supervised_trainer(
     deterministic: bool = False,
     amp_mode: Optional[str] = None,
     scaler: Union[bool, "torch.cuda.amp.GradScaler"] = False,
+    gradient_accumulation_steps: int = 1,
 ) -> Engine:
     """Factory function for creating a trainer for supervised models.
 
@@ -356,6 +414,8 @@ def create_supervised_trainer(
             and ``amp_mode`` is ``amp``. If ``amp_mode`` is ``apex``, this argument will be ignored.
             If True, will create default GradScaler. If GradScaler instance is passed, it will be used instead.
             (default: False)
+        gradient_accumulation_steps: Number of steps the gradients should be accumulated across.
+            (default: 1 (means no gradient accumulation))
 
     Returns:
         a trainer engine with supervised update function.
@@ -388,6 +448,9 @@ def create_supervised_trainer(
 
         - Added ``amp_mode`` argument for automatic mixed precision.
         - Added ``scaler`` argument for gradient scaling.
+
+    .. versionchanged:: 0.5.0
+        Added Gradient Accumulation argument for all supervised training methods.
     """
 
     device_type = device.type if isinstance(device, torch.device) else device
@@ -396,19 +459,48 @@ def create_supervised_trainer(
 
     if mode == "amp":
         _update = supervised_training_step_amp(
-            model, optimizer, loss_fn, device, non_blocking, prepare_batch, output_transform, _scaler
+            model,
+            optimizer,
+            loss_fn,
+            device,
+            non_blocking,
+            prepare_batch,
+            output_transform,
+            _scaler,
+            gradient_accumulation_steps,
         )
     elif mode == "apex":
         _update = supervised_training_step_apex(
-            model, optimizer, loss_fn, device, non_blocking, prepare_batch, output_transform
+            model,
+            optimizer,
+            loss_fn,
+            device,
+            non_blocking,
+            prepare_batch,
+            output_transform,
+            gradient_accumulation_steps,
         )
     elif mode == "tpu":
         _update = supervised_training_step_tpu(
-            model, optimizer, loss_fn, device, non_blocking, prepare_batch, output_transform
+            model,
+            optimizer,
+            loss_fn,
+            device,
+            non_blocking,
+            prepare_batch,
+            output_transform,
+            gradient_accumulation_steps,
         )
     else:
         _update = supervised_training_step(
-            model, optimizer, loss_fn, device, non_blocking, prepare_batch, output_transform
+            model,
+            optimizer,
+            loss_fn,
+            device,
+            non_blocking,
+            prepare_batch,
+            output_transform,
+            gradient_accumulation_steps,
         )
 
     trainer = Engine(_update) if not deterministic else DeterministicEngine(_update)


Test patch:
diff --git a/tests/ignite/engine/test_create_supervised.py b/tests/ignite/engine/test_create_supervised.py
--- a/tests/ignite/engine/test_create_supervised.py
+++ b/tests/ignite/engine/test_create_supervised.py
@@ -25,7 +25,8 @@
 from ignite.metrics import MeanSquaredError
 
 
-def _test_create_supervised_trainer(
+def _default_create_supervised_trainer(
+    gradient_accumulation_steps=1,
     model_device: Optional[str] = None,
     trainer_device: Optional[str] = None,
     trace: bool = False,
@@ -42,7 +43,7 @@ def _test_create_supervised_trainer(
     optimizer = SGD(model.parameters(), 0.1)
 
     if trace:
-        example_input = torch.randn(1, 1)
+        example_input = torch.randn(1)
         model = torch.jit.trace(model, example_input)
 
     if amp_mode == "apex" and model_device == trainer_device == "cuda":
@@ -58,21 +59,44 @@ def _test_create_supervised_trainer(
         output_transform=lambda x, y, y_pred, loss: (y_pred, loss.item()),
         amp_mode=amp_mode,
         scaler=scaler,
+        gradient_accumulation_steps=gradient_accumulation_steps,
     )
-
-    x = torch.tensor([[0.1], [0.2]])
-    y = torch.tensor([[0.3], [0.5]])
-    data = [(x, y)]
-
     assert model.weight.data[0, 0].item() == approx(0.0)
     assert model.bias.item() == approx(0.0)
 
+    return trainer, model
+
+
+def _test_create_supervised_trainer(
+    gradient_accumulation_steps=3,
+    loss=0.0045,
+    weight=0.0540,
+    bias=0.1133,
+    model_device: Optional[str] = None,
+    trainer_device: Optional[str] = None,
+    trace: bool = False,
+    amp_mode: str = None,
+    scaler: Union[bool, "torch.cuda.amp.GradScaler"] = False,
+):
+    trainer, model = _default_create_supervised_trainer(
+        gradient_accumulation_steps=gradient_accumulation_steps,
+        model_device=model_device,
+        trainer_device=trainer_device,
+        trace=trace,
+        amp_mode=amp_mode,
+        scaler=scaler,
+    )
+
+    x = torch.tensor([[0.1], [0.3], [0.7], [0.9], [1.3]])
+    y = torch.tensor([[0.3], [0.5], [0.9], [1.3], [0.3]])
+    data = [(_x, _y) for _x, _y in zip(x, y)]
+
     if model_device == trainer_device or ((model_device == "cpu") ^ (trainer_device == "cpu")):
         state = trainer.run(data)
 
-        assert state.output[-1] == approx(0.17), state.output[-1]
-        assert round(model.weight.data[0, 0].item(), 3) == approx(0.013), model.weight.item()
-        assert round(model.bias.item(), 3) == approx(0.08), model.bias.item()
+        assert round(state.output[-1], 4) == loss, state.output[-1]
+        assert round(model.weight.data[0, 0].item(), 4) == weight, model.weight.item()
+        assert round(model.bias.item(), 4) == bias, model.bias.item()
 
         if amp_mode == "amp":
             assert state.output[0].dtype is torch.half
@@ -80,6 +104,26 @@ def _test_create_supervised_trainer(
                 assert hasattr(state, "scaler")
             else:
                 assert not hasattr(state, "scaler")
+
+        # Test for Gradient Accumulation Turned Off
+        trainer, model = _default_create_supervised_trainer(
+            model_device=model_device, trainer_device=trainer_device, trace=trace, amp_mode=amp_mode, scaler=scaler,
+        )
+        x = torch.tensor([[1.0], [1.0], [1.0], [1.0], [1.0]])
+        data = [(_x, _y) for _x, _y in zip(x, x)]
+
+        for i in range(len(data)):
+            original_weights = model.weight.data[0, 0].item()
+            original_bias = model.bias.item()
+            state = trainer.run([data[i]])
+            assert state.output[-1] == pytest.approx((1 - (original_weights + original_bias)) ** 2), state.output[-1]
+            assert model.weight.data[0, 0].item() == pytest.approx(
+                original_weights + 2 * 0.1 * (1 - (original_weights + original_bias))
+            ), model.weight.item()
+            assert model.bias.item() == pytest.approx(
+                original_bias + 2 * 0.1 * (1 - (original_weights + original_bias))
+            ), model.bias.item()
+
     else:
         if LooseVersion(torch.__version__) >= LooseVersion("1.7.0"):
             # This is broken in 1.6.0 but will be probably fixed with 1.7.0
@@ -89,23 +133,10 @@ def _test_create_supervised_trainer(
 
 @pytest.mark.skipif(LooseVersion(torch.__version__) < LooseVersion("1.6.0"), reason="Skip if < 1.6.0")
 def test_create_supervised_training_scalar_assignment():
-    model = Linear(1, 1)
-
-    model.weight.data.zero_()
-    model.bias.data.zero_()
-    optimizer = SGD(model.parameters(), 0.1)
 
     with mock.patch("ignite.engine._check_arg") as check_arg_mock:
         check_arg_mock.return_value = None, torch.cuda.amp.GradScaler(enabled=False)
-        trainer = create_supervised_trainer(
-            model,
-            optimizer,
-            mse_loss,
-            device="cpu",
-            output_transform=lambda x, y, y_pred, loss: (y_pred, loss.item()),
-            amp_mode=None,
-            scaler=True,
-        )
+        trainer, _ = _default_create_supervised_trainer(model_device="cpu", trainer_device="cpu", scaler=True)
         assert hasattr(trainer.state, "scaler")
         assert isinstance(trainer.state.scaler, torch.cuda.amp.GradScaler)
 
@@ -121,30 +152,11 @@ def _test_create_mocked_supervised_trainer(
         with mock.patch("ignite.engine.supervised_training_step_apex") as training_step_apex_mock:
             with mock.patch("ignite.engine.supervised_training_step_tpu") as training_step_tpu_mock:
                 with mock.patch("ignite.engine.supervised_training_step") as training_step_mock:
-                    model = Linear(1, 1)
 
-                    if model_device:
-                        model.to(model_device)
-
-                    model.weight.data.zero_()
-                    model.bias.data.zero_()
-                    optimizer = SGD(model.parameters(), 0.1)
-
-                    if trace:
-                        example_input = torch.randn(1, 1)
-                        model = torch.jit.trace(model, example_input)
-
-                    if amp_mode == "apex" and model_device == trainer_device == "cuda":
-                        from apex import amp
-
-                        model, optimizer = amp.initialize(model, optimizer, opt_level="O2")
-
-                    trainer = create_supervised_trainer(
-                        model,
-                        optimizer,
-                        mse_loss,
-                        device=trainer_device,
-                        output_transform=lambda x, y, y_pred, loss: (y_pred, loss.item()),
+                    trainer, _ = _default_create_supervised_trainer(
+                        model_device=model_device,
+                        trainer_device=trainer_device,
+                        trace=trace,
                         amp_mode=amp_mode,
                         scaler=scaler,
                     )
@@ -153,9 +165,6 @@ def _test_create_mocked_supervised_trainer(
                     y = torch.tensor([[0.3], [0.5]])
                     data = [(x, y)]
 
-                    assert model.weight.data[0, 0].item() == approx(0.0)
-                    assert model.bias.item() == approx(0.0)
-
                     on_tpu = "xla" in trainer_device if trainer_device is not None else False
                     mode, _ = _check_arg(on_tpu, amp_mode, scaler)
 
@@ -172,7 +181,14 @@ def _test_create_mocked_supervised_trainer(
                             assert training_step_mock.called
 
 
-def _test_create_supervised_evaluator(
+def _test_create_supervised_trainer_wrong_accumulation(model_device=None, trainer_device=None, amp_mode=None):
+    with pytest.raises(ValueError, match="Gradient_accumulation_steps must be strictly positive."):
+        _default_create_supervised_trainer(
+            gradient_accumulation_steps=0, model_device=model_device, trainer_device=trainer_device, amp_mode=amp_mode
+        )
+
+
+def _default_create_supervised_evaluator(
     model_device: Optional[str] = None,
     evaluator_device: Optional[str] = None,
     trace: bool = False,
@@ -192,6 +208,21 @@ def _test_create_supervised_evaluator(
 
     evaluator = create_supervised_evaluator(model, device=evaluator_device, amp_mode=amp_mode)
 
+    assert model.weight.data[0, 0].item() == approx(0.0)
+    assert model.bias.item() == approx(0.0)
+
+    return model, evaluator
+
+
+def _test_create_supervised_evaluator(
+    model_device: Optional[str] = None,
+    evaluator_device: Optional[str] = None,
+    trace: bool = False,
+    amp_mode: str = None,
+):
+    model, evaluator = _default_create_supervised_evaluator(
+        model_device=model_device, evaluator_device=evaluator_device, trace=trace, amp_mode=amp_mode
+    )
     x = torch.tensor([[1.0], [2.0]])
     y = torch.tensor([[3.0], [5.0]])
     data = [(x, y)]
@@ -224,26 +255,16 @@ def _test_mocked_supervised_evaluator(
 ):
     with mock.patch("ignite.engine.supervised_evaluation_step") as evaluation_step:
         with mock.patch("ignite.engine.supervised_evaluation_step_amp") as evaluation_step_amp:
-            model = Linear(1, 1)
-
-            if model_device:
-                model.to(model_device)
-
-            model.weight.data.zero_()
-            model.bias.data.zero_()
-
-            if trace:
-                example_input = torch.randn(1, 1)
-                model = torch.jit.trace(model, example_input)
-
-            evaluator = create_supervised_evaluator(model, device=evaluator_device, amp_mode=amp_mode)
+            _, evaluator = _default_create_supervised_evaluator(
+                model_device=model_device, evaluator_device=evaluator_device, trace=trace, amp_mode=amp_mode
+            )
 
             x = torch.tensor([[1.0], [2.0]])
             y = torch.tensor([[3.0], [5.0]])
             data = [(x, y)]
 
             if model_device == evaluator_device or ((model_device == "cpu") ^ (evaluator_device == "cpu")):
-                state = evaluator.run(data)
+                evaluator.run(data)
 
                 if amp_mode == "amp":
                     assert evaluation_step_amp.called
@@ -327,22 +348,29 @@ def _test_create_evaluation_step(
 
 
 def test_create_supervised_trainer():
+    _test_create_supervised_trainer_wrong_accumulation()
     _test_create_supervised_trainer()
     _test_create_mocked_supervised_trainer()
 
 
 def test_create_supervised_trainer_with_cpu():
+    _test_create_supervised_trainer_wrong_accumulation(trainer_device="cpu")
     _test_create_supervised_trainer(trainer_device="cpu")
     _test_create_mocked_supervised_trainer(trainer_device="cpu")
 
 
 def test_create_supervised_trainer_traced_with_cpu():
+    _test_create_supervised_trainer_wrong_accumulation(trainer_device="cpu")
     _test_create_supervised_trainer(trainer_device="cpu", trace=True)
     _test_create_mocked_supervised_trainer(trainer_device="cpu", trace=True)
 
 
 @pytest.mark.skipif(find_spec("apex"), reason="Skip if APEX")
 def test_create_supervised_trainer_apex_error():
+    with pytest.raises(
+        ModuleNotFoundError, match="Please install apex from https://github.com/nvidia/apex to use amp_mode='apex'."
+    ):
+        _test_create_supervised_trainer_wrong_accumulation(trainer_device="cpu", amp_mode="apex")
     with pytest.raises(
         ModuleNotFoundError, match="Please install apex from https://github.com/nvidia/apex to use amp_mode='apex'."
     ):
@@ -359,15 +387,17 @@ def mock_torch_cuda_amp_module():
 
 
 def test_create_supervised_trainer_amp_error(mock_torch_cuda_amp_module):
+    with pytest.raises(ImportError, match="Please install torch>=1.6.0 to use amp_mode='amp'."):
+        _test_create_supervised_trainer_wrong_accumulation(trainer_device="cpu", amp_mode="amp")
     with pytest.raises(ImportError, match="Please install torch>=1.6.0 to use amp_mode='amp'."):
         _test_create_supervised_trainer(amp_mode="amp")
     with pytest.raises(ImportError, match="Please install torch>=1.6.0 to use scaler argument."):
         _test_create_supervised_trainer(amp_mode="amp", scaler=True)
 
 
-@pytest.mark.skipif(LooseVersion(torch.__version__) < LooseVersion("1.6.0"), reason="Skip if < 1.6.0")
 def test_create_supervised_trainer_scaler_not_amp():
     scaler = torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available())
+
     with pytest.raises(ValueError, match=f"scaler argument is {scaler}, but amp_mode is None."):
         _test_create_supervised_trainer(amp_mode=None, scaler=scaler)
     with pytest.raises(ValueError, match="scaler argument is True, but amp_mode is None."):
@@ -381,6 +411,7 @@ def test_create_supervised_trainer_scaler_not_amp():
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="Skip if no GPU")
 def test_create_supervised_trainer_on_cuda():
     model_device = trainer_device = "cuda"
+    _test_create_supervised_trainer_wrong_accumulation(model_device=model_device, trainer_device=trainer_device)
     _test_create_supervised_trainer(model_device=model_device, trainer_device=trainer_device)
     _test_create_mocked_supervised_trainer(model_device=model_device, trainer_device=trainer_device)
 
@@ -389,7 +420,12 @@ def test_create_supervised_trainer_on_cuda():
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="Skip if no GPU")
 def test_create_supervised_trainer_on_cuda_amp():
     model_device = trainer_device = "cuda"
-    _test_create_supervised_trainer(model_device=model_device, trainer_device=trainer_device, amp_mode="amp")
+    _test_create_supervised_trainer_wrong_accumulation(
+        model_device=model_device, trainer_device=trainer_device, amp_mode="amp"
+    )
+    _test_create_supervised_trainer(
+        model_device=model_device, trainer_device=trainer_device, amp_mode="amp",
+    )
     _test_create_mocked_supervised_trainer(model_device=model_device, trainer_device=trainer_device, amp_mode="amp")
 
 
@@ -397,8 +433,12 @@ def test_create_supervised_trainer_on_cuda_amp():
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="Skip if no GPU")
 def test_create_supervised_trainer_on_cuda_amp_scaler():
     model_device = trainer_device = "cuda"
+    _test_create_supervised_trainer_wrong_accumulation(
+        model_device=model_device, trainer_device=trainer_device, amp_mode="amp"
+    )
+
     _test_create_supervised_trainer(
-        model_device=model_device, trainer_device=trainer_device, amp_mode="amp", scaler=True
+        model_device=model_device, trainer_device=trainer_device, amp_mode="amp", scaler=True,
     )
     _test_create_mocked_supervised_trainer(
         model_device=model_device, trainer_device=trainer_device, amp_mode="amp", scaler=True
@@ -406,7 +446,7 @@ def test_create_supervised_trainer_on_cuda_amp_scaler():
 
     scaler = torch.cuda.amp.GradScaler(enabled=torch.cuda.is_available())
     _test_create_supervised_trainer(
-        model_device=model_device, trainer_device=trainer_device, amp_mode="amp", scaler=scaler
+        model_device=model_device, trainer_device=trainer_device, amp_mode="amp", scaler=scaler,
     )
     _test_create_mocked_supervised_trainer(
         model_device=model_device, trainer_device=trainer_device, amp_mode="amp", scaler=scaler
@@ -417,7 +457,14 @@ def test_create_supervised_trainer_on_cuda_amp_scaler():
 @pytest.mark.skipif(not find_spec("apex"), reason="Skip if no APEX")
 def test_create_supervised_trainer_on_cuda_apex():
     model_device = trainer_device = "cuda"
-    _test_create_supervised_trainer(model_device=model_device, trainer_device=trainer_device, amp_mode="apex")
+    _test_create_supervised_trainer_wrong_accumulation(
+        model_device=model_device, trainer_device=trainer_device, amp_mode="apex"
+    )
+
+    _test_create_supervised_trainer(
+        model_device=model_device, trainer_device=trainer_device, amp_mode="apex",
+    )
+
     _test_create_mocked_supervised_trainer(model_device=model_device, trainer_device=trainer_device, amp_mode="apex")
 
 
@@ -440,6 +487,7 @@ def test_create_supervised_trainer_on_tpu_no_xla():
 @pytest.mark.skipif(not idist.has_xla_support, reason="Skip if no PyTorch XLA package")
 def test_create_supervised_trainer_on_tpu():
     model_device = trainer_device = "xla"
+    _test_create_supervised_trainer_wrong_accumulation(model_device=model_device, trainer_device=trainer_device)
     _test_create_supervised_trainer(model_device=model_device, trainer_device=trainer_device)
     _test_create_mocked_supervised_trainer(model_device=model_device, trainer_device=trainer_device)
 
@@ -454,6 +502,7 @@ def test_create_supervised_trainer_on_tpu_amp():
 
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="Skip if no GPU")
 def test_create_supervised_trainer_on_cuda_with_model_on_cpu():
+    _test_create_supervised_trainer_wrong_accumulation(trainer_device="cuda")
     _test_create_supervised_trainer(trainer_device="cuda")
     _test_create_mocked_supervised_trainer(trainer_device="cuda")