diff --git a/docs/source/accelerator.mdx b/docs/source/accelerator.mdx
index 6e7a93a3d..5202aeb8c 100644
--- a/docs/source/accelerator.mdx
+++ b/docs/source/accelerator.mdx
@@ -38,4 +38,27 @@ should search for and replace by the corresponding methods of your `accelerator`
 - Use [`~Accelerator.clip_grad_norm_`] instead of `torch.nn.utils.clip_grad_norm_` and
   [`~Accelerator.clip_grad_value_`] instead of `torch.nn.utils.clip_grad_value_`.
 
+To perform gradient accumulation use [`~Accelerator.accumulate`] and specify a `gradient_accumulation_steps`. 
+This will also automatically ensure the gradients are synced or unsynced when on multi-device training, check if the step should
+actually be performed, and auto-scale the loss:
+
+```python
+accelerator = Accelerator(gradient_accumulation_steps=2)
+
+for (input, label) in enumerate(training_dataloader):
+    with accelerator.accumulate(model):
+        predictions = model(input)
+        loss = loss_function(predictions, labels)
+        accelerator.backward(loss)
+        optimizer.step()
+        scheduler.step()
+        optimizer.zero_grad()
+```
+
+<Tip warning={true}>
+
+Using this with `dispatch_batches=True`  (which is the default for iterable datasets) is currently not supported.
+
+</Tip>
+
 [[autodoc]] Accelerator
diff --git a/docs/source/internal.mdx b/docs/source/internal.mdx
index e1eec36be..8da23bee8 100644
--- a/docs/source/internal.mdx
+++ b/docs/source/internal.mdx
@@ -12,6 +12,10 @@ specific language governing permissions and limitations under the License.
 
 # Internals
 
+## Gradient Accumulation states
+
+[[autodoc]] state.GradientState
+
 ## Optimizer
 
 [[autodoc]] optimizer.AcceleratedOptimizer
diff --git a/examples/by_feature/gradient_accumulation.py b/examples/by_feature/gradient_accumulation.py
index b69978023..d3b253bce 100644
--- a/examples/by_feature/gradient_accumulation.py
+++ b/examples/by_feature/gradient_accumulation.py
@@ -102,19 +102,21 @@ def collate_fn(examples):
 
 
 def training_function(config, args):
-    # Initialize accelerator
-    accelerator = Accelerator(cpu=args.cpu, mixed_precision=args.mixed_precision)
-    # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
-    lr = config["lr"]
-    num_epochs = int(config["num_epochs"])
-    seed = int(config["seed"])
-    batch_size = int(config["batch_size"])
     # New Code #
     gradient_accumulation_steps = int(args.gradient_accumulation_steps)
+    # Initialize accelerator
+    accelerator = Accelerator(
+        cpu=args.cpu, mixed_precision=args.mixed_precision, gradient_accumulation_steps=gradient_accumulation_steps
+    )
     if accelerator.distributed_type == DistributedType.TPU and gradient_accumulation_steps > 1:
         raise NotImplementedError(
             "Gradient accumulation on TPUs is currently not supported. Pass `gradient_accumulation_steps=1`"
         )
+    # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
+    lr = config["lr"]
+    num_epochs = int(config["num_epochs"])
+    seed = int(config["seed"])
+    batch_size = int(config["batch_size"])
 
     metric = evaluate.load("glue", "mrpc")
 
@@ -152,20 +154,11 @@ def training_function(config, args):
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch.to(accelerator.device)
             # New code #
-            # We use the new `no_sync` context manager to prevent gradient averaging
-            # until we want to at the proper step if we happen to be in a distributed setup
-            # otherwise it does nothing
+            # We use the new `accumulate` context manager to perform gradient accumulation
             # We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
-            if step % gradient_accumulation_steps != 0:
-                # Accumulate gradients locally
-                with accelerator.no_sync(model):
-                    output = model(**batch)
-                    loss = output.loss / gradient_accumulation_steps
-                    accelerator.backward(loss)
-            else:
-                # Sync gradients and step
+            with accelerator.accumulate(model):
                 output = model(**batch)
-                loss = output.loss / gradient_accumulation_steps
+                loss = output.loss
                 accelerator.backward(loss)
                 optimizer.step()
                 lr_scheduler.step()
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 1b3fa27e9..1148262af 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -28,7 +28,7 @@
 from .logging import get_logger
 from .optimizer import AcceleratedOptimizer
 from .scheduler import AcceleratedScheduler
-from .state import AcceleratorState
+from .state import AcceleratorState, GradientState
 from .tracking import LOGGER_TYPE_TO_CLASS, GeneralTracker, filter_trackers
 from .utils import (
     DeepSpeedPlugin,
@@ -92,6 +92,9 @@ class Accelerator:
             default to the value in the environment variable `MIXED_PRECISION`, which will use the default value in the
             accelerate config of the current system or the flag passed with the `accelerate.launch` command. 'fp16'
             requires pytorch 1.6 or higher. 'bf16' requires pytorch 1.10 or higher.
+        gradient_accumulation_steps (`int`, *optional*, default to 1):
+            The number of steps that should pass before gradients are accumulated. A number > 1 should be combined with
+            `Accelerator.accumulate`.
         cpu (`bool`, *optional*):
             Whether or not to force the script to execute on CPU. Will ignore GPU available if set to `True` and force
             the execution on one process only.
@@ -146,6 +149,7 @@ def __init__(
         split_batches: bool = False,
         fp16: bool = None,
         mixed_precision: Union[PrecisionType, str] = None,
+        gradient_accumulation_steps: int = 1,
         cpu: bool = False,
         deepspeed_plugin: DeepSpeedPlugin = None,
         fsdp_plugin: FullyShardedDataParallelPlugin = None,
@@ -231,6 +235,17 @@ def __init__(
             **kwargs,
         )
 
+        if gradient_accumulation_steps > 1:
+            if self.state.distributed_type == DistributedType.TPU:
+                raise NotImplementedError(
+                    "Gradient accumulation on TPU is not supported. Pass in `gradient_accumulation_steps=1`"
+                )
+            if dispatch_batches:
+                raise NotImplementedError(
+                    "Gradient accumulation with dispatched dataloaders is not supported. Pass in `gradient_accumulation_steps=1` or `dispatch_batches=False`"
+                )
+
+        self.gradient_accumulation_steps = gradient_accumulation_steps
         self.device_placement = device_placement
         self.split_batches = split_batches
         self.dispatch_batches = dispatch_batches
@@ -262,6 +277,10 @@ def __init__(
                 kwargs = self.scaler_handler.to_kwargs() if self.scaler_handler is not None else {}
                 self.scaler = torch.cuda.amp.GradScaler(**kwargs)
 
+        # Start of internal step tracking
+        self.step = 0
+        self.gradient_state = GradientState()
+
         # Internal references to the training objects
         self._optimizers = []
         self._models = []
@@ -367,6 +386,37 @@ def no_sync(self, model):
         with context():
             yield
 
+    def _do_sync(self):
+        "Sets the right `sync_gradients` context and either resets or increases `self.step`"
+        if self.gradient_state.end_of_dataloader:
+            self.step = 0
+            self.gradient_state._set_sync_gradients(True)
+        else:
+            self.step += 1
+        self.gradient_state._set_sync_gradients((self.step % self.gradient_accumulation_steps) == 0)
+
+    @property
+    def sync_gradients(self):
+        return self.gradient_state.sync_gradients
+
+    @contextmanager
+    def accumulate(self, model):
+        """
+        A context manager that will lightly wrap around and perform gradient accumulation automatically
+
+        Args:
+            model (`torch.nn.Module`):
+                PyTorch Module that was prepared with `Accelerator.prepare`
+        """
+        self._do_sync()
+        if self.sync_gradients:
+            context = contextlib.nullcontext
+        else:
+            context = self.no_sync
+
+        with context(model):
+            yield
+
     def print(self, *args, **kwargs):
         """
         Use in replacement of `print()` to only print once per server.
@@ -734,6 +784,7 @@ def backward(self, loss, **kwargs):
         """
         Use `accelerator.backward(loss)` in lieu of `loss.backward()`.
         """
+        loss /= self.gradient_accumulation_steps
         if self.distributed_type == DistributedType.DEEPSPEED:
             self.deepspeed_engine_wrapped.backward(loss, **kwargs)
         elif self.scaler is not None:
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index 806fb7ddc..524034eb1 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -18,7 +18,8 @@
 import torch
 from torch.utils.data import BatchSampler, DataLoader, IterableDataset
 
-from .state import AcceleratorState, DistributedType, is_tpu_available
+from .logging import get_logger
+from .state import AcceleratorState, DistributedType, GradientState, is_tpu_available
 from .utils import (
     RNGType,
     broadcast,
@@ -37,6 +38,7 @@
 if is_tpu_available(check_device=False):
     import torch_xla.distributed.parallel_loader as xpl
 
+logger = get_logger(__name__)
 
 # kwargs of the DataLoader in min version 1.4.0.
 _PYTORCH_DATALOADER_KWARGS = {
@@ -294,31 +296,44 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
         self.device = device
         self.rng_types = rng_types
         self.generator = generator
+        self.gradient_state = GradientState()
 
     def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
-        for batch in super().__iter__():
-            yield batch if self.device is None else send_to_device(batch, self.device)
+        self.gradient_state._set_end_of_dataloader(False)
+        dataloader_iter = super().__iter__()
+        # We iterate one batch ahead to check when we are at the end
+        try:
+            current_batch = next(dataloader_iter)
+        except StopIteration:
+            yield
+        while True:
+            try:
+                # But we still move it to the device so it is done before `StopIteration` is reached
+                if self.device is not None:
+                    current_batch = send_to_device(current_batch, self.device)
+                next_batch = next(dataloader_iter)
+                yield current_batch
+                current_batch = next_batch
+            except StopIteration:
+                self.gradient_state._set_end_of_dataloader(True)
+                yield current_batch
+                break
 
 
 class DataLoaderDispatcher(DataLoader):
     """
+    Args:
     Subclass of a PyTorch `DataLoader` that will iterate and preprocess on process 0 only, then dispatch on each
     process their part of the batch.
-
-    Args:
         split_batches (`bool`, *optional*, defaults to `False`):
             Whether the resulting `DataLoader` should split the batches of the original data loader across devices or
             yield full batches (in which case it will yield batches starting at the `process_index`-th and advancing of
-            `num_processes` batches at each iteration).
-
-            Another way to see this is that the observed batch size will be the same as the initial `dataloader` if
-            this option is set to `True`, the batch size of the initial `dataloader` multiplied by `num_processes`
-            otherwise.
-
-            Setting this option to `True` requires that the batch size of the `dataloader` is a round multiple of
-            `batch_size`.
+            `num_processes` batches at each iteration). Another way to see this is that the observed batch size will be
+            the same as the initial `dataloader` if this option is set to `True`, the batch size of the initial
+            `dataloader` multiplied by `num_processes` otherwise. Setting this option to `True` requires that the batch
+            size of the `dataloader` is a round multiple of `batch_size`.
     """
 
     def __init__(self, dataset, split_batches: bool = False, **kwargs):
@@ -338,6 +353,9 @@ def __init__(self, dataset, split_batches: bool = False, **kwargs):
         if shuffle:
             torch.utils.data.graph_settings.apply_shuffle_settings(dataset, shuffle=shuffle)
 
+        self.gradient_state = GradientState()
+        self.state = AcceleratorState()
+
     def __iter__(self):
         state = AcceleratorState()
         if state.process_index == 0:
@@ -408,12 +426,11 @@ def __iter__(self):
             yield slice_tensors(batch, data_slice)
 
     def __len__(self):
-        state = AcceleratorState()
         whole_length = super().__len__()
         if self.drop_last:
-            return whole_length // state.num_processes
+            return whole_length // self.state.num_processes
         else:
-            return math.ceil(whole_length / state.num_processes)
+            return math.ceil(whole_length / self.state.num_processes)
 
 
 def prepare_data_loader(
diff --git a/src/accelerate/optimizer.py b/src/accelerate/optimizer.py
index 046d5ce5b..c5ed66cd4 100644
--- a/src/accelerate/optimizer.py
+++ b/src/accelerate/optimizer.py
@@ -17,7 +17,7 @@
 
 import torch
 
-from .state import AcceleratorState
+from .state import AcceleratorState, GradientState
 from .utils import DistributedType, honor_type, is_torch_version, is_tpu_available
 
 
@@ -53,6 +53,7 @@ def __init__(self, optimizer, device_placement=True, scaler=None):
         self.optimizer = optimizer
         self.scaler = scaler
         self.accelerator_state = AcceleratorState()
+        self.gradient_state = GradientState()
         self.device_placement = device_placement
         self._is_overflow = False
 
@@ -101,37 +102,39 @@ def state_dict(self):
         return self.optimizer.state_dict()
 
     def zero_grad(self, set_to_none=None):
-        if is_torch_version("<", "1.7.0"):
-            if set_to_none is not None:
-                raise ValueError(
-                    "`set_to_none` for Optimizer.zero_grad` was introduced in PyTorch 1.7.0 and can't be used for "
-                    f"earlier versions (found version {torch.__version__})."
-                )
-            self.optimizer.zero_grad()
-        else:
-            accept_arg = "set_to_none" in inspect.signature(self.optimizer.zero_grad).parameters
-            if accept_arg:
-                if set_to_none is None:
-                    set_to_none = False
-                self.optimizer.zero_grad(set_to_none=set_to_none)
-            else:
+        if self.gradient_state.sync_gradients:
+            if is_torch_version("<", "1.7.0"):
                 if set_to_none is not None:
-                    raise ValueError("`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.")
+                    raise ValueError(
+                        "`set_to_none` for Optimizer.zero_grad` was introduced in PyTorch 1.7.0 and can't be used for "
+                        f"earlier versions (found version {torch.__version__})."
+                    )
                 self.optimizer.zero_grad()
+            else:
+                accept_arg = "set_to_none" in inspect.signature(self.optimizer.zero_grad).parameters
+                if accept_arg:
+                    if set_to_none is None:
+                        set_to_none = False
+                    self.optimizer.zero_grad(set_to_none=set_to_none)
+                else:
+                    if set_to_none is not None:
+                        raise ValueError("`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.")
+                    self.optimizer.zero_grad()
 
     def step(self, closure=None):
-        if self.accelerator_state.distributed_type == DistributedType.TPU:
-            optimizer_args = {"closure": closure} if closure is not None else {}
-            xm.optimizer_step(self.optimizer, optimizer_args=optimizer_args)
-        elif self.scaler is not None:
-            scale_before = self.scaler.get_scale()
-            self.scaler.step(self.optimizer, closure)
-            self.scaler.update()
-            scale_after = self.scaler.get_scale()
-            # If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
-            self._is_overflow = scale_after < scale_before
-        else:
-            self.optimizer.step(closure)
+        if self.gradient_state.sync_gradients:
+            if self.accelerator_state.distributed_type == DistributedType.TPU:
+                optimizer_args = {"closure": closure} if closure is not None else {}
+                xm.optimizer_step(self.optimizer, optimizer_args=optimizer_args)
+            elif self.scaler is not None:
+                scale_before = self.scaler.get_scale()
+                self.scaler.step(self.optimizer, closure)
+                self.scaler.update()
+                scale_after = self.scaler.get_scale()
+                # If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
+                self._is_overflow = scale_after < scale_before
+            else:
+                self.optimizer.step(closure)
 
     def _switch_parameters(self, parameters_map):
         for param_group in self.optimizer.param_groups:
diff --git a/src/accelerate/scheduler.py b/src/accelerate/scheduler.py
index 1fac5f012..cfd6d719f 100644
--- a/src/accelerate/scheduler.py
+++ b/src/accelerate/scheduler.py
@@ -12,7 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .state import AcceleratorState
+# We ignore warnings about stepping the scheduler since we step it ourselves during gradient accumulation
+
+import warnings
+
+from .state import AcceleratorState, GradientState
+
+
+warnings.filterwarnings("ignore", category=UserWarning, module="torch.optim.lr_scheduler")
 
 
 class AcceleratedScheduler:
@@ -40,6 +47,7 @@ def __init__(self, scheduler, optimizers, step_with_optimizer: bool = True, spli
         self.scheduler = scheduler
         self.optimizers = optimizers if isinstance(optimizers, (list, tuple)) else [optimizers]
         self.split_batches = split_batches
+        self.gradient_state = GradientState()
         self.step_with_optimizer = step_with_optimizer
 
     def step(self, *args, **kwargs):
@@ -52,7 +60,6 @@ def step(self, *args, **kwargs):
         for opt in self.optimizers:
             if opt.step_was_skipped:
                 return
-
         if self.split_batches:
             # Split batches -> the training dataloader batch size is not changed so one step per training step
             self.scheduler.step(*args, **kwargs)
diff --git a/src/accelerate/state.py b/src/accelerate/state.py
index 63527124c..330830eb5 100644
--- a/src/accelerate/state.py
+++ b/src/accelerate/state.py
@@ -52,6 +52,7 @@ class AcceleratorState:
     Attributes:
 
         - **device** (`torch.device`) -- The device to use.
+        - **sync_gradients** (`bool`) -- Whether to sync the gradients or not
         - **distributed_type** (`~accelerate.state.DistributedType`) -- The type of distributed environment currently
           in use.
         - **num_processes** (`int`) -- The number of processes currently launched in parallel.
@@ -209,3 +210,36 @@ def _check_initialized(self, mixed_precision=None, cpu=None):
                 raise ValueError(err.format(flag="cpu=True"))
             if mixed_precision is not None and mixed_precision != self.mixed_precision:
                 raise ValueError(err.format(flag=f"mixed_precision='{mixed_precision}'"))
+
+
+class GradientState:
+    """
+    This is a variation of a [singleton class](https://en.wikipedia.org/wiki/Singleton_pattern) in the sense that all
+    instance of `GradientState` share the same state, which is initialized on the first instantiation.
+
+    This specific state revolves around whether gradients should be synced and if we have reached the end of a prepared
+    dataloader Attributes:
+
+        - **sync_gradients** (`bool`) -- Whether the gradients should be synced
+        - **end_of_dataloader** (`bool`) -- Whether we have reached the end the current dataloader
+    """
+
+    _shared_state = {}
+
+    def __init__(self):
+        self.__dict__ = self._shared_state
+        if not getattr(self, "initialized", False):
+            self.sync_gradients = True
+            self.end_of_dataloader = False
+        self.initialized = True
+
+    def __repr__(self):
+        return f"Sync Gradients: {self.sync_gradients}\n" f"At end of current dataloader: {self.end_of_dataloader}\n"
+
+    def _set_sync_gradients(self, sync_gradients):
+        "Private function that sets whether gradients should be synchronized. Users should not have to call this."
+        self.sync_gradients = sync_gradients
+
+    def _set_end_of_dataloader(self, end_of_dataloader):
+        "Private function that sets whether the end of the current dataloader has been reached. Users should not have to call this."
+        self.end_of_dataloader = end_of_dataloader
diff --git a/src/accelerate/test_utils/scripts/test_sync.py b/src/accelerate/test_utils/scripts/test_sync.py
index b58a8c092..0b0dc7abd 100644
--- a/src/accelerate/test_utils/scripts/test_sync.py
+++ b/src/accelerate/test_utils/scripts/test_sync.py
@@ -16,6 +16,8 @@
 
 import torch
 import torch.nn.functional as F
+from torch.optim import AdamW
+from torch.optim.lr_scheduler import LambdaLR
 from torch.utils.data import DataLoader
 
 from accelerate import Accelerator
@@ -23,30 +25,61 @@
 from accelerate.utils import DistributedType, set_seed
 
 
-def step_model(model, input, target, accelerator):
+def check_model_parameters(model_a, model_b, did_step):
+    for param, grad_param in zip(model_a.parameters(), model_b.parameters()):
+        if not param.requires_grad:
+            continue
+        if not did_step:
+            # Grads should not be in sync
+            assert (
+                torch.allclose(param.grad, grad_param.grad) is False
+            ), f"Gradients in sync when they should not be:\nmodel_a grad ({param.grad}) == model_b grad ({grad_param.grad})"
+        else:
+            # Grads should be in sync
+            assert (
+                torch.allclose(param.grad, grad_param.grad) is True
+            ), f"Gradients not in sync when they should be:\nmodel_a grad ({param.grad}) != model_b grad ({grad_param.grad})"
+
+
+def step_model(model, input, target, accelerator, do_backward=True):
     model.train()
     output = model(input)
     loss = F.mse_loss(output, target.to(output.device))
-    accelerator.backward(loss)
+    if not do_backward:
+        loss /= accelerator.gradient_accumulation_steps
+        loss.backward()
+    else:
+        accelerator.backward(loss)
 
 
-def get_training_setup(accelerator):
+def get_training_setup(accelerator, sched=False):
     "Returns everything needed to perform basic training"
     set_seed(42)
     model = RegressionModel()
-    model.to(accelerator.device)
-    dset = RegressionDataset()
+    ddp_model = deepcopy(model)
+    dset = RegressionDataset(length=80)
     dataloader = DataLoader(dset, batch_size=16)
+    model.to(accelerator.device)
+    if sched:
+        opt = AdamW(params=model.parameters(), lr=1e-3)
+        ddp_opt = AdamW(params=ddp_model.parameters(), lr=1e-3)
+        sched = LambdaLR(opt, lr_lambda=lambda epoch: epoch**0.65)
+        ddp_sched = LambdaLR(ddp_opt, lr_lambda=lambda epoch: epoch**0.65)
     # Make a copy of `model`
-    ddp_model, dataloader = accelerator.prepare(deepcopy(model), dataloader)
-    # Use a single batch for all of the tests
-    ddp_input, ddp_target = next(iter(dataloader)).values()
-    return model, ddp_model, ddp_input, ddp_target
+    if sched:
+        ddp_model, ddp_opt, ddp_sched, dataloader = accelerator.prepare(ddp_model, ddp_opt, ddp_sched, dataloader)
+    else:
+        ddp_model, dataloader = accelerator.prepare(ddp_model, dataloader)
+    if sched:
+        return (model, opt, sched, dataloader, ddp_model, ddp_opt, ddp_sched)
+    return model, ddp_model, dataloader
 
 
 def test_noop_sync(accelerator):
     # Test when on a single CPU or GPU that the context manager does nothing
-    model, ddp_model, ddp_input, ddp_target = get_training_setup(accelerator)
+    model, ddp_model, dataloader = get_training_setup(accelerator)
+    # Use a single batch
+    ddp_input, ddp_target = next(iter(dataloader)).values()
     for iteration in range(3):
         # Gather the distributed inputs and targs for the base model
         input, target = accelerator.gather((ddp_input, ddp_target))
@@ -63,6 +96,7 @@ def test_noop_sync(accelerator):
             step_model(ddp_model, ddp_input, ddp_target, accelerator)
 
         # Since `no_sync` is a noop, `ddp_model` and `model` grads should always be in sync
+        check_model_parameters(model, ddp_model, True)
         for param, ddp_param in zip(model.parameters(), ddp_model.parameters()):
             if not param.requires_grad:
                 continue
@@ -77,7 +111,9 @@ def test_noop_sync(accelerator):
 
 def test_distributed_sync(accelerator):
     # Test on distributed setup that context manager behaves properly
-    model, ddp_model, ddp_input, ddp_target = get_training_setup(accelerator)
+    model, ddp_model, dataloader = get_training_setup(accelerator)
+    # Use a single batch
+    ddp_input, ddp_target = next(iter(dataloader)).values()
     for iteration in range(3):
         # Gather the distributed inputs and targs for the base model
         input, target = accelerator.gather((ddp_input, ddp_target))
@@ -113,17 +149,92 @@ def test_distributed_sync(accelerator):
         ddp_input = ddp_input[torch.randperm(16)]
 
 
+def test_gradient_accumulation():
+    accelerator = Accelerator(gradient_accumulation_steps=2)
+    # Test that context manager behaves properly
+    model, ddp_model, dataloader = get_training_setup(accelerator)
+    for iteration, batch in enumerate(dataloader):
+        ddp_input, ddp_target = batch.values()
+        # Gather the distributed inputs and targs for the base model
+        input, target = accelerator.gather((ddp_input, ddp_target))
+        input, target = input.to(accelerator.device), target.to(accelerator.device)
+        # Perform our initial ground truth step in non "DDP"
+        step_model(model, input, target, accelerator, False)
+        # Do "gradient accumulation" (noop)
+        with accelerator.accumulate(ddp_model):
+            step_model(ddp_model, ddp_input, ddp_target, accelerator)
+
+        # DDP model and model should only be in sync when not (iteration % 2 == 0)
+        for param, ddp_param in zip(model.parameters(), ddp_model.parameters()):
+            if not param.requires_grad:
+                continue
+            if ((iteration + 1) % 2 == 0) or (iteration == len(dataloader) - 1):
+                # Grads should be in sync
+                assert (
+                    torch.allclose(param.grad, ddp_param.grad) is True
+                ), f"Gradients not in sync when they should be at iteration {iteration}:\nModel grad ({param.grad}) != DDP grad ({ddp_param.grad})"
+            else:
+                # Grads should not be in sync
+                assert (
+                    torch.allclose(param.grad, ddp_param.grad) is False
+                ), f"Gradients in sync when they should not be at iteration {iteration}:\nModel grad ({param.grad}) == DDP grad ({ddp_param.grad})"
+
+        # Shuffle ddp_input on each iteration
+        torch.manual_seed(1337 + iteration)
+        ddp_input = ddp_input[torch.randperm(16)]
+
+
+def test_gradient_accumulation_with_opt_and_scheduler():
+    accelerator = Accelerator(gradient_accumulation_steps=2)
+    # Test that context manager behaves properly
+    model, opt, sched, dataloader, ddp_model, ddp_opt, ddp_sched = get_training_setup(accelerator, True)
+    for iteration, batch in enumerate(dataloader):
+        ddp_input, ddp_target = batch.values()
+        # Gather the distributed inputs and targs for the base model
+        input, target = accelerator.gather((ddp_input, ddp_target))
+        input, target = input.to(accelerator.device), target.to(accelerator.device)
+        # Perform our initial ground truth step in non "DDP"
+        model.train()
+        ddp_model.train()
+        step_model(model, input, target, accelerator, False)
+        opt.step()
+        for _ in range(accelerator.num_processes):
+            sched.step()
+        opt.zero_grad()
+        # Perform gradient accumulation under wrapper
+        with accelerator.accumulate(ddp_model):
+            step_model(ddp_model, ddp_input, ddp_target, accelerator)
+            ddp_opt.step()
+            ddp_sched.step()
+            ddp_opt.zero_grad()
+
+        # Learning rates should be the same
+        assert opt.param_groups[0]["lr"] == ddp_opt.param_groups[0]["lr"]
+        did_step = (((iteration + 1) % 2) == 0) or (iteration == (len(dataloader) - 1))
+        if accelerator.num_processes > 1:
+            check_model_parameters(model, ddp_model, did_step)
+        # Shuffle ddp_input on each iteration
+        torch.manual_seed(1337 + iteration)
+
+
 def main():
     accelerator = Accelerator()
     state = accelerator.state
     if state.distributed_type == DistributedType.NO:
         if state.local_process_index == 0:
-            print("**NOOP `no_sync` gradient accumulation**")
+            print("**Test NOOP `no_sync` context manager**")
         test_noop_sync(accelerator)
     if state.distributed_type in (DistributedType.MULTI_GPU, DistributedType.MULTI_CPU):
         if state.local_process_index == 0:
-            print("**Distributed `no_sync` gradient accumulation**")
+            print("**Test Distributed `no_sync` context manager**")
         test_distributed_sync(accelerator)
+    if state.distributed_type == DistributedType.MULTI_GPU:
+        if state.local_process_index == 0:
+            print("**Test `accumulate` gradient accumulation**")
+        test_gradient_accumulation()
+    if state.local_process_index == 0:
+        print("**Test `accumulate` gradient accumulation with optimizer and scheduler**")
+    test_gradient_accumulation_with_opt_and_scheduler()
 
 
 def _mp_fn(index):
diff --git a/tests/test_grad_sync.py b/tests/test_grad_sync.py
index c066bd41a..182d3ef95 100644
--- a/tests/test_grad_sync.py
+++ b/tests/test_grad_sync.py
@@ -36,20 +36,19 @@ def setUp(self):
         self.test_file_path = os.path.sep.join(mod_file.split(os.path.sep)[:-1] + ["scripts", "test_sync.py"])
 
     @require_cpu
-    def test_gradient_sync_single_cpu_noop(self):
-        debug_launcher(test_sync.main)
+    def test_gradient_sync_cpu_noop(self):
         debug_launcher(test_sync.main, num_processes=1)
 
     @require_cpu
-    def test_gradient_sync_multi_cpu(self):
+    def test_gradient_sync_cpu_multi(self):
         debug_launcher(test_sync.main)
 
     @require_single_gpu
-    def test_gradient_sync_single_gpu(self):
-        debug_launcher(test_sync.main, num_processes=1)
+    def test_gradient_sync_gpu(self):
+        test_sync.main()
 
     @require_multi_gpu
-    def test_gradient_sync_multi_gpu(self):
+    def test_gradient_sync_gpu_multi(self):
         print(f"Found {torch.cuda.device_count()} devices.")
         cmd = get_launch_prefix() + [f"--nproc_per_node={torch.cuda.device_count()}", self.test_file_path]
         with patch_environment(omp_num_threads=1):
diff --git a/tests/test_kwargs_handlers.py b/tests/test_kwargs_handlers.py
index 8b438d1c6..d8c893234 100644
--- a/tests/test_kwargs_handlers.py
+++ b/tests/test_kwargs_handlers.py
@@ -21,6 +21,7 @@
 import torch
 
 from accelerate import Accelerator, DistributedDataParallelKwargs, GradScalerKwargs
+from accelerate.state import AcceleratorState
 from accelerate.test_utils import execute_subprocess_async, require_cuda, require_multi_gpu
 from accelerate.utils import KwargsHandler
 
@@ -44,7 +45,8 @@ def test_kwargs_handler(self):
     def test_grad_scaler_kwargs(self):
         # If no defaults are changed, `to_kwargs` returns an empty dict.
         scaler_handler = GradScalerKwargs(init_scale=1024, growth_factor=2)
-        accelerator = Accelerator(fp16=True, kwargs_handlers=[scaler_handler])
+        AcceleratorState._reset_state()
+        accelerator = Accelerator(mixed_precision="fp16", kwargs_handlers=[scaler_handler])
         print(accelerator.use_fp16)
         scaler = accelerator.scaler
 


Review comments: body: Note that the first part of the test is implied by the second ;-)
diff_hunk: @@ -367,6 +380,39 @@ def no_sync(self, model):
         with context():
             yield
 
+    @property
+    def _do_sync(self) -> bool:
+        "Checks if gradients should be synchronized and the optimizers + schedulers should be stepped"
+        if self.state.end_of_dataloader:
+            self.step = 0
+            return True
+        self.step += 1
+        if (self.gradient_accumulation_steps == 1) or ((self.step % self.gradient_accumulation_steps) == 0):
body: This shouldn't be in the state, the state only should contain the environment info, not some things that are subject to change. So store it directly in self.
diff_hunk: @@ -367,6 +380,39 @@ def no_sync(self, model):
         with context():
             yield
 
+    @property
+    def _do_sync(self) -> bool:
+        "Checks if gradients should be synchronized and the optimizers + schedulers should be stepped"
+        if self.state.end_of_dataloader:
body: Same here, this should be stored somewhere else.
diff_hunk: @@ -367,6 +380,39 @@ def no_sync(self, model):
         with context():
             yield
 
+    @property
+    def _do_sync(self) -> bool:
+        "Checks if gradients should be synchronized and the optimizers + schedulers should be stepped"
+        if self.state.end_of_dataloader:
+            self.step = 0
+            return True
+        self.step += 1
+        if (self.gradient_accumulation_steps == 1) or ((self.step % self.gradient_accumulation_steps) == 0):
+            return True
+        return False
+
+    @contextmanager
+    def accumulate(self, model):
+        """
+        A context manager that will lightly wrap around and perform gradient accumulation automatically
+
+        Args:
+            model (`torch.nn.Module`):
+                PyTorch Module that was prepared with `Accelerator.prepare`
+            dataloader (`torch.utils.DataLoader`)
+                PyTorch DataLoader that was prepared with `Accelerator.prepare`
+        """
+
+        if self._do_sync:
+            context = contextlib.nullcontext
+            AcceleratorState._set_state("sync_gradients", True)
+        else:
+            context = self.no_sync
+            AcceleratorState._set_state("sync_gradients", False)
body: ```suggestion
        gradient_accumulation_steps (`int`, *optional*, defaults to 1):
```
diff_hunk: @@ -92,6 +92,9 @@ class Accelerator:
             default to the value in the environment variable `MIXED_PRECISION`, which will use the default value in the
             accelerate config of the current system or the flag passed with the `accelerate.launch` command. 'fp16'
             requires pytorch 1.6 or higher. 'bf16' requires pytorch 1.10 or higher.
+        gradient_accumulation_steps (`int`, *optional*):
body: ```suggestion
            [`Accelerator.accumulate`].
```
Link to the doc.
diff_hunk: @@ -92,6 +92,9 @@ class Accelerator:
             default to the value in the environment variable `MIXED_PRECISION`, which will use the default value in the
             accelerate config of the current system or the flag passed with the `accelerate.launch` command. 'fp16'
             requires pytorch 1.6 or higher. 'bf16' requires pytorch 1.10 or higher.
+        gradient_accumulation_steps (`int`, *optional*):
+            The number of steps that should pass before gradients are accumulated. Should be combined with
+            `Accelerator.accumulate`
body: I think the class should have a static method to set this.
diff_hunk: @@ -367,6 +380,37 @@ def no_sync(self, model):
         with context():
             yield
 
+    def _do_sync(self) -> bool:
+        "Checks if gradients should be synchronized and the optimizers + schedulers should be stepped"
+        if GradientState().end_of_dataloader:
+            self.step = 0
+            return True
+        else:
+            self.step += 1
+        return (self.step % self.gradient_accumulation_steps) == 0
+
+    @contextmanager
+    def accumulate(self, model):
+        """
+        A context manager that will lightly wrap around and perform gradient accumulation automatically
+
+        Args:
+            model (`torch.nn.Module`):
+                PyTorch Module that was prepared with `Accelerator.prepare`
+            dataloader (`torch.utils.DataLoader`)
+                PyTorch DataLoader that was prepared with `Accelerator.prepare`
+        """
+
+        if self._do_sync():
+            context = contextlib.nullcontext
+            GradientState._set_state("sync_gradients", True)
body: Same here
diff_hunk: @@ -367,6 +380,37 @@ def no_sync(self, model):
         with context():
             yield
 
+    def _do_sync(self) -> bool:
+        "Checks if gradients should be synchronized and the optimizers + schedulers should be stepped"
+        if GradientState().end_of_dataloader:
+            self.step = 0
+            return True
+        else:
+            self.step += 1
+        return (self.step % self.gradient_accumulation_steps) == 0
+
+    @contextmanager
+    def accumulate(self, model):
+        """
+        A context manager that will lightly wrap around and perform gradient accumulation automatically
+
+        Args:
+            model (`torch.nn.Module`):
+                PyTorch Module that was prepared with `Accelerator.prepare`
+            dataloader (`torch.utils.DataLoader`)
+                PyTorch DataLoader that was prepared with `Accelerator.prepare`
+        """
+
+        if self._do_sync():
+            context = contextlib.nullcontext
+            GradientState._set_state("sync_gradients", True)
+        else:
+            context = self.no_sync
+            GradientState._set_state("sync_gradients", False)
body: The dataloader may still have no length, so set to `False at the start of `__iter__` and to `True` at the end.
diff_hunk: @@ -298,7 +298,8 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
     def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
-        for batch in super().__iter__():
+        for i, batch in enumerate(super().__iter__()):
+            GradientState._set_state("end_of_dataloader", i == (len(self) - 1))
body: ```suggestion
        GradientState._set_state("end_of_dataloader", True)
```
Like this.

diff_hunk: @@ -406,6 +408,8 @@ def __iter__(self):
 
             data_slice = slice(state.process_index * batch_size, (state.process_index + 1) * batch_size)
             yield slice_tensors(batch, data_slice)
+            current_idx += 1
+            GradientState._set_state("end_of_dataloader", current_idx == len(self) - 1)
body: For the scheduler, I'm not entirely sure how to proceed: we step for each process to account for Distributed training, not sure if we should step all the time or just when synchronizing gradients (since the length set for the scheduler probably does not account for gradient accumulation steps).
diff_hunk: @@ -52,18 +53,18 @@ def step(self, *args, **kwargs):
         for opt in self.optimizers:
             if opt.step_was_skipped:
                 return
-
-        if self.split_batches:
-            # Split batches -> the training dataloader batch size is not changed so one step per training step
-            self.scheduler.step(*args, **kwargs)
-        else:
-            # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
-            # num_processes steps per training step
-            num_processes = AcceleratorState().num_processes
-            for _ in range(num_processes):
-                # Special case when using OneCycle and `drop_last` was not used
-                if getattr(self.scheduler, "total_steps", 0) <= self.scheduler.last_epoch:
-                    self.scheduler.step(*args, **kwargs)
+        if self.gradient_state.sync_gradients:
+            if self.split_batches:
+                # Split batches -> the training dataloader batch size is not changed so one step per training step
+                self.scheduler.step(*args, **kwargs)
+            else:
+                # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
+                # num_processes steps per training step
+                num_processes = AcceleratorState().num_processes
+                for _ in range(num_processes):
+                    # Special case when using OneCycle and `drop_last` was not used
+                    if getattr(self.scheduler, "total_steps", 0) <= self.scheduler.last_epoch:
+                        self.scheduler.step(*args, **kwargs)
body: Add two static methods to quickly set `sync_gradients` and `end_of_dataloader` (also wondering why you can't jsut set them directly on an instance? It should automatically update everything no?)
diff_hunk: @@ -209,3 +210,35 @@ def _check_initialized(self, mixed_precision=None, cpu=None):
                 raise ValueError(err.format(flag="cpu=True"))
             if mixed_precision is not None and mixed_precision != self.mixed_precision:
                 raise ValueError(err.format(flag=f"mixed_precision='{mixed_precision}'"))
+
+
+class GradientState:
+    """
+    This is a variation of a [singleton class](https://en.wikipedia.org/wiki/Singleton_pattern) in the sense that all
+    instance of `GradientState` share the same state, which is initialized on the first instantiation.
+
+    This specific state revolves around whether gradients should be synced and if we have reached the end of a prepared
+    dataloader Attributes:
+
+        - **sync_gradients** (`bool`) -- Whether the gradients should be synced
+        - **end_of_dataloader** (`bool`) -- Whether we have reached the end the current dataloader
+    """
+
+    _shared_state = {}
+
+    def __init__(self):
+        self.__dict__ = self._shared_state
+        if not getattr(self, "initialized", False):
+            self.sync_gradients = True
+            self.end_of_dataloader = False
+        self.initialized = True
+
+    @staticmethod
+    def _set_state(key, val):
+        "Sets `key` to `val` in GradientState"
+        if GradientState._shared_state != {} and key not in GradientState._shared_state.keys():
+            raise KeyError(f"{key} is not a valid key of `GradientState`, {GradientState._shared_state.keys()}")
+        GradientState._shared_state[key] = val
+
body: Resolved: we update every time. 
diff_hunk: @@ -52,18 +53,18 @@ def step(self, *args, **kwargs):
         for opt in self.optimizers:
             if opt.step_was_skipped:
                 return
-
-        if self.split_batches:
-            # Split batches -> the training dataloader batch size is not changed so one step per training step
-            self.scheduler.step(*args, **kwargs)
-        else:
-            # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
-            # num_processes steps per training step
-            num_processes = AcceleratorState().num_processes
-            for _ in range(num_processes):
-                # Special case when using OneCycle and `drop_last` was not used
-                if getattr(self.scheduler, "total_steps", 0) <= self.scheduler.last_epoch:
-                    self.scheduler.step(*args, **kwargs)
+        if self.gradient_state.sync_gradients:
+            if self.split_batches:
+                # Split batches -> the training dataloader batch size is not changed so one step per training step
+                self.scheduler.step(*args, **kwargs)
+            else:
+                # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
+                # num_processes steps per training step
+                num_processes = AcceleratorState().num_processes
+                for _ in range(num_processes):
+                    # Special case when using OneCycle and `drop_last` was not used
+                    if getattr(self.scheduler, "total_steps", 0) <= self.scheduler.last_epoch:
+                        self.scheduler.step(*args, **kwargs)
body: This is unnecessary complex. Why are you not doing `self.gradient_state._set_end_of_dataloader(True)` at the end of `iter`?
diff_hunk: @@ -294,12 +296,26 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
         self.device = device
         self.rng_types = rng_types
         self.generator = generator
+        self.gradient_state = GradientState()
 
     def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
+        self.gradient_state._set_end_of_dataloader(False)
+        _current_batch = 0
+        _dataloader_length = len(self) - 1 if hasattr(self, "__len__") else False
         for batch in super().__iter__():
+            if _dataloader_length:
+                self.gradient_state._set_end_of_dataloader(_current_batch == _dataloader_length)
+                _current_batch += 1
+            else:
+                self.gradient_state._set_end_of_dataloader(False)
             yield batch if self.device is None else send_to_device(batch, self.device)
+        if not self.gradient_state.end_of_dataloader:
+            self.gradient_state._set_end_of_dataloader(True)
+            logger.warn(
+                "Warning! DataLoader had no length and finished iterating. Backwards pass and stepping must be manually performed one last time."
+            )
body: Not sure about this second test. If `step_with_optimizer=True` we then only step for sync gradients, which is not what we want.
diff_hunk: @@ -40,10 +47,11 @@ def __init__(self, scheduler, optimizers, step_with_optimizer: bool = True, spli
         self.scheduler = scheduler
         self.optimizers = optimizers if isinstance(optimizers, (list, tuple)) else [optimizers]
         self.split_batches = split_batches
+        self.gradient_state = GradientState()
         self.step_with_optimizer = step_with_optimizer
 
     def step(self, *args, **kwargs):
-        if not self.step_with_optimizer:
+        if not self.step_with_optimizer or not self.gradient_state.sync_gradients:
body: The filter should be done in the scheduler, not the test scripts.
diff_hunk: @@ -12,41 +12,62 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import warnings
 from copy import deepcopy
 
 import torch
 import torch.nn.functional as F
+from torch.optim import AdamW
+from torch.optim.lr_scheduler import LambdaLR
 from torch.utils.data import DataLoader
 
 from accelerate import Accelerator
 from accelerate.test_utils import RegressionDataset, RegressionModel
 from accelerate.utils import DistributedType, set_seed
 
 
-def step_model(model, input, target, accelerator):
+warnings.filterwarnings("ignore", category=UserWarning, module="torch.optim.lr_scheduler")
body: This check has to happen as the last batch is being pushed out, otherwise it doesn't occur until outside the iteration loop. 

Example in python of what doesn't happen:

```python
a = [0,1,2,3]
class A:
    def __init__(self,a):
        self.a,self.is_end = a,False
    def __iter__(self):
        for item in a: yield item
        self.is_end = True
aa = A(a)
for i,item in enumerate(aa):
    print(f'At end at {i}?: {aa.is_end}')
print(aa.is_end)
```
Result:
```
At end at 0?: False
At end at 1?: False
At end at 2?: False
At end at 3?: False
True
```
What we want to have happen:
```python
class A:
    def __init__(self,a):
        self.a,self.is_end = a,False
    def __iter__(self):
        for i,item in enumerate(a):
            if i == len(a)-1:
                self.is_end = True 
            yield item
        self.is_end = True
```
```
At end at 0?: False
At end at 1?: False
At end at 2?: False
At end at 3?: True
True
```
Since we cannot check the length sometimes, this extra step + warning needs to be done.
diff_hunk: @@ -294,12 +296,26 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
         self.device = device
         self.rng_types = rng_types
         self.generator = generator
+        self.gradient_state = GradientState()
 
     def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
+        self.gradient_state._set_end_of_dataloader(False)
+        _current_batch = 0
+        _dataloader_length = len(self) - 1 if hasattr(self, "__len__") else False
         for batch in super().__iter__():
+            if _dataloader_length:
+                self.gradient_state._set_end_of_dataloader(_current_batch == _dataloader_length)
+                _current_batch += 1
+            else:
+                self.gradient_state._set_end_of_dataloader(False)
             yield batch if self.device is None else send_to_device(batch, self.device)
+        if not self.gradient_state.end_of_dataloader:
+            self.gradient_state._set_end_of_dataloader(True)
+            logger.warn(
+                "Warning! DataLoader had no length and finished iterating. Backwards pass and stepping must be manually performed one last time."
+            )
body: I think there's a way to do this easier, will toy with it today. 
diff_hunk: @@ -294,12 +296,26 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
         self.device = device
         self.rng_types = rng_types
         self.generator = generator
+        self.gradient_state = GradientState()
 
     def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
+        self.gradient_state._set_end_of_dataloader(False)
+        _current_batch = 0
+        _dataloader_length = len(self) - 1 if hasattr(self, "__len__") else False
         for batch in super().__iter__():
+            if _dataloader_length:
+                self.gradient_state._set_end_of_dataloader(_current_batch == _dataloader_length)
+                _current_batch += 1
+            else:
+                self.gradient_state._set_end_of_dataloader(False)
             yield batch if self.device is None else send_to_device(batch, self.device)
+        if not self.gradient_state.end_of_dataloader:
+            self.gradient_state._set_end_of_dataloader(True)
+            logger.warn(
+                "Warning! DataLoader had no length and finished iterating. Backwards pass and stepping must be manually performed one last time."
+            )
body: Much simpler way for us to iterate over the dataloader, and given we only move to device right before we yield there shouldn't be too much trouble with multiple batches in memory at one time. If so let me know.
diff_hunk: @@ -294,12 +296,27 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
         self.device = device
         self.rng_types = rng_types
         self.generator = generator
+        self.gradient_state = GradientState()
 
     def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
-        for batch in super().__iter__():
-            yield batch if self.device is None else send_to_device(batch, self.device)
+        self.gradient_state._set_end_of_dataloader(False)
+        dataloader_iter = super().__iter__()
+        # We iterate one batch ahead to check when we are at the end
+        current_batch = next(dataloader_iter)
+        while True:
+            try:
+                # But we still move it to the device so it is done before `StopIteration` is reached
+                if self.device is not None:
+                    current_batch = send_to_device(current_batch, self.device)
+                next_batch = next(dataloader_iter)
+                yield current_batch
+                current_batch = next_batch
+            except StopIteration:
+                self.gradient_state._set_end_of_dataloader(True)
+                yield current_batch
+                break
body: We should have a try except here too in case of an empty dataloader.
diff_hunk: @@ -294,12 +296,27 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
         self.device = device
         self.rng_types = rng_types
         self.generator = generator
+        self.gradient_state = GradientState()
 
     def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
-        for batch in super().__iter__():
-            yield batch if self.device is None else send_to_device(batch, self.device)
+        self.gradient_state._set_end_of_dataloader(False)
+        dataloader_iter = super().__iter__()
+        # We iterate one batch ahead to check when we are at the end
+        current_batch = next(dataloader_iter)
body: Very clever!
diff_hunk: @@ -294,12 +296,27 @@ def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwarg
         self.device = device
         self.rng_types = rng_types
         self.generator = generator
+        self.gradient_state = GradientState()
 
     def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
-        for batch in super().__iter__():
-            yield batch if self.device is None else send_to_device(batch, self.device)
+        self.gradient_state._set_end_of_dataloader(False)
+        dataloader_iter = super().__iter__()
+        # We iterate one batch ahead to check when we are at the end
+        current_batch = next(dataloader_iter)
+        while True:
+            try:
+                # But we still move it to the device so it is done before `StopIteration` is reached
+                if self.device is not None:
+                    current_batch = send_to_device(current_batch, self.device)
+                next_batch = next(dataloader_iter)
+                yield current_batch
+                current_batch = next_batch
+            except StopIteration:
+                self.gradient_state._set_end_of_dataloader(True)
+                yield current_batch
+                break
body: Small docstrings here would be great :-) (with a mention that it's private for a good reason and shouldn't be touched by the user directly!)
diff_hunk: @@ -209,3 +210,34 @@ def _check_initialized(self, mixed_precision=None, cpu=None):
                 raise ValueError(err.format(flag="cpu=True"))
             if mixed_precision is not None and mixed_precision != self.mixed_precision:
                 raise ValueError(err.format(flag=f"mixed_precision='{mixed_precision}'"))
+
+
+class GradientState:
+    """
+    This is a variation of a [singleton class](https://en.wikipedia.org/wiki/Singleton_pattern) in the sense that all
+    instance of `GradientState` share the same state, which is initialized on the first instantiation.
+
+    This specific state revolves around whether gradients should be synced and if we have reached the end of a prepared
+    dataloader Attributes:
+
+        - **sync_gradients** (`bool`) -- Whether the gradients should be synced
+        - **end_of_dataloader** (`bool`) -- Whether we have reached the end the current dataloader
+    """
+
+    _shared_state = {}
+
+    def __init__(self):
+        self.__dict__ = self._shared_state
+        if not getattr(self, "initialized", False):
+            self.sync_gradients = True
+            self.end_of_dataloader = False
+        self.initialized = True
+
+    def __repr__(self):
+        return f"Sync Gradients: {self.sync_gradients}\n" f"At end of current dataloader: {self.end_of_dataloader}\n"
+
+    def _set_sync_gradients(self, sync_gradients):
+        self.sync_gradients = sync_gradients
+
+    def _set_end_of_dataloader(self, end_of_dataloader):
+        self.end_of_dataloader = end_of_dataloader
body: No we shouldn't raise here, just yield nothing. A training loop over an empty dataloader works, just doesn't do anything.
We can put a warning however, just in case the user didn't intend that.
diff_hunk: @@ -304,7 +304,10 @@ def __iter__(self):
         self.gradient_state._set_end_of_dataloader(False)
         dataloader_iter = super().__iter__()
         # We iterate one batch ahead to check when we are at the end
-        current_batch = next(dataloader_iter)
+        try:
+            current_batch = next(dataloader_iter)
+        except StopIteration:
+            raise StopIteration("Tried iterating over an empty dataloader")
body: ```suggestion
This will also automatically ensure the gradients are synced or unsynced when on multi-device training, check if the step should
```
diff_hunk: @@ -38,4 +38,23 @@ should search for and replace by the corresponding methods of your `accelerator`
 - Use [`~Accelerator.clip_grad_norm_`] instead of `torch.nn.utils.clip_grad_norm_` and
   [`~Accelerator.clip_grad_value_`] instead of `torch.nn.utils.clip_grad_value_`.
 
+To perform gradient accumulation use [`~Accelerator.accumulate`] and specify a `gradient_accumulation_steps`. 
+This will also automatically ensure the gradients are synced or unsynced when on multi-node, check if the step should
body: Use the proper syntax for a warning here
```suggestion
<Tip warning={true}>

Using this with `dispatch_batches=True`  (which is the default for iterable datasets) is currently not supported.

</Tip>
```
diff_hunk: @@ -38,4 +38,23 @@ should search for and replace by the corresponding methods of your `accelerator`
 - Use [`~Accelerator.clip_grad_norm_`] instead of `torch.nn.utils.clip_grad_norm_` and
   [`~Accelerator.clip_grad_value_`] instead of `torch.nn.utils.clip_grad_value_`.
 
+To perform gradient accumulation use [`~Accelerator.accumulate`] and specify a `gradient_accumulation_steps`. 
+This will also automatically ensure the gradients are synced or unsynced when on multi-node, check if the step should
+actually be performed, and auto-scale the loss:
+
+```python
+accelerator = Accelerator(gradient_accumulation_steps=2)
+
+for (input, label) in enumerate(training_dataloader):
+    with accelerator.accumulate(model):
+        predictions = model(input)
+        loss = loss_function(predictions, labels)
+        accelerator.backward(loss)
+        optimizer.step()
+        scheduler.step()
+        optimizer.zero_grad()
+```
+
+**Note**: Using this with `dispatch_batches=True` is currently not supported.
body: ```suggestion
        gradient_accumulation_steps (`int`, *optional*, default to 1):
```
diff_hunk: @@ -92,6 +92,9 @@ class Accelerator:
             default to the value in the environment variable `MIXED_PRECISION`, which will use the default value in the
             accelerate config of the current system or the flag passed with the `accelerate.launch` command. 'fp16'
             requires pytorch 1.6 or higher. 'bf16' requires pytorch 1.10 or higher.
+        gradient_accumulation_steps (`int`, *optional*):
body: ```suggestion
            The number of steps that should pass before gradients are accumulated. A number > 1 should be combined with
```
diff_hunk: @@ -92,6 +92,9 @@ class Accelerator:
             default to the value in the environment variable `MIXED_PRECISION`, which will use the default value in the
             accelerate config of the current system or the flag passed with the `accelerate.launch` command. 'fp16'
             requires pytorch 1.6 or higher. 'bf16' requires pytorch 1.10 or higher.
+        gradient_accumulation_steps (`int`, *optional*):
+            The number of steps that should pass before gradients are accumulated. Should be combined with

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._