diff --git a/examples/by_feature/checkpointing.py b/examples/by_feature/checkpointing.py
index 5e364491e..3860875e9 100644
--- a/examples/by_feature/checkpointing.py
+++ b/examples/by_feature/checkpointing.py
@@ -241,7 +241,7 @@ def training_function(config, args):
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
             # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), eval_dataloader)
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/by_feature/cross_validation.py b/examples/by_feature/cross_validation.py
index 79a207fa8..cd5b2fd02 100644
--- a/examples/by_feature/cross_validation.py
+++ b/examples/by_feature/cross_validation.py
@@ -207,9 +207,7 @@ def training_function(config, args):
                 with torch.no_grad():
                     outputs = model(**batch)
                 predictions = outputs.logits.argmax(dim=-1)
-                predictions, references = accelerator.gather_for_metrics(
-                    (predictions, batch["labels"]), eval_dataloader
-                )
+                predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
                 metric.add_batch(
                     predictions=predictions,
                     references=references,
@@ -228,7 +226,7 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits
-            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), test_dataloader)
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             fold_predictions.append(predictions.cpu())
             if i == 0:
                 # We need all of the test predictions
diff --git a/examples/by_feature/fsdp_with_peak_mem_tracking.py b/examples/by_feature/fsdp_with_peak_mem_tracking.py
index 8046b4557..1426dd0cd 100644
--- a/examples/by_feature/fsdp_with_peak_mem_tracking.py
+++ b/examples/by_feature/fsdp_with_peak_mem_tracking.py
@@ -279,9 +279,7 @@ def collate_fn(examples):
                     outputs = model(**batch)
                 predictions = outputs.logits.argmax(dim=-1)
                 # It is slightly faster to call this once, than multiple times
-                predictions, references = accelerator.gather_for_metrics(
-                    (predictions, batch["labels"]), eval_dataloader
-                )
+                predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
                 metric.add_batch(
                     predictions=predictions,
                     references=references,
diff --git a/examples/by_feature/gradient_accumulation.py b/examples/by_feature/gradient_accumulation.py
index fe3d948f5..2e6de6a32 100644
--- a/examples/by_feature/gradient_accumulation.py
+++ b/examples/by_feature/gradient_accumulation.py
@@ -174,7 +174,7 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
-            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), eval_dataloader)
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/by_feature/memory.py b/examples/by_feature/memory.py
index f71283561..6f7cf00fd 100644
--- a/examples/by_feature/memory.py
+++ b/examples/by_feature/memory.py
@@ -186,9 +186,7 @@ def inner_training_loop(batch_size):
                 with torch.no_grad():
                     outputs = model(**batch)
                 predictions = outputs.logits.argmax(dim=-1)
-                predictions, references = accelerator.gather_for_metrics(
-                    (predictions, batch["labels"]), eval_dataloader
-                )
+                predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
                 metric.add_batch(
                     predictions=predictions,
                     references=references,
diff --git a/examples/by_feature/multi_process_metrics.py b/examples/by_feature/multi_process_metrics.py
index d69c0bc90..f39b064d6 100644
--- a/examples/by_feature/multi_process_metrics.py
+++ b/examples/by_feature/multi_process_metrics.py
@@ -191,7 +191,7 @@ def training_function(config, args):
                     # Otherwise we add the number of samples seen
                     samples_seen += references.shape[0]
             # All of this can be avoided if you use `Accelerator.gather_for_metrics` instead of `Accelerator.gather`:
-            # accelerator.gather_for_metrics((predictions, references), eval_dataloader)
+            # accelerator.gather_for_metrics((predictions, references))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/by_feature/tracking.py b/examples/by_feature/tracking.py
index d453b770c..d8c985e88 100644
--- a/examples/by_feature/tracking.py
+++ b/examples/by_feature/tracking.py
@@ -199,7 +199,7 @@ def training_function(config, args):
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
             # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), eval_dataloader)
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/complete_cv_example.py b/examples/complete_cv_example.py
index 17cc83170..ee051a0c5 100644
--- a/examples/complete_cv_example.py
+++ b/examples/complete_cv_example.py
@@ -239,7 +239,7 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(inputs)
             predictions = outputs.argmax(dim=-1)
-            predictions, labels = accelerator.gather_for_metrics((predictions, batch["label"]), eval_dataloader)
+            predictions, labels = accelerator.gather_for_metrics((predictions, batch["label"]))
             accurate_preds = predictions == labels
             accurate += accurate_preds.long().sum()
 
diff --git a/examples/complete_nlp_example.py b/examples/complete_nlp_example.py
index dc0cf43ed..88d379c8e 100644
--- a/examples/complete_nlp_example.py
+++ b/examples/complete_nlp_example.py
@@ -217,7 +217,7 @@ def collate_fn(examples):
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
             # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), eval_dataloader)
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/cv_example.py b/examples/cv_example.py
index aa97ce2d7..6663b2fe9 100644
--- a/examples/cv_example.py
+++ b/examples/cv_example.py
@@ -173,7 +173,7 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(inputs)
             predictions = outputs.argmax(dim=-1)
-            predictions, labels = accelerator.gather_for_metrics((predictions, batch["label"]), eval_dataloader)
+            predictions, labels = accelerator.gather_for_metrics((predictions, batch["label"]))
             accurate_preds = predictions == labels
             num_elems += accurate_preds.shape[0]
             accurate += accurate_preds.long().sum()
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index d64ab3bb7..2ef18598b 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -158,7 +158,7 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
-            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), eval_dataloader)
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index acf960026..977cd5d24 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -934,7 +934,7 @@ def gather(self, tensor):
         """
         return gather(tensor)
 
-    def gather_for_metrics(self, tensor, dataloader):
+    def gather_for_metrics(self, tensor):
         """
         Gathers `tensor` and potentially drops duplicates in the last batch if on a distributed system. Should be used
         for gathering the inputs and targets for metric calculation.
@@ -942,17 +942,20 @@ def gather_for_metrics(self, tensor, dataloader):
         Args:
             tensor (`torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`):
                 The tensors for calculating metrics across all processes.
-            dataloader (`torch.utils.data.DataLoader`):
-                A dataloader prepared with `Accelerator.prepare`
         """
         tensor = self.gather(tensor)
         if self.use_distributed:
+            if self.gradient_state.remainder == -1:
+                logger.info(
+                    "The used dataset had no length, returning gathered tensors. You should drop the remainder yourself."
+                )
+                return tensor
             try:
                 # Then see if we're on the last batch of our eval dataloader
                 if self.gradient_state.end_of_dataloader:
                     # Last batch needs to be truncated on distributed systems as it contains additional samples
                     def _adjust_samples(tensor):
-                        return tensor[: dataloader.total_dataset_length - self.gradient_state.samples_seen]
+                        return tensor[: self.gradient_state.remainder]
 
                     return recursively_apply(_adjust_samples, tensor)
                 else:
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index ba3338546..531f8eff9 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -138,6 +138,10 @@ def __init__(
         self.batch_size = batch_sampler.batch_size
         self.drop_last = batch_sampler.drop_last
 
+    @property
+    def total_length(self):
+        return len(self.batch_sampler)
+
     def __len__(self):
         if self.split_batches:
             return len(self.batch_sampler)
@@ -332,7 +336,12 @@ def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
         self.gradient_state._set_end_of_dataloader(False)
-        self.gradient_state._set_samples_seen(0)
+        try:
+            length = getattr(self.dataset, "total_dataset_length", len(self.dataset))
+            self.gradient_state._set_remainder(length % self.total_batch_size)
+        except Exception:
+            # We can safely pass because the default is -1
+            pass
         dataloader_iter = super().__iter__()
         # We iterate one batch ahead to check when we are at the end
         try:
@@ -346,12 +355,10 @@ def __iter__(self):
                 if self.device is not None:
                     current_batch = send_to_device(current_batch, self.device)
                 next_batch = next(dataloader_iter)
-                self.gradient_state._iterate_samples_seen(find_batch_size(current_batch))
                 yield current_batch
                 current_batch = next_batch
             except StopIteration:
                 self.gradient_state._set_end_of_dataloader(True)
-                self.gradient_state._iterate_samples_seen(find_batch_size(current_batch))
                 yield current_batch
                 break
 
@@ -365,7 +372,10 @@ def total_batch_size(self):
 
     @property
     def total_dataset_length(self):
-        return len(self.dataset)
+        if hasattr("total_length", self.dataset):
+            return self.dataset.total_length
+        else:
+            return len(self.dataset)
 
 
 class DataLoaderDispatcher(DataLoader):
@@ -453,7 +463,11 @@ def _fetch_batches(self, iterator):
 
     def __iter__(self):
         self.gradient_state._set_end_of_dataloader(False)
-        self.gradient_state._set_samples_seen(0)
+        try:
+            length = getattr(self.dataset, "total_dataset_length", len(self.dataset))
+            self.gradient_state._set_remainder(length % self.batch_size)
+        except:
+            self.gradient_state._set_remainder(-1)
         main_iterator = None
         if self.state.process_index == 0:
             # We only iterate through the DataLoader on process 0.
@@ -487,11 +501,9 @@ def __iter__(self):
             next_batch, next_batch_info, next_skip = self._fetch_batches(main_iterator)
             batch = slice_tensors(batch, data_slice)
             if not self._stop_iteration:
-                self.gradient_state._iterate_samples_seen(batch_size)
                 yield batch
                 batch, batch_info, skip = next_batch, next_batch_info, next_skip
             else:
-                self.gradient_state._iterate_samples_seen(batch_size)
                 self.gradient_state._set_end_of_dataloader(True)
                 yield batch
                 break
diff --git a/src/accelerate/state.py b/src/accelerate/state.py
index 06a25bc4e..27a627ae9 100644
--- a/src/accelerate/state.py
+++ b/src/accelerate/state.py
@@ -258,6 +258,7 @@ class GradientState:
 
         - **sync_gradients** (`bool`) -- Whether the gradients should be synced
         - **end_of_dataloader** (`bool`) -- Whether we have reached the end the current dataloader
+        - **remainder** (`int`) -- The number of extra samples that were added from padding the dataloader
     """
 
     _shared_state = {}
@@ -267,14 +268,14 @@ def __init__(self):
         if not getattr(self, "initialized", False):
             self.sync_gradients = True
             self.end_of_dataloader = False
-            self.samples_seen = 0
+            self.remainder = -1
         self.initialized = True
 
     def __repr__(self):
         return (
             f"Sync Gradients: {self.sync_gradients}\n"
             f"At end of current dataloader: {self.end_of_dataloader}\n"
-            f"Samples seen: {self.samples_seen}"
+            f"Extra samples added: {self.remainder}"
         )
 
     def _set_sync_gradients(self, sync_gradients):
@@ -285,10 +286,6 @@ def _set_end_of_dataloader(self, end_of_dataloader):
         "Private function that sets whether the end of the current dataloader has been reached. Users should not have to call this."
         self.end_of_dataloader = end_of_dataloader
 
-    def _set_samples_seen(self, samples_seen):
-        "Private function that sets the number of samples iterated over. Users should not have to call this."
-        self.samples_seen = samples_seen
-
-    def _iterate_samples_seen(self, iteration: int = 1):
-        "Private function that iterates the number of samples seen by an iteration. Users should not have to call this."
-        self._set_samples_seen(self.samples_seen + iteration)
+    def _set_remainder(self, remainder):
+        "Private function that sets the number of remaining samples at the end of the dataloader"
+        self.remainder = remainder
diff --git a/src/accelerate/test_utils/scripts/test_metrics.py b/src/accelerate/test_utils/scripts/test_metrics.py
index c426d1fc5..8f057ccc5 100644
--- a/src/accelerate/test_utils/scripts/test_metrics.py
+++ b/src/accelerate/test_utils/scripts/test_metrics.py
@@ -22,50 +22,63 @@
 from accelerate.utils import set_seed
 
 
-def get_setup(accelerator):
+def get_setup(accelerator, num_samples=82):
     "Returns everything needed to perform basic training"
     set_seed(42)
     model = RegressionModel()
     ddp_model = deepcopy(model)
-    dset = RegressionDataset(length=80)
+    dset = RegressionDataset(length=num_samples)
     dataloader = DataLoader(dset, batch_size=16)
     model.to(accelerator.device)
     ddp_model, dataloader = accelerator.prepare(ddp_model, dataloader)
     return model, ddp_model, dataloader
 
 
-def accuracy(predictions, labels) -> float:
-    """
-    Get the accuracy with respect to the most likely label
-    """
-    return (predictions == labels).float().mean()
-
-
-def test_torch_metrics():
-    accelerator = Accelerator()
-    model, ddp_model, dataloader = get_setup(accelerator)
+def generate_predictions(model, dataloader, accelerator):
+    logits_and_targets = []
     for batch in dataloader:
-        ddp_input, ddp_target = batch.values()
-        # First do single process
-        input, target = accelerator.gather((ddp_input, ddp_target))
-        input, target = input.to(accelerator.device), target.to(accelerator.device)
+        input, target = batch.values()
         with torch.no_grad():
             logits = model(input)
-            accuracy_single = accuracy(logits.argmax(dim=-1), target)
-        # Then do multiprocess
-        with torch.no_grad():
-            logits = ddp_model(ddp_input)
-            logits, target = accelerator.gather_for_metrics((logits, ddp_target), dataloader)
-            accuracy_multi = accuracy(logits.argmax(dim=-1), target)
-        assert torch.allclose(accuracy_single, accuracy_multi), "The two accuracies were not the same!"
+            logits, target = accelerator.gather_for_metrics((logits, target))
+            logits_and_targets.append((logits, target))
+    inps, targs = [], []
+    for (inp, targ) in logits_and_targets:
+        inps.append(inp)
+        targs.append(targ)
+    inps, targs = torch.cat(inps), torch.cat(targs)
+    return inps, targs
+
+
+def test_torch_metrics(accelerator: Accelerator, num_samples=82):
+    model, ddp_model, dataloader = get_setup(accelerator, num_samples)
+    inps, targs = generate_predictions(ddp_model, dataloader, accelerator)
+    assert (
+        len(inps) == num_samples
+    ), f"Unexpected number of inputs:\n    Expected: {num_samples}\n    Actual: {len(inps)}"
 
 
 def main():
-    accelerator = Accelerator()
-    state = accelerator.state
-    if state.local_process_index == 0:
+    accelerator = Accelerator(split_batches=False, dispatch_batches=False)
+    if accelerator.is_local_main_process:
         print("**Test torch metrics**")
-    test_torch_metrics()
+        print("With: `split_batches=False`, `dispatch_batches=False`")
+    test_torch_metrics(accelerator)
+    accelerator.state._reset_state()
+    accelerator = Accelerator(split_batches=True, dispatch_batches=False)
+    if accelerator.is_local_main_process:
+        print("With: `split_batches=True`, `dispatch_batches=False`")
+    test_torch_metrics(accelerator)
+    accelerator.state._reset_state()
+    accelerator = Accelerator(split_batches=False, dispatch_batches=True)
+    if accelerator.is_local_main_process:
+        print("With: `split_batches=False`, `dispatch_batches=True`")
+    test_torch_metrics(accelerator)
+    accelerator.state._reset_state()
+    accelerator = Accelerator(split_batches=True, dispatch_batches=True)
+    if accelerator.is_local_main_process:
+        print("With: `split_batches=True`, `dispatch_batches=True`")
+    test_torch_metrics(accelerator)
 
 
 def _mp_fn(index):


Review comments: body: I think this can be an info, no need for a warning here.
diff_hunk: @@ -934,25 +934,28 @@ def gather(self, tensor):
         """
         return gather(tensor)
 
-    def gather_for_metrics(self, tensor, dataloader):
+    def gather_for_metrics(self, tensor):
         """
         Gathers `tensor` and potentially drops duplicates in the last batch if on a distributed system. Should be used
         for gathering the inputs and targets for metric calculation.
 
         Args:
             tensor (`torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`):
                 The tensors for calculating metrics across all processes.
-            dataloader (`torch.utils.data.DataLoader`):
-                A dataloader prepared with `Accelerator.prepare`
         """
         tensor = self.gather(tensor)
         if self.use_distributed:
+            if self.gradient_state.remainder == -1:
+                logger.warn(
+                    "The used dataset had no length, returning gathered tensors. You should drop the remainder yourself."
+                )
body: There should be an exception ehre :-)
diff_hunk: @@ -332,7 +336,11 @@ def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
         self.gradient_state._set_end_of_dataloader(False)
-        self.gradient_state._set_samples_seen(0)
+        try:
+            length = getattr(self.dataset, "total_dataset_length", len(self.dataset))
+            self.gradient_state._set_remainder(length % self.batch_size)
+        except:
body: Nice new test!
diff_hunk: @@ -22,50 +22,63 @@
 from accelerate.utils import set_seed
 
 
-def get_setup(accelerator):
+def get_setup(accelerator, num_samples=82):
     "Returns everything needed to perform basic training"
     set_seed(42)
     model = RegressionModel()
     ddp_model = deepcopy(model)
-    dset = RegressionDataset(length=80)
+    dset = RegressionDataset(length=num_samples)
     dataloader = DataLoader(dset, batch_size=16)
     model.to(accelerator.device)
     ddp_model, dataloader = accelerator.prepare(ddp_model, dataloader)
     return model, ddp_model, dataloader
 
 
-def accuracy(predictions, labels) -> float:
-    """
-    Get the accuracy with respect to the most likely label
-    """
-    return (predictions == labels).float().mean()
-
-
-def test_torch_metrics():
-    accelerator = Accelerator()
-    model, ddp_model, dataloader = get_setup(accelerator)
+def generate_predictions(model, dataloader, accelerator):
+    logits_and_targets = []
     for batch in dataloader:
-        ddp_input, ddp_target = batch.values()
-        # First do single process
-        input, target = accelerator.gather((ddp_input, ddp_target))
-        input, target = input.to(accelerator.device), target.to(accelerator.device)
+        input, target = batch.values()
         with torch.no_grad():
             logits = model(input)
-            accuracy_single = accuracy(logits.argmax(dim=-1), target)
-        # Then do multiprocess
-        with torch.no_grad():
-            logits = ddp_model(ddp_input)
-            logits, target = accelerator.gather_for_metrics((logits, ddp_target), dataloader)
-            accuracy_multi = accuracy(logits.argmax(dim=-1), target)
-        assert torch.allclose(accuracy_single, accuracy_multi), "The two accuracies were not the same!"
+            logits, target = accelerator.gather_for_metrics((logits, target))
+            logits_and_targets.append((logits, target))
+    inps, targs = [], []
+    for (inp, targ) in logits_and_targets:
+        inps.append(inp)
+        targs.append(targ)
+    inps, targs = torch.cat(inps), torch.cat(targs)
+    return inps, targs
+
+
+def test_torch_metrics(accelerator: Accelerator, num_samples=82):
+    model, ddp_model, dataloader = get_setup(accelerator, num_samples)
+    inps, targs = generate_predictions(ddp_model, dataloader, accelerator)
+    assert (
+        len(inps) == num_samples
+    ), f"Unexpected number of inputs:\n    Expected: {num_samples}\n    Actual: {len(inps)}"
body: ```suggestion
self.gradient_state._set_remainder(length % self.total_batch_size)
```
diff_hunk: @@ -453,7 +463,11 @@ def _fetch_batches(self, iterator):
 
     def __iter__(self):
         self.gradient_state._set_end_of_dataloader(False)
-        self.gradient_state._set_samples_seen(0)
+        try:
+            length = getattr(self.dataset, "total_dataset_length", len(self.dataset))
+            self.gradient_state._set_remainder(length % self.batch_size)
body: Irrespective of the above change, `test_metrics.py` fails when `dispatch_batches=True` when the `num_samples % num_processes != 0` , e.g., num_samples=97 and num_processes=2. Below is the modified test with print statements. I think, `IterableDatasetShard` should be used in conjunction with  `dispatch_batches=True` for expected behaviour or `DataLoaderDispatcher` needs to be fixed to handle normal dataloaders. 

![Screenshot 2022-07-28 at 10 54 03 AM](https://user-images.githubusercontent.com/13534540/181427059-38d54e57-894e-4ae9-bf6c-9add9549c1a5.png)

diff_hunk: @@ -453,7 +463,11 @@ def _fetch_batches(self, iterator):
 
     def __iter__(self):
         self.gradient_state._set_end_of_dataloader(False)
-        self.gradient_state._set_samples_seen(0)
+        try:
+            length = getattr(self.dataset, "total_dataset_length", len(self.dataset))
+            self.gradient_state._set_remainder(length % self.batch_size)
body: Thanks, will change the test case slightly for this and resolve today, though a bit weird that 82 didn't catch this
diff_hunk: @@ -453,7 +463,11 @@ def _fetch_batches(self, iterator):
 
     def __iter__(self):
         self.gradient_state._set_end_of_dataloader(False)
-        self.gradient_state._set_samples_seen(0)
+        try:
+            length = getattr(self.dataset, "total_dataset_length", len(self.dataset))
+            self.gradient_state._set_remainder(length % self.batch_size)

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._