diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx
index 2e9cbe203..4b144ba05 100644
--- a/docs/source/quicktour.mdx
+++ b/docs/source/quicktour.mdx
@@ -120,14 +120,13 @@ validation_dataloader = accelerator.prepare(validation_dataloader)
 
 As for your training dataloader, it will mean that (should you run your script on multiple devices) each device will
 only see part of the evaluation data. This means you will need to group your predictions together. This is very easy to
-do with the [`~Accelerator.gather`] method.
+do with the [`~Accelerator.gather_for_metrics`] method.
 
 ```python
 for inputs, targets in validation_dataloader:
     predictions = model(inputs)
     # Gather all predictions and targets
-    all_predictions = accelerator.gather(predictions)
-    all_targets = accelerator.gather(targets)
+    all_predictions, all_targets = accelerator.gather_for_metrics((predictions, targets))
     # Example of use with a *Datasets.Metric*
     metric.add_batch(all_predictions, all_targets)
 ```
@@ -141,11 +140,17 @@ As for the training dataloader, passing your validation dataloader through
 Any instruction using your training dataloader length (for instance if you need the number of total training steps
 to create a learning rate scheduler) should go after the call to [`~Accelerator.prepare`].
 
+As some data at the end of the dataset may be duplicated so the batch can divide equally to all workers, metrics should be 
+calculated through the [`~Accelerator.gather_for_metrics`] method to automatically remove the duplicated data.
+
+If for some reason you don't wish to have this automatically done, [`~Accelerator.gather`] can be used instead to gather 
+the data across all processes and this can manually be done instead.
+
 </Tip>
 
 <Tip warning={true}>
 
-The [`~Accelerator.gather`] method requires the tensors to be all the same size on each process. If
+The [`~Accelerator.gather`] and [`~Accelerator.gather_for_metrics`] methods require the tensors to be all the same size on each process. If
 you have tensors of different sizes on each process (for instance when dynamically padding to the maximum length in
 a batch), you should use the [`~Accelerator.pad_across_processes`] method to pad you tensor to the
 biggest size across processes.
diff --git a/examples/by_feature/checkpointing.py b/examples/by_feature/checkpointing.py
index 6880f4626..1b21ebed7 100644
--- a/examples/by_feature/checkpointing.py
+++ b/examples/by_feature/checkpointing.py
@@ -238,7 +238,7 @@ def training_function(config, args):
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
             # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), eval_dataloader)
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/by_feature/cross_validation.py b/examples/by_feature/cross_validation.py
index 95af2a6c9..79a207fa8 100644
--- a/examples/by_feature/cross_validation.py
+++ b/examples/by_feature/cross_validation.py
@@ -207,7 +207,9 @@ def training_function(config, args):
                 with torch.no_grad():
                     outputs = model(**batch)
                 predictions = outputs.logits.argmax(dim=-1)
-                predictions, references = accelerator.gather((predictions, batch["labels"]))
+                predictions, references = accelerator.gather_for_metrics(
+                    (predictions, batch["labels"]), eval_dataloader
+                )
                 metric.add_batch(
                     predictions=predictions,
                     references=references,
@@ -226,7 +228,7 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), test_dataloader)
             fold_predictions.append(predictions.cpu())
             if i == 0:
                 # We need all of the test predictions
diff --git a/examples/by_feature/fsdp_with_peak_mem_tracking.py b/examples/by_feature/fsdp_with_peak_mem_tracking.py
index 8da906ee9..8e3b9e130 100644
--- a/examples/by_feature/fsdp_with_peak_mem_tracking.py
+++ b/examples/by_feature/fsdp_with_peak_mem_tracking.py
@@ -270,7 +270,6 @@ def collate_fn(examples):
         # context manager to track the peak memory usage during the evaluation
         with TorchTracemalloc() as tracemalloc:
             model.eval()
-            samples_seen = 0
             for step, batch in enumerate(eval_dataloader):
                 # We could avoid this line since we set the accelerator with `device_placement=True`.
                 batch.to(accelerator.device)
@@ -278,15 +277,9 @@ def collate_fn(examples):
                     outputs = model(**batch)
                 predictions = outputs.logits.argmax(dim=-1)
                 # It is slightly faster to call this once, than multiple times
-                predictions, references = accelerator.gather(
-                    (predictions, batch["labels"])
-                )  # If we are in a multiprocess environment, the last batch has duplicates
-                if accelerator.use_distributed:
-                    if step == len(eval_dataloader) - 1:
-                        predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                        references = references[: len(eval_dataloader.dataset) - samples_seen]
-                    else:
-                        samples_seen += references.shape[0]
+                predictions, references = accelerator.gather_for_metrics(
+                    (predictions, batch["labels"]), eval_dataloader
+                )
                 metric.add_batch(
                     predictions=predictions,
                     references=references,
diff --git a/examples/by_feature/gradient_accumulation.py b/examples/by_feature/gradient_accumulation.py
index 403587f10..d95f9bef3 100644
--- a/examples/by_feature/gradient_accumulation.py
+++ b/examples/by_feature/gradient_accumulation.py
@@ -171,7 +171,7 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), eval_dataloader)
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/by_feature/memory.py b/examples/by_feature/memory.py
index 65feed51c..b11c80fd0 100644
--- a/examples/by_feature/memory.py
+++ b/examples/by_feature/memory.py
@@ -183,7 +183,9 @@ def inner_training_loop(batch_size):
                 with torch.no_grad():
                     outputs = model(**batch)
                 predictions = outputs.logits.argmax(dim=-1)
-                predictions, references = accelerator.gather((predictions, batch["labels"]))
+                predictions, references = accelerator.gather_for_metrics(
+                    (predictions, batch["labels"]), eval_dataloader
+                )
                 metric.add_batch(
                     predictions=predictions,
                     references=references,
diff --git a/examples/by_feature/multi_process_metrics.py b/examples/by_feature/multi_process_metrics.py
index 741e62b8c..dc47f3db6 100644
--- a/examples/by_feature/multi_process_metrics.py
+++ b/examples/by_feature/multi_process_metrics.py
@@ -187,6 +187,8 @@ def training_function(config, args):
                 else:
                     # Otherwise we add the number of samples seen
                     samples_seen += references.shape[0]
+            # All of this can be avoided if you use `Accelerator.gather_for_metrics` instead of `Accelerator.gather`:
+            # accelerator.gather_for_metrics((predictions, references), eval_dataloader)
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/by_feature/tracking.py b/examples/by_feature/tracking.py
index 3278a913a..78d4af422 100644
--- a/examples/by_feature/tracking.py
+++ b/examples/by_feature/tracking.py
@@ -197,7 +197,7 @@ def training_function(config, args):
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
             # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), eval_dataloader)
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/complete_cv_example.py b/examples/complete_cv_example.py
index 6e414bc9c..8f893b7c4 100644
--- a/examples/complete_cv_example.py
+++ b/examples/complete_cv_example.py
@@ -233,7 +233,6 @@ def training_function(config, args):
                     accelerator.save_state(output_dir)
         model.eval()
         accurate = 0
-        samples_seen = 0
         for step, batch in enumerate(eval_dataloader):
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch = {k: v.to(accelerator.device) for k, v in batch.items()}
@@ -241,19 +240,11 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(inputs)
             predictions = outputs.argmax(dim=-1)
-            predictions, references = accelerator.gather((predictions, batch["label"]))
-            if accelerator.use_distributed:
-                if step == len(eval_dataloader) - 1:
-                    predictions = predictions[: len(eval_dataloader) - samples_seen]
-                    references = references[: len(eval_dataloader) - samples_seen]
-                else:
-                    samples_seen += references.shape[0]
-            else:
-                samples_seen += references.shape[0]
-            accurate_preds = predictions == references
+            predictions, labels = accelerator.gather_for_metrics((predictions, batch["label"]), eval_dataloader)
+            accurate_preds = predictions == labels
             accurate += accurate_preds.long().sum()
 
-        eval_metric = accurate.item() / samples_seen
+        eval_metric = accurate.item() / accelerator.gradient_state.samples_seen
         # Use accelerator.print to print only on the main process.
         accelerator.print(f"epoch {epoch}: {100 * eval_metric:.2f}")
         if args.with_tracking:
diff --git a/examples/complete_nlp_example.py b/examples/complete_nlp_example.py
index 6e6aec0ff..572bc9a23 100644
--- a/examples/complete_nlp_example.py
+++ b/examples/complete_nlp_example.py
@@ -211,7 +211,6 @@ def collate_fn(examples):
                     accelerator.save_state(output_dir)
 
         model.eval()
-        samples_seen = 0
         for step, batch in enumerate(eval_dataloader):
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch.to(accelerator.device)
@@ -219,15 +218,7 @@ def collate_fn(examples):
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
             # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather(
-                (predictions, batch["labels"])
-            )  # If we are in a multiprocess environment, the last batch has duplicates
-            if accelerator.use_distributed:
-                if step == len(eval_dataloader) - 1:
-                    predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                    references = references[: len(eval_dataloader.dataset) - samples_seen]
-                else:
-                    samples_seen += references.shape[0]
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), eval_dataloader)
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/cv_example.py b/examples/cv_example.py
index 483bddac3..aa97ce2d7 100644
--- a/examples/cv_example.py
+++ b/examples/cv_example.py
@@ -173,7 +173,8 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(inputs)
             predictions = outputs.argmax(dim=-1)
-            accurate_preds = accelerator.gather(predictions) == accelerator.gather(batch["label"])
+            predictions, labels = accelerator.gather_for_metrics((predictions, batch["label"]), eval_dataloader)
+            accurate_preds = predictions == labels
             num_elems += accurate_preds.shape[0]
             accurate += accurate_preds.long().sum()
 
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index 55dedae5e..d64ab3bb7 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -158,7 +158,7 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]), eval_dataloader)
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 3cc1059e1..6a2aa9703 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -51,6 +51,7 @@
     is_torch_version,
     is_tpu_available,
     pad_across_processes,
+    recursively_apply,
     reduce,
     save,
     wait_for_everyone,
@@ -874,6 +875,35 @@ def gather(self, tensor):
         """
         return gather(tensor)
 
+    def gather_for_metrics(self, tensor, dataloader):
+        """
+        Gathers `tensor` and potentially drops duplicates in the last batch if on a distributed system. Should be used
+        for gathering the inputs and targets for metric calculation.
+
+        Args:
+            tensor (`torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`):
+                The tensors for calculating metrics across all processes.
+            dataloader (`torch.utils.data.DataLoader`):
+                A dataloader prepared with `Accelerator.prepare`
+        """
+        tensor = self.gather(tensor)
+        if self.use_distributed:
+            try:
+                # Then see if we're on the last batch of our eval dataloader
+                if self.gradient_state.end_of_dataloader:
+                    # Last batch needs to be truncated on distributed systems as it contains additional samples
+                    def _adjust_samples(tensor):
+                        return tensor[: dataloader.total_dataset_length - self.gradient_state.samples_seen]
+
+                    return recursively_apply(_adjust_samples, tensor)
+                else:
+                    # Not at the end of the dataloader, no need to adjust the tensors
+                    return tensor
+            except:
+                # Dataset had no length or raised an error
+                return tensor
+        return tensor
+
     def reduce(self, tensor, reduction="sum"):
         """
         Reduce the values in *tensor* across all processes based on *reduction*.
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index fddfe8955..ba3338546 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -47,12 +47,18 @@ class MpDeviceLoaderWrapper(xpl.MpDeviceLoader):
         - **total_batch_size** (`int`) -- Total batch size of the dataloader across all processes.
             Equal to the original batch size when `split_batches=True`; otherwise the original batch size * the total
             number of processes
+
+        - **total_dataset_length** (`int`) -- Total length of the inner dataset across all processes.
         """
 
         @property
         def total_batch_size(self):
             return self._loader.total_batch_size
 
+        @property
+        def total_dataset_length(self):
+            return self._loader.total_dataset_length
+
 
 logger = get_logger(__name__)
 
@@ -311,6 +317,8 @@ class DataLoaderShard(DataLoader):
         - **total_batch_size** (`int`) -- Total batch size of the dataloader across all processes.
             Equal to the original batch size when `split_batches=True`; otherwise the original batch size * the total
             number of processes
+
+        - **total_dataset_length** (`int`) -- Total length of the inner dataset across all processes.
     """
 
     def __init__(self, dataset, device=None, rng_types=None, generator=None, **kwargs):
@@ -324,11 +332,13 @@ def __iter__(self):
         if self.rng_types is not None:
             synchronize_rng_states(self.rng_types, self.generator)
         self.gradient_state._set_end_of_dataloader(False)
+        self.gradient_state._set_samples_seen(0)
         dataloader_iter = super().__iter__()
         # We iterate one batch ahead to check when we are at the end
         try:
             current_batch = next(dataloader_iter)
         except StopIteration:
+            self.gradient_state._iterate_samples_seen(find_batch_size(current_batch))
             yield
         while True:
             try:
@@ -336,10 +346,12 @@ def __iter__(self):
                 if self.device is not None:
                     current_batch = send_to_device(current_batch, self.device)
                 next_batch = next(dataloader_iter)
+                self.gradient_state._iterate_samples_seen(find_batch_size(current_batch))
                 yield current_batch
                 current_batch = next_batch
             except StopIteration:
                 self.gradient_state._set_end_of_dataloader(True)
+                self.gradient_state._iterate_samples_seen(find_batch_size(current_batch))
                 yield current_batch
                 break
 
@@ -351,6 +363,10 @@ def total_batch_size(self):
             else (self.batch_sampler.batch_size * self.batch_sampler.num_processes)
         )
 
+    @property
+    def total_dataset_length(self):
+        return len(self.dataset)
+
 
 class DataLoaderDispatcher(DataLoader):
     """
@@ -370,6 +386,8 @@ class DataLoaderDispatcher(DataLoader):
         - **total_batch_size** (`int`) -- Total batch size of the dataloader across all processes.
             Equal to the original batch size when `split_batches=True`; otherwise the original batch size * the total
             number of processes
+
+        - **total_dataset_length** (`int`) -- Total length of the inner dataset across all processes.
     """
 
     def __init__(self, dataset, split_batches: bool = False, **kwargs):
@@ -435,6 +453,7 @@ def _fetch_batches(self, iterator):
 
     def __iter__(self):
         self.gradient_state._set_end_of_dataloader(False)
+        self.gradient_state._set_samples_seen(0)
         main_iterator = None
         if self.state.process_index == 0:
             # We only iterate through the DataLoader on process 0.
@@ -466,12 +485,15 @@ def __iter__(self):
 
             data_slice = slice(self.state.process_index * batch_size, (self.state.process_index + 1) * batch_size)
             next_batch, next_batch_info, next_skip = self._fetch_batches(main_iterator)
+            batch = slice_tensors(batch, data_slice)
             if not self._stop_iteration:
-                yield slice_tensors(batch, data_slice)
+                self.gradient_state._iterate_samples_seen(batch_size)
+                yield batch
                 batch, batch_info, skip = next_batch, next_batch_info, next_skip
             else:
+                self.gradient_state._iterate_samples_seen(batch_size)
                 self.gradient_state._set_end_of_dataloader(True)
-                yield slice_tensors(batch, data_slice)
+                yield batch
                 break
 
     def __len__(self):
@@ -489,6 +511,10 @@ def total_batch_size(self):
             self.dataset.batch_size if self.split_batches else (self.dataset.batch_size * self.dataset.num_processes)
         )
 
+    @property
+    def total_dataset_length(self):
+        return len(self.dataset)
+
 
 def prepare_data_loader(
     dataloader: DataLoader,
diff --git a/src/accelerate/state.py b/src/accelerate/state.py
index 3c06968aa..6d3e7d3ff 100644
--- a/src/accelerate/state.py
+++ b/src/accelerate/state.py
@@ -255,10 +255,15 @@ def __init__(self):
         if not getattr(self, "initialized", False):
             self.sync_gradients = True
             self.end_of_dataloader = False
+            self.samples_seen = 0
         self.initialized = True
 
     def __repr__(self):
-        return f"Sync Gradients: {self.sync_gradients}\n" f"At end of current dataloader: {self.end_of_dataloader}\n"
+        return (
+            f"Sync Gradients: {self.sync_gradients}\n"
+            f"At end of current dataloader: {self.end_of_dataloader}\n"
+            f"Samples seen: {self.samples_seen}"
+        )
 
     def _set_sync_gradients(self, sync_gradients):
         "Private function that sets whether gradients should be synchronized. Users should not have to call this."
@@ -267,3 +272,11 @@ def _set_sync_gradients(self, sync_gradients):
     def _set_end_of_dataloader(self, end_of_dataloader):
         "Private function that sets whether the end of the current dataloader has been reached. Users should not have to call this."
         self.end_of_dataloader = end_of_dataloader
+
+    def _set_samples_seen(self, samples_seen):
+        "Private function that sets the number of samples iterated over. Users should not have to call this."
+        self.samples_seen = samples_seen
+
+    def _iterate_samples_seen(self, iteration: int = 1):
+        "Private function that iterates the number of samples seen by an iteration. Users should not have to call this."
+        self._set_samples_seen(self.samples_seen + iteration)
diff --git a/src/accelerate/test_utils/__init__.py b/src/accelerate/test_utils/__init__.py
index 2d8527be2..d65099602 100644
--- a/src/accelerate/test_utils/__init__.py
+++ b/src/accelerate/test_utils/__init__.py
@@ -15,4 +15,4 @@
 from .training import RegressionDataset, RegressionModel
 
 
-from .scripts import test_script, test_sync  # isort:skip
+from .scripts import test_metrics, test_script, test_sync  # isort:skip
diff --git a/src/accelerate/test_utils/scripts/test_metrics.py b/src/accelerate/test_utils/scripts/test_metrics.py
new file mode 100644
index 000000000..442053213
--- /dev/null
+++ b/src/accelerate/test_utils/scripts/test_metrics.py
@@ -0,0 +1,98 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from copy import deepcopy
+
+import torch
+from torch.utils.data import DataLoader
+
+import evaluate
+from accelerate import Accelerator
+from accelerate.test_utils import RegressionDataset, RegressionModel
+from accelerate.utils import set_seed
+
+
+def get_setup(accelerator):
+    "Returns everything needed to perform basic training"
+    set_seed(42)
+    model = RegressionModel()
+    ddp_model = deepcopy(model)
+    dset = RegressionDataset(length=80)
+    dataloader = DataLoader(dset, batch_size=16)
+    model.to(accelerator.device)
+    ddp_model, dataloader = accelerator.prepare(ddp_model, dataloader)
+    return model, ddp_model, dataloader
+
+
+def accuracy(predictions, labels) -> float:
+    """
+    Get the accuracy with respect to the most likely label
+    """
+    return (predictions == labels).float().mean()
+
+
+def test_torch_metrics():
+    accelerator = Accelerator()
+    model, ddp_model, dataloader = get_setup(accelerator)
+    for batch in dataloader:
+        ddp_input, ddp_target = batch.values()
+        # First do single process
+        input, target = accelerator.gather((ddp_input, ddp_target))
+        input, target = input.to(accelerator.device), target.to(accelerator.device)
+        with torch.no_grad():
+            logits = model(input)
+            accuracy_single = accuracy(logits.argmax(dim=-1), target)
+        # Then do multiprocess
+        with torch.no_grad():
+            logits = ddp_model(ddp_input)
+            logits, target = accelerator.gather_for_metrics((logits, ddp_target), dataloader)
+            accuracy_multi = accuracy(logits.argmax(dim=-1), target)
+        assert torch.allclose(accuracy_single, accuracy_multi), "The two accuracies were not the same!"
+
+
+def test_evaluate_metrics():
+    metric = evaluate.load("accuracy")
+    accelerator = Accelerator()
+    model, ddp_model, dataloader = get_setup(accelerator)
+    for batch in dataloader:
+        ddp_input, ddp_target = batch.values()
+        # First do single process
+        input, target = accelerator.gather((ddp_input, ddp_target))
+        input, target = input.to(accelerator.device), target.to(accelerator.device)
+        with torch.no_grad():
+            logits = model(input)
+            accuracy_single = metric.compute(logits, target)
+        # Then do multiprocess
+        with torch.no_grad():
+            logits = ddp_model(ddp_input)
+            logits, target = accelerator.gather_for_metrics((logits, ddp_target), dataloader)
+            accuracy_multi = metric.compute(logits, target)
+        assert torch.allclose(accuracy_single, accuracy_multi), "The two accuracies were not the same!"
+
+
+def main():
+    accelerator = Accelerator()
+    state = accelerator.state
+    if state.local_process_index == 0:
+        print("**Test torch metrics**")
+    test_torch_metrics()
+
+
+def _mp_fn(index):
+    # For xla_spawn (TPUs)
+    main()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
new file mode 100644
index 000000000..c744a2fdb
--- /dev/null
+++ b/tests/test_metrics.py
@@ -0,0 +1,55 @@
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import os
+import unittest
+
+import torch
+
+import accelerate
+from accelerate import debug_launcher
+from accelerate.test_utils import (
+    execute_subprocess_async,
+    require_cpu,
+    require_multi_gpu,
+    require_single_gpu,
+    test_metrics,
+)
+from accelerate.utils import get_launch_prefix, patch_environment
+
+
+class MetricTester(unittest.TestCase):
+    def setUp(self):
+        mod_file = inspect.getfile(accelerate.test_utils)
+        self.test_file_path = os.path.sep.join(mod_file.split(os.path.sep)[:-1] + ["scripts", "test_metrics.py"])
+
+    @require_cpu
+    def test_metric_cpu_noop(self):
+        debug_launcher(test_metrics.main, num_processes=1)
+
+    @require_cpu
+    def test_metric_cpu_multi(self):
+        debug_launcher(test_metrics.main)
+
+    @require_single_gpu
+    def test_metric_gpu(self):
+        test_metrics.main()
+
+    @require_multi_gpu
+    def test_metric_gpu_multi(self):
+        print(f"Found {torch.cuda.device_count()} devices.")
+        cmd = get_launch_prefix() + [f"--nproc_per_node={torch.cuda.device_count()}", self.test_file_path]
+        with patch_environment(omp_num_threads=1):
+            execute_subprocess_async(cmd, env=os.environ.copy())


Review comments: body: Maybe mention it that if the user doesn't want that behavior for some reason, they can just use `gather`?
diff_hunk: @@ -141,11 +140,14 @@ As for the training dataloader, passing your validation dataloader through
 Any instruction using your training dataloader length (for instance if you need the number of total training steps
 to create a learning rate scheduler) should go after the call to [`~Accelerator.prepare`].
 
+As some data at the end of the dataset may be duplicated so the batch can divide equally to all workers, metrics should be 
+calculated through the [`~Accelerator.gather_for_metrics`] method to automatically remove the duplicated data.

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: is this functionality in 0.11.0?
body: It is not, we suggest not using it for now and doing the check manually as shown in the metric example script as some bugs were discovered: https://github.com/huggingface/accelerate/issues/575
body: Cool, thank you, excited for this change when it happens