diff --git a/examples/by_feature/checkpointing.py b/examples/by_feature/checkpointing.py
index 1b462a1f4..bffd843d8 100644
--- a/examples/by_feature/checkpointing.py
+++ b/examples/by_feature/checkpointing.py
@@ -236,23 +236,13 @@ def training_function(config, args):
                     accelerator.save_state(output_dir)
 
         model.eval()
-        samples_seen = 0
         for step, batch in enumerate(eval_dataloader):
             # We could avoid this line since we set the accelerator with `device_placement=True` (the default).
             batch.to(accelerator.device)
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
-            # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
-            if accelerator.use_distributed:
-                if step == len(eval_dataloader) - 1:
-                    # Last batch needs to be truncated on distributed systems as it contains additional samples
-                    predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                    references = references[: len(eval_dataloader.dataset) - samples_seen]
-                else:
-                    # Otherwise we add the number of samples seen
-                    samples_seen += references.shape[0]
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/by_feature/cross_validation.py b/examples/by_feature/cross_validation.py
index 6dd61bbf8..707c93c4c 100644
--- a/examples/by_feature/cross_validation.py
+++ b/examples/by_feature/cross_validation.py
@@ -203,23 +203,13 @@ def training_function(config, args):
                     optimizer.zero_grad()
 
             model.eval()
-            samples_seen = 0
             for step, batch in enumerate(eval_dataloader):
                 # We could avoid this line since we set the accelerator with `device_placement=True`.
                 batch.to(accelerator.device)
                 with torch.no_grad():
                     outputs = model(**batch)
                 predictions = outputs.logits.argmax(dim=-1)
-                # It is slightly faster to call this once, than multiple times
-                predictions, references = accelerator.gather((predictions, batch["labels"]))
-                if accelerator.use_distributed:
-                    if step == len(eval_dataloader) - 1:
-                        # Last batch needs to be truncated on distributed systems as it contains additional samples
-                        predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                        references = references[: len(eval_dataloader.dataset) - samples_seen]
-                    else:
-                        # Otherwise we add the number of samples seen
-                        samples_seen += references.shape[0]
+                predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
                 metric.add_batch(
                     predictions=predictions,
                     references=references,
diff --git a/examples/by_feature/fsdp_with_peak_mem_tracking.py b/examples/by_feature/fsdp_with_peak_mem_tracking.py
index a279e64eb..274ccdfce 100644
--- a/examples/by_feature/fsdp_with_peak_mem_tracking.py
+++ b/examples/by_feature/fsdp_with_peak_mem_tracking.py
@@ -274,23 +274,13 @@ def collate_fn(examples):
         # context manager to track the peak memory usage during the evaluation
         with TorchTracemalloc() as tracemalloc:
             model.eval()
-            samples_seen = 0
             for step, batch in enumerate(eval_dataloader):
                 # We could avoid this line since we set the accelerator with `device_placement=True`.
                 batch.to(accelerator.device)
                 with torch.no_grad():
                     outputs = model(**batch)
                 predictions = outputs.logits.argmax(dim=-1)
-                # It is slightly faster to call this once, than multiple times
-                predictions, references = accelerator.gather((predictions, batch["labels"]))
-                if accelerator.use_distributed:
-                    if step == len(eval_dataloader) - 1:
-                        # Last batch needs to be truncated on distributed systems as it contains additional samples
-                        predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                        references = references[: len(eval_dataloader.dataset) - samples_seen]
-                    else:
-                        # Otherwise we add the number of samples seen
-                        samples_seen += references.shape[0]
+                predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
                 metric.add_batch(
                     predictions=predictions,
                     references=references,
diff --git a/examples/by_feature/gradient_accumulation.py b/examples/by_feature/gradient_accumulation.py
index 69856c97b..170a885e3 100644
--- a/examples/by_feature/gradient_accumulation.py
+++ b/examples/by_feature/gradient_accumulation.py
@@ -170,23 +170,13 @@ def training_function(config, args):
                 optimizer.zero_grad()
 
         model.eval()
-        samples_seen = 0
         for step, batch in enumerate(eval_dataloader):
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch.to(accelerator.device)
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
-            # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
-            if accelerator.use_distributed:
-                if step == len(eval_dataloader) - 1:
-                    # Last batch needs to be truncated on distributed systems as it contains additional samples
-                    predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                    references = references[: len(eval_dataloader.dataset) - samples_seen]
-                else:
-                    # Otherwise we add the number of samples seen
-                    samples_seen += references.shape[0]
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/by_feature/memory.py b/examples/by_feature/memory.py
index 91f3e41d1..9f0d55c69 100644
--- a/examples/by_feature/memory.py
+++ b/examples/by_feature/memory.py
@@ -182,23 +182,13 @@ def inner_training_loop(batch_size):
                     optimizer.zero_grad()
 
             model.eval()
-            samples_seen = 0
             for step, batch in enumerate(eval_dataloader):
                 # We could avoid this line since we set the accelerator with `device_placement=True`.
                 batch.to(accelerator.device)
                 with torch.no_grad():
                     outputs = model(**batch)
                 predictions = outputs.logits.argmax(dim=-1)
-                # It is slightly faster to call this once, than multiple times
-                predictions, references = accelerator.gather((predictions, batch["labels"]))
-                if accelerator.use_distributed:
-                    if step == len(eval_dataloader) - 1:
-                        # Last batch needs to be truncated on distributed systems as it contains additional samples
-                        predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                        references = references[: len(eval_dataloader.dataset) - samples_seen]
-                    else:
-                        # Otherwise we add the number of samples seen
-                        samples_seen += references.shape[0]
+                predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
                 metric.add_batch(
                     predictions=predictions,
                     references=references,
diff --git a/examples/by_feature/multi_process_metrics.py b/examples/by_feature/multi_process_metrics.py
index 522cc571b..cb9534c4a 100644
--- a/examples/by_feature/multi_process_metrics.py
+++ b/examples/by_feature/multi_process_metrics.py
@@ -192,6 +192,8 @@ def training_function(config, args):
                 else:
                     # Otherwise we add the number of samples seen
                     samples_seen += references.shape[0]
+            # All of this can be avoided if you use `Accelerator.gather_for_metrics` instead of `Accelerator.gather`:
+            # accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/by_feature/tracking.py b/examples/by_feature/tracking.py
index d7248682d..76ad7a64b 100644
--- a/examples/by_feature/tracking.py
+++ b/examples/by_feature/tracking.py
@@ -194,23 +194,13 @@ def training_function(config, args):
                 optimizer.zero_grad()
 
         model.eval()
-        samples_seen = 0
         for step, batch in enumerate(eval_dataloader):
             # We could avoid this line since we set the accelerator with `device_placement=True` (the default).
             batch.to(accelerator.device)
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
-            # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
-            if accelerator.use_distributed:
-                if step == len(eval_dataloader) - 1:
-                    # Last batch needs to be truncated on distributed systems as it contains additional samples
-                    predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                    references = references[: len(eval_dataloader.dataset) - samples_seen]
-                else:
-                    # Otherwise we add the number of samples seen
-                    samples_seen += references.shape[0]
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/complete_cv_example.py b/examples/complete_cv_example.py
index f984406bf..99d109bd1 100644
--- a/examples/complete_cv_example.py
+++ b/examples/complete_cv_example.py
@@ -232,7 +232,6 @@ def training_function(config, args):
                     accelerator.save_state(output_dir)
         model.eval()
         accurate = 0
-        samples_seen = 0
         for step, batch in enumerate(eval_dataloader):
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch = {k: v.to(accelerator.device) for k, v in batch.items()}
@@ -240,16 +239,7 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(inputs)
             predictions = outputs.argmax(dim=-1)
-            # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather((predictions, batch["label"]))
-            if accelerator.use_distributed:
-                if step == len(eval_dataloader) - 1:
-                    # Last batch needs to be truncated on distributed systems as it contains additional samples
-                    predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                    references = references[: len(eval_dataloader.dataset) - samples_seen]
-                else:
-                    # Otherwise we add the number of samples seen
-                    samples_seen += references.shape[0]
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["label"]))
             accurate_preds = predictions == references
             accurate += accurate_preds.long().sum()
 
diff --git a/examples/complete_nlp_example.py b/examples/complete_nlp_example.py
index 000da6038..094f6cfb5 100644
--- a/examples/complete_nlp_example.py
+++ b/examples/complete_nlp_example.py
@@ -212,23 +212,13 @@ def collate_fn(examples):
                     accelerator.save_state(output_dir)
 
         model.eval()
-        samples_seen = 0
         for step, batch in enumerate(eval_dataloader):
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch.to(accelerator.device)
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
-            # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
-            if accelerator.use_distributed:
-                if step == len(eval_dataloader) - 1:
-                    # Last batch needs to be truncated on distributed systems as it contains additional samples
-                    predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                    references = references[: len(eval_dataloader.dataset) - samples_seen]
-                else:
-                    # Otherwise we add the number of samples seen
-                    samples_seen += references.shape[0]
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/examples/cv_example.py b/examples/cv_example.py
index 875b68398..1118a2f0e 100644
--- a/examples/cv_example.py
+++ b/examples/cv_example.py
@@ -166,7 +166,6 @@ def training_function(config, args):
         model.eval()
         accurate = 0
         num_elems = 0
-        samples_seen = 0
         for _, batch in enumerate(eval_dataloader):
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch = {k: v.to(accelerator.device) for k, v in batch.items()}
@@ -174,16 +173,7 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(inputs)
             predictions = outputs.argmax(dim=-1)
-            # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather((predictions, batch["label"]))
-            if accelerator.use_distributed:
-                if step == len(eval_dataloader) - 1:
-                    # Last batch needs to be truncated on distributed systems as it contains additional samples
-                    predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                    references = references[: len(eval_dataloader.dataset) - samples_seen]
-                else:
-                    # Otherwise we add the number of samples seen
-                    samples_seen += references.shape[0]
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["label"]))
             accurate_preds = predictions == references
             num_elems += accurate_preds.shape[0]
             accurate += accurate_preds.long().sum()
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index b1e7cba27..a126b5dd5 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -154,23 +154,13 @@ def training_function(config, args):
                 optimizer.zero_grad()
 
         model.eval()
-        samples_seen = 0
         for step, batch in enumerate(eval_dataloader):
             # We could avoid this line since we set the accelerator with `device_placement=True`.
             batch.to(accelerator.device)
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits.argmax(dim=-1)
-            # It is slightly faster to call this once, than multiple times
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
-            if accelerator.use_distributed:
-                if step == len(eval_dataloader) - 1:
-                    # Last batch needs to be truncated on distributed systems as it contains additional samples
-                    predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
-                    references = references[: len(eval_dataloader.dataset) - samples_seen]
-                else:
-                    # Otherwise we add the number of samples seen
-                    samples_seen += references.shape[0]
+            predictions, references = accelerator.gather_for_metrics((predictions, batch["labels"]))
             metric.add_batch(
                 predictions=predictions,
                 references=references,
diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index bbd8db435..ef421a22b 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -956,9 +956,6 @@ def gather_for_metrics(self, tensor):
             tensor (`torch.Tensor`, or a nested tuple/list/dictionary of `torch.Tensor`):
                 The tensors for calculating metrics across all processes.
         """
-        raise NotImplementedError(
-            "Currently there are a number of bugs with this method. You should use `Accelerator.gather()` and drop the samples yourself for the time being."
-        )
         tensor = self.gather(tensor)
         if self.use_distributed:
             if self.gradient_state.remainder == -1:
diff --git a/src/accelerate/data_loader.py b/src/accelerate/data_loader.py
index 4cedd8f1a..b8152b302 100644
--- a/src/accelerate/data_loader.py
+++ b/src/accelerate/data_loader.py
@@ -470,11 +470,6 @@ def _fetch_batches(self, iterator):
 
     def __iter__(self):
         self.gradient_state._set_end_of_dataloader(False)
-        try:
-            length = getattr(self.dataset, "total_dataset_length", len(self.dataset))
-            self.gradient_state._set_remainder(length % self.batch_size)
-        except:
-            self.gradient_state._set_remainder(-1)
         main_iterator = None
         if self.state.process_index == 0:
             # We only iterate through the DataLoader on process 0.
@@ -501,7 +496,16 @@ def __iter__(self):
             observed_batch_size = find_batch_size(batch)
             batch_size = observed_batch_size // self.state.num_processes
 
-            if not self._drop_last and self._stop_iteration and observed_batch_size % self.state.num_processes != 0:
+            stop_iteration = self._stop_iteration
+            if not stop_iteration:
+                # We may still be at the end of the dataloader without knowing it yet: if there is nothing left in
+                # the dataloader since the number of batches is a round multiple of the number of processes.
+                next_batch, next_batch_info, next_skip = self._fetch_batches(main_iterator)
+                # next_batch_info[0] is None when there are no more batches, otherwise we still need to process them.
+                if self._stop_iteration and next_batch_info[0] is None:
+                    stop_iteration = True
+
+            if not self._drop_last and stop_iteration and observed_batch_size % self.state.num_processes != 0:
                 # If the last batch is not complete, let's add the first batch to it.
                 batch = concatenate([batch, first_batch], dim=0)
                 # Batch size computation above is wrong, it's off by 1 so we fix it.
@@ -510,16 +514,8 @@ def __iter__(self):
             data_slice = slice(self.state.process_index * batch_size, (self.state.process_index + 1) * batch_size)
             batch = slice_tensors(batch, data_slice)
 
-            stop_iteration = self._stop_iteration
-            if not stop_iteration:
-                # We may still be at the end of the dataloader without knowing it yet: if there is nothing left
-                # because by change the dataset had a round multiple of samples.
-                next_batch, next_batch_info, next_skip = self._fetch_batches(main_iterator)
-                # next_batch_info[0] is None when there are no more batches, otherwise we still need to process them.
-                if self._stop_iteration and next_batch_info[0] is None:
-                    stop_iteration = True
-
             if stop_iteration:
+                self.gradient_state._set_remainder(observed_batch_size)
                 self.gradient_state._set_end_of_dataloader(True)
             yield batch
 
diff --git a/src/accelerate/test_utils/__init__.py b/src/accelerate/test_utils/__init__.py
index 382bf81bf..43d5ed7ef 100644
--- a/src/accelerate/test_utils/__init__.py
+++ b/src/accelerate/test_utils/__init__.py
@@ -7,6 +7,7 @@
     execute_subprocess_async,
     require_cpu,
     require_cuda,
+    require_huggingface_suite,
     require_multi_gpu,
     require_single_gpu,
     require_tpu,
@@ -16,4 +17,4 @@
 from .training import RegressionDataset, RegressionModel
 
 
-from .scripts import test_metrics, test_script, test_sync  # isort:skip
+from .scripts import test_script, test_sync  # isort:skip
diff --git a/src/accelerate/test_utils/scripts/external_deps/__init__.py b/src/accelerate/test_utils/scripts/external_deps/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/accelerate/test_utils/scripts/external_deps/test_metrics.py b/src/accelerate/test_utils/scripts/external_deps/test_metrics.py
new file mode 100755
index 000000000..d234ee14a
--- /dev/null
+++ b/src/accelerate/test_utils/scripts/external_deps/test_metrics.py
@@ -0,0 +1,170 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from copy import deepcopy
+
+import torch
+from torch.utils.data import DataLoader
+
+import datasets
+import evaluate
+import transformers
+from accelerate import Accelerator
+from accelerate.test_utils import RegressionDataset, RegressionModel
+from accelerate.utils import is_tpu_available, set_seed
+from datasets import load_dataset
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+
+def get_basic_setup(accelerator, num_samples=82):
+    "Returns everything needed to perform basic training"
+    set_seed(42)
+    model = RegressionModel()
+    ddp_model = deepcopy(model)
+    dset = RegressionDataset(length=num_samples)
+    dataloader = DataLoader(dset, batch_size=16)
+    model.to(accelerator.device)
+    ddp_model, dataloader = accelerator.prepare(ddp_model, dataloader)
+    return model, ddp_model, dataloader
+
+
+def get_dataloader(accelerator: Accelerator, use_longest=False):
+    tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/mrpc-bert-base-cased")
+    dataset = load_dataset("glue", "mrpc", split="validation")
+
+    def tokenize_function(examples):
+        outputs = tokenizer(examples["sentence1"], examples["sentence2"], truncation=True, max_length=None)
+        return outputs
+
+    with accelerator.main_process_first():
+        tokenized_datasets = dataset.map(
+            tokenize_function,
+            batched=True,
+            remove_columns=["idx", "sentence1", "sentence2"],
+        )
+
+    tokenized_datasets = tokenized_datasets.rename_column("label", "labels")
+
+    def collate_fn(examples):
+        if use_longest:
+            return tokenizer.pad(examples, padding="longest", return_tensors="pt")
+        return tokenizer.pad(examples, padding="max_length", max_length=128, return_tensors="pt")
+
+    return DataLoader(tokenized_datasets, shuffle=False, collate_fn=collate_fn, batch_size=16)
+
+
+def get_mrpc_setup(dispatch_batches, split_batches):
+    accelerator = Accelerator(dispatch_batches=dispatch_batches, split_batches=split_batches)
+    dataloader = get_dataloader(accelerator, not dispatch_batches)
+    model = AutoModelForSequenceClassification.from_pretrained(
+        "hf-internal-testing/mrpc-bert-base-cased", return_dict=True
+    )
+    ddp_model, ddp_dataloader = accelerator.prepare(model, dataloader)
+    return {"ddp": [ddp_model, ddp_dataloader, "cuda:0"], "no": [model, dataloader, accelerator.device]}, accelerator
+
+
+def generate_predictions(model, dataloader, accelerator):
+    logits_and_targets = []
+    for batch in dataloader:
+        input, target = batch.values()
+        with torch.no_grad():
+            logit = model(input)
+            logit, target = accelerator.gather_for_metrics((logit, target))
+            logits_and_targets.append((logit, target))
+    logits, targs = [], []
+    for (logit, targ) in logits_and_targets:
+        logits.append(logit)
+        targs.append(targ)
+    logits, targs = torch.cat(logits), torch.cat(targs)
+    return logits, targs
+
+
+def test_torch_metrics(accelerator: Accelerator, num_samples=82, dispatch_batches=False, split_batches=False):
+    model, ddp_model, dataloader = get_basic_setup(accelerator, num_samples)
+    logits, targs = generate_predictions(ddp_model, dataloader, accelerator)
+    assert (
+        len(logits) == num_samples
+    ), f"Unexpected number of inputs:\n    Expected: {num_samples}\n    Actual: {len(logits)}"
+
+
+def test_mrpc(dispatch_batches: bool = False, split_batches: bool = False):
+    metric = evaluate.load("glue", "mrpc")
+    setup, accelerator = get_mrpc_setup(dispatch_batches, split_batches)
+    # First do baseline
+    model, dataloader, device = setup["no"]
+    model.to(device)
+    model.eval()
+    for batch in dataloader:
+        batch.to(device)
+        with torch.inference_mode():
+            outputs = model(**batch)
+        preds = outputs.logits.argmax(dim=-1)
+        metric.add_batch(predictions=preds, references=batch["labels"])
+    baseline = metric.compute()
+
+    # Then do distributed
+    model, dataloader, device = setup["ddp"]
+    model.eval()
+    for batch in dataloader:
+        with torch.inference_mode():
+            outputs = model(**batch)
+        preds = outputs.logits.argmax(dim=-1)
+        references = batch["labels"]
+        preds, references = accelerator.gather_for_metrics((preds, references))
+        metric.add_batch(predictions=preds, references=references)
+    distributed = metric.compute()
+
+    for key in "accuracy f1".split():
+        assert math.isclose(
+            baseline[key], distributed[key]
+        ), f"Baseline and Distributed are not the same for key {key}:\n\tBaseline: {baseline[key]}\n\tDistributed: {distributed[key]}\n"
+
+
+def main():
+    accelerator = Accelerator(split_batches=False, dispatch_batches=False)
+    if accelerator.is_local_main_process:
+        datasets.utils.logging.set_verbosity_warning()
+        transformers.utils.logging.set_verbosity_warning()
+    else:
+        datasets.utils.logging.set_verbosity_error()
+        transformers.utils.logging.set_verbosity_error()
+    # These are a bit slower so they should only be ran on the GPU or TPU
+    if torch.cuda.is_available() or is_tpu_available():
+        if accelerator.is_local_main_process:
+            print("**Testing gather_for_metrics**")
+        for split_batches in [True, False]:
+            for dispatch_batches in [True, False]:
+                if accelerator.is_local_main_process:
+                    print(f"With: `split_batches={split_batches}`, `dispatch_batches={dispatch_batches}`")
+                test_mrpc(dispatch_batches, split_batches)
+                accelerator.state._reset_state()
+    if accelerator.is_local_main_process:
+        print("**Test torch metrics**")
+    for split_batches in [True, False]:
+        for dispatch_batches in [True, False]:
+            accelerator = Accelerator(split_batches=split_batches, dispatch_batches=dispatch_batches)
+            if accelerator.is_local_main_process:
+                print(f"With: `split_batches={split_batches}`, `dispatch_batches={dispatch_batches}`, length=99")
+            test_torch_metrics(accelerator, 99)
+            accelerator.state._reset_state()
+
+
+def _mp_fn(index):
+    # For xla_spawn (TPUs)
+    main()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/accelerate/test_utils/scripts/test_metrics.py b/src/accelerate/test_utils/scripts/test_metrics.py
deleted file mode 100644
index 8f057ccc5..000000000
--- a/src/accelerate/test_utils/scripts/test_metrics.py
+++ /dev/null
@@ -1,90 +0,0 @@
-# Copyright 2022 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from copy import deepcopy
-
-import torch
-from torch.utils.data import DataLoader
-
-from accelerate import Accelerator
-from accelerate.test_utils import RegressionDataset, RegressionModel
-from accelerate.utils import set_seed
-
-
-def get_setup(accelerator, num_samples=82):
-    "Returns everything needed to perform basic training"
-    set_seed(42)
-    model = RegressionModel()
-    ddp_model = deepcopy(model)
-    dset = RegressionDataset(length=num_samples)
-    dataloader = DataLoader(dset, batch_size=16)
-    model.to(accelerator.device)
-    ddp_model, dataloader = accelerator.prepare(ddp_model, dataloader)
-    return model, ddp_model, dataloader
-
-
-def generate_predictions(model, dataloader, accelerator):
-    logits_and_targets = []
-    for batch in dataloader:
-        input, target = batch.values()
-        with torch.no_grad():
-            logits = model(input)
-            logits, target = accelerator.gather_for_metrics((logits, target))
-            logits_and_targets.append((logits, target))
-    inps, targs = [], []
-    for (inp, targ) in logits_and_targets:
-        inps.append(inp)
-        targs.append(targ)
-    inps, targs = torch.cat(inps), torch.cat(targs)
-    return inps, targs
-
-
-def test_torch_metrics(accelerator: Accelerator, num_samples=82):
-    model, ddp_model, dataloader = get_setup(accelerator, num_samples)
-    inps, targs = generate_predictions(ddp_model, dataloader, accelerator)
-    assert (
-        len(inps) == num_samples
-    ), f"Unexpected number of inputs:\n    Expected: {num_samples}\n    Actual: {len(inps)}"
-
-
-def main():
-    accelerator = Accelerator(split_batches=False, dispatch_batches=False)
-    if accelerator.is_local_main_process:
-        print("**Test torch metrics**")
-        print("With: `split_batches=False`, `dispatch_batches=False`")
-    test_torch_metrics(accelerator)
-    accelerator.state._reset_state()
-    accelerator = Accelerator(split_batches=True, dispatch_batches=False)
-    if accelerator.is_local_main_process:
-        print("With: `split_batches=True`, `dispatch_batches=False`")
-    test_torch_metrics(accelerator)
-    accelerator.state._reset_state()
-    accelerator = Accelerator(split_batches=False, dispatch_batches=True)
-    if accelerator.is_local_main_process:
-        print("With: `split_batches=False`, `dispatch_batches=True`")
-    test_torch_metrics(accelerator)
-    accelerator.state._reset_state()
-    accelerator = Accelerator(split_batches=True, dispatch_batches=True)
-    if accelerator.is_local_main_process:
-        print("With: `split_batches=True`, `dispatch_batches=True`")
-    test_torch_metrics(accelerator)
-
-
-def _mp_fn(index):
-    # For xla_spawn (TPUs)
-    main()
-
-
-if __name__ == "__main__":
-    main()
diff --git a/src/accelerate/test_utils/testing.py b/src/accelerate/test_utils/testing.py
index 145cc80b2..381f92c58 100644
--- a/src/accelerate/test_utils/testing.py
+++ b/src/accelerate/test_utils/testing.py
@@ -30,10 +30,12 @@
 from ..utils import (
     gather,
     is_comet_ml_available,
+    is_datasets_available,
     is_deepspeed_available,
     is_tensorboard_available,
     is_torch_version,
     is_tpu_available,
+    is_transformers_available,
     is_wandb_available,
 )
 
@@ -84,6 +86,15 @@ def require_cuda(test_case):
     return unittest.skipUnless(torch.cuda.is_available(), "test requires a GPU")(test_case)
 
 
+def require_huggingface_suite(test_case):
+    """
+    Decorator marking a test that requires transformers and datasets. These tests are skipped when they are not.
+    """
+    return unittest.skipUnless(
+        is_transformers_available() and is_datasets_available(), "test requires the Hugging Face suite"
+    )(test_case)
+
+
 def require_tpu(test_case):
     """
     Decorator marking a test that requires TPUs. These tests are skipped when there are no TPUs available.
diff --git a/src/accelerate/utils/__init__.py b/src/accelerate/utils/__init__.py
index d1a015b93..db21afc3a 100644
--- a/src/accelerate/utils/__init__.py
+++ b/src/accelerate/utils/__init__.py
@@ -25,6 +25,7 @@
     is_boto3_available,
     is_ccl_available,
     is_comet_ml_available,
+    is_datasets_available,
     is_deepspeed_available,
     is_sagemaker_available,
     is_tensorboard_available,
diff --git a/src/accelerate/utils/imports.py b/src/accelerate/utils/imports.py
index bc4e3b4cd..074d02e4a 100644
--- a/src/accelerate/utils/imports.py
+++ b/src/accelerate/utils/imports.py
@@ -89,6 +89,10 @@ def is_transformers_available():
     return importlib.util.find_spec("transformers") is not None
 
 
+def is_datasets_available():
+    return importlib.util.find_spec("datasets") is not None
+
+
 def is_tensorboard_available():
     return importlib.util.find_spec("tensorboard") is not None or importlib.util.find_spec("tensorboardX") is not None
 
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 7e42c793e..b26ef00b2 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -23,31 +23,36 @@
 from accelerate.test_utils import (
     execute_subprocess_async,
     require_cpu,
+    require_huggingface_suite,
     require_multi_gpu,
     require_single_gpu,
-    skip,
-    test_metrics,
 )
 from accelerate.utils import get_launch_prefix, patch_environment
 
 
-@skip
+@require_huggingface_suite
 class MetricTester(unittest.TestCase):
     def setUp(self):
         mod_file = inspect.getfile(accelerate.test_utils)
-        self.test_file_path = os.path.sep.join(mod_file.split(os.path.sep)[:-1] + ["scripts", "test_metrics.py"])
+        self.test_file_path = os.path.sep.join(
+            mod_file.split(os.path.sep)[:-1] + ["scripts", "external_deps", "test_metrics.py"]
+        )
+
+        from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401
+
+        self.test_metrics = test_metrics
 
     @require_cpu
     def test_metric_cpu_noop(self):
-        debug_launcher(test_metrics.main, num_processes=1)
+        debug_launcher(self.test_metrics.main, num_processes=1)
 
     @require_cpu
     def test_metric_cpu_multi(self):
-        debug_launcher(test_metrics.main)
+        debug_launcher(self.test_metrics.main)
 
     @require_single_gpu
     def test_metric_gpu(self):
-        test_metrics.main()
+        self.test_metrics.main()
 
     @require_multi_gpu
     def test_metric_gpu_multi(self):



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._