diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index d52cf5e1e..0e8f31407 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -35,6 +35,7 @@ jobs:
       - name: Run examples on GPUs
         run: |
           source activate accelerate
+          pip uninstall comet_ml -y
           make test_examples
 
   run_all_tests_multi_gpu:
@@ -64,4 +65,5 @@ jobs:
       - name: Run examples on GPUs
         run: |
           source activate accelerate
+          pip uninstall comet_ml -y
           make test_examples
\ No newline at end of file
diff --git a/.github/workflows/on-merge.yml b/.github/workflows/on-merge.yml
index b95474758..59677b34e 100644
--- a/.github/workflows/on-merge.yml
+++ b/.github/workflows/on-merge.yml
@@ -34,6 +34,7 @@ jobs:
       - name: Run examples on GPUs
         run: |
           source activate accelerate
+          pip uninstall comet_ml -y
           make test_examples
 
   run_all_tests_multi_gpu:
@@ -61,4 +62,5 @@ jobs:
       - name: Run examples on GPUs
         run: |
           source activate accelerate
+          pip uninstall comet_ml -y
           make test_examples
\ No newline at end of file
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c585e742a..c0871b28c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -38,6 +38,7 @@ jobs:
       run: |
         pip install --upgrade pip
         pip install -e .[test,test_trackers]
+        if [ ${{ matrix.test-kind }} = test_rest ]; then pip uninstall comet_ml -y; fi
     
     - name: Run Tests
       run: |
diff --git a/examples/by_feature/cross_validation.py b/examples/by_feature/cross_validation.py
index e380d2d85..95af2a6c9 100644
--- a/examples/by_feature/cross_validation.py
+++ b/examples/by_feature/cross_validation.py
@@ -125,7 +125,6 @@ def collate_fn(examples):
 
 def training_function(config, args):
     # New Code #
-    test_labels = None
     test_predictions = []
     # Download the dataset
     datasets = load_dataset("glue", "mrpc")
@@ -152,17 +151,15 @@ def training_function(config, args):
     # New Code #
     # Create our folds:
     folds = kfold.split(np.zeros(datasets["train"].num_rows), datasets["train"]["label"])
-
+    test_references = []
     # Iterate over them
-    for train_idxs, valid_idxs in folds:
+    for i, (train_idxs, valid_idxs) in enumerate(folds):
         train_dataloader, eval_dataloader, test_dataloader = get_fold_dataloaders(
             accelerator,
             datasets,
             train_idxs,
             valid_idxs,
         )
-        if test_labels is None:
-            test_labels = datasets["validation"]["label"]
         # Instantiate the model (we build the model here so that the seed also control new weights initialization)
         model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", return_dict=True)
 
@@ -231,19 +228,18 @@ def training_function(config, args):
             predictions = outputs.logits
             predictions, references = accelerator.gather((predictions, batch["labels"]))
             fold_predictions.append(predictions.cpu())
-            metric.add_batch(
-                predictions=predictions.argmax(dim=-1),
-                references=references,
-            )
-        test_metric = metric.compute()
+            if i == 0:
+                # We need all of the test predictions
+                test_references.append(references.cpu())
         # Use accelerator.print to print only on the main process.
         test_predictions.append(torch.cat(fold_predictions, dim=0))
         # We now need to release all our memory and get rid of the current model, optimizer, etc
         accelerator.free_memory()
     # New Code #
     # Finally we check the accuracy of our folded results:
-    preds = torch.stack(test_predictions, dim=0).sum(dim=0).div(int(config["n_splits"])).argmax(dim=-1)
-    test_metric = metric.compute(predictions=preds, references=test_labels)
+    test_references = torch.cat(test_references, dim=0)
+    preds = torch.stack(test_predictions, dim=0).sum(dim=0).div(int(args.num_folds)).argmax(dim=-1)
+    test_metric = metric.compute(predictions=preds, references=test_references)
     accelerator.print("Average test metrics from all folds:", test_metric)
 
 
diff --git a/examples/by_feature/tracking.py b/examples/by_feature/tracking.py
index ef2b1388b..3278a913a 100644
--- a/examples/by_feature/tracking.py
+++ b/examples/by_feature/tracking.py
@@ -165,8 +165,6 @@ def training_function(config, args):
     if args.with_tracking:
         if accelerator.is_main_process:
             run = os.path.split(__file__)[-1].split(".")[0]
-            if args.logging_dir:
-                run = os.path.join(args.logging_dir, run)
             accelerator.init_trackers(run, config)
 
     # Now we train the model
diff --git a/src/accelerate/test_utils/testing.py b/src/accelerate/test_utils/testing.py
index 017f66d06..0b3145070 100644
--- a/src/accelerate/test_utils/testing.py
+++ b/src/accelerate/test_utils/testing.py
@@ -15,6 +15,7 @@
 import asyncio
 import os
 import shutil
+import subprocess
 import sys
 import tempfile
 import unittest
@@ -129,6 +130,22 @@ def require_comet_ml(test_case):
     return unittest.skipUnless(is_comet_ml_available(), "test requires comet_ml")(test_case)
 
 
+_atleast_one_tracker_available = (
+    any([is_wandb_available(), is_tensorboard_available()]) and not is_comet_ml_available()
+)
+
+
+def require_trackers(test_case):
+    """
+    Decorator marking that a test requires at least one tracking library installed. These tests are skipped when none
+    are installed
+    """
+    return unittest.skipUnless(
+        _atleast_one_tracker_available,
+        "test requires at least one tracker to be available and for `comet_ml` to not be installed",
+    )(test_case)
+
+
 class TempDirTestCase(unittest.TestCase):
     """
     A TestCase class that keeps a single `tempfile.TemporaryDirectory` open for the duration of the class, wipes its
@@ -279,3 +296,24 @@ def execute_subprocess_async(cmd, env=None, stdin=None, timeout=180, quiet=False
         )
 
     return result
+
+
+class SubprocessCallException(Exception):
+    pass
+
+
+def run_command(command: List[str], return_stdout=False):
+    """
+    Runs `command` with `subprocess.check_output` and will potentially return the `stdout`. Will also properly capture
+    if an error occured while running `command`
+    """
+    try:
+        output = subprocess.check_output(command, stderr=subprocess.STDOUT)
+        if return_stdout:
+            if hasattr(output, "decode"):
+                output = output.decode("utf-8")
+            return output
+    except subprocess.CalledProcessError as e:
+        raise SubprocessCallException(
+            f"Command `{' '.join(command)}` failed with the following error:\n\n{e.output.decode()}"
+        ) from e
diff --git a/tests/test_examples.py b/tests/test_examples.py
index 4b298d253..9e1854cd8 100644
--- a/tests/test_examples.py
+++ b/tests/test_examples.py
@@ -16,7 +16,6 @@
 import os
 import re
 import shutil
-import subprocess
 import tempfile
 import unittest
 from unittest import mock
@@ -24,7 +23,7 @@
 import torch
 
 from accelerate.test_utils.examples import compare_against_test
-from accelerate.test_utils.testing import TempDirTestCase, slow
+from accelerate.test_utils.testing import TempDirTestCase, require_trackers, run_command, slow
 from accelerate.utils import write_basic_config
 
 
@@ -145,7 +144,7 @@ def test_checkpointing_by_epoch(self):
         --checkpointing_steps epoch
         --output_dir {self.tmpdir}
         """.split()
-        _ = subprocess.run(self._launch_args + testargs, stdout=subprocess.PIPE)
+        run_command(self._launch_args + testargs)
         self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "epoch_1")))
 
     def test_checkpointing_by_steps(self):
@@ -154,7 +153,7 @@ def test_checkpointing_by_steps(self):
         --checkpointing_steps 1
         --output_dir {self.tmpdir}
         """.split()
-        _ = subprocess.run(self._launch_args + testargs, stdout=subprocess.PIPE, env=os.environ)
+        _ = run_command(self._launch_args + testargs)
         self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "step_5")))
 
     def test_load_states_by_epoch(self):
@@ -162,9 +161,7 @@ def test_load_states_by_epoch(self):
         examples/by_feature/checkpointing.py
         --resume_from_checkpoint {os.path.join(self.tmpdir, "epoch_1")}
         """.split()
-        output = subprocess.run(
-            self._launch_args + testargs, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-        ).stdout
+        output = run_command(self._launch_args + testargs, return_stdout=True)
         self.assertNotIn("epoch 0:", output)
         self.assertNotIn("epoch 1:", output)
         self.assertIn("epoch 2:", output)
@@ -174,9 +171,7 @@ def test_load_states_by_steps(self):
         examples/by_feature/checkpointing.py
         --resume_from_checkpoint {os.path.join(self.tmpdir, "step_5")}
         """.split()
-        output = subprocess.run(
-            self._launch_args + testargs, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-        ).stdout
+        output = run_command(self._launch_args + testargs, return_stdout=True)
         if torch.cuda.is_available():
             num_processes = torch.cuda.device_count()
         else:
@@ -184,11 +179,10 @@ def test_load_states_by_steps(self):
         if num_processes > 1:
             self.assertNotIn("epoch 0:", output)
             self.assertNotIn("epoch 1:", output)
-            self.assertIn("epoch 2:", output)
         else:
             self.assertNotIn("epoch 0:", output)
             self.assertIn("epoch 1:", output)
-            self.assertIn("epoch 2:", output)
+        self.assertIn("epoch 2:", output)
 
     @slow
     def test_cross_validation(self):
@@ -197,16 +191,16 @@ def test_cross_validation(self):
         --num_folds 2
         """.split()
         with mock.patch.dict(os.environ, {"TESTING_MOCKED_DATALOADERS": "0"}):
-            output = subprocess.run(
-                self._launch_args + testargs, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-            ).stdout
+            output = run_command(self._launch_args + testargs, return_stdout=True)
             results = ast.literal_eval(re.findall("({.+})", output)[-1])
             self.assertGreaterEqual(results["accuracy"], 0.75)
 
     def test_multi_process_metrics(self):
         testargs = ["examples/by_feature/multi_process_metrics.py"]
-        _ = subprocess.run(self._launch_args + testargs, stdout=subprocess.PIPE)
+        run_command(self._launch_args + testargs)
 
+    @require_trackers
+    @mock.patch.dict(os.environ, {"WANDB_MODE": "offline"})
     def test_tracking(self):
         with tempfile.TemporaryDirectory() as tmpdir:
             testargs = f"""
@@ -214,9 +208,9 @@ def test_tracking(self):
             --with_tracking
             --logging_dir {tmpdir}
             """.split()
-            _ = subprocess.run(self._launch_args + testargs, stdout=subprocess.PIPE)
+            run_command(self._launch_args + testargs)
             self.assertTrue(os.path.exists(os.path.join(tmpdir, "tracking")))
 
     def test_gradient_accumulation(self):
         testargs = ["examples/by_feature/gradient_accumulation.py"]
-        _ = subprocess.run(self._launch_args + testargs, stdout=subprocess.PIPE)
+        run_command(self._launch_args + testargs)


Review comments: body: This will stop working on distributed setup. The predictions will need to be gathered (maybe not the labels).
diff_hunk: @@ -229,20 +229,14 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits
-            predictions, references = accelerator.gather((predictions, batch["labels"]))
body: Should be fixed now
diff_hunk: @@ -229,20 +229,14 @@ def training_function(config, args):
             with torch.no_grad():
                 outputs = model(**batch)
             predictions = outputs.logits
-            predictions, references = accelerator.gather((predictions, batch["labels"]))

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: All slow ci is now passing here: https://github.com/huggingface/accelerate/runs/7358153325