diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c0871b28c..fe44d5f61 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,7 +12,9 @@ jobs:
     strategy:
       matrix:
         test-kind: [
-          test,
+          test_prod,
+          test_core,
+          test_big_modeling,
           test_deepspeed,
           test_example_differences,
           test_checkpoint_step,
@@ -37,8 +39,9 @@ jobs:
     - name: Install the library
       run: |
         pip install --upgrade pip
-        pip install -e .[test,test_trackers]
-        if [ ${{ matrix.test-kind }} = test_rest ]; then pip uninstall comet_ml -y; fi
+        if [[ ${{ matrix.test-kind }} = test_prod ]]; then pip install -e .[test_prod]; fi
+        if [[ ${{ matrix.test-kind }} != test_prod ]]; then pip install -e .[test,test_trackers]; fi
+        if [[ ${{ matrix.test-kind }} = test_rest ]]; then pip uninstall comet_ml -y; fi
     
     - name: Run Tests
       run: |
diff --git a/Makefile b/Makefile
index 6dab6af11..4104f1bba 100644
--- a/Makefile
+++ b/Makefile
@@ -27,6 +27,12 @@ style:
 test:
 	python -m pytest -s -v ./tests/ --ignore=./tests/test_examples.py
 
+test_big_modeling:
+	python -m pytest -s -v ./tests/test_big_modeling.py
+
+test_core:
+	python -m pytest -s -v ./tests/ --ignore=./tests/test_examples.py --ignore=./tests/deepspeed --ignore=./tests/test_big_modeling.py
+
 test_deepspeed:
 	python -m pytest -s -v ./tests/deepspeed
 
@@ -43,5 +49,9 @@ test_checkpoint_epoch:
 test_checkpoint_step:
 	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "by_step"
 
+# Same as test but used to install only the base dependencies
+test_prod:
+	$(MAKE) test_core
+
 test_rest:
 	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests -k "not by_step and not by_epoch"
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 77679a51e..dedd077f3 100644
--- a/setup.py
+++ b/setup.py
@@ -18,18 +18,9 @@
 extras = {}
 extras["quality"] = ["black ~= 22.0", "isort >= 5.5.4", "flake8 >= 3.8.3", "hf-doc-builder >= 0.3.0"]
 extras["docs"] = []
-extras["test"] = [
-    "pytest",
-    "pytest-xdist",
-    "pytest-subtests",
-    "datasets<=2.2.2",
-    "evaluate",
-    "transformers",
-    "scipy",
-    "sklearn",
-    "parameterized",
-    "deepspeed",
-]
+extras["test_prod"] = ["pytest", "pytest-xdist", "pytest-subtests", "parameterized"]
+extras["test_dev"] = ["datasets<=2.2.2", "evaluate", "transformers", "scipy", "sklearn", "deepspeed"]
+extras["test"] = extras["test_prod"] + extras["test_dev"]
 
 extras["test_trackers"] = ["wandb", "comet-ml", "tensorboard"]
 extras["dev"] = extras["quality"] + extras["test"]
diff --git a/src/accelerate/test_utils/scripts/test_metrics.py b/src/accelerate/test_utils/scripts/test_metrics.py
index 442053213..c426d1fc5 100644
--- a/src/accelerate/test_utils/scripts/test_metrics.py
+++ b/src/accelerate/test_utils/scripts/test_metrics.py
@@ -17,7 +17,6 @@
 import torch
 from torch.utils.data import DataLoader
 
-import evaluate
 from accelerate import Accelerator
 from accelerate.test_utils import RegressionDataset, RegressionModel
 from accelerate.utils import set_seed
@@ -61,26 +60,6 @@ def test_torch_metrics():
         assert torch.allclose(accuracy_single, accuracy_multi), "The two accuracies were not the same!"
 
 
-def test_evaluate_metrics():
-    metric = evaluate.load("accuracy")
-    accelerator = Accelerator()
-    model, ddp_model, dataloader = get_setup(accelerator)
-    for batch in dataloader:
-        ddp_input, ddp_target = batch.values()
-        # First do single process
-        input, target = accelerator.gather((ddp_input, ddp_target))
-        input, target = input.to(accelerator.device), target.to(accelerator.device)
-        with torch.no_grad():
-            logits = model(input)
-            accuracy_single = metric.compute(logits, target)
-        # Then do multiprocess
-        with torch.no_grad():
-            logits = ddp_model(ddp_input)
-            logits, target = accelerator.gather_for_metrics((logits, ddp_target), dataloader)
-            accuracy_multi = metric.compute(logits, target)
-        assert torch.allclose(accuracy_single, accuracy_multi), "The two accuracies were not the same!"
-
-
 def main():
     accelerator = Accelerator()
     state = accelerator.state



Discussion: body: _The documentation is not available anymore as the PR was closed or merged._