diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e9d797083..d764d3927 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -7,12 +7,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.6
+    - name: Set up Python 3.7
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6
+        python-version: 3.7
     - name: Install Python dependencies
-      run: pip install setuptools==59.5.0; pip install -e .[test,test_trackers]
+      run: |
+        pip install --upgrade pip
+        pip install -e .[test,test_trackers]
     - name: Run Tests
       run: make test
       
@@ -20,11 +22,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.6
+    - name: Set up Python 3.7
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6
+        python-version: 3.7
     - name: Install Python dependencies
-      run: pip install setuptools==59.5.0; pip install -e .[test] tensorboard
+      run: |
+        pip install --upgrade pip
+        pip install -e .[test,tensorboard]
     - name: Run Tests
       run: make test_examples
diff --git a/examples/README.md b/examples/README.md
index bb248e9d1..271ffd3c5 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -23,7 +23,7 @@ The [nlp_example.py](./nlp_example.py) script is a simple example to train a Ber
 Prior to running it you should install 🤗 Dataset and 🤗 Transformers:
 
 ```bash
-pip install datasets transformers
+pip install datasets evaluate transformers
 ```
 
 The same script can be run in any of the following configurations:
diff --git a/examples/by_feature/checkpointing.py b/examples/by_feature/checkpointing.py
index 06edd8540..c06d06ee9 100644
--- a/examples/by_feature/checkpointing.py
+++ b/examples/by_feature/checkpointing.py
@@ -18,8 +18,9 @@
 import torch
 from torch.utils.data import DataLoader
 
+import evaluate
 from accelerate import Accelerator, DistributedType
-from datasets import load_dataset, load_metric
+from datasets import load_dataset
 from transformers import (
     AdamW,
     AutoModelForSequenceClassification,
@@ -137,7 +138,7 @@ def training_function(config, args):
     set_seed(seed)
 
     train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)
-    metric = load_metric("glue", "mrpc")
+    metric = evaluate.load("glue", "mrpc")
 
     # If the batch size is too big we use gradient accumulation
     gradient_accumulation_steps = 1
diff --git a/examples/by_feature/cross_validation.py b/examples/by_feature/cross_validation.py
index 9fdc3fe72..53d582c67 100644
--- a/examples/by_feature/cross_validation.py
+++ b/examples/by_feature/cross_validation.py
@@ -19,8 +19,9 @@
 import torch
 from torch.utils.data import DataLoader
 
+import evaluate
 from accelerate import Accelerator, DistributedType
-from datasets import DatasetDict, load_dataset, load_metric
+from datasets import DatasetDict, load_dataset
 
 # New Code #
 # We'll be using StratifiedKFold for this example
@@ -144,7 +145,7 @@ def training_function(config, args):
     seed = int(config["seed"])
     batch_size = int(config["batch_size"])
 
-    metric = load_metric("glue", "mrpc")
+    metric = evaluate.load("glue", "mrpc")
 
     # If the batch size is too big we use gradient accumulation
     gradient_accumulation_steps = 1
diff --git a/examples/by_feature/fsdp_with_peak_mem_tracking.py b/examples/by_feature/fsdp_with_peak_mem_tracking.py
index 8304bcefe..4f0311785 100644
--- a/examples/by_feature/fsdp_with_peak_mem_tracking.py
+++ b/examples/by_feature/fsdp_with_peak_mem_tracking.py
@@ -19,8 +19,9 @@
 import torch
 from torch.utils.data import DataLoader
 
+import evaluate
 from accelerate import Accelerator, DistributedType
-from datasets import load_dataset, load_metric
+from datasets import load_dataset
 from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed
 
 
@@ -119,7 +120,7 @@ def training_function(config, args):
 
     tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
     datasets = load_dataset("glue", "mrpc")
-    metric = load_metric("glue", "mrpc")
+    metric = evaluate.load("glue", "mrpc")
 
     def tokenize_function(examples):
         # max_length=None => use the model max length (it's actually the default)
diff --git a/examples/by_feature/memory.py b/examples/by_feature/memory.py
index 140f14d75..ac4b45866 100644
--- a/examples/by_feature/memory.py
+++ b/examples/by_feature/memory.py
@@ -17,11 +17,11 @@
 import torch
 from torch.utils.data import DataLoader
 
-from accelerate import Accelerator, DistributedType
-
 # New Code #
+import evaluate
+from accelerate import Accelerator, DistributedType
 from accelerate.utils import find_executable_batch_size
-from datasets import load_dataset, load_metric
+from datasets import load_dataset
 from transformers import (
     AdamW,
     AutoModelForSequenceClassification,
@@ -121,7 +121,7 @@ def training_function(config, args):
     seed = int(config["seed"])
     batch_size = int(config["batch_size"])
 
-    metric = load_metric("glue", "mrpc")
+    metric = evaluate.load("glue", "mrpc")
 
     # If the batch size is too big we use gradient accumulation
     gradient_accumulation_steps = 1
diff --git a/examples/by_feature/multi_process_metrics.py b/examples/by_feature/multi_process_metrics.py
index 6babf879a..0307d251f 100644
--- a/examples/by_feature/multi_process_metrics.py
+++ b/examples/by_feature/multi_process_metrics.py
@@ -18,8 +18,9 @@
 import torch
 from torch.utils.data import DataLoader
 
+import evaluate
 from accelerate import Accelerator, DistributedType
-from datasets import load_dataset, load_metric
+from datasets import load_dataset
 from transformers import (
     AdamW,
     AutoModelForSequenceClassification,
@@ -122,7 +123,7 @@ def training_function(config, args):
     seed = int(config["seed"])
     batch_size = int(config["batch_size"])
 
-    metric = load_metric("glue", "mrpc")
+    metric = evaluate.load("glue", "mrpc")
 
     # If the batch size is too big we use gradient accumulation
     gradient_accumulation_steps = 1
diff --git a/examples/by_feature/tracking.py b/examples/by_feature/tracking.py
index 9538b21b1..4020b3382 100644
--- a/examples/by_feature/tracking.py
+++ b/examples/by_feature/tracking.py
@@ -18,8 +18,9 @@
 import torch
 from torch.utils.data import DataLoader
 
+import evaluate
 from accelerate import Accelerator, DistributedType
-from datasets import load_dataset, load_metric
+from datasets import load_dataset
 from transformers import (
     AdamW,
     AutoModelForSequenceClassification,
@@ -132,7 +133,7 @@ def training_function(config, args):
     set_seed(seed)
 
     train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)
-    metric = load_metric("glue", "mrpc")
+    metric = evaluate.load("glue", "mrpc")
 
     # If the batch size is too big we use gradient accumulation
     gradient_accumulation_steps = 1
diff --git a/examples/complete_nlp_example.py b/examples/complete_nlp_example.py
index 9c8b89706..8544d1279 100644
--- a/examples/complete_nlp_example.py
+++ b/examples/complete_nlp_example.py
@@ -18,8 +18,9 @@
 import torch
 from torch.utils.data import DataLoader
 
+import evaluate
 from accelerate import Accelerator, DistributedType
-from datasets import load_dataset, load_metric
+from datasets import load_dataset
 from transformers import (
     AdamW,
     AutoModelForSequenceClassification,
@@ -89,7 +90,7 @@ def training_function(config, args):
 
     tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
     datasets = load_dataset("glue", "mrpc")
-    metric = load_metric("glue", "mrpc")
+    metric = evaluate.load("glue", "mrpc")
 
     def tokenize_function(examples):
         # max_length=None => use the model max length (it's actually the default)
diff --git a/examples/nlp_example.py b/examples/nlp_example.py
index 87cfd7698..84c24fd11 100644
--- a/examples/nlp_example.py
+++ b/examples/nlp_example.py
@@ -17,8 +17,9 @@
 import torch
 from torch.utils.data import DataLoader
 
+import evaluate
 from accelerate import Accelerator, DistributedType
-from datasets import load_dataset, load_metric
+from datasets import load_dataset
 from transformers import (
     AdamW,
     AutoModelForSequenceClassification,
@@ -106,7 +107,7 @@ def training_function(config, args):
     seed = int(config["seed"])
     batch_size = int(config["batch_size"])
 
-    metric = load_metric("glue", "mrpc")
+    metric = evaluate.load("glue", "mrpc")
 
     # If the batch size is too big we use gradient accumulation
     gradient_accumulation_steps = 1
diff --git a/setup.py b/setup.py
index 5f6e128c1..51a6c1a01 100644
--- a/setup.py
+++ b/setup.py
@@ -24,11 +24,13 @@
     "pytest-xdist",
     "pytest-subtests",
     "datasets",
+    "evaluate",
     "transformers",
     "scipy",
     "sklearn"
 ]
-extras["test_trackers"] = ["wandb", "comet-ml", "tensorflow>=2.6.2", "tensorboard"]
+extras["tensorboard"] = ["protobuf<=3.20.1", "tensorflow>=2.6.2", "tensorboard"]
+extras["test_trackers"] = extras["tensorboard"] + ["wandb", "comet-ml"]
 extras["dev"] = extras["quality"] + extras["test"]
 
 extras["sagemaker"] = [


Review comments: body: There's some micro optimization that can be done I found that can save us a number of minutes on each CI, and will actually let this no longer need to be a thing. I'll make a PR after this with that, it'll simplify some things quite a bit :) 
diff_hunk: @@ -7,24 +7,28 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.6
+    - name: Set up Python 3.7
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6
+        python-version: 3.7
     - name: Install Python dependencies
-      run: pip install setuptools==59.5.0; pip install -e .[test,test_trackers]
+      run: |
+        pip install --upgrade pip
+        pip install -e .[test,test_trackers]

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._