diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml
index 6b7987e51..de8542359 100644
--- a/.github/workflows/build-docker-images.yml
+++ b/.github/workflows/build-docker-images.yml
@@ -1,7 +1,7 @@
 name: Build Docker images (scheduled)
 
 on:
-  repository_dispatch:
+  workflow_dispatch:
   schedule:
     - cron: "0 1 * * *"
 
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
new file mode 100644
index 000000000..11946b544
--- /dev/null
+++ b/.github/workflows/nightly.yml
@@ -0,0 +1,72 @@
+name: Self-hosted runner (scheduled)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 2 * * *"
+
+jobs:
+  run_all_tests_single_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      CUDA_VISIBLE_DEVICES: "0"
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone & pip install
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
+          git fetch && git checkout ${{ github.sha }}
+          pip install tensorflow -U 
+          pip install -e . --no-deps
+
+# Note: tensorflow upgrade is needed until dropped py 3.6 support
+
+      - name: Run test on GPUs
+        run: |
+          source activate accelerate
+          make test_cuda
+      - name: Run examples on GPUs
+        run: |
+          source activate accelerate
+          make test_examples
+
+  run_all_tests_multi_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      ALL_SLOW: "yes"
+      CUDA_VISIBLE_DEVICES: "0,1"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
+          git fetch && git checkout ${{ github.sha }}
+          pip install tensorflow -U
+          pip install -e . --no-deps
+
+        # Note: tensorflow upgrade is needed until dropped py 3.6 support
+
+      - name: Run test on GPUs
+        run: |
+          source activate accelerate
+          make test_cuda
+
+      - name: Run examples on GPUs
+        run: |
+          source activate accelerate
+          make test_examples
\ No newline at end of file
diff --git a/docker/accelerate-cpu/Dockerfile b/docker/accelerate-cpu/Dockerfile
index c536fb70c..1d99c4035 100644
--- a/docker/accelerate-cpu/Dockerfile
+++ b/docker/accelerate-cpu/Dockerfile
@@ -21,8 +21,8 @@ WORKDIR /workspace
 RUN python3 -m pip install --upgrade --no-cache-dir pip
 RUN python3 -m pip install --no-cache-dir \
     jupyter \
-    torch --extra-index-url https://download.pytorch.org/whl/cpu \
-    git+https://github.com/huggingface/accelerate#egg=accelerate[dev]
+    git+https://github.com/huggingface/accelerate#egg=accelerate[test,test_trackers] \
+    --extra-index-url https://download.pytorch.org/whl/cpu
     
 # Stage 2
 FROM python:3.6-slim AS build-image
diff --git a/docker/accelerate-gpu/Dockerfile b/docker/accelerate-gpu/Dockerfile
index 299f573d8..6195ae2ae 100644
--- a/docker/accelerate-gpu/Dockerfile
+++ b/docker/accelerate-gpu/Dockerfile
@@ -4,7 +4,7 @@
 # Use base conda image to reduce time
 FROM continuumio/miniconda3:latest AS compile-image
 # Specify py version
-ENV PYTHON_VERSION=3.6 
+ENV PYTHON_VERSION=3.6
 # Install apt libs
 RUN apt-get update && \
     apt-get install -y curl git wget && \
@@ -22,17 +22,21 @@ SHELL ["/bin/bash", "-c"]
 # Activate the conda env and install torch + accelerate
 RUN source activate accelerate && \
     python3 -m pip install --no-cache-dir \
-    torch --extra-index-url https://download.pytorch.org/whl/cu113 \
-    git+https://github.com/huggingface/accelerate#egg=accelerate[dev]
+    git+https://github.com/huggingface/accelerate@nightly-runner#egg=accelerate[test,test_trackers] \
+    --extra-index-url https://download.pytorch.org/whl/cu113
 
 # Stage 2
 FROM nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 AS build-image
 COPY --from=compile-image /opt/conda /opt/conda
 ENV PATH /opt/conda/bin:$PATH
 
-RUN echo "source activate accelerate" >> /.bashrc
+# Install apt libs
+RUN apt-get update && \
+    apt-get install -y curl git wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists*
+
+RUN echo "source activate accelerate" >> ~/.profile
 
-RUN useradd -ms /bin/bash user
-USER user
 # Activate the virtualenv
 CMD ["/bin/bash"]
\ No newline at end of file
diff --git a/setup.py b/setup.py
index dcfe3fd8e..5f6e128c1 100644
--- a/setup.py
+++ b/setup.py
@@ -19,6 +19,7 @@
 extras["quality"] = ["black ~= 22.0", "isort >= 5.5.4", "flake8 >= 3.8.3"]
 extras["docs"] = []
 extras["test"] = [
+    "psutil",
     "pytest",
     "pytest-xdist",
     "pytest-subtests",
@@ -27,7 +28,7 @@
     "scipy",
     "sklearn"
 ]
-extras["test_trackers"] = ["wandb", "comet-ml", "tensorflow"]
+extras["test_trackers"] = ["wandb", "comet-ml", "tensorflow>=2.6.2", "tensorboard"]
 extras["dev"] = extras["quality"] + extras["test"]
 
 extras["sagemaker"] = [
diff --git a/src/accelerate/test_utils/training.py b/src/accelerate/test_utils/training.py
index c26587abe..7345b93cb 100644
--- a/src/accelerate/test_utils/training.py
+++ b/src/accelerate/test_utils/training.py
@@ -17,8 +17,6 @@
 from torch.utils.data import DataLoader
 
 from accelerate.utils.dataclasses import DistributedType
-from datasets import load_dataset
-from transformers import AutoTokenizer
 
 
 class RegressionDataset:
@@ -51,6 +49,9 @@ def forward(self, x=None):
 
 
 def mocked_dataloaders(accelerator, batch_size: int = 16):
+    from datasets import load_dataset
+    from transformers import AutoTokenizer
+
     tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
     data_files = {"train": "tests/test_samples/MRPC/train.csv", "validation": "tests/test_samples/MRPC/dev.csv"}
     datasets = load_dataset("csv", data_files=data_files)
diff --git a/tests/test_examples.py b/tests/test_examples.py
index 6ec2a143a..8f277b932 100644
--- a/tests/test_examples.py
+++ b/tests/test_examples.py
@@ -21,6 +21,8 @@
 import unittest
 from unittest import mock
 
+import torch
+
 from accelerate.test_utils.examples import compare_against_test
 from accelerate.test_utils.testing import TempDirTestCase, slow
 from accelerate.utils import write_basic_config
@@ -142,11 +144,11 @@ def test_checkpointing_by_epoch(self):
     def test_checkpointing_by_steps(self):
         testargs = f"""
         examples/by_feature/checkpointing.py
-        --checkpointing_steps 2
+        --checkpointing_steps 1
         --output_dir {self.tmpdir}
         """.split()
         _ = subprocess.run(self._launch_args + testargs, stdout=subprocess.PIPE, env=os.environ)
-        self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "step_4")))
+        self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "step_5")))
 
     def test_load_states_by_epoch(self):
         testargs = f"""
@@ -163,14 +165,23 @@ def test_load_states_by_epoch(self):
     def test_load_states_by_steps(self):
         testargs = f"""
         examples/by_feature/checkpointing.py
-        --resume_from_checkpoint {os.path.join(self.tmpdir, "step_4")}
+        --resume_from_checkpoint {os.path.join(self.tmpdir, "step_5")}
         """.split()
         output = subprocess.run(
             self._launch_args + testargs, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
         ).stdout
-        self.assertNotIn("epoch 0:", output)
-        self.assertIn("epoch 1:", output)
-        self.assertIn("epoch 2:", output)
+        if torch.cuda.is_available():
+            num_processes = torch.cuda.device_count()
+        else:
+            num_processes = 1
+        if num_processes > 1:
+            self.assertNotIn("epoch 0:", output)
+            self.assertNotIn("epoch 1:", output)
+            self.assertIn("epoch 2:", output)
+        else:
+            self.assertNotIn("epoch 0:", output)
+            self.assertIn("epoch 1:", output)
+            self.assertIn("epoch 2:", output)
 
     @slow
     def test_cross_validation(self):


Review comments: body: Why was this necessary?
diff_hunk: @@ -0,0 +1,65 @@
+name: Self-hosted runner (scheduled)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 2 * * *"
+
+jobs:
+  run_all_tests_single_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      CUDA_VISIBLE_DEVICES: "0"
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone & pip install
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
+          git fetch && git checkout ${{ github.sha }}
+          pip install -e . --no-deps
+          pip install tensorflow -U
+      - name: Run test on GPUs
+        run: |
+          source activate accelerate
+          make test_cuda
+      - name: Run examples on GPUs
+        run: |
+          source activate accelerate
+          make test_examples
+
+  run_all_tests_multi_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
body: Is `tensorflow` required?
diff_hunk: @@ -0,0 +1,65 @@
+name: Self-hosted runner (scheduled)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 2 * * *"
+
+jobs:
+  run_all_tests_single_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      CUDA_VISIBLE_DEVICES: "0"
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone & pip install
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
+          git fetch && git checkout ${{ github.sha }}
+          pip install -e . --no-deps
+          pip install tensorflow -U
body: I was getting odd permission errors, and this was the documented solution I found. (Will post the link in the AM)
diff_hunk: @@ -0,0 +1,65 @@
+name: Self-hosted runner (scheduled)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 2 * * *"
+
+jobs:
+  run_all_tests_single_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      CUDA_VISIBLE_DEVICES: "0"
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone & pip install
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
+          git fetch && git checkout ${{ github.sha }}
+          pip install -e . --no-deps
+          pip install tensorflow -U
+      - name: Run test on GPUs
+        run: |
+          source activate accelerate
+          make test_cuda
+      - name: Run examples on GPUs
+        run: |
+          source activate accelerate
+          make test_examples
+
+  run_all_tests_multi_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
body: It won't be after the prior prs are merged, nice catch!
diff_hunk: @@ -0,0 +1,65 @@
+name: Self-hosted runner (scheduled)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 2 * * *"
+
+jobs:
+  run_all_tests_single_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      CUDA_VISIBLE_DEVICES: "0"
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone & pip install
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
+          git fetch && git checkout ${{ github.sha }}
+          pip install -e . --no-deps
+          pip install tensorflow -U
body: @LysandreJik here's the specific thread: https://github.com/actions/checkout/issues/760#issuecomment-1097519290

How we got here is the runner has a clone of accelerate inside the working directory for the action, and that clone is then updated, checked out, etc. But we run into this permission denied unsafe repository from github, so github needs to be told we can safely adjust the repository. 
diff_hunk: @@ -0,0 +1,65 @@
+name: Self-hosted runner (scheduled)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 2 * * *"
+
+jobs:
+  run_all_tests_single_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      CUDA_VISIBLE_DEVICES: "0"
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone & pip install
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
+          git fetch && git checkout ${{ github.sha }}
+          pip install -e . --no-deps
+          pip install tensorflow -U
+      - name: Run test on GPUs
+        run: |
+          source activate accelerate
+          make test_cuda
+      - name: Run examples on GPUs
+        run: |
+          source activate accelerate
+          make test_examples
+
+  run_all_tests_multi_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
body: Okay, interesting. Thanks!
diff_hunk: @@ -0,0 +1,65 @@
+name: Self-hosted runner (scheduled)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 2 * * *"
+
+jobs:
+  run_all_tests_single_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      CUDA_VISIBLE_DEVICES: "0"
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone & pip install
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'
+          git fetch && git checkout ${{ github.sha }}
+          pip install -e . --no-deps
+          pip install tensorflow -U
+      - name: Run test on GPUs
+        run: |
+          source activate accelerate
+          make test_cuda
+      - name: Run examples on GPUs
+        run: |
+          source activate accelerate
+          make test_examples
+
+  run_all_tests_multi_gpu:
+    runs-on: [self-hosted, docker-gpu, multi-gpu]
+    env:
+      ALL_SLOW: "yes"
+    container:
+      image: huggingface/accelerate-gpu:latest
+      options: --gpus all --shm-size "16gb"
+    defaults:
+      run:
+        working-directory: accelerate/
+        shell: bash
+    steps:
+      - name: Update clone
+        run: |
+          source activate accelerate
+          git config --global --add safe.directory '*'

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._