diff --git a/src/accelerate/__init__.py b/src/accelerate/__init__.py
index e0b96cd10..f77a26a56 100644
--- a/src/accelerate/__init__.py
+++ b/src/accelerate/__init__.py
@@ -6,6 +6,6 @@
 
 from .accelerator import Accelerator
 from .kwargs_handlers import DistributedDataParallelKwargs, GradScalerKwargs
-from .notebook_launcher import notebook_launcher
+from .launchers import debug_launcher, notebook_launcher
 from .state import DistributedType
 from .utils import DeepSpeedPlugin, synchronize_rng_states
diff --git a/src/accelerate/notebook_launcher.py b/src/accelerate/launchers.py
similarity index 75%
rename from src/accelerate/notebook_launcher.py
rename to src/accelerate/launchers.py
index 90b4f826a..a8429db96 100644
--- a/src/accelerate/notebook_launcher.py
+++ b/src/accelerate/launchers.py
@@ -14,6 +14,7 @@
 
 import os
 import sys
+import tempfile
 import warnings
 
 import torch
@@ -21,7 +22,7 @@
 from packaging import version
 
 from .state import AcceleratorState
-from .utils import PrepareForLaunch
+from .utils import PrepareForLaunch, patch_environment
 
 
 def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, mixed_precision="no", use_port="29500"):
@@ -106,12 +107,6 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, mix
                     "function."
                 )
 
-            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
-            # process here (the other ones will be set be the launcher).
-            os.environ["WORLD_SIZE"] = str(num_processes)
-            os.environ["MASTER_ADDR"] = "127.0.0.1"
-            os.environ["MASTER_PORT"] = str(use_port)
-
             mixed_precision = mixed_precision.lower()
             if mixed_precision not in ["no", "fp16", "bf16"]:
                 raise ValueError(
@@ -122,17 +117,15 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, mix
                 warnings.warn('use_fp16=True is deprecated. Use mixed_precision="fp16" instead.', DeprecationWarning)
                 mixed_precision = "fp16"
 
-            os.environ["MIXED_PRECISION"] = str(mixed_precision)
+            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
+            # process here (the other ones will be set be the launcher).
+            with patch_environment(
+                world_size=num_processes, master_addr="127.0.01", master_port=use_port, mixed_precision=mixed_precision
+            ):
+                launcher = PrepareForLaunch(function, distributed_type="MULTI_GPU")
 
-            launcher = PrepareForLaunch(function, distributed_type="MULTI_GPU")
-            try:
                 print(f"Launching training on {num_processes} GPUs.")
                 start_processes(launcher, args=args, nprocs=num_processes, start_method="fork")
-            finally:
-                # Clean up the environment variables set.
-                del os.environ["WORLD_SIZE"]
-                del os.environ["MASTER_ADDR"]
-                del os.environ["MASTER_PORT"]
 
         else:
             # No need for a distributed launch otherwise as it's either CPU or one GPU.
@@ -141,3 +134,43 @@ def notebook_launcher(function, args=(), num_processes=None, use_fp16=False, mix
             else:
                 print("Launching training on CPU.")
             function(*args)
+
+
+def debug_launcher(function, args=(), num_processes=2):
+    """
+    Launches a training function using several processes on CPU for debugging purposes.
+
+    .. warning::
+
+        This function is provided for internal testing and debugging, but it's not intended for real trainings. It will
+        only use the CPU.
+
+
+    Args:
+        function (:obj:`Callable`):
+            The training function to execute.
+        args (:obj:`Tuple`):
+            Tuple of arguments to pass to the function (it will receive :obj:`*args`).
+        num_processes (:obj:`int`, *optional*, defaults to 2):
+            The number of processes to use for training.
+    """
+    if version.parse(torch.__version__) < version.parse("1.5.0"):
+        raise ImportError(
+            "Using `debug_launcher` for distributed training on GPUs require torch >= 1.5.0, got "
+            f"{torch.__version__}."
+        )
+
+    from torch.multiprocessing import start_processes
+
+    with tempfile.NamedTemporaryFile() as tmp_file:
+        # torch.distributed will expect a few environment variable to be here. We set the ones common to each
+        # process here (the other ones will be set be the launcher).
+        with patch_environment(
+            world_size=num_processes,
+            master_addr="127.0.01",
+            master_port="29500",
+            mixed_precision="no",
+            accelerate_debug_rdv_file=tmp_file.name,
+        ):
+            launcher = PrepareForLaunch(function, debug=True)
+            start_processes(launcher, args=args, nprocs=num_processes, start_method="fork")
diff --git a/src/accelerate/state.py b/src/accelerate/state.py
index 6693a34ca..59aebef2d 100644
--- a/src/accelerate/state.py
+++ b/src/accelerate/state.py
@@ -217,7 +217,14 @@ def __init__(
                 rank = get_int_from_env(["RANK", "PMI_RANK", "OMPI_COMM_WORLD_RANK", "MV2_COMM_WORLD_RANK"], 0)
                 size = get_int_from_env(["WORLD_SIZE", "PMI_SIZE", "OMPI_COMM_WORLD_SIZE", "MV2_COMM_WORLD_SIZE"], 1)
                 local_rank = get_int_from_env(
-                    ["LOCAL_RANK", "MPI_LOCALRANKID", "OMPI_COMM_WORLD_LOCAL_RANK", "MV2_COMM_WORLD_LOCAL_RANK"], 0
+                    [
+                        "LOCAL_RANK",
+                        "CPU_LOCAL_RANK",
+                        "MPI_LOCALRANKID",
+                        "OMPI_COMM_WORLD_LOCAL_RANK",
+                        "MV2_COMM_WORLD_LOCAL_RANK",
+                    ],
+                    0,
                 )
                 local_size = get_int_from_env(
                     ["MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE"], 1
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 36eaea592..443c55622 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -16,6 +16,7 @@
 import os
 import random
 from collections.abc import Mapping
+from contextlib import contextmanager
 from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, List, Optional, Union
@@ -594,14 +595,28 @@ class PrepareForLaunch:
             The function to launch.
         distributed_type (:class:`~accelerate.state.DistributedType`):
             The distributed type to prepare for.
+        debug (:obj:`bool`, *optional*, defaults to :obj:`False`):
+            Whether or not this is a debug launch.
     """
 
-    def __init__(self, launcher, distributed_type="NO"):
+    def __init__(self, launcher, distributed_type="NO", debug=False):
         self.launcher = launcher
         self.distributed_type = DistributedType(distributed_type)
+        self.debug = debug
 
     def __call__(self, index, *args):
-        if self.distributed_type == DistributedType.MULTI_GPU or self.distributed_type == DistributedType.MULTI_CPU:
+        if self.debug:
+            world_size = int(os.environ.get("WORLD_SIZE"))
+            rdv_file = os.environ.get("ACCELERATE_DEBUG_RDV_FILE")
+            torch.distributed.init_process_group(
+                "gloo",
+                rank=index,
+                store=torch.distributed.FileStore(rdv_file, world_size),
+                world_size=world_size,
+            )
+            # Prepare the environment for torch.distributed
+            os.environ["CPU_LOCAL_RANK"] = str(index)
+        elif self.distributed_type == DistributedType.MULTI_GPU or self.distributed_type == DistributedType.MULTI_CPU:
             # Prepare the environment for torch.distributed
             os.environ["LOCAL_RANK"] = str(index)
             os.environ["RANK"] = str(index)
@@ -654,3 +669,19 @@ def __post_init__(self):
             "steps_per_print": float("inf"),  # this will stop deepspeed from logging @ stdout
             "zero_allow_untested_optimizer": True,
         }
+
+
+@contextmanager
+def patch_environment(**kwargs):
+    """
+    A context manager that will add each keyword argument passed to ``os.environ`` and remove them when exiting.
+
+    Will convert the values in :obj:`kwargs` to strings and upper-case all the keys.
+    """
+    for key, value in kwargs.items():
+        os.environ[key.upper()] = str(value)
+
+    yield
+
+    for key in kwargs:
+        del os.environ[key.upper()]
diff --git a/tests/test_cpu.py b/tests/test_cpu.py
new file mode 100644
index 000000000..72a8b2e7d
--- /dev/null
+++ b/tests/test_cpu.py
@@ -0,0 +1,23 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from accelerate import debug_launcher
+from accelerate.test_utils import test_script
+
+
+class MultiTPUTester(unittest.TestCase):
+    def test_cpu(self):
+        debug_launcher(test_script.main)
diff --git a/tests/test_utils.py b/tests/test_utils.py
index e515df8fe..a31c45d31 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -12,12 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import unittest
 from collections import UserDict, namedtuple
 
 import torch
 
-from accelerate.utils import send_to_device
+from accelerate.utils import patch_environment, send_to_device
 
 
 TestNamedTuple = namedtuple("TestNamedTuple", "a b c")
@@ -62,3 +63,11 @@ def test_send_to_device(self):
         self.assertTrue(torch.equal(result4["b"][0].cpu(), tensor))
         self.assertTrue(torch.equal(result4["b"][1].cpu(), tensor))
         self.assertEqual(result4["c"], 1)
+
+    def test_patch_environment(self):
+        with patch_environment(aa=1, BB=2):
+            self.assertEqual(os.environ.get("AA"), "1")
+            self.assertEqual(os.environ.get("BB"), "2")
+
+        self.assertNotIn("AA", os.environ)
+        self.assertNotIn("BB", os.environ)