diff --git a/src/accelerate/commands/config/cluster.py b/src/accelerate/commands/config/cluster.py
index 534ea1526..15eaaecb6 100644
--- a/src/accelerate/commands/config/cluster.py
+++ b/src/accelerate/commands/config/cluster.py
@@ -52,6 +52,17 @@ def get_cluster_input():
                 lambda x: int(x),
             )
 
+    if distributed_type == DistributedType.NO:
+        use_cpu = _ask_field(
+            "Do you want to run your training on CPU only (even if a GPU is available)? [no]:",
+            lambda x: bool(x),
+            default=False,
+        )
+    elif distributed_type == DistributedType.MULTI_CPU:
+        use_cpu = True
+    else:
+        use_cpu = False
+
     deepspeed_config = None
     if distributed_type in [DistributedType.MULTI_GPU, DistributedType.NO]:
         use_deepspeed = _ask_field(
@@ -122,4 +133,5 @@ def get_cluster_input():
         main_process_port=main_process_port,
         main_training_function=main_training_function,
         deepspeed_config=deepspeed_config,
+        use_cpu=use_cpu,
     )
diff --git a/src/accelerate/commands/config/config_args.py b/src/accelerate/commands/config/config_args.py
index 891580ab5..5af912fc3 100644
--- a/src/accelerate/commands/config/config_args.py
+++ b/src/accelerate/commands/config/config_args.py
@@ -67,6 +67,7 @@ class BaseConfig:
     compute_environment: ComputeEnvironment
     distributed_type: Union[DistributedType, SageMakerDistributedType]
     mixed_precision: str
+    use_cpu: bool
 
     def to_dict(self):
         result = self.__dict__
@@ -87,6 +88,8 @@ def from_json_file(cls, json_file=None):
             config_dict["mixed_precision"] = "fp16" if ("fp16" in config_dict and config_dict["fp16"]) else "no"
         if "fp16" in config_dict:  # Convert the config to the new format.
             del config_dict["fp16"]
+        if "use_cpu" not in config_dict:
+            config_dict["use_cpu"] = False
         return cls(**config_dict)
 
     def to_json_file(self, json_file):
@@ -106,6 +109,8 @@ def from_yaml_file(cls, yaml_file=None):
             config_dict["mixed_precision"] = "fp16" if ("fp16" in config_dict and config_dict["fp16"]) else "no"
         if "fp16" in config_dict:  # Convert the config to the new format.
             del config_dict["fp16"]
+        if "use_cpu" not in config_dict:
+            config_dict["use_cpu"] = False
 
         return cls(**config_dict)
 
diff --git a/src/accelerate/launchers.py b/src/accelerate/launchers.py
index a8429db96..a47eb9a4c 100644
--- a/src/accelerate/launchers.py
+++ b/src/accelerate/launchers.py
@@ -171,6 +171,7 @@ def debug_launcher(function, args=(), num_processes=2):
             master_port="29500",
             mixed_precision="no",
             accelerate_debug_rdv_file=tmp_file.name,
+            use_cpu="yes",
         ):
             launcher = PrepareForLaunch(function, debug=True)
             start_processes(launcher, args=args, nprocs=num_processes, start_method="fork")
diff --git a/src/accelerate/state.py b/src/accelerate/state.py
index 59aebef2d..a029acfae 100644
--- a/src/accelerate/state.py
+++ b/src/accelerate/state.py
@@ -153,6 +153,8 @@ def __init__(
         **kwargs,
     ):
         self.__dict__ = self._shared_state
+        if parse_flag_from_env("USE_CPU"):
+            cpu = True
         if not getattr(self, "initialized", False):
             self.backend = None
             self.deepspeed_plugin = None
@@ -217,14 +219,7 @@ def __init__(
                 rank = get_int_from_env(["RANK", "PMI_RANK", "OMPI_COMM_WORLD_RANK", "MV2_COMM_WORLD_RANK"], 0)
                 size = get_int_from_env(["WORLD_SIZE", "PMI_SIZE", "OMPI_COMM_WORLD_SIZE", "MV2_COMM_WORLD_SIZE"], 1)
                 local_rank = get_int_from_env(
-                    [
-                        "LOCAL_RANK",
-                        "CPU_LOCAL_RANK",
-                        "MPI_LOCALRANKID",
-                        "OMPI_COMM_WORLD_LOCAL_RANK",
-                        "MV2_COMM_WORLD_LOCAL_RANK",
-                    ],
-                    0,
+                    ["LOCAL_RANK", "MPI_LOCALRANKID", "OMPI_COMM_WORLD_LOCAL_RANK", "MV2_COMM_WORLD_LOCAL_RANK"], 0
                 )
                 local_size = get_int_from_env(
                     ["MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE"], 1
diff --git a/src/accelerate/utils.py b/src/accelerate/utils.py
index 6eba4b295..83a40bf86 100644
--- a/src/accelerate/utils.py
+++ b/src/accelerate/utils.py
@@ -614,8 +614,6 @@ def __call__(self, index, *args):
                 store=torch.distributed.FileStore(rdv_file, world_size),
                 world_size=world_size,
             )
-            # Prepare the environment for torch.distributed
-            os.environ["CPU_LOCAL_RANK"] = str(index)
         elif self.distributed_type == DistributedType.MULTI_GPU or self.distributed_type == DistributedType.MULTI_CPU:
             # Prepare the environment for torch.distributed
             os.environ["LOCAL_RANK"] = str(index)