diff --git a/.github/workflows/run_merge_tests.yml b/.github/workflows/run_merge_tests.yml
index a794cd72b..fcbb62f53 100644
--- a/.github/workflows/run_merge_tests.yml
+++ b/.github/workflows/run_merge_tests.yml
@@ -29,6 +29,11 @@ jobs:
           pip install -e .[testing,test_trackers]
           pip install pytest-reportlog
 
+      - name: Run CLI tests
+        run: |
+          source activate accelerate
+          make test_cli
+          
       - name: Run test on GPUs
         run: |
           source activate accelerate
@@ -62,6 +67,11 @@ jobs:
           pip install -e .[testing,test_trackers]
           pip install pytest-reportlog
 
+      - name: Run CLI tests
+        run: |
+          source activate accelerate
+          make test_cli
+
       - name: Run test on GPUs
         run: |
           source activate accelerate
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 593eec741..2959316d8 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,6 +28,7 @@ jobs:
         test-kind: [
           test_prod,
           test_core,
+          test_cli,
           test_big_modeling,
           test_deepspeed,
           test_fsdp,
diff --git a/Makefile b/Makefile
index 52c594ae7..558e98347 100644
--- a/Makefile
+++ b/Makefile
@@ -32,7 +32,10 @@ test_big_modeling:
 
 test_core:
 	python -m pytest -s -v ./tests/ --ignore=./tests/test_examples.py --ignore=./tests/deepspeed --ignore=./tests/test_big_modeling.py \
-	--ignore=./tests/fsdp $(if $(IS_GITHUB_CI),--report-log 'core.log',)
+	--ignore=./tests/fsdp --ignore=./tests/test_cli.py $(if $(IS_GITHUB_CI),--report-log 'core.log',)
+
+test_cli:
+	python -m pytest -s -v ./tests/test_cli.py $(if $(IS_GITHUB_CI),--report-log 'cli.log',)
 
 test_deepspeed:
 	python -m pytest -s -v ./tests/deepspeed $(if $(IS_GITHUB_CI),--report-log 'deepspeed.log',)
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index 65e9be9c7..fa7588b0a 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -432,7 +432,7 @@ def simple_launcher(args):
     current_env["USE_MPS_DEVICE"] = str(args.use_mps_device)
     if args.use_mps_device:
         current_env["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
-    elif args.gpu_ids != "all":
+    elif args.gpu_ids != "all" and args.gpu_ids is not None:
         current_env["CUDA_VISIBLE_DEVICES"] = args.gpu_ids
     if args.num_machines > 1:
         current_env["MASTER_ADDR"] = args.main_process_ip
@@ -489,7 +489,7 @@ def multi_gpu_launcher(args):
 
     current_env = os.environ.copy()
     gpu_ids = getattr(args, "gpu_ids")
-    if gpu_ids != "all":
+    if gpu_ids != "all" and args.gpu_ids is not None:
         current_env["CUDA_VISIBLE_DEVICES"] = gpu_ids
     mixed_precision = args.mixed_precision.lower()
     try:
@@ -637,7 +637,7 @@ def deepspeed_launcher(args):
 
     current_env = os.environ.copy()
     gpu_ids = getattr(args, "gpu_ids")
-    if gpu_ids != "all":
+    if gpu_ids != "all" and args.gpu_ids is not None:
         current_env["CUDA_VISIBLE_DEVICES"] = gpu_ids
     try:
         mixed_precision = PrecisionType(args.mixed_precision.lower())
@@ -947,7 +947,7 @@ def launch_command(args):
     else:
         if args.num_processes is None:
             args.num_processes = torch.cuda.device_count() if args.multi_gpu else 1
-            warned.append("\t`--num_processes` was set to a value of `{args.num_processes}`")
+            warned.append(f"\t`--num_processes` was set to a value of `{args.num_processes}`")
         if args.num_machines is None:
             warned.append("\t`--num_machines` was set to a value of `1`")
             args.num_machines = 1
diff --git a/src/accelerate/test_utils/scripts/test_cli.py b/src/accelerate/test_utils/scripts/test_cli.py
new file mode 100644
index 000000000..491410e5f
--- /dev/null
+++ b/src/accelerate/test_utils/scripts/test_cli.py
@@ -0,0 +1,13 @@
+import torch
+
+
+def main():
+    if torch.cuda.is_available():
+        num_gpus = torch.cuda.device_count()
+    else:
+        num_gpus = 0
+    print(f"Successfully ran on {num_gpus} GPUs")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/test_cli.py b/tests/test_cli.py
new file mode 100644
index 000000000..ceed58290
--- /dev/null
+++ b/tests/test_cli.py
@@ -0,0 +1,65 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import os
+import unittest
+from pathlib import Path
+
+import torch
+
+import accelerate
+from accelerate.test_utils import execute_subprocess_async
+
+
+class AccelerateLauncherTester(unittest.TestCase):
+    """
+    Test case for verifying the `accelerate launch` CLI operates correctly.
+    If a `default_config.yaml` file is located in the cache it will temporarily move it
+    for the duration of the tests.
+    """
+
+    mod_file = inspect.getfile(accelerate.test_utils)
+    test_file_path = os.path.sep.join(mod_file.split(os.path.sep)[:-1] + ["scripts", "test_cli.py"])
+
+    base_cmd = ["accelerate", "launch"]
+    config_folder = Path.home() / ".cache/huggingface/accelerate"
+    config_file = "default_config.yaml"
+    config_path = config_folder / config_file
+    changed_path = config_folder / "_default_config.yaml"
+
+    test_config_path = Path("tests/test_configs")
+
+    @classmethod
+    def setUpClass(cls):
+        if cls.config_path.is_file():
+            cls.config_path.rename(cls.changed_path)
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.changed_path.is_file():
+            cls.changed_path.rename(cls.config_path)
+
+    def test_no_config(self):
+        cmd = self.base_cmd
+        if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
+            cmd += ["--multi_gpu"]
+        execute_subprocess_async(cmd + [self.test_file_path], env=os.environ.copy())
+
+    def test_config_compatibility(self):
+        for config in sorted(self.test_config_path.glob("**/*.yaml")):
+            with self.subTest(config_file=config):
+                execute_subprocess_async(
+                    self.base_cmd + ["--config_file", str(config), self.test_file_path], env=os.environ.copy()
+                )
diff --git a/tests/test_configs/0_11_0.yaml b/tests/test_configs/0_11_0.yaml
new file mode 100644
index 000000000..9ef829e6b
--- /dev/null
+++ b/tests/test_configs/0_11_0.yaml
@@ -0,0 +1,12 @@
+compute_environment: LOCAL_MACHINE
+deepspeed_config: {}
+distributed_type: 'NO'
+fsdp_config: {}
+machine_rank: 0
+main_process_ip: null
+main_process_port: null
+main_training_function: main
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 1
+use_cpu: false
\ No newline at end of file
diff --git a/tests/test_configs/0_12_0.yaml b/tests/test_configs/0_12_0.yaml
new file mode 100644
index 000000000..00d06aab9
--- /dev/null
+++ b/tests/test_configs/0_12_0.yaml
@@ -0,0 +1,13 @@
+compute_environment: LOCAL_MACHINE
+deepspeed_config: {}
+distributed_type: 'NO'
+downcast_bf16: 'no'
+fsdp_config: {}
+machine_rank: 0
+main_process_ip: null
+main_process_port: null
+main_training_function: main
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 1
+use_cpu: false
\ No newline at end of file
diff --git a/tests/test_configs/README.md b/tests/test_configs/README.md
new file mode 100644
index 000000000..fd88d066c
--- /dev/null
+++ b/tests/test_configs/README.md
@@ -0,0 +1,2 @@
+This folder contains test configs for `accelerate config`. These should be generated for each major version
+and are written based on `accelerate config` and selecting the "No distributed training" option.
\ No newline at end of file
diff --git a/tests/test_configs/latest.yaml b/tests/test_configs/latest.yaml
new file mode 100644
index 000000000..87b294cf8
--- /dev/null
+++ b/tests/test_configs/latest.yaml
@@ -0,0 +1,17 @@
+compute_environment: LOCAL_MACHINE
+deepspeed_config: {}
+distributed_type: 'NO'
+downcast_bf16: 'no'
+fsdp_config: {}
+gpu_ids: all
+machine_rank: 0
+main_process_ip: null
+main_process_port: null
+main_training_function: main
+megatron_lm_config: {}
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 1
+rdzv_backend: static
+same_network: true
+use_cpu: false


Review comments: body: This addition is needed for the test case of `CUDA_VISIBLE_DEVICES=""`
diff_hunk: @@ -489,7 +489,7 @@ def multi_gpu_launcher(args):
 
     current_env = os.environ.copy()
     gpu_ids = getattr(args, "gpu_ids")
-    if gpu_ids != "all":
+    if gpu_ids != "all" and args.gpu_ids is not None:

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._