diff --git a/docs/source/package_reference/cli.mdx b/docs/source/package_reference/cli.mdx
index 092b72850..a9e955d43 100644
--- a/docs/source/package_reference/cli.mdx
+++ b/docs/source/package_reference/cli.mdx
@@ -94,6 +94,7 @@ The following arguments are useful for customization of worker machines
 * `--machine_rank MACHINE_RANK` (`int`) -- The rank of the machine on which this script is launched.
 * `--num_machines NUM_MACHINES` (`int`) -- The total number of machines used in this training.
 * `--num_processes NUM_PROCESSES` (`int`) -- The total number of processes to be launched in parallel.
+* `--gpu_ids` (`str`) -- What GPUs (by id) should be used for training on this machine as a comma-seperated list
 * `--main_process_ip MAIN_PROCESS_IP` (`str`) -- The IP address of the machine of rank 0.
 * `--main_process_port MAIN_PROCESS_PORT` (`int`) -- The port to use to communicate with the machine of rank 0.
 * `--num_cpu_threads_per_process NUM_CPU_THREADS_PER_PROCESS` (`int`) -- The number of CPU threads per process. Can be tuned for optimal performance.
diff --git a/src/accelerate/commands/config/cluster.py b/src/accelerate/commands/config/cluster.py
index a174115b1..01c419f4f 100644
--- a/src/accelerate/commands/config/cluster.py
+++ b/src/accelerate/commands/config/cluster.py
@@ -35,6 +35,7 @@ def get_cluster_input():
 
     machine_rank = 0
     num_machines = 1
+    gpu_ids = None
     main_process_ip = None
     main_process_port = None
     rdzv_backend = "static"
@@ -293,6 +294,12 @@ def get_cluster_input():
             default=1,
             error_message="Please enter an integer.",
         )
+
+    if distributed_type in [DistributedType.MULTI_GPU, DistributedType.NO] and not use_cpu:
+        gpu_ids = _ask_field(
+            "What GPU(s) (by id) should be used for training on this machine as a comma-seperated list? [all]:",
+            default="all",
+        )
     elif distributed_type in [DistributedType.FSDP, DistributedType.DEEPSPEED]:
         num_processes = _ask_field(
             "How many GPU(s) should be used for distributed training? [1]:",
@@ -325,6 +332,7 @@ def get_cluster_input():
         compute_environment=ComputeEnvironment.LOCAL_MACHINE,
         distributed_type=distributed_type,
         num_processes=num_processes,
+        gpu_ids=gpu_ids,
         mixed_precision=mixed_precision,
         downcast_bf16=downcast_bf16,
         machine_rank=machine_rank,
diff --git a/src/accelerate/commands/config/config_args.py b/src/accelerate/commands/config/config_args.py
index 68da64963..43faf45ca 100644
--- a/src/accelerate/commands/config/config_args.py
+++ b/src/accelerate/commands/config/config_args.py
@@ -135,6 +135,7 @@ class ClusterConfig(BaseConfig):
     num_processes: int
     machine_rank: int = 0
     num_machines: int = 1
+    gpu_ids: Optional[str] = None
     main_process_ip: Optional[str] = None
     main_process_port: Optional[int] = None
     rdzv_backend: Optional[str] = "static"
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index d1905d1c0..982293a33 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -259,6 +259,11 @@ def launch_command_parser(subparsers=None):
     parser.add_argument(
         "--num_machines", type=int, default=None, help="The total number of machines used in this training."
     )
+    parser.add_argument(
+        "--gpu_ids",
+        default=None,
+        help="What GPUs (by id) should be used for training on this machine as a comma-seperated list",
+    )
     parser.add_argument(
         "--machine_rank", type=int, default=None, help="The rank of the machine on which this script is launched."
     )
@@ -366,6 +371,8 @@ def simple_launcher(args):
     current_env["USE_MPS_DEVICE"] = str(args.use_mps_device)
     if args.use_mps_device:
         current_env["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+    elif args.gpu_ids != "all":
+        current_env["CUDA_VISIBLE_DEVICES"] = args.gpu_ids
     if args.num_machines > 1:
         current_env["MASTER_ADDR"] = args.main_process_ip
         current_env["MASTER_PORT"] = str(args.main_process_port)
@@ -420,6 +427,9 @@ def multi_gpu_launcher(args):
         setattr(args, "no_python", True)
 
     current_env = os.environ.copy()
+    gpu_ids = getattr(args, "gpu_ids")
+    if gpu_ids != "all":
+        current_env["CUDA_VISIBLE_DEVICES"] = gpu_ids
     mixed_precision = args.mixed_precision.lower()
     try:
         mixed_precision = PrecisionType(mixed_precision)
@@ -549,6 +559,9 @@ def deepspeed_launcher(args):
         setattr(args, "no_python", True)
 
     current_env = os.environ.copy()
+    gpu_ids = getattr(args, "gpu_ids")
+    if gpu_ids != "all":
+        current_env["CUDA_VISIBLE_DEVICES"] = gpu_ids
     try:
         mixed_precision = PrecisionType(args.mixed_precision.lower())
     except ValueError:
@@ -817,6 +830,14 @@ def launch_command(args):
             args.tpu = defaults.distributed_type == DistributedType.TPU
             args.use_fsdp = defaults.distributed_type == DistributedType.FSDP
             args.use_mps_device = defaults.distributed_type == DistributedType.MPS
+        if not args.use_mps_device:
+            if args.gpu_ids is None:
+                if defaults.gpu_ids is not None:
+                    args.gpu_ids = defaults.gpu_ids
+                else:
+                    args.gpu_ids = "all"
+            if len(args.gpu_ids.split(",")) < 2 and args.multi_gpu and (args.gpu_ids != "all"):
+                args.multi_gpu = False
         if defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:
             # Update args with the defaults
             for name, attr in defaults.__dict__.items():


Review comments: body: I meant with respect to these changes

diff_hunk: @@ -817,6 +830,13 @@ def launch_command(args):
             args.tpu = defaults.distributed_type == DistributedType.TPU
             args.use_fsdp = defaults.distributed_type == DistributedType.FSDP
             args.use_mps_device = defaults.distributed_type == DistributedType.MPS
+        if args.gpu_ids is None:
+            if defaults.gpu_ids is not None:
+                args.gpu_ids = defaults.gpu_ids
+            else:
+                args.gpu_ids = "all"
+        if len(args.gpu_ids.split(",")) < 2 and args.multi_gpu and (args.gpu_ids != "all"):
+            args.multi_gpu = False
body: Let me know what you think about my solution here: https://github.com/huggingface/accelerate/pull/732/commits/0467d4ff0bacda7ffb0d6087a7ef636cfbbb4960
diff_hunk: @@ -817,6 +830,13 @@ def launch_command(args):
             args.tpu = defaults.distributed_type == DistributedType.TPU
             args.use_fsdp = defaults.distributed_type == DistributedType.FSDP
             args.use_mps_device = defaults.distributed_type == DistributedType.MPS
+        if args.gpu_ids is None:
+            if defaults.gpu_ids is not None:
+                args.gpu_ids = defaults.gpu_ids
+            else:
+                args.gpu_ids = "all"
+        if len(args.gpu_ids.split(",")) < 2 and args.multi_gpu and (args.gpu_ids != "all"):
+            args.multi_gpu = False
body: Nice! This should have expected behaviour. 🤗
diff_hunk: @@ -817,6 +830,13 @@ def launch_command(args):
             args.tpu = defaults.distributed_type == DistributedType.TPU
             args.use_fsdp = defaults.distributed_type == DistributedType.FSDP
             args.use_mps_device = defaults.distributed_type == DistributedType.MPS
+        if args.gpu_ids is None:
+            if defaults.gpu_ids is not None:
+                args.gpu_ids = defaults.gpu_ids
+            else:
+                args.gpu_ids = "all"
+        if len(args.gpu_ids.split(",")) < 2 and args.multi_gpu and (args.gpu_ids != "all"):
+            args.multi_gpu = False

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: @pacman100 correct, this is solely for non-M1 GPUs, I'll specify that in the info description for it