diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index f348bb0a6..3e5eff45e 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -878,15 +878,17 @@ def launch_command(args):
             args.use_cpu = args.cpu
 
     if args.num_cpu_threads_per_process is None:
-        local_size = get_int_from_env(
-            ["MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE"], 1
-        )
-        args.num_cpu_threads_per_process = int(psutil.cpu_count(logical=False) / local_size)
-        if args.num_cpu_threads_per_process == 0:
-            args.num_cpu_threads_per_process = 1
-        warned.append(
-            f"\t`--num_cpu_threads_per_process` was set to `{args.num_cpu_threads_per_process}` to improve out-of-box performance"
-        )
+        args.num_cpu_threads_per_process = 1
+        if args.use_cpu and args.num_processes > 1:
+            local_size = get_int_from_env(
+                ["MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE"], 1
+            )
+            threads_per_process = int(psutil.cpu_count(logical=False) / local_size)
+            if args.num_cpu_threads_per_process > 1:
+                args.num_cpu_threads_per_process = threads_per_process
+                warned.append(
+                    f"\t`--num_cpu_threads_per_process` was set to `{args.num_cpu_threads_per_process}` to improve out-of-box performance when training on CPUs"
+                )
 
     if any(warned):
         message = "The following values were not passed to `accelerate launch` and had defaults used instead:\n"


Review comments: body: This means a chosen value of 1 is erased. Let's leave the default at `None` and change it at 1 when not in distributed CPU, or the other default when in distributed CPU.
diff_hunk: @@ -877,15 +877,15 @@ def launch_command(args):
         if not hasattr(args, "use_cpu"):
             args.use_cpu = args.cpu
 
-    if args.num_cpu_threads_per_process is None:
+    if args.num_cpu_threads_per_process == 1 and args.use_cpu:
body: Adjusted :) 
diff_hunk: @@ -877,15 +877,15 @@ def launch_command(args):
         if not hasattr(args, "use_cpu"):
             args.use_cpu = args.cpu
 
-    if args.num_cpu_threads_per_process is None:
+    if args.num_cpu_threads_per_process == 1 and args.use_cpu:
body: ```suggestion
    if args.num_cpu_threads_per_process is None:
```
Let's not rely on Python bool conversion magic please :-)
diff_hunk: @@ -877,16 +877,18 @@ def launch_command(args):
         if not hasattr(args, "use_cpu"):
             args.use_cpu = args.cpu
 
-    if args.num_cpu_threads_per_process is None:
-        local_size = get_int_from_env(
-            ["MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE"], 1
-        )
-        args.num_cpu_threads_per_process = int(psutil.cpu_count(logical=False) / local_size)
-        if args.num_cpu_threads_per_process == 0:
-            args.num_cpu_threads_per_process = 1
-        warned.append(
-            f"\t`--num_cpu_threads_per_process` was set to `{args.num_cpu_threads_per_process}` to improve out-of-box performance"
-        )
+    if not args.num_cpu_threads_per_process:

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._