diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 1d6f7ee10..85fc1b4d2 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -365,7 +365,10 @@ def __init__(
             and self.distributed_type != DistributedType.FSDP
             and self.distributed_type != DistributedType.MEGATRON_LM
         ):
-            self.native_amp = is_bf16_available(True)
+            if self.device.type == "cpu":
+                self.native_amp = is_torch_version(">=", "1.10")
+            else:
+                self.native_amp = is_bf16_available(True)
             if mixed_precision == "bf16" and not self.native_amp and not is_tpu_available():
                 raise ValueError(err.format(mode="bf16", requirement="PyTorch >= 1.10 and a supported device."))
 
@@ -948,8 +951,7 @@ def prepare_model(self, model: torch.nn.Module, device_placement=None):
             if self.mixed_precision == "fp16" and is_torch_version(">=", "1.10"):
                 model.forward = torch.cuda.amp.autocast(dtype=torch.float16)(model.forward)
             elif self.mixed_precision == "bf16" and self.distributed_type != DistributedType.TPU:
-                device_type = "cuda" if torch.cuda.is_available() else "cpu"
-                model.forward = torch.autocast(device_type=device_type, dtype=torch.bfloat16)(model.forward)
+                model.forward = torch.autocast(device_type=self.device.type, dtype=torch.bfloat16)(model.forward)
             else:
                 model.forward = torch.cuda.amp.autocast()(model.forward)
             model.forward = convert_outputs_to_fp32(model.forward)
@@ -1847,8 +1849,7 @@ def autocast(self):
                 autocast_context = torch.cuda.amp.autocast(dtype=torch.float16)
             elif self.mixed_precision == "bf16" and is_bf16_available():
                 if self.distributed_type in [DistributedType.NO, DistributedType.MULTI_CPU, DistributedType.MULTI_GPU]:
-                    device_type = "cpu" if not torch.cuda.is_available() else "cuda"
-                    autocast_context = torch.autocast(dtype=torch.bfloat16, device_type=device_type)
+                    autocast_context = torch.autocast(dtype=torch.bfloat16, device_type=self.device.type)
             else:
                 autocast_context = torch.cuda.amp.autocast()
 
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index 12f67088a..af3ba0691 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -1089,7 +1089,7 @@ def launch_command(args):
                 ["MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE"], 1
             )
             threads_per_process = int(psutil.cpu_count(logical=False) / local_size)
-            if args.num_cpu_threads_per_process > 1:
+            if threads_per_process > 1:
                 args.num_cpu_threads_per_process = threads_per_process
                 warned.append(
                     f"\t`--num_cpu_threads_per_process` was set to `{args.num_cpu_threads_per_process}` to improve out-of-box performance when training on CPUs"


Review comments: body: This line (and the next occurrence) can be removed entirely and just pass in `self.device.type` to `torch.autocast`
diff_hunk: @@ -948,7 +951,7 @@ def prepare_model(self, model: torch.nn.Module, device_placement=None):
             if self.mixed_precision == "fp16" and is_torch_version(">=", "1.10"):
                 model.forward = torch.cuda.amp.autocast(dtype=torch.float16)(model.forward)
             elif self.mixed_precision == "bf16" and self.distributed_type != DistributedType.TPU:
-                device_type = "cuda" if torch.cuda.is_available() else "cpu"
+                device_type = "cuda" if self.device.type != "cpu" and torch.cuda.is_available() else "cpu"
body: good point. done
diff_hunk: @@ -948,7 +951,7 @@ def prepare_model(self, model: torch.nn.Module, device_placement=None):
             if self.mixed_precision == "fp16" and is_torch_version(">=", "1.10"):
                 model.forward = torch.cuda.amp.autocast(dtype=torch.float16)(model.forward)
             elif self.mixed_precision == "bf16" and self.distributed_type != DistributedType.TPU:
-                device_type = "cuda" if torch.cuda.is_available() else "cpu"
+                device_type = "cuda" if self.device.type != "cpu" and torch.cuda.is_available() else "cpu"
body: done
diff_hunk: @@ -948,7 +951,7 @@ def prepare_model(self, model: torch.nn.Module, device_placement=None):
             if self.mixed_precision == "fp16" and is_torch_version(">=", "1.10"):
                 model.forward = torch.cuda.amp.autocast(dtype=torch.float16)(model.forward)
             elif self.mixed_precision == "bf16" and self.distributed_type != DistributedType.TPU:
-                device_type = "cuda" if torch.cuda.is_available() else "cpu"
+                device_type = "cuda" if self.device.type != "cpu" and torch.cuda.is_available() else "cpu"

Discussion: body: @sgugger @yao-matrix  @jiqing-feng  please notice the issue
body: _The documentation is not available anymore as the PR was closed or merged._