diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py
index 9b6e7f069..e781a5cea 100644
--- a/src/accelerate/accelerator.py
+++ b/src/accelerate/accelerator.py
@@ -272,7 +272,7 @@ def __init__(
                 raise ValueError(err.format(mode="bf16", requirement="PyTorch >= 1.10 and a supported device."))
 
             # Only on the GPU do we care about scaling the gradients
-            if torch.cuda.is_available():
+            if torch.cuda.is_available() and self.device.type != "cpu":
                 kwargs = self.scaler_handler.to_kwargs() if self.scaler_handler is not None else {}
                 self.scaler = torch.cuda.amp.GradScaler(**kwargs)
 
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index cffe9990b..bda9e6b38 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -256,7 +256,14 @@ def simple_launcher(args):
     cmd.extend(args.training_script_args)
 
     current_env = os.environ.copy()
-    current_env["USE_CPU"] = str(args.cpu)
+    current_env["USE_CPU"] = str(args.cpu or args.use_cpu)
+    if args.num_machines > 1:
+        current_env["MASTER_ADDR"] = args.main_process_ip
+        current_env["MASTER_PORT"] = str(args.main_process_port)
+    elif args.num_processes > 1:
+        current_env["MASTER_ADDR"] = args.main_process_ip if args.main_process_ip is not None else "127.0.0.1"
+        current_env["MASTER_PORT"] = str(args.main_process_port) if args.main_process_port is not None else "29500"
+
     try:
         mixed_precision = PrecisionType(args.mixed_precision.lower())
     except ValueError:
@@ -269,6 +276,7 @@ def simple_launcher(args):
         mixed_precision = "fp16"
 
     current_env["MIXED_PRECISION"] = str(mixed_precision)
+    current_env["OMP_NUM_THREADS"] = str(args.num_cpu_threads_per_process)
 
     process = subprocess.Popen(cmd, env=current_env)
     process.wait()


Review comments: body: Should this be under and `if args.num_machines > 1`? It seems related to multinode training.
diff_hunk: @@ -257,6 +257,8 @@ def simple_launcher(args):
 
     current_env = os.environ.copy()
     current_env["USE_CPU"] = str(args.cpu)
+    current_env["MASTER_ADDR"] = args.main_process_ip
+    current_env["MASTER_PORT"] = str(args.main_process_port)
body: thanks for the suggestion, a patch is uploaded to fix it
diff_hunk: @@ -257,6 +257,8 @@ def simple_launcher(args):
 
     current_env = os.environ.copy()
     current_env["USE_CPU"] = str(args.cpu)
+    current_env["MASTER_ADDR"] = args.main_process_ip
+    current_env["MASTER_PORT"] = str(args.main_process_port)
body: If they are needed, I would rather use the ones set par the user than hardcoding a default, which may create hard-to-debug issues for users.
diff_hunk: @@ -257,6 +257,12 @@ def simple_launcher(args):
 
     current_env = os.environ.copy()
     current_env["USE_CPU"] = str(args.cpu)
+    if args.num_machines > 1:
+        current_env["MASTER_ADDR"] = args.main_process_ip
+        current_env["MASTER_PORT"] = str(args.main_process_port)
+    elif args.num_processes > 1:
+        current_env["MASTER_ADDR"] = "127.0.0.1"
+        current_env["MASTER_PORT"] = "29500"
body: hi, sgugger, now in the accelerate config, if num_machines == 1, the master ip and port is not set. and if num_process > 1 and DDP is used in one machine, we still need the master ip and port
diff_hunk: @@ -257,6 +257,12 @@ def simple_launcher(args):
 
     current_env = os.environ.copy()
     current_env["USE_CPU"] = str(args.cpu)
+    if args.num_machines > 1:
+        current_env["MASTER_ADDR"] = args.main_process_ip
+        current_env["MASTER_PORT"] = str(args.main_process_port)
+    elif args.num_processes > 1:
+        current_env["MASTER_ADDR"] = "127.0.0.1"
+        current_env["MASTER_PORT"] = "29500"
body: But they could be set in the args passed, so maybe only use this default if the args is empty/None?
diff_hunk: @@ -257,6 +257,12 @@ def simple_launcher(args):
 
     current_env = os.environ.copy()
     current_env["USE_CPU"] = str(args.cpu)
+    if args.num_machines > 1:
+        current_env["MASTER_ADDR"] = args.main_process_ip
+        current_env["MASTER_PORT"] = str(args.main_process_port)
+    elif args.num_processes > 1:
+        current_env["MASTER_ADDR"] = "127.0.0.1"
+        current_env["MASTER_PORT"] = "29500"
body: done, thanks for the suggestion
diff_hunk: @@ -257,6 +257,12 @@ def simple_launcher(args):
 
     current_env = os.environ.copy()
     current_env["USE_CPU"] = str(args.cpu)
+    if args.num_machines > 1:
+        current_env["MASTER_ADDR"] = args.main_process_ip
+        current_env["MASTER_PORT"] = str(args.main_process_port)
+    elif args.num_processes > 1:
+        current_env["MASTER_ADDR"] = "127.0.0.1"
+        current_env["MASTER_PORT"] = "29500"

Discussion: body: @sgugger @yao-matrix please help review
body: _The documentation is not available anymore as the PR was closed or merged._
body: Thansk again for your contribution!