diff --git a/setup.py b/setup.py
index 8247aa5fe..4bc977029 100644
--- a/setup.py
+++ b/setup.py
@@ -50,7 +50,7 @@
         ]
     },
     python_requires=">=3.7.0",
-    install_requires=["numpy>=1.17", "packaging>=20.0", "psutil", "pyyaml", "torch>=1.4.0"],
+    install_requires=["numpy>=1.17", "packaging>=20.0", "psutil", "pyyaml", "torch>=1.4.0", "rich"],
     extras_require=extras,
     classifiers=[
         "Development Status :: 5 - Production/Stable",
diff --git a/src/accelerate/__init__.py b/src/accelerate/__init__.py
index 145b0aa27..03a7198ac 100644
--- a/src/accelerate/__init__.py
+++ b/src/accelerate/__init__.py
@@ -17,5 +17,6 @@
     find_executable_batch_size,
     infer_auto_device_map,
     load_checkpoint_in_model,
+    rich,
     synchronize_rng_states,
 )
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index fcb26228a..ea27afb75 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -46,11 +46,15 @@
 )
 from accelerate.utils.constants import DEEPSPEED_MULTINODE_LAUNCHERS
 from accelerate.utils.dataclasses import SageMakerDistributedType
+from rich import get_console
+from rich.logging import RichHandler
 
 
 if is_torch_version(">=", "1.9.0"):
     import torch.distributed.run as distrib_run
 
+FORMAT = "%(message)s"
+logging.basicConfig(format=FORMAT, datefmt="[%X]", handlers=[RichHandler()])
 
 logger = logging.getLogger(__name__)
 
@@ -301,7 +305,12 @@ def launch_command_parser(subparsers=None):
         "--aws_secret_access_key",
         type=str,
         default=None,
-        help="The AWS_SECRET_ACCESS_KEY used to launch the Amazon SageMaker training job",
+        help="The AWS_SECRET_ACCESS_KEY used to launch the Amazon SageMaker training job.",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Whether to print out the torch.distributed stack trace when something fails.",
     )
     parser.add_argument(
         "training_script",
@@ -443,9 +452,17 @@ def multi_gpu_launcher(args):
             current_env["FSDP_STATE_DICT_TYPE"] = str(args.fsdp_state_dict_type)
     current_env["OMP_NUM_THREADS"] = str(args.num_cpu_threads_per_process)
     if is_torch_version(">=", "1.9.0"):
+        debug = getattr(args, "debug", False)
         args = _filter_args(args)
         with patch_environment(**current_env):
-            distrib_run.run(args)
+            console = get_console()
+
+            try:
+                distrib_run.run(args)
+            except:
+                if debug:
+                    console.print("\n[bold red]Using --debug, `torch.distributed` Stack Trace:[/bold red]")
+                    console.print_exception(suppress=[__file__], show_locals=False)
     else:
         # We still have to use subprocess, the user won't get a clean traceback as a result
         cmd = get_launch_prefix()
diff --git a/src/accelerate/utils/launch.py b/src/accelerate/utils/launch.py
index 1a452ca63..31d71ad21 100644
--- a/src/accelerate/utils/launch.py
+++ b/src/accelerate/utils/launch.py
@@ -23,8 +23,6 @@
 
 if is_torch_version(">=", "1.9.0"):
     import torch.distributed.run as distrib_run
-else:
-    import torch.distributed.launch as distrib_run
 
 
 def get_launch_prefix():
diff --git a/src/accelerate/utils/rich.py b/src/accelerate/utils/rich.py
new file mode 100644
index 000000000..7a54c8b5a
--- /dev/null
+++ b/src/accelerate/utils/rich.py
@@ -0,0 +1,18 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from rich.traceback import install
+
+
+install(show_locals=False)


Review comments: body: I think this is a leftover of your tests?
diff_hunk: @@ -97,6 +98,8 @@ def collate_fn(examples):
 def training_function(config, args):
     # Initialize accelerator
     accelerator = Accelerator(cpu=args.cpu, mixed_precision=args.mixed_precision)
+    if accelerator.process_index == 1:
+        raise ValueError()
body: You put `rich` in the core install here but use an `is_rich_available` later on.
diff_hunk: @@ -50,7 +50,7 @@
         ]
     },
     python_requires=">=3.7.0",
-    install_requires=["numpy>=1.17", "packaging>=20.0", "psutil", "pyyaml", "torch>=1.4.0"],
+    install_requires=["numpy>=1.17", "packaging>=20.0", "psutil", "pyyaml", "torch>=1.4.0", "rich"],
body: Should be its own list in a constant probably, so it's easier to read/add/see diff
diff_hunk: @@ -444,10 +441,29 @@ def multi_gpu_launcher(args):
         if args.fsdp_state_dict_type is not None:
             current_env["FSDP_STATE_DICT_TYPE"] = str(args.fsdp_state_dict_type)
     current_env["OMP_NUM_THREADS"] = str(args.num_cpu_threads_per_process)
-    process = subprocess.Popen(cmd, env=current_env)
-    process.wait()
-    if process.returncode != 0:
-        raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
+    # use torchrun directly here and try to call it
+
+    with patch_environment(**current_env):
+        import torch.distributed.run as distrib_run
+        debug = getattr(args, "debug", False)
+        keys = """nnodes,nproc_per_node,rdzv_backend,rdzv_endpoint,rdzv_id,rdzv_conf,standalone,max_restarts,monitor_interval,start_method,role,module,m,no_python,run_path,log_dir,r,redirects,t,tee,node_rank,master_addr,master_port,training_script,training_script_args""".split(",")
body: Thanks, fixed this now! 🤗 
diff_hunk: @@ -97,6 +98,8 @@ def collate_fn(examples):
 def training_function(config, args):
     # Initialize accelerator
     accelerator = Accelerator(cpu=args.cpu, mixed_precision=args.mixed_precision)
+    if accelerator.process_index == 1:
+        raise ValueError()
body: As discussed offline, opted to make rich a true dep in Accelerate
diff_hunk: @@ -50,7 +50,7 @@
         ]
     },
     python_requires=">=3.7.0",
-    install_requires=["numpy>=1.17", "packaging>=20.0", "psutil", "pyyaml", "torch>=1.4.0"],
+    install_requires=["numpy>=1.17", "packaging>=20.0", "psutil", "pyyaml", "torch>=1.4.0", "rich"],

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._
body: Nice utility! LGTM 🤗, I think Sylvain's suggestion would be a really good way to use this feature. Also, I think this won't result in rich traceback if the issue was in creating an `Accelerate` object itself, right? if so, should that be mentioned as part of the documentation?
body: 
In Multi GPU launcher, if one isn't using `--debug` then they won't see any error stack trace at all (which might be hard to read but important for root cause and resolution). This makes `--debug` sort of mandatory instead of optional. Can the user have normal error traceback if `--debug` isn't given? Please let me know if I am missing something or way off. 


Went over it again with a sample code, 
```python
from rich.traceback import 
install install(show_locals=True)
``` 
makes rich the default traceback handler. `--debug` meaning is no more what Sylvain was referring to, hence my confusion.