diff --git a/src/accelerate/commands/config/cluster.py b/src/accelerate/commands/config/cluster.py
index 907122036..0051d899a 100644
--- a/src/accelerate/commands/config/cluster.py
+++ b/src/accelerate/commands/config/cluster.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 from ...utils import ComputeEnvironment, DistributedType, is_deepspeed_available, is_transformers_available
+from ...utils.constants import DEEPSPEED_MULTINODE_LAUNCHERS
 from .config_args import ClusterConfig
 from .config_utils import _ask_field, _convert_distributed_mode, _convert_yes_no_to_bool
 
@@ -144,6 +145,51 @@ def get_cluster_input():
                         "Please run `pip3 install transformers`."
                     )
 
+            if num_machines > 1:
+                launcher_query = "Which Type of launcher do you want to use "
+                for i, launcher in enumerate(DEEPSPEED_MULTINODE_LAUNCHERS):
+                    launcher_query += f"[{i}] {launcher}, "
+                launcher_query = launcher_query[:-2] + ")? [0]: "
+                deepspeed_config["deepspeed_multinode_launcher"] = _ask_field(
+                    launcher_query,
+                    lambda x: DEEPSPEED_MULTINODE_LAUNCHERS[int(x)],
+                    default=DEEPSPEED_MULTINODE_LAUNCHERS[0],
+                )
+
+                if deepspeed_config["deepspeed_multinode_launcher"] != DEEPSPEED_MULTINODE_LAUNCHERS[1]:
+                    deepspeed_config["deepspeed_hostfile"] = _ask_field(
+                        "DeepSpeed configures multi-node compute resources with hostfile. "
+                        "Each row is of the format `hostname slots=[num_gpus]`, e.g., `localhost slots=2`; "
+                        "for more information please refer official [documentation]"
+                        "(https://www.deepspeed.ai/getting-started/#resource-configuration-multi-node). "
+                        "Please specify the location of hostfile: ",
+                        lambda x: str(x),
+                    )
+
+                    is_exclusion_filter = _ask_field(
+                        "Do you want to specify exclusion filter string? [yes/NO]: ",
+                        _convert_yes_no_to_bool,
+                        default=False,
+                        error_message="Please enter yes or no.",
+                    )
+                    if is_exclusion_filter:
+                        deepspeed_config["deepspeed_exclusion_filter"] = _ask_field(
+                            "DeepSpeed exclusion filter string: ",
+                            lambda x: str(x),
+                        )
+
+                    is_inclusion_filter = _ask_field(
+                        "Do you want to specify inclusion filter string? [yes/NO]: ",
+                        _convert_yes_no_to_bool,
+                        default=False,
+                        error_message="Please enter yes or no.",
+                    )
+                    if is_inclusion_filter:
+                        deepspeed_config["deepspeed_inclusion_filter"] = _ask_field(
+                            "DeepSpeed inclusion filter string: ",
+                            lambda x: str(x),
+                        )
+
     fsdp_config = {}
     if distributed_type in [DistributedType.MULTI_GPU]:
         use_fsdp = _ask_field(
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
index ac91ad1e9..cffe9990b 100644
--- a/src/accelerate/commands/launch.py
+++ b/src/accelerate/commands/launch.py
@@ -35,6 +35,7 @@
     is_deepspeed_available,
     is_sagemaker_available,
 )
+from accelerate.utils.constants import DEEPSPEED_MULTINODE_LAUNCHERS
 from accelerate.utils.dataclasses import SageMakerDistributedType
 
 
@@ -109,6 +110,30 @@ def launch_command_parser(subparsers=None):
         help="Decides Whether (true|false) to save 16-bit model weights when using ZeRO Stage-3. "
         "Only applicable with DeepSpeed ZeRO Stage-3.",
     )
+    parser.add_argument(
+        "--deepspeed_hostfile",
+        default=None,
+        type=str,
+        help="DeepSpeed hostfile for configuring multi-node compute resources.",
+    )
+    parser.add_argument(
+        "--deepspeed_exclusion_filter",
+        default=None,
+        type=str,
+        help="DeepSpeed exclusion filter string when using mutli-node setup.",
+    )
+    parser.add_argument(
+        "--deepspeed_inclusion_filter",
+        default=None,
+        type=str,
+        help="DeepSpeed inclusion filter string when using mutli-node setup.",
+    )
+    parser.add_argument(
+        "--deepspeed_multinode_launcher",
+        default=None,
+        type=str,
+        help="DeepSpeed multi-node launcher to use.",
+    )
     parser.add_argument(
         "--use_fsdp",
         default=False,
@@ -312,20 +337,42 @@ def deepspeed_launcher(args):
         raise ImportError("DeepSpeed is not installed => run `pip3 install deepspeed` or build it from source.")
     cmd = ["deepspeed", "--no_local_rank"]
     if args.num_machines > 1:
-        cmd.extend(
-            [
-                "--num_gpus",
-                str(args.num_processes // args.num_machines),
-                "--num_nodes",
-                str(args.num_machines),
-                "--node_rank",
-                str(args.machine_rank),
-                "--master_addr",
-                args.main_process_ip,
-                "--master_port",
-                str(args.main_process_port),
-            ]
-        )
+        if args.deepspeed_multinode_launcher == DEEPSPEED_MULTINODE_LAUNCHERS[1]:
+            cmd = get_launch_prefix()
+            cmd.extend(
+                [
+                    "--nproc_per_node",
+                    str(args.num_processes // args.num_machines),
+                    "--nnodes",
+                    str(args.num_machines),
+                    "--node_rank",
+                    str(args.machine_rank),
+                    "--master_addr",
+                    args.main_process_ip,
+                    "--master_port",
+                    str(args.main_process_port),
+                ]
+            )
+        else:
+            cmd.extend(
+                ["--hostfile", str(args.deepspeed_hostfile), "--launcher", str(args.deepspeed_multinode_launcher)]
+            )
+            if args.deepspeed_exclusion_filter is not None:
+                cmd.extend(
+                    [
+                        "--exclude",
+                        str(args.deepspeed_exclusion_filter),
+                    ]
+                )
+            elif args.deepspeed_inclusion_filter is not None:
+                cmd.extend(
+                    [
+                        "--include",
+                        str(args.deepspeed_inclusion_filter),
+                    ]
+                )
+            else:
+                cmd.extend(["--num_gpus", str(args.num_processes // args.num_machines)])
     else:
         cmd.extend(["--num_gpus", str(args.num_processes)])
 
@@ -350,6 +397,7 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
+    current_env["PYTHONPATH"] = sys.executable
     current_env["MIXED_PRECISION"] = str(mixed_precision)
     current_env["USE_DEEPSPEED"] = "true"
     current_env["DEEPSPEED_ZERO_STAGE"] = str(args.zero_stage)
@@ -361,6 +409,13 @@ def deepspeed_launcher(args):
     current_env["DEEPSPEED_ZERO3_SAVE_16BIT_MODEL"] = str(args.zero3_save_16bit_model).lower()
     current_env["DEEPSPEED_CONFIG_FILE"] = str(args.deepspeed_config_file).lower()
 
+    if args.num_machines > 1 and args.deepspeed_multinode_launcher != DEEPSPEED_MULTINODE_LAUNCHERS[1]:
+        with open(".deepspeed_env", "a") as f:
+            for key, value in current_env.items():
+                if ";" in value or " " in value:
+                    continue
+                f.write(f"{key}={value}\n")
+
     process = subprocess.Popen(cmd, env=current_env)
     process.wait()
     if process.returncode != 0:
diff --git a/src/accelerate/utils/constants.py b/src/accelerate/utils/constants.py
index 173cfa790..5e31a96c9 100644
--- a/src/accelerate/utils/constants.py
+++ b/src/accelerate/utils/constants.py
@@ -24,5 +24,6 @@
 SAGEMAKER_PYTHON_VERSION = "py38"
 SAGEMAKER_TRANSFORMERS_VERSION = "4.17.0"
 SAGEMAKER_PARALLEL_EC2_INSTANCES = ["ml.p3.16xlarge", "ml.p3dn.24xlarge", "ml.p4dn.24xlarge"]
+DEEPSPEED_MULTINODE_LAUNCHERS = ["pdsh", "standard", "openmpi", "mvapich"]
 
 STR_OPERATION_TO_FUNC = {">": op.gt, ">=": op.ge, "==": op.eq, "!=": op.ne, "<=": op.le, "<": op.lt}


Review comments: body: Could we add some documentation on the format of the hostfile? Then we can link it from here.
diff_hunk: @@ -144,6 +145,46 @@ def get_cluster_input():
                         "Please run `pip3 install transformers`."
                     )
 
+            if num_machines > 1:
+                deepspeed_config["deepspeed_hostfile"] = _ask_field(
+                    "DeepSpeed configures multi-node compute resources with hostfile, please specify the location of hostfile: ",
body: Maybe start with this question and ask if users want to use the traditional PyTorch DDP launcher here, in which case the other questions are moot.
diff_hunk: @@ -144,6 +145,46 @@ def get_cluster_input():
                         "Please run `pip3 install transformers`."
                     )
 
+            if num_machines > 1:
+                deepspeed_config["deepspeed_hostfile"] = _ask_field(
+                    "DeepSpeed configures multi-node compute resources with hostfile, please specify the location of hostfile: ",
+                    lambda x: str(x),
+                )
+
+                is_exclusion_filter = _ask_field(
+                    "Do you want to specify exclusion filter string? [yes/NO]: ",
+                    _convert_yes_no_to_bool,
+                    default=False,
+                    error_message="Please enter yes or no.",
+                )
+                if is_exclusion_filter:
+                    deepspeed_config["deepspeed_exclusion_filter"] = _ask_field(
+                        "DeepSpeed exclusion filter string: ",
+                        lambda x: str(x),
+                    )
+
+                is_inclusion_filter = _ask_field(
+                    "Do you want to specify inclusion filter string? [yes/NO]: ",
+                    _convert_yes_no_to_bool,
+                    default=False,
+                    error_message="Please enter yes or no.",
+                )
+                if is_inclusion_filter:
+                    deepspeed_config["deepspeed_inclusion_filter"] = _ask_field(
+                        "DeepSpeed inclusion filter string: ",
+                        lambda x: str(x),
+                    )
+
+                launcher_query = "Which Type of launcher do you want to use "
+                for i, launcher in enumerate(DEEPSPEED_MULTINODE_LAUNCHERS):
+                    launcher_query += f"[{i}] {launcher}, "
+                launcher_query = launcher_query[:-2] + ")? [0]: "
+                deepspeed_config["deepspeed_multinode_launcher"] = _ask_field(
body: Done.
diff_hunk: @@ -144,6 +145,46 @@ def get_cluster_input():
                         "Please run `pip3 install transformers`."
                     )
 
+            if num_machines > 1:
+                deepspeed_config["deepspeed_hostfile"] = _ask_field(
+                    "DeepSpeed configures multi-node compute resources with hostfile, please specify the location of hostfile: ",
body: this is a problem, @pacman100 - how can a user then use `PYTHONPATH` if it's being hijacked by the launcher?

If you must manipulate `PYTHONPATH` you need to extend it and not override it.
diff_hunk: @@ -350,6 +397,7 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
+    current_env["PYTHONPATH"] = sys.executable
body: but also why would you set something that should be `path1:...:pathn` to find python libraries to a path to an executable which is not a directory.
diff_hunk: @@ -350,6 +397,7 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
+    current_env["PYTHONPATH"] = sys.executable
body: Proposed fix https://github.com/huggingface/accelerate/pull/741

diff_hunk: @@ -350,6 +397,7 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
+    current_env["PYTHONPATH"] = sys.executable
body: I don't remember but it was needed to know the python env on other nodes to run the code. If I don't do that, deepspeed code will still hijack it: https://github.com/microsoft/DeepSpeed/blob/master/deepspeed/launcher/runner.py#L480-L484.

Below is the run with this line commented and you can notice that PYTHONPATH is still overridden by the above deepspeed code:

<img width="1502" alt="Screenshot 2022-10-05 at 12 35 26 PM" src="https://user-images.githubusercontent.com/13534540/194000694-8d5aa407-7ff3-42a0-9f84-908e8ae67713.png">
  
diff_hunk: @@ -350,6 +397,7 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
+    current_env["PYTHONPATH"] = sys.executable
body: The code you linked to:
```
        curr_path = os.path.abspath('.')
        if 'PYTHONPATH' in env:
            env['PYTHONPATH'] = curr_path + ":" + env['PYTHONPATH']
        else:
            env['PYTHONPATH'] = curr_path
```

1. it doesn't override the setting, it appends to it:
2. it doesn't add `sys.executable` but the current working directory. I think sort of pushing ``pwd`` of the launched-from-dir.
diff_hunk: @@ -350,6 +397,7 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
+    current_env["PYTHONPATH"] = sys.executable
body: Hello Stas, yeah, you are correct, my bad. After spending quite some time diving deep and setting up the multi-node setup to test things out, I have found the reason.

In Accelerate, all the DeepSpeed config from `accelerate config` command is passed as env variables, as shown towards the end of the above screenshot. DeepSpeed uses `.deepspeed_env` file for all these 
additional env variables instead of using the passed env variables as mentioned here: https://www.deepspeed.ai/getting-started/#multi-node-environment-variables

So, I was adding all the env variables to `.deepspeed_env` file as seen here: https://github.com/huggingface/accelerate/blob/main/src/accelerate/commands/launch.py#L590-L594
. The DeepSpeed snippet from the above message would extend/write to PYTHONPATH. However, just after that things will be read from `.deepspeed_env` and env variables will be reset or newly added. Here, the `pwd` extension would be reset and the original PYTHONPATH if sent from accelerate will be used instead of extended one. I overlooked extension usecase and confused `pwd` for python env path and just added that directly to the launch env variables leading to unintended behaviour.   
diff_hunk: @@ -350,6 +397,7 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
+    current_env["PYTHONPATH"] = sys.executable
body: Thank you for the extended commentary that gives more context to the problem, @pacman100 

And it looks like your suggestion in the PR resolving this issue. which has been merged now, so resolving this conversation. 

Thank you!
diff_hunk: @@ -350,6 +397,7 @@ def deepspeed_launcher(args):
         warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning)
         mixed_precision = "fp16"
 
+    current_env["PYTHONPATH"] = sys.executable

Discussion: body: _The documentation is not available anymore as the PR was closed or merged._