Repository: pytorch/ignite

Issue:
Improve idist to use Tcp and FileStore for init_method
## 🚀 Feature

`idist` supports default initilialization `env://`. It could be useful to support `tcp://` and `file://` modes (to tackle distributed computations on windows for instance).
 
Main issue is how to configure these modes ?


Discussion:
How about users customize this using environment variables? like this:
```
self._init_method = os.eviron["INIT_METHOD"]
dist.init_process_group(backend, init_method=self._init_method, **init_pg_kwargs)
```
in `native.py`

What are the drawbacks of this?
@ahmedo42 this is interesting idea. Initially, I was thinking about an argument here:
https://github.com/pytorch/ignite/blob/6d490c8f3083f21ba8f82eb646d9052a06fae3e2/ignite/distributed/comp_models/native.py#L78-L82

but maybe we can think about two options like that... 
So the end result would be something like that? 

``` python
       def _create_from_backend(self, backend: str, timeout: Optional[int] = None, 
                   init_method:Optional[str] = "env://", **kwargs: Any) -> None:

            if backend == dist.Backend.NCCL and not torch.cuda.is_available():
                raise RuntimeError("Nccl backend is required but no cuda capable devices")

            self.setup_env_vars()

            self._local_rank = int(os.environ["LOCAL_RANK"])
            # for debug purposes
            self._master_port = int(os.environ["MASTER_PORT"])  # type: Optional[int]
            self._master_addr = os.environ["MASTER_ADDR"]  # type: Optional[str]

            init_pg_kwargs = {}
            if timeout is not None:
                init_pg_kwargs["timeout"] = timeout

           if  not _valid_init_method(init_method):
                #raise error 
                
           dist.init_process_group(backend, init_method=init_method, **init_pg_kwargs)

```
Yes, something like that. Maybe, `init_method:Optional[str] = None` and we check `os.eviron["INIT_METHOD"]` and if `init_method is not None` (priority to the argument) and set up `dist.init_process_group`.

Resolving patch:
diff --git a/ignite/distributed/comp_models/native.py b/ignite/distributed/comp_models/native.py
--- a/ignite/distributed/comp_models/native.py
+++ b/ignite/distributed/comp_models/native.py
@@ -46,40 +46,83 @@ def create_from_context() -> Optional["_NativeDistModel"]:
             return _NativeDistModel()
 
         @staticmethod
-        def create_from_backend(backend: str, **kwargs: Any) -> "_NativeDistModel":
+        def create_from_backend(
+            backend: str,
+            init_method: Optional[str] = None,
+            world_size: Optional[int] = None,
+            rank: Optional[int] = None,
+            **kwargs: Any,
+        ) -> "_NativeDistModel":
             if backend not in _NativeDistModel.available_backends:
                 raise ValueError(f"Backend should be one of '{_NativeDistModel.available_backends}'")
 
             if dist.is_available() and dist.is_initialized():
                 raise RuntimeError("Can not create new distributed process group if default one is already initialized")
-            return _NativeDistModel(backend=backend, **kwargs)
 
-        def __init__(self, backend: Optional[str] = None, timeout: Optional[int] = None, **kwargs: Any) -> None:
+            if init_method is None:
+                if world_size is not None or rank is not None:
+                    raise ValueError("Arguments rank and world_size should be None if no init_method is provided")
+            else:
+                has_rank = rank is not None
+                has_ws = world_size is not None
+                if (has_rank or has_ws) and (not has_rank or not has_ws):
+                    raise ValueError(f"Both rank and world_size should be provided, but given {rank} and {world_size}")
+
+            return _NativeDistModel(
+                backend=backend, init_method=init_method, world_size=world_size, rank=rank, **kwargs
+            )
+
+        def __init__(
+            self,
+            backend: Optional[str] = None,
+            timeout: Optional[int] = None,
+            init_method: Optional[str] = None,
+            world_size: Optional[int] = None,
+            rank: Optional[int] = None,
+            **kwargs: Any,
+        ) -> None:
             """This is a private method. Please, use `create_from_backend` or `create_from_context`
             """
             super(_NativeDistModel, self).__init__()
             self._env_backup = None  # type: Optional[Dict[str, str]]
+            self._master_port = None  # type: Optional[int]
+            self._master_addr = None  # type: Optional[str]
+            self._init_method = None  # type: Optional[str]
             if backend is not None:
-                self._create_from_backend(backend, timeout=timeout, **kwargs)
+                self._create_from_backend(
+                    backend, timeout=timeout, init_method=init_method, world_size=world_size, rank=rank, **kwargs
+                )
             else:
                 self._init_from_context()
 
-        def _create_from_backend(self, backend: str, timeout: Optional[int] = None, **kwargs: Any) -> None:
+        def _create_from_backend(
+            self,
+            backend: str,
+            timeout: Optional[int] = None,
+            init_method: Optional[str] = None,
+            world_size: Optional[int] = None,
+            rank: Optional[int] = None,
+            **kwargs: Any,
+        ) -> None:
             if backend == dist.Backend.NCCL and not torch.cuda.is_available():
                 raise RuntimeError("Nccl backend is required but no cuda capable devices")
-
-            self.setup_env_vars()
-
-            self._local_rank = int(os.environ["LOCAL_RANK"])
-            # for debug purposes
-            self._master_port = int(os.environ["MASTER_PORT"])  # type: Optional[int]
-            self._master_addr = os.environ["MASTER_ADDR"]  # type: Optional[str]
+            self._backend = backend
+            self.setup_env_vars(rank, world_size)
 
             init_pg_kwargs = {}
             if timeout is not None:
                 init_pg_kwargs["timeout"] = timeout
 
-            dist.init_process_group(backend, init_method="env://", **init_pg_kwargs)
+            if init_method is None:
+                init_method = "env://"
+
+            if "env" not in init_method:
+                init_pg_kwargs["world_size"] = int(os.environ["WORLD_SIZE"])
+                init_pg_kwargs["rank"] = int(os.environ["RANK"])
+            self._init_method = init_method
+
+            dist.init_process_group(backend, init_method=init_method, **init_pg_kwargs)
+
             # https://github.com/facebookresearch/maskrcnn-benchmark/issues/172
             dist.barrier()
 
@@ -89,12 +132,8 @@ def _create_from_backend(self, backend: str, timeout: Optional[int] = None, **kw
             self._setup_attrs()
 
         def _init_from_context(self) -> None:
-
+            self._backend = dist.get_backend()
             self._identify_local_rank()
-
-            # for debug purposes
-            self._master_port = None
-            self._master_addr = None
             self._setup_attrs()
 
         def _compute_nproc_per_node(self) -> int:
@@ -165,11 +204,13 @@ def _identify_local_rank(self) -> None:
                 # use socket gethostname heuristic to determine number of nodes => local rank
                 self._local_rank = self._compute_local_rank_via_hostname()
 
-        def setup_env_vars(self) -> None:
+        def setup_env_vars(self, rank: Optional[int] = None, world_size: Optional[int] = None) -> None:
 
             self._env_backup = os.environ.copy()
 
             if "SLURM_JOBID" in os.environ:
+                if rank is not None or world_size is not None:
+                    raise ValueError("Arguments rank and world_size should not be specified with SLURM")
                 self._setup_env_in_slurm()
                 return
 
@@ -182,11 +223,14 @@ def setup_env_vars(self) -> None:
                     f"PyTorch distributed configuration should define env variables '{necessary_env_vars}'"
                 )
 
-            os.environ["RANK"] = os.environ.get("RANK", "0")
+            os.environ["RANK"] = os.environ.get("RANK", f"{rank if rank is not None else 0}")
+            os.environ["WORLD_SIZE"] = os.environ.get("WORLD_SIZE", f"{world_size if world_size is not None else 1}")
             os.environ["LOCAL_RANK"] = os.environ.get("LOCAL_RANK", "0")
-            os.environ["WORLD_SIZE"] = os.environ.get("WORLD_SIZE", "1")
-            os.environ["MASTER_ADDR"] = os.environ.get("MASTER_ADDR", "127.0.0.1")
             os.environ["MASTER_PORT"] = os.environ.get("MASTER_PORT", "15000")
+            os.environ["MASTER_ADDR"] = os.environ.get("MASTER_ADDR", "127.0.0.1")
+            self._local_rank = int(os.environ["LOCAL_RANK"])
+            self._master_addr = os.environ["MASTER_ADDR"]
+            self._master_port = int(os.environ["MASTER_PORT"])
 
         def _setup_env_in_slurm(self) -> None:
             for k in ["SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS", "SLURM_JOB_NODELIST"]:
@@ -253,21 +297,31 @@ def _dist_worker_task_fn(
             world_size: int,
             nprocs_per_node: int,
             node_rank: int,
-            master_addr: str,
-            master_port: str,
+            master_addr: Optional[str],
+            master_port: Optional[str],
+            init_method: str,
             kw: Any,
         ) -> None:
             from ignite.distributed.utils import _set_model, finalize
 
             copy_env_vars = os.environ.copy()
 
+            rank = node_rank * nprocs_per_node + local_rank
             os.environ["LOCAL_RANK"] = str(local_rank)
-            os.environ["RANK"] = str(node_rank * nprocs_per_node + local_rank)
+            os.environ["RANK"] = str(rank)
             os.environ["WORLD_SIZE"] = str(world_size)
-            os.environ["MASTER_ADDR"] = str(master_addr)
-            os.environ["MASTER_PORT"] = str(master_port)
 
-            model = _NativeDistModel.create_from_backend(backend, **kw)
+            arg_world_size = world_size  # type: Optional[int]
+            arg_rank = rank  # type: Optional[int]
+            if init_method == "env://":
+                os.environ["MASTER_ADDR"] = str(master_addr)
+                os.environ["MASTER_PORT"] = str(master_port)
+                arg_world_size = None
+                arg_rank = None
+
+            model = _NativeDistModel.create_from_backend(
+                backend, init_method=init_method, world_size=arg_world_size, rank=arg_rank, **kw
+            )
             _set_model(model)
             fn(local_rank, *args, **kw_dict)
             finalize()
@@ -283,9 +337,10 @@ def spawn(  # type: ignore[override]
             nproc_per_node: int = 1,
             nnodes: int = 1,
             node_rank: int = 0,
-            master_addr: str = "127.0.0.1",
-            master_port: int = 2222,
+            master_addr: Optional[str] = None,
+            master_port: Optional[int] = None,
             backend: str = "nccl",
+            init_method: Optional[str] = None,
             **kwargs: Any,
         ) -> None:
             world_size = nnodes * nproc_per_node
@@ -301,6 +356,11 @@ def spawn(  # type: ignore[override]
                 spawn_kwargs["start_method"] = kwargs.get("start_method", "spawn")
                 start_processes = mp.start_processes
 
+            if init_method in [None, "env://"]:
+                init_method = "env://"
+                master_addr = "127.0.0.1"
+                master_port = 2222
+
             start_processes(
                 _NativeDistModel._dist_worker_task_fn,
                 nprocs=nproc_per_node,
@@ -314,6 +374,7 @@ def spawn(  # type: ignore[override]
                     node_rank,
                     master_addr,
                     master_port,
+                    init_method,
                     kwargs,
                 ),
                 **spawn_kwargs,
diff --git a/ignite/distributed/launcher.py b/ignite/distributed/launcher.py
--- a/ignite/distributed/launcher.py
+++ b/ignite/distributed/launcher.py
@@ -11,14 +11,18 @@
 class Parallel:
     """Distributed launcher context manager to simplify distributed configuration setup for multiple backends:
 
-    - backends from native torch distributed configuration: "nccl", "gloo", "mpi" (if available)
+    - backends from native torch distributed configuration: "nccl", "gloo" and "mpi" (if available)
 
     - XLA on TPUs via `pytorch/xla <https://github.com/pytorch/xla>`_ (if installed)
 
     - using `Horovod distributed framework <https://horovod.readthedocs.io>`_ (if installed)
 
-    Namely, it can 1) spawn ``nproc_per_node`` child processes and initialize a processing group according to
-    provided ``backend`` (useful for standalone scripts) or 2) only initialize a processing group given the ``backend``
+    Namely, it can:
+
+    1) Spawn ``nproc_per_node`` child processes and initialize a processing group according to
+    provided ``backend`` (useful for standalone scripts).
+
+    2) Only initialize a processing group given the ``backend``
     (useful with tools like `torch.distributed.launch`_, `horovodrun`_, etc).
 
     Examples:
@@ -88,10 +92,25 @@ def training(local_rank, config, **kwargs):
 
             backend = "nccl"  # or "horovod" if package is installed
 
+            # no "init_method" was specified , "env://" will be used
             with idist.Parallel(backend=backend, nproc_per_node=4) as parallel:
                 parallel.run(training, config, a=1, b=2)
 
 
+        Initializing the process using ``file://``
+
+        .. code-block:: python
+
+            with idist.Parallel(backend=backend,init_method='file:///d:/tmp/some_file', nproc_per_node=4) as parallel:
+                parallel.run(training, config, a=1, b=2)
+
+        Initializing the process using ``tcp://``
+
+        .. code-block:: python
+
+            with idist.Parallel(backend=backend,init_method='tcp://10.1.1.20:23456', nproc_per_node=4) as parallel:
+                parallel.run(training, config, a=1, b=2)
+
         3) Single node, Multi-TPU training launched with `python`
 
         .. code-block:: bash
@@ -148,6 +167,8 @@ def training(local_rank, config, **kwargs):
             with idist.Parallel(backend="nccl", **dist_config) as parallel:
                 parallel.run(training, config, a=1, b=2)
 
+
+
     .. _torch.distributed.launch: https://pytorch.org/docs/stable/distributed.html#launch-utility
     .. _horovodrun: https://horovod.readthedocs.io/en/latest/api.html#module-horovod.run
 
@@ -170,10 +191,18 @@ def training(local_rank, config, **kwargs):
             (`nccl`, `gloo`). Mandatory argument if ``nnodes`` is specified and larger than one.
         master_port: optional argument, master node port for torch native backends
             (`nccl`, `gloo`). Mandatory argument if ``master_addr`` is specified.
+        init_method: optional argument to specify processing group initialization method for torch native
+            backends (`nccl`, `gloo`). Default, "env://".
+            See more info: `dist.init_process_group`_.
         spawn_kwargs: kwargs to ``idist.spawn`` function.
 
+    .. _dist.init_process_group: https://pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group
     .. versionchanged:: 0.4.2
         ``backend`` now accepts `horovod` distributed framework.
+
+    .. versionchanged:: 0.5.0
+        ``init_method`` added.
+
     """
 
     def __init__(
@@ -184,6 +213,7 @@ def __init__(
         node_rank: Optional[int] = None,
         master_addr: Optional[str] = None,
         master_port: Optional[int] = None,
+        init_method: Optional[str] = None,
         **spawn_kwargs: Any,
     ) -> None:
         if backend is not None:
@@ -198,13 +228,14 @@ def __init__(
 
         self.backend = backend
         self._spawn_params = None
+        self.init_method = init_method
         self.logger = setup_logger(__name__ + "." + self.__class__.__name__, distributed_rank=0)
-        # distributed_rank=0 <=> explicit rank 0, avoid call idist. Critical for TPU on Colab, avoid context is setup
+        # distributed_rank=0 <=> explicit rank 0, avoid call idist. Critical for TPU on Colab, avoid context setup
 
         if self.backend is not None:
             if nproc_per_node is not None:
                 self._spawn_params = self._setup_spawn_params(
-                    nproc_per_node, nnodes, node_rank, master_addr, master_port, **spawn_kwargs
+                    nproc_per_node, nnodes, node_rank, master_addr, master_port, init_method, **spawn_kwargs
                 )
 
         if self._spawn_params is not None:
@@ -219,6 +250,7 @@ def _setup_spawn_params(
         node_rank: Optional[int] = None,
         master_addr: Optional[str] = None,
         master_port: Optional[int] = None,
+        init_method: Optional[str] = None,
         **spawn_kwargs: Any,
     ) -> Dict:
         if nproc_per_node < 1:
@@ -233,10 +265,11 @@ def _setup_spawn_params(
             node_rank = 0
         if node_rank >= nnodes or node_rank < 0:
             raise ValueError(f"Argument node_rank should be between 0 and {nnodes - 1}, but given {node_rank}")
-        if nnodes > 1 and (master_addr is None or master_port is None):
+        if nnodes > 1 and (master_addr is None or master_port is None or init_method is None):
             raise ValueError(
-                "If number of nodes larger than one, arguments master_addr and master_port "
-                f"should be specified, but given master_addr={master_addr} and master_port={master_port}"
+                "If number of nodes larger than one, arguments master_addr and master_port or init_method"
+                f"should be specified, but given master_addr={master_addr}, master_port={master_port} and "
+                f"init_method={init_method}."
             )
         params = {
             "nproc_per_node": nproc_per_node,
@@ -244,6 +277,7 @@ def _setup_spawn_params(
             "node_rank": node_rank,
             "master_addr": master_addr,
             "master_port": master_port,
+            "init_method": init_method,
         }
         params.update(spawn_kwargs)
         return {k: v for k, v in params.items() if v is not None}
@@ -282,7 +316,7 @@ def training(local_rank, config, **kwargs):
 
     def __enter__(self) -> "Parallel":
         if (self.backend is not None) and self._spawn_params is None:
-            idist.initialize(self.backend)
+            idist.initialize(self.backend, init_method=self.init_method)
             self.logger = setup_logger(__name__ + "." + self.__class__.__name__)
             self.logger.info(f"Initialized processing group with backend: '{self.backend}'")
 
diff --git a/ignite/distributed/utils.py b/ignite/distributed/utils.py
--- a/ignite/distributed/utils.py
+++ b/ignite/distributed/utils.py
@@ -294,14 +294,15 @@ def train_fn(local_rank, a, b, c, d=12):
         nproc_per_node: number of processes to spawn on a single node. Default, 1.
         kwargs: acceptable kwargs according to provided backend:
 
-            - | "nccl" or "gloo" : `nnodes` (default, 1), `node_rank` (default, 0), `master_addr`
-              | (default, "127.0.0.1"), `master_port` (default, 2222), `timeout` to `dist.init_process_group`_ function
+            - | "nccl" or "gloo" : ``nnodes`` (default, 1), ``node_rank`` (default, 0), ``master_addr``
+              | (default, "127.0.0.1"), ``master_port`` (default, 2222), ``init_method`` (default, "env://"),
+              | `timeout` to `dist.init_process_group`_ function
               | and kwargs for `mp.start_processes`_ function.
 
-            - | "xla-tpu" : `nnodes` (default, 1), `node_rank` (default, 0) and kwargs to `xmp.spawn`_ function.
+            - | "xla-tpu" : ``nnodes`` (default, 1), ``node_rank`` (default, 0) and kwargs to `xmp.spawn`_ function.
 
-            - | "horovod": `hosts` (default, None) and other kwargs to `hvd_run`_ function. Arguments `nnodes=1`
-              | and `node_rank=0` are tolerated and ignored, otherwise an exception is raised.
+            - | "horovod": ``hosts`` (default, None) and other kwargs to `hvd_run`_ function. Arguments ``nnodes=1``
+              | and ``node_rank=0`` are tolerated and ignored, otherwise an exception is raised.
 
     .. _dist.init_process_group: https://pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group
     .. _mp.start_processes: https://pytorch.org/docs/stable/multiprocessing.html#torch.multiprocessing.spawn
@@ -479,7 +480,10 @@ def train_fn(local_rank, a, b, c):
                 assert device == torch.device(f"cuda:{local_rank}")
 
 
-            idist.initialize("nccl")
+            backend = "nccl"  # or "gloo" or "horovod" or "xla-tpu"
+            idist.initialize(backend)
+            # or for torch native distributed on Windows:
+            # idist.initialize("nccl", init_method="file://tmp/shared")
             local_rank = idist.get_local_rank()
             train_fn(local_rank, a, b, c)
             idist.finalize()
@@ -489,14 +493,20 @@ def train_fn(local_rank, a, b, c):
         backend: backend: `nccl`, `gloo`, `xla-tpu`, `horovod`.
         kwargs: acceptable kwargs according to provided backend:
 
-            - "nccl" or "gloo" : timeout(=timedelta(minutes=30)).
+            - | "nccl" or "gloo" : ``timeout(=timedelta(minutes=30))``, ``init_method(=None)``,
+              | ``rank(=None)``, ``world_size(=None)``.
+              | By default, ``init_method`` will be "env://". See more info about parameters: `torch_init`_.
 
-            - "horovod" : comm(=None), more info: `hvd_init`_.
+            - | "horovod" : comm(=None), more info: `hvd_init`_.
 
+    .. _torch_init: https://pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group
     .. _hvd_init: https://horovod.readthedocs.io/en/latest/api.html#horovod.torch.init
 
     .. versionchanged:: 0.4.2
         ``backend`` now accepts `horovod` distributed framework.
+
+    .. versionchanged:: 0.5.0
+        ``kwargs`` now accepts ``init_method``, ``rank``, ``world_size`` for PyTorch native distributed backend.
     """
     if not (has_xla_support or has_native_dist_support or has_hvd_support):
         # nothing to do => serial model


Test patch:
diff --git a/tests/ignite/conftest.py b/tests/ignite/conftest.py
--- a/tests/ignite/conftest.py
+++ b/tests/ignite/conftest.py
@@ -1,6 +1,8 @@
+import os
 import shutil
 import sys
 import tempfile
+import time
 
 import pytest
 import torch
@@ -14,6 +16,21 @@ def dirname():
     shutil.rmtree(path)
 
 
+@pytest.fixture()
+def fixed_dirname(worker_id):
+    # multi-proc friendly fixed tmp dirname
+    path = "/tmp/fixed_tmp_dirname"
+    lrank = int(worker_id.replace("gw", "")) if "gw" in worker_id else 0
+    time.sleep(0.5 * lrank)
+    os.makedirs(path, exist_ok=True)
+    yield path
+    time.sleep(0.5 * lrank)
+    if os.path.exists(path):
+        shutil.rmtree(path)
+    # sort of sync
+    time.sleep(1.0)
+
+
 @pytest.fixture()
 def get_rank_zero_dirname(dirname):
     def func():
@@ -28,7 +45,6 @@ def func():
 @pytest.fixture()
 def local_rank(worker_id):
     """ use a different account in each xdist worker """
-    import os
 
     if "gw" in worker_id:
         lrank = int(worker_id.replace("gw", ""))
@@ -46,7 +62,6 @@ def local_rank(worker_id):
 
 @pytest.fixture()
 def world_size():
-    import os
 
     remove_env_var = False
 
@@ -62,7 +77,6 @@ def world_size():
 
 @pytest.fixture()
 def clean_env():
-    import os
 
     for k in ["RANK", "LOCAL_RANK", "WORLD_SIZE"]:
         if k in os.environ:
@@ -102,8 +116,6 @@ def _find_free_port():
 
 
 def _setup_free_port(local_rank):
-    import os
-    import time
 
     port_file = "/tmp/free_port"
 
@@ -174,7 +186,6 @@ def distributed_context_single_node_gloo(local_rank, world_size):
 
 @pytest.fixture()
 def multi_node_conf(local_rank):
-    import os
 
     assert "node_id" in os.environ
     assert "nnodes" in os.environ
@@ -213,8 +224,6 @@ def _destroy_mnodes_dist_context():
 @pytest.fixture()
 def distributed_context_multi_node_gloo(multi_node_conf):
 
-    import os
-
     assert "MASTER_ADDR" in os.environ
     assert "MASTER_PORT" in os.environ
 
@@ -231,8 +240,6 @@ def distributed_context_multi_node_gloo(multi_node_conf):
 @pytest.fixture()
 def distributed_context_multi_node_nccl(multi_node_conf):
 
-    import os
-
     assert "MASTER_ADDR" in os.environ
     assert "MASTER_PORT" in os.environ
 
@@ -256,7 +263,6 @@ def _xla_template_worker_task(index, fn, args):
 
 
 def _xla_execute(fn, args, nprocs):
-    import os
 
     import torch_xla.distributed.xla_multiprocessing as xmp
 
diff --git a/tests/ignite/distributed/check_idist_parallel.py b/tests/ignite/distributed/check_idist_parallel.py
--- a/tests/ignite/distributed/check_idist_parallel.py
+++ b/tests/ignite/distributed/check_idist_parallel.py
@@ -20,6 +20,14 @@ def training(local_rank, config, **kwargs):
     assert t == ws * (ws - 1) / 2, f"{t} vs {ws}"
     assert local_rank == idist.get_local_rank()
 
+    # Test init method:
+    if idist.model_name() == "native-dist":
+        from ignite.distributed.utils import _model
+
+        true_init_method = config.get("true_init_method", None)
+        assert true_init_method is not None, true_init_method
+        assert _model._init_method == true_init_method
+
 
 if __name__ == "__main__":
     """
@@ -74,10 +82,16 @@ def training(local_rank, config, **kwargs):
     parser.add_argument("--node_rank", type=int, default=None)
     parser.add_argument("--master_addr", type=str, default=None)
     parser.add_argument("--master_port", type=str, default=None)
+    parser.add_argument("--init_method", type=str, default=None)
 
     args = parser.parse_args()
 
-    config = {"model": "resnet18", "lr": 0.01}
+    config = {
+        "model": "resnet18",
+        "lr": 0.01,
+    }
+    if args.backend in ["gloo", "nccl"]:
+        config["true_init_method"] = args.init_method if args.init_method is not None else "env://"
 
     dist_config = dict(
         nproc_per_node=args.nproc_per_node,
@@ -86,6 +100,8 @@ def training(local_rank, config, **kwargs):
         master_addr=args.master_addr,
         master_port=args.master_port,
     )
+    if args.init_method is not None:
+        dist_config["init_method"] = args.init_method
 
     with idist.Parallel(backend=args.backend, **dist_config) as parallel:
         parallel.run(training, config, a=1, b=2)
diff --git a/tests/ignite/distributed/comp_models/test_native.py b/tests/ignite/distributed/comp_models/test_native.py
--- a/tests/ignite/distributed/comp_models/test_native.py
+++ b/tests/ignite/distributed/comp_models/test_native.py
@@ -94,7 +94,8 @@ def _test__native_dist_model_create_from_backend_no_dist(backend, true_device):
     model.finalize()
 
 
-def _test__native_dist_model_create_from_backend_dist(local_rank, rank, world_size, backend, true_device):
+def _test__native_dist_model_create_from_backend_dist(init_method, local_rank, rank, world_size, backend, true_device):
+
     import os
     from datetime import timedelta
 
@@ -104,7 +105,7 @@ def _test__native_dist_model_create_from_backend_dist(local_rank, rank, world_si
     assert "MASTER_ADDR" not in os.environ
     assert "MASTER_PORT" not in os.environ
 
-    model = _NativeDistModel.create_from_backend(backend=backend, timeout=timeout)
+    model = _NativeDistModel.create_from_backend(backend=backend, timeout=timeout, init_method=init_method)
 
     assert dist.is_available() and dist.is_initialized()
     assert dist.get_backend() == backend
@@ -125,6 +126,11 @@ def _test__native_dist_model_create_from_backend_dist(local_rank, rank, world_si
         },
     )
 
+    if init_method is None:
+        assert model._init_method == "env://"
+    else:
+        assert model._init_method == init_method
+
     model.finalize()
 
     del os.environ["RANK"]
@@ -249,8 +255,12 @@ def test__native_dist_model_create_no_dist_nccl(clean_env):
 
 
 @pytest.mark.distributed
-def test__native_dist_model_create_dist_gloo_1(local_rank, world_size):
-    _test__native_dist_model_create_from_backend_dist(local_rank, local_rank, world_size, "gloo", "cpu")
+@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+def test__native_dist_model_create_dist_gloo_1(init_method, fixed_dirname, local_rank, world_size):
+    if init_method == "FILE":
+        init_method = f"file://{fixed_dirname}/shared"
+
+    _test__native_dist_model_create_from_backend_dist(init_method, local_rank, local_rank, world_size, "gloo", "cpu")
 
 
 @pytest.mark.distributed
@@ -260,8 +270,14 @@ def test__native_dist_model_create_dist_gloo_2(local_rank, world_size):
 
 @pytest.mark.distributed
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-def test__native_dist_model_create_dist_nccl_1(local_rank, world_size):
-    _test__native_dist_model_create_from_backend_dist(local_rank, local_rank, world_size, "nccl", f"cuda:{local_rank}")
+@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+def test__native_dist_model_create_dist_nccl_1(init_method, fixed_dirname, local_rank, world_size):
+    if init_method == "FILE":
+        init_method = f"file://{fixed_dirname}/shared"
+
+    _test__native_dist_model_create_from_backend_dist(
+        init_method, local_rank, local_rank, world_size, "nccl", f"cuda:{local_rank}"
+    )
 
 
 @pytest.mark.distributed
@@ -307,26 +323,60 @@ def _test_dist_spawn_fn(local_rank, backend, world_size, device):
         assert _model.device() == torch.device(device)
 
 
-def _test__native_dist_model_spawn(backend, num_workers_per_machine, device, **spawn_kwargs):
+def _test__native_dist_model_spawn(backend, num_workers_per_machine, device, init_method=None, **spawn_kwargs):
     _NativeDistModel.spawn(
         _test_dist_spawn_fn,
         args=(backend, num_workers_per_machine, device),
         kwargs_dict={},
         backend=backend,
         nproc_per_node=num_workers_per_machine,
+        init_method=init_method,
         **spawn_kwargs,
     )
 
 
 @pytest.mark.distributed
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
-def test__native_dist_model_spawn_gloo():
-    _test__native_dist_model_spawn("gloo", num_workers_per_machine=4, device="cpu")
-    _test__native_dist_model_spawn("gloo", num_workers_per_machine=4, device="cpu", start_method="fork")
+@pytest.mark.parametrize("init_method", [None, "env://", "tcp://0.0.0.0:22334", "FILE"])
+def test__native_dist_model_spawn_gloo(init_method, dirname):
+    if init_method == "FILE":
+        init_method = f"file://{dirname}/shared"
+
+    _test__native_dist_model_spawn("gloo", num_workers_per_machine=4, device="cpu", init_method=init_method)
+    _test__native_dist_model_spawn(
+        "gloo", num_workers_per_machine=4, device="cpu", start_method="fork", init_method=init_method
+    )
 
 
 @pytest.mark.distributed
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-def test__native_dist_model_spawn_nccl():
-    _test__native_dist_model_spawn("nccl", num_workers_per_machine=torch.cuda.device_count(), device="cuda")
+@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+def test__native_dist_model_spawn_nccl(init_method, dirname):
+    if init_method == "FILE":
+        init_method = f"file://{dirname}/shared"
+
+    num_workers_per_machine = torch.cuda.device_count()
+    _test__native_dist_model_spawn(
+        "nccl", num_workers_per_machine=num_workers_per_machine, device="cuda", init_method=init_method
+    )
+
+
+@pytest.mark.distributed
+@pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
+@pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
+def test__native_dist_model_init_method_is_none(world_size):
+    with pytest.raises(ValueError, match=r"Arguments rank and world_size should be None"):
+        _NativeDistModel.create_from_backend(backend="gloo", world_size=world_size)
+
+
+@pytest.mark.distributed
+@pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
+@pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
+def test__native_dist_model_init_method_is_not_none(world_size, local_rank, fixed_dirname):
+    init_method = f"file://{fixed_dirname}/shared"
+    with pytest.raises(ValueError, match=r"Both rank and world_size should be provided"):
+        _NativeDistModel.create_from_backend(backend="gloo", world_size=world_size, init_method=init_method)
+
+    with pytest.raises(ValueError, match=r"Both rank and world_size should be provided"):
+        _NativeDistModel.create_from_backend(backend="gloo", rank=local_rank, init_method=init_method)
diff --git a/tests/ignite/distributed/test_launcher.py b/tests/ignite/distributed/test_launcher.py
--- a/tests/ignite/distributed/test_launcher.py
+++ b/tests/ignite/distributed/test_launcher.py
@@ -62,7 +62,7 @@ def test_check_idist_parallel_no_dist(exec_filepath):
     assert "End of run" in out
 
 
-def _test_check_idist_parallel_torch_launch(fp, backend, nprocs):
+def _test_check_idist_parallel_torch_launch(init_method, fp, backend, nprocs):
     # python -m torch.distributed.launch --nproc_per_node=nprocs --use_env \
     #   tests/ignite/distributed/check_idist_parallel.py --backend=backend
 
@@ -75,6 +75,8 @@ def _test_check_idist_parallel_torch_launch(fp, backend, nprocs):
         fp,
         f"--backend={backend}",
     ]
+    if init_method is not None:
+        cmd.append(f"--init_method={init_method}")
 
     out = execute(cmd)
     assert f"backend={backend}" in out
@@ -85,16 +87,24 @@ def _test_check_idist_parallel_torch_launch(fp, backend, nprocs):
 @pytest.mark.distributed
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip because test uses torch launch")
-def test_check_idist_parallel_torch_launch_n_procs_gloo(exec_filepath):
-    _test_check_idist_parallel_torch_launch(exec_filepath, "gloo", 4)
+@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+def test_check_idist_parallel_torch_launch_n_procs_gloo(init_method, dirname, exec_filepath):
+    if init_method == "FILE":
+        init_method = f"file://{dirname}/shared"
+
+    _test_check_idist_parallel_torch_launch(init_method, exec_filepath, "gloo", 4)
 
 
 @pytest.mark.distributed
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip because test uses torch launch")
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-def test_check_idist_parallel_torch_launch_n_procs_nccl(exec_filepath):
-    _test_check_idist_parallel_torch_launch(exec_filepath, "nccl", torch.cuda.device_count())
+@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+def test_check_idist_parallel_torch_launch_n_procs_nccl(init_method, dirname, exec_filepath):
+    if init_method == "FILE":
+        init_method = f"file://{dirname}/shared"
+
+    _test_check_idist_parallel_torch_launch(init_method, exec_filepath, "nccl", torch.cuda.device_count())
 
 
 def _test_check_idist_parallel_hvdrun(fp, backend, nprocs):
@@ -168,59 +178,66 @@ def test_check_idist_parallel_spawn_n_procs_hvd(exec_filepath):
     _test_check_idist_parallel_spawn(exec_filepath, "horovod", np)
 
 
-def _test_func(index, ws, device):
+def _test_func(index, ws, device, backend, true_init_method):
     assert 0 <= index < ws
+    assert index == idist.get_local_rank()
     assert ws == idist.get_world_size()
     assert device in idist.device().type
+    assert backend == idist.backend()
 
+    if idist.model_name() == "native-dist":
+        from ignite.distributed.utils import _model
 
-@pytest.mark.distributed
-@pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
-@pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
-def test_idist_parallel_gloo():
-    with idist.Parallel(backend="gloo", nproc_per_node=4) as parallel:
-        parallel.run(_test_func, ws=4, device="cpu")
-
-
-@pytest.mark.distributed
-@pytest.mark.skipif("WORLD_SIZE" not in os.environ, reason="Skip if not launched as multiproc")
-@pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
-def test_idist_parallel_gloo_nprocs(local_rank, world_size):
-    os.environ["RANK"] = str(local_rank)
-    with idist.Parallel(backend="gloo") as parallel:
-        parallel.run(_test_func, ws=world_size, device="cpu")
+        assert _model._init_method == true_init_method
 
 
 @pytest.mark.distributed
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
-@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-def test_idist_parallel_nccl():
-    with idist.Parallel(backend="nccl", nproc_per_node=torch.cuda.device_count()) as parallel:
-        parallel.run(_test_func, ws=torch.cuda.device_count(), device="cuda")
+@pytest.mark.parametrize("init_method", ["env://", "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize(
+    "backend",
+    ["gloo", pytest.param("nccl", marks=pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU"))],
+)
+def test_idist_parallel_spawn_n_procs_native(init_method, backend, dirname):
+    if init_method == "FILE":
+        init_method = f"file://{dirname}/shared"
+
+    nproc_per_node = 4 if "gloo" == backend else torch.cuda.device_count()
+    device = "cpu" if "gloo" == backend else "cuda"
+    with idist.Parallel(backend=backend, nproc_per_node=nproc_per_node, init_method=init_method) as parallel:
+        parallel.run(_test_func, ws=nproc_per_node, device=device, backend=backend, true_init_method=init_method)
 
 
 @pytest.mark.distributed
 @pytest.mark.skipif("WORLD_SIZE" not in os.environ, reason="Skip if not launched as multiproc")
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
-@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-def test_idist_parallel_nccl_nprocs(local_rank, world_size):
+@pytest.mark.parametrize("init_method", ["env://", "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize(
+    "backend",
+    ["gloo", pytest.param("nccl", marks=pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU"))],
+)
+def test_idist_parallel_n_procs_native(init_method, backend, fixed_dirname, local_rank, world_size):
+    if init_method == "FILE":
+        init_method = f"file://{fixed_dirname}/shared"
+
     os.environ["RANK"] = str(local_rank)
-    with idist.Parallel(backend="nccl") as parallel:
-        parallel.run(_test_func, ws=world_size, device="cuda")
+    device = "cuda" if "nccl" in backend else "cpu"
+    with idist.Parallel(backend=backend, init_method=init_method) as parallel:
+        parallel.run(_test_func, ws=world_size, device=device, backend=backend, true_init_method=init_method)
 
 
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
 def test_idist_parallel_no_dist():
     device = "cuda" if torch.cuda.is_available() else "cpu"
     with idist.Parallel(backend=None) as parallel:
-        parallel.run(_test_func, ws=1, device=device)
+        parallel.run(_test_func, ws=1, device=device, backend=None, true_init_method=None)
 
 
 @pytest.mark.tpu
 @pytest.mark.skipif("NUM_TPU_WORKERS" in os.environ, reason="Skip if no NUM_TPU_WORKERS in env vars")
 @pytest.mark.skipif(not has_xla_support, reason="Skip if no PyTorch XLA package")
-def test_idist_parallel_spawn_params():
+def test_idist_parallel_spawn_params_xla():
 
     res = idist.Parallel._setup_spawn_params(
         nproc_per_node=8, nnodes=None, node_rank=None, master_addr=None, master_port=None, start_method="fork"
diff --git a/tests/ignite/distributed/utils/__init__.py b/tests/ignite/distributed/utils/__init__.py
--- a/tests/ignite/distributed/utils/__init__.py
+++ b/tests/ignite/distributed/utils/__init__.py
@@ -16,7 +16,7 @@ def _sanity_check():
     assert _model.get_node_rank() < _model.get_nnodes()
 
 
-def _test_distrib_config(local_rank, backend, ws, true_device, rank=None):
+def _test_distrib_config(local_rank, backend, ws, true_device, rank=None, true_init_method=None):
     assert idist.backend() == backend, f"{idist.backend()} vs {backend}"
 
     this_device = idist.device()
@@ -43,6 +43,12 @@ def _test_distrib_config(local_rank, backend, ws, true_device, rank=None):
 
     _sanity_check()
 
+    if idist.model_name() == "native-dist":
+        from ignite.distributed.utils import _model
+
+        if true_init_method is not None:
+            assert _model._init_method == true_init_method
+
 
 def _test_sync(cls):
     from ignite.distributed.utils import _SerialModel, _set_model
diff --git a/tests/ignite/distributed/utils/test_native.py b/tests/ignite/distributed/utils/test_native.py
--- a/tests/ignite/distributed/utils/test_native.py
+++ b/tests/ignite/distributed/utils/test_native.py
@@ -19,59 +19,84 @@
 )
 
 
-@pytest.mark.distributed
-@pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
-def test_native_distrib_single_node_launch_tool_gloo(local_rank, world_size):
+def _test_native_distrib_single_node_launch_tool(backend, device, local_rank, world_size, init_method=None, **kwargs):
     import os
-    from datetime import timedelta
 
-    timeout = timedelta(seconds=20)
     rank = local_rank
     os.environ["RANK"] = f"{rank}"
 
-    idist.initialize("gloo", timeout=timeout)
-    _test_distrib_config(local_rank, "gloo", world_size, "cpu", rank)
+    idist.initialize(backend, init_method=init_method, **kwargs)
+    _test_distrib_config(local_rank, backend, world_size, device, rank, true_init_method=init_method)
     idist.finalize()
 
 
+@pytest.mark.distributed
+@pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
+@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+def test_native_distrib_single_node_launch_tool_gloo(init_method, fixed_dirname, local_rank, world_size):
+
+    from datetime import timedelta
+
+    timeout = timedelta(seconds=20)
+
+    if init_method == "FILE":
+        init_method = f"file://{fixed_dirname}/shared"
+
+    _test_native_distrib_single_node_launch_tool(
+        "gloo", "cpu", local_rank, world_size, timeout=timeout, init_method=init_method
+    )
+
+
 @pytest.mark.distributed
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-def test_native_distrib_single_node_launch_tool_nccl(local_rank, world_size):
-    import os
+@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+def test_native_distrib_single_node_launch_tool_nccl(init_method, fixed_dirname, local_rank, world_size):
 
-    rank = local_rank
-    os.environ["RANK"] = f"{rank}"
+    if init_method == "FILE":
+        init_method = f"file://{fixed_dirname}/shared"
 
-    idist.initialize("nccl")
-    _test_distrib_config(local_rank, "nccl", world_size, "cuda", rank)
-    idist.finalize()
+    _test_native_distrib_single_node_launch_tool("nccl", "cuda", local_rank, world_size, init_method=init_method)
+
+
+def _test_native_distrib_single_node_spawn(init_method, backend, device, **kwargs):
+    world_size = 4 if device == "cpu" else torch.cuda.device_count()
+    idist.spawn(
+        backend,
+        _test_distrib_config,
+        args=(backend, world_size, device),
+        nproc_per_node=world_size,
+        init_method=init_method,
+        **kwargs,
+    )
 
 
 @pytest.mark.distributed
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
-def test_native_distrib_single_node_spawn_gloo():
+@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+def test_native_distrib_single_node_spawn_gloo(init_method, dirname):
 
     from datetime import timedelta
 
     timeout = timedelta(seconds=20)
 
-    world_size = 4
+    if init_method == "FILE":
+        init_method = f"file://{dirname}/shared"
 
-    idist.spawn(
-        "gloo", _test_distrib_config, args=("gloo", world_size, "cpu"), nproc_per_node=world_size, timeout=timeout
-    )
+    _test_native_distrib_single_node_spawn(init_method, "gloo", "cpu", timeout=timeout)
 
 
 @pytest.mark.distributed
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-def test_native_distrib_single_node_spawn_nccl():
-    world_size = torch.cuda.device_count()
+@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+def test_native_distrib_single_node_spawn_nccl(init_method, dirname):
+    if init_method == "FILE":
+        init_method = f"file://{dirname}/shared"
 
-    idist.spawn("nccl", _test_distrib_config, args=("nccl", world_size, "cuda"), nproc_per_node=world_size)
+    _test_native_distrib_single_node_spawn(init_method, "nccl", "cuda")
 
 
 @pytest.mark.distributed