Repository: pytorch/ignite

Issue:
ignite.distributed.Parallel: master_port seems always to be 2222 when using env:// init_method
I want to use another master_port in my machine. My code is below:

```python
def run(
    backend='nccl',
    resume=True,
    nproc_per_node=4,
    with_clearml=False,
    with_amp=False,
    **spawn_kwargs
):
    config = locals()
    config.update(config["spawn_kwargs"])
    del config["spawn_kwargs"]
    spawn_kwargs["nproc_per_node"] = nproc_per_node
    if backend == "xla-tpu" and with_amp:
        raise RuntimeError("The value of with_amp should be False if backend is xla")

    os.environ['CUDA_VISIBLE_DEVICES']   = "0,1,2,3"

    with ignite.distributed.Parallel(backend=backend, **spawn_kwargs) as parallel:

        parallel.run(training, config)


if __name__ == '__main__':
    fire.Fire(run)
```

I use the command to run this:

`python normal_train_ddp.py  --matser-port 50001`

I found it always use master port 2222.

I found this code.  It seems ignore my master_port setting when init_method is None or env://. 

https://github.com/pytorch/ignite/blob/8a1de70e2152936ffcb051f3bd618138130626ec/ignite/distributed/comp_models/native.py#L385-L388

## Environment

- PyTorch Version (e.g., 1.4): 1.8
- Ignite Version (e.g., 0.3.0): 0.4.7
- OS (e.g., Linux): Linux
- How you installed Ignite (`conda`, `pip`, source): conda
- Python version: 3.8



Discussion:
@kong13661 thanks a lot for the report! It looks like it is a bug propagated since https://github.com/pytorch/ignite/pull/1767

Meanwhile you can do the following: you can use `init_method` as tcp where you can specify master address and port:
```python
import os
import time
import ignite.distributed as idist
import torch.distributed as dist
import torch.distributed.distributed_c10d as c10d


def training(lrank):    
    time.sleep(idist.get_rank() * 0.05)
    print(idist.get_rank(), " - used init method: ", c10d._default_pg_init_method)

if __name__ == "__main__":
    
    spawn_kwargs = {
        "nproc_per_node": 4,
        "init_method": "tcp://0.0.0.0:50001"
    }
    
    os.environ["CUDA_VISIBLE_DEVICES"] = ""

    with idist.Parallel(backend="gloo", **spawn_kwargs) as parallel:
        parallel.run(training)
```
>
```
2021-11-09 06:49:50,807 ignite.distributed.launcher.Parallel INFO: Initialized distributed launcher with backend: 'gloo'
2021-11-09 06:49:50,807 ignite.distributed.launcher.Parallel INFO: - Parameters to spawn processes: 
        nproc_per_node: 4
        nnodes: 1
        node_rank: 0
        init_method: tcp://0.0.0.0:50001
2021-11-09 06:49:50,807 ignite.distributed.launcher.Parallel INFO: Spawn function '<function training at 0x7fc14dadb280>' in 4 processes
0  - used init method:  tcp://0.0.0.0:50001
1  - used init method:  tcp://0.0.0.0:50001
2  - used init method:  tcp://0.0.0.0:50001
3  - used init method:  tcp://0.0.0.0:50001
2021-11-09 06:49:53,402 ignite.distributed.launcher.Parallel INFO: End of run
```

> @kong13661 thanks a lot for the report! It looks like it is a bug propagated since #1767
> 
> Meanwhile you can do the following: you can use `init_method` as tcp where you can specify master address and port:
> 
> ```python
> import os
> import time
> import ignite.distributed as idist
> import torch.distributed as dist
> import torch.distributed.distributed_c10d as c10d
> 
> 
> def training(lrank):    
>     time.sleep(idist.get_rank() * 0.05)
>     print(idist.get_rank(), " - used init method: ", c10d._default_pg_init_method)
> 
> if __name__ == "__main__":
>     
>     spawn_kwargs = {
>         "nproc_per_node": 4,
>         "init_method": "tcp://0.0.0.0:50001"
>     }
>     
>     os.environ["CUDA_VISIBLE_DEVICES"] = ""
> 
>     with idist.Parallel(backend="gloo", **spawn_kwargs) as parallel:
>         parallel.run(training)
> ```
> 
> > 
> 
> ```
> 2021-11-09 06:49:50,807 ignite.distributed.launcher.Parallel INFO: Initialized distributed launcher with backend: 'gloo'
> 2021-11-09 06:49:50,807 ignite.distributed.launcher.Parallel INFO: - Parameters to spawn processes: 
>         nproc_per_node: 4
>         nnodes: 1
>         node_rank: 0
>         init_method: tcp://0.0.0.0:50001
> 2021-11-09 06:49:50,807 ignite.distributed.launcher.Parallel INFO: Spawn function '<function training at 0x7fc14dadb280>' in 4 processes
> 0  - used init method:  tcp://0.0.0.0:50001
> 1  - used init method:  tcp://0.0.0.0:50001
> 2  - used init method:  tcp://0.0.0.0:50001
> 3  - used init method:  tcp://0.0.0.0:50001
> 2021-11-09 06:49:53,402 ignite.distributed.launcher.Parallel INFO: End of run
> ```

It works. Thanks.

Resolving patch:
diff --git a/ignite/distributed/comp_models/native.py b/ignite/distributed/comp_models/native.py
--- a/ignite/distributed/comp_models/native.py
+++ b/ignite/distributed/comp_models/native.py
@@ -382,8 +382,14 @@ def spawn(  # type: ignore[override]
 
             if init_method in [None, "env://"]:
                 init_method = "env://"
-                master_addr = "127.0.0.1"
-                master_port = 2222
+                if master_port is None:
+                    master_port = 2222
+                if master_addr is None:
+                    master_addr = "127.0.0.1"
+            elif master_addr is not None:
+                raise ValueError("master_addr should be None if init_method is provided other then 'env://'")
+            elif master_port is not None:
+                raise ValueError("master_port should be None if init_method is provided other then 'env://'")
 
             start_processes(
                 _NativeDistModel._dist_worker_task_fn,


Test patch:
diff --git a/tests/ignite/distributed/comp_models/test_native.py b/tests/ignite/distributed/comp_models/test_native.py
--- a/tests/ignite/distributed/comp_models/test_native.py
+++ b/tests/ignite/distributed/comp_models/test_native.py
@@ -472,7 +472,7 @@ def test__native_dist_model_warning_index_less_localrank(local_rank, world_size)
     dist.destroy_process_group()
 
 
-def _test_dist_spawn_fn(local_rank, backend, world_size, device):
+def _test_dist_spawn_fn(local_rank, backend, world_size, device, **kwargs):
     from ignite.distributed.utils import _model
 
     assert dist.is_available() and dist.is_initialized()
@@ -484,12 +484,22 @@ def _test_dist_spawn_fn(local_rank, backend, world_size, device):
     assert _model.get_world_size() == world_size
     assert _model.device().type == torch.device(device).type
 
+    if "master_addr" in kwargs:
+        assert os.environ["MASTER_ADDR"] == kwargs["master_addr"]
+    if "master_port" in kwargs:
+        assert os.environ["MASTER_PORT"] == str(kwargs["master_port"])
+
 
 def _test__native_dist_model_spawn(backend, num_workers_per_machine, device, init_method=None, **spawn_kwargs):
+    kwargs_dict = {}
+    for key in ["master_addr", "master_port"]:
+        if key in spawn_kwargs:
+            kwargs_dict[key] = spawn_kwargs[key]
+
     _NativeDistModel.spawn(
         _test_dist_spawn_fn,
         args=(backend, num_workers_per_machine, device),
-        kwargs_dict={},
+        kwargs_dict=kwargs_dict,
         backend=backend,
         nproc_per_node=num_workers_per_machine,
         init_method=init_method,
@@ -499,31 +509,56 @@ def _test__native_dist_model_spawn(backend, num_workers_per_machine, device, ini
 
 @pytest.mark.distributed
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
-@pytest.mark.parametrize("init_method", [None, "env://", "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize("init_method", [None, "CUSTOM_ADDR_PORT", "env://", "tcp://0.0.0.0:22334", "FILE"])
 def test__native_dist_model_spawn_gloo(init_method, dirname):
+    spawn_kwargs = {}
+
     if init_method == "FILE":
         init_method = f"file://{dirname}/shared"
+    elif init_method == "CUSTOM_ADDR_PORT":
+        init_method = None
+        spawn_kwargs["master_addr"] = "0.0.0.0"
+        spawn_kwargs["master_port"] = 2345
 
     nproc = torch.cuda.device_count() if torch.cuda.is_available() else 4
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    _test__native_dist_model_spawn("gloo", num_workers_per_machine=nproc, device=device, init_method=init_method)
+    _test__native_dist_model_spawn(
+        "gloo", num_workers_per_machine=nproc, device=device, init_method=init_method, **spawn_kwargs
+    )
     if device.type == "cpu":
+        spawn_kwargs["start_method"] = "fork"
         _test__native_dist_model_spawn(
-            "gloo", num_workers_per_machine=nproc, device=device, start_method="fork", init_method=init_method
+            "gloo", num_workers_per_machine=nproc, device=device, init_method=init_method, **spawn_kwargs
         )
 
+    if init_method not in [None, "env://"]:
+        with pytest.raises(ValueError, match=r"master_addr should be None if init_method is provided"):
+            _test__native_dist_model_spawn(
+                "gloo", num_workers_per_machine=nproc, device=device, init_method=init_method, master_addr="abc"
+            )
+        with pytest.raises(ValueError, match=r"master_port should be None if init_method is provided"):
+            _test__native_dist_model_spawn(
+                "gloo", num_workers_per_machine=nproc, device=device, init_method=init_method, master_port=123
+            )
+
 
 @pytest.mark.distributed
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize("init_method", [None, "CUSTOM_ADDR_PORT", "tcp://0.0.0.0:22334", "FILE"])
 def test__native_dist_model_spawn_nccl(init_method, dirname):
+    spawn_kwargs = {}
+
     if init_method == "FILE":
         init_method = f"file://{dirname}/shared"
+    elif init_method == "CUSTOM_ADDR_PORT":
+        init_method = None
+        spawn_kwargs["master_addr"] = "0.0.0.0"
+        spawn_kwargs["master_port"] = 2345
 
-    num_workers_per_machine = torch.cuda.device_count()
+    nproc = torch.cuda.device_count()
     _test__native_dist_model_spawn(
-        "nccl", num_workers_per_machine=num_workers_per_machine, device="cuda", init_method=init_method
+        "nccl", num_workers_per_machine=nproc, device="cuda", init_method=init_method, **spawn_kwargs
     )