Repository: pytorch/ignite

Issue:
Inspect warnings from Visdom that make CI flaky
## 🐛 Bug description

LR finder catch errors from Visdom server and thus fails.

```
tests/ignite/contrib/handlers/test_lr_finder.py:202: AssertionError

------------------------------ Captured log call -------------------------------

ERROR    visdom:__init__.py:619 [Errno 111] Connection refused
ERROR    visdom:__init__.py:619 [Errno 111] Connection refused
```
https://travis-ci.org/github/pytorch/ignite/jobs/698932904#L3758

Need to inspect and fix that.


## Environment

 - PyTorch Version (e.g., 1.4):
 - Ignite Version (e.g., 0.3.0):
 - OS (e.g., Linux):
 - How you installed Ignite (`conda`, `pip`, source):
 - Python version:
 - Any other relevant information:


Discussion:
vfdev-5: We still have `[Errno 111] Connection refused`

https://github.com/pytorch/ignite/pull/1149/checks?check_run_id=795148276#step:5:802
sdesrozis: I think we have more messages. I just added a connexion check. I will inspect more precisely.
vfdev-5: @sdesrozis seems like there is `offline` mode in `Visdom`: https://github.com/facebookresearch/visdom/blob/v0.1.8.9/py/visdom/__init__.py#L413

Probably, we can use it without setup of visdom server for our tests. I let you explore that.

By the way, I could reproduce the issue by adding 
```python
def test_logger_init_hostname_port(visdom_server):
    # Explicit hostname, port
    with pytest.warns(None) as record:
        vd_logger = VisdomLogger(server=visdom_server[0], port=visdom_server[1], num_workers=0)
        assert "main" in vd_logger.vis.get_env_list()
        vd_logger.close()
        assert len(record) == 0
```
and running
```
CUDA_VISIBLE_DEVICES="" py.test -vvv tests/ignite/contrib/engines/test_common.py tests/ignite/contrib/handlers/test_visdom_logger.py 
``` 


Patch:
diff --git a/ignite/contrib/handlers/visdom_logger.py b/ignite/contrib/handlers/visdom_logger.py
index a4b9a337..01e6d98c 100644
--- a/ignite/contrib/handlers/visdom_logger.py
+++ b/ignite/contrib/handlers/visdom_logger.py
@@ -454,7 +454,7 @@ class VisdomLogger(BaseLogger):
 
     """
 
-    def __init__(self, server=None, port=None, num_workers=1, **kwargs):
+    def __init__(self, server=None, port=None, num_workers=1, raise_exceptions=True, **kwargs):
         try:
             import visdom
         except ImportError:
@@ -491,11 +491,11 @@ class VisdomLogger(BaseLogger):
             password = os.environ.get("VISDOM_PASSWORD", None)
             kwargs["password"] = password
 
-        self.vis = visdom.Visdom(server=server, port=port, **kwargs)
+        self.vis = visdom.Visdom(server=server, port=port, raise_exceptions=raise_exceptions, **kwargs)
 
-        if not self.vis.check_connection():
+        if not self.vis.offline and not self.vis.check_connection():
             raise RuntimeError(
-                "Failed to connect to Visdom server at {}. " "Did you run python -m visdom.server ?".format(server)
+                "Failed to connect to Visdom server at {}. Did you run python -m visdom.server ?".format(server)
             )
 
         self.executor = _DummyExecutor()
@@ -508,8 +508,8 @@ class VisdomLogger(BaseLogger):
         self.vis.save([self.vis.env])
 
     def close(self):
-        self.vis = None
         self.executor.shutdown()
+        self.vis = None
 
     def _create_output_handler(self, *args, **kwargs):
         return OutputHandler(*args, **kwargs)


Test patch:
diff --git a/tests/ignite/contrib/conftest.py b/tests/ignite/contrib/conftest.py
index 16bfe3cc..489e0b51 100644
--- a/tests/ignite/contrib/conftest.py
+++ b/tests/ignite/contrib/conftest.py
@@ -1,21 +1,63 @@
+import random
+
 import pytest
 
 
 @pytest.fixture()
-def visdom_server():
+def visdom_offline_logfile(dirname):
 
     import os
-    import signal
-    import subprocess
-    import time
 
-    from visdom.server import download_scripts
+    log_file = os.path.join(dirname, "logs.visdom")
+    yield log_file
+
+
+vd_hostname = None
+vd_port = None
+vd_server_process = None
+
+
+@pytest.fixture()
+def visdom_server():
+    # Start Visdom server once and stop it with visdom_server_stop
+    global vd_hostname, vd_port, vd_server_process
+
+    if vd_server_process is None:
+
+        import subprocess
+        import time
+
+        from visdom import Visdom
+        from visdom.server import download_scripts
 
-    download_scripts()
+        download_scripts()
+
+        vd_hostname = "localhost"
+        vd_port = random.randint(8089, 8887)
+
+        try:
+            vis = Visdom(server=vd_hostname, port=vd_port, raise_exceptions=True)
+        except ConnectionError:
+            pass
+
+        vd_server_process = subprocess.Popen(
+            ["python", "-m", "visdom.server", "--hostname", vd_hostname, "-port", str(vd_port)]
+        )
+        time.sleep(5)
+
+        vis = Visdom(server=vd_hostname, port=vd_port)
+        assert vis.check_connection()
+        vis.close()
+
+    yield (vd_hostname, vd_port)
+
+
+@pytest.fixture()
+def visdom_server_stop():
+
+    yield None
+
+    import time
 
-    hostname = "localhost"
-    port = 8098
-    p = subprocess.Popen("visdom --hostname {} -port {}".format(hostname, port), shell=True, preexec_fn=os.setsid)
-    time.sleep(5)
-    yield (hostname, port)
-    os.killpg(os.getpgid(p.pid), signal.SIGTERM)
+    vd_server_process.kill()
+    time.sleep(2)
diff --git a/tests/ignite/contrib/engines/test_common.py b/tests/ignite/contrib/engines/test_common.py
index 8d9a8ee7..af43a826 100644
--- a/tests/ignite/contrib/engines/test_common.py
+++ b/tests/ignite/contrib/engines/test_common.py
@@ -403,10 +403,10 @@ def test_setup_tb_logging(dirname):
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
-def test_setup_visdom_logging(visdom_server):
+def test_setup_visdom_logging(visdom_offline_logfile):
     vis_logger = _test_setup_logging(
         setup_logging_fn=setup_visdom_logging,
-        kwargs_dict={"server": visdom_server[0], "port": str(visdom_server[1])},
+        kwargs_dict={"offline": True, "log_to_filename": visdom_offline_logfile},
         output_handler_cls=handlers.visdom_logger.OutputHandler,
         opt_params_handler_cls=handlers.visdom_logger.OptimizerParamsHandler,
         with_eval=False,
@@ -416,7 +416,7 @@ def test_setup_visdom_logging(visdom_server):
 
     vis_logger = _test_setup_logging(
         setup_logging_fn=setup_visdom_logging,
-        kwargs_dict={"server": visdom_server[0], "port": str(visdom_server[1])},
+        kwargs_dict={"offline": True, "log_to_filename": visdom_offline_logfile},
         output_handler_cls=handlers.visdom_logger.OutputHandler,
         opt_params_handler_cls=handlers.visdom_logger.OptimizerParamsHandler,
         with_eval=True,
diff --git a/tests/ignite/contrib/handlers/test_lr_finder.py b/tests/ignite/contrib/handlers/test_lr_finder.py
index 6e8471fa..30482302 100644
--- a/tests/ignite/contrib/handlers/test_lr_finder.py
+++ b/tests/ignite/contrib/handlers/test_lr_finder.py
@@ -199,14 +199,13 @@ def test_num_iter_is_not_enough(lr_finder, to_save, dummy_engine, dataloader):
         assert dummy_engine.state.iteration == len(dataloader)
 
 
-def test_detach_terminates(lr_finder, to_save, dummy_engine, dataloader):
+def test_detach_terminates(lr_finder, to_save, dummy_engine, dataloader, recwarn):
     with lr_finder.attach(dummy_engine, to_save, end_lr=100, diverge_th=2) as trainer_with_finder:
-        with pytest.warns(None) as record:
-            trainer_with_finder.run(dataloader)
-            assert len(record) == 0
+        trainer_with_finder.run(dataloader)
 
     dummy_engine.run(dataloader, max_epochs=3)
     assert dummy_engine.state.epoch == 3
+    assert len(recwarn) == 0
 
 
 def test_lr_suggestion(lr_finder, to_save, dummy_engine, dataloader):
diff --git a/tests/ignite/contrib/handlers/test_visdom_logger.py b/tests/ignite/contrib/handlers/test_visdom_logger.py
index 31c56413..a8bc76cd 100644
--- a/tests/ignite/contrib/handlers/test_visdom_logger.py
+++ b/tests/ignite/contrib/handlers/test_visdom_logger.py
@@ -788,11 +788,8 @@ def test_grads_scalar_handler():
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
 def test_integration_no_server():
 
-    with pytest.warns(
-        PendingDeprecationWarning, match="Visdom is eventually changing to default to raising exceptions"
-    ):
-        with pytest.raises(RuntimeError, match="Failed to connect to Visdom server"):
-            VisdomLogger()
+    with pytest.raises(ConnectionError, match="Error connecting to Visdom server"):
+        VisdomLogger()
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
@@ -800,6 +797,7 @@ def test_logger_init_hostname_port(visdom_server):
     # Explicit hostname, port
     vd_logger = VisdomLogger(server=visdom_server[0], port=visdom_server[1], num_workers=0)
     assert "main" in vd_logger.vis.get_env_list()
+    vd_logger.close()
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
@@ -811,6 +809,7 @@ def test_logger_init_env_vars(visdom_server):
     os.environ["VISDOM_PORT"] = str(visdom_server[1])
     vd_logger = VisdomLogger(server=visdom_server[0], port=visdom_server[1], num_workers=0)
     assert "main" in vd_logger.vis.get_env_list()
+    vd_logger.close()
 
 
 def _parse_content(content):
@@ -852,6 +851,7 @@ def test_integration_no_executor(visdom_server):
     x_vals, y_vals = data["x"], data["y"]
     assert all([int(x) == x_true for x, x_true in zip(x_vals, list(range(1, n_epochs * len(data) + 1)))])
     assert all([y == y_true for y, y_true in zip(y_vals, losses)])
+    vd_logger.close()
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
@@ -892,7 +892,7 @@ def test_integration_with_executor(visdom_server):
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Skip on Windows")
-def test_integration_with_executor_as_context_manager(visdom_server):
+def test_integration_with_executor_as_context_manager(visdom_server, visdom_server_stop):
 
     n_epochs = 5
     data = list(range(50))