Repository: pytorch/ignite

Issue:
BaseOutputHandler plots all available metrics
Idea is to improve loggers in a way that any OutputHandler accepts `metric_names='all'` to plot all provided metrics (by `engine.state.metrics`):
```python
    tb_logger = TensorboardLogger(log_dir=path)
    tb_logger.attach(trainer,
                     log_handler=tbOutputHandler(tag="training", metric_names="all"),
                     event_name=Events.ITERATION_COMPLETED)
```


Discussion:

Patch:
diff --git a/ignite/contrib/handlers/base_logger.py b/ignite/contrib/handlers/base_logger.py
index a617006b..f965aaf9 100644
--- a/ignite/contrib/handlers/base_logger.py
+++ b/ignite/contrib/handlers/base_logger.py
@@ -67,8 +67,10 @@ class BaseOutputHandler(BaseHandler):
 
     def __init__(self, tag, metric_names=None, output_transform=None, another_engine=None, global_step_transform=None):
 
-        if metric_names is not None and not isinstance(metric_names, list):
-            raise TypeError("metric_names should be a list, got {} instead.".format(type(metric_names)))
+        if metric_names is not None:
+            if not (isinstance(metric_names, list) or (isinstance(metric_names, str) and metric_names == "all")):
+                raise TypeError("metric_names should be either a list or equal 'all', "
+                                "got {} instead.".format(type(metric_names)))
 
         if output_transform is not None and not callable(output_transform):
             raise TypeError("output_transform should be a function, got {} instead."
@@ -103,12 +105,15 @@ class BaseOutputHandler(BaseHandler):
         """
         metrics = {}
         if self.metric_names is not None:
-            for name in self.metric_names:
-                if name not in engine.state.metrics:
-                    warnings.warn("Provided metric name '{}' is missing "
-                                  "in engine's state metrics: {}".format(name, list(engine.state.metrics.keys())))
-                    continue
-                metrics[name] = engine.state.metrics[name]
+            if isinstance(self.metric_names, str) and self.metric_names == "all":
+                metrics = engine.state.metrics
+            else:
+                for name in self.metric_names:
+                    if name not in engine.state.metrics:
+                        warnings.warn("Provided metric name '{}' is missing "
+                                      "in engine's state metrics: {}".format(name, list(engine.state.metrics.keys())))
+                        continue
+                    metrics[name] = engine.state.metrics[name]
 
         if self.output_transform is not None:
             output_dict = self.output_transform(engine.state.output)
diff --git a/ignite/contrib/handlers/polyaxon_logger.py b/ignite/contrib/handlers/polyaxon_logger.py
index 4954461e..be24ef4b 100644
--- a/ignite/contrib/handlers/polyaxon_logger.py
+++ b/ignite/contrib/handlers/polyaxon_logger.py
@@ -63,7 +63,8 @@ class OutputHandler(BaseOutputHandler):
 
     Args:
         tag (str): common title for all produced plots. For example, 'training'
-        metric_names (list of str, optional): list of metric names to plot.
+        metric_names (list of str, optional): list of metric names to plot or a string "all" to plot all available
+            metrics.
         output_transform (callable, optional): output transform function to prepare `engine.state.output` as a number.
             For example, `output_transform = lambda output: output`
             This function can also return a dictionary, e.g `{'loss': loss1, `another_loss`: loss2}` to label the plot
diff --git a/ignite/contrib/handlers/tensorboard_logger.py b/ignite/contrib/handlers/tensorboard_logger.py
index b2c9256a..1265dc03 100644
--- a/ignite/contrib/handlers/tensorboard_logger.py
+++ b/ignite/contrib/handlers/tensorboard_logger.py
@@ -65,7 +65,8 @@ class OutputHandler(BaseOutputHandler):
 
     Args:
         tag (str): common title for all produced plots. For example, 'training'
-        metric_names (list of str, optional): list of metric names to plot.
+        metric_names (list of str, optional): list of metric names to plot or a string "all" to plot all available
+            metrics.
         output_transform (callable, optional): output transform function to prepare `engine.state.output` as a number.
             For example, `output_transform = lambda output: output`
             This function can also return a dictionary, e.g `{'loss': loss1, `another_loss`: loss2}` to label the plot
diff --git a/ignite/contrib/handlers/tqdm_logger.py b/ignite/contrib/handlers/tqdm_logger.py
index 2a66cde3..f3069286 100644
--- a/ignite/contrib/handlers/tqdm_logger.py
+++ b/ignite/contrib/handlers/tqdm_logger.py
@@ -143,7 +143,8 @@ class ProgressBar(BaseLogger):
 
         Args:
             engine (Engine): engine object.
-            metric_names (list, optional): list of the metrics names to log as the bar progresses
+            metric_names (list of str, optional): list of metric names to plot or a string "all" to plot all available
+                metrics.
             output_transform (callable, optional): a function to select what you want to print from the engine's
                 output. This function may return either a dictionary with entries in the format of ``{name: value}``,
                 or a single scalar, which will be displayed with the default name `output`.
@@ -173,7 +174,8 @@ class _OutputHandler(BaseOutputHandler):
 
     Args:
         description (str): progress bar description.
-        metric_names (list of str, optional): list of metric names to plot.
+        metric_names (list of str, optional): list of metric names to plot or a string "all" to plot all available
+            metrics.
         output_transform (callable, optional): output transform function to prepare `engine.state.output` as a number.
             For example, `output_transform = lambda output: output`
             This function can also return a dictionary, e.g `{'loss': loss1, `another_loss`: loss2}` to label the plot
diff --git a/ignite/contrib/handlers/visdom_logger.py b/ignite/contrib/handlers/visdom_logger.py
index 3103f9ec..9561432a 100644
--- a/ignite/contrib/handlers/visdom_logger.py
+++ b/ignite/contrib/handlers/visdom_logger.py
@@ -114,7 +114,8 @@ class OutputHandler(BaseOutputHandler, _BaseVisDrawer):
 
     Args:
         tag (str): common title for all produced plots. For example, 'training'
-        metric_names (list of str, optional): list of metric names to plot.
+        metric_names (list of str, optional): list of metric names to plot or a string "all" to plot all available
+            metrics.
         output_transform (callable, optional): output transform function to prepare `engine.state.output` as a number.
             For example, `output_transform = lambda output: output`
             This function can also return a dictionary, e.g `{'loss': loss1, `another_loss`: loss2}` to label the plot


Test patch:
diff --git a/tests/ignite/contrib/handlers/test_base_logger.py b/tests/ignite/contrib/handlers/test_base_logger.py
index 48dc8a10..d9e01b6d 100644
--- a/tests/ignite/contrib/handlers/test_base_logger.py
+++ b/tests/ignite/contrib/handlers/test_base_logger.py
@@ -22,7 +22,7 @@ class DummyOutputHandler(BaseOutputHandler):
 
 def test_base_output_handler_wrong_setup():
 
-    with pytest.raises(TypeError, match="metric_names should be a list"):
+    with pytest.raises(TypeError, match="metric_names should be either a list or equal 'all'"):
         DummyOutputHandler("tag", metric_names="abc", output_transform=None)
 
     with pytest.raises(TypeError, match="output_transform should be a function"):
@@ -81,6 +81,11 @@ def test_base_output_handler_setup_output_metrics():
     metrics = handler._setup_output_metrics(engine=engine)
     assert metrics == {"a": 0, "b": 1, "loss": engine.state.output}
 
+    # All metrics
+    handler = DummyOutputHandler("tag", metric_names="all", output_transform=None)
+    metrics = handler._setup_output_metrics(engine=engine)
+    assert metrics == true_metrics
+
 
 def test_attach():
 
diff --git a/tests/ignite/contrib/handlers/test_polyaxon_logger.py b/tests/ignite/contrib/handlers/test_polyaxon_logger.py
index c0937d71..4f6f1cb3 100644
--- a/tests/ignite/contrib/handlers/test_polyaxon_logger.py
+++ b/tests/ignite/contrib/handlers/test_polyaxon_logger.py
@@ -104,6 +104,25 @@ def test_output_handler_metric_names():
         call(step=7, **{"tag/a": 55.56})
     ], any_order=True)
 
+    # all metrics
+    wrapper = OutputHandler("tag", metric_names="all")
+    mock_logger = MagicMock(spec=PolyaxonLogger)
+    mock_logger.log_metrics = MagicMock()
+
+    mock_engine = MagicMock()
+    mock_engine.state = State(metrics={"a": 12.23, "b": 23.45, "c": torch.tensor(10.0)})
+    mock_engine.state.iteration = 5
+
+    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)
+
+    assert mock_logger.log_metrics.call_count == 1
+    mock_logger.log_metrics.assert_called_once_with(
+        step=5,
+        **{"tag/a": 12.23,
+           "tag/b": 23.45,
+           "tag/c": 10.0}
+    )
+
 
 def test_output_handler_both():
 
diff --git a/tests/ignite/contrib/handlers/test_tensorboard_logger.py b/tests/ignite/contrib/handlers/test_tensorboard_logger.py
index 0ada180d..3c2e6df9 100644
--- a/tests/ignite/contrib/handlers/test_tensorboard_logger.py
+++ b/tests/ignite/contrib/handlers/test_tensorboard_logger.py
@@ -142,6 +142,23 @@ def test_output_handler_metric_names(dirname):
         call("tag/a", 55.56, 7),
     ], any_order=True)
 
+    # all metrics
+    wrapper = OutputHandler("tag", metric_names="all")
+    mock_logger = MagicMock(spec=TensorboardLogger)
+    mock_logger.writer = MagicMock()
+
+    mock_engine = MagicMock()
+    mock_engine.state = State(metrics={"a": 12.23, "b": 23.45})
+    mock_engine.state.iteration = 5
+
+    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)
+
+    assert mock_logger.writer.add_scalar.call_count == 2
+    mock_logger.writer.add_scalar.assert_has_calls([
+        call("tag/a", 12.23, 5),
+        call("tag/b", 23.45, 5),
+    ], any_order=True)
+
 
 def test_output_handler_both(dirname):
 
diff --git a/tests/ignite/contrib/handlers/test_tqdm_logger.py b/tests/ignite/contrib/handlers/test_tqdm_logger.py
index 5f9fb9c5..386b3d66 100644
--- a/tests/ignite/contrib/handlers/test_tqdm_logger.py
+++ b/tests/ignite/contrib/handlers/test_tqdm_logger.py
@@ -5,7 +5,7 @@ import torch
 
 from ignite.contrib.handlers import ProgressBar
 from ignite.engine import Engine, Events
-from ignite.metrics import Accuracy
+from ignite.metrics import RunningAverage
 from ignite.contrib.handlers import CustomPeriodicEvent
 from ignite.handlers import TerminateOnNan
 
@@ -55,32 +55,63 @@ def test_attach_fail_with_string():
         pbar.attach(engine, 'a')
 
 
-def test_pbar_with_metric():
+def test_pbar_with_metric(capsys):
 
-    n_iters = 20
-    batch_size = 10
-    n_classes = 2
+    n_iters = 2
     data = list(range(n_iters))
-    y_true_batch_values = iter(np.random.randint(0, n_classes, size=(n_iters, batch_size)))
-    y_pred_batch_values = iter(np.random.rand(n_iters, batch_size, n_classes))
     loss_values = iter(range(n_iters))
 
     def step(engine, batch):
         loss_value = next(loss_values)
-        y_true_batch = next(y_true_batch_values)
-        y_pred_batch = next(y_pred_batch_values)
-        return loss_value, torch.from_numpy(y_pred_batch), torch.from_numpy(y_true_batch)
+        return loss_value
 
     trainer = Engine(step)
 
-    accuracy = Accuracy(output_transform=lambda x: (x[1], x[2]))
-    accuracy.attach(trainer, "avg_accuracy")
+    RunningAverage(alpha=0.5, output_transform=lambda x: x).attach(trainer, "batchloss")
 
     pbar = ProgressBar()
-    pbar.attach(trainer, ['avg_accuracy'])
+    pbar.attach(trainer, metric_names=['batchloss', ])
 
-    with pytest.warns(UserWarning):
-        trainer.run(data=data, max_epochs=1)
+    trainer.run(data=data, max_epochs=1)
+
+    captured = capsys.readouterr()
+    err = captured.err.split('\r')
+    err = list(map(lambda x: x.strip(), err))
+    err = list(filter(None, err))
+    actual = err[-1]
+    expected = u'Epoch: [1/2]  50%|█████     , batchloss=5.00e-01 [00:00<00:00]'
+    assert actual == expected
+
+
+def test_pbar_with_all_metric(capsys):
+
+    n_iters = 2
+    data = list(range(n_iters))
+    loss_values = iter(range(n_iters))
+    another_loss_values = iter(range(1, n_iters + 1))
+
+    def step(engine, batch):
+        loss_value = next(loss_values)
+        another_loss_value = next(another_loss_values)
+        return loss_value, another_loss_value
+
+    trainer = Engine(step)
+
+    RunningAverage(alpha=0.5, output_transform=lambda x: x[0]).attach(trainer, "batchloss")
+    RunningAverage(alpha=0.5, output_transform=lambda x: x[1]).attach(trainer, "another batchloss")
+
+    pbar = ProgressBar()
+    pbar.attach(trainer, metric_names="all")
+
+    trainer.run(data=data, max_epochs=1)
+
+    captured = capsys.readouterr()
+    err = captured.err.split('\r')
+    err = list(map(lambda x: x.strip(), err))
+    err = list(filter(None, err))
+    actual = err[-1]
+    expected = u'Epoch: [1/2]  50%|█████     , another batchloss=1.50e+00, batchloss=5.00e-01 [00:00<00:00]'
+    assert actual == expected
 
 
 def test_pbar_no_metric_names(capsys):
diff --git a/tests/ignite/contrib/handlers/test_visdom_logger.py b/tests/ignite/contrib/handlers/test_visdom_logger.py
index 86209f82..b26b6a3a 100644
--- a/tests/ignite/contrib/handlers/test_visdom_logger.py
+++ b/tests/ignite/contrib/handlers/test_visdom_logger.py
@@ -229,6 +229,33 @@ def test_output_handler_metric_names(dirname):
              opts=wrapper.windows['tag/a']['opts'], name="tag/a"),
     ], any_order=True)
 
+    # all metrics
+    wrapper = OutputHandler("tag", metric_names="all")
+    mock_logger = MagicMock(spec=VisdomLogger)
+    mock_logger.vis = MagicMock()
+    mock_logger.executor = _DummyExecutor()
+
+    mock_engine = MagicMock()
+    mock_engine.state = State(metrics={"a": 12.23, "b": 23.45})
+    mock_engine.state.iteration = 5
+
+    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)
+
+    assert len(wrapper.windows) == 2 and \
+        "tag/a" in wrapper.windows and "tag/b" in wrapper.windows
+    assert wrapper.windows["tag/a"]['win'] is not None
+    assert wrapper.windows["tag/b"]['win'] is not None
+
+    assert mock_logger.vis.line.call_count == 2
+    mock_logger.vis.line.assert_has_calls([
+        call(X=[5, ], Y=[12.23, ], env=mock_logger.vis.env,
+             win=None, update=None,
+             opts=wrapper.windows['tag/a']['opts'], name="tag/a"),
+        call(X=[5, ], Y=[23.45, ], env=mock_logger.vis.env,
+             win=None, update=None,
+             opts=wrapper.windows['tag/b']['opts'], name="tag/b"),
+    ], any_order=True)
+
 
 def test_output_handler_both(dirname):