Repository: pytorch/ignite

Issue:
Improve Frequency
## 🚀 Feature

If we would like to log datapoints/second every 100 iterations, we most probably do like this 
```python
wps_metric = Frequency(output_transformer=lambda x: x['ntokens'])
wps_metric.attach(trainer, name='wps', event_name=Events.ITERATION_COMPLETED(every=100))
```
however, seems like this wont take into account all other iterations while computing the total number of tokens.
```python
class Frequency(Metric):
    ....
    def attach(self, engine, name, event_name=Events.ITERATION_COMPLETED):
        engine.add_event_handler(Events.EPOCH_STARTED, self.started)
        engine.add_event_handler(event_name, self.iteration_completed)
        engine.add_event_handler(event_name, self.completed, name)
```
IMO, should be 
```python
class Frequency(Metric):
    ....
    def attach(self, engine, name, event_name=Events.ITERATION_COMPLETED):
        engine.add_event_handler(Events.EPOCH_STARTED, self.started)
        engine.add_event_handler(Events.ITERATION_COMPLETED, self.iteration_completed)
        engine.add_event_handler(event_name, self.completed, name)
```

cc @erip 

Improve Frequency
## 🚀 Feature

If we would like to log datapoints/second every 100 iterations, we most probably do like this 
```python
wps_metric = Frequency(output_transformer=lambda x: x['ntokens'])
wps_metric.attach(trainer, name='wps', event_name=Events.ITERATION_COMPLETED(every=100))
```
however, seems like this wont take into account all other iterations while computing the total number of tokens.
```python
class Frequency(Metric):
    ....
    def attach(self, engine, name, event_name=Events.ITERATION_COMPLETED):
        engine.add_event_handler(Events.EPOCH_STARTED, self.started)
        engine.add_event_handler(event_name, self.iteration_completed)
        engine.add_event_handler(event_name, self.completed, name)
```
IMO, should be 
```python
class Frequency(Metric):
    ....
    def attach(self, engine, name, event_name=Events.ITERATION_COMPLETED):
        engine.add_event_handler(Events.EPOCH_STARTED, self.started)
        engine.add_event_handler(Events.ITERATION_COMPLETED, self.iteration_completed)
        engine.add_event_handler(event_name, self.completed, name)
```

cc @erip 



Discussion:
> however, seems like this wont take into account all other iterations while computing the totla number of tokens.

@vfdev-5 is this because the user can change the event?

Doesn't the above example of Frequency of wps work as proposed?
Not sure how I missed the cc here - sorry about that! Let me reacquaint myself quickly and I'll try to submit a PR.
Yes - this seems right. I'll write some tests to confirm this works as expected first.
I'm actually having trouble getting the original tests to pass on my Mac. 😬 

When I run

```sh
$ WORLD_SIZE=2 py.test --dist=each --tx "2*popen//python=python3" tests/ignite/metrics/test_frequency.py -m distributed
```

I see a nasty stack trace which seems to imply that the `_elapsed` is being invoked a) as a raw float (which it probably shouldn't be - rather it should probably be `torch.tensor(0.0)`), b) that `Frequency.update` isn't being called... perhaps another exception is occurring and being swallowed by pytest?

```
================================================================================ test session starts ================================================================================
platform darwin -- Python 3.6.10, pytest-5.3.4, py-1.8.1, pluggy-0.13.1
rootdir: /Users/erip/Code/ignite, inifile: setup.cfg
plugins: xdist-1.31.0, forked-1.1.3, cov-2.8.1
gw0 [1] / gw1 [1]
FF                                                                                                                                                                            [100%] [100%]
===================================================================================== FAILURES ======================================================================================
______________________________________________________________________ test_frequency_with_engine_distributed _______________________________________________________________________
[gw1] darwin -- Python 3.6.10 /Users/erip/miniconda3/envs/ignite-dev/bin/python3

distributed_context_single_node_gloo = {'local_rank': 1}

    @pytest.mark.distributed
    def test_frequency_with_engine_distributed(distributed_context_single_node_gloo):
        device = "cpu"
>       _test_frequency_with_engine(device, workers=dist.get_world_size())

tests/ignite/metrics/test_frequency.py:59:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/ignite/metrics/test_frequency.py:48: in _test_frequency_with_engine
    engine.run(data, max_epochs=1)
ignite/engine/engine.py:691: in run
    return self._internal_run()
ignite/engine/engine.py:803: in _internal_run
    self._handle_exception(e)
ignite/engine/engine.py:509: in _handle_exception
    raise e
ignite/engine/engine.py:788: in _internal_run
    hours, mins, secs = self._run_once_on_dataset()
ignite/engine/engine.py:498: in _run_once_on_dataset
    self._handle_exception(e)
ignite/engine/engine.py:509: in _handle_exception
    raise e
ignite/engine/engine.py:482: in _run_once_on_dataset
    self._fire_event(Events.ITERATION_COMPLETED)
ignite/engine/engine.py:396: in _fire_event
    func(*first, *(event_args + others), **kwargs)
ignite/metrics/frequency.py:58: in completed
    engine.state.metrics[name] = int(self.compute())
ignite/metrics/metric.py:319: in another_wrapper
    return func(self, *args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <ignite.metrics.frequency.Frequency object at 0x1257ebba8>

    @sync_all_reduce("_n", "_elapsed")
    def compute(self):
        time_divisor = 1.0

        if dist.is_available() and dist.is_initialized():
            time_divisor *= dist.get_world_size()

        # Returns the average processed objects per second across all workers
>       return self._n / self._elapsed.item() * time_divisor
E       AttributeError: 'float' object has no attribute 'item'

ignite/metrics/frequency.py:55: AttributeError
------------------------------------------------------------------------------- Captured stderr setup -------------------------------------------------------------------------------
Warning: Unable to resolve hostname to a (local) address. Using the loopback address as fallback. Manually set the network interface to bind to with GLOO_SOCKET_IFNAME. (operator() at /Users/distiller/project/conda/conda-bld/pytorch_1579022036889/work/torch/lib/c10d/ProcessGroupGloo.cpp:543)
--------------------------------------------------------------------------------- Captured log call ---------------------------------------------------------------------------------
ERROR    ignite.engine.engine.Engine:engine.py:497 Current run is terminating due to exception: 'float' object has no attribute 'item'.
ERROR    ignite.engine.engine.Engine:engine.py:802 Engine run is terminating due to exception: 'float' object has no attribute 'item'.
______________________________________________________________________ test_frequency_with_engine_distributed _______________________________________________________________________
[gw0] darwin -- Python 3.6.10 /Users/erip/miniconda3/envs/ignite-dev/bin/python3

distributed_context_single_node_gloo = {'local_rank': 0}

    @pytest.mark.distributed
    def test_frequency_with_engine_distributed(distributed_context_single_node_gloo):
        device = "cpu"
>       _test_frequency_with_engine(device, workers=dist.get_world_size())

tests/ignite/metrics/test_frequency.py:59:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/ignite/metrics/test_frequency.py:48: in _test_frequency_with_engine
    engine.run(data, max_epochs=1)
ignite/engine/engine.py:691: in run
    return self._internal_run()
ignite/engine/engine.py:803: in _internal_run
    self._handle_exception(e)
ignite/engine/engine.py:509: in _handle_exception
    raise e
ignite/engine/engine.py:788: in _internal_run
    hours, mins, secs = self._run_once_on_dataset()
ignite/engine/engine.py:498: in _run_once_on_dataset
    self._handle_exception(e)
ignite/engine/engine.py:509: in _handle_exception
    raise e
ignite/engine/engine.py:482: in _run_once_on_dataset
    self._fire_event(Events.ITERATION_COMPLETED)
ignite/engine/engine.py:396: in _fire_event
    func(*first, *(event_args + others), **kwargs)
ignite/metrics/frequency.py:58: in completed
    engine.state.metrics[name] = int(self.compute())
ignite/metrics/metric.py:319: in another_wrapper
    return func(self, *args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <ignite.metrics.frequency.Frequency object at 0x1215259e8>

    @sync_all_reduce("_n", "_elapsed")
    def compute(self):
        time_divisor = 1.0

        if dist.is_available() and dist.is_initialized():
            time_divisor *= dist.get_world_size()

        # Returns the average processed objects per second across all workers
>       return self._n / self._elapsed.item() * time_divisor
E       AttributeError: 'float' object has no attribute 'item'

ignite/metrics/frequency.py:55: AttributeError
------------------------------------------------------------------------------- Captured stderr setup -------------------------------------------------------------------------------
Warning: Unable to resolve hostname to a (local) address. Using the loopback address as fallback. Manually set the network interface to bind to with GLOO_SOCKET_IFNAME. (operator() at /Users/distiller/project/conda/conda-bld/pytorch_1579022036889/work/torch/lib/c10d/ProcessGroupGloo.cpp:543)
--------------------------------------------------------------------------------- Captured log call ---------------------------------------------------------------------------------
ERROR    ignite.engine.engine.Engine:engine.py:497 Current run is terminating due to exception: 'float' object has no attribute 'item'.
ERROR    ignite.engine.engine.Engine:engine.py:802 Engine run is terminating due to exception: 'float' object has no attribute 'item'.
================================================================================ 2 failed in 10.11s =================================================================================
```
@erip I'm on Mac too, the following test passes on `master` on my computer.

Anyway, please send the PR and we can see what happens.
I think we also need to improve the docs and explain better the purpose of `event_name` argument. 

> however, seems like this wont take into account all other iterations while computing the totla number of tokens.

@vfdev-5 is this because the user can change the event?

Doesn't the above example of Frequency of wps work as proposed?
Not sure how I missed the cc here - sorry about that! Let me reacquaint myself quickly and I'll try to submit a PR.
Yes - this seems right. I'll write some tests to confirm this works as expected first.
I'm actually having trouble getting the original tests to pass on my Mac. 😬 

When I run

```sh
$ WORLD_SIZE=2 py.test --dist=each --tx "2*popen//python=python3" tests/ignite/metrics/test_frequency.py -m distributed
```

I see a nasty stack trace which seems to imply that the `_elapsed` is being invoked a) as a raw float (which it probably shouldn't be - rather it should probably be `torch.tensor(0.0)`), b) that `Frequency.update` isn't being called... perhaps another exception is occurring and being swallowed by pytest?

```
================================================================================ test session starts ================================================================================
platform darwin -- Python 3.6.10, pytest-5.3.4, py-1.8.1, pluggy-0.13.1
rootdir: /Users/erip/Code/ignite, inifile: setup.cfg
plugins: xdist-1.31.0, forked-1.1.3, cov-2.8.1
gw0 [1] / gw1 [1]
FF                                                                                                                                                                            [100%] [100%]
===================================================================================== FAILURES ======================================================================================
______________________________________________________________________ test_frequency_with_engine_distributed _______________________________________________________________________
[gw1] darwin -- Python 3.6.10 /Users/erip/miniconda3/envs/ignite-dev/bin/python3

distributed_context_single_node_gloo = {'local_rank': 1}

    @pytest.mark.distributed
    def test_frequency_with_engine_distributed(distributed_context_single_node_gloo):
        device = "cpu"
>       _test_frequency_with_engine(device, workers=dist.get_world_size())

tests/ignite/metrics/test_frequency.py:59:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/ignite/metrics/test_frequency.py:48: in _test_frequency_with_engine
    engine.run(data, max_epochs=1)
ignite/engine/engine.py:691: in run
    return self._internal_run()
ignite/engine/engine.py:803: in _internal_run
    self._handle_exception(e)
ignite/engine/engine.py:509: in _handle_exception
    raise e
ignite/engine/engine.py:788: in _internal_run
    hours, mins, secs = self._run_once_on_dataset()
ignite/engine/engine.py:498: in _run_once_on_dataset
    self._handle_exception(e)
ignite/engine/engine.py:509: in _handle_exception
    raise e
ignite/engine/engine.py:482: in _run_once_on_dataset
    self._fire_event(Events.ITERATION_COMPLETED)
ignite/engine/engine.py:396: in _fire_event
    func(*first, *(event_args + others), **kwargs)
ignite/metrics/frequency.py:58: in completed
    engine.state.metrics[name] = int(self.compute())
ignite/metrics/metric.py:319: in another_wrapper
    return func(self, *args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <ignite.metrics.frequency.Frequency object at 0x1257ebba8>

    @sync_all_reduce("_n", "_elapsed")
    def compute(self):
        time_divisor = 1.0

        if dist.is_available() and dist.is_initialized():
            time_divisor *= dist.get_world_size()

        # Returns the average processed objects per second across all workers
>       return self._n / self._elapsed.item() * time_divisor
E       AttributeError: 'float' object has no attribute 'item'

ignite/metrics/frequency.py:55: AttributeError
------------------------------------------------------------------------------- Captured stderr setup -------------------------------------------------------------------------------
Warning: Unable to resolve hostname to a (local) address. Using the loopback address as fallback. Manually set the network interface to bind to with GLOO_SOCKET_IFNAME. (operator() at /Users/distiller/project/conda/conda-bld/pytorch_1579022036889/work/torch/lib/c10d/ProcessGroupGloo.cpp:543)
--------------------------------------------------------------------------------- Captured log call ---------------------------------------------------------------------------------
ERROR    ignite.engine.engine.Engine:engine.py:497 Current run is terminating due to exception: 'float' object has no attribute 'item'.
ERROR    ignite.engine.engine.Engine:engine.py:802 Engine run is terminating due to exception: 'float' object has no attribute 'item'.
______________________________________________________________________ test_frequency_with_engine_distributed _______________________________________________________________________
[gw0] darwin -- Python 3.6.10 /Users/erip/miniconda3/envs/ignite-dev/bin/python3

distributed_context_single_node_gloo = {'local_rank': 0}

    @pytest.mark.distributed
    def test_frequency_with_engine_distributed(distributed_context_single_node_gloo):
        device = "cpu"
>       _test_frequency_with_engine(device, workers=dist.get_world_size())

tests/ignite/metrics/test_frequency.py:59:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/ignite/metrics/test_frequency.py:48: in _test_frequency_with_engine
    engine.run(data, max_epochs=1)
ignite/engine/engine.py:691: in run
    return self._internal_run()
ignite/engine/engine.py:803: in _internal_run
    self._handle_exception(e)
ignite/engine/engine.py:509: in _handle_exception
    raise e
ignite/engine/engine.py:788: in _internal_run
    hours, mins, secs = self._run_once_on_dataset()
ignite/engine/engine.py:498: in _run_once_on_dataset
    self._handle_exception(e)
ignite/engine/engine.py:509: in _handle_exception
    raise e
ignite/engine/engine.py:482: in _run_once_on_dataset
    self._fire_event(Events.ITERATION_COMPLETED)
ignite/engine/engine.py:396: in _fire_event
    func(*first, *(event_args + others), **kwargs)
ignite/metrics/frequency.py:58: in completed
    engine.state.metrics[name] = int(self.compute())
ignite/metrics/metric.py:319: in another_wrapper
    return func(self, *args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <ignite.metrics.frequency.Frequency object at 0x1215259e8>

    @sync_all_reduce("_n", "_elapsed")
    def compute(self):
        time_divisor = 1.0

        if dist.is_available() and dist.is_initialized():
            time_divisor *= dist.get_world_size()

        # Returns the average processed objects per second across all workers
>       return self._n / self._elapsed.item() * time_divisor
E       AttributeError: 'float' object has no attribute 'item'

ignite/metrics/frequency.py:55: AttributeError
------------------------------------------------------------------------------- Captured stderr setup -------------------------------------------------------------------------------
Warning: Unable to resolve hostname to a (local) address. Using the loopback address as fallback. Manually set the network interface to bind to with GLOO_SOCKET_IFNAME. (operator() at /Users/distiller/project/conda/conda-bld/pytorch_1579022036889/work/torch/lib/c10d/ProcessGroupGloo.cpp:543)
--------------------------------------------------------------------------------- Captured log call ---------------------------------------------------------------------------------
ERROR    ignite.engine.engine.Engine:engine.py:497 Current run is terminating due to exception: 'float' object has no attribute 'item'.
ERROR    ignite.engine.engine.Engine:engine.py:802 Engine run is terminating due to exception: 'float' object has no attribute 'item'.
================================================================================ 2 failed in 10.11s =================================================================================
```
@erip I'm on Mac too, the following test passes on `master` on my computer.

Anyway, please send the PR and we can see what happens.
I think we also need to improve the docs and explain better the purpose of `event_name` argument. 


Resolving patch:
diff --git a/ignite/metrics/frequency.py b/ignite/metrics/frequency.py
--- a/ignite/metrics/frequency.py
+++ b/ignite/metrics/frequency.py
@@ -19,8 +19,21 @@ class Frequency(Metric):
             wps_metric.attach(trainer, name='wps')
             # Logging with TQDM
             ProgressBar(persist=True).attach(trainer, metric_names=['wps'])
-            # Progress bar will looks like
+            # Progress bar will look like
             # Epoch [2/10]: [12/24]  50%|█████      , wps=400 [00:17<1:23]
+
+
+        To compute examples processed per second every 50th iteration:
+
+        .. code-block:: python
+
+            # Compute number of tokens processed
+            wps_metric = Frequency(output_transform=lambda x: x['ntokens'])
+            wps_metric.attach(trainer, name='wps', event_name=Events.ITERATION_COMPLETED(every=50))
+            # Logging with TQDM
+            ProgressBar(persist=True).attach(trainer, metric_names=['wps'])
+            # Progress bar will look like
+            # Epoch [2/10]: [50/100]  50%|█████      , wps=400 [00:17<00:35]
     """
 
     def __init__(self, output_transform=lambda x: x, device=None):
@@ -59,5 +72,5 @@ def completed(self, engine, name):
 
     def attach(self, engine, name, event_name=Events.ITERATION_COMPLETED):
         engine.add_event_handler(Events.EPOCH_STARTED, self.started)
-        engine.add_event_handler(event_name, self.iteration_completed)
+        engine.add_event_handler(Events.ITERATION_COMPLETED, self.iteration_completed)
         engine.add_event_handler(event_name, self.completed, name)


Test patch:
diff --git a/tests/ignite/metrics/test_frequency.py b/tests/ignite/metrics/test_frequency.py
--- a/tests/ignite/metrics/test_frequency.py
+++ b/tests/ignite/metrics/test_frequency.py
@@ -21,7 +21,7 @@ def test_nondistributed_average():
     assert average_lower_bound < average < average_upper_bound
 
 
-def _test_frequency_with_engine(device, workers):
+def _test_frequency_with_engine(device, workers, every=1):
 
     artificial_time = 0.1 / workers  # seconds
     total_tokens = 1200 // workers
@@ -35,9 +35,10 @@ def update_fn(engine, batch):
 
     engine = Engine(update_fn)
     wps_metric = Frequency(output_transform=lambda x: x["ntokens"], device=device)
-    wps_metric.attach(engine, "wps")
+    event = Events.ITERATION_COMPLETED(every=every)
+    wps_metric.attach(engine, "wps", event_name=event)
 
-    @engine.on(Events.ITERATION_COMPLETED)
+    @engine.on(event)
     def assert_wps(e):
         wps = e.state.metrics["wps"]
         assert estimated_wps * 0.80 < wps < estimated_wps, "{}: {} < {} < {}".format(
@@ -57,3 +58,18 @@ def test_frequency_with_engine():
 def test_frequency_with_engine_distributed(distributed_context_single_node_gloo):
     device = "cpu"
     _test_frequency_with_engine(device, workers=dist.get_world_size())
+
+
+def test_frequency_with_engine_with_every():
+    device = "cpu"
+    _test_frequency_with_engine(device, workers=1, every=1)
+    _test_frequency_with_engine(device, workers=1, every=2)
+    _test_frequency_with_engine(device, workers=1, every=10)
+
+
+@pytest.mark.distributed
+def test_frequency_with_engine_distributed_with_every(distributed_context_single_node_gloo):
+    device = "cpu"
+    _test_frequency_with_engine(device, workers=dist.get_world_size(), every=1)
+    _test_frequency_with_engine(device, workers=dist.get_world_size(), every=2)
+    _test_frequency_with_engine(device, workers=dist.get_world_size(), every=10)