Repository: pytorch/ignite

Issue:
Use BasicTimeProfiler during the run
## 🚀 Feature

1. Idea is to improve the docs with a complete example of usage of `BasicTimeProfiler`
2. Seems like using `BasicTimeProfiler` during the run does not work. It would be interesting to get an intermediate results.


Discussion:

Resolving patch:
diff --git a/ignite/contrib/handlers/time_profilers.py b/ignite/contrib/handlers/time_profilers.py
--- a/ignite/contrib/handlers/time_profilers.py
+++ b/ignite/contrib/handlers/time_profilers.py
@@ -23,8 +23,18 @@ class BasicTimeProfiler:
         profiler.attach(trainer)
         trainer.run(dataloader, max_epochs=3)
 
+        @trainer.on(Events.EPOCH_COMPLETED)
+        def log_intermediate_results():
+            profiler.print_results(profiler.get_results())
+
+        profiler.write_results('path_to_dir/time_profiling.csv')
+
     """
 
+    events_to_ignore = [
+        Events.EXCEPTION_RAISED,
+    ]
+
     def __init__(self):
         self._dataflow_timer = Timer()
         self._processing_timer = Timer()
@@ -90,9 +100,10 @@ def _as_first_started(self, engine):
             e: [
                 h.__qualname__ if hasattr(h, "__qualname__") else h.__class__.__name__
                 for (h, _, _) in engine._event_handlers[e]
+                if "BasicTimeProfiler." not in repr(h)  # avoid adding internal handlers into output
             ]
             for e in Events
-            if e != Events.EXCEPTION_RAISED
+            if e not in self.events_to_ignore
         }
 
         # Setup all other handlers:
@@ -194,15 +205,17 @@ def attach(self, engine):
 
     @staticmethod
     def _compute_basic_stats(data):
-        return OrderedDict(
-            [
+        # compute on non-zero data:
+        data = data[data > 0]
+        out = [("total", torch.sum(data).item() if len(data) > 0 else "not yet triggered")]
+        if len(data) > 1:
+            out += [
                 ("min/index", (torch.min(data).item(), torch.argmin(data).item())),
                 ("max/index", (torch.max(data).item(), torch.argmax(data).item())),
                 ("mean", torch.mean(data).item()),
                 ("std", torch.std(data).item()),
-                ("total", torch.sum(data).item()),
             ]
-        )
+        return OrderedDict(out)
 
     def get_results(self):
         """
@@ -213,8 +226,7 @@ def get_results(self):
             results = profiler.get_results()
 
         """
-        events_to_ignore = [Events.EXCEPTION_RAISED]
-        total_eh_time = sum([sum(self.event_handlers_times[e]) for e in Events if e not in events_to_ignore])
+        total_eh_time = sum([(self.event_handlers_times[e]).sum() for e in Events if e not in self.events_to_ignore])
 
         return OrderedDict(
             [
@@ -226,7 +238,7 @@ def get_results(self):
                         [
                             (str(e.name).replace(".", "_"), self._compute_basic_stats(self.event_handlers_times[e]))
                             for e in Events
-                            if e not in events_to_ignore
+                            if e not in self.events_to_ignore
                         ]
                         + [("total_time", total_eh_time)]
                     ),
@@ -333,32 +345,29 @@ def print_results(results):
             - Time profiling results:
             --------------------------------------------
             Processing function time stats (in seconds):
-                min/index: (2.9754010029137135e-05, 0)
-                max/index: (2.9754010029137135e-05, 0)
-                mean: 2.9754010029137135e-05
-                std: nan
-                total: 2.9754010029137135e-05
+                min/index: (1.3081999895803165e-05, 1)
+                max/index: (1.433099987480091e-05, 0)
+                mean: 1.3706499885302037e-05
+                std: 8.831763693706307e-07
+                total: 2.7412999770604074e-05
 
             Dataflow time stats (in seconds):
-                min/index: (0.2523871660232544, 0)
-                max/index: (0.2523871660232544, 0)
-                mean: 0.2523871660232544
-                std: nan
-                total: 0.2523871660232544
+                min/index: (5.199999941396527e-05, 0)
+                max/index: (0.00010925399692496285, 1)
+                mean: 8.062699635047466e-05
+                std: 4.048469054396264e-05
+                total: 0.0001612539927009493
+
 
             Time stats of event handlers (in seconds):
             - Total time spent:
                 1.0080009698867798
 
             - Events.STARTED:
-                min/index: (0.1256754994392395, 0)
-                max/index: (0.1256754994392395, 0)
-                mean: 0.1256754994392395
-                std: nan
                 total: 0.1256754994392395
 
             Handlers names:
-            ['BasicTimeProfiler._as_first_started', 'delay_start']
+            ['delay_start']
             --------------------------------------------
         """
 


Test patch:
diff --git a/tests/ignite/contrib/handlers/test_time_profilers.py b/tests/ignite/contrib/handlers/test_time_profilers.py
--- a/tests/ignite/contrib/handlers/test_time_profilers.py
+++ b/tests/ignite/contrib/handlers/test_time_profilers.py
@@ -1,9 +1,7 @@
 import time
 import os
-import shutil
 
 from ignite.engine import Engine, Events
-from ignite.handlers import Timer
 from ignite.contrib.handlers.time_profilers import BasicTimeProfiler
 
 from pytest import approx
@@ -13,6 +11,44 @@ def _do_nothing_update_fn(engine, batch):
     pass
 
 
+def get_prepared_engine(true_event_handler_time):
+    dummy_trainer = Engine(_do_nothing_update_fn)
+
+    @dummy_trainer.on(Events.STARTED)
+    def delay_start(engine):
+        time.sleep(true_event_handler_time)
+
+    @dummy_trainer.on(Events.COMPLETED)
+    def delay_complete(engine):
+        time.sleep(true_event_handler_time)
+
+    @dummy_trainer.on(Events.EPOCH_STARTED)
+    def delay_epoch_start(engine):
+        time.sleep(true_event_handler_time)
+
+    @dummy_trainer.on(Events.EPOCH_COMPLETED)
+    def delay_epoch_complete(engine):
+        time.sleep(true_event_handler_time)
+
+    @dummy_trainer.on(Events.ITERATION_STARTED)
+    def delay_iter_start(engine):
+        time.sleep(true_event_handler_time)
+
+    @dummy_trainer.on(Events.ITERATION_COMPLETED)
+    def delay_iter_complete(engine):
+        time.sleep(true_event_handler_time)
+
+    @dummy_trainer.on(Events.GET_BATCH_STARTED)
+    def delay_get_batch_started(engine):
+        time.sleep(true_event_handler_time)
+
+    @dummy_trainer.on(Events.GET_BATCH_COMPLETED)
+    def delay_get_batch_completed(engine):
+        time.sleep(true_event_handler_time)
+
+    return dummy_trainer
+
+
 def test_dataflow_timer():
     true_dataflow_time_per_ele = 0.1
     true_max_epochs = 1
@@ -79,9 +115,7 @@ def delay_start(engine):
     results = profiler.get_results()
     event_results = results["event_handlers_stats"]["STARTED"]
 
-    assert event_results["min/index"][0] == approx(true_event_handler_time, abs=1e-1)
-    assert event_results["max/index"][0] == approx(true_event_handler_time, abs=1e-1)
-    assert event_results["mean"] == approx(true_event_handler_time, abs=1e-1)
+    assert event_results["total"] == approx(true_event_handler_time, abs=1e-1)
 
 
 def test_event_handler_completed():
@@ -101,9 +135,7 @@ def delay_complete(engine):
     results = profiler.get_results()
     event_results = results["event_handlers_stats"]["COMPLETED"]
 
-    assert event_results["min/index"][0] == approx(true_event_handler_time, abs=1e-1)
-    assert event_results["max/index"][0] == approx(true_event_handler_time, abs=1e-1)
-    assert event_results["mean"] == approx(true_event_handler_time, abs=1e-1)
+    assert event_results["total"] == approx(true_event_handler_time, abs=1e-1)
 
 
 def test_event_handler_epoch_started():
@@ -298,86 +330,64 @@ def delay_get_batch_completed(engine):
     assert event_results["total_time"].item() == approx(true_event_handler_time * 8, abs=1e-1)
 
 
-def test_write_results():
+def test_write_results(dirname):
     true_event_handler_time = 0.125
     true_max_epochs = 3
     true_num_iters = 2
-    test_folder = "./test_log_folder"
-
-    if os.path.exists(test_folder):
-        shutil.rmtree(test_folder)
-    os.makedirs(test_folder)
 
     profiler = BasicTimeProfiler()
-    dummy_trainer = Engine(_do_nothing_update_fn)
+    dummy_trainer = get_prepared_engine(true_event_handler_time)
     profiler.attach(dummy_trainer)
 
-    @dummy_trainer.on(Events.STARTED)
-    def delay_start(engine):
-        time.sleep(true_event_handler_time)
-
-    @dummy_trainer.on(Events.COMPLETED)
-    def delay_complete(engine):
-        time.sleep(true_event_handler_time)
-
-    @dummy_trainer.on(Events.EPOCH_STARTED)
-    def delay_epoch_start(engine):
-        time.sleep(true_event_handler_time)
-
-    @dummy_trainer.on(Events.EPOCH_COMPLETED)
-    def delay_epoch_complete(engine):
-        time.sleep(true_event_handler_time)
-
-    @dummy_trainer.on(Events.ITERATION_STARTED)
-    def delay_iter_start(engine):
-        time.sleep(true_event_handler_time)
-
-    @dummy_trainer.on(Events.ITERATION_COMPLETED)
-    def delay_iter_complete(engine):
-        time.sleep(true_event_handler_time)
-
-    @dummy_trainer.on(Events.GET_BATCH_STARTED)
-    def delay_get_batch_started(engine):
-        time.sleep(true_event_handler_time)
-
-    @dummy_trainer.on(Events.GET_BATCH_COMPLETED)
-    def delay_get_batch_completed(engine):
-        time.sleep(true_event_handler_time)
-
     dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs)
-    profiler.write_results(test_folder + "/test_log.csv")
+    fp = os.path.join(dirname, "test_log.csv")
+    profiler.write_results(fp)
 
-    assert os.path.isfile(test_folder + "/test_log.csv")
+    assert os.path.isfile(fp)
 
     file_length = 0
-    with open(test_folder + "/test_log.csv") as f:
+    with open(fp) as f:
         for l in f:
             file_length += 1
 
     assert file_length == (true_max_epochs * true_num_iters) + 1
 
-    # cleanup test log directory
-    shutil.rmtree(test_folder)
-
 
 def test_print_results(capsys):
-    true_event_handler_time = 0.0
+
     true_max_epochs = 1
-    true_num_iters = 1
+    true_num_iters = 5
 
     profiler = BasicTimeProfiler()
-    dummy_trainer = Engine(_do_nothing_update_fn)
+    dummy_trainer = get_prepared_engine(true_event_handler_time=0.0125)
     profiler.attach(dummy_trainer)
 
-    @dummy_trainer.on(Events.GET_BATCH_COMPLETED)
-    def delay_get_batch_completed(engine):
-        time.sleep(true_event_handler_time)
-
     dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs)
-    results = profiler.get_results()
     BasicTimeProfiler.print_results(profiler.get_results())
 
     captured = capsys.readouterr()
     out = captured.out
+    assert "BasicTimeProfiler._" not in out
+    assert "nan" not in out
+
+
+def test_get_intermediate_results_during_run(capsys):
+    true_event_handler_time = 0.0645
+    true_max_epochs = 2
+    true_num_iters = 5
 
-    assert out[0] == "\n"
+    profiler = BasicTimeProfiler()
+    dummy_trainer = get_prepared_engine(true_event_handler_time)
+    profiler.attach(dummy_trainer)
+
+    @dummy_trainer.on(Events.ITERATION_COMPLETED(every=3))
+    def log_results(_):
+        results = profiler.get_results()
+        profiler.print_results(results)
+        captured = capsys.readouterr()
+        out = captured.out
+        assert "BasicTimeProfiler._" not in out
+        assert "nan" not in out
+        assert " min/index: (0.0, " not in out, out
+
+    dummy_trainer.run(range(true_num_iters), max_epochs=true_max_epochs)