Repository: mosaicml/composer

Issue:
`resize_size` on `val_dataset` changes `wall_clock_train`
** Environment **

- image: `pytorch_internal`
- branch: `dev`
- Hardware (GPU, or instance type): r1z1 8x A100

** To reproduce

Steps to reproduce the behavior:

1. Run `composer -n 8 examples/run_composer_trainer.py -f composer/yamls/models/resnet50.yaml --val_dataset.imagenet.resize_size 232` (will need to provide your own dataset path/hparams)
2. Run `composer -n 8 examples/run_composer_trainer.py -f composer/yamls/models/resnet50.yaml`
3. Compare `wall_clock_time` for each run

## Expected behavior

`wall_clock_train` should be unaffected by changes to validation. Instead, setting `resize_size` (in this case to `232`) increases `wall_clock_train` by about 8%. [See W&B runs](https://wandb.ai/mosaic-ml/resnet-high-score/reports/Composer-ResNet50--VmlldzoxODc4NTMx#-validation-resizing-slows-wall_clock_train).



Discussion:
Thanks @growlix, good catch.

The underlying issue is that the [Speed Monitor](https://github.com/mosaicml/composer/blob/0e4955e6e60d96cc37680b99a77c16966d342803/composer/callbacks/speed_monitor.py#L17) does not stop time tracking during evaluation.

We can stop time tracking between `Event.EVAL_START` and `Event.EVAL_END`.

Would it sense to track the total wall clock time separately (say, as 'wall_clock_total'), from the train-only wall clock time? Or do we only want to track training time?

Resolving patch:
diff --git a/composer/callbacks/speed_monitor.py b/composer/callbacks/speed_monitor.py
--- a/composer/callbacks/speed_monitor.py
+++ b/composer/callbacks/speed_monitor.py
@@ -20,8 +20,8 @@ class SpeedMonitor(Callback):
 
     The training throughput in terms of number of samples per second is logged on the
     :attr:`~composer.core.event.Event.BATCH_END` event if we have reached the ``window_size`` threshold.  Per epoch
-    average throughput and wall clock train time is also logged on the :attr:`~composer.core.event.Event.EPOCH_END`
-    event.
+    average throughput and wall clock train, validation, and total time is also logged on the
+    :attr:`~composer.core.event.Event.EPOCH_END` event.
 
     Example
 
@@ -55,7 +55,11 @@ class SpeedMonitor(Callback):
     |                       | Number of samples processed per second (averaged over       |
     | ``throughput/epoch``  | an entire epoch)                                            |
     +-----------------------+-------------------------------------------------------------+
-    |``wall_clock_train``   | Total elapsed training time                                 |
+    |``wall_clock/train``   | Total elapsed training time                                 |
+    +-----------------------+-------------------------------------------------------------+
+    |``wall_clock/val``     | Total elapsed validation time                               |
+    +-----------------------+-------------------------------------------------------------+
+    |``wall_clock/total``   | Total elapsed time (wall_clock/train + wall_clock/val)      |
     +-----------------------+-------------------------------------------------------------+
 
     Args:
@@ -66,8 +70,19 @@ class SpeedMonitor(Callback):
     def __init__(self, window_size: int = 100):
         super().__init__()
         self.train_examples_per_epoch = 0
+
+        # log the total wall clock time that this program has been running for
+        self.wall_clock_total = 0.0
+
+        # log the total wall clock_time that has been spent in training
         self.wall_clock_train = 0.0
+
+        # log the total wall clock_time that has been spent in validation
+        self.wall_clock_val = 0.0
+
         self.epoch_start_time = 0.0
+        self.validation_start_time = 0.0
+        self.epoch_time_in_validation = 0.0
         self.batch_start_num_samples = None
         self.batch_end_times: Deque[float] = deque(maxlen=window_size + 1)  # rolling list of batch end times
         self.batch_num_samples: Deque[int] = deque(maxlen=window_size)  # rolling list of num samples in batch.
@@ -85,7 +100,9 @@ def state_dict(self) -> Dict[str, Any]:
         current_time = time.time()
         return {
             "train_examples_per_epoch": self.train_examples_per_epoch,
-            "wall_clock_train": self.wall_clock_train,
+            "wall_clock/train": self.wall_clock_train,
+            "wall_clock/val": self.wall_clock_val,
+            "wall_clock/total": self.wall_clock_total,
             "epoch_duration": current_time - self.epoch_start_time,
             "batch_durations": [current_time - x for x in self.batch_end_times],
             "batch_num_samples": self.batch_num_samples,
@@ -104,7 +121,9 @@ def _load_state(self) -> None:
         current_time = time.time()
         if self.loaded_state is not None:
             self.train_examples_per_epoch = self.loaded_state["train_examples_per_epoch"]
-            self.wall_clock_train = self.loaded_state["wall_clock_train"]
+            self.wall_clock_train = self.loaded_state["wall_clock/train"]
+            self.wall_clock_val = self.loaded_state["wall_clock/val"]
+            self.wall_clock_total = self.loaded_state["wall_clock/total"]
             self.epoch_start_time = current_time - self.loaded_state["epoch_duration"]
             self.batch_end_times = deque([current_time - x for x in self.loaded_state["batch_durations"]],
                                          maxlen=self.window_size + 1)
@@ -133,15 +152,35 @@ def batch_end(self, state: State, logger: Logger):
         self.train_examples_per_epoch += batch_num_samples
         if len(self.batch_end_times) == self.window_size + 1:
             throughput = sum(self.batch_num_samples) / (self.batch_end_times[-1] - self.batch_end_times[0])
-            logger.data_batch({'throughput/step': throughput})
+            logger.data_batch({'samples/step': throughput})
+
+    def eval_start(self, state: State, logger: Logger):
+        del state, logger  # unused
+        self.validation_start_time = time.time()
+
+    def eval_end(self, state: State, logger: Logger):
+        del state, logger  # unused
+        self.epoch_time_in_validation += time.time() - self.validation_start_time
 
     def epoch_end(self, state: State, logger: Logger):
         del state  # unused
-        epoch_time = time.time() - self.epoch_start_time
-        self.wall_clock_train += epoch_time
+
+        epoch_time_in_train = time.time() - self.epoch_start_time - self.epoch_time_in_validation
+        self.wall_clock_train += epoch_time_in_train
+        self.wall_clock_val += self.epoch_time_in_validation
+        self.wall_clock_total += epoch_time_in_train + self.epoch_time_in_validation
+        assert (self.wall_clock_train + self.wall_clock_val) == self.wall_clock_total
+
+        logger.data_epoch({
+            "wall_clock/train": self.wall_clock_train,
+        })
+        logger.data_epoch({
+            "wall_clock/val": self.wall_clock_val,
+        })
         logger.data_epoch({
-            "wall_clock_train": self.wall_clock_train,
+            "wall_clock/total": self.wall_clock_total,
         })
         logger.data_epoch({
-            "throughput/epoch": self.train_examples_per_epoch / epoch_time,
+            "samples/epoch": self.train_examples_per_epoch / epoch_time_in_train,
         })
+        self.epoch_time_in_validation = 0.0


Test patch:
diff --git a/tests/callbacks/test_speed_monitor.py b/tests/callbacks/test_speed_monitor.py
--- a/tests/callbacks/test_speed_monitor.py
+++ b/tests/callbacks/test_speed_monitor.py
@@ -26,15 +26,21 @@ def test_speed_monitor(composer_trainer_hparams: TrainerHparams):
 
     throughput_epoch_calls = 0
     wall_clock_train_calls = 0
+    wall_clock_val_calls = 0
+    wall_clock_total_calls = 0
     throughput_step_calls = 0
     for call_ in log_destination.log_data.mock_calls:
         metrics = call_[1][2]
-        if "throughput/step" in metrics:
+        if "samples/step" in metrics:
             throughput_step_calls += 1
-        if "throughput/epoch" in metrics:
+        if "samples/epoch" in metrics:
             throughput_epoch_calls += 1
-        if 'wall_clock_train' in metrics:
+        if 'wall_clock/train' in metrics:
             wall_clock_train_calls += 1
+        if 'wall_clock/val' in metrics:
+            wall_clock_val_calls += 1
+        if 'wall_clock/total' in metrics:
+            wall_clock_total_calls += 1
 
     assert isinstance(trainer.state.dataloader, collections.abc.Sized)
     assert trainer.state.dataloader_label is not None
@@ -42,4 +48,6 @@ def test_speed_monitor(composer_trainer_hparams: TrainerHparams):
     expected_step_calls = (trainer.state.dataloader_len - speed_monitor_hparams.window_size) * max_epochs
     assert throughput_step_calls == expected_step_calls
     assert throughput_epoch_calls == max_epochs
+    assert wall_clock_total_calls == max_epochs
     assert wall_clock_train_calls == max_epochs
+    assert wall_clock_val_calls == max_epochs