Repository: open-mmlab/mmengine

Issue:
[Bug] when log_with_hierarchy is `True`, the running error appears in `_collect_scalars` for log processor
### Prerequisite

- [X] I have searched [Issues](https://github.com/open-mmlab/mmengine/issues) and [Discussions](https://github.com/open-mmlab/mmengine/discussions) but cannot get the expected help.
- [X] The bug has not been fixed in the latest version(https://github.com/open-mmlab/mmengine).

### Environment

OrderedDict([('sys.platform', 'linux'), ('Python', '3.8.0 | packaged by conda-forge | (default, Nov 22 2019, 19:11:38) [GCC 7.3.0]'), ('CUDA available', True), ('numpy_random_seed', 2147483648), ('GPU 0', 'NVIDIA GeForce RTX 3090'), ('CUDA_HOME', '/usr/local/cuda-11.7'), ('NVCC', 'Cuda compilation tools, release 11.7, V11.7.99'), ('GCC', 'gcc (Ubuntu 11.2.0-19ubuntu1) 11.2.0'), ('PyTorch', '1.13.1'), ('PyTorch compiling details', 'PyTorch built with:\n  - GCC 9.3\n  - C++ Version: 201402\n  - Intel(R) oneAPI Math Kernel Library Version 2021.4-Product Build 20210904 for Intel(R) 64 architecture applications\n  - Intel(R) MKL-DNN v2.6.0 (Git Hash 52b5f107dd9cf10910aaa19cb47f3abf9b349815)\n  - OpenMP 201511 (a.k.a. OpenMP 4.5)\n  - LAPACK is enabled (usually provided by MKL)\n  - NNPACK is enabled\n  - CPU capability usage: AVX2\n  - CUDA Runtime 11.7\n  - NVCC architecture flags: -gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86;-gencode;arch=compute_37,code=compute_37\n  - CuDNN 8.5\n  - Magma 2.6.1\n  - Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CUDA_VERSION=11.7, CUDNN_VERSION=8.5.0, CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, CXX_FLAGS= -fabi-version=11 -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -fopenmp -DNDEBUG -DUSE_KINETO -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -DEDGE_PROFILER_USE_KINETO -O2 -fPIC -Wno-narrowing -Wall -Wextra -Werror=return-type -Werror=non-virtual-dtor -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wunused-local-typedefs -Wno-unused-parameter -Wno-unused-function -Wno-unused-result -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-stringop-overflow -Wno-psabi -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Werror=cast-function-type -Wno-stringop-overflow, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_VERSION=1.13.1, USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=ON, USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, \n'), ('TorchVision', '0.14.1'), ('OpenCV', '4.7.0'), ('MMEngine', '0.7.0')])

### Reproduces the problem - code sample

    log_processor = dict(
        by_epoch=False,
        window_size=50,
        custom_cfg=[
            dict(data_src='decode.acc_seg',
                 method_name='mean',
                 log_name='acc_seg_large_window',
                 window_size=50)
        ],
        log_with_hierarchy=True,
    )

### Reproduces the problem - command or script

    log_processor = dict(
        by_epoch=False,
        window_size=50,
        custom_cfg=[
            dict(data_src='decode.acc_seg',
                 method_name='mean',
                 log_name='acc_seg_large_window',
                 window_size=50)
        ],
        log_with_hierarchy=True,
    )

### Reproduces the problem - error message

Traceback (most recent call last):
  File "/home/akide/anaconda3/envs/xxx_project/lib/python3.8/site-packages/mmengine/runner/runner.py", line 1763, in call_hook
    getattr(hook, fn_name)(self, **kwargs)
  File "/home/akide/anaconda3/envs/xxx_project/lib/python3.8/site-packages/mmengine/hooks/logger_hook.py", line 167, in after_train_iter
    tag, log_str = runner.log_processor.get_log_after_iter(
  File "/home/akide/anaconda3/envs/xxx_projectlib/python3.8/site-packages/mmengine/runner/log_processor.py", line 145, in get_log_after_iter
    tag = self._collect_scalars(parsed_cfg, runner, mode, True)
  File "/home/akide/anaconda3/envs/xxx_project/lib/python3.8/site-packages/mmengine/runner/log_processor.py", line 368, in _collect_scalars
    data_src = log_cfg.pop('data_src')
KeyError: 'data_src'

### Additional information

1. in the following line, it's popped out the keys`data_src`https://github.com/open-mmlab/mmengine/blob/093068e4ffdb0bb6ab0b95c45d9616835e930ce0/mmengine/runner/log_processor.py#L367
2. however, if log_with_hierarchy is enabled, the pop will try to access the same keys twice which occurred a runtime error.https://github.com/open-mmlab/mmengine/blob/093068e4ffdb0bb6ab0b95c45d9616835e930ce0/mmengine/runner/log_processor.py#L140-L145
3. let me know if it is a feature or bug, if it's a bug I can open a PR to fix it.


Discussion:

Resolving patch:
diff --git a/mmengine/runner/log_processor.py b/mmengine/runner/log_processor.py
--- a/mmengine/runner/log_processor.py
+++ b/mmengine/runner/log_processor.py
@@ -341,6 +341,7 @@ def _collect_scalars(self,
         Returns:
             dict: Statistical values of logs.
         """
+        custom_cfg = copy.deepcopy(custom_cfg)
         tag = OrderedDict()
         # history_scalars of train/val/test phase.
         history_scalars = runner.message_hub.log_scalars


Test patch:
diff --git a/tests/test_runner/test_log_processor.py b/tests/test_runner/test_log_processor.py
--- a/tests/test_runner/test_log_processor.py
+++ b/tests/test_runner/test_log_processor.py
@@ -5,12 +5,14 @@
 import numpy as np
 import pytest
 import torch
+from parameterized import parameterized
 
 from mmengine.logging import HistoryBuffer, MessageHub, MMLogger
 from mmengine.runner import LogProcessor
+from mmengine.testing import RunnerTestCase
 
 
-class TestLogProcessor:
+class TestLogProcessor(RunnerTestCase):
 
     def test_init(self):
         log_processor = LogProcessor(
@@ -69,12 +71,13 @@ def test_parse_windows_size(self):
         with pytest.raises(TypeError):
             log_processor._parse_windows_size(self.runner, 1, custom_cfg)
 
-    @pytest.mark.parametrize(
-        'by_epoch,mode,log_with_hierarchy',
+    # yapf: disable
+    @parameterized.expand(
         ([True, 'train', True], [True, 'train', False], [False, 'train', True],
          [False, 'train', False], [True, 'val', True], [True, 'val', False],
          [False, 'val', True], [False, 'val', False], [True, 'test', True],
          [True, 'test', False], [False, 'test', True], [False, 'test', False]))
+    # yapf: enable
     def test_get_log_after_iter(self, by_epoch, mode, log_with_hierarchy):
         # Prepare LoggerHook
         log_processor = LogProcessor(
@@ -141,8 +144,7 @@ def test_get_log_after_iter(self, by_epoch, mode, log_with_hierarchy):
                 log_str += f"loss_cls: {train_logs['loss_cls']:.4f}"
             assert out == log_str
 
-    @pytest.mark.parametrize(
-        'by_epoch,mode,log_with_hierarchy',
+    @parameterized.expand(
         ([True, 'val', True], [True, 'val', False], [False, 'val', True],
          [False, 'val', False], [True, 'test', True], [False, 'test', False]))
     def test_log_val(self, by_epoch, mode, log_with_hierarchy):
@@ -188,7 +190,7 @@ def test_collect_scalars(self):
         custom_cfg = [
             dict(data_src='time', method_name='max', log_name='time_max')
         ]
-        logger_hook = LogProcessor(custom_cfg=custom_cfg)
+        log_processor = LogProcessor(custom_cfg=custom_cfg)
         # Collect with prefix.
         log_scalars = {
             'train/time': history_time_buffer,
@@ -197,7 +199,7 @@ def test_collect_scalars(self):
             'val/metric': history_metric_buffer
         }
         self.runner.message_hub._log_scalars = log_scalars
-        tag = logger_hook._collect_scalars(
+        tag = log_processor._collect_scalars(
             copy.deepcopy(custom_cfg), self.runner, mode='train')
         # Test training key in tag.
         assert list(tag.keys()) == ['time', 'loss_cls', 'time_max']
@@ -206,7 +208,7 @@ def test_collect_scalars(self):
         assert tag['time_max'] == time_scalars.max()
         assert tag['loss_cls'] == loss_cls_scalars[-10:].mean()
 
-        tag = logger_hook._collect_scalars(
+        tag = log_processor._collect_scalars(
             copy.deepcopy(custom_cfg), self.runner, mode='val')
         assert list(tag.keys()) == ['metric']
         assert tag['metric'] == metric_scalars[-1]
@@ -268,7 +270,7 @@ def test_get_cur_loop(self):
         loop = log_processor._get_cur_loop(self.runner, 'test')
         assert len(loop.dataloader) == 5
 
-    def setup_method(self):
+    def setUp(self):
         runner = MagicMock()
         runner.epoch = 1
         runner.max_epochs = 10
@@ -289,3 +291,36 @@ def setup_method(self):
             message_hub.update_scalar('val/acc', i * 0.1)
         runner.message_hub = message_hub
         self.runner = runner
+        super().setUp()
+
+    def test_with_runner(self):
+        cfg = self.epoch_based_cfg.copy()
+        cfg.log_processor = dict(
+            custom_cfg=[
+                dict(
+                    data_src='time',
+                    window_size='epoch',
+                    log_name='iter_time',
+                    method_name='mean')
+            ],
+            log_with_hierarchy=True)
+        runner = self.build_runner(cfg)
+        runner.train()
+        runner.val()
+        runner.test()
+
+        cfg = self.iter_based_cfg.copy()
+        cfg.log_processor = dict(
+            by_epoch=False,
+            custom_cfg=[
+                dict(
+                    data_src='time',
+                    window_size=100,
+                    log_name='iter_time',
+                    method_name='mean')
+            ],
+            log_with_hierarchy=True)
+        runner = self.build_runner(cfg)
+        runner.train()
+        runner.val()
+        runner.test()