Repository: open-mmlab/mmdetection

Issue:
Support Evaluation over ConcatDataset
When combining multiple datasets for validation, evaluation function does not work.
`AttributeError: 'ConcatDataset' object has no attribute 'evaluate'
`

**Motivation**
Usually you download datasets for different sources and you keep them in the same format (say COCO). You can now train with all data using concat method but it would be very useful to validate on both too.

**Related resources**
None

**Additional context**

I am not sure about which would be the best approach... maybe edit `ConcatDataset` wrapper to merge different datasets first and then call parent class constructor. 

I am willing to implement this feature myself.


Discussion:

Resolving patch:
diff --git a/mmdet/datasets/builder.py b/mmdet/datasets/builder.py
--- a/mmdet/datasets/builder.py
+++ b/mmdet/datasets/builder.py
@@ -29,11 +29,15 @@ def _concat_dataset(cfg, default_args=None):
     img_prefixes = cfg.get('img_prefix', None)
     seg_prefixes = cfg.get('seg_prefix', None)
     proposal_files = cfg.get('proposal_file', None)
+    separate_eval = cfg.get('separate_eval', True)
 
     datasets = []
     num_dset = len(ann_files)
     for i in range(num_dset):
         data_cfg = copy.deepcopy(cfg)
+        # pop 'separate_eval' since it is not a valid key for common datasets.
+        if 'separate_eval' in data_cfg:
+            data_cfg.pop('separate_eval')
         data_cfg['ann_file'] = ann_files[i]
         if isinstance(img_prefixes, (list, tuple)):
             data_cfg['img_prefix'] = img_prefixes[i]
@@ -43,7 +47,7 @@ def _concat_dataset(cfg, default_args=None):
             data_cfg['proposal_file'] = proposal_files[i]
         datasets.append(build_dataset(data_cfg, default_args))
 
-    return ConcatDataset(datasets)
+    return ConcatDataset(datasets, separate_eval)
 
 
 def build_dataset(cfg, default_args=None):
@@ -51,6 +55,10 @@ def build_dataset(cfg, default_args=None):
                                    ClassBalancedDataset)
     if isinstance(cfg, (list, tuple)):
         dataset = ConcatDataset([build_dataset(c, default_args) for c in cfg])
+    elif cfg['type'] == 'ConcatDataset':
+        dataset = ConcatDataset(
+            [build_dataset(c, default_args) for c in cfg['datasets']],
+            cfg.get('separate_eval', True))
     elif cfg['type'] == 'RepeatDataset':
         dataset = RepeatDataset(
             build_dataset(cfg['dataset'], default_args), cfg['times'])
diff --git a/mmdet/datasets/custom.py b/mmdet/datasets/custom.py
--- a/mmdet/datasets/custom.py
+++ b/mmdet/datasets/custom.py
@@ -24,7 +24,7 @@ class CustomDataset(Dataset):
                 'width': 1280,
                 'height': 720,
                 'ann': {
-                    'bboxes': <np.ndarray> (n, 4),
+                    'bboxes': <np.ndarray> (n, 4) in (x1, y1, x2, y2) order.
                     'labels': <np.ndarray> (n, ),
                     'bboxes_ignore': <np.ndarray> (k, 4), (optional field)
                     'labels_ignore': <np.ndarray> (k, 4) (optional field)
diff --git a/mmdet/datasets/dataset_wrappers.py b/mmdet/datasets/dataset_wrappers.py
--- a/mmdet/datasets/dataset_wrappers.py
+++ b/mmdet/datasets/dataset_wrappers.py
@@ -3,9 +3,11 @@
 from collections import defaultdict
 
 import numpy as np
+from mmcv.utils import print_log
 from torch.utils.data.dataset import ConcatDataset as _ConcatDataset
 
 from .builder import DATASETS
+from .coco import CocoDataset
 
 
 @DATASETS.register_module()
@@ -17,11 +19,24 @@ class ConcatDataset(_ConcatDataset):
 
     Args:
         datasets (list[:obj:`Dataset`]): A list of datasets.
+        separate_eval (bool): Whether to evaluate the results
+            separately if it is used as validation dataset.
+            Defaults to True.
     """
 
-    def __init__(self, datasets):
+    def __init__(self, datasets, separate_eval=True):
         super(ConcatDataset, self).__init__(datasets)
         self.CLASSES = datasets[0].CLASSES
+        self.separate_eval = separate_eval
+        if not separate_eval:
+            if any([isinstance(ds, CocoDataset) for ds in datasets]):
+                raise NotImplementedError(
+                    'Evaluating concatenated CocoDataset as a whole is not'
+                    ' supported! Please set "separate_eval=True"')
+            elif len(set([type(ds) for ds in datasets])) != 1:
+                raise NotImplementedError(
+                    'All the datasets should have same types')
+
         if hasattr(datasets[0], 'flag'):
             flags = []
             for i in range(0, len(datasets)):
@@ -50,6 +65,64 @@ def get_cat_ids(self, idx):
             sample_idx = idx - self.cumulative_sizes[dataset_idx - 1]
         return self.datasets[dataset_idx].get_cat_ids(sample_idx)
 
+    def evaluate(self, results, logger=None, **kwargs):
+        """Evaluate the results.
+
+        Args:
+            results (list[list | tuple]): Testing results of the dataset.
+            logger (logging.Logger | str | None): Logger used for printing
+                related information during evaluation. Default: None.
+
+        Returns:
+            dict[str: float]: AP results of the total dataset or each separate
+            dataset if `self.separate_eval=True`.
+        """
+        assert len(results) == self.cumulative_sizes[-1], \
+            ('Dataset and results have different sizes: '
+             f'{self.cumulative_sizes[-1]} v.s. {len(results)}')
+
+        # Check whether all the datasets support evaluation
+        for dataset in self.datasets:
+            assert hasattr(dataset, 'evaluate'), \
+                    f'{type(dataset)} does not implement evaluate function'
+
+        if self.separate_eval:
+            dataset_idx = -1
+            total_eval_results = dict()
+            for size, dataset in zip(self.cumulative_sizes, self.datasets):
+                start_idx = 0 if dataset_idx == -1 else \
+                    self.cumulative_sizes[dataset_idx]
+                end_idx = self.cumulative_sizes[dataset_idx + 1]
+
+                results_per_dataset = results[start_idx:end_idx]
+                print_log(
+                    f'\nEvaluateing {dataset.ann_file} with '
+                    f'{len(results_per_dataset)} images now',
+                    logger=logger)
+
+                eval_results_per_dataset = dataset.evaluate(
+                    results_per_dataset, logger=logger, **kwargs)
+                dataset_idx += 1
+                for k, v in eval_results_per_dataset.items():
+                    total_eval_results.update({f'{dataset_idx}_{k}': v})
+
+            return total_eval_results
+        elif any([isinstance(ds, CocoDataset) for ds in self.datasets]):
+            raise NotImplementedError(
+                'Evaluating concatenated CocoDataset as a whole is not'
+                ' supported! Please set "separate_eval=True"')
+        elif len(set([type(ds) for ds in self.datasets])) != 1:
+            raise NotImplementedError(
+                'All the datasets should have same types')
+        else:
+            original_data_infos = self.datasets[0].data_infos
+            self.datasets[0].data_infos = sum(
+                [dataset.data_infos for dataset in self.datasets], [])
+            eval_results = self.datasets[0].evaluate(
+                results, logger=logger, **kwargs)
+            self.datasets[0].data_infos = original_data_infos
+            return eval_results
+
 
 @DATASETS.register_module()
 class RepeatDataset(object):
@@ -135,8 +208,8 @@ def __init__(self, dataset, oversample_thr):
 
         repeat_factors = self._get_repeat_factors(dataset, oversample_thr)
         repeat_indices = []
-        for dataset_index, repeat_factor in enumerate(repeat_factors):
-            repeat_indices.extend([dataset_index] * math.ceil(repeat_factor))
+        for dataset_idx, repeat_factor in enumerate(repeat_factors):
+            repeat_indices.extend([dataset_idx] * math.ceil(repeat_factor))
         self.repeat_indices = repeat_indices
 
         flags = []


Test patch:
diff --git a/tests/test_data/test_dataset.py b/tests/test_data/test_dataset.py
--- a/tests/test_data/test_dataset.py
+++ b/tests/test_data/test_dataset.py
@@ -1,13 +1,201 @@
 import bisect
 import math
+import os.path as osp
+import tempfile
 from collections import defaultdict
 from unittest.mock import MagicMock
 
+import mmcv
 import numpy as np
 import pytest
 
-from mmdet.datasets import (DATASETS, ClassBalancedDataset, ConcatDataset,
-                            CustomDataset, RepeatDataset)
+from mmdet.datasets import (DATASETS, ClassBalancedDataset, CocoDataset,
+                            ConcatDataset, CustomDataset, RepeatDataset,
+                            build_dataset)
+
+
+def _create_dummy_coco_json(json_name):
+    image = {
+        'id': 0,
+        'width': 640,
+        'height': 640,
+        'file_name': 'fake_name.jpg',
+    }
+
+    annotation_1 = {
+        'id': 1,
+        'image_id': 0,
+        'category_id': 0,
+        'area': 400,
+        'bbox': [50, 60, 20, 20],
+        'iscrowd': 0,
+    }
+
+    annotation_2 = {
+        'id': 2,
+        'image_id': 0,
+        'category_id': 0,
+        'area': 900,
+        'bbox': [100, 120, 30, 30],
+        'iscrowd': 0,
+    }
+
+    annotation_3 = {
+        'id': 3,
+        'image_id': 0,
+        'category_id': 0,
+        'area': 1600,
+        'bbox': [150, 160, 40, 40],
+        'iscrowd': 0,
+    }
+
+    annotation_4 = {
+        'id': 4,
+        'image_id': 0,
+        'category_id': 0,
+        'area': 10000,
+        'bbox': [250, 260, 100, 100],
+        'iscrowd': 0,
+    }
+
+    categories = [{
+        'id': 0,
+        'name': 'car',
+        'supercategory': 'car',
+    }]
+
+    fake_json = {
+        'images': [image],
+        'annotations':
+        [annotation_1, annotation_2, annotation_3, annotation_4],
+        'categories': categories
+    }
+
+    mmcv.dump(fake_json, json_name)
+
+
+def _create_dummy_custom_pkl(pkl_name):
+    fake_pkl = [{
+        'filename': 'fake_name.jpg',
+        'width': 640,
+        'height': 640,
+        'ann': {
+            'bboxes':
+            np.array([[50, 60, 70, 80], [100, 120, 130, 150],
+                      [150, 160, 190, 200], [250, 260, 350, 360]]),
+            'labels':
+            np.array([0, 0, 0, 0])
+        }
+    }]
+    mmcv.dump(fake_pkl, pkl_name)
+
+
+def _create_dummy_results():
+    boxes = [
+        np.array([[50, 60, 70, 80, 1.0], [100, 120, 130, 150, 0.98],
+                  [150, 160, 190, 200, 0.96], [250, 260, 350, 360, 0.95]])
+    ]
+    return [boxes]
+
+
+def test_dataset_evaluation():
+    tmp_dir = tempfile.TemporaryDirectory()
+    # create dummy data
+    fake_json_file = osp.join(tmp_dir.name, 'fake_data.json')
+    _create_dummy_coco_json(fake_json_file)
+
+    # test single coco dataset evaluation
+    coco_dataset = CocoDataset(
+        ann_file=fake_json_file, classes=('car', ), pipeline=[])
+    fake_results = _create_dummy_results()
+    eval_results = coco_dataset.evaluate(fake_results, classwise=True)
+    assert eval_results['bbox_mAP'] == 1
+    assert eval_results['bbox_mAP_50'] == 1
+    assert eval_results['bbox_mAP_75'] == 1
+
+    # test concat dataset evaluation
+    fake_concat_results = _create_dummy_results() + _create_dummy_results()
+
+    # build concat dataset through two config dict
+    coco_cfg = dict(
+        type='CocoDataset',
+        ann_file=fake_json_file,
+        classes=('car', ),
+        pipeline=[])
+    concat_cfgs = [coco_cfg, coco_cfg]
+    concat_dataset = build_dataset(concat_cfgs)
+    eval_results = concat_dataset.evaluate(fake_concat_results)
+    assert eval_results['0_bbox_mAP'] == 1
+    assert eval_results['0_bbox_mAP_50'] == 1
+    assert eval_results['0_bbox_mAP_75'] == 1
+    assert eval_results['1_bbox_mAP'] == 1
+    assert eval_results['1_bbox_mAP_50'] == 1
+    assert eval_results['1_bbox_mAP_75'] == 1
+
+    # build concat dataset through concatenated ann_file
+    coco_cfg = dict(
+        type='CocoDataset',
+        ann_file=[fake_json_file, fake_json_file],
+        classes=('car', ),
+        pipeline=[])
+    concat_dataset = build_dataset(coco_cfg)
+    eval_results = concat_dataset.evaluate(fake_concat_results)
+    assert eval_results['0_bbox_mAP'] == 1
+    assert eval_results['0_bbox_mAP_50'] == 1
+    assert eval_results['0_bbox_mAP_75'] == 1
+    assert eval_results['1_bbox_mAP'] == 1
+    assert eval_results['1_bbox_mAP_50'] == 1
+    assert eval_results['1_bbox_mAP_75'] == 1
+
+    # create dummy data
+    fake_pkl_file = osp.join(tmp_dir.name, 'fake_data.pkl')
+    _create_dummy_custom_pkl(fake_pkl_file)
+
+    # test single custom dataset evaluation
+    custom_dataset = CustomDataset(
+        ann_file=fake_pkl_file, classes=('car', ), pipeline=[])
+    fake_results = _create_dummy_results()
+    eval_results = custom_dataset.evaluate(fake_results)
+    assert eval_results['mAP'] == 1
+
+    # test concat dataset evaluation
+    fake_concat_results = _create_dummy_results() + _create_dummy_results()
+
+    # build concat dataset through two config dict
+    custom_cfg = dict(
+        type='CustomDataset',
+        ann_file=fake_pkl_file,
+        classes=('car', ),
+        pipeline=[])
+    concat_cfgs = [custom_cfg, custom_cfg]
+    concat_dataset = build_dataset(concat_cfgs)
+    eval_results = concat_dataset.evaluate(fake_concat_results)
+    assert eval_results['0_mAP'] == 1
+    assert eval_results['1_mAP'] == 1
+
+    # build concat dataset through concatenated ann_file
+    concat_cfg = dict(
+        type='CustomDataset',
+        ann_file=[fake_pkl_file, fake_pkl_file],
+        classes=('car', ),
+        pipeline=[])
+    concat_dataset = build_dataset(concat_cfg)
+    eval_results = concat_dataset.evaluate(fake_concat_results)
+    assert eval_results['0_mAP'] == 1
+    assert eval_results['1_mAP'] == 1
+
+    # build concat dataset through explict type
+    concat_cfg = dict(
+        type='ConcatDataset',
+        datasets=[custom_cfg, custom_cfg],
+        separate_eval=False)
+    concat_dataset = build_dataset(concat_cfg)
+    eval_results = concat_dataset.evaluate(fake_concat_results, metric='mAP')
+    assert eval_results['mAP'] == 1
+    assert len(concat_dataset.datasets[0].data_infos) == \
+        len(concat_dataset.datasets[1].data_infos)
+    assert len(concat_dataset.datasets[0].data_infos) == 1
+    tmp_dir.cleanup()
 
 
 @pytest.mark.parametrize('dataset',
diff --git a/tools/test.py b/tools/test.py
--- a/tools/test.py
+++ b/tools/test.py
@@ -91,7 +91,13 @@ def main():
         if cfg.model.neck.get('rfp_backbone'):
             if cfg.model.neck.rfp_backbone.get('pretrained'):
                 cfg.model.neck.rfp_backbone.pretrained = None
-    cfg.data.test.test_mode = True
+
+    # in case the test dataset is concatenated
+    if isinstance(cfg.data.test, dict):
+        cfg.data.test.test_mode = True
+    elif isinstance(cfg.data.test, list):
+        for ds_cfg in cfg.data.test:
+            ds_cfg.test_mode = True
 
     # init distributed env first, since logger depends on the dist info.
     if args.launcher == 'none':