Repository: open-mmlab/mmdetection

Issue:
Add ability to serve models
**Describe the feature**

There are libaries such as Tensorflow Serve and Pytorch Serve. It would be useful is MMDetection had the ability to serve models, maybe inside a container, through an API endpoint.

**Motivation**
It would allow us to easily use models trained using MMDetection in other applications for inference.

**Related resources**
If there is an official code release or third-party implementations, please also provide the information here, which would be very helpful.

**Additional context**
At the moment I have a script which calls an MMDetection script that I use as the entrypoint for a Docker container; it starts up whenever I need it, it's pretty slow. It would be nice if there was an easier way.



Discussion:
Thanks for your kind suggestion. Due to limited resources, this may not be supported soon. But we will consider this valuable feature in the future.

Resolving patch:
diff --git a/tools/deployment/mmdet2torchserve.py b/tools/deployment/mmdet2torchserve.py
new file mode 100644
--- /dev/null
+++ b/tools/deployment/mmdet2torchserve.py
@@ -0,0 +1,107 @@
+from argparse import ArgumentParser, Namespace
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import mmcv
+
+try:
+    from model_archiver.model_packaging import package_model
+    from model_archiver.model_packaging_utils import ModelExportUtils
+except ImportError:
+    package_model = None
+
+
+def mmdet2torchserve(
+    config_file: str,
+    checkpoint_file: str,
+    output_folder: str,
+    model_name: str,
+    model_version: str = '1.0',
+    force: bool = False,
+):
+    """Converts MMDetection model (config + checkpoint) to TorchServe `.mar`.
+
+    Args:
+        config_file:
+            In MMDetection config format.
+            The contents vary for each task repository.
+        checkpoint_file:
+            In MMDetection checkpoint format.
+            The contents vary for each task repository.
+        output_folder:
+            Folder where `{model_name}.mar` will be created.
+            The file created will be in TorchServe archive format.
+        model_name:
+            If not None, used for naming the `{model_name}.mar` file
+            that will be created under `output_folder`.
+            If None, `{Path(checkpoint_file).stem}` will be used.
+        model_version:
+            Model's version.
+        force:
+            If True, if there is an existing `{model_name}.mar`
+            file under `output_folder` it will be overwritten.
+    """
+    config = mmcv.Config.fromfile(config_file)
+
+    with TemporaryDirectory() as tmpdir:
+        config.dump(f'{tmpdir}/config.py')
+
+        args = Namespace(
+            **{
+                'model_file': f'{tmpdir}/config.py',
+                'serialized_file': checkpoint_file,
+                'handler': f'{Path(__file__).parent}/mmdet_handler.py',
+                'model_name': model_name or Path(checkpoint_file).stem,
+                'version': model_version,
+                'export_path': output_folder,
+                'force': force,
+                'requirements_file': None,
+                'extra_files': None,
+                'runtime': 'python',
+                'archive_format': 'default'
+            })
+        manifest = ModelExportUtils.generate_manifest_json(args)
+        package_model(args, manifest)
+
+
+def parse_args():
+    parser = ArgumentParser(
+        description='Convert MMDetection models to TorchServe `.mar` format.')
+    parser.add_argument('config', type=str, help='config file path')
+    parser.add_argument('checkpoint', type=str, help='checkpoint file path')
+    parser.add_argument(
+        '--output-folder',
+        type=str,
+        required=True,
+        help='Folder where `{model_name}.mar` will be created.')
+    parser.add_argument(
+        '--model-name',
+        type=str,
+        default=None,
+        help='If not None, used for naming the `{model_name}.mar`'
+        'file that will be created under `output_folder`.'
+        'If None, `{Path(checkpoint_file).stem}` will be used.')
+    parser.add_argument(
+        '--model-version',
+        type=str,
+        default='1.0',
+        help='Number used for versioning.')
+    parser.add_argument(
+        '-f',
+        '--force',
+        action='store_true',
+        help='overwrite the existing `{model_name}.mar`')
+    args = parser.parse_args()
+
+    return args
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    if package_model is None:
+        raise ImportError('`torch-model-archiver` is required.'
+                          'Try: pip install torch-model-archiver')
+
+    mmdet2torchserve(args.config, args.checkpoint, args.output_folder,
+                     args.model_name, args.model_version, args.force)
diff --git a/tools/deployment/mmdet_handler.py b/tools/deployment/mmdet_handler.py
new file mode 100644
--- /dev/null
+++ b/tools/deployment/mmdet_handler.py
@@ -0,0 +1,69 @@
+import base64
+import os
+
+import mmcv
+import torch
+from ts.torch_handler.base_handler import BaseHandler
+
+from mmdet.apis import inference_detector, init_detector
+
+
+class MMdetHandler(BaseHandler):
+    threshold = 0.5
+
+    def initialize(self, context):
+        properties = context.system_properties
+        self.map_location = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.device = torch.device(self.map_location + ':' +
+                                   str(properties.get('gpu_id')) if torch.cuda.
+                                   is_available() else self.map_location)
+        self.manifest = context.manifest
+
+        model_dir = properties.get('model_dir')
+        serialized_file = self.manifest['model']['serializedFile']
+        checkpoint = os.path.join(model_dir, serialized_file)
+        self.config_file = os.path.join(model_dir, 'config.py')
+
+        self.model = init_detector(self.config_file, checkpoint, self.device)
+        self.initialized = True
+
+    def preprocess(self, data):
+        images = []
+
+        for row in data:
+            image = row.get('data') or row.get('body')
+            if isinstance(image, str):
+                image = base64.b64decode(image)
+            image = mmcv.imfrombytes(image)
+            images.append(image)
+
+        return images
+
+    def inference(self, data, *args, **kwargs):
+        results = inference_detector(self.model, data)
+        return results
+
+    def postprocess(self, data):
+        # Format output following the example ObjectDetectionHandler format
+        output = []
+        for image_index, image_result in enumerate(data):
+            output.append([])
+            if isinstance(image_result, tuple):
+                bbox_result, segm_result = image_result
+                if isinstance(segm_result, tuple):
+                    segm_result = segm_result[0]  # ms rcnn
+            else:
+                bbox_result, segm_result = image_result, None
+
+            for class_index, class_result in enumerate(bbox_result):
+                class_name = self.model.CLASSES[class_index]
+                for bbox in class_result:
+                    bbox_coords = bbox[:-1].tolist()
+                    score = float(bbox[-1])
+                    if score >= self.threshold:
+                        output[image_index].append({
+                            class_name: bbox_coords,
+                            'score': score
+                        })
+
+        return output


Test patch: