Spaces:
Sleeping
Sleeping
| # Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """TODO: Add a description here.""" | |
| from typing import List, Tuple, Literal | |
| from deprecated import deprecated | |
| import evaluate | |
| import datasets | |
| import numpy as np | |
| from seametrics.detection import PrecisionRecallF1Support | |
| from seametrics.payload import Payload | |
| _CITATION = """\ | |
| @InProceedings{coco:2020, | |
| title = {Microsoft {COCO:} Common Objects in Context}, | |
| authors={Tsung{-}Yi Lin and | |
| Michael Maire and | |
| Serge J. Belongie and | |
| James Hays and | |
| Pietro Perona and | |
| Deva Ramanan and | |
| Piotr Dollar and | |
| C. Lawrence Zitnick}, | |
| booktitle = {Computer Vision - {ECCV} 2014 - 13th European Conference, Zurich, | |
| Switzerland, September 6-12, 2014, Proceedings, Part {V}}, | |
| series = {Lecture Notes in Computer Science}, | |
| volume = {8693}, | |
| pages = {740--755}, | |
| publisher = {Springer}, | |
| year={2014} | |
| } | |
| """ | |
| _DESCRIPTION = """\ | |
| This evaluation metric is designed to give provide object detection metrics at | |
| different object size levels. It is based on a modified version of the commonly used | |
| COCO-evaluation metrics. | |
| """ | |
| _KWARGS_DESCRIPTION = """ | |
| Calculates object detection metrics given predicted and ground truth bounding boxes for | |
| a single image. | |
| Args: | |
| predictions: list of predictions for each image. Each prediction should | |
| be a dict containing the following | |
| - 'boxes': list of bounding boxes, xywh in absolute pixel values | |
| - 'labels': list of labels for each bounding box | |
| - 'scores': list of scores for each bounding box | |
| references: list of ground truth annotations for each image. Each reference should | |
| be a dict containing the following | |
| - 'boxes': list of bounding boxes, xywh in absolute pixel values | |
| - 'labels': list of labels for each bounding box | |
| - 'area': list of areas for each bounding box | |
| Returns: | |
| dict containing dicts for each specified area range with following items: | |
| 'range': specified area with [max_px_area, max_px_area] | |
| 'iouThr': min. IOU-threshold of a prediction with a ground truth box | |
| to be considered a correct prediction | |
| 'maxDets': maximum number of detections | |
| 'tp': number of true positive (correct) predictions | |
| 'fp': number of false positive (incorrect) predictions | |
| 'fn': number of false negative (missed) predictions | |
| 'duplicates': number of duplicate predictions | |
| 'precision': best possible score = 1, worst possible score = 0 | |
| large if few false positive predictions | |
| formula: tp/(fp+tp) | |
| 'recall' best possible score = 1, worst possible score = 0 | |
| large if few missed predictions | |
| formula: tp/(tp+fn) | |
| 'f1': best possible score = 1, worst possible score = 0 | |
| trades off precision and recall | |
| formula: 2*(precision*recall)/(precision+recall) | |
| 'support': number of ground truth bounding boxes considered in the evaluation, | |
| 'fpi': number of images with no ground truth but false positive predictions, | |
| 'nImgs': number of images considered in evaluation | |
| Examples: | |
| >>> import evaluate | |
| >>> from seametrics.payload.processor import PayloadProcessor | |
| >>> payload = PayloadProcessor(...).payload | |
| >>> module = evaluate.load("SEA-AI/det-metrics", ...) | |
| >>> module.add_payload(payload) | |
| >>> result = module.compute() | |
| >>> print(result) | |
| {'all': { | |
| 'range': [0, 10000000000.0], | |
| 'iouThr': '0.00', | |
| 'maxDets': 100, | |
| 'tp': 1, | |
| 'fp': 3, | |
| 'fn': 1, | |
| 'duplicates': 0, | |
| 'precision': 0.25, | |
| 'recall': 0.5, | |
| 'f1': 0.3333333333333333, | |
| 'support': 2, | |
| 'fpi': 0, | |
| 'nImgs': 2 | |
| } | |
| } | |
| """ | |
| class DetectionMetric(evaluate.Metric): | |
| def __init__( | |
| self, | |
| area_ranges_tuples: List[Tuple[str, List[int]]] = [("all", [0, 1e5**2])], | |
| iou_threshold: List[float] = [1e-10], | |
| class_agnostic: bool = True, | |
| bbox_format: str = "xywh", | |
| iou_type: Literal["bbox", "segm"] = "bbox", | |
| **kwargs | |
| ): | |
| super().__init__(**kwargs) | |
| self.coco_metric = PrecisionRecallF1Support( | |
| iou_thresholds=( | |
| iou_threshold if isinstance(iou_threshold, list) else [iou_threshold] | |
| ), | |
| area_ranges=[v for _, v in area_ranges_tuples], | |
| area_ranges_labels=[k for k, _ in area_ranges_tuples], | |
| class_agnostic=class_agnostic, | |
| iou_type=iou_type, | |
| box_format=bbox_format, | |
| ) | |
| def _info(self): | |
| return evaluate.MetricInfo( | |
| # This is the description that will appear on the modules page. | |
| module_type="metric", | |
| description=_DESCRIPTION, | |
| citation=_CITATION, | |
| inputs_description=_KWARGS_DESCRIPTION, | |
| # This defines the format of each prediction and reference | |
| features=datasets.Features( | |
| { | |
| "predictions": [ | |
| datasets.Features( | |
| { | |
| "boxes": datasets.Sequence( | |
| datasets.Sequence(datasets.Value("float")) | |
| ), | |
| "labels": datasets.Sequence(datasets.Value("int64")), | |
| "scores": datasets.Sequence(datasets.Value("float")), | |
| } | |
| ) | |
| ], | |
| "references": [ | |
| datasets.Features( | |
| { | |
| "boxes": datasets.Sequence( | |
| datasets.Sequence(datasets.Value("float")) | |
| ), | |
| "labels": datasets.Sequence(datasets.Value("int64")), | |
| "area": datasets.Sequence(datasets.Value("float")), | |
| } | |
| ) | |
| ], | |
| } | |
| ), | |
| # Additional links to the codebase or references | |
| codebase_urls=[ | |
| "https://github.com/SEA-AI/seametrics/tree/main", | |
| "https://lightning.ai/docs/torchmetrics/stable/detection/mean_average_precision.html", | |
| ], | |
| ) | |
| def add(self, *, prediction, reference, **kwargs): | |
| """Adds a batch of predictions and references to the metric""" | |
| # in case the inputs are lists, convert them to numpy arrays | |
| prediction = self._preprocess(prediction) | |
| reference = self._preprocess(reference) | |
| self.coco_metric.update(prediction, reference) | |
| # does not impact the metric, but is required for the interface x_x | |
| super(evaluate.Metric, self).add( | |
| prediction=self._postprocess(prediction), | |
| references=self._postprocess(reference), | |
| **kwargs | |
| ) | |
| def add_batch(self, payload: Payload, model_name: str = None): | |
| """Takes as input a payload and adds the batch to the metric""" | |
| self.add_payload(payload, model_name) | |
| def _compute(self, *, predictions, references, **kwargs): | |
| """Called within the evaluate.Metric.compute() method""" | |
| return self.coco_metric.compute()["metrics"] | |
| def add_payload(self, payload: Payload, model_name: str = None): | |
| """Converts the payload to the format expected by the metric""" | |
| # import only if needed since fiftyone is not a direct dependency | |
| from seametrics.detection.utils import payload_to_det_metric | |
| predictions, references = payload_to_det_metric(payload, model_name) | |
| self.add(prediction=predictions, reference=references) | |
| return self | |
| def _preprocess(self, list_of_dicts): | |
| """Converts the lists to numpy arrays for type checking""" | |
| return [self._lists_to_np(d) for d in list_of_dicts] | |
| def _postprocess(self, list_of_dicts): | |
| """Converts the numpy arrays to lists for type checking""" | |
| return [self._np_to_lists(d) for d in list_of_dicts] | |
| def _np_to_lists(self, d): | |
| """datasets does not support numpy arrays for type checking""" | |
| for k, v in d.items(): | |
| if isinstance(v, dict): | |
| self._np_to_lists(v) | |
| elif isinstance(v, np.ndarray): | |
| d[k] = v.tolist() | |
| return d | |
| def _lists_to_np(self, d): | |
| """datasets does not support numpy arrays for type checking""" | |
| for k, v in d.items(): | |
| if isinstance(v, dict): | |
| self._lists_to_np(v) | |
| elif isinstance(v, list): | |
| d[k] = np.array(v) | |
| return d | |