# Copyright 2017 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Evaluator class for Visual Relations Detection. VRDDetectionEvaluator is a class which manages ground truth information of a visual relations detection (vrd) dataset, and computes frequently used detection metrics such as Precision, Recall, Recall@k, of the provided vrd detection results. It supports the following operations: 1) Adding ground truth information of images sequentially. 2) Adding detection results of images sequentially. 3) Evaluating detection metrics on already inserted detection results. Note1: groundtruth should be inserted before evaluation. Note2: This module operates on numpy boxes and box lists. """ from abc import abstractmethod import collections import logging import numpy as np from object_detection.core import standard_fields from object_detection.utils import metrics from object_detection.utils import object_detection_evaluation from object_detection.utils import per_image_vrd_evaluation # Below standard input numpy datatypes are defined: # box_data_type - datatype of the groundtruth visual relations box annotations; # this datatype consists of two named boxes: subject bounding box and object # bounding box. Each box is of the format [y_min, x_min, y_max, x_max], each # coordinate being of type float32. # label_data_type - corresponding datatype of the visual relations label # annotaions; it consists of three numerical class labels: subject class label, # object class label and relation class label, each class label being of type # int32. vrd_box_data_type = np.dtype([('subject', 'f4', (4,)), ('object', 'f4', (4,))]) single_box_data_type = np.dtype([('box', 'f4', (4,))]) label_data_type = np.dtype([('subject', 'i4'), ('object', 'i4'), ('relation', 'i4')]) class VRDDetectionEvaluator(object_detection_evaluation.DetectionEvaluator): """A class to evaluate VRD detections. This class serves as a base class for VRD evaluation in two settings: - phrase detection - relation detection. """ def __init__(self, matching_iou_threshold=0.5, metric_prefix=None): """Constructor. Args: matching_iou_threshold: IOU threshold to use for matching groundtruth boxes to detection boxes. metric_prefix: (optional) string prefix for metric name; if None, no prefix is used. """ super(VRDDetectionEvaluator, self).__init__([]) self._matching_iou_threshold = matching_iou_threshold self._evaluation = _VRDDetectionEvaluation( matching_iou_threshold=self._matching_iou_threshold) self._image_ids = set([]) self._metric_prefix = (metric_prefix + '_') if metric_prefix else '' self._evaluatable_labels = {} self._negative_labels = {} @abstractmethod def _process_groundtruth_boxes(self, groundtruth_box_tuples): """Pre-processes boxes before adding them to the VRDDetectionEvaluation. Phrase detection and Relation detection subclasses re-implement this method depending on the task. Args: groundtruth_box_tuples: A numpy array of structures with the shape [M, 1], each structure containing the same number of named bounding boxes. Each box is of the format [y_min, x_min, y_max, x_max] (see datatype vrd_box_data_type, single_box_data_type above). """ raise NotImplementedError( '_process_groundtruth_boxes method should be implemented in subclasses' 'of VRDDetectionEvaluator.') @abstractmethod def _process_detection_boxes(self, detections_box_tuples): """Pre-processes boxes before adding them to the VRDDetectionEvaluation. Phrase detection and Relation detection subclasses re-implement this method depending on the task. Args: detections_box_tuples: A numpy array of structures with the shape [M, 1], each structure containing the same number of named bounding boxes. Each box is of the format [y_min, x_min, y_max, x_max] (see datatype vrd_box_data_type, single_box_data_type above). """ raise NotImplementedError( '_process_detection_boxes method should be implemented in subclasses' 'of VRDDetectionEvaluator.') def add_single_ground_truth_image_info(self, image_id, groundtruth_dict): """Adds groundtruth for a single image to be used for evaluation. Args: image_id: A unique string/integer identifier for the image. groundtruth_dict: A dictionary containing - standard_fields.InputDataFields.groundtruth_boxes: A numpy array of structures with the shape [M, 1], representing M tuples, each tuple containing the same number of named bounding boxes. Each box is of the format [y_min, x_min, y_max, x_max] (see datatype vrd_box_data_type, single_box_data_type above). standard_fields.InputDataFields.groundtruth_classes: A numpy array of structures shape [M, 1], representing the class labels of the corresponding bounding boxes and possibly additional classes (see datatype label_data_type above). standard_fields.InputDataFields.groundtruth_image_classes: numpy array of shape [K] containing verified labels. Raises: ValueError: On adding groundtruth for an image more than once. """ if image_id in self._image_ids: raise ValueError('Image with id {} already added.'.format(image_id)) groundtruth_class_tuples = ( groundtruth_dict[standard_fields.InputDataFields.groundtruth_classes]) groundtruth_box_tuples = ( groundtruth_dict[standard_fields.InputDataFields.groundtruth_boxes]) self._evaluation.add_single_ground_truth_image_info( image_key=image_id, groundtruth_box_tuples=self._process_groundtruth_boxes( groundtruth_box_tuples), groundtruth_class_tuples=groundtruth_class_tuples) self._image_ids.update([image_id]) all_classes = [] for field in groundtruth_box_tuples.dtype.fields: all_classes.append(groundtruth_class_tuples[field]) groudtruth_positive_classes = np.unique(np.concatenate(all_classes)) verified_labels = groundtruth_dict.get( standard_fields.InputDataFields.groundtruth_image_classes, np.array([], dtype=int)) self._evaluatable_labels[image_id] = np.unique( np.concatenate((verified_labels, groudtruth_positive_classes))) self._negative_labels[image_id] = np.setdiff1d(verified_labels, groudtruth_positive_classes) def add_single_detected_image_info(self, image_id, detections_dict): """Adds detections for a single image to be used for evaluation. Args: image_id: A unique string/integer identifier for the image. detections_dict: A dictionary containing - standard_fields.DetectionResultFields.detection_boxes: A numpy array of structures with shape [N, 1], representing N tuples, each tuple containing the same number of named bounding boxes. Each box is of the format [y_min, x_min, y_max, x_max] (as an example see datatype vrd_box_data_type, single_box_data_type above). standard_fields.DetectionResultFields.detection_scores: float32 numpy array of shape [N] containing detection scores for the boxes. standard_fields.DetectionResultFields.detection_classes: A numpy array of structures shape [N, 1], representing the class labels of the corresponding bounding boxes and possibly additional classes (see datatype label_data_type above). """ if image_id not in self._image_ids: logging.warn('No groundtruth for the image with id %s.', image_id) # Since for the correct work of evaluator it is assumed that groundtruth # is inserted first we make sure to break the code if is it not the case. self._image_ids.update([image_id]) self._negative_labels[image_id] = np.array([]) self._evaluatable_labels[image_id] = np.array([]) num_detections = detections_dict[ standard_fields.DetectionResultFields.detection_boxes].shape[0] detection_class_tuples = detections_dict[ standard_fields.DetectionResultFields.detection_classes] detection_box_tuples = detections_dict[ standard_fields.DetectionResultFields.detection_boxes] negative_selector = np.zeros(num_detections, dtype=bool) selector = np.ones(num_detections, dtype=bool) # Only check boxable labels for field in detection_box_tuples.dtype.fields: # Verify if one of the labels is negative (this is sure FP) negative_selector |= np.isin(detection_class_tuples[field], self._negative_labels[image_id]) # Verify if all labels are verified selector &= np.isin(detection_class_tuples[field], self._evaluatable_labels[image_id]) selector |= negative_selector self._evaluation.add_single_detected_image_info( image_key=image_id, detected_box_tuples=self._process_detection_boxes( detection_box_tuples[selector]), detected_scores=detections_dict[ standard_fields.DetectionResultFields.detection_scores][selector], detected_class_tuples=detection_class_tuples[selector]) def evaluate(self, relationships=None): """Compute evaluation result. Args: relationships: A dictionary of numerical label-text label mapping; if specified, returns per-relationship AP. Returns: A dictionary of metrics with the following fields - summary_metrics: 'weightedAP@IOU' : weighted average precision at the specified IOU threshold. 'AP@IOU/' : AP per relationship. 'mAP@IOU': mean average precision at the specified IOU threshold. 'Recall@50@IOU': recall@50 at the specified IOU threshold. 'Recall@100@IOU': recall@100 at the specified IOU threshold. if relationships is specified, returns in AP metrics as readable names, otherwise the names correspond to class numbers. """ (weighted_average_precision, mean_average_precision, average_precisions, _, _, recall_50, recall_100, _, _) = ( self._evaluation.evaluate()) vrd_metrics = { (self._metric_prefix + 'weightedAP@{}IOU'.format( self._matching_iou_threshold)): weighted_average_precision, self._metric_prefix + 'mAP@{}IOU'.format(self._matching_iou_threshold): mean_average_precision, self._metric_prefix + 'Recall@50@{}IOU'.format( self._matching_iou_threshold): recall_50, self._metric_prefix + 'Recall@100@{}IOU'.format( self._matching_iou_threshold): recall_100, } if relationships: for key, average_precision in average_precisions.iteritems(): vrd_metrics[self._metric_prefix + 'AP@{}IOU/{}'.format( self._matching_iou_threshold, relationships[key])] = average_precision else: for key, average_precision in average_precisions.iteritems(): vrd_metrics[self._metric_prefix + 'AP@{}IOU/{}'.format( self._matching_iou_threshold, key)] = average_precision return vrd_metrics def clear(self): """Clears the state to prepare for a fresh evaluation.""" self._evaluation = _VRDDetectionEvaluation( matching_iou_threshold=self._matching_iou_threshold) self._image_ids.clear() self._negative_labels.clear() self._evaluatable_labels.clear() class VRDRelationDetectionEvaluator(VRDDetectionEvaluator): """A class to evaluate VRD detections in relations setting. Expected groundtruth box datatype is vrd_box_data_type, expected groudtruth labels datatype is label_data_type. Expected detection box datatype is vrd_box_data_type, expected detection labels datatype is label_data_type. """ def __init__(self, matching_iou_threshold=0.5): super(VRDRelationDetectionEvaluator, self).__init__( matching_iou_threshold=matching_iou_threshold, metric_prefix='VRDMetric_Relationships') def _process_groundtruth_boxes(self, groundtruth_box_tuples): """Pre-processes boxes before adding them to the VRDDetectionEvaluation. Args: groundtruth_box_tuples: A numpy array of structures with the shape [M, 1], each structure containing the same number of named bounding boxes. Each box is of the format [y_min, x_min, y_max, x_max]. Returns: Unchanged input. """ return groundtruth_box_tuples def _process_detection_boxes(self, detections_box_tuples): """Pre-processes boxes before adding them to the VRDDetectionEvaluation. Phrase detection and Relation detection subclasses re-implement this method depending on the task. Args: detections_box_tuples: A numpy array of structures with the shape [M, 1], each structure containing the same number of named bounding boxes. Each box is of the format [y_min, x_min, y_max, x_max] (see datatype vrd_box_data_type, single_box_data_type above). Returns: Unchanged input. """ return detections_box_tuples class VRDPhraseDetectionEvaluator(VRDDetectionEvaluator): """A class to evaluate VRD detections in phrase setting. Expected groundtruth box datatype is vrd_box_data_type, expected groudtruth labels datatype is label_data_type. Expected detection box datatype is single_box_data_type, expected detection labels datatype is label_data_type. """ def __init__(self, matching_iou_threshold=0.5): super(VRDPhraseDetectionEvaluator, self).__init__( matching_iou_threshold=matching_iou_threshold, metric_prefix='VRDMetric_Phrases') def _process_groundtruth_boxes(self, groundtruth_box_tuples): """Pre-processes boxes before adding them to the VRDDetectionEvaluation. In case of phrase evaluation task, evaluation expects exactly one bounding box containing all objects in the phrase. This bounding box is computed as an enclosing box of all groundtruth boxes of a phrase. Args: groundtruth_box_tuples: A numpy array of structures with the shape [M, 1], each structure containing the same number of named bounding boxes. Each box is of the format [y_min, x_min, y_max, x_max]. See vrd_box_data_type for an example of structure. Returns: result: A numpy array of structures with the shape [M, 1], each structure containing exactly one named bounding box. i-th output structure corresponds to the result of processing i-th input structure, where the named bounding box is computed as an enclosing bounding box of all bounding boxes of the i-th input structure. """ first_box_key = groundtruth_box_tuples.dtype.fields.keys()[0] miny = groundtruth_box_tuples[first_box_key][:, 0] minx = groundtruth_box_tuples[first_box_key][:, 1] maxy = groundtruth_box_tuples[first_box_key][:, 2] maxx = groundtruth_box_tuples[first_box_key][:, 3] for fields in groundtruth_box_tuples.dtype.fields: miny = np.minimum(groundtruth_box_tuples[fields][:, 0], miny) minx = np.minimum(groundtruth_box_tuples[fields][:, 1], minx) maxy = np.maximum(groundtruth_box_tuples[fields][:, 2], maxy) maxx = np.maximum(groundtruth_box_tuples[fields][:, 3], maxx) data_result = [] for i in range(groundtruth_box_tuples.shape[0]): data_result.append(([miny[i], minx[i], maxy[i], maxx[i]],)) result = np.array(data_result, dtype=[('box', 'f4', (4,))]) return result def _process_detection_boxes(self, detections_box_tuples): """Pre-processes boxes before adding them to the VRDDetectionEvaluation. In case of phrase evaluation task, evaluation expects exactly one bounding box containing all objects in the phrase. This bounding box is computed as an enclosing box of all groundtruth boxes of a phrase. Args: detections_box_tuples: A numpy array of structures with the shape [M, 1], each structure containing the same number of named bounding boxes. Each box is of the format [y_min, x_min, y_max, x_max]. See vrd_box_data_type for an example of this structure. Returns: result: A numpy array of structures with the shape [M, 1], each structure containing exactly one named bounding box. i-th output structure corresponds to the result of processing i-th input structure, where the named bounding box is computed as an enclosing bounding box of all bounding boxes of the i-th input structure. """ first_box_key = detections_box_tuples.dtype.fields.keys()[0] miny = detections_box_tuples[first_box_key][:, 0] minx = detections_box_tuples[first_box_key][:, 1] maxy = detections_box_tuples[first_box_key][:, 2] maxx = detections_box_tuples[first_box_key][:, 3] for fields in detections_box_tuples.dtype.fields: miny = np.minimum(detections_box_tuples[fields][:, 0], miny) minx = np.minimum(detections_box_tuples[fields][:, 1], minx) maxy = np.maximum(detections_box_tuples[fields][:, 2], maxy) maxx = np.maximum(detections_box_tuples[fields][:, 3], maxx) data_result = [] for i in range(detections_box_tuples.shape[0]): data_result.append(([miny[i], minx[i], maxy[i], maxx[i]],)) result = np.array(data_result, dtype=[('box', 'f4', (4,))]) return result VRDDetectionEvalMetrics = collections.namedtuple('VRDDetectionEvalMetrics', [ 'weighted_average_precision', 'mean_average_precision', 'average_precisions', 'precisions', 'recalls', 'recall_50', 'recall_100', 'median_rank_50', 'median_rank_100' ]) class _VRDDetectionEvaluation(object): """Performs metric computation for the VRD task. This class is internal. """ def __init__(self, matching_iou_threshold=0.5): """Constructor. Args: matching_iou_threshold: IOU threshold to use for matching groundtruth boxes to detection boxes. """ self._per_image_eval = per_image_vrd_evaluation.PerImageVRDEvaluation( matching_iou_threshold=matching_iou_threshold) self._groundtruth_box_tuples = {} self._groundtruth_class_tuples = {} self._num_gt_instances = 0 self._num_gt_imgs = 0 self._num_gt_instances_per_relationship = {} self.clear_detections() def clear_detections(self): """Clears detections.""" self._detection_keys = set() self._scores = [] self._relation_field_values = [] self._tp_fp_labels = [] self._average_precisions = {} self._precisions = [] self._recalls = [] def add_single_ground_truth_image_info( self, image_key, groundtruth_box_tuples, groundtruth_class_tuples): """Adds groundtruth for a single image to be used for evaluation. Args: image_key: A unique string/integer identifier for the image. groundtruth_box_tuples: A numpy array of structures with the shape [M, 1], representing M tuples, each tuple containing the same number of named bounding boxes. Each box is of the format [y_min, x_min, y_max, x_max]. groundtruth_class_tuples: A numpy array of structures shape [M, 1], representing the class labels of the corresponding bounding boxes and possibly additional classes. """ if image_key in self._groundtruth_box_tuples: logging.warn( 'image %s has already been added to the ground truth database.', image_key) return self._groundtruth_box_tuples[image_key] = groundtruth_box_tuples self._groundtruth_class_tuples[image_key] = groundtruth_class_tuples self._update_groundtruth_statistics(groundtruth_class_tuples) def add_single_detected_image_info(self, image_key, detected_box_tuples, detected_scores, detected_class_tuples): """Adds detections for a single image to be used for evaluation. Args: image_key: A unique string/integer identifier for the image. detected_box_tuples: A numpy array of structures with shape [N, 1], representing N tuples, each tuple containing the same number of named bounding boxes. Each box is of the format [y_min, x_min, y_max, x_max]. detected_scores: A float numpy array of shape [N, 1], representing the confidence scores of the detected N object instances. detected_class_tuples: A numpy array of structures shape [N, 1], representing the class labels of the corresponding bounding boxes and possibly additional classes. """ self._detection_keys.add(image_key) if image_key in self._groundtruth_box_tuples: groundtruth_box_tuples = self._groundtruth_box_tuples[image_key] groundtruth_class_tuples = self._groundtruth_class_tuples[image_key] else: groundtruth_box_tuples = np.empty( shape=[0, 4], dtype=detected_box_tuples.dtype) groundtruth_class_tuples = np.array([], dtype=detected_class_tuples.dtype) scores, tp_fp_labels, mapping = ( self._per_image_eval.compute_detection_tp_fp( detected_box_tuples=detected_box_tuples, detected_scores=detected_scores, detected_class_tuples=detected_class_tuples, groundtruth_box_tuples=groundtruth_box_tuples, groundtruth_class_tuples=groundtruth_class_tuples)) self._scores += [scores] self._tp_fp_labels += [tp_fp_labels] self._relation_field_values += [detected_class_tuples[mapping]['relation']] def _update_groundtruth_statistics(self, groundtruth_class_tuples): """Updates grouth truth statistics. Args: groundtruth_class_tuples: A numpy array of structures shape [M, 1], representing the class labels of the corresponding bounding boxes and possibly additional classes. """ self._num_gt_instances += groundtruth_class_tuples.shape[0] self._num_gt_imgs += 1 for relation_field_value in np.unique(groundtruth_class_tuples['relation']): if relation_field_value not in self._num_gt_instances_per_relationship: self._num_gt_instances_per_relationship[relation_field_value] = 0 self._num_gt_instances_per_relationship[relation_field_value] += np.sum( groundtruth_class_tuples['relation'] == relation_field_value) def evaluate(self): """Computes evaluation result. Returns: A named tuple with the following fields - average_precision: a float number corresponding to average precision. precisions: an array of precisions. recalls: an array of recalls. recall@50: recall computed on 50 top-scoring samples. recall@100: recall computed on 100 top-scoring samples. median_rank@50: median rank computed on 50 top-scoring samples. median_rank@100: median rank computed on 100 top-scoring samples. """ if self._num_gt_instances == 0: logging.warn('No ground truth instances') if not self._scores: scores = np.array([], dtype=float) tp_fp_labels = np.array([], dtype=bool) else: scores = np.concatenate(self._scores) tp_fp_labels = np.concatenate(self._tp_fp_labels) relation_field_values = np.concatenate(self._relation_field_values) for relation_field_value, _ in ( self._num_gt_instances_per_relationship.iteritems()): precisions, recalls = metrics.compute_precision_recall( scores[relation_field_values == relation_field_value], tp_fp_labels[relation_field_values == relation_field_value], self._num_gt_instances_per_relationship[relation_field_value]) self._average_precisions[ relation_field_value] = metrics.compute_average_precision( precisions, recalls) self._mean_average_precision = np.mean(self._average_precisions.values()) self._precisions, self._recalls = metrics.compute_precision_recall( scores, tp_fp_labels, self._num_gt_instances) self._weighted_average_precision = metrics.compute_average_precision( self._precisions, self._recalls) self._recall_50 = ( metrics.compute_recall_at_k(self._tp_fp_labels, self._num_gt_instances, 50)) self._median_rank_50 = ( metrics.compute_median_rank_at_k(self._tp_fp_labels, 50)) self._recall_100 = ( metrics.compute_recall_at_k(self._tp_fp_labels, self._num_gt_instances, 100)) self._median_rank_100 = ( metrics.compute_median_rank_at_k(self._tp_fp_labels, 100)) return VRDDetectionEvalMetrics( self._weighted_average_precision, self._mean_average_precision, self._average_precisions, self._precisions, self._recalls, self._recall_50, self._recall_100, self._median_rank_50, self._median_rank_100)