Repository: lightly-ai/lightly

Issue:
Add classification scores to object detection scorer



Discussion:

Resolving patch:
diff --git a/lightly/active_learning/scorers/detection.py b/lightly/active_learning/scorers/detection.py
--- a/lightly/active_learning/scorers/detection.py
+++ b/lightly/active_learning/scorers/detection.py
@@ -2,6 +2,7 @@
 
 import numpy as np
 
+from lightly.active_learning.scorers import ScorerClassification
 from lightly.active_learning.scorers.scorer import Scorer
 from lightly.active_learning.utils.object_detection_output import ObjectDetectionOutput
 
@@ -44,7 +45,7 @@ def _object_frequency(model_output: List[ObjectDetectionOutput],
     return np.asarray(scores)
 
 
-def _prediction_margin(model_output: List[ObjectDetectionOutput]):
+def _objectness_least_confidence(model_output: List[ObjectDetectionOutput]) -> np.ndarray:
     """Score which prefers samples with low max(class prob) * objectness.
 
     Args:
@@ -65,24 +66,79 @@ def _prediction_margin(model_output: List[ObjectDetectionOutput]):
             # set the score to 0 if there was no bounding box detected
             score = 0.
         scores.append(score)
+
     return np.asarray(scores)
 
 
+def _reduce_classification_scores_over_boxes(
+        model_output: List[ObjectDetectionOutput],
+        reduce_fn_over_bounding_boxes: Callable[[np.ndarray], float] = np.max,
+        default_value_no_bounding_box: float = 0
+) -> Dict[str, List[float]]:
+    """Calculates classification scores over the mean of all found objects
+
+    Args:
+        model_output:
+            Predictions of the model of length N.
+        reduce_fn_over_bounding_boxes:
+            This function reduces the scores for each bounding box of an image
+            to one score per image.
+        default_value_no_bounding_box:
+            This is the default score if the image does not have any bounding boxes found.
+
+    Returns:
+        Numpy array of length N with the computed scores.
+
+    """
+    # calculate a score dictionary for each sample
+    scores_dict_list: List[dict[str, np.ndarray]] = []
+    for index_sample, output in enumerate(model_output):
+        probs = np.array(output.class_probabilities)
+        scores_dict_this_sample = ScorerClassification(model_output=probs).calculate_scores()
+        scores_dict_list.append(scores_dict_this_sample)
+
+    score_names = ScorerClassification.score_names()
+
+    # reduce it to one score per sample
+
+    # Initialize the dictionary
+    output_scores_dict = {score_name: [] for score_name in score_names}
+
+    # Fill the dictionary
+    for scores_dict in scores_dict_list:
+        for score_name in score_names:
+            score: np.ndarray = scores_dict[score_name]
+            if len(score) > 0:
+                scalar_score = float(reduce_fn_over_bounding_boxes(score))
+            else:
+                scalar_score = default_value_no_bounding_box
+            output_scores_dict[score_name].append(scalar_score)
+
+    return output_scores_dict
+
+
 class ScorerObjectDetection(Scorer):
     """Class to compute active learning scores from the model_output of an object detection task.
 
     Currently supports the following scorers:
 
-        `object-frequency`:
+        `object_frequency`:
             This scorer uses model predictions to focus more on images which
             have many objects in them. Use this scorer if you want scenes
             with lots of objects in them like we usually want in
             computer vision tasks such as perception in autonomous driving.
 
-        `prediction-margin`:
-            This scorer uses the margin between 1.0 and the highest confidence
-            prediction. Use this scorer to select images where the model is
-            insecure.
+        `objectness_least_confidence`:
+            This score is 1 - the mean of the highest confidence prediction. Use this scorer
+            to select images where the model is insecure about both whether it found an object
+            at all and the class of the object.
+
+        scores from ScorerClassification:
+            These scores are computed for each object detection out of
+            the class probability prediction for this detection.
+            Then these scores are reduced to one score per image
+            by taking the maximum. The scores are named as
+            f"classification_{score_name}".
 
     Attributes:
         model_output:
@@ -104,6 +160,7 @@ class only 50%. Lowering this value results in a more balanced
                 the sampler focus more on samples with many objects.
                 (default: 0.9)
 
+
     Examples:
         >>> # typical model output
         >>> predictions = [{
@@ -193,17 +250,27 @@ def calculate_scores(self) -> Dict[str, np.ndarray]:
             to the scores (as a single-dimensional numpy array).
         """
         scores = dict()
-        scores['object-frequency'] = self._get_object_frequency()
-        scores['prediction-margin'] = self._get_prediction_margin()
+        scores_with_names = [
+            self._get_object_frequency(),
+            self._get_objectness_least_confidence()
+        ]
+        for score, score_name in scores_with_names:
+            score = np.nan_to_num(score)
+            scores[score_name] = score
+
+        # add classification scores
+        scores_dict_classification = \
+            _reduce_classification_scores_over_boxes(model_output=self.model_output)
+        for score_name, score in scores_dict_classification.items():
+            scores[f"classification_{score_name}"] = score
+
         return scores
 
     def _get_object_frequency(self):
-        scores = _object_frequency(
+        return _object_frequency(
             self.model_output,
             self.config['frequency_penalty'],
-            self.config['min_score'])
-        return scores
+            self.config['min_score']), "object_frequency"
 
-    def _get_prediction_margin(self):
-        scores = _prediction_margin(self.model_output)
-        return scores
+    def _get_objectness_least_confidence(self):
+        return _objectness_least_confidence(self.model_output), "objectness_least_confidence"


Test patch:
diff --git a/tests/active_learning/test_ScorerObjectDetection.py b/tests/active_learning/test_ScorerObjectDetection.py
--- a/tests/active_learning/test_ScorerObjectDetection.py
+++ b/tests/active_learning/test_ScorerObjectDetection.py
@@ -1,5 +1,8 @@
 import unittest
 
+import numpy as np
+
+from lightly.active_learning.scorers.classification import _entropy
 from lightly.active_learning.utils.bounding_box import BoundingBox
 from lightly.active_learning.utils.object_detection_output import ObjectDetectionOutput
 from lightly.active_learning.scorers.detection import ScorerObjectDetection
@@ -73,14 +76,28 @@ def test_object_detection_scorer(self):
 
         self.assertTrue(set(scores.keys()), set(ScorerObjectDetection.score_names()))
 
-        res = scores['object-frequency']
+        res = scores['object_frequency']
         self.assertEqual(len(res), len(self.dummy_data))
         self.assertListEqual(res.tolist(), [1.0, 0.95, 0.9])
 
-        res = scores['prediction-margin']
+        res = scores['objectness_least_confidence']
         self.assertEqual(len(res), len(self.dummy_data))
         self.assertListEqual(res.tolist(), [0.5514945, 0.9488, 0.])
 
+        for score_name, score in scores.items():
+            if "classification" in score_name:
+                self.assertEqual(len(res), len(self.dummy_data))
+            if score_name == "classification_uncertainty_least_confidence":
+                self.assertListEqual(list(score), [max(1 - 0.7, 1 - 0.5)/(1 - 1/3), (1 - 0.5)/(1 - 1/3), 0])
+            elif score_name == "classification_uncertainty_margin":
+                self.assertListEqual(list(score), [max(1 - (0.7 - 0.2), 1 - (0.5 - 0.4)), 1 - (0.5 - 0.41), 0])
+            elif score_name == "classification_uncertainty_entropy":
+                entropies_0 = _entropy(np.array(self.dummy_data[0].class_probabilities))/np.log2(3)
+                entropies_1 = _entropy(np.array(self.dummy_data[1].class_probabilities))/np.log2(3)
+                score_target = [float(max(entropies_0)), float(max(entropies_1)), 0]
+                for val1, val2 in zip(score, score_target):
+                    self.assertAlmostEqual(val1, val2, places=8)
+
 
     def test_object_detection_scorer_config(self):