Repository: lightly-ai/lightly

Issue:
Update scorers and documentation
### Goal

Our implementations of scorers match with commonly used ones. I would follow the conventions from the papers. There are also good descriptions [here](https://modal-python.readthedocs.io/en/latest/content/query_strategies/uncertainty_sampling.html) or [here](https://towardsdatascience.com/uncertainty-sampling-cheatsheet-ec57bc067c0b). and from a [academic active learning literature survey](http://burrsettles.com/pub/settles.activelearning.pdf)
It should be very transparent how the scores are computed and how the different scorers work.

Make sure we have the following for the existing samplers (uncertainty and margin):

- [ ] implementation matches "commonly used standards"
- [ ] we have references to the papers in the docs if possible
- [ ] we describe the algorithms in the docs


Discussion:

Resolving patch:
diff --git a/lightly/active_learning/scorers/classification.py b/lightly/active_learning/scorers/classification.py
--- a/lightly/active_learning/scorers/classification.py
+++ b/lightly/active_learning/scorers/classification.py
@@ -25,20 +25,48 @@ def _entropy(probs: np.ndarray, axis: int = 1) -> np.ndarray:
     entropies = -1 * np.sum(probs * log_probs, axis=axis)
     return entropies
 
+def _margin_largest_secondlargest(probs: np.ndarray) -> np.ndarray:
+    """Computes the margin of a probability matrix
+
+        Args:
+            probs:
+                A probability matrix of shape (N, M)
+
+        Exammple:
+            if probs.shape = (N, C) then margins.shape = (N, )
+
+        Returns:
+            The margin of the prediction vectors
+        """
+    sorted_probs = np.partition(probs, -2, axis=1)
+    margins = sorted_probs[:, -1] - sorted_probs[:, -2]
+    return margins
+
 
 class ScorerClassification(Scorer):
     """Class to compute active learning scores from the model_output of a classification task.
 
     Currently supports the following scorers:
 
-        `prediction-margin`:
-            This scorer uses the margin between 1.0 and the highest confidence
-            prediction. Use this scorer to select images where the model is
-            insecure.
+        The following three uncertainty scores are taken from
+        http://burrsettles.com/pub/settles.activelearning.pdf, Section 3.1, page 12f
+        and also explained in https://towardsdatascience.com/uncertainty-sampling-cheatsheet-ec57bc067c0b
+        They all have in common, that the score is highest if all classes have the
+        same confidence and are 0 if the model assigns 100% probability to a single class.
+        The differ in the number of class confidences they take into account.
 
-        `prediction-entropy`:
-            This scorer computes the entropy of the prediction. All
-            confidences are considered to compute the entropy of a sample.
+        `uncertainty_least_confidence`:
+            This score is 1 - the highest confidence prediction. It is high
+            when the confidence about the most probable class is low.
+
+        `uncertainty_margin`
+            This score is 1- the margin between the highest conficence
+            and second highest confidence prediction. It is high when the model
+            cannot decide between the two most probable classes.
+
+        `uncertainty_entropy`:
+            This scorer computes the entropy of the prediction. The confidences
+             for all classes are considered to compute the entropy of a sample.
 
     Attributes:
         model_output:
@@ -64,25 +92,73 @@ class ScorerClassification(Scorer):
         >>> scorer = ScorerClassification(predictions)
 
     """
-    def __init__(self, model_output: np.ndarray):
+    def __init__(self, model_output: Union[np.ndarray, List[List[float]]]):
+        if isinstance(model_output, List):
+            model_output = np.array(model_output)
         super(ScorerClassification, self).__init__(model_output)
 
-    def calculate_scores(self) -> Dict[str, np.ndarray]:
+    @classmethod
+    def score_names(cls) -> List[str]:
+        """Returns the names of the calculated active learning scores
+        """
+        score_names = list(cls(model_output=[[0.5, 0.5]]).calculate_scores().keys())
+        return score_names
+
+    def calculate_scores(self, normalize_to_0_1: bool = True) -> Dict[str, np.ndarray]:
         """Calculates and returns the active learning scores.
 
+        Args:
+            normalize_to_0_1:
+                If this is true, each score is normalized to have a
+                theoretical minimum of 0 and a theoretical maximum of 1.
+
         Returns:
             A dictionary mapping from the score name (as string)
             to the scores (as a single-dimensional numpy array).
         """
+        if len(self.model_output) == 0:
+            return {score_name: [] for score_name in self.score_names()}
+
+        scores_with_names = [
+            self._get_scores_uncertainty_least_confidence(),
+            self._get_scores_uncertainty_margin(),
+            self._get_scores_uncertainty_entropy()
+        ]
+
         scores = dict()
-        scores["prediction-margin"] = self._get_prediction_margin_score()
-        scores["prediction-entropy"] = self._get_prediction_entropy_score()
+        for score, score_name in scores_with_names:
+            score = np.nan_to_num(score)
+            scores[score_name] = score
+
+        if normalize_to_0_1:
+            scores = self.normalize_scores_0_1(scores)
+
+        return scores
+
+    def normalize_scores_0_1(self, scores: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
+        num_classes = self.model_output.shape[1]
+        model_output_very_sure = np.zeros(shape=(1,num_classes))
+        model_output_very_sure[0, 0] = 1
+        model_output_very_unsure = np.ones_like(model_output_very_sure)/num_classes
+
+        scores_minimum = ScorerClassification(model_output_very_sure).calculate_scores(normalize_to_0_1=False)
+        scores_maximum = ScorerClassification(model_output_very_unsure).calculate_scores(normalize_to_0_1=False)
+
+        for score_name in scores.keys():
+            interp_xp = [float(scores_minimum[score_name]), float(scores_maximum[score_name])]
+            interp_fp = [0, 1]
+            scores[score_name] = np.interp(scores[score_name], interp_xp, interp_fp)
+
         return scores
 
-    def _get_prediction_margin_score(self):
-        uncertainties = np.array([1 - max(class_probabilities) for class_probabilities in self.model_output])
-        return uncertainties
+    def _get_scores_uncertainty_least_confidence(self):
+        scores = 1 - np.max(self.model_output, axis=1)
+        return scores, "uncertainty_least_confidence"
+
+    def _get_scores_uncertainty_margin(self):
+        scores = 1 - _margin_largest_secondlargest(self.model_output)
+        return scores, "uncertainty_margin"
 
-    def _get_prediction_entropy_score(self):
-        uncertainties = _entropy(self.model_output, axis=1)
-        return uncertainties
+    def _get_scores_uncertainty_entropy(self):
+        scores = _entropy(self.model_output, axis=1)
+        return scores, "uncertainty_entropy"
diff --git a/lightly/active_learning/scorers/detection.py b/lightly/active_learning/scorers/detection.py
--- a/lightly/active_learning/scorers/detection.py
+++ b/lightly/active_learning/scorers/detection.py
@@ -20,7 +20,7 @@ def _object_frequency(model_output: List[ObjectDetectionOutput],
             object only as 0.25.
         min_score:
             The minimum score a single sample can have
-        
+
     Returns:
         Numpy array of length N with the computed scores
 
@@ -177,6 +177,14 @@ def _check_config(self):
         else:
             self.config = default_conf
 
+    @classmethod
+    def score_names(cls) -> List[str]:
+        """Returns the names of the calculated active learning scores
+        """
+        scorer = cls(model_output=[ObjectDetectionOutput([], [], [])])
+        score_names = list(scorer.calculate_scores().keys())
+        return score_names
+
     def calculate_scores(self) -> Dict[str, np.ndarray]:
         """Calculates and returns the active learning scores.
 
@@ -199,4 +207,3 @@ def _get_object_frequency(self):
     def _get_prediction_margin(self):
         scores = _prediction_margin(self.model_output)
         return scores
-
diff --git a/lightly/active_learning/scorers/scorer.py b/lightly/active_learning/scorers/scorer.py
--- a/lightly/active_learning/scorers/scorer.py
+++ b/lightly/active_learning/scorers/scorer.py
@@ -12,3 +12,9 @@ def calculate_scores(self) -> Dict[str, np.ndarray]:
         """Calculates and returns active learning scores in a dictionary.
         """
         raise NotImplementedError
+
+    @classmethod
+    def score_names(cls) -> List[str]:
+        """Returns the names of the calculated active learning scores
+        """
+        raise NotImplementedError


Test patch:
diff --git a/tests/active_learning/test_ScorerClassification.py b/tests/active_learning/test_ScorerClassification.py
--- a/tests/active_learning/test_ScorerClassification.py
+++ b/tests/active_learning/test_ScorerClassification.py
@@ -1,24 +1,46 @@
 import unittest
 import numpy as np
 
-from lightly.active_learning.scorers.classification import ScorerClassification
+from lightly.active_learning.scorers.classification import ScorerClassification, _entropy
 
 
 class TestScorerClassification(unittest.TestCase):
 
-    def test_score_calculation(self):
-        n_samples = 100
+    def test_score_calculation_random(self):
+        n_samples = 10000
         n_classes = 10
+        np.random.seed(42)
         predictions = np.random.rand(n_samples, n_classes)
         predictions_normalized = predictions / np.sum(predictions, axis=1)[:, np.newaxis]
         model_output = predictions_normalized
         scorer = ScorerClassification(model_output)
         scores = scorer.calculate_scores()
-        scores_prediction_entropy = scores["prediction-entropy"]
-        scores_prediction_margin = scores["prediction-margin"]
-
-        assert scores_prediction_entropy.shape == (n_samples, )
-        assert scores_prediction_margin.shape == (n_samples, )
-        assert all(scores_prediction_entropy >= 0)
-        assert all(scores_prediction_margin >= 0)
-        assert all(scores_prediction_margin <= 1)
+
+        self.assertEqual(set(scores.keys()), set(ScorerClassification.score_names()))
+
+        for score_name, score in scores.items():
+            self.assertEqual(score.shape, (n_samples,))
+            self.assertTrue(all(score >= 0))
+            self.assertTrue(all(score <= 1))
+
+    def test_score_calculation_specific(self):
+        model_output = [
+                    [0.7, 0.2, 0.1],
+                    [0.4, 0.5, 0.1]
+                ]
+        model_output = np.array(model_output)
+        scorer = ScorerClassification(model_output)
+        scores = scorer.calculate_scores()
+
+        self.assertListEqual(list(scores["uncertainty_least_confidence"]), [(1 - 0.7) / (1 - 1./3.), (1 - 0.5) / (1 - 1./3.)])
+        self.assertListEqual(list(scores["uncertainty_margin"]), [1 - (0.7 - 0.2), 1 - (0.5 - 0.4)])
+        for val1, val2 in zip(scores["uncertainty_entropy"], _entropy(model_output)/np.log2(3)):
+            self.assertAlmostEqual(val1, val2,places=8)
+
+
+    def test_scorer_classification_empty_model_output(self):
+        scorer = ScorerClassification(model_output=[])
+        scores = scorer.calculate_scores()
+        self.assertEqual(set(scores.keys()), set(ScorerClassification.score_names()))
+
+
diff --git a/tests/active_learning/test_ScorerObjectDetection.py b/tests/active_learning/test_ScorerObjectDetection.py
--- a/tests/active_learning/test_ScorerObjectDetection.py
+++ b/tests/active_learning/test_ScorerObjectDetection.py
@@ -71,6 +71,8 @@ def test_object_detection_scorer(self):
         scorer = ScorerObjectDetection(self.dummy_data)
         scores = scorer.calculate_scores()
 
+        self.assertTrue(set(scores.keys()), set(ScorerObjectDetection.score_names()))
+
         res = scores['object-frequency']
         self.assertEqual(len(res), len(self.dummy_data))
         self.assertListEqual(res.tolist(), [1.0, 0.95, 0.9])