Repository: Lightning-AI/torchmetrics

Issue:
MeanAveragePrecision incorrect calculations when predictions are empty
## 🐛 Bug

There seem to be some strange results when an image contains GTs but no predictions. This image is not considered when summarizing the results from all images. I've created a test to illustrate this

#### Code sample

```python
@pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed")
def test_one_correct_one_missing_prediction():
    """Test one correct prediction and one missing prediction"""
    metric = MeanAveragePrecision()

    preds = [
        dict(
            boxes=torch.Tensor([[214.1500, 41.2900, 562.4100, 285.0700]]),
            scores=torch.Tensor([0.5]),
            labels=torch.IntTensor([4]),
        ),
        dict(
            boxes=torch.Tensor([]),
            scores=torch.Tensor([]),
            labels=torch.IntTensor([]), # Empty prediction
        ),
    ]
    gts = [
        dict(boxes=torch.Tensor([[214.1500, 41.2900, 562.4100, 285.0700]]), labels=torch.IntTensor([4])),
        dict(boxes=torch.Tensor([[214.1500, 41.2900, 562.4100, 285.0700]]), labels=torch.IntTensor([4])), # Non-empty gt
    ]

    metric.update(preds, gts)
    result = metric.compute()
    # Since the second ground truth isn't found, the recall can not be 1. Thus mAP can not be 1
    assert result["map"] != 1
```

I believe that there is some error in `torchmetrics/detection/map.py MeanAveragePrecision._evaluate_image()`.

```python
# in _evaluate_image

if len(gt_label_mask) == 0 or len(det_label_mask) == 0:
    return None
gt = gt[gt_label_mask]
det = det[det_label_mask]
if len(gt) == 0 and len(det) == 0:
    return None
```

When `len(det_label_mask) == 0` the function returns None, which will disregard that image in downstream calculations. I’d argue that the model should be “punished” if it doesn’t find the GT.
Another thing is that I think `len(gt) == 0` should be replaced with `gt.numel() == 0` since empty `len(gt)` evaluates to 1 if the the gts are empty.

I've experienced this on the latest version of torchmetrics=0.7.0.


Discussion:
cc: @tkupek, @twsl 
@OlofHarrysson could you please clarify what TM version you are talking about? :rabbit: 
Hi @OlofHarrysson,
I just found the exact same bug, and started searching if someone else also found it.

I completely agree with you, the model should be punished when it doesn't find any object and there is some ground truth.

Did you manage to fix it? Is it enough to change
`if len(gt_label_mask) == 0 or len(det_label_mask) == 0:`
by
`if len(gt_label_mask) == 0 and len(det_label_mask) == 0:`
?

Thanks in advance
@mtailanian mind send a PR with also adding a test for this corner case? 
Hi @Borda.
Sure, I'm working in a possible solution. For sure only changing the OR by an AND is not enough. 
My attempt right now is returning a dict of tensors filled with False:

``` 
if len(gt_label_mask) == 0 or len(det_label_mask) == 0:
	# return None
	nb_iou_thrs = len(self.iou_thresholds)
	nb_gt = len(gt)
	nb_det = len(det)
	return {
		"dtMatches": torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device),
		"gtMatches": torch.zeros((nb_iou_thrs, nb_gt), dtype=torch.bool, device=self.device),
		"dtScores": torch.zeros(nb_det, dtype=torch.bool, device=self.device),
		"gtIgnore": torch.zeros(nb_gt, dtype=torch.bool, device=self.device),
		"dtIgnore": torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device),
	}
```
But I still need to test it... 

For testing the code of @OlofHarrysson works perfectly...

Thanks,

Resolving patch:
diff --git a/torchmetrics/detection/map.py b/torchmetrics/detection/map.py
--- a/torchmetrics/detection/map.py
+++ b/torchmetrics/detection/map.py
@@ -367,6 +367,60 @@ def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor:
         ious = box_iou(det, gt)
         return ious
 
+    def __evaluate_image_gt_no_preds(
+        self, gt: Tensor, gt_label_mask: Tensor, area_range: Tuple[int, int], nb_iou_thrs: int
+    ) -> Dict[str, Any]:
+        """Some GT but no predictions."""
+        # GTs
+        gt = gt[gt_label_mask]
+        nb_gt = len(gt)
+        areas = box_area(gt)
+        ignore_area = (areas < area_range[0]) | (areas > area_range[1])
+        gt_ignore, _ = torch.sort(ignore_area.to(torch.uint8))
+        gt_ignore = gt_ignore.to(torch.bool)
+
+        # Detections
+        nb_det = 0
+        det_ignore = torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device)
+
+        return {
+            "dtMatches": torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device),
+            "gtMatches": torch.zeros((nb_iou_thrs, nb_gt), dtype=torch.bool, device=self.device),
+            "dtScores": torch.zeros(nb_det, dtype=torch.bool, device=self.device),
+            "gtIgnore": gt_ignore,
+            "dtIgnore": det_ignore,
+        }
+
+    def __evaluate_image_preds_no_gt(
+        self, det: Tensor, idx: int, det_label_mask: Tensor, max_det: int, area_range: Tuple[int, int], nb_iou_thrs: int
+    ) -> Dict[str, Any]:
+        """Some predictions but no GT."""
+        # GTs
+        nb_gt = 0
+        gt_ignore = torch.zeros(nb_gt, dtype=torch.bool, device=self.device)
+
+        # Detections
+        det = det[det_label_mask]
+        scores = self.detection_scores[idx]
+        scores_filtered = scores[det_label_mask]
+        scores_sorted, dtind = torch.sort(scores_filtered, descending=True)
+        det = det[dtind]
+        if len(det) > max_det:
+            det = det[:max_det]
+        nb_det = len(det)
+        det_areas = box_area(det).to(self.device)
+        det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1])
+        ar = det_ignore_area.reshape((1, nb_det))
+        det_ignore = torch.repeat_interleave(ar, nb_iou_thrs, 0)
+
+        return {
+            "dtMatches": torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device),
+            "gtMatches": torch.zeros((nb_iou_thrs, nb_gt), dtype=torch.bool, device=self.device),
+            "dtScores": scores_sorted,
+            "gtIgnore": gt_ignore,
+            "dtIgnore": det_ignore,
+        }
+
     def _evaluate_image(
         self, idx: int, class_id: int, area_range: Tuple[int, int], max_det: int, ious: dict
     ) -> Optional[dict]:
@@ -388,11 +442,24 @@ def _evaluate_image(
         det = self.detection_boxes[idx]
         gt_label_mask = self.groundtruth_labels[idx] == class_id
         det_label_mask = self.detection_labels[idx] == class_id
-        if len(gt_label_mask) == 0 or len(det_label_mask) == 0:
+
+        # No Gt and No predictions --> ignore image
+        if len(gt_label_mask) == 0 and len(det_label_mask) == 0:
             return None
+
+        nb_iou_thrs = len(self.iou_thresholds)
+
+        # Some GT but no predictions
+        if len(gt_label_mask) > 0 and len(det_label_mask) == 0:
+            return self.__evaluate_image_gt_no_preds(gt, gt_label_mask, area_range, nb_iou_thrs)
+
+        # Some predictions but no GT
+        if len(gt_label_mask) == 0 and len(det_label_mask) >= 0:
+            return self.__evaluate_image_preds_no_gt(det, idx, det_label_mask, max_det, area_range, nb_iou_thrs)
+
         gt = gt[gt_label_mask]
         det = det[det_label_mask]
-        if len(gt) == 0 and len(det) == 0:
+        if gt.numel() == 0 and det.numel() == 0:
             return None
 
         areas = box_area(gt)
@@ -424,10 +491,11 @@ def _evaluate_image(
             for idx_iou, t in enumerate(self.iou_thresholds):
                 for idx_det, _ in enumerate(det):
                     m = MeanAveragePrecision._find_best_gt_match(t, gt_matches, idx_iou, gt_ignore, ious, idx_det)
-                    if m != -1:
-                        det_ignore[idx_iou, idx_det] = gt_ignore[m]
-                        det_matches[idx_iou, idx_det] = 1
-                        gt_matches[idx_iou, m] = 1
+                    if m == -1:
+                        continue
+                    det_ignore[idx_iou, idx_det] = gt_ignore[m]
+                    det_matches[idx_iou, idx_det] = 1
+                    gt_matches[idx_iou, m] = 1
 
         # set unmatched detections outside of area range to ignore
         det_areas = box_area(det)


Test patch:
diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py
--- a/tests/detection/test_map.py
+++ b/tests/detection/test_map.py
@@ -266,6 +266,50 @@ def test_empty_metric():
     metric.compute()
 
 
+@pytest.mark.skipif(_pytest_condition, reason="test requires that pycocotools and torchvision=>0.8.0 is installed")
+def test_missing_pred():
+    """One good detection, one false negative.
+
+    Map should be lower than 1. Actually it is 0.5, but the exact value depends on where we are sampling (i.e. recall's
+    values)
+    """
+    gts = [
+        dict(boxes=torch.Tensor([[10, 20, 15, 25]]), labels=torch.IntTensor([0])),
+        dict(boxes=torch.Tensor([[10, 20, 15, 25]]), labels=torch.IntTensor([0])),
+    ]
+    preds = [
+        dict(boxes=torch.Tensor([[10, 20, 15, 25]]), scores=torch.Tensor([0.9]), labels=torch.IntTensor([0])),
+        # Empty prediction
+        dict(boxes=torch.Tensor([]), scores=torch.Tensor([]), labels=torch.IntTensor([])),
+    ]
+    metric = MeanAveragePrecision()
+    metric.update(preds, gts)
+    result = metric.compute()
+    assert result["map"] < 1, "MAP cannot be 1, as there is a missing prediction."
+
+
+@pytest.mark.skipif(_pytest_condition, reason="test requires that pycocotools and torchvision=>0.8.0 is installed")
+def test_missing_gt():
+    """The symmetric case of test_missing_pred.
+
+    One good detection, one false positive. Map should be lower than 1. Actually it is 0.5, but the exact value depends
+    on where we are sampling (i.e. recall's values)
+    """
+    gts = [
+        dict(boxes=torch.Tensor([[10, 20, 15, 25]]), labels=torch.IntTensor([0])),
+        dict(boxes=torch.Tensor([]), labels=torch.IntTensor([])),
+    ]
+    preds = [
+        dict(boxes=torch.Tensor([[10, 20, 15, 25]]), scores=torch.Tensor([0.9]), labels=torch.IntTensor([0])),
+        dict(boxes=torch.Tensor([[10, 20, 15, 25]]), scores=torch.Tensor([0.95]), labels=torch.IntTensor([0])),
+    ]
+
+    metric = MeanAveragePrecision()
+    metric.update(preds, gts)
+    result = metric.compute()
+    assert result["map"] < 1, "MAP cannot be 1, as there is an image with no ground truth, but some predictions."
+
+
 @pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed")
 def test_error_on_wrong_input():
     """Test class input validation."""