Repository: huggingface/transformers
Pull request: Fix cardinality error for DETR models without explicit background class

For DETR-derived models that use sigmoid/focal loss (Deformable DETR, Grounding DINO, LW-DETR, RT-DETR, D-FINE), the cardinality error calculation was incorrect. These models don't have an explicit background class, so checking `argmax(-1) != shape[-1] - 1` is meaningless.

This PR fixes the cardinality calculation to use `sigmoid().max(-1).values > 0.5` to determine if a query confidently predicts an object.

The original DETR's `ImageLoss.loss_cardinality()` remains unchanged since it correctly uses softmax with an explicit background class.

Fixes #42199

Changed files:
filename: src/transformers/loss/loss_deformable_detr.py
patch: @@ -64,6 +64,22 @@ def __init__(self, matcher, num_classes, focal_alpha, losses):
         self.focal_alpha = focal_alpha
         self.losses = losses
 
+    @torch.no_grad()
+    def loss_cardinality(self, outputs, targets, indices, num_boxes):
+        """
+        Compute the cardinality error, i.e. the absolute error in the number of predicted non-empty boxes.
+
+        This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients.
+        """
+        logits = outputs["logits"]
+        device = logits.device
+        target_lengths = torch.as_tensor([len(v["class_labels"]) for v in targets], device=device)
+        # Count the number of predictions that are NOT "no-object" (sigmoid > 0.5 threshold)
+        card_pred = (logits.sigmoid().max(-1).values > 0.5).sum(1)
+        card_err = nn.functional.l1_loss(card_pred.float(), target_lengths.float())
+        losses = {"cardinality_error": card_err}
+        return losses
+
     # removed logging parameter, which was part of the original implementation
     def loss_labels(self, outputs, targets, indices, num_boxes):
         """

filename: src/transformers/loss/loss_grounding_dino.py
patch: @@ -146,6 +146,22 @@ def __init__(self, matcher, focal_alpha, losses):
         self.focal_alpha = focal_alpha
         self.losses = losses
 
+    @torch.no_grad()
+    def loss_cardinality(self, outputs, targets, indices, num_boxes):
+        """
+        Compute the cardinality error, i.e. the absolute error in the number of predicted non-empty boxes.
+
+        This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients.
+        """
+        logits = outputs["logits"]
+        device = logits.device
+        target_lengths = torch.as_tensor([len(v["class_labels"]) for v in targets], device=device)
+        # Count the number of predictions that are NOT "no-object" (sigmoid > 0.5 threshold)
+        card_pred = (logits.sigmoid().max(-1).values > 0.5).sum(1)
+        card_err = nn.functional.l1_loss(card_pred.float(), target_lengths.float())
+        losses = {"cardinality_error": card_err}
+        return losses
+
     def _get_target_classes_one_hot(self, outputs, targets, indices):
         """
         Create one_hot based on the matching indices

filename: src/transformers/loss/loss_lw_detr.py
patch: @@ -144,7 +144,6 @@ def loss_labels(self, outputs, targets, indices, num_boxes):
 
         return losses
 
-    # Copied from loss.loss_for_object_detection.ImageLoss.loss_cardinality
     @torch.no_grad()
     def loss_cardinality(self, outputs, targets, indices, num_boxes):
         """
@@ -155,8 +154,8 @@ def loss_cardinality(self, outputs, targets, indices, num_boxes):
         logits = outputs["logits"]
         device = logits.device
         target_lengths = torch.as_tensor([len(v["class_labels"]) for v in targets], device=device)
-        # Count the number of predictions that are NOT "no-object" (which is the last class)
-        card_pred = (logits.argmax(-1) != logits.shape[-1] - 1).sum(1)
+        # Count the number of predictions that are NOT "no-object" (sigmoid > 0.5 threshold)
+        card_pred = (logits.sigmoid().max(-1).values > 0.5).sum(1)
         card_err = nn.functional.l1_loss(card_pred.float(), target_lengths.float())
         losses = {"cardinality_error": card_err}
         return losses

filename: src/transformers/loss/loss_rt_detr.py
patch: @@ -222,8 +222,8 @@ def loss_cardinality(self, outputs, targets, indices, num_boxes):
         logits = outputs["logits"]
         device = logits.device
         target_lengths = torch.as_tensor([len(v["class_labels"]) for v in targets], device=device)
-        # Count the number of predictions that are NOT "no-object" (which is the last class)
-        card_pred = (logits.argmax(-1) != logits.shape[-1] - 1).sum(1)
+        # Count the number of predictions that are NOT "no-object" (sigmoid > 0.5 threshold)
+        card_pred = (logits.sigmoid().max(-1).values > 0.5).sum(1)
         card_err = nn.functional.l1_loss(card_pred.float(), target_lengths.float())
         losses = {"cardinality_error": card_err}
         return losses