Repository: microsoft/torchgeo

Issue:
Bug: missing "not" operator to set ignore_zeros in segmentation trainer
I believe there's a bug in the translation from ignore_zeros to ignore_index in the [\_\_init__ of SemanticSegmentationTask](https://github.com/microsoft/torchgeo/blob/c933694ddc9a121af9415389949aefcc84760472/torchgeo/trainers/segmentation.py#L89).
`self.ignore_zeros = None if kwargs["ignore_zeros"] else 0` should be:
`self.ignore_zeros = None if kwargs["ignore_zeros"] is None else 0` as is done [when define loss function](https://github.com/microsoft/torchgeo/blob/c933694ddc9a121af9415389949aefcc84760472/torchgeo/trainers/segmentation.py#L57)

`self.ignore_zeros` is later used to define the "self.ignore_index" of the self.train_metrics. For sake of clarity, I'd suggest renaming it to  `self.ignore_index` directly in the \_\_init__. 

Therefore, this
```
...
        self.ignore_zeros = None if kwargs["ignore_zeros"] else 0

        self.config_task()

        self.train_metrics = MetricCollection(
            [
                Accuracy(
                    num_classes=self.hparams["num_classes"],
                    ignore_index=self.ignore_zeros,
...
```
would become
```
...
        self.ignore_index = None if kwargs["ignore_zeros"] is None else 0

        self.config_task()

        self.train_metrics = MetricCollection(
            [
                Accuracy(
                    num_classes=self.hparams["num_classes"],
                    ignore_index=self.ignore_index,
...
```
If you'd like, I can definitely push a PR for to fix this bug if the solution seems ok to all. Let me know.


Discussion:
@calebrob6 were you the one who wrote this logic? Can you take a look at this?
@remtav, the problem with defining this in __init__ is that nn.CrossEntropLoss uses -1000 as a "None" value (https://github.com/microsoft/torchgeo/blob/c933694ddc9a121af9415389949aefcc84760472/torchgeo/trainers/segmentation.py#L57). Some changes that I think make sense:
- Add an assert to make sure `kwargs["ignore_zeros"]` is boolean
- Throw an error if `kwargs["ignore_zeros"] == True` and `loss="jaccard"` (as the jaccard loss doesn't have an `ignore_index`)
- Say that it should be a boolean in the docstring
- Rename `self.ignore_zeros` to `self.ignore_index`
@calebrob6 can you submit a PR for this? I'll include it in 0.2.1 (releasing this week).
@adamjstewart aiming this for 0.3

Resolving patch:
diff --git a/torchgeo/trainers/segmentation.py b/torchgeo/trainers/segmentation.py
--- a/torchgeo/trainers/segmentation.py
+++ b/torchgeo/trainers/segmentation.py
@@ -3,6 +3,7 @@
 
 """Segmentation tasks."""
 
+import warnings
 from typing import Any, Dict, cast
 
 import segmentation_models_pytorch as smp
@@ -53,16 +54,15 @@ def config_task(self) -> None:
             )
 
         if self.hyperparams["loss"] == "ce":
-            self.loss = nn.CrossEntropyLoss(
-                ignore_index=-1000 if self.ignore_zeros is None else 0
-            )
+            ignore_value = -1000 if self.ignore_index is None else self.ignore_index
+            self.loss = nn.CrossEntropyLoss(ignore_index=ignore_value)
         elif self.hyperparams["loss"] == "jaccard":
             self.loss = smp.losses.JaccardLoss(
                 mode="multiclass", classes=self.hyperparams["num_classes"]
             )
         elif self.hyperparams["loss"] == "focal":
             self.loss = smp.losses.FocalLoss(
-                "multiclass", ignore_index=self.ignore_zeros, normalized=True
+                "multiclass", ignore_index=self.ignore_index, normalized=True
             )
         else:
             raise ValueError(f"Loss type '{self.hyperparams['loss']}' is not valid.")
@@ -78,7 +78,7 @@ def __init__(self, **kwargs: Any) -> None:
             in_channels: Number of channels in input image
             num_classes: Number of semantic classes to predict
             loss: Name of the loss function
-            ignore_zeros: Whether to ignore the "0" class value in the loss and metrics
+            ignore_index: Optional integer class index to ignore in the loss and metrics
 
         Raises:
             ValueError: if kwargs arguments are invalid
@@ -89,20 +89,26 @@ def __init__(self, **kwargs: Any) -> None:
         self.save_hyperparameters()  # type: ignore[operator]
         self.hyperparams = cast(Dict[str, Any], self.hparams)
 
-        self.ignore_zeros = None if kwargs["ignore_zeros"] else 0
-
+        if not isinstance(kwargs["ignore_index"], (int, type(None))):
+            raise ValueError("ignore_index must be an int or None")
+        if (kwargs["ignore_index"] is not None) and (kwargs["loss"] == "jaccard"):
+            warnings.warn(
+                "ignore_index has no effect on training when loss='jaccard'",
+                UserWarning,
+            )
+        self.ignore_index = kwargs["ignore_index"]
         self.config_task()
 
         self.train_metrics = MetricCollection(
             [
                 Accuracy(
                     num_classes=self.hyperparams["num_classes"],
-                    ignore_index=self.ignore_zeros,
+                    ignore_index=self.ignore_index,
                     mdmc_average="global",
                 ),
                 JaccardIndex(
                     num_classes=self.hyperparams["num_classes"],
-                    ignore_index=self.ignore_zeros,
+                    ignore_index=self.ignore_index,
                 ),
             ],
             prefix="train_",


Test patch:
diff --git a/tests/conf/chesapeake_cvpr_5.yaml b/tests/conf/chesapeake_cvpr_5.yaml
--- a/tests/conf/chesapeake_cvpr_5.yaml
+++ b/tests/conf/chesapeake_cvpr_5.yaml
@@ -11,7 +11,7 @@ experiment:
     in_channels: 4
     num_classes: 5
     num_filters: 1
-    ignore_zeros: False
+    ignore_index: null
     imagenet_pretraining: False
   datamodule:
     root_dir: "tests/data/chesapeake/cvpr"
diff --git a/tests/conf/chesapeake_cvpr_7.yaml b/tests/conf/chesapeake_cvpr_7.yaml
--- a/tests/conf/chesapeake_cvpr_7.yaml
+++ b/tests/conf/chesapeake_cvpr_7.yaml
@@ -11,7 +11,7 @@ experiment:
     in_channels: 4
     num_classes: 7
     num_filters: 1
-    ignore_zeros: False
+    ignore_index: null
     imagenet_pretraining: True
   datamodule:
     root_dir: "tests/data/chesapeake/cvpr"
diff --git a/tests/conf/chesapeake_cvpr_prior.yaml b/tests/conf/chesapeake_cvpr_prior.yaml
--- a/tests/conf/chesapeake_cvpr_prior.yaml
+++ b/tests/conf/chesapeake_cvpr_prior.yaml
@@ -11,7 +11,7 @@ experiment:
     in_channels: 4
     num_classes: 5
     num_filters: 1
-    ignore_zeros: False
+    ignore_index: null
     imagenet_pretraining: False
   datamodule:
     root_dir: "tests/data/chesapeake/cvpr"
diff --git a/tests/conf/etci2021.yaml b/tests/conf/etci2021.yaml
--- a/tests/conf/etci2021.yaml
+++ b/tests/conf/etci2021.yaml
@@ -9,7 +9,7 @@ experiment:
     learning_rate_schedule_patience: 6
     in_channels: 7
     num_classes: 2
-    ignore_zeros: True
+    ignore_index: 0
   datamodule:
     root_dir: "tests/data/etci2021"
     batch_size: 1
diff --git a/tests/conf/inria.yaml b/tests/conf/inria.yaml
--- a/tests/conf/inria.yaml
+++ b/tests/conf/inria.yaml
@@ -9,7 +9,7 @@ experiment:
     learning_rate_schedule_patience: 6
     in_channels: 3
     num_classes: 2
-    ignore_zeros: True  # class 0 not used for scoring
+    ignore_index: 0  # class 0 not used for scoring
   datamodule:
       root_dir: "tests/data/inria"
       batch_size: 1
diff --git a/tests/conf/landcoverai.yaml b/tests/conf/landcoverai.yaml
--- a/tests/conf/landcoverai.yaml
+++ b/tests/conf/landcoverai.yaml
@@ -11,7 +11,7 @@ experiment:
     in_channels: 3
     num_classes: 6
     num_filters: 1
-    ignore_zeros: False
+    ignore_index: null
   datamodule:
     root_dir: "tests/data/landcoverai"
     batch_size: 1
diff --git a/tests/conf/naipchesapeake.yaml b/tests/conf/naipchesapeake.yaml
--- a/tests/conf/naipchesapeake.yaml
+++ b/tests/conf/naipchesapeake.yaml
@@ -11,7 +11,7 @@ experiment:
     in_channels: 4
     num_classes: 14
     num_filters: 1
-    ignore_zeros: False
+    ignore_index: null
   datamodule:
     naip_root_dir: "tests/data/naip"
     chesapeake_root_dir: "tests/data/chesapeake/BAYWIDE"
diff --git a/tests/conf/oscd_all.yaml b/tests/conf/oscd_all.yaml
--- a/tests/conf/oscd_all.yaml
+++ b/tests/conf/oscd_all.yaml
@@ -11,7 +11,7 @@ experiment:
     in_channels: 26
     num_classes: 2
     num_filters: 1
-    ignore_zeros: True
+    ignore_index: null
   datamodule:
     root_dir: "tests/data/oscd"
     batch_size: 1
diff --git a/tests/conf/oscd_rgb.yaml b/tests/conf/oscd_rgb.yaml
--- a/tests/conf/oscd_rgb.yaml
+++ b/tests/conf/oscd_rgb.yaml
@@ -11,7 +11,7 @@ experiment:
     in_channels: 6
     num_classes: 2
     num_filters: 1
-    ignore_zeros: True
+    ignore_index: null
   datamodule:
     root_dir: "tests/data/oscd"
     batch_size: 1
diff --git a/tests/conf/sen12ms_all.yaml b/tests/conf/sen12ms_all.yaml
--- a/tests/conf/sen12ms_all.yaml
+++ b/tests/conf/sen12ms_all.yaml
@@ -10,7 +10,7 @@ experiment:
     learning_rate_schedule_patience: 2
     in_channels: 15
     num_classes: 11
-    ignore_zeros: False
+    ignore_index: null
   datamodule:
     root_dir: "tests/data/sen12ms"
     band_set: "all"
diff --git a/tests/conf/sen12ms_s1.yaml b/tests/conf/sen12ms_s1.yaml
--- a/tests/conf/sen12ms_s1.yaml
+++ b/tests/conf/sen12ms_s1.yaml
@@ -11,7 +11,7 @@ experiment:
     learning_rate_schedule_patience: 2
     in_channels: 2
     num_classes: 11
-    ignore_zeros: False
+    ignore_index: null
   datamodule:
     root_dir: "tests/data/sen12ms"
     band_set: "s1"
diff --git a/tests/conf/sen12ms_s2_all.yaml b/tests/conf/sen12ms_s2_all.yaml
--- a/tests/conf/sen12ms_s2_all.yaml
+++ b/tests/conf/sen12ms_s2_all.yaml
@@ -10,7 +10,7 @@ experiment:
     learning_rate_schedule_patience: 2
     in_channels: 13
     num_classes: 11
-    ignore_zeros: False
+    ignore_index: null
   datamodule:
     root_dir: "tests/data/sen12ms"
     band_set: "s2-all"
diff --git a/tests/conf/sen12ms_s2_reduced.yaml b/tests/conf/sen12ms_s2_reduced.yaml
--- a/tests/conf/sen12ms_s2_reduced.yaml
+++ b/tests/conf/sen12ms_s2_reduced.yaml
@@ -10,7 +10,7 @@ experiment:
     learning_rate_schedule_patience: 2
     in_channels: 6
     num_classes: 11
-    ignore_zeros: False
+    ignore_index: null
   datamodule:
     root_dir: "tests/data/sen12ms"
     band_set: "s2-reduced"
diff --git a/tests/trainers/test_segmentation.py b/tests/trainers/test_segmentation.py
--- a/tests/trainers/test_segmentation.py
+++ b/tests/trainers/test_segmentation.py
@@ -97,9 +97,9 @@ def model_kwargs(self) -> Dict[Any, Any]:
             "encoder_name": "resnet18",
             "encoder_weights": None,
             "in_channels": 1,
-            "num_classes": 1,
+            "num_classes": 2,
             "loss": "ce",
-            "ignore_zeros": True,
+            "ignore_index": 0,
         }
 
     def test_invalid_model(self, model_kwargs: Dict[Any, Any]) -> None:
@@ -113,3 +113,16 @@ def test_invalid_loss(self, model_kwargs: Dict[Any, Any]) -> None:
         match = "Loss type 'invalid_loss' is not valid."
         with pytest.raises(ValueError, match=match):
             SemanticSegmentationTask(**model_kwargs)
+
+    def test_invalid_ignoreindex(self, model_kwargs: Dict[Any, Any]) -> None:
+        model_kwargs["ignore_index"] = "0"
+        match = "ignore_index must be an int or None"
+        with pytest.raises(ValueError, match=match):
+            SemanticSegmentationTask(**model_kwargs)
+
+    def test_ignoreindex_with_jaccard(self, model_kwargs: Dict[Any, Any]) -> None:
+        model_kwargs["loss"] = "jaccard"
+        model_kwargs["ignore_index"] = 0
+        match = "ignore_index has no effect on training when loss='jaccard'"
+        with pytest.warns(UserWarning, match=match):
+            SemanticSegmentationTask(**model_kwargs)