Repository: microsoft/torchgeo

Issue:
Inria dataset validation set
https://project.inria.fr/aerialimagelabeling/contest/ says, "For validation, we suggest to remove the first five images of every location (e.g., austin{1-5}.tif, chicago{1-5}.tif) from the training set."

It would be nice to implement this in our Dataset, then have the option to ignore it or not in the DataModule (i.e. when you want to train with as many labels as possible).


Discussion:

Resolving patch:
diff --git a/torchgeo/datamodules/inria.py b/torchgeo/datamodules/inria.py
--- a/torchgeo/datamodules/inria.py
+++ b/torchgeo/datamodules/inria.py
@@ -12,7 +12,6 @@
 from ..transforms import AugmentationSequential
 from ..transforms.transforms import _RandomNCrop
 from .geo import NonGeoDataModule
-from .utils import dataset_split
 
 
 class InriaAerialImageLabelingDataModule(NonGeoDataModule):
@@ -29,8 +28,6 @@ def __init__(
         batch_size: int = 64,
         patch_size: Union[tuple[int, int], int] = 64,
         num_workers: int = 0,
-        val_split_pct: float = 0.1,
-        test_split_pct: float = 0.1,
         **kwargs: Any,
     ) -> None:
         """Initialize a new InriaAerialImageLabelingDataModule instance.
@@ -40,16 +37,12 @@ def __init__(
             patch_size: Size of each patch, either ``size`` or ``(height, width)``.
                 Should be a multiple of 32 for most segmentation architectures.
             num_workers: Number of workers for parallel data loading.
-            val_split_pct: Percentage of the dataset to use as a validation set.
-            test_split_pct: Percentage of the dataset to use as a test set.
             **kwargs: Additional keyword arguments passed to
                 :class:`~torchgeo.datasets.InriaAerialImageLabeling`.
         """
         super().__init__(InriaAerialImageLabeling, 1, num_workers, **kwargs)
 
         self.patch_size = _to_tuple(patch_size)
-        self.val_split_pct = val_split_pct
-        self.test_split_pct = test_split_pct
 
         self.train_aug = AugmentationSequential(
             K.Normalize(mean=self.mean, std=self.std),
@@ -75,11 +68,10 @@ def setup(self, stage: str) -> None:
         Args:
             stage: Either 'fit', 'validate', 'test', or 'predict'.
         """
-        if stage in ["fit", "validate", "test"]:
-            self.dataset = InriaAerialImageLabeling(split="train", **self.kwargs)
-            self.train_dataset, self.val_dataset, self.test_dataset = dataset_split(
-                self.dataset, self.val_split_pct, self.test_split_pct
-            )
+        if stage in ["fit"]:
+            self.train_dataset = InriaAerialImageLabeling(split="train", **self.kwargs)
+        if stage in ["fit", "validate"]:
+            self.val_dataset = InriaAerialImageLabeling(split="val", **self.kwargs)
         if stage in ["predict"]:
             # Test set masks are not public, use for prediction instead
             self.predict_dataset = InriaAerialImageLabeling(split="test", **self.kwargs)
diff --git a/torchgeo/datasets/inria.py b/torchgeo/datasets/inria.py
--- a/torchgeo/datasets/inria.py
+++ b/torchgeo/datasets/inria.py
@@ -5,6 +5,7 @@
 
 import glob
 import os
+import re
 from typing import Any, Callable, Optional
 
 import matplotlib.pyplot as plt
@@ -45,6 +46,9 @@ class InriaAerialImageLabeling(NonGeoDataset):
     * https://doi.org/10.1109/IGARSS.2017.8127684
 
     .. versionadded:: 0.3
+
+    .. versionchanged:: 0.5
+       Added support for a *val* split.
     """
 
     directory = "AerialImageDataset"
@@ -62,7 +66,7 @@ def __init__(
 
         Args:
             root: root directory where dataset can be found
-            split: train/test split
+            split: train/val/test split
             transforms: a function/transform that takes input sample and its target as
                 entry and returns a transformed version.
             checksum: if True, check the MD5 of the downloaded files (may be slow)
@@ -72,7 +76,7 @@ def __init__(
             RuntimeError: if dataset is missing
         """
         self.root = root
-        assert split in {"train", "test"}
+        assert split in {"train", "val", "test"}
         self.split = split
         self.transforms = transforms
         self.checksum = checksum
@@ -90,15 +94,25 @@ def _load_files(self, root: str) -> list[dict[str, str]]:
             list of dicts containing paths for each pair of image and label
         """
         files = []
-        root_dir = os.path.join(root, self.directory, self.split)
+        split = "train" if self.split in ["train", "val"] else "test"
+        root_dir = os.path.join(root, self.directory, split)
+        pattern = re.compile(r"([A-Za-z]+)(\d+)")
+
         images = glob.glob(os.path.join(root_dir, "images", "*.tif"))
         images = sorted(images)
-        if self.split == "train":
+
+        if split == "train":
             labels = glob.glob(os.path.join(root_dir, "gt", "*.tif"))
             labels = sorted(labels)
 
             for img, lbl in zip(images, labels):
-                files.append({"image": img, "label": lbl})
+                if match := pattern.search(img):
+                    idx = int(match.group(2))
+                    # For validation, use the first 5 images of every location
+                    if self.split == "train" and idx > 5:
+                        files.append({"image": img, "label": lbl})
+                    elif self.split == "val" and idx < 6:
+                        files.append({"image": img, "label": lbl})
         else:
             for img in images:
                 files.append({"image": img})


Test patch:
diff --git a/tests/conf/inria.yaml b/tests/conf/inria.yaml
--- a/tests/conf/inria.yaml
+++ b/tests/conf/inria.yaml
@@ -12,7 +12,5 @@ data:
   init_args:
     batch_size: 1
     patch_size: 2
-    val_split_pct: 0.2
-    test_split_pct: 0.2
   dict_kwargs:
     root: "tests/data/inria"
diff --git a/tests/conf/inria_deeplab.yaml b/tests/conf/inria_deeplab.yaml
--- a/tests/conf/inria_deeplab.yaml
+++ b/tests/conf/inria_deeplab.yaml
@@ -10,7 +10,5 @@ data:
   init_args:
     batch_size: 1
     patch_size: 2
-    val_split_pct: 0.2
-    test_split_pct: 0.2
   dict_kwargs:
     root: "tests/data/inria"
diff --git a/tests/conf/inria_fcn.yaml b/tests/conf/inria_fcn.yaml
--- a/tests/conf/inria_fcn.yaml
+++ b/tests/conf/inria_fcn.yaml
@@ -10,7 +10,5 @@ data:
   init_args:
     batch_size: 1
     patch_size: 2
-    val_split_pct: 0.2
-    test_split_pct: 0.2
   dict_kwargs:
     root: "tests/data/inria"
diff --git a/tests/conf/inria_unet.yaml b/tests/conf/inria_unet.yaml
--- a/tests/conf/inria_unet.yaml
+++ b/tests/conf/inria_unet.yaml
@@ -10,7 +10,5 @@ data:
   init_args:
     batch_size: 1
     patch_size: 2
-    val_split_pct: 0.2
-    test_split_pct: 0.2
   dict_kwargs:
     root: "tests/data/inria"
diff --git a/tests/data/inria/AerialImageDataset/test/images/austin10.tif b/tests/data/inria/AerialImageDataset/test/images/austin10.tif
Binary files a/tests/data/inria/AerialImageDataset/test/images/austin10.tif and b/tests/data/inria/AerialImageDataset/test/images/austin10.tif differ
diff --git a/tests/data/inria/AerialImageDataset/test/images/austin11.tif b/tests/data/inria/AerialImageDataset/test/images/austin11.tif
Binary files a/tests/data/inria/AerialImageDataset/test/images/austin11.tif and b/tests/data/inria/AerialImageDataset/test/images/austin11.tif differ
diff --git a/tests/data/inria/AerialImageDataset/test/images/austin12.tif b/tests/data/inria/AerialImageDataset/test/images/austin12.tif
Binary files a/tests/data/inria/AerialImageDataset/test/images/austin12.tif and b/tests/data/inria/AerialImageDataset/test/images/austin12.tif differ
diff --git a/tests/data/inria/AerialImageDataset/test/images/austin13.tif b/tests/data/inria/AerialImageDataset/test/images/austin13.tif
Binary files a/tests/data/inria/AerialImageDataset/test/images/austin13.tif and b/tests/data/inria/AerialImageDataset/test/images/austin13.tif differ
diff --git a/tests/data/inria/AerialImageDataset/test/images/austin14.tif b/tests/data/inria/AerialImageDataset/test/images/austin14.tif
Binary files a/tests/data/inria/AerialImageDataset/test/images/austin14.tif and b/tests/data/inria/AerialImageDataset/test/images/austin14.tif differ
diff --git a/tests/data/inria/AerialImageDataset/test/images/austin15.tif b/tests/data/inria/AerialImageDataset/test/images/austin15.tif
new file mode 100644
Binary files /dev/null and b/tests/data/inria/AerialImageDataset/test/images/austin15.tif differ
diff --git a/tests/data/inria/AerialImageDataset/test/images/austin16.tif b/tests/data/inria/AerialImageDataset/test/images/austin16.tif
new file mode 100644
Binary files /dev/null and b/tests/data/inria/AerialImageDataset/test/images/austin16.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/gt/austin1.tif b/tests/data/inria/AerialImageDataset/train/gt/austin1.tif
Binary files a/tests/data/inria/AerialImageDataset/train/gt/austin1.tif and b/tests/data/inria/AerialImageDataset/train/gt/austin1.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/gt/austin2.tif b/tests/data/inria/AerialImageDataset/train/gt/austin2.tif
Binary files a/tests/data/inria/AerialImageDataset/train/gt/austin2.tif and b/tests/data/inria/AerialImageDataset/train/gt/austin2.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/gt/austin3.tif b/tests/data/inria/AerialImageDataset/train/gt/austin3.tif
Binary files a/tests/data/inria/AerialImageDataset/train/gt/austin3.tif and b/tests/data/inria/AerialImageDataset/train/gt/austin3.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/gt/austin4.tif b/tests/data/inria/AerialImageDataset/train/gt/austin4.tif
Binary files a/tests/data/inria/AerialImageDataset/train/gt/austin4.tif and b/tests/data/inria/AerialImageDataset/train/gt/austin4.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/gt/austin5.tif b/tests/data/inria/AerialImageDataset/train/gt/austin5.tif
Binary files a/tests/data/inria/AerialImageDataset/train/gt/austin5.tif and b/tests/data/inria/AerialImageDataset/train/gt/austin5.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/gt/austin6.tif b/tests/data/inria/AerialImageDataset/train/gt/austin6.tif
new file mode 100644
Binary files /dev/null and b/tests/data/inria/AerialImageDataset/train/gt/austin6.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/gt/austin7.tif b/tests/data/inria/AerialImageDataset/train/gt/austin7.tif
new file mode 100644
Binary files /dev/null and b/tests/data/inria/AerialImageDataset/train/gt/austin7.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/images/austin1.tif b/tests/data/inria/AerialImageDataset/train/images/austin1.tif
Binary files a/tests/data/inria/AerialImageDataset/train/images/austin1.tif and b/tests/data/inria/AerialImageDataset/train/images/austin1.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/images/austin2.tif b/tests/data/inria/AerialImageDataset/train/images/austin2.tif
Binary files a/tests/data/inria/AerialImageDataset/train/images/austin2.tif and b/tests/data/inria/AerialImageDataset/train/images/austin2.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/images/austin3.tif b/tests/data/inria/AerialImageDataset/train/images/austin3.tif
Binary files a/tests/data/inria/AerialImageDataset/train/images/austin3.tif and b/tests/data/inria/AerialImageDataset/train/images/austin3.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/images/austin4.tif b/tests/data/inria/AerialImageDataset/train/images/austin4.tif
Binary files a/tests/data/inria/AerialImageDataset/train/images/austin4.tif and b/tests/data/inria/AerialImageDataset/train/images/austin4.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/images/austin5.tif b/tests/data/inria/AerialImageDataset/train/images/austin5.tif
Binary files a/tests/data/inria/AerialImageDataset/train/images/austin5.tif and b/tests/data/inria/AerialImageDataset/train/images/austin5.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/images/austin6.tif b/tests/data/inria/AerialImageDataset/train/images/austin6.tif
new file mode 100644
Binary files /dev/null and b/tests/data/inria/AerialImageDataset/train/images/austin6.tif differ
diff --git a/tests/data/inria/AerialImageDataset/train/images/austin7.tif b/tests/data/inria/AerialImageDataset/train/images/austin7.tif
new file mode 100644
Binary files /dev/null and b/tests/data/inria/AerialImageDataset/train/images/austin7.tif differ
diff --git a/tests/data/inria/NEW2-AerialImageDataset.zip b/tests/data/inria/NEW2-AerialImageDataset.zip
Binary files a/tests/data/inria/NEW2-AerialImageDataset.zip and b/tests/data/inria/NEW2-AerialImageDataset.zip differ
diff --git a/tests/data/inria/data.py b/tests/data/inria/data.py
--- a/tests/data/inria/data.py
+++ b/tests/data/inria/data.py
@@ -81,10 +81,9 @@ def generate_test_data(root: str, n_samples: int = 2) -> str:
     shutil.make_archive(
         archive_path, "zip", root_dir=root, base_dir="AerialImageDataset"
     )
-    shutil.rmtree(folder_path)
     return calculate_md5(f"{archive_path}.zip")
 
 
 if __name__ == "__main__":
-    md5_hash = generate_test_data(os.getcwd(), 5)
+    md5_hash = generate_test_data(os.getcwd(), 7)
     print(md5_hash)
diff --git a/tests/datasets/test_inria.py b/tests/datasets/test_inria.py
--- a/tests/datasets/test_inria.py
+++ b/tests/datasets/test_inria.py
@@ -15,12 +15,12 @@
 
 
 class TestInriaAerialImageLabeling:
-    @pytest.fixture(params=["train", "test"])
+    @pytest.fixture(params=["train", "val", "test"])
     def dataset(
         self, request: SubRequest, monkeypatch: MonkeyPatch
     ) -> InriaAerialImageLabeling:
         root = os.path.join("tests", "data", "inria")
-        test_md5 = "478688944e4797c097d9387fd0b3f038"
+        test_md5 = "3ecbe95eb84aea064e455c4321546be1"
         monkeypatch.setattr(InriaAerialImageLabeling, "md5", test_md5)
         transforms = nn.Identity()
         return InriaAerialImageLabeling(
@@ -38,7 +38,12 @@ def test_getitem(self, dataset: InriaAerialImageLabeling) -> None:
         assert x["image"].ndim == 3
 
     def test_len(self, dataset: InriaAerialImageLabeling) -> None:
-        assert len(dataset) == 5
+        if dataset.split == "train":
+            assert len(dataset) == 2
+        elif dataset.split == "val":
+            assert len(dataset) == 5
+        elif dataset.split == "test":
+            assert len(dataset) == 7
 
     def test_already_downloaded(self, dataset: InriaAerialImageLabeling) -> None:
         InriaAerialImageLabeling(root=dataset.root)