Repository: microsoft/torchgeo

Issue:
Microsoft Canadian Building Footprints Dataset
https://github.com/Microsoft/CanadianBuildingFootprints

This dataset contains 11,842,186 computer generated building footprints in all Canadian provinces and territories. This data is freely available for download and use.


Discussion:
A problem with using this dataset (and the similar USA building footprints) is that we don't know which imagery they used.

Resolving patch:
diff --git a/torchgeo/datasets/__init__.py b/torchgeo/datasets/__init__.py
--- a/torchgeo/datasets/__init__.py
+++ b/torchgeo/datasets/__init__.py
@@ -1,6 +1,7 @@
 """TorchGeo datasets."""
 
 from .benin_cashews import BeninSmallHolderCashews
+from .cbf import CanadianBuildingFootprints
 from .cdl import CDL
 from .chesapeake import (
     Chesapeake,
@@ -42,6 +43,7 @@
 __all__ = (
     "BeninSmallHolderCashews",
     "BoundingBox",
+    "CanadianBuildingFootprints",
     "CDL",
     "collate_dict",
     "Chesapeake",
diff --git a/torchgeo/datasets/cbf.py b/torchgeo/datasets/cbf.py
new file mode 100644
--- /dev/null
+++ b/torchgeo/datasets/cbf.py
@@ -0,0 +1,216 @@
+"""Canadian Building Footprints dataset."""
+
+import glob
+import os
+import sys
+from typing import Any, Callable, Dict, Optional
+
+import fiona
+import fiona.transform
+import matplotlib.pyplot as plt
+import rasterio
+import torch
+from rasterio.crs import CRS
+from rtree.index import Index, Property
+from torch import Tensor
+
+from .geo import GeoDataset
+from .utils import BoundingBox, check_integrity, download_and_extract_archive
+
+_crs = CRS.from_epsg(4326)
+
+
+class CanadianBuildingFootprints(GeoDataset):
+    """Canadian Building Footprints dataset.
+
+    The `Canadian Building Footprints
+    <https://github.com/Microsoft/CanadianBuildingFootprints>`_ dataset contains
+    11,842,186 computer generated building footprints in all Canadian provinces and
+    territories in GeoJSON format. This data is freely available for download and use.
+    """
+
+    # TODO: how does one cite this dataset?
+    # https://github.com/microsoft/CanadianBuildingFootprints/issues/11
+
+    url = "https://usbuildingdata.blob.core.windows.net/canadian-buildings-v2/"
+    provinces_territories = [
+        "Alberta",
+        "BritishColumbia",
+        "Manitoba",
+        "NewBrunswick",
+        "NewfoundlandAndLabrador",
+        "NorthwestTerritories",
+        "NovaScotia",
+        "Nunavut",
+        "Ontario",
+        "PrinceEdwardIsland",
+        "Quebec",
+        "Saskatchewan",
+        "YukonTerritory",
+    ]
+    md5s = [
+        "8b4190424e57bb0902bd8ecb95a9235b",
+        "fea05d6eb0006710729c675de63db839",
+        "adf11187362624d68f9c69aaa693c46f",
+        "44269d4ec89521735389ef9752ee8642",
+        "65dd92b1f3f5f7222ae5edfad616d266",
+        "346d70a682b95b451b81b47f660fd0e2",
+        "bd57cb1a7822d72610215fca20a12602",
+        "c1f29b73cdff9a6a9dd7d086b31ef2cf",
+        "76ba4b7059c5717989ce34977cad42b2",
+        "2e4a3fa47b3558503e61572c59ac5963",
+        "9ff4417ae00354d39a0cf193c8df592c",
+        "a51078d8e60082c7d3a3818240da6dd5",
+        "c11f3bd914ecabd7cac2cb2871ec0261",
+    ]
+
+    def __init__(
+        self,
+        root: str = "data",
+        crs: CRS = _crs,
+        res: float = 0.00001,
+        transforms: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None,
+        download: bool = False,
+        checksum: bool = False,
+    ) -> None:
+        """Initialize a new Canadian Building Footprints dataset.
+
+        Args:
+            root: root directory where dataset can be found
+            crs: :term:`coordinate reference system (CRS)` to project to
+            res: resolution to use when rasterizing features
+            transforms: a function/transform that takes input sample and its target as
+                entry and returns a transformed version
+            download: if True, download dataset and store it in the root directory
+            checksum: if True, check the MD5 of the downloaded files (may be slow)
+
+        Raises:
+            RuntimeError: if ``download=False`` and data is not found, or
+                ``checksum=True`` and checksums don't match
+        """
+        self.root = root
+        self.crs = crs
+        self.res = res
+        self.transforms = transforms
+        self.checksum = checksum
+
+        if download:
+            self._download()
+
+        if not self._check_integrity():
+            raise RuntimeError(
+                "Dataset not found or corrupted. "
+                + "You can use download=True to download it"
+            )
+
+        # Create an R-tree to index the dataset
+        self.index = Index(interleaved=False, properties=Property(dimension=3))
+        fileglob = os.path.join(root, "**.geojson")
+        for i, filename in enumerate(glob.iglob(fileglob, recursive=True)):
+            with fiona.open(filename) as src:
+                minx, miny, maxx, maxy = src.bounds
+                (minx, maxx), (miny, maxy) = fiona.transform.transform(
+                    src.crs, crs.to_dict(), [minx, maxx], [miny, maxy]
+                )
+            mint = 0
+            maxt = sys.maxsize
+            coords = (minx, maxx, miny, maxy, mint, maxt)
+            self.index.insert(i, coords, filename)
+
+    def __getitem__(self, query: BoundingBox) -> Dict[str, Any]:
+        """Retrieve image and metadata indexed by query.
+
+        Args:
+            query: (minx, maxx, miny, maxy, mint, maxt) coordinates to index
+
+        Returns:
+            sample of labels and metadata at that index
+
+        Raises:
+            IndexError: if query is not within bounds of the index
+        """
+        if not query.intersects(self.bounds):
+            raise IndexError(
+                f"query: {query} is not within bounds of the index: {self.bounds}"
+            )
+
+        hits = self.index.intersection(query, objects=True)
+        filename = next(hits).object  # TODO: this assumes there is only a single hit
+        shapes = []
+        with fiona.open(filename) as src:
+            # We need to know the bounding box of the query in the source CRS
+            (minx, maxx), (miny, maxy) = fiona.transform.transform(
+                self.crs.to_dict(),
+                src.crs,
+                [query.minx, query.maxx],
+                [query.miny, query.maxy],
+            )
+
+            # Filter geometries to those that intersect with the bounding box
+            for feature in src.filter(bbox=(minx, miny, maxx, maxy)):
+                # Warp geometries to requested CRS
+                shape = fiona.transform.transform_geom(
+                    src.crs, self.crs.to_dict(), feature["geometry"]
+                )
+                shapes.append(shape)
+
+        # Rasterize geometries
+        width = (query.maxx - query.minx) / self.res
+        height = (query.maxy - query.miny) / self.res
+        transform = rasterio.transform.from_bounds(
+            query.minx, query.miny, query.maxx, query.maxy, width, height
+        )
+        masks = rasterio.features.rasterize(
+            shapes, out_shape=(int(height), int(width)), transform=transform
+        )
+
+        sample = {
+            "masks": torch.tensor(masks),  # type: ignore[attr-defined]
+            "crs": self.crs,
+            "bbox": query,
+        }
+
+        if self.transforms is not None:
+            sample = self.transforms(sample)
+
+        return sample
+
+    def _check_integrity(self) -> bool:
+        """Check integrity of dataset.
+
+        Returns:
+            True if dataset files are found and/or MD5s match, else False
+        """
+        for prov_terr, md5 in zip(self.provinces_territories, self.md5s):
+            filepath = os.path.join(self.root, prov_terr + ".zip")
+            if not check_integrity(filepath, md5 if self.checksum else None):
+                return False
+        return True
+
+    def _download(self) -> None:
+        """Download the dataset and extract it."""
+        if self._check_integrity():
+            print("Files already downloaded and verified")
+            return
+
+        for prov_terr, md5 in zip(self.provinces_territories, self.md5s):
+            download_and_extract_archive(
+                self.url + prov_terr + ".zip",
+                self.root,
+                md5=md5 if self.checksum else None,
+            )
+
+    def plot(self, image: Tensor) -> None:
+        """Plot an image on a map.
+
+        Args:
+            image: the image to plot
+        """
+        array = image.squeeze().numpy()
+
+        # Plot the image
+        ax = plt.axes()
+        ax.imshow(array)
+        ax.axis("off")
+        plt.show()
+        plt.close()


Test patch:
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -62,7 +62,7 @@ jobs:
       if: ${{ runner.os == 'Windows' }}
     - name: Install conda dependencies (Windows)
       run: |
-        conda install h5py 'rasterio>=1.0'
+        conda install fiona h5py 'rasterio>=1.0'
         conda list
         conda info
       if: ${{ runner.os == 'Windows' }}
diff --git a/tests/data/README.md b/tests/data/README.md
--- a/tests/data/README.md
+++ b/tests/data/README.md
@@ -1,9 +1,11 @@
-This directory contains fake data used to test torchgeo. Depending on the type of dataset, fake data can be created in one of two ways:
+This directory contains fake data used to test torchgeo. Depending on the type of dataset, fake data can be created in multiple ways:
 
 ## GeoDataset
 
 GeoDataset data can be created like so. We first open an existing data example and use it to copy the driver/CRS/transform to the fake data.
 
+### Raster data
+
 ```python
 import os
 
@@ -25,7 +27,21 @@ cmap = src.colormap(1)
 dst.write_colormap(1, cmap)
 ```
 
-If the dataset expects multiple files, you can simply copy and rename the file you created.
+### Vector data
+
+```python
+import os
+
+import fiona
+
+ROOT = "/mnt/blobfuse/adam-scratch/cbf"
+FILENAME = "Ontario.geojson"
+
+src = fiona.open(os.path.join(ROOT, FILENAME))
+dst = fiona.open(FILENAME, "w", **src.meta)
+rec = next(iter(src))
+dst.write(rec)
+```
 
 ## VisionDataset
 
diff --git a/tests/data/cbf/Alberta.zip b/tests/data/cbf/Alberta.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/Alberta.zip differ
diff --git a/tests/data/cbf/BritishColumbia.zip b/tests/data/cbf/BritishColumbia.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/BritishColumbia.zip differ
diff --git a/tests/data/cbf/Manitoba.zip b/tests/data/cbf/Manitoba.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/Manitoba.zip differ
diff --git a/tests/data/cbf/NewBrunswick.zip b/tests/data/cbf/NewBrunswick.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/NewBrunswick.zip differ
diff --git a/tests/data/cbf/NewfoundlandAndLabrador.zip b/tests/data/cbf/NewfoundlandAndLabrador.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/NewfoundlandAndLabrador.zip differ
diff --git a/tests/data/cbf/NorthwestTerritories.zip b/tests/data/cbf/NorthwestTerritories.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/NorthwestTerritories.zip differ
diff --git a/tests/data/cbf/NovaScotia.zip b/tests/data/cbf/NovaScotia.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/NovaScotia.zip differ
diff --git a/tests/data/cbf/Nunavut.zip b/tests/data/cbf/Nunavut.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/Nunavut.zip differ
diff --git a/tests/data/cbf/Ontario.zip b/tests/data/cbf/Ontario.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/Ontario.zip differ
diff --git a/tests/data/cbf/PrinceEdwardIsland.zip b/tests/data/cbf/PrinceEdwardIsland.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/PrinceEdwardIsland.zip differ
diff --git a/tests/data/cbf/Quebec.zip b/tests/data/cbf/Quebec.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/Quebec.zip differ
diff --git a/tests/data/cbf/Saskatchewan.zip b/tests/data/cbf/Saskatchewan.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/Saskatchewan.zip differ
diff --git a/tests/data/cbf/YukonTerritory.zip b/tests/data/cbf/YukonTerritory.zip
new file mode 100644
Binary files /dev/null and b/tests/data/cbf/YukonTerritory.zip differ
diff --git a/tests/datasets/test_cbf.py b/tests/datasets/test_cbf.py
new file mode 100644
--- /dev/null
+++ b/tests/datasets/test_cbf.py
@@ -0,0 +1,92 @@
+import os
+import shutil
+from pathlib import Path
+from typing import Generator
+
+import matplotlib.pyplot as plt
+import pytest
+import torch
+from _pytest.fixtures import SubRequest
+from pytest import MonkeyPatch
+from rasterio.crs import CRS
+
+import torchgeo.datasets.utils
+from torchgeo.datasets import BoundingBox, CanadianBuildingFootprints, ZipDataset
+from torchgeo.transforms import Identity
+
+
+def download_url(url: str, root: str, *args: str) -> None:
+    shutil.copy(url, root)
+
+
+class TestCanadianBuildingFootprints:
+    @pytest.fixture
+    def dataset(
+        self,
+        monkeypatch: Generator[MonkeyPatch, None, None],
+        tmp_path: Path,
+        request: SubRequest,
+    ) -> CanadianBuildingFootprints:
+        monkeypatch.setattr(  # type: ignore[attr-defined]
+            torchgeo.datasets.utils, "download_url", download_url
+        )
+        md5s = [
+            "aef9a3deb3297f225d6cdb221cb48527",
+            "2b7872c4121116fda8f96490daf89d29",
+            "c71ded923e22569b62b00da2d2a61076",
+            "75a8f652531790c3c3aefc0655400d6d",
+            "89ff9c6257efa99365a8b709dde9579b",
+            "d4d6a36ed834df5cbf5254effca78a4d",
+            "cce85f6183427e3034704cf35919c985",
+            "0149c7ec5101c0309c79b7e695dcb394",
+            "b05216155725f48937804371b945f8ae",
+            "72d0e6d7196345ca520c825697cc4947",
+            "77e1c6c71ff0efbdd221b7e7d4a5f2df",
+            "86e32374f068c7bbb76aa81af0736733",
+            "5e453a3426b0bb986b2837b85e8b8850",
+        ]
+        monkeypatch.setattr(  # type: ignore[attr-defined]
+            CanadianBuildingFootprints, "md5s", md5s
+        )
+        url = os.path.join("tests", "data", "cbf") + os.sep
+        monkeypatch.setattr(  # type: ignore[attr-defined]
+            CanadianBuildingFootprints, "url", url
+        )
+        monkeypatch.setattr(  # type: ignore[attr-defined]
+            plt, "show", lambda *args: None
+        )
+        (tmp_path / "cbf").mkdir()
+        root = str(tmp_path)
+        transforms = Identity()
+        return CanadianBuildingFootprints(
+            root, transforms=transforms, download=True, checksum=True
+        )
+
+    def test_getitem(self, dataset: CanadianBuildingFootprints) -> None:
+        x = dataset[dataset.bounds]
+        assert isinstance(x, dict)
+        assert isinstance(x["crs"], CRS)
+        assert isinstance(x["masks"], torch.Tensor)
+
+    def test_add(self, dataset: CanadianBuildingFootprints) -> None:
+        ds = dataset + dataset
+        assert isinstance(ds, ZipDataset)
+
+    def test_already_downloaded(self, dataset: CanadianBuildingFootprints) -> None:
+        CanadianBuildingFootprints(root=dataset.root, download=True)
+
+    def test_plot(self, dataset: CanadianBuildingFootprints) -> None:
+        query = dataset.bounds
+        x = dataset[query]
+        dataset.plot(x["masks"])
+
+    def test_not_downloaded(self, tmp_path: Path) -> None:
+        with pytest.raises(RuntimeError, match="Dataset not found or corrupted."):
+            CanadianBuildingFootprints(str(tmp_path))
+
+    def test_invalid_query(self, dataset: CanadianBuildingFootprints) -> None:
+        query = BoundingBox(0, 0, 0, 0, 0, 0)
+        with pytest.raises(
+            IndexError, match="query: .* is not within bounds of the index:"
+        ):
+            dataset[query]