Repository: microsoft/torchgeo

Issue:
NCCM checksum error
### Description

NCCM dataset with download=True, checksum=True fails with checksum error, because the zip file seems to be created by the server on-the-fly.

### Steps to reproduce

Code:

```python
from torchgeo.datasets import NCCM
NCCM(download=True, checksum=True, paths="./data/nccm/")
```

Error:

```
Downloading https://figshare.com/ndownloader/articles/13090442/versions/1 to ./data/nccm/13090442.zip
RuntimeError: File not found or corrupted.
```

It seems the zip file changes each time so the md5 is different.

```
$ md5sum file1.zip
eb29f8e5b6f41ed2c32ba5ed2b0bdc90  file1.zip
$ md5sum file2.zip
179c53feddaa81c609b0cd0433db669a  file2.zip
$ head -c 14 file1.zip | hd
00000000  50 4b 03 04 0a 00 08 08  00 00 c9 8b 45 58        |PK..........EX|
0000000e
$ head -c 14 file2.zip | hd
00000000  50 4b 03 04 0a 00 08 08  00 00 a0 8c 45 58        |PK..........EX|
0000000e
```
According to https://stackoverflow.com/questions/67902342/how-to-read-zip-file-header-manually the last four bytes there indicate file modification time. (Other bytes are different too though.) So it is possible the server is generating the zip files on the fly.

### Version

0.6.0.dev0 (a52f88f614400513cb6ef432c7caa7a8c46d878f)


Discussion:
This is annoying. Luckily NCCM is CC-BY-4.0. We should probably rehost the dataset on HF and download it from there instead so the checksum is stable.
@shreyakannan1205 @yichiac do we already have a copy of NCCM downloaded somewhere? How big is it? I can upload it to HF (or show you how) and then we can change the download URL/checksum, should be simple.
Could also checksum the files inside the archive.
I have the downloaded NCCM on my local machine. The size is ~2.4GB. 

I agree with @favyen2. The zip has only five files (three TIF masks and two GEE codes). The checksums for all individual files are the same, regardless of the download time. 

We can also consider downloading only mask files using separate links instead of the whole zip file.
5 files isn't bad, although it does open up the possibility of a zip bomb if we first extract the zip file before checksumming. But we already default to skipping the checksum, and TorchGeo isn't a security-focused application, so I'm not too worried.

I'm fine with either:

1. Download 1 zip file, extract, checksum 3 mask files
2. Download 3 mask files, checksum 3 mask files

Who wants to submit the PR?
I will submit a PR for this issue today.

Resolving patch:
diff --git a/torchgeo/datasets/nccm.py b/torchgeo/datasets/nccm.py
--- a/torchgeo/datasets/nccm.py
+++ b/torchgeo/datasets/nccm.py
@@ -3,8 +3,6 @@
 
 """Northeastern China Crop Map Dataset."""
 
-import glob
-import os
 from collections.abc import Iterable
 from typing import Any, Callable, Optional, Union
 
@@ -14,7 +12,7 @@
 from rasterio.crs import CRS
 
 from .geo import RasterDataset
-from .utils import BoundingBox, DatasetNotFoundError, download_url, extract_archive
+from .utils import BoundingBox, DatasetNotFoundError, download_url
 
 
 class NCCM(RasterDataset):
@@ -55,12 +53,24 @@ class NCCM(RasterDataset):
 
     filename_regex = r"CDL(?P<year>\d{4})_clip"
     filename_glob = "CDL*.*"
-    zipfile_glob = "13090442.zip"
 
     date_format = "%Y"
     is_image = False
-    url = "https://figshare.com/ndownloader/articles/13090442/versions/1"
-    md5 = "eae952f1b346d7e649d027e8139a76f5"
+    urls = {
+        2019: "https://figshare.com/ndownloader/files/25070540",
+        2018: "https://figshare.com/ndownloader/files/25070624",
+        2017: "https://figshare.com/ndownloader/files/25070582",
+    }
+    md5s = {
+        2019: "0d062bbd42e483fdc8239d22dba7020f",
+        2018: "b3bb4894478d10786aa798fb11693ec1",
+        2017: "d047fbe4a85341fa6248fd7e0badab6c",
+    }
+    fnames = {
+        2019: "CDL2019_clip.tif",
+        2018: "CDL2018_clip1.tif",
+        2017: "CDL2017_clip.tif",
+    }
 
     cmap = {
         0: (0, 255, 0, 255),
@@ -75,6 +85,7 @@ def __init__(
         paths: Union[str, Iterable[str]] = "data",
         crs: Optional[CRS] = None,
         res: Optional[float] = None,
+        years: list[int] = [2019],
         transforms: Optional[Callable[[dict[str, Any]], dict[str, Any]]] = None,
         cache: bool = True,
         download: bool = False,
@@ -88,6 +99,7 @@ def __init__(
                 (defaults to the CRS of the first file found)
             res: resolution of the dataset in units of CRS
                 (defaults to the resolution of the first file found)
+            years: list of years for which to use nccm layers
             transforms: a function/transform that takes an input sample
                 and returns a transformed version
             cache: if True, cache file handle to speed up repeated sampling
@@ -97,7 +109,12 @@ def __init__(
         Raises:
             DatasetNotFoundError: If dataset is not found and *download* is False.
         """
+        assert set(years) <= self.md5s.keys(), (
+            "NCCM data product only exists for the following years: "
+            f"{list(self.md5s.keys())}."
+        )
         self.paths = paths
+        self.years = years
         self.download = download
         self.checksum = checksum
         self.ordinal_map = torch.full((max(self.cmap.keys()) + 1,), 4, dtype=self.dtype)
@@ -128,37 +145,26 @@ def __getitem__(self, query: BoundingBox) -> dict[str, Any]:
 
     def _verify(self) -> None:
         """Verify the integrity of the dataset."""
-        # Check if the extracted files already exist
+        # Check if the files already exist
         if self.files:
             return
 
-        # Check if the zip file has already been downloaded
-        assert isinstance(self.paths, str)
-        pathname = os.path.join(self.paths, "**", self.zipfile_glob)
-        if glob.glob(pathname, recursive=True):
-            self._extract()
-            return
-
         # Check if the user requested to download the dataset
         if not self.download:
             raise DatasetNotFoundError(self)
 
         # Download the dataset
         self._download()
-        self._extract()
 
     def _download(self) -> None:
         """Download the dataset."""
-        filename = "13090442.zip"
-        download_url(
-            self.url, self.paths, filename, md5=self.md5 if self.checksum else None
-        )
-
-    def _extract(self) -> None:
-        """Extract the dataset."""
-        assert isinstance(self.paths, str)
-        pathname = os.path.join(self.paths, "**", self.zipfile_glob)
-        extract_archive(glob.glob(pathname, recursive=True)[0], self.paths)
+        for year in self.years:
+            download_url(
+                self.urls[year],
+                self.paths,
+                filename=self.fnames[year],
+                md5=self.md5s[year] if self.checksum else None,
+            )
 
     def plot(
         self,


Test patch:
diff --git a/tests/data/nccm/13090442.zip b/tests/data/nccm/13090442.zip
deleted file mode 100644
Binary files a/tests/data/nccm/13090442.zip and /dev/null differ
diff --git a/tests/data/nccm/13090442/CDL2017_clip.tif b/tests/data/nccm/13090442/CDL2017_clip.tif
deleted file mode 100644
Binary files a/tests/data/nccm/13090442/CDL2017_clip.tif and /dev/null differ
diff --git a/tests/data/nccm/13090442/CDL2018_clip1.tif b/tests/data/nccm/13090442/CDL2018_clip1.tif
deleted file mode 100644
Binary files a/tests/data/nccm/13090442/CDL2018_clip1.tif and /dev/null differ
diff --git a/tests/data/nccm/13090442/CDL2019_clip.tif b/tests/data/nccm/13090442/CDL2019_clip.tif
deleted file mode 100644
Binary files a/tests/data/nccm/13090442/CDL2019_clip.tif and /dev/null differ
diff --git a/tests/data/nccm/CDL2017_clip.tif b/tests/data/nccm/CDL2017_clip.tif
new file mode 100644
Binary files /dev/null and b/tests/data/nccm/CDL2017_clip.tif differ
diff --git a/tests/data/nccm/CDL2018_clip1.tif b/tests/data/nccm/CDL2018_clip1.tif
new file mode 100644
Binary files /dev/null and b/tests/data/nccm/CDL2018_clip1.tif differ
diff --git a/tests/data/nccm/CDL2019_clip.tif b/tests/data/nccm/CDL2019_clip.tif
new file mode 100644
Binary files /dev/null and b/tests/data/nccm/CDL2019_clip.tif differ
diff --git a/tests/data/nccm/data.py b/tests/data/nccm/data.py
--- a/tests/data/nccm/data.py
+++ b/tests/data/nccm/data.py
@@ -5,7 +5,6 @@
 
 import hashlib
 import os
-import shutil
 
 import numpy as np
 import rasterio
@@ -48,20 +47,14 @@ def create_file(path: str, dtype: str):
 
 
 if __name__ == "__main__":
-    dir = os.path.join(os.getcwd(), "13090442")
-
-    if os.path.exists(dir) and os.path.isdir(dir):
-        shutil.rmtree(dir)
-
+    dir = os.path.join(os.getcwd())
     os.makedirs(dir, exist_ok=True)
 
     for file in files:
         create_file(os.path.join(dir, file), dtype="int8")
 
-    # Compress data
-    shutil.make_archive("13090442", "zip", ".", dir)
-
     # Compute checksums
-    with open("13090442.zip", "rb") as f:
-        md5 = hashlib.md5(f.read()).hexdigest()
-        print(f"13090442.zip: {md5}")
+    for file in files:
+        with open(file, "rb") as f:
+            md5 = hashlib.md5(f.read()).hexdigest()
+            print(f"{file}: {md5}")
diff --git a/tests/datasets/test_nccm.py b/tests/datasets/test_nccm.py
--- a/tests/datasets/test_nccm.py
+++ b/tests/datasets/test_nccm.py
@@ -25,9 +25,19 @@ class TestNCCM:
     @pytest.fixture
     def dataset(self, monkeypatch: MonkeyPatch, tmp_path: Path) -> NCCM:
         monkeypatch.setattr(torchgeo.datasets.nccm, "download_url", download_url)
-        url = os.path.join("tests", "data", "nccm", "13090442.zip")
+        md5s = {
+            2017: "ae5c390d0ffb8970d544b8a09142759f",
+            2018: "0d453bdb8ea5b7318c33e62513760580",
+            2019: "d4ab7ab00bb57623eafb6b27747e5639",
+        }
+        monkeypatch.setattr(NCCM, "md5s", md5s)
+        urls = {
+            2017: os.path.join("tests", "data", "nccm", "CDL2017_clip.tif"),
+            2018: os.path.join("tests", "data", "nccm", "CDL2018_clip1.tif"),
+            2019: os.path.join("tests", "data", "nccm", "CDL2019_clip.tif"),
+        }
+        monkeypatch.setattr(NCCM, "urls", urls)
         transforms = nn.Identity()
-        monkeypatch.setattr(NCCM, "url", url)
         root = str(tmp_path)
         return NCCM(root, transforms=transforms, download=True, checksum=True)
 
@@ -48,11 +58,8 @@ def test_or(self, dataset: NCCM) -> None:
     def test_already_extracted(self, dataset: NCCM) -> None:
         NCCM(dataset.paths, download=True)
 
-    def test_already_downloaded(self, tmp_path: Path) -> None:
-        pathname = os.path.join("tests", "data", "nccm", "13090442.zip")
-        root = str(tmp_path)
-        shutil.copy(pathname, root)
-        NCCM(root)
+    def test_already_downloaded(self, dataset: NCCM) -> None:
+        NCCM(dataset.paths, download=True)
 
     def test_plot(self, dataset: NCCM) -> None:
         query = dataset.bounds