Repository: opendatacube/datacube-core

Issue:
Netcdf reader doesn't handle missing layer definition correctly
This was reported by @jankovicgd in the #815 issue. Mis-configuration in dataset yaml document leads to confusing error messages that do not guide user to possible solutions.

```
Error opening source dataset: netCDF:"/home/nikola/Data/SULAMOSA/SD022_sbas_short.nc":None
Traceback (most recent call last):
  File "rasterio/_base.pyx", line 213, in rasterio._base.DatasetBase.__init__
  File "rasterio/_shim.pyx", line 64, in rasterio._shim.open_dataset
  File "rasterio/_err.pyx", line 205, in rasterio._err.exc_wrap_pointer
rasterio._err.CPLE_OpenFailedError: netCDF:"/home/nikola/Data/SULAMOSA/SD022_sbas_short.nc":None: No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python3.6/dist-packages/datacube-1.7rc1+3.gd3f447fa.dirty-py3.6.egg/datacube/api/core.py", line 316, in load
    **legacy_args)
  File "/usr/local/lib/python3.6/dist-packages/datacube-1.7rc1+3.gd3f447fa.dirty-py3.6.egg/datacube/api/core.py", line 610, in load_data
    progress_cbk=progress_cbk)
  File "/usr/local/lib/python3.6/dist-packages/datacube-1.7rc1+3.gd3f447fa.dirty-py3.6.egg/datacube/api/core.py", line 515, in _xr_load
    progress_cbk=_cbk)
  File "/usr/local/lib/python3.6/dist-packages/datacube-1.7rc1+3.gd3f447fa.dirty-py3.6.egg/datacube/api/core.py", line 732, in _fuse_measurement
    progress_cbk=progress_cbk)
  File "/usr/local/lib/python3.6/dist-packages/datacube-1.7rc1+3.gd3f447fa.dirty-py3.6.egg/datacube/storage/_load.py", line 71, in reproject_and_fuse
    with datasources[0].open() as rdr:
  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
    return next(self.gen)
  File "/usr/local/lib/python3.6/dist-packages/datacube-1.7rc1+3.gd3f447fa.dirty-py3.6.egg/datacube/storage/_rio.py", line 201, in open
    raise e
  File "/usr/local/lib/python3.6/dist-packages/datacube-1.7rc1+3.gd3f447fa.dirty-py3.6.egg/datacube/storage/_rio.py", line 167, in open
    with rasterio.open(self.filename, sharing=False) as src:
  File "/usr/local/lib/python3.6/dist-packages/rasterio/env.py", line 423, in wrapper
    return f(*args, **kwds)
  File "/usr/local/lib/python3.6/dist-packages/rasterio/__init__.py", line 216, in open
    s = DatasetReader(path, driver=driver, **kwargs)
  File "rasterio/_base.pyx", line 215, in rasterio._base.DatasetBase.__init__
rasterio.errors.RasterioIOError: netCDF:"/home/nikola/Data/SULAMOSA/SD022_sbas_short.nc":None: No such file or directory
```

For netcdf sources `layer` key in the measurement section ought to contain the name of the netcdf variable to read. This same key is expected to contain band index for other types of sources like GeoTIFF, so it's easy to generate an incorrect dataset yaml that contains `layer: 1`. Since we perform zero verification at index time, datasets will be added to the database without any warning or errors. But at load time an error like above will be printed when user actually try to read pixels.

This function and possibly calling methods of this one should check that `layer` contains a valid string or at least is not `None` and report a meaningful error:

https://github.com/opendatacube/datacube-core/blob/7f1b81272f53e14ff9f2da9b256b13a66e47e241/datacube/storage/_rio.py#L275-L294







Discussion:

Resolving patch:
diff --git a/datacube/storage/_rio.py b/datacube/storage/_rio.py
--- a/datacube/storage/_rio.py
+++ b/datacube/storage/_rio.py
@@ -215,10 +215,10 @@ def __init__(self, band: BandInfo):
         :param measurement_id: measurement to read. a single 'band' or 'slice'
         """
         self._band_info = band
-        self._netcdf = ('netcdf' in band.format.lower())
+        self._hdf = _is_hdf(band.format)
         self._part = get_part_from_uri(band.uri)
         filename = _url2rasterio(band.uri, band.format, band.layer)
-        lock = _HDF5_LOCK if self._netcdf else None
+        lock = _HDF5_LOCK if self._hdf else None
         super(RasterDatasetDataSource, self).__init__(filename, nodata=band.nodata, lock=lock)
 
     def get_bandnumber(self, src=None) -> Optional[int]:
@@ -228,10 +228,10 @@ def get_bandnumber(self, src=None) -> Optional[int]:
         if bi.band is not None:
             return bi.band
 
-        if not self._netcdf:
+        if not self._hdf:
             return 1
 
-        # Netcdf only below
+        # Netcdf/hdf only below
         if self._part is not None:
             return self._part + 1  # Convert to rasterio 1-based indexing
 
@@ -272,20 +272,29 @@ def get_crs(self):
         return self._band_info.crs
 
 
-def _url2rasterio(url_str, fmt, layer):
+def _is_hdf(fmt: str) -> bool:
+    """ Check if format is of HDF type (this includes netcdf variants)
+    """
+    fmt = fmt.lower()
+    return any(f in fmt for f in ('netcdf', 'hdf'))
+
+
+def _url2rasterio(url_str: str, fmt: str, layer: Optional[str]) -> str:
     """
     turn URL into a string that could be passed to raterio.open
     """
     url = urlparse(url_str)
     assert url.scheme, "Expecting URL with scheme here"
 
-    # if format is NETCDF or HDF need to pass NETCDF:path:band as filename to rasterio/GDAL
-    for nasty_format in ('netcdf', 'hdf'):
-        if nasty_format in fmt.lower():
-            if url.scheme != 'file':
-                raise RuntimeError("Can't access %s over %s" % (fmt, url.scheme))
-            filename = '%s:"%s":%s' % (fmt, uri_to_local_path(url_str), layer)
-            return filename
+    if _is_hdf(fmt):
+        # if format is NETCDF or HDF need to pass NETCDF:"path":band as filename to rasterio/GDAL
+        if url.scheme != 'file':
+            raise RuntimeError("Can't access %s over %s" % (fmt, url.scheme))
+        if layer is None:
+            raise ValueError("Missing layer for hdf/netcdf format dataset")
+
+        filename = '%s:"%s":%s' % (fmt, uri_to_local_path(url_str), layer)
+        return filename
 
     if url.scheme and url.scheme != 'file':
         return url_str


Test patch:
diff --git a/tests/api/test_masking.py b/tests/api/test_masking.py
--- a/tests/api/test_masking.py
+++ b/tests/api/test_masking.py
@@ -152,7 +152,7 @@ class SimpleVariableWithFlagsDef(object):
             16383: true
         """
 
-    flags_definition = yaml.load(bits_def_yaml)
+    flags_definition = yaml.safe_load(bits_def_yaml)
 
 
 class VariableWithMultiBitFlags(object):
@@ -225,7 +225,7 @@ class VariableWithMultiBitFlags(object):
             1: True
     """
 
-    flags_definition = yaml.load(bits_def_yaml)
+    flags_definition = yaml.safe_load(bits_def_yaml)
 
 
 bits_def_json = {
diff --git a/tests/storage/test_netcdfwriter.py b/tests/storage/test_netcdfwriter.py
--- a/tests/storage/test_netcdfwriter.py
+++ b/tests/storage/test_netcdfwriter.py
@@ -60,6 +60,7 @@
                                        SPHEROID["WGS84",6378137,6556752.3141]],
                                        PRIMEM["Greenwich",0],
                                        UNIT["degree",0.0174532925199433]],
+                               UNIT["metre",1, AUTHORITY["EPSG","9001"]],
                                PROJECTION["Lambert_Conformal_Conic_2SP"],
                                PARAMETER["standard_parallel_1",17.5],
                                PARAMETER["standard_parallel_2",29.5],
diff --git a/tests/storage/test_storage.py b/tests/storage/test_storage.py
--- a/tests/storage/test_storage.py
+++ b/tests/storage/test_storage.py
@@ -15,7 +15,7 @@
 from datacube.storage import BandInfo
 from datacube.drivers.netcdf import create_netcdf_storage_unit, write_dataset_to_netcdf, Variable
 from datacube.storage import reproject_and_fuse
-from datacube.storage._rio import RasterDatasetDataSource
+from datacube.storage._rio import RasterDatasetDataSource, _url2rasterio, _is_hdf
 from datacube.storage._read import read_time_slice
 from datacube.utils.geometry import GeoBox
 
@@ -710,3 +710,18 @@ def test_rasterio_nodata(tmpdir):
 
     yy = dc_read(mm.path)
     np.testing.assert_array_equal(xx, yy)
+
+
+def test_rio_driver_specifics():
+    assert _url2rasterio('file:///f.nc', 'NetCDF', 'band') == 'NetCDF:"/f.nc":band'
+    assert _url2rasterio('file:///f.nc', 'HDF5', 'band') == 'HDF5:"/f.nc":band'
+    assert _url2rasterio('file:///f.nc', 'HDF4_EOS:EOS_GRID', 'band') == 'HDF4_EOS:EOS_GRID:"/f.nc":band'
+    assert _url2rasterio('file:///f.tiff', 'GeoTIFF', None) == '/f.tiff'
+    s3_url = 's3://bucket/file'
+    assert _url2rasterio(s3_url, 'GeoTIFF', None) is s3_url
+
+    with pytest.raises(ValueError):
+        _url2rasterio('file:///f.nc', 'NetCDF', None)
+
+    with pytest.raises(RuntimeError):
+        _url2rasterio('http://example.com/f.nc', 'NetCDF', 'aa')