Repository: opendatacube/datacube-core

Issue:
`dc.load` is not thread-safe when reading netcdf files
HDF5 library used via `datacube->rasterio->gdal->netcdf->hdf5`  is not thread-safe. It is not safe to call any method of HDF5 library from two threads simultaneously. It doesn't matter if operation is on different files or the same file, calling `read` from one file while calling `open` on another might cause memory corruption within HDF5 library.

My understanding is that GDAL does not guarantee that HDF5 library calls are serialised. It is possible to compile HDF5 library with "threads supports", which would lock-protect access to the entire library API, this option does not make HDF5 library concurrent, but it does make it thread-safe. This option is however rarely enabled.

RasterIO based IO driver should detect that file being read is netcdf and make sure that only one thread is calling `.open|.read`. This is particularly important when using `dask`.

References: 

- https://support.hdfgroup.org/HDF5/faq/threadsafe.html


Discussion:

Resolving patch:
diff --git a/datacube/storage/_rio.py b/datacube/storage/_rio.py
--- a/datacube/storage/_rio.py
+++ b/datacube/storage/_rio.py
@@ -3,7 +3,9 @@
 Driver implementation for Rasterio based reader.
 """
 import logging
+import contextlib
 from contextlib import contextmanager
+from threading import RLock
 import numpy as np
 from affine import Affine
 import rasterio
@@ -18,6 +20,7 @@
 from . import DataSource, GeoRasterReader, RasterShape, RasterWindow, BandInfo
 
 _LOG = logging.getLogger(__name__)
+_HDF5_LOCK = RLock()
 
 GDAL_NETCDF_DIM = ('NETCDF_DIM_'
                    if str(rasterio.__gdal_version__) >= '1.10.0' else
@@ -31,6 +34,12 @@ def _rasterio_crs_wkt(src):
         return ''
 
 
+def maybe_lock(lock):
+    if lock is None:
+        return contextlib.suppress()
+    return lock
+
+
 class BandDataSource(GeoRasterReader):
     """
     Wrapper for a :class:`rasterio.Band` object
@@ -38,12 +47,14 @@ class BandDataSource(GeoRasterReader):
     :type source: rasterio.Band
     """
 
-    def __init__(self, source, nodata=None):
+    def __init__(self, source, nodata=None,
+                 lock: Optional[RLock] = None):
         self.source = source
         if nodata is None:
             nodata = self.source.ds.nodatavals[self.source.bidx-1]
 
         self._nodata = num2numpy(nodata, source.dtype)
+        self._lock = lock
 
     @property
     def nodata(self):
@@ -69,7 +80,8 @@ def read(self, window: Optional[RasterWindow] = None,
              out_shape: Optional[RasterShape] = None) -> Optional[np.ndarray]:
         """Read data in the native format, returning a numpy array
         """
-        return self.source.ds.read(indexes=self.source.bidx, window=window, out_shape=out_shape)
+        with maybe_lock(self._lock):
+            return self.source.ds.read(indexes=self.source.bidx, window=window, out_shape=out_shape)
 
 
 class OverrideBandDataSource(GeoRasterReader):
@@ -85,11 +97,13 @@ def __init__(self,
                  source: rasterio.Band,
                  nodata,
                  crs: geometry.CRS,
-                 transform: Affine):
+                 transform: Affine,
+                 lock: Optional[RLock] = None):
         self.source = source
         self._nodata = num2numpy(nodata, source.dtype)
         self._crs = crs
         self._transform = transform
+        self._lock = lock
 
     @property
     def crs(self) -> geometry.CRS:
@@ -115,7 +129,8 @@ def read(self, window: Optional[RasterWindow] = None,
              out_shape: Optional[RasterShape] = None) -> Optional[np.ndarray]:
         """Read data in the native format, returning a native array
         """
-        return self.source.ds.read(indexes=self.source.bidx, window=window, out_shape=out_shape)
+        with maybe_lock(self._lock):
+            return self.source.ds.read(indexes=self.source.bidx, window=window, out_shape=out_shape)
 
 
 class RasterioDataSource(DataSource):
@@ -124,9 +139,10 @@ class RasterioDataSource(DataSource):
 
     """
 
-    def __init__(self, filename, nodata):
+    def __init__(self, filename, nodata, lock=None):
         self.filename = filename
         self.nodata = nodata
+        self._lock = lock
 
     def get_bandnumber(self, src):
         raise NotImplementedError()
@@ -143,6 +159,9 @@ def open(self) -> Iterator[GeoRasterReader]:
 
         activate_from_config()  # check if settings changed and apply new
 
+        lock = self._lock
+        locked = False if lock is None else lock.acquire(blocking=True)
+
         try:
             _LOG.debug("opening %s", self.filename)
             with rasterio.open(self.filename, sharing=False) as src:
@@ -168,14 +187,21 @@ def open(self) -> Iterator[GeoRasterReader]:
                 nodata = src.nodatavals[band.bidx-1] if src.nodatavals[band.bidx-1] is not None else self.nodata
                 nodata = num2numpy(nodata, band.dtype)
 
+                if locked:
+                    locked = False
+                    lock.release()
+
                 if override:
-                    yield OverrideBandDataSource(band, nodata=nodata, crs=crs, transform=transform)
+                    yield OverrideBandDataSource(band, nodata=nodata, crs=crs, transform=transform, lock=lock)
                 else:
-                    yield BandDataSource(band, nodata=nodata)
+                    yield BandDataSource(band, nodata=nodata, lock=lock)
 
         except Exception as e:
             _LOG.error("Error opening source dataset: %s", self.filename)
             raise e
+        finally:
+            if locked:
+                lock.release()
 
 
 class RasterDatasetDataSource(RasterioDataSource):
@@ -192,7 +218,8 @@ def __init__(self, band: BandInfo):
         self._netcdf = ('netcdf' in band.format.lower())
         self._part = get_part_from_uri(band.uri)
         filename = _url2rasterio(band.uri, band.format, band.layer)
-        super(RasterDatasetDataSource, self).__init__(filename, nodata=band.nodata)
+        lock = _HDF5_LOCK if self._netcdf else None
+        super(RasterDatasetDataSource, self).__init__(filename, nodata=band.nodata, lock=lock)
 
     def get_bandnumber(self, src=None) -> Optional[int]:
 
diff --git a/datacube/testutils/io.py b/datacube/testutils/io.py
--- a/datacube/testutils/io.py
+++ b/datacube/testutils/io.py
@@ -12,8 +12,8 @@
 class RasterFileDataSource(RasterioDataSource):
     """ This is only used in test code
     """
-    def __init__(self, filename, bandnumber, nodata=None, crs=None, transform=None):
-        super(RasterFileDataSource, self).__init__(filename, nodata)
+    def __init__(self, filename, bandnumber, nodata=None, crs=None, transform=None, lock=None):
+        super(RasterFileDataSource, self).__init__(filename, nodata, lock=lock)
         self.bandnumber = bandnumber
         self.crs = crs
         self.transform = transform


Test patch:
diff --git a/tests/test_load_data.py b/tests/test_load_data.py
--- a/tests/test_load_data.py
+++ b/tests/test_load_data.py
@@ -11,6 +11,7 @@
     mk_test_image,
 )
 from datacube.testutils.io import write_gtiff, rio_slurp
+from datacube.testutils.iodriver import NetCDF
 from datacube.utils import ignore_exceptions_if
 
 
@@ -109,6 +110,28 @@ def progress_cbk(n, nt):
     assert progress_call_data == [(1, 2), (2, 2)]
 
 
+def test_hdf5_lock_release_on_failure():
+    from datacube.storage._rio import RasterDatasetDataSource, _HDF5_LOCK
+    from datacube.storage import BandInfo
+
+    band = dict(name='xx',
+                layer='xx',
+                dtype='uint8',
+                units='K',
+                nodata=33)
+
+    ds = mk_sample_dataset([band],
+                           uri='file:///tmp/this_probably_doesnot_exist_37237827513/xx.nc',
+                           format=NetCDF)
+    src = RasterDatasetDataSource(BandInfo(ds, 'xx'))
+
+    with pytest.raises(OSError):
+        with src.open():
+            assert False and "Did not expect to get here"
+
+    assert not _HDF5_LOCK._is_owned()
+
+
 def test_rio_slurp(tmpdir):
     w, h, dtype, nodata, ndw = 96, 64, 'int16', -999, 7