Repository: opendatacube/datacube-core

Issue:
Capture more classes of error when `skip_broken_datasets=True`
Currently we only capture `OSError`, which covers missing files, with s3 and http this probably won't work. 

This function needs to be generalised to capture wider range of exceptions, and load code needs to be updated to supply a broader list of exceptions to capture, possibly user selected. 

https://github.com/opendatacube/datacube-core/blob/166fecff31fcef2334ce1e74a3c9ad05173a6376/datacube/utils/py.py#L27-L36




Discussion:
@dunkgray relevant to processing in the cloud

Resolving patch:
diff --git a/datacube/api/core.py b/datacube/api/core.py
--- a/datacube/api/core.py
+++ b/datacube/api/core.py
@@ -128,6 +128,7 @@ def _list_measurements(self):
 
     #: pylint: disable=too-many-arguments, too-many-locals
     def load(self, product=None, measurements=None, output_crs=None, resolution=None, resampling=None,
+             skip_broken_datasets=False,
              dask_chunks=None, like=None, fuse_func=None, align=None, datasets=None, **query):
         """
         Load data as an ``xarray`` object.  Each measurement will be a data variable in the :class:`xarray.Dataset`.
@@ -293,6 +294,7 @@ def load(self, product=None, measurements=None, output_crs=None, resolution=None
                                 resampling=resampling,
                                 fuse_func=fuse_func,
                                 dask_chunks=dask_chunks,
+                                skip_broken_datasets=skip_broken_datasets,
                                 **legacy_args)
 
         return apply_aliases(result, datacube_product, measurements)
diff --git a/datacube/storage/_rio.py b/datacube/storage/_rio.py
--- a/datacube/storage/_rio.py
+++ b/datacube/storage/_rio.py
@@ -141,7 +141,7 @@ def open(self) -> Iterator[GeoRasterReader]:
         """Context manager which returns a :class:`BandDataSource`"""
         try:
             _LOG.debug("opening %s", self.filename)
-            with rasterio.open(self.filename) as src:
+            with rasterio.open(self.filename, sharing=False) as src:
                 override = False
 
                 transform = src.transform
diff --git a/datacube/utils/py.py b/datacube/utils/py.py
--- a/datacube/utils/py.py
+++ b/datacube/utils/py.py
@@ -25,12 +25,15 @@ def import_function(func_ref):
 
 
 @contextmanager
-def ignore_exceptions_if(ignore_errors):
+def ignore_exceptions_if(ignore_errors, errors=None):
     """Ignore Exceptions raised within this block if ignore_errors is True"""
+    if errors is None:
+        errors = (Exception,)
+
     if ignore_errors:
         try:
             yield
-        except OSError as e:
+        except errors as e:
             _LOG.warning('Ignoring Exception: %s', e)
     else:
         yield


Test patch:
diff --git a/tests/test_load_data.py b/tests/test_load_data.py
--- a/tests/test_load_data.py
+++ b/tests/test_load_data.py
@@ -3,6 +3,7 @@
 import numpy as np
 from collections import Sequence
 from types import SimpleNamespace
+import pytest
 
 from pathlib import Path
 from datacube.testutils import (
@@ -10,6 +11,7 @@
     mk_test_image,
 )
 from datacube.testutils.io import write_gtiff, rio_slurp
+from datacube.utils import ignore_exceptions_if
 
 
 def gen_tiff_dataset(bands,
@@ -169,3 +171,22 @@ def test_rio_slurp_with_gbox(tmpdir):
     aa, mm = rio_slurp(mm.path, mm.gbox)
     assert aa.shape == aa0.shape
     np.testing.assert_array_equal(aa, aa0)
+
+
+def test_missing_file_handling():
+    with pytest.raises(IOError):
+        rio_slurp('no-such-file.tiff')
+
+    # by default should catch any exception
+    with ignore_exceptions_if(True):
+        rio_slurp('no-such-file.tiff')
+
+    # this is equivalent to previous default behaviour, note that missing http
+    # resources are not OSError
+    with ignore_exceptions_if(True, (OSError,)):
+        rio_slurp('no-such-file.tiff')
+
+    # check that only requested exceptions are caught
+    with pytest.raises(IOError):
+        with ignore_exceptions_if(True, (ValueError, ArithmeticError)):
+            rio_slurp('no-such-file.tiff')