Repository: opendatacube/datacube-core

Issue:
Support GDAL "/vsi*" paths directly
Normally we rely on URL syntax for defining paths to assets of a dataset. `rasterio` library converts those to GDAL's VSI syntax for us. For example `https://host.tld/file.tif` will become `/vsicurl/https://host.tld/file.tif`. Nesting is also supported via `+`, for example `http+zip://`.

Sometimes however one needs to fallback to raw GDAL syntax. It should be possible to define dataset location as GDAL vsi path.


Discussion:
Looks like netcdf over http case requires `vsicurl/` paths right now as it's not clear how to specify band names in the url syntax and `vsi` fallback can be used in the interim.
This used to be something we could simply handle on the driver level by having our compatibility layer return `Dataset` objects with a custom `uri_scheme` defined that would route the rasters through our driver, but v1.7 started parsing the uri_scheme from the uri rather than pulling it from the `Dataset` objects. I've been tinkering with restoring that sort of scheme on our end but it'll involve some gross stuff like prepending `vsicurl://` then making required modifications to the uri elsewhere.  

Re: netCDF vsicurl usage, we're formatting 'uri's that look like: `netCDF:/vsicurl/"https://{url}":{band_name}"'` and it works fine. Unfortunately GDAL/rasterio doesn't recognize netCDF files properly without the vsicurl designation. 
Thanks @alfredoahds so we need to detect all "HDF:" prefixes as well as `/vsi.*` for "raw gdal syntax" paths.
I guess alternative is to still "wrap" path in `NETCDF:"{file|vsi path}":band`, and just detect `vsi_path` strings.
I think the simplest solution would be to add support in rasterio directly. In this scenario, no `vsicurl` support needed in datacube. Issue about it here: https://github.com/mapbox/rasterio/issues/1838. 
@snowman2 agree, but having fallback to gdal vsi paths be available is still useful.
> having fallback to gdal vsi paths be available is still useful.

Agreed 👍

Resolving patch:
diff --git a/datacube/storage/_rio.py b/datacube/storage/_rio.py
--- a/datacube/storage/_rio.py
+++ b/datacube/storage/_rio.py
@@ -15,7 +15,7 @@
 from datacube.utils import datetime_to_seconds_since_1970
 from datacube.utils import geometry
 from datacube.utils.math import num2numpy
-from datacube.utils import uri_to_local_path, get_part_from_uri
+from datacube.utils import uri_to_local_path, get_part_from_uri, is_vsipath
 from datacube.utils.rio import activate_from_config
 from . import DataSource, GeoRasterReader, RasterShape, RasterWindow, BandInfo
 
@@ -279,25 +279,40 @@ def _is_hdf(fmt: str) -> bool:
     return any(f in fmt for f in ('netcdf', 'hdf'))
 
 
+def _build_hdf_uri(url_str: str, fmt: str, layer: str) -> str:
+    if is_vsipath(url_str):
+        base = url_str
+    else:
+        url = urlparse(url_str)
+        if url.scheme in (None, ''):
+            raise ValueError("Expect either URL or /vsi path")
+
+        if url.scheme != 'file':
+            raise RuntimeError("Can't access %s over %s" % (fmt, url.scheme))
+        base = str(uri_to_local_path(url_str))
+
+    return '{}:"{}":{}'.format(fmt, base, layer)
+
+
 def _url2rasterio(url_str: str, fmt: str, layer: Optional[str]) -> str:
     """
     turn URL into a string that could be passed to raterio.open
     """
-    url = urlparse(url_str)
-    assert url.scheme, "Expecting URL with scheme here"
-
     if _is_hdf(fmt):
-        # if format is NETCDF or HDF need to pass NETCDF:"path":band as filename to rasterio/GDAL
-        if url.scheme != 'file':
-            raise RuntimeError("Can't access %s over %s" % (fmt, url.scheme))
         if layer is None:
             raise ValueError("Missing layer for hdf/netcdf format dataset")
 
-        filename = '%s:"%s":%s' % (fmt, uri_to_local_path(url_str), layer)
-        return filename
+        return _build_hdf_uri(url_str, fmt, layer)
 
-    if url.scheme and url.scheme != 'file':
+    if is_vsipath(url_str):
         return url_str
 
-    # if local path strip scheme and other gunk
-    return str(uri_to_local_path(url_str))
+    url = urlparse(url_str)
+    if url.scheme in (None, ''):
+        raise ValueError("Expect either URL or /vsi path")
+
+    if url.scheme == 'file':
+        # if local path strip scheme and other gunk
+        return str(uri_to_local_path(url_str))
+
+    return url_str
diff --git a/datacube/utils/__init__.py b/datacube/utils/__init__.py
--- a/datacube/utils/__init__.py
+++ b/datacube/utils/__init__.py
@@ -5,7 +5,7 @@
 from .dates import datetime_to_seconds_since_1970, parse_time
 from .py import cached_property, ignore_exceptions_if, import_function
 from .serialise import jsonify_document
-from .uris import is_url, uri_to_local_path, get_part_from_uri, mk_part_uri
+from .uris import is_url, uri_to_local_path, get_part_from_uri, mk_part_uri, is_vsipath
 from .io import slurp, check_write_path, write_user_secret_file
 from .documents import (
     InvalidDocException,
@@ -43,6 +43,7 @@
     "import_function",
     "jsonify_document",
     "is_url",
+    "is_vsipath",
     "uri_to_local_path",
     "get_part_from_uri",
     "mk_part_uri",
diff --git a/datacube/utils/uris.py b/datacube/utils/uris.py
--- a/datacube/utils/uris.py
+++ b/datacube/utils/uris.py
@@ -34,6 +34,21 @@ def is_url(url_str: str) -> bool:
         return False
 
 
+def is_vsipath(path: str) -> bool:
+    """ Check if string is a GDAL "/vsi.*" path
+    """
+    path = path.lower()
+    return path.startswith("/vsi")
+
+
+def vsi_join(base: str, path: str) -> str:
+    """ Extend GDAL's vsi path
+
+        Basically just base/path, but taking care of trailing `/` in base
+    """
+    return base.rstrip('/') + '/' + path
+
+
 def uri_to_local_path(local_uri: Optional[str]) -> Optional[pathlib.Path]:
     """
     Transform a URI to a platform dependent Path.
@@ -163,21 +178,24 @@ def norm(p):
 
 def uri_resolve(base: str, path: Optional[str]) -> str:
     """
-    path                  -- if path is a uri
+    path                  -- if path is a uri or /vsi.* style path
     Path(path).as_uri()   -- if path is absolute filename
     base/path             -- in all other cases
     """
     if not path:
         return base
 
-    if is_url(path):
+    if is_vsipath(path) or is_url(path):
         return path
 
     p = Path(path)
     if p.is_absolute():
         return p.as_uri()
 
-    return urljoin(base, path)
+    if is_vsipath(base):
+        return vsi_join(base, path)
+    else:
+        return urljoin(base, path)
 
 
 def pick_uri(uris: List[str], scheme: Optional[str] = None) -> str:


Test patch:
diff --git a/tests/storage/test_storage.py b/tests/storage/test_storage.py
--- a/tests/storage/test_storage.py
+++ b/tests/storage/test_storage.py
@@ -720,8 +720,19 @@ def test_rio_driver_specifics():
     s3_url = 's3://bucket/file'
     assert _url2rasterio(s3_url, 'GeoTIFF', None) is s3_url
 
+    vsi_url = '/vsicurl/https://host.tld/path'
+    assert _url2rasterio(vsi_url, 'GeoTIFF', None) is vsi_url
+    assert _url2rasterio(vsi_url, 'NetCDF', 'aa') == 'NetCDF:"{}":aa'.format(vsi_url)
+    assert _url2rasterio(vsi_url, 'HDF5', 'aa') == 'HDF5:"{}":aa'.format(vsi_url)
+
     with pytest.raises(ValueError):
         _url2rasterio('file:///f.nc', 'NetCDF', None)
 
     with pytest.raises(RuntimeError):
         _url2rasterio('http://example.com/f.nc', 'NetCDF', 'aa')
+
+    with pytest.raises(ValueError):
+        _url2rasterio('/some/path/', 'GeoTIFF', None)
+
+    with pytest.raises(ValueError):
+        _url2rasterio('/some/path/', 'NetCDF', 'aa')
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -106,6 +106,8 @@ def test_uri_resolve():
     some_uri = 'http://example.com/file.txt'
     s3_base = 's3://foo'
     gs_base = 'gs://foo'
+    vsi_base = '/vsizip//vsicurl/https://host.tld/some/path'
+
     assert uri_resolve(s3_base, abs_path) == "file://" + abs_path
     assert uri_resolve(s3_base, some_uri) is some_uri
     assert uri_resolve(s3_base, None) is s3_base
@@ -117,6 +119,9 @@ def test_uri_resolve():
     assert uri_resolve(gs_base, '') is gs_base
     assert uri_resolve(gs_base, 'relative/path') == gs_base + '/relative/path'
 
+    assert uri_resolve(vsi_base, 'relative/path') == vsi_base + '/relative/path'
+    assert uri_resolve(vsi_base + '/', 'relative/path') == vsi_base + '/relative/path'
+
 
 def test_pick_uri():
     f, s, h = ('file://a', 's3://b', 'http://c')