Repository: opendatacube/datacube-core

Issue:
Azure Blob support
### Expected behaviour
A user in the slack group has asked if ODC supports Azure blob store 

### Actual behaviour

ODC only supports s3 and gs cloud storage: (and vsicurl etc.) 
https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/uris.py 

### Required work 

To add support for azure blob storage we should add wasb[s]:// to:
https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/uris.py  

To enable it to actually be indexed we should probably create a wasb_to_tar script to the odc tools repo https://github.com/opendatacube/odc-tools/tree/master/apps/cloud/odc/apps/cloud 



Discussion:
You don't think it'll work? I guess we should try! 

This identifies the URI for the Azure blob, which is `/vsiaz/` https://github.com/opendatacube/datacube-core/blob/develop/datacube/utils/uris.py#L37
Issue if not with GDAL but with python URL parsing libraries, for some reason it needs a whitelist of supported protocols. As Tom mentioned we just need another call to `register_scheme`  like this one is for `gs`. 

https://github.com/opendatacube/datacube-core/blob/e327e8f3508111635dfee2cffa36ae079384737e/datacube/utils/uris.py#L242-L244

Users can do that themselves too, but for standard cloud formats we should put in to core


Resolving patch:
diff --git a/datacube/api/query.py b/datacube/api/query.py
--- a/datacube/api/query.py
+++ b/datacube/api/query.py
@@ -27,8 +27,9 @@
 import numpy as np
 
 
-from ..model import Range
+from ..model import Range, Dataset
 from ..utils import geometry, datetime_to_seconds_since_1970
+from ..utils.dates import normalise_dt
 
 _LOG = logging.getLogger(__name__)
 
@@ -156,19 +157,23 @@ def query_geopolygon(geopolygon=None, **kwargs):
     return geopolygon
 
 
+def _extract_time_from_ds(ds: Dataset) -> datetime.datetime:
+    return normalise_dt(ds.center_time)
+
+
 def query_group_by(group_by='time', **kwargs):
     if not isinstance(group_by, str):
         return group_by
 
     time_grouper = GroupBy(dimension='time',
-                           group_by_func=lambda ds: ds.center_time,
+                           group_by_func=_extract_time_from_ds,
                            units='seconds since 1970-01-01 00:00:00',
-                           sort_key=lambda ds: ds.center_time)
+                           sort_key=_extract_time_from_ds)
 
     solar_day_grouper = GroupBy(dimension='time',
                                 group_by_func=solar_day,
                                 units='seconds since 1970-01-01 00:00:00',
-                                sort_key=lambda ds: ds.center_time)
+                                sort_key=_extract_time_from_ds)
 
     group_by_map = {
         None: time_grouper,
diff --git a/datacube/utils/geometry/_base.py b/datacube/utils/geometry/_base.py
--- a/datacube/utils/geometry/_base.py
+++ b/datacube/utils/geometry/_base.py
@@ -2,7 +2,7 @@
 import itertools
 import math
 from collections import namedtuple, OrderedDict
-from typing import Tuple, Iterable, List, Union, Optional, Any, Callable, Mapping, Hashable, Dict
+from typing import Tuple, Iterable, List, Union, Optional, Any, Callable, Hashable, Dict
 from collections.abc import Sequence
 from distutils.version import LooseVersion
 
@@ -943,7 +943,7 @@ def xr_coords(self, with_crs: Union[bool, str] = False) -> Dict[Hashable, xr.Dat
             attrs['crs'] = str(crs)
 
         coords = dict((n, _coord_to_xr(n, c, **attrs))
-                      for n, c in coords.items()) # type: Dict[Hashable, xr.DataArray]
+                      for n, c in coords.items())  # type: Dict[Hashable, xr.DataArray]
 
         if with_crs and crs is not None:
             coords[spatial_ref] = _mk_crs_coord(crs, spatial_ref)
diff --git a/datacube/utils/uris.py b/datacube/utils/uris.py
--- a/datacube/utils/uris.py
+++ b/datacube/utils/uris.py
@@ -14,19 +14,6 @@
 def is_url(url_str: str) -> bool:
     """
     Check if url_str tastes like a url (starts with blah://)
-
-    >>> is_url('file:///etc/blah')
-    True
-    >>> is_url('http://greg.com/greg.txt')
-    True
-    >>> is_url('s3:///etc/blah')
-    True
-    >>> is_url('gs://data/etc/blah.yaml')
-    True
-    >>> is_url('/etc/blah')
-    False
-    >>> is_url('C:/etc/blah')
-    False
     """
     try:
         return URL_RE.match(url_str) is not None
@@ -235,10 +222,12 @@ def register_scheme(*schemes):
     urllib.parse.uses_params.extend(schemes)
 
 
-# s3:// not recognised by python by default
-#  without this `urljoin` might be broken for s3 urls
-register_scheme('s3')
-
-# gs:// not recognised by python by default
-#  without this `urljoin` might be broken for google storage urls
-register_scheme('gs')
+# `urljoin`, that we use for relative path computation, needs to know which url
+# schemes support relative offsets. By default only well known types are
+# understood. So here we register more common blob store url protocols.
+register_scheme(
+    's3',         # `s3://...`      -- AWS S3 Object Store
+    'gs',         # `gs://...`      -- Google Cloud Storage
+    'wasb',       # `wasb[s]://...` -- Windows Azure Storage Blob
+    'wasbs'
+)


Test patch:
diff --git a/tests/api/test_core.py b/tests/api/test_core.py
--- a/tests/api/test_core.py
+++ b/tests/api/test_core.py
@@ -9,6 +9,7 @@
 from datacube.api.core import _calculate_chunk_sizes
 from datacube import Datacube
 from datacube.testutils.geom import AlbersGS
+from datacube.testutils import mk_sample_dataset
 
 
 def test_grouping_datasets():
@@ -38,6 +39,23 @@ def group_func(d):
     assert grouped.loc['2016-01-01':'2016-01-15']
 
 
+def test_group_datasets_by_time():
+    bands = [dict(name='a')]
+    # Same time instant but one explicitly marked as UTC
+    ds1 = mk_sample_dataset(bands, timestamp="2019-01-01T23:24:00Z")
+    ds2 = mk_sample_dataset(bands, timestamp="2019-01-01T23:24:00")
+    # Same "time" but in a different timezone, and actually later
+    ds3 = mk_sample_dataset(bands, timestamp="2019-01-01T23:24:00-1")
+    assert ds1.center_time.tzinfo is not None
+    assert ds2.center_time.tzinfo is None
+    assert ds3.center_time.tzinfo is not None
+
+    xx = Datacube.group_datasets([ds1, ds2, ds3], 'time')
+    assert xx.time.shape == (2,)
+    assert len(xx.data[0]) == 2
+    assert len(xx.data[1]) == 1
+
+
 def test_grouped_datasets_should_be_in_consistent_order():
     datasets = [
         {'time': datetime.datetime(2016, 1, 1, 0, 1), 'value': 'foo'},
@@ -73,7 +91,7 @@ def test_dask_chunks():
 
     sources = xr.DataArray(coords['time'],
                            coords=coords,
-                           dims=coords.keys())
+                           dims=list(coords))
     geobox = AlbersGS.tile_geobox((0, 0))[:6, :7]
 
     assert geobox.dimensions == ('y', 'x')
diff --git a/tests/test_geometry.py b/tests/test_geometry.py
--- a/tests/test_geometry.py
+++ b/tests/test_geometry.py
@@ -227,6 +227,19 @@ def test_ops():
     assert all(p.crs == poly_2_parts.crs for p in pp)
 
 
+def test_to_crs():
+    poly = geometry.polygon([(0, 0), (0, 5), (10, 5)], epsg4326)
+    assert poly.crs is epsg4326
+    assert poly.to_crs(epsg3857).crs is epsg3857
+    assert poly.to_crs('EPSG:3857').crs == 'EPSG:3857'
+    assert poly.to_crs('EPSG:3857', 0.1).crs == epsg3857
+
+    poly = geometry.polygon([(0, 0), (0, 5), (10, 5)], None)
+    assert poly.crs is None
+    with pytest.raises(ValueError):
+        poly.to_crs(epsg3857)
+
+
 def test_bbox_union():
     b1 = BoundingBox(0, 1, 10, 20)
     b2 = BoundingBox(5, 6, 11, 22)
diff --git a/tests/test_utils_other.py b/tests/test_utils_other.py
--- a/tests/test_utils_other.py
+++ b/tests/test_utils_other.py
@@ -35,7 +35,7 @@
 )
 from datacube.utils.py import sorted_items
 from datacube.utils.uris import (uri_to_local_path, mk_part_uri, get_part_from_uri, as_url, is_url,
-                                 pick_uri, uri_resolve,
+                                 pick_uri, uri_resolve, is_vsipath,
                                  normalise_path, default_base_dir)
 from datacube.utils.io import check_write_path
 from datacube.testutils import mk_sample_product, remove_crs
@@ -104,26 +104,27 @@ def test_uri_to_local_path():
         uri_to_local_path('ftp://example.com/tmp/something.txt')
 
 
-def test_uri_resolve():
+@pytest.mark.parametrize("base", [
+    "s3://foo",
+    "gs://foo",
+    "wasb://foo",
+    "wasbs://foo",
+    "/vsizip//vsicurl/https://host.tld/some/path",
+])
+def test_uri_resolve(base):
     abs_path = '/abs/path/to/something'
     some_uri = 'http://example.com/file.txt'
-    s3_base = 's3://foo'
-    gs_base = 'gs://foo'
-    vsi_base = '/vsizip//vsicurl/https://host.tld/some/path'
 
-    assert uri_resolve(s3_base, abs_path) == "file://" + abs_path
-    assert uri_resolve(s3_base, some_uri) is some_uri
-    assert uri_resolve(s3_base, None) is s3_base
-    assert uri_resolve(s3_base, '') is s3_base
-    assert uri_resolve(s3_base, 'relative/path') == s3_base + '/relative/path'
-    assert uri_resolve(gs_base, abs_path) == "file://" + abs_path
-    assert uri_resolve(gs_base, some_uri) is some_uri
-    assert uri_resolve(gs_base, None) is gs_base
-    assert uri_resolve(gs_base, '') is gs_base
-    assert uri_resolve(gs_base, 'relative/path') == gs_base + '/relative/path'
+    assert uri_resolve(base, abs_path) == "file://" + abs_path
+    assert uri_resolve(base, some_uri) is some_uri
+    assert uri_resolve(base, None) is base
+    assert uri_resolve(base, '') is base
+    assert uri_resolve(base, 'relative/path') == base + '/relative/path'
+    assert uri_resolve(base + '/', 'relative/path') == base + '/relative/path'
+    assert uri_resolve(base + '/some/dir/', 'relative/path') == base + '/some/dir/relative/path'
 
-    assert uri_resolve(vsi_base, 'relative/path') == vsi_base + '/relative/path'
-    assert uri_resolve(vsi_base + '/', 'relative/path') == vsi_base + '/relative/path'
+    if not is_vsipath(base):
+        assert uri_resolve(base + '/some/dir/file.txt', 'relative/path') == base + '/some/dir/relative/path'
 
 
 def test_pick_uri():
@@ -511,6 +512,9 @@ def test_testutils_geobox():
     ("file:///foo/bar/file.txt", True),
     ("test.bar", False),
     ("s3://mybucket/objname.tiff", True),
+    ("gs://mybucket/objname.tiff", True),
+    ("wasb://mybucket/objname.tiff", True),
+    ("wasbs://mybucket/objname.tiff", True),
     ("ftp://host.name/filename.txt", True),
     ("https://host.name.com/path/file.txt", True),
     ("http://host.name.com/path/file.txt", True),