Repository: dcs4cop/xcube

Issue:
Writing to zarr fails with message "specified zarr chunks would overlap multiple dask chunks"
For some data it is necessary to rechunk it in order to deal with it efficiently. This rechunking process might result in that the final chunk of a dimension has a smaller size than the previous ones. E.g., if we have a dimension of 500, originally split into ten chunks of size 50, we can rechunk it to sizes of 200, 200, and 100. This is perfectly valid and supported by xarray.
However, when the affected dimension/variable is latitude and the latitude is ascending, the current implementation of the normalization in xcube will revert the order of chunks and cause that the smaller chunk will be at the start - which is not supported by xarray and might result in an error as the one in the description, for example when writing to zarr.

For the time being, I will work around this issue by ensuring that all chunks are of the same size after rechunking, which will make this issue a bit harder to reproduce. I open this issue to document that the current solution of dealing with ascending latitudes is not optimal. The best solution will probably be to get rid of the part where latitudes are reverted during the normalization and support ascending latitudes in the data.

See also https://github.com/dcs4cop/xcube/issues/251 and https://github.com/dcs4cop/xcube/issues/327.

Discussion:

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index be15df9a..df4ef834 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,7 +1,11 @@
-## Changes in 0.7.1.dev1 (in development)
-
-* Added `s3fs`  requirement that has been removed by accident.
-* Added missing requirements `requests` and `urllib3`.
+## Changes in 0.7.1
+
+* Dataset normalisation no longer includes reordering increasing
+  latitude coordinates, as this creates datasets that are no longer writable 
+  to Zarr. (#347)
+* Updated package requirements
+  - Added `s3fs`  requirement that has been removed by accident.
+  - Added missing requirements `requests` and `urllib3`.
 
 ## Changes in 0.7.0
 
diff --git a/xcube/core/normalize.py b/xcube/core/normalize.py
index 9ac3e503..d0a39570 100644
--- a/xcube/core/normalize.py
+++ b/xcube/core/normalize.py
@@ -56,34 +56,11 @@ def normalize_dataset(ds: xr.Dataset) -> xr.Dataset:
     ds = _normalize_lat_lon_2d(ds)
     ds = _normalize_dim_order(ds)
     ds = _normalize_lon_360(ds)
-
-    # xcube viewer currently requires decreasing latitude co-ordinates, so
-    # we invert them here if necessary.
-    ds = _ensure_lat_decreasing(ds)
-
     ds = normalize_missing_time(ds)
     ds = _normalize_jd2datetime(ds)
     return ds
 
 
-def _ensure_lat_decreasing(ds: xr.Dataset) -> xr.Dataset:
-    """
-    If the latitude is increasing, invert it to make it decreasing.
-    :param ds: some xarray dataset
-    :return: a normalized xarray dataset
-    """
-    try:
-        if not _is_lat_decreasing(ds.lat):
-            ds = ds.sel(lat=slice(None, None, -1))
-    except AttributeError:
-        # The dataset doesn't have 'lat', probably not geospatial
-        pass
-    except ValueError:
-        # The dataset still has an ND 'lat' array
-        pass
-    return ds
-
-
 def _normalize_lat_lon(ds: xr.Dataset) -> xr.Dataset:
     """
     Rename variables named 'longitude' or 'long' to 'lon', and 'latitude' to 'lon'.
@@ -366,7 +343,7 @@ def normalize_missing_time(ds: xr.Dataset) -> xr.Dataset:
     return ds
 
 
-def adjust_spatial_attrs(ds: xr.Dataset, allow_point: bool=False) -> xr.Dataset:
+def adjust_spatial_attrs(ds: xr.Dataset, allow_point: bool = False) -> xr.Dataset:
     """
     Adjust the global spatial attributes of the dataset by doing some
     introspection of the dataset and adjusting the appropriate attributes
@@ -692,7 +669,6 @@ def _is_lat_decreasing(lat: xr.DataArray) -> bool:
 
 
 def _normalize_dim_order(ds: xr.Dataset) -> xr.Dataset:
-
     copy_created = False
 
     for var_name in ds.data_vars:
diff --git a/xcube/version.py b/xcube/version.py
index 2a007fc7..b30ba38b 100644
--- a/xcube/version.py
+++ b/xcube/version.py
@@ -19,4 +19,4 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-version = '0.7.1.dev1'
+version = '0.7.1'


Test patch:
diff --git a/test/core/test_normalize.py b/test/core/test_normalize.py
index 3f0b1099..37441a6c 100644
--- a/test/core/test_normalize.py
+++ b/test/core/test_normalize.py
@@ -12,11 +12,11 @@ import xarray as xr
 from jdcal import gcal2jd
 from numpy.testing import assert_array_almost_equal
 
+from xcube.core.normalize import adjust_spatial_attrs
+from xcube.core.normalize import adjust_temporal_attrs
 from xcube.core.normalize import normalize_coord_vars
 from xcube.core.normalize import normalize_dataset
 from xcube.core.normalize import normalize_missing_time
-from xcube.core.normalize import adjust_spatial_attrs
-from xcube.core.normalize import adjust_temporal_attrs
 
 
 # noinspection PyPep8Naming
@@ -85,25 +85,23 @@ class TestNormalize(TestCase):
         dataset = xr.Dataset({'first': (['latitude',
                                          'longitude'], [[1, 2, 3],
                                                         [2, 3, 4]])})
-        # Since normalization puts latitudes into descending order, we
-        # expect the rows to be swapped.
-        expected = xr.Dataset({'first': (['lat', 'lon'], [[2, 3, 4],
-                                                          [1, 2, 3]])})
+        expected = xr.Dataset({'first': (['lat', 'lon'], [[1, 2, 3],
+                                                          [2, 3, 4]])})
         actual = normalize_dataset(dataset)
         assertDatasetEqual(actual, expected)
 
         dataset = xr.Dataset({'first': (['lat', 'long'], [[1, 2, 3],
                                                           [2, 3, 4]])})
-        expected = xr.Dataset({'first': (['lat', 'lon'], [[2, 3, 4],
-                                                          [1, 2, 3]])})
+        expected = xr.Dataset({'first': (['lat', 'lon'], [[1, 2, 3],
+                                                          [2, 3, 4]])})
         actual = normalize_dataset(dataset)
         assertDatasetEqual(actual, expected)
 
         dataset = xr.Dataset({'first': (['latitude',
                                          'spacetime'], [[1, 2, 3],
                                                         [2, 3, 4]])})
-        expected = xr.Dataset({'first': (['lat', 'spacetime'], [[2, 3, 4],
-                                                                [1, 2, 3]])})
+        expected = xr.Dataset({'first': (['lat', 'spacetime'], [[1, 2, 3],
+                                                                [2, 3, 4]])})
         actual = normalize_dataset(dataset)
         assertDatasetEqual(actual, expected)
 
@@ -114,7 +112,7 @@ class TestNormalize(TestCase):
         actual = normalize_dataset(dataset)
         assertDatasetEqual(actual, expected)
 
-    def test_normalize_inverted_lat(self):
+    def test_normalize_does_not_reorder_increasing_lat(self):
         first = np.zeros([3, 45, 90])
         first[0, :, :] = np.eye(45, 90)
         ds = xr.Dataset({
@@ -125,18 +123,8 @@ class TestNormalize(TestCase):
             'time': [datetime(2000, x, 1) for x in range(1, 4)]}).chunk(
             chunks={'time': 1})
 
-        first = np.zeros([3, 45, 90])
-        first[0, :, :] = np.flip(np.eye(45, 90), axis=0)
-        expected = xr.Dataset({
-            'first': (['time', 'lat', 'lon'], first),
-            'second': (['time', 'lat', 'lon'], np.zeros([3, 45, 90])),
-            'lat': np.linspace(88, -88, 45),
-            'lon': np.linspace(-178, 178, 90),
-            'time': [datetime(2000, x, 1) for x in range(1, 4)]}).chunk(
-            chunks={'time': 1})
-
         actual = normalize_dataset(ds)
-        xr.testing.assert_equal(actual, expected)
+        xr.testing.assert_equal(actual, ds)
 
     def test_normalize_with_missing_time_dim(self):
         ds = xr.Dataset({'first': (['lat', 'lon'], np.zeros([90, 180])),