Repository: dcs4cop/xcube-sh

Issue:
Specify nodata value when generating a cube.
It would be really useful to be able to specify a `nodata` value during the `xcube` generation ( as an argument of `CubeConfig`). This would enable to set the nodata values that are set by the user in BYOC collections.

Discussion:

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 66bd980..247c3f6 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -3,10 +3,18 @@
 
 * It is now possible accessing BYOC/BYOD and DEM datasets without
   providing the `time_period` cube configuration parameter. 
-  (#67, #35. #25)
   In case such a dataset has no associated time information,  
   we assume a single time step (size of dimension `time` is one)
-  and assign the query time range to this time step.
+  and assign the query time range to this time step. (#75, #67, #35)
+
+* The cube configuration parameter `band_names` can now be omitted
+  also for BYOC collections. In this case, the returned data cube 
+  will contain all available using their native sample types. (#76)
+
+* Introduced a new cube configuration parameter `band_fill_values` 
+  that can be used to specify the fill value (= no-data value)
+  for either all bands if given as a scalar numbers or for individual bands,
+  if given as list or tuple of numbers. (#34)
 
 
 ## Changes in 0.9.1
diff --git a/xcube_sh/chunkstore.py b/xcube_sh/chunkstore.py
index b26c497..e06df3d 100644
--- a/xcube_sh/chunkstore.py
+++ b/xcube_sh/chunkstore.py
@@ -21,6 +21,7 @@
 
 import itertools
 import json
+import math
 import time
 from abc import abstractmethod, ABCMeta
 from collections import MutableMapping
@@ -575,7 +576,8 @@ class SentinelHubChunkStore(RemoteStore):
         if time_period is not None:
             return super().get_time_ranges()
 
-        if self._cube_config.dataset_name == 'CUSTOM':
+        if self._cube_config.dataset_name is None \
+                or self._cube_config.dataset_name.upper() == 'CUSTOM':
             collection_name = self._cube_config.collection_id
         else:
             collection_name = self._METADATA.dataset_collection_name(
@@ -620,23 +622,44 @@ class SentinelHubChunkStore(RemoteStore):
         return SentinelHub.features_to_time_ranges(features)
 
     def get_band_encoding(self, band_name: str) -> Dict[str, Any]:
-        fill_value = self._METADATA.dataset_band_fill_value(
-            self.cube_config.dataset_name,
-            band_name, default=None
-        )
+
         band_sample_types = self.cube_config.band_sample_types
         if not band_sample_types:
             sample_type = self._METADATA.dataset_band_sample_type(
-                self.cube_config.dataset_name,
-                band_name, default='FLOAT32'
+                self.cube_config.dataset_name, band_name, default='FLOAT32'
             )
         elif isinstance(band_sample_types, (tuple, list)):
             index = self.cube_config.band_names.index(band_name)
             sample_type = band_sample_types[index]
         else:  # isinstance(band_sample_types, str)
             sample_type = band_sample_types
-
-        dtype = self._SAMPLE_TYPE_TO_DTYPE[sample_type]
+        # Convert to sample type name to Zarr dtype value
+        dtype = self._SAMPLE_TYPE_TO_DTYPE.get(sample_type)
+        if dtype is None:
+            raise TypeError(f'Invalid sample type {sample_type!r},'
+                            f' must be one of'
+                            f' {tuple(self._SAMPLE_TYPE_TO_DTYPE.keys())}.')
+
+        band_fill_values = self.cube_config.band_fill_values
+        if band_fill_values is None:
+            fill_value = self._METADATA.dataset_band_fill_value(
+                self.cube_config.dataset_name, band_name, default=None
+            )
+        elif isinstance(band_fill_values, (tuple, list)):
+            index = self.cube_config.band_names.index(band_name)
+            fill_value = band_fill_values[index]
+        else:  # isinstance(band_fill_values, Number)
+            fill_value = band_fill_values
+        # Force fill value to be serializable
+        if isinstance(fill_value, float):
+            fill_value = float(fill_value)
+            if math.isnan(fill_value):
+                fill_value = None
+        elif isinstance(fill_value, int):
+            fill_value = int(fill_value)
+        elif fill_value is not None:
+            raise TypeError(f'Invalid fill value {fill_value!r}, must be'
+                            f' int, float, or None')
         return dict(dtype=dtype,
                     fill_value=fill_value,
                     compressor=dict(id='zlib', level=8),
diff --git a/xcube_sh/config.py b/xcube_sh/config.py
index c6e0ea1..df0386f 100644
--- a/xcube_sh/config.py
+++ b/xcube_sh/config.py
@@ -23,6 +23,7 @@ import math
 import warnings
 from datetime import datetime
 from typing import Tuple, Union, Optional, Sequence, Dict, Any
+from numbers import Number
 
 import pandas as pd
 
@@ -58,8 +59,9 @@ class CubeConfig:
         *dataset_name* must be omitted or set to "CUSTOM".
     :param band_names: Optional sequence of band names. If omitted (=None)
         all bands are included.
-    :param band_units: Band units. Optional.
+    :param band_fill_values: Band fill values. Optional.
     :param band_sample_types: Band sample types. Optional.
+    :param band_units: Band units. Optional.
     :param tile_size: Tile size as tuple (width, height). Optional.
     :param chunk_size: Deprecated. Use *tile_size*.
     :param bbox: tuple of 4 numbers: (x1, y1, x2, y2)
@@ -93,8 +95,9 @@ class CubeConfig:
     def __init__(self,
                  dataset_name: str = None,
                  band_names: Sequence[str] = None,
-                 band_units: Union[str, Sequence[str]] = None,
                  band_sample_types: Union[str, Sequence[str]] = None,
+                 band_fill_values: Union[Number, Sequence[Number]] = None,
+                 band_units: Union[str, Sequence[str]] = None,
                  tile_size: Union[str, Tuple[int, int]] = None,
                  chunk_size: Union[str, Tuple[int, int]] = None,
                  bbox: Bbox = None,
@@ -129,7 +132,7 @@ class CubeConfig:
             assert_given(collection_id, 'collection_id')
             dataset_name = 'CUSTOM'
         if collection_id:
-            assert_true(dataset_name == 'CUSTOM',
+            assert_true(dataset_name.upper() == 'CUSTOM',
                         'dataset_name must be "CUSTOM"')
 
         assert_given(spatial_res, 'spatial_res')
@@ -246,8 +249,9 @@ class CubeConfig:
         self._dataset_name = dataset_name
         self._band_names = tuple(band_names) \
             if band_names is not None else None
-        self._band_units = band_units or None
+        self._band_fill_values = band_fill_values
         self._band_sample_types = band_sample_types or None
+        self._band_units = band_units or None
         self._bbox = bbox
         self._spatial_res = spatial_res
         self._crs = crs
@@ -306,8 +310,9 @@ class CubeConfig:
             if self.time_tolerance else None
         return dict(dataset_name=self.dataset_name,
                     band_names=self.band_names,
-                    band_units=self.band_units,
+                    band_fill_values=self.band_fill_values,
                     band_sample_types=self.band_sample_types,
+                    band_units=self.band_units,
                     tile_size=self.tile_size,
                     bbox=self.bbox,
                     spatial_res=self.spatial_res,
@@ -334,14 +339,18 @@ class CubeConfig:
     def band_names(self) -> Optional[Tuple[str, ...]]:
         return self._band_names
 
-    @property
-    def band_units(self) -> Union[None, str, Sequence[str]]:
-        return self._band_units
-
     @property
     def band_sample_types(self) -> Union[None, str, Sequence[str]]:
         return self._band_sample_types
 
+    @property
+    def band_fill_values(self) -> Union[None, Number, Sequence[Number]]:
+        return self._band_fill_values
+
+    @property
+    def band_units(self) -> Union[None, str, Sequence[str]]:
+        return self._band_units
+
     @property
     def crs(self) -> str:
         return self._crs
diff --git a/xcube_sh/metadata.py b/xcube_sh/metadata.py
index 6659ce2..d92ae68 100644
--- a/xcube_sh/metadata.py
+++ b/xcube_sh/metadata.py
@@ -92,7 +92,7 @@ class SentinelHubMetadata:
         return band.get('fill_value', default) if band else default
 
     def _dataset_direct(self, dataset_name: str) -> Optional[Dict]:
-        return self._metadata['datasets'].get(dataset_name)
+        return self._metadata['datasets'].get(dataset_name.upper())
 
     def _dataset_bands_direct(self, dataset_name: str) -> Optional[Dict]:
         dataset = self._dataset_direct(dataset_name)
diff --git a/xcube_sh/store.py b/xcube_sh/store.py
index 35ab388..8c04abb 100644
--- a/xcube_sh/store.py
+++ b/xcube_sh/store.py
@@ -23,7 +23,6 @@ from typing import Iterator, Tuple, Optional, Dict, Any, Union, Container
 
 import xarray as xr
 import zarr
-
 from xcube.core.store import DATASET_TYPE
 from xcube.core.store import DataDescriptor
 from xcube.core.store import DataOpener
@@ -41,6 +40,7 @@ from xcube.util.jsonschema import JsonIntegerSchema
 from xcube.util.jsonschema import JsonNumberSchema
 from xcube.util.jsonschema import JsonObjectSchema
 from xcube.util.jsonschema import JsonStringSchema
+
 from .chunkstore import SentinelHubChunkStore
 from .config import CubeConfig
 from .constants import CRS_ID_TO_URI
@@ -214,27 +214,41 @@ class SentinelHubDataOpener(DataOpener):
         cube_params = dict(
             dataset_name=JsonStringSchema(min_length=1),
             variable_names=JsonArraySchema(
-                items=JsonStringSchema(
-                    enum=dsd.data_vars.keys()
-                    if dsd and dsd.data_vars
-                    else None)),
-            variable_units=JsonArraySchema(),
-            variable_sample_types=JsonArraySchema(),
-            tile_size=JsonArraySchema(items=(
-                JsonNumberSchema(minimum=1,
-                                 maximum=2500,
-                                 default=DEFAULT_TILE_SIZE),
-                JsonNumberSchema(minimum=1,
-                                 maximum=2500,
-                                 default=DEFAULT_TILE_SIZE)
+                # Note, in xcube-sh < 0.9.2 formerly used an enum here.
+                # However, that doesn't work for BYOC, so we omit that
+                # limitation.
+                items=JsonStringSchema()
+            ),
+            variable_fill_values=JsonArraySchema(
+                items=JsonNumberSchema(nullable=True)
+            ),
+            variable_sample_types=JsonArraySchema(
+                items=JsonStringSchema()
+            ),
+            variable_units=JsonArraySchema(
+                items=JsonStringSchema()
+            ),
+            tile_size=JsonArraySchema(
+                items=(
+                    JsonNumberSchema(minimum=1,
+                                     maximum=2500,
+                                     default=DEFAULT_TILE_SIZE),
+                    JsonNumberSchema(minimum=1,
+                                     maximum=2500,
+                                     default=DEFAULT_TILE_SIZE)
+                ),
+                default=(DEFAULT_TILE_SIZE, DEFAULT_TILE_SIZE)
+            ),
+            crs=JsonStringSchema(
+                default=DEFAULT_CRS,
+                enum=CRS_ID_TO_URI.keys()
+            ),
+            bbox=JsonArraySchema(
+                items=(JsonNumberSchema(),
+                       JsonNumberSchema(),
+                       JsonNumberSchema(),
+                       JsonNumberSchema())
             ),
-                default=(DEFAULT_TILE_SIZE, DEFAULT_TILE_SIZE)),
-            crs=JsonStringSchema(default=DEFAULT_CRS,
-                                 enum=CRS_ID_TO_URI.keys()),
-            bbox=JsonArraySchema(items=(JsonNumberSchema(),
-                                        JsonNumberSchema(),
-                                        JsonNumberSchema(),
-                                        JsonNumberSchema())),
             spatial_res=JsonNumberSchema(exclusive_minimum=0.0),
             upsampling=JsonStringSchema(
                 default=DEFAULT_RESAMPLING,
@@ -248,9 +262,11 @@ class SentinelHubDataOpener(DataOpener):
                 default=DEFAULT_MOSAICKING_ORDER,
                 enum=MOSAICKING_ORDERS
             ),
-            time_range=JsonDateSchema.new_range(min_date=min_date,
-                                                max_date=max_date,
-                                                nullable=True),
+            time_range=JsonDateSchema.new_range(
+                min_date=min_date,
+                max_date=max_date,
+                nullable=True
+            ),
             # TODO: add pattern
             time_period=JsonStringSchema(
                 default='1D', nullable=True,
@@ -263,8 +279,7 @@ class SentinelHubDataOpener(DataOpener):
                 format='^([1-9]*[0-9]*)[NULSTH]$'
             ),
             collection_id=JsonStringSchema(),
-            four_d=JsonBooleanSchema(
-                default=False),
+            four_d=JsonBooleanSchema(default=False),
         )
         cache_params = dict(
             max_cache_size=JsonIntegerSchema(minimum=0),
@@ -292,11 +307,12 @@ class SentinelHubDataOpener(DataOpener):
         )
 
     def _describe_data(self, data_id: str) -> DatasetDescriptor:
+        assert_not_none(data_id, 'data_id')
         dataset_metadata, collection_metadata = \
             self._get_dataset_and_collection_metadata(data_id)
         bands = dict(dataset_metadata.get('bands', {}))
 
-        if self._sentinel_hub is not None and data_id != 'CUSTOM':
+        if self._sentinel_hub is not None and data_id.upper() != 'CUSTOM':
             # If we are connected to the API, we return band names by API
             remote_bands = self._sentinel_hub.bands(data_id)
             if remote_bands:


Test patch:
diff --git a/test/test_chunkstore.py b/test/test_chunkstore.py
index 9559bfc..d525b82 100644
--- a/test/test_chunkstore.py
+++ b/test/test_chunkstore.py
@@ -29,7 +29,6 @@ import numpy as np
 import pandas as pd
 import xarray as xr
 import zarr
-
 from xcube_sh.chunkstore import SentinelHubChunkStore
 from xcube_sh.config import CubeConfig
 from xcube_sh.metadata import S2_BAND_NAMES
@@ -261,6 +260,48 @@ class SentinelHubStore3DTestWithTiles(SentinelHubStoreTest):
             sorted(list(self.observed_kwargs.keys())))
 
 
+# noinspection PyTypeChecker
+class SentinelHubStore3DTestWithFillValue(unittest.TestCase):
+    default_kwargs = dict(
+        dataset_name='S2L1C',
+        band_names=['B01', 'B08', 'B12'],
+        bbox=(10.2, 53.5, 10.3, 53.6),
+        spatial_res=0.1 / 4000,
+        time_range=('2017-08-01', '2017-08-31'),
+        time_period='1D'
+    )
+
+    def test_all_zero(self):
+        cube_config = CubeConfig(band_fill_values=0,
+                                 **self.default_kwargs)
+        self.store = SentinelHubChunkStore(SentinelHubMock(cube_config),
+                                           cube_config)
+        cube = xr.open_zarr(self.store)
+        self.assertEqual(0, cube.B01.encoding.get('_FillValue'))
+        self.assertEqual(0, cube.B08.encoding.get('_FillValue'))
+        self.assertEqual(0, cube.B12.encoding.get('_FillValue'))
+
+    def test_individual(self):
+        cube_config = CubeConfig(band_fill_values=[1, 2, 3],
+                                 **self.default_kwargs)
+        self.store = SentinelHubChunkStore(SentinelHubMock(cube_config),
+                                           cube_config)
+        cube = xr.open_zarr(self.store)
+        self.assertEqual(1, cube.B01.encoding.get('_FillValue'))
+        self.assertEqual(2, cube.B08.encoding.get('_FillValue'))
+        self.assertEqual(3, cube.B12.encoding.get('_FillValue'))
+
+    def test_individual_nan(self):
+        cube_config = CubeConfig(band_fill_values=[np.nan, np.nan, np.nan],
+                                 **self.default_kwargs)
+        self.store = SentinelHubChunkStore(SentinelHubMock(cube_config),
+                                           cube_config)
+        cube = xr.open_zarr(self.store)
+        self.assertEqual(None, cube.B01.encoding.get('_FillValue'))
+        self.assertEqual(None, cube.B08.encoding.get('_FillValue'))
+        self.assertEqual(None, cube.B12.encoding.get('_FillValue'))
+
+
 class SentinelHubStore4DTest(SentinelHubStoreTest):
     def get_cube_config(self):
         return CubeConfig(dataset_name='S2L1C',
diff --git a/test/test_config.py b/test/test_config.py
index 773d218..9b05135 100644
--- a/test/test_config.py
+++ b/test/test_config.py
@@ -135,6 +135,7 @@ class CubeConfigTest(unittest.TestCase):
     def test_from_and_to_dict(self):
         config = CubeConfig.from_dict(dict(dataset_name='S2L2A',
                                            band_names=('B01', 'B02', 'B03'),
+                                           band_fill_values=0,
                                            bbox=(10.11, 54.17, 10.14, 54.19),
                                            spatial_res=0.00001,
                                            tile_size=(512, 512),
@@ -142,6 +143,7 @@ class CubeConfigTest(unittest.TestCase):
         expected = {
             'band_names': ('B01', 'B02', 'B03'),
             'band_sample_types': None,
+            'band_fill_values': 0,
             'band_units': None,
             'collection_id': None,
             'crs': 'WGS84',