Repository: dcs4cop/xcube (python)

Issue:
Default input processor
We currently must specify a class name for the input processor to be used for each input dataset.

If datasets are already in xcube's "standard format", that is:

* Have dimensions `lat`, `lon`, optionally `time` of length 1;
* have coordinate variables `lat[lat]`, `lon[lat]`, `time[time]` (opt.), `time_bnds[time, 2]` (opt.);
* have any data variables of form `<var>[time, lat, lon]` or `<var>[lat, lon]` if `time` coordinate variable is missing;
* have global attribute pair `time_coverage_start`, `time_coverage_end` (or pair `time_start`, `time_stop`, and others) if `time` variable coordinate is missing,

then we could use a default input processor that would be configurable w.r.t. the variables to be processed, and which mask to apply, etc.




Pull request:
Forman fix reprojection of wgs84 sources
This will close #1

Patch:
diff --git a/xcube/genl2c/default/inputprocessor.py b/xcube/genl2c/default/inputprocessor.py
index 72ce4881..7ae5d416 100644
--- a/xcube/genl2c/default/inputprocessor.py
+++ b/xcube/genl2c/default/inputprocessor.py
@@ -67,12 +67,15 @@ class DefaultInputProcessor(InputProcessor, metaclass=ABCMeta):
 
     def pre_process(self, dataset: xr.Dataset) -> xr.Dataset:
         self._validate(dataset)
+        if "time" in dataset:
+            return dataset.squeeze("time")
         return dataset
 
     def get_reprojection_info(self, dataset: xr.Dataset) -> ReprojectionInfo:
         return ReprojectionInfo(xy_var_names=('lon', 'lat'),
                                 xy_crs=CRS_WKT_EPSG_4326,
-                                xy_gcp_step=1)
+                                xy_gcp_step=(max(1, len(dataset.lon) // 4),
+                                             max(1, len(dataset.lat) // 4)))
 
     def get_time_range(self, dataset: xr.Dataset) -> Tuple[float, float]:
         time_coverage_start, time_coverage_end = None, None
@@ -113,10 +116,10 @@ class DefaultInputProcessor(InputProcessor, metaclass=ABCMeta):
         return time_start, time_stop
 
     def _validate(self, dataset):
-        self._check_coordinate_var(dataset, "lon", 2)
-        self._check_coordinate_var(dataset, "lat", 2)
+        self._check_coordinate_var(dataset, "lon", min_length=2)
+        self._check_coordinate_var(dataset, "lat", min_length=2)
         if "time" in dataset.dims:
-            self._check_coordinate_var(dataset, "time", 1)
+            self._check_coordinate_var(dataset, "time", max_length=1)
             required_dims = ("time", "lat", "lon")
         else:
             required_dims = ("lat", "lon")
@@ -128,7 +131,8 @@ class DefaultInputProcessor(InputProcessor, metaclass=ABCMeta):
         if count == 0:
             raise ValueError(f"dataset has no variables with required dimensions {required_dims!r}")
 
-    def _check_coordinate_var(self, dataset: xr.Dataset, coord_var_name: str, min_length: int):
+    def _check_coordinate_var(self, dataset: xr.Dataset, coord_var_name: str,
+                              min_length: int = None, max_length: int = None):
         if coord_var_name not in dataset.coords:
             raise ValueError(f'missing coordinate variable "{coord_var_name}"')
         coord_var = dataset.coords[coord_var_name]
@@ -140,8 +144,11 @@ class DefaultInputProcessor(InputProcessor, metaclass=ABCMeta):
             expected_shape = (len(coord_var), 2)
             if coord_bnds_var.shape != expected_shape:
                 raise ValueError(f'coordinate bounds variable "{coord_bnds_var}" must have shape {expected_shape!r}')
-        elif len(coord_var) < min_length:
-            raise ValueError(f'coordinate variable "{coord_var_name}" must have at least {min_length} value(s)')
+        else:
+            if min_length is not None and len(coord_var) < min_length:
+                raise ValueError(f'coordinate variable "{coord_var_name}" must have at least {min_length} value(s)')
+            if max_length is not None and len(coord_var) > max_length:
+                raise ValueError(f'coordinate variable "{coord_var_name}" must have no more than {max_length} value(s)')
 
 
 def init_plugin():
diff --git a/xcube/genl2c/inputprocessor.py b/xcube/genl2c/inputprocessor.py
index 4a22d469..124bd8ca 100644
--- a/xcube/genl2c/inputprocessor.py
+++ b/xcube/genl2c/inputprocessor.py
@@ -115,7 +115,7 @@ class InputProcessor(metaclass=ABCMeta):
 
     def pre_process(self, dataset: xr.Dataset) -> xr.Dataset:
         """
-        Do any pre-processing before reprojection.
+        Do any pre-processing before reprojection. The output must be a 2D array.
         For example, perform dataset validation, masking, and/or filtering using provided configuration parameters.
         The default implementation returns the unchanged *dataset*.
         :param dataset: The dataset.
@@ -125,7 +125,7 @@ class InputProcessor(metaclass=ABCMeta):
 
     def post_process(self, dataset: xr.Dataset) -> xr.Dataset:
         """
-        Do any post-processing after reprojection.
+        Do any post-processing after reprojection. The input is a 3D array with dimensions ("time", "lat", "lon").
         For example, generate new "wavelength" dimension for variables whose name follow a certain pattern.
         The default implementation returns the unchanged *dataset*.
         :param dataset: The dataset.
diff --git a/xcube/reproject.py b/xcube/reproject.py
index e0669632..1a3a339a 100644
--- a/xcube/reproject.py
+++ b/xcube/reproject.py
@@ -110,12 +110,14 @@ def reproject_to_wgs84(src_dataset: xr.Dataset,
     _assert(x_name in src_dataset)
     _assert(y_name in src_dataset)
     x_var = src_dataset[x_name]
+    y_var = src_dataset[y_name]
+    if len(x_var.dims) == 1 and len(y_var.dims) == 1:
+        y_var, x_var = xr.broadcast(y_var, x_var)
     _assert(len(x_var.dims) == 2)
+    _assert(y_var.dims == x_var.dims)
     _assert(x_var.shape[-1] >= 2)
     _assert(x_var.shape[-2] >= 2)
-    y_var = src_dataset[y_name]
     _assert(y_var.shape == x_var.shape)
-    _assert(y_var.dims == x_var.dims)
 
     src_width = x_var.shape[-1]
     src_height = x_var.shape[-2]


Test patch:
diff --git a/test/genl2c/default/test_inputprocessor.py b/test/genl2c/default/test_inputprocessor.py
index 703b6e54..55627710 100644
--- a/test/genl2c/default/test_inputprocessor.py
+++ b/test/genl2c/default/test_inputprocessor.py
@@ -49,7 +49,16 @@ class DefaultInputProcessorTest(unittest.TestCase):
         self.assertEqual("invalid input: missing time coverage information in dataset", f"{cm.exception}")
 
     def test_pre_process(self):
-        ds1 = create_default_dataset()
+        ds1 = create_default_dataset(time_mode="time")
+        ds2 = self.processor.pre_process(ds1)
+        self.assertIsNot(ds1, ds2)
+        ds1 = create_default_dataset(time_mode="time_bnds")
+        ds2 = self.processor.pre_process(ds1)
+        self.assertIsNot(ds1, ds2)
+        ds1 = create_default_dataset(time_mode="time_coverage")
+        ds2 = self.processor.pre_process(ds1)
+        self.assertIs(ds1, ds2)
+        ds1 = create_default_dataset(time_mode="start_stop_time")
         ds2 = self.processor.pre_process(ds1)
         self.assertIs(ds1, ds2)