Repository: dcs4cop/xcube (python)

Issue:
False cube creation when output name is not set using xcube-genl2c
When the user does not set an output name for the generated data cube when using xcube-genl2c, the output is one output file per input files. This is because the default output name depends on the input file, which changes if there is more than one input file used for the cube generation. The default name should be set to a not changing name. 

Pull request:
Forman 18 xcube verify
Closes #19.

@AliceBalfanz, @dzelge  in the future, please apply the new tool `xcube verify INPUT` on each generated data cube.

@gunbra32, FYI.

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 59eb8efb..415165d9 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,6 +2,7 @@
 
 ### Enhancements
 
+* Added new CLI command `xcube verify` performing data cube verification. (#19)
 * Reworked `xcube extract` to be finally useful and effective for point data extraction. (#102) 
 * `xcube server`can now filter datasets by point coordinate, e.g. `/datasets?point=12.5,52.8`. (#50) 
 * `xcube server`can now limit time series to a maximum number of 
diff --git a/README.md b/README.md
index f99c7232..4b31bce1 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,7 @@ Data cubes with [xarray](http://xarray.pydata.org/).
   - [`xcube resample`](#xcube-resample)
   - [`xcube serve`](#xcube-serve)
   - [`xcube vars2dim`](#xcube-vars2dim)
+  - [`xcube verify`](#xcube-verify)
 
 
 # Installation
@@ -118,8 +119,12 @@ To start a demo using docker use the following commands
       gen       Generate data cube.
       grid      Find spatial data cube resolutions and adjust bounding boxes.
       level     Generate multi-resolution levels.
+      prune     Delete empty chunks.
+      resample  Resample data along the time dimension.
       serve     Serve data cubes via web service.
       vars2dim  Convert cube variables into new dimension.
+      verify    Perform cube verification.
+
     
 
 ## `xcube chunk`
@@ -779,3 +784,27 @@ in the [demo's HTML file](https://github.com/dcs4cop/xcube/blob/master/xcube/web
 * Feature: WMTS GetFeatureInfo
 * Feature: collect Path entry of any Dataset and observe if the file are modified, if so remove dataset from cache
   to force its reopening.
+
+
+## `xcube verify`
+
+Perform cube verification.
+
+    Usage: xcube verify [OPTIONS] INPUT
+    
+      Perform cube verification.
+    
+      The tool verifies that INPUT
+      * defines the dimensions "time", "lat", "lon";
+      * has corresponding "time", "lat", "lon" coordinate variables and that they
+        are valid, e.g. 1-D, non-empty, using correct units;
+      * has valid  bounds variables for "time", "lat", "lon" coordinate
+        variables, if any;
+      * has any data variables and that they are valid, e.g. min. 3-D, all have
+        same dimensions, have at least dimensions "time", "lat", "lon".
+    
+      If INPUT is a valid data cube, the tool returns exit code 0. Otherwise a
+      violation report is written to stdout and the tool returns exit code 3.
+    
+    Options:
+      --help  Show this message and exit.
diff --git a/xcube/api/verify.py b/xcube/api/verify.py
index b1c5842a..34f2f7fc 100644
--- a/xcube/api/verify.py
+++ b/xcube/api/verify.py
@@ -1,3 +1,24 @@
+# The MIT License (MIT)
+# Copyright (c) 2019 by the xcube development team and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
 from typing import List
 
 import numpy as np
@@ -24,6 +45,15 @@ def verify_cube(dataset: xr.Dataset) -> List[str]:
     """
     Verify the given *dataset* for being a valid data cube.
 
+    The tool verifies that *dataset*
+    * defines the dimensions "time", "lat", "lon";
+    * has corresponding "time", "lat", "lon" coordinate variables and that they
+      are valid, e.g. 1-D, non-empty, using correct units;
+    * has valid  bounds variables for "time", "lat", "lon" coordinate
+      variables, if any;
+    * has any data variables and that they are valid, e.g. min. 3-D, all have
+      same dimensions, have at least dimensions "time", "lat", "lon".
+
     Returns a list of issues, which is empty if *dataset* is a valid data cube.
 
     :param dataset: A dataset to be verified.
diff --git a/xcube/cli/cli.py b/xcube/cli/cli.py
index d0c92a0e..84c6dea0 100644
--- a/xcube/cli/cli.py
+++ b/xcube/cli/cli.py
@@ -10,6 +10,7 @@ from xcube.cli.prune import prune
 from xcube.cli.resample import resample
 from xcube.cli.serve import serve
 from xcube.cli.timeit import timeit
+from xcube.cli.verify import verify
 from xcube.util.cliutil import parse_cli_kwargs, new_cli_ctx_obj, handle_cli_exception, cli_option_traceback, \
     cli_option_scheduler
 from xcube.version import version
@@ -213,6 +214,7 @@ cli.add_command(resample)
 cli.add_command(serve)
 cli.add_command(timeit)
 cli.add_command(vars2dim)
+cli.add_command(verify)
 
 
 def main(args=None):
diff --git a/xcube/cli/verify.py b/xcube/cli/verify.py
new file mode 100644
index 00000000..ad051fbd
--- /dev/null
+++ b/xcube/cli/verify.py
@@ -0,0 +1,64 @@
+# The MIT License (MIT)
+# Copyright (c) 2019 by the xcube development team and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import sys
+
+import click
+
+
+# noinspection PyShadowingBuiltins
+@click.command(name='verify')
+@click.argument('input')
+def verify(input):
+    """
+    Perform cube verification.
+
+    \b
+    The tool verifies that INPUT
+    * defines the dimensions "time", "lat", "lon";
+    * has corresponding "time", "lat", "lon" coordinate variables and that they
+      are valid, e.g. 1-D, non-empty, using correct units;
+    * has valid  bounds variables for "time", "lat", "lon" coordinate
+      variables, if any;
+    * has any data variables and that they are valid, e.g. min. 3-D, all have
+      same dimensions, have at least dimensions "time", "lat", "lon".
+
+    If INPUT is a valid data cube, the tool returns exit code 0.
+    Otherwise a violation report is written to stdout and the tool returns exit code 3.
+    """
+    return _verify(input_path=input, monitor=print)
+
+
+def _verify(input_path: str = None, monitor=None):
+    from xcube.api import open_dataset, verify_cube
+
+    monitor(f'Opening cube from {input_path!r}...')
+    with open_dataset(input_path) as cube:
+        report = verify_cube(cube)
+
+    if not report:
+        monitor("INPUT is a valid cube.")
+        return
+
+    monitor('INPUT is not a valid cube due to the following reasons:')
+    monitor('- ' + '\n- '.join(report))
+    # According to http://tldp.org/LDP/abs/html/exitcodes.html, exit code 3 is not reserved
+    sys.exit(3)


Test patch:
diff --git a/test/cli/test_verify.py b/test/cli/test_verify.py
new file mode 100644
index 00000000..da08f08d
--- /dev/null
+++ b/test/cli/test_verify.py
@@ -0,0 +1,52 @@
+import numpy as np
+import xarray as xr
+
+from test.cli.helpers import CliTest
+from xcube.api import new_cube
+from xcube.api.readwrite import write_cube
+from xcube.util.dsio import rimraf
+
+
+class VerifyTest(CliTest):
+
+    def test_help_option(self):
+        result = self.invoke_cli(['verify', '--help'])
+        self.assertEqual(0, result.exit_code)
+
+
+class VerifyDataTest(CliTest):
+    TEST_CUBE = "test.zarr"
+
+    def setUp(self) -> None:
+        rimraf(self.TEST_CUBE)
+
+    def tearDown(self) -> None:
+        rimraf(self.TEST_CUBE)
+
+    def test_verify_ok(self):
+        cube = new_cube(variables=dict(precipitation=0.5))
+        write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
+        result = self.invoke_cli(['verify', self.TEST_CUBE])
+        self.assertEqual(0, result.exit_code)
+        self.assertEqual("Opening cube from 'test.zarr'...\n"
+                         "INPUT is a valid cube.\n",
+                         result.stdout)
+
+    def test_verify_failure(self):
+        cube = new_cube(variables=dict(precipitation=0.5))
+        cube["chl"] = xr.DataArray(np.random.rand(cube.dims["lat"], cube.dims["lon"]),
+                                   dims=("lat", "lon"),
+                                   coords=dict(lat=cube.lat, lon=cube.lon))
+        write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
+
+        result = self.invoke_cli(['verify', self.TEST_CUBE])
+        self.assertEqual(3, result.exit_code)
+        self.assertEqual("Opening cube from 'test.zarr'...\n"
+                         "INPUT is not a valid cube due to the following reasons:\n"
+                         "- dimensions of data variable 'chl' must be ('time', ..., 'lat', 'lon'),"
+                         " but were ('lat', 'lon') for 'chl'\n"
+                         "- dimensions of all data variables must be same, but found ('lat', 'lon')"
+                         " for 'chl' and ('time', 'lat', 'lon') for 'precipitation'\n"
+                         "- all data variables must have same chunk sizes, but found ((90, 90), (360,))"
+                         " for 'chl' and ((3, 2), (90, 90), (180, 180)) for 'precipitation'\n",
+                         result.stdout)