Repository: dcs4cop/xcube
Pull request #17 (2019-01-24 13:40:28)

New Xcube CLI and API
* New top-level CLI `xcube` implemented in module `xcube.cli` with first two commands 
  * `chunk` 
  * `dump`
* New initial Python API implemented in module `xcube.api` with functions:
  * `read_dataset(path) -> Dataset`
  * `write_dataset(dataset, path) -> Dataset`
  * `chunk_dataset(dataset) -> Dataset`
  * `dump_dataset(dataset) -> str`
* Extended existing module `xcube.dsio`
  * Function `guess_dataset_format(path) -> str | None`
  * Function `guess_dataset_ios(path) -> [(DatasetIO, float)]`
  * Abstract method `DatasetIO.fitness(path) -> float`




Patch:
diff --git a/environment.yml b/environment.yml
index a8221a2f..5566aadf 100644
--- a/environment.yml
+++ b/environment.yml
@@ -9,6 +9,7 @@ dependencies:
   #
   - boto3 =1.7.51
   - botocore =1.10.51
+  - click =6.7
   - dask=0.17.2
   - gdal=2.2.4
   - matplotlib=2.2.2
diff --git a/setup.py b/setup.py
index 1a1cc63c..33c77f5e 100644
--- a/setup.py
+++ b/setup.py
@@ -26,6 +26,7 @@ from setuptools import setup, find_packages
 
 # in alphabetical oder
 requirements = [
+    'click',
     'dask',
     'gdal',
     'matplotlib',
@@ -53,6 +54,7 @@ setup(
     packages=packages,
     entry_points={
         'console_scripts': [
+            'xcube = xcube.cli:main',
             'xcube-genl2c = xcube.genl2c.cli:main',
             'xcube-genl3 = xcube.genl3.cli:main',
             'xcube-grid = xcube.grid.main:main',
diff --git a/xcube/api.py b/xcube/api.py
new file mode 100644
index 00000000..eaba8a57
--- /dev/null
+++ b/xcube/api.py
@@ -0,0 +1,155 @@
+from contextlib import contextmanager
+from typing import Dict
+
+import xarray as xr
+
+from xcube.dsio import find_dataset_io, guess_dataset_format, FORMAT_NAME_ZARR, FORMAT_NAME_NETCDF4
+
+
+@contextmanager
+def open_dataset(input_path: str,
+                 format_name: str = None,
+                 **kwargs) -> xr.Dataset:
+    """
+    The ``read_dataset`` function as context manager that auto-closes the dataset read.
+
+    :param input_path: input path
+    :param format_name: format, e.g. "zarr" or "netcdf4"
+    :param kwargs: format-specific keyword arguments
+    :return: dataset object
+    """
+    dataset = read_dataset(input_path, format_name, **kwargs)
+    try:
+        yield dataset
+    finally:
+        dataset.close()
+
+
+def read_dataset(input_path: str,
+                 format_name: str = None,
+                 **kwargs) -> xr.Dataset:
+    """
+    Read a dataset.
+
+    :param input_path: input path
+    :param format_name: format, e.g. "zarr" or "netcdf4"
+    :param kwargs: format-specific keyword arguments
+    :return: dataset object
+    """
+    format_name = format_name if format_name else guess_dataset_format(input_path)
+    if format_name is None:
+        raise ValueError("Unknown input format")
+    dataset_io = find_dataset_io(format_name, modes=["r"])
+    if dataset_io is None:
+        raise ValueError(f"Unknown input format {format_name!r} for {input_path}")
+
+    return dataset_io.read(input_path, **kwargs)
+
+
+def write_dataset(dataset: xr.Dataset,
+                  output_path: str,
+                  format_name: str = None,
+                  **kwargs) -> xr.Dataset:
+    """
+    Write dataset.
+
+    :param dataset: Dataset to be written.
+    :param output_path: output path
+    :param format_name: format, e.g. "zarr" or "netcdf4"
+    :param kwargs: format-specific keyword arguments
+    :return: the input dataset
+    """
+    format_name = format_name if format_name else guess_dataset_format(output_path)
+    if format_name is None:
+        raise ValueError("Unknown output format")
+    dataset_io = find_dataset_io(format_name, modes=["w"])
+    if dataset_io is None:
+        raise ValueError(f"Unknown output format {format_name!r} for {output_path}")
+
+    dataset_io.write(dataset, output_path, **kwargs)
+
+    return dataset
+
+
+def chunk_dataset(dataset: xr.Dataset,
+                  chunk_sizes: Dict[str, int] = None,
+                  format_name: str = None) -> xr.Dataset:
+    """
+    Chunk dataset and update encodings for given format.
+
+    :param dataset: input dataset
+    :param chunk_sizes: mapping from dimension name to new chunk size
+    :param format_name: format, e.g. "zarr" or "netcdf4"
+    :return: the re-chunked dataset
+    """
+    chunked_ds = dataset.chunk(chunks=chunk_sizes)
+
+    # Update encoding so writing of chunked_ds recognizes new chunks
+    chunk_sizes_attr_name = None
+    if format_name == FORMAT_NAME_ZARR:
+        chunk_sizes_attr_name = "chunks"
+    if format_name == FORMAT_NAME_NETCDF4:
+        chunk_sizes_attr_name = "chunksizes"
+    if chunk_sizes_attr_name:
+        for var_name in chunked_ds.variables:
+            var = chunked_ds[var_name]
+            if chunk_sizes:
+                sizes = tuple(chunk_sizes[dim_name] if dim_name in chunk_sizes
+                              else var.shape[var.dims.index(dim_name)]
+                              for dim_name in var.dims)
+                var.encoding.update({chunk_sizes_attr_name: sizes})
+            elif chunk_sizes_attr_name in var.encoding:
+                # Remove any explicit and wrong specification so writing will use Dask chunks (TBC!)
+                del var.encoding[chunk_sizes_attr_name]
+
+    return chunked_ds
+
+
+def dump_dataset(dataset: xr.Dataset,
+                 variable_names=None,
+                 show_var_encoding=False) -> str:
+    """
+    Dumps a dataset or variables into a text string.
+
+    :param dataset: input dataset
+    :param variable_names: names of variables to be dumped
+    :param show_var_encoding: also dump variable encodings?
+    :return: the dataset dump
+    """
+    lines = []
+    if not variable_names:
+        lines.append(str(dataset))
+        if show_var_encoding:
+            for var_name, var in dataset.coords.items():
+                if var.encoding:
+                    lines.append(dump_var_encoding(var, header=f"Encoding for coordinate variable {var_name!r}:"))
+            for var_name, var in dataset.data_vars.items():
+                if var.encoding:
+                    lines.append(dump_var_encoding(var, header=f"Encoding for data variable {var_name!r}:"))
+    else:
+        for var_name in variable_names:
+            var = dataset[var_name]
+            lines.append(str(var))
+            if show_var_encoding and var.encoding:
+                lines.append(dump_var_encoding(var))
+    return "\n".join(lines)
+
+
+def dump_var_encoding(var: xr.DataArray, header="Encoding:", indent=4) -> str:
+    """
+    Dump the encoding information of a variable into a text string.
+
+    :param var: Dataset variable.
+    :param header: Title/header string.
+    :param indent: Indention in spaces.
+    :return: the variable dump
+    """
+    lines = [header]
+    max_len = 0
+    for k in var.encoding:
+        max_len = max(max_len, len(k))
+    indent_spaces = indent * " "
+    for k, v in var.encoding.items():
+        tab_spaces = (2 + max_len - len(k)) * " "
+        lines.append(f"{indent_spaces}{k}:{tab_spaces}{v!r}")
+    return "\n".join(lines)
diff --git a/xcube/cli.py b/xcube/cli.py
new file mode 100644
index 00000000..add55d52
--- /dev/null
+++ b/xcube/cli.py
@@ -0,0 +1,101 @@
+from typing import Any, Dict
+
+import click
+
+from .version import version
+
+
+def _parse_kwargs(value: str, metavar: str=None) -> Dict[str, Any]:
+    if value:
+        try:
+            return eval(f"dict({value})", {}, {})
+        except Exception:
+            if metavar:
+                message = f"Invalid value for {metavar}: {value!r}"
+            else:
+                message = f"Invalid value: {value!r}"
+            raise click.ClickException(message)
+    else:
+        return dict()
+
+
+# noinspection PyShadowingBuiltins
+@click.command(name="chunk")
+@click.argument('input', metavar='<input>')
+@click.argument('output', metavar='<output>')
+@click.option('--format', '-f', metavar='<format>', type=click.Choice(['zarr', 'netcdf']),
+              help="Format of the output. If not given, guessed from <output>.")
+@click.option('--params', '-p', metavar='<params>',
+              help="Parameters specific for the output format."
+                   " Comma-separated list of <key>=<value> pairs.")
+@click.option('--chunks', '-c', metavar='<chunks>', nargs=1, default=None,
+              help='Chunk sizes for each dimension.'
+                   ' Comma-separated list of <dim>=<size> pairs,'
+                   ' e.g. "time=1,lat=270,lon=270"')
+def chunk(input, output, format=None, params=None, chunks=None):
+    """
+    Write a new dataset with identical data and compression but with new chunk sizes.
+    """
+    chunk_sizes = None
+    if chunks:
+        chunk_sizes = _parse_kwargs(chunks, metavar="<chunks>")
+        for k, v in chunk_sizes.items():
+            if not isinstance(v, int) or v <= 0:
+                raise click.ClickException("Invalid value for <chunks>, "
+                                           f"chunk sizes must be positive integers: {chunks}")
+
+    write_kwargs = dict()
+    if params:
+        write_kwargs = _parse_kwargs(params, metavar="<params>")
+
+    from .api import guess_dataset_format
+    format_name = format if format else guess_dataset_format(output)
+
+    from .api import open_dataset, chunk_dataset, write_dataset
+
+    with open_dataset(input_path=input) as ds:
+        if chunk_sizes:
+            for k in chunk_sizes:
+                if k not in ds.dims:
+                    raise click.ClickException("Invalid value for <chunks>, "
+                                               f"{k!r} is not the name of any dimension: {chunks}")
+
+        chunked_dataset = chunk_dataset(ds, chunk_sizes=chunk_sizes, format_name=format_name)
+        write_dataset(chunked_dataset, output_path=output, format_name=format_name, **write_kwargs)
+
+
+@click.command(name="dump")
+@click.argument('dataset', metavar='<path>')
+@click.option('--variable', '-v', metavar='<variable>', multiple=True,
+              help="Name of a variable (multiple allowed).")
+@click.option('--encoding', '-e', is_flag=True, flag_value=True,
+              help="Dump also variable encoding information.")
+def dump(dataset, variable, encoding):
+    """
+    Dump contents of dataset.
+    """
+    from .api import open_dataset, dump_dataset
+    with open_dataset(dataset) as ds:
+        text = dump_dataset(ds, variable_names=variable, show_var_encoding=encoding)
+        print(text)
+
+
+# noinspection PyShadowingBuiltins
+@click.group()
+@click.version_option(version)
+def cli():
+    """
+    Xcube Toolkit
+    """
+
+
+cli.add_command(chunk)
+cli.add_command(dump)
+
+
+def main(args=None):
+    cli.main(args=args)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/xcube/dsio.py b/xcube/dsio.py
index f73799e8..9cf09f69 100644
--- a/xcube/dsio.py
+++ b/xcube/dsio.py
@@ -20,8 +20,9 @@
 # SOFTWARE.
 
 import glob
+import os
 from abc import abstractmethod, ABCMeta
-from typing import Set, Callable, List, Optional, Dict, Iterable
+from typing import Set, Callable, List, Optional, Dict, Iterable, Tuple
 
 import s3fs
 import xarray as xr
@@ -29,6 +30,10 @@ import zarr
 
 from xcube.objreg import get_obj_registry
 
+FORMAT_NAME_MEM = "mem"
+FORMAT_NAME_ZARR = "zarr"
+FORMAT_NAME_NETCDF4 = "netcdf4"
+
 
 def open_from_fs(paths: str, recursive: bool = False, **kwargs):
     """
@@ -94,6 +99,18 @@ class DatasetIO(metaclass=ABCMeta):
     def modes(self) -> Set[str]:
         pass
 
+    @abstractmethod
+    def fitness(self, path: str, path_type: str = None) -> float:
+        """
+        Compute a fitness in the interval [0 to 1] for reading/writing from/to
+        the given *path*.
+
+        :param path: The path or URL.
+        :param path_type: Either "file", "dir", "url", or None.
+        :return: the chance in range [0 to 1]
+        """
+        return 0.0
+
     def read(self, input_path: str, **kwargs) -> xr.Dataset:
         raise NotImplementedError()
 
@@ -128,6 +145,57 @@ def find_dataset_io(format_name: str, modes: Iterable[str] = None, default: Data
     return default
 
 
+def guess_dataset_format(path: str) -> Optional[str]:
+    """
+    Guess a dataset format for a file system path or URL given by *path*.
+
+    :param path: A file system path or URL.
+    :return: The name of a dataset format guessed from *path*.
+    """
+    dataset_io_fitness_list = guess_dataset_ios(path)
+    if dataset_io_fitness_list:
+        return dataset_io_fitness_list[0][0].name
+    return None
+
+
+def guess_dataset_ios(path: str) -> List[Tuple[DatasetIO, float]]:
+    """
+    Guess suitable DatasetIO objects for a file system path or URL given by *path*.
+
+    Returns a list of (DatasetIO, fitness) tuples, sorted by descending fitness values.
+    Fitness values are in the interval (0, 1].
+
+    The first entry is the most appropriate DatasetIO object.
+
+    :param path: A file system path or URL.
+    :return: A list of (DatasetIO, fitness) tuples.
+    """
+    if os.path.isfile(path):
+        input_type = "file"
+    elif os.path.isdir(path):
+        input_type = "dir"
+    elif path.find("://") > 0:
+        input_type = "url"
+    else:
+        input_type = None
+
+    dataset_ios = get_obj_registry().get_all(type=DatasetIO)
+
+    dataset_io_fitness_list = []
+    for dataset_io in dataset_ios:
+        fitness = dataset_io.fitness(path, path_type=input_type)
+        if fitness > 0.0:
+            dataset_io_fitness_list.append((dataset_io, fitness))
+
+    dataset_io_fitness_list.sort(key=lambda item: -item[1])
+    return dataset_io_fitness_list
+
+
+def _get_ext(path: str) -> Optional[str]:
+    _, ext = os.path.splitext(path)
+    return ext.lower()
+
+
 def query_dataset_io(filter_fn: Callable[[DatasetIO], bool] = None) -> List[DatasetIO]:
     dataset_ios = get_obj_registry().get_all(type=DatasetIO)
     if filter_fn is None:
@@ -141,7 +209,7 @@ class MemDatasetIO(DatasetIO):
 
     @property
     def name(self) -> str:
-        return 'mem'
+        return FORMAT_NAME_MEM
 
     @property
     def description(self) -> str:
@@ -155,6 +223,13 @@ class MemDatasetIO(DatasetIO):
     def modes(self) -> Set[str]:
         return {'r', 'w', 'a'}
 
+    def fitness(self, path: str, path_type: str = None) -> float:
+        if path in self.datasets:
+            return 1.0
+        ext_value = _get_ext(path) == ".mem"
+        type_value = 0.0
+        return (3 * ext_value + type_value) / 4
+
     def read(self, path: str, **kwargs) -> xr.Dataset:
         if path in self.datasets:
             return self.datasets[path]
@@ -180,7 +255,7 @@ class Netcdf4DatasetIO(DatasetIO):
 
     @property
     def name(self) -> str:
-        return 'netcdf4'
+        return FORMAT_NAME_NETCDF4
 
     @property
     def description(self) -> str:
@@ -194,6 +269,18 @@ class Netcdf4DatasetIO(DatasetIO):
     def modes(self) -> Set[str]:
         return {'r', 'w', 'a'}
 
+    def fitness(self, path: str, path_type: str = None) -> float:
+        ext = _get_ext(path)
+        ext_value = ext in {'.nc', '.hdf', '.h5'}
+        type_value = 0.0
+        if path_type is "file":
+            type_value = 1.0
+        elif path_type is None:
+            type_value = 0.5
+        else:
+            ext_value = 0.0
+        return (3 * ext_value + type_value) / 4
+
     def read(self, input_path: str, **kwargs) -> xr.Dataset:
         return xr.open_dataset(input_path, **kwargs)
 
@@ -222,7 +309,7 @@ class ZarrDatasetIO(DatasetIO):
 
     @property
     def name(self) -> str:
-        return 'zarr'
+        return FORMAT_NAME_ZARR
 
     @property
     def description(self) -> str:
@@ -236,6 +323,35 @@ class ZarrDatasetIO(DatasetIO):
     def modes(self) -> Set[str]:
         return {'r', 'w', 'a'}
 
+    def fitness(self, path: str, path_type: str = None) -> float:
+        ext = _get_ext(path)
+        ext_value = 0.0
+        type_value = 0.0
+        if ext == ".zarr":
+            ext_value = 1.0
+            if path_type is "dir":
+                type_value = 1.0
+            elif path_type == "url" or path_type is None:
+                type_value = 0.5
+            else:
+                ext_value = 0.0
+        else:
+            lower_path = path.lower()
+            if lower_path.endswith(".zarr.zip"):
+                ext_value = 1.0
+                if path_type == "file":
+                    type_value = 1.0
+                elif path_type is None:
+                    type_value = 0.5
+                else:
+                    ext_value = 0.0
+            else:
+                if path_type is "dir":
+                    type_value = 1.0
+                elif path_type == "url":
+                    type_value = 0.5
+        return (3 * ext_value + type_value) / 4
+
     def read(self, path: str, **kwargs) -> xr.Dataset:
         return xr.open_zarr(path, **kwargs)
 


Test patch:
diff --git a/test/sampledata.py b/test/sampledata.py
index ed832d4b..2054ab00 100644
--- a/test/sampledata.py
+++ b/test/sampledata.py
@@ -4,6 +4,15 @@ import xarray as xr
 
 
 def new_test_dataset(time, height=180, **indexers):
+    """
+    Create a test dataset with dimensions ("time", "lat", "lon") and data variables given by *indexers*.
+
+    :param time: Single date/time string or sequence of date-time strings.
+    :param height: Size of the latitude dimension.
+    :param indexers: Variable name to value mapping.
+           Value may be a scalar or a vector of same length as *time*.
+    :return: test dataset
+    """
     time = [time] if isinstance(time, str) else time
     width = height * 2
     num_times = len(time)
@@ -25,6 +34,20 @@ def new_test_dataset(time, height=180, **indexers):
                                   lon=(['lon'], np.linspace(-180 + res, +180 - res, width))))
 
 
+def new_test_cube(size=100, periods=5):
+    dims = ("time", "lat", "lon")
+    w = 2 * size
+    h = size
+    time = pd.date_range('2010-01-01', periods=periods)
+    lon = np.linspace(0, 4, w)
+    lat = np.linspace(50, 52, h)
+    precipitation_var = xr.DataArray(np.random.rand(periods, h, w), coords=(time, lat, lon), dims=dims)
+    temperature_var = xr.DataArray(np.random.rand(periods, h, w), coords=(time, lat, lon), dims=dims)
+    ds = xr.Dataset({"precipitation": precipitation_var, "temperature": temperature_var})
+    ds.attrs.update(dict(time_coverage_start=str(time[0]), time_coverage_end=str(time[-1])))
+    return ds
+
+
 def create_highroc_dataset(no_spectra=False):
     """
     Simulates a HIGHROC OLCI L2 product in NetCDF 4 format
diff --git a/test/test_api.py b/test/test_api.py
new file mode 100644
index 00000000..e0922b0f
--- /dev/null
+++ b/test/test_api.py
@@ -0,0 +1,113 @@
+import os
+import unittest
+
+from test.sampledata import new_test_cube, new_test_dataset
+from xcube.api import open_dataset, read_dataset, write_dataset, dump_dataset, chunk_dataset
+
+TEST_NC_FILE = "test.nc"
+
+
+class OpenReadWriteDatasetTest(unittest.TestCase):
+    def setUp(self):
+        super().setUp()
+        dataset = new_test_cube()
+        dataset.to_netcdf(TEST_NC_FILE, mode="w")
+        dataset.close()
+
+    def tearDown(self):
+        os.remove(TEST_NC_FILE)
+        super().tearDown()
+
+    def test_open_dataset(self):
+        with open_dataset(TEST_NC_FILE) as ds:
+            self.assertIsNotNone(ds)
+
+    def test_read_dataset(self):
+        ds = read_dataset(TEST_NC_FILE)
+        self.assertIsNotNone(ds)
+        ds.close()
+
+    def test_write_dataset(self):
+        TEST_NC_FILE_2 = "test-2.nc"
+
+        dataset = new_test_cube()
+        try:
+            write_dataset(dataset, TEST_NC_FILE_2)
+            self.assertTrue(os.path.isfile(TEST_NC_FILE_2))
+        finally:
+            if os.path.isfile(TEST_NC_FILE_2):
+                os.remove(TEST_NC_FILE_2)
+
+
+class ChunkDatasetTest(unittest.TestCase):
+    def test_chunk_dataset(self):
+        dataset = new_test_dataset(["2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05"],
+                                   precipitation=0.4, temperature=275.2)
+
+        chunked_dataset = chunk_dataset(dataset,
+                                        chunk_sizes=dict(time=1, lat=10, lon=20),
+                                        format_name="zarr")
+        self.assertEqual({'chunks': (1, 10, 20)}, chunked_dataset.precipitation.encoding)
+        self.assertEqual({'chunks': (1, 10, 20)}, chunked_dataset.temperature.encoding)
+
+        chunked_dataset = chunk_dataset(dataset,
+                                        chunk_sizes=dict(time=1, lat=20, lon=40),
+                                        format_name="netcdf4")
+        self.assertEqual({'chunksizes': (1, 20, 40)}, chunked_dataset.precipitation.encoding)
+        self.assertEqual({'chunksizes': (1, 20, 40)}, chunked_dataset.temperature.encoding)
+
+        chunked_dataset = chunk_dataset(dataset,
+                                        chunk_sizes=dict(time=1, lat=20, lon=40))
+        self.assertEqual({}, chunked_dataset.precipitation.encoding)
+        self.assertEqual({}, chunked_dataset.temperature.encoding)
+
+    def test_unchunk_dataset(self):
+        dataset = new_test_dataset(["2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05"],
+                                   precipitation=0.4, temperature=275.2)
+
+        for var in dataset.data_vars.values():
+            var.encoding.update({"chunks": (5, 180, 360), "_FillValue": -999.0})
+
+        chunked_dataset = chunk_dataset(dataset, format_name="zarr")
+        self.assertEqual({"_FillValue": -999.0}, chunked_dataset.precipitation.encoding)
+        self.assertEqual({"_FillValue": -999.0}, chunked_dataset.temperature.encoding)
+
+
+class DumpDatasetTest(unittest.TestCase):
+    def test_dump_dataset(self):
+        dataset = new_test_dataset(["2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05"],
+                                   precipitation=0.4, temperature=275.2)
+        for var in dataset.variables.values():
+            var.encoding.update({"_FillValue": 999.0})
+
+        text = dump_dataset(dataset)
+        self.assertIn("<xarray.Dataset>", text)
+        self.assertIn("Dimensions:        (lat: 180, lon: 360, time: 5)\n", text)
+        self.assertIn("Coordinates:\n", text)
+        self.assertIn("  * lon            (lon) float64 ", text)
+        self.assertIn("Data variables:\n", text)
+        self.assertIn("    precipitation  (time, lat, lon) float64 ", text)
+        self.assertNotIn("Encoding for coordinate variable 'lat':\n", text)
+        self.assertNotIn("Encoding for data variable 'temperature':\n", text)
+        self.assertNotIn("    _FillValue:  999.0\n", text)
+
+        text = dump_dataset(dataset, show_var_encoding=True)
+        self.assertIn("<xarray.Dataset>", text)
+        self.assertIn("Dimensions:        (lat: 180, lon: 360, time: 5)\n", text)
+        self.assertIn("Coordinates:\n", text)
+        self.assertIn("  * lon            (lon) float64 ", text)
+        self.assertIn("Data variables:\n", text)
+        self.assertIn("    precipitation  (time, lat, lon) float64 ", text)
+        self.assertIn("Encoding for coordinate variable 'lat':\n", text)
+        self.assertIn("Encoding for data variable 'temperature':\n", text)
+        self.assertIn("    _FillValue:  999.0\n", text)
+
+        text = dump_dataset(dataset, ["precipitation"])
+        self.assertIn("<xarray.DataArray 'precipitation' (time: 5, lat: 180, lon: 360)>\n", text)
+        self.assertNotIn("Encoding:\n", text)
+        self.assertNotIn("    _FillValue:  999.0", text)
+
+        text = dump_dataset(dataset, ["precipitation"], show_var_encoding=True)
+        self.assertIn("<xarray.DataArray 'precipitation' (time: 5, lat: 180, lon: 360)>\n", text)
+        self.assertIn("Encoding:\n", text)
+        self.assertIn("    _FillValue:  999.0", text)
diff --git a/test/test_cli.py b/test/test_cli.py
new file mode 100644
index 00000000..e6c67e38
--- /dev/null
+++ b/test/test_cli.py
@@ -0,0 +1,156 @@
+import os
+import shutil
+import unittest
+from abc import ABCMeta
+from typing import List
+
+import click
+import click.testing
+import numpy as np
+import pandas as pd
+import xarray as xr
+
+from test.sampledata import new_test_cube
+from xcube.cli import cli, _parse_kwargs
+
+TEST_NC_FILE = "test.nc"
+TEST_ZARR_DIR = "test.zarr"
+
+
+class CliTest(unittest.TestCase, metaclass=ABCMeta):
+
+    def invoke_cli(self, args: List[str]):
+        self.runner = click.testing.CliRunner()
+        return self.runner.invoke(cli, args, catch_exceptions=False)
+
+    def setUp(self):
+        super().setUp()
+        dataset = new_test_cube()
+        dataset.to_netcdf(TEST_NC_FILE, mode="w")
+        dataset.to_zarr(TEST_ZARR_DIR, mode="w")
+
+    def tearDown(self):
+        if os.path.isdir(TEST_ZARR_DIR):
+            shutil.rmtree(TEST_ZARR_DIR, ignore_errors=True)
+        os.remove(TEST_NC_FILE)
+        super().tearDown()
+
+
+class DumpTest(CliTest):
+
+    def test_dump_ds(self):
+        result = self.invoke_cli(["dump", TEST_NC_FILE])
+        self.assertEqual((
+            "<xarray.Dataset>\n"
+            "Dimensions:        (lat: 100, lon: 200, time: 5)\n"
+            "Coordinates:\n"
+            "  * time           (time) datetime64[ns] 2010-01-01 2010-01-02 2010-01-03 ...\n"
+            "  * lat            (lat) float64 50.0 50.02 50.04 50.06 50.08 50.1 50.12 ...\n"
+            "  * lon            (lon) float64 0.0 0.0201 0.0402 0.0603 0.0804 0.1005 ...\n"
+            "Data variables:\n"
+            "    precipitation  (time, lat, lon) float64 ...\n"
+            "    temperature    (time, lat, lon) float64 ...\n"
+            "Attributes:\n"
+            "    time_coverage_start:  2010-01-01 00:00:00\n"
+            "    time_coverage_end:    2010-01-05 00:00:00\n"
+        ), result.output)
+        self.assertEqual(0, result.exit_code)
+
+
+class ChunkTest(CliTest):
+
+    def test_chunk_zarr(self):
+        output_path = "test-chunked.zarr"
+        result = self.invoke_cli(["chunk",
+                                  TEST_ZARR_DIR,
+                                  output_path,
+                                  "-c", "time=1,lat=20,lon=40"])
+        self.assertEqual("", result.output)
+        self.assertEqual(0, result.exit_code)
+        self.assertTrue(os.path.isdir(output_path))
+        try:
+            ds = xr.open_zarr(output_path)
+            self.assertIn("precipitation", ds)
+            precipitation = ds["precipitation"]
+            self.assertTrue(hasattr(precipitation, "encoding"))
+            self.assertIn("chunks", precipitation.encoding)
+            self.assertEqual(precipitation.encoding["chunks"], (1, 20, 40))
+        finally:
+            shutil.rmtree(output_path, ignore_errors=True)
+
+    # TODO (forman): this test fails
+    # netCDF4\_netCDF4.pyx:2437: in netCDF4._netCDF4.Dataset.createVariable
+    # ValueError: cannot specify chunksizes for a contiguous dataset
+    #
+    # def test_chunk_nc(self):
+    #     output_path = "test-chunked.nc"
+    #     result = self.invoke_cli(["chunk",
+    #                               TEST_NC_FILE,
+    #                               output_path,
+    #                               "-c", "time=1,lat=20,lon=40"])
+    #     self.assertEqual("", result.output)
+    #     self.assertEqual(0, result.exit_code)
+    #     self.assertTrue(os.path.isdir(output_path))
+    #     try:
+    #         ds = xr.open_zarr(output_path)
+    #         self.assertIn("precipitation", ds)
+    #         precipitation = ds["precipitation"]
+    #         self.assertTrue(hasattr(precipitation, "encoding"))
+    #         self.assertIn("chunksizes", precipitation.encoding)
+    #         self.assertEqual(precipitation.encoding["chunksizes"], (1, 20, 40))
+    #     finally:
+    #         os.remove(output_path)
+
+    def test_chunk_size_syntax(self):
+        result = self.invoke_cli(["chunk",
+                                  TEST_NC_FILE,
+                                  "test-chunked.zarr",
+                                  "-c", "time=1,lat!gnnn,lon=40"])
+        self.assertEqual("Error: Invalid value for <chunks>:"
+                         " 'time=1,lat!gnnn,lon=40'\n",
+                         result.output)
+        self.assertEqual(1, result.exit_code)
+
+    def test_chunk_size_not_an_int(self):
+        result = self.invoke_cli(["chunk",
+                                  TEST_NC_FILE,
+                                  "test-chunked.zarr",
+                                  "-c", "time=1,lat=20.3,lon=40"])
+        self.assertEqual("Error: Invalid value for <chunks>,"
+                         " chunk sizes must be positive integers:"
+                         " time=1,lat=20.3,lon=40\n",
+                         result.output)
+        self.assertEqual(1, result.exit_code)
+
+    def test_chunk_size_not_a_dim(self):
+        result = self.invoke_cli(["chunk",
+                                  TEST_NC_FILE,
+                                  "test-chunked.zarr",
+                                  "-c", "time=1,lati=20,lon=40"])
+        self.assertEqual("Error: Invalid value for <chunks>,"
+                         " 'lati' is not the name of any dimension:"
+                         " time=1,lati=20,lon=40\n",
+                         result.output)
+        self.assertEqual(1, result.exit_code)
+
+
+class ParseTest(unittest.TestCase):
+
+    def test_parse_kwargs(self):
+        self.assertEqual(dict(),
+                         _parse_kwargs("", metavar="<chunks>"))
+        self.assertEqual(dict(time=1, lat=256, lon=512),
+                         _parse_kwargs("time=1, lat=256, lon=512", metavar="<chunks>"))
+
+        with self.assertRaises(click.ClickException) as cm:
+            _parse_kwargs("45 * 'A'", metavar="<chunks>")
+        self.assertEqual("Invalid value for <chunks>: \"45 * 'A'\"",
+                         f"{cm.exception}")
+
+        with self.assertRaises(click.ClickException) as cm:
+            _parse_kwargs("9==2")
+        self.assertEqual("Invalid value: '9==2'",
+                         f"{cm.exception}")
+
+
+
diff --git a/test/test_dsio.py b/test/test_dsio.py
index 161550a4..1a3931ae 100644
--- a/test/test_dsio.py
+++ b/test/test_dsio.py
@@ -28,6 +28,9 @@ class MyDatasetIO(DatasetIO):
     def modes(self) -> Set[str]:
         return set()
 
+    def fitness(self, input_path: str, path_type: str = None) -> float:
+        return 0.0
+
 
 class DatasetIOTest(unittest.TestCase):
 
@@ -58,6 +61,25 @@ class MemDatasetIOTest(unittest.TestCase):
         self.assertEqual('In-memory dataset I/O', ds_io.description)
         self.assertEqual({'r', 'w', 'a'}, ds_io.modes)
 
+    def test_fitness(self):
+        ds_io = MemDatasetIO()
+
+        self.assertEqual(0.75, ds_io.fitness("test.mem", path_type=None))
+        self.assertEqual(0.75, ds_io.fitness("test.mem", path_type="file"))
+        self.assertEqual(0.75, ds_io.fitness("test.mem", path_type="dir"))
+        self.assertEqual(0.75, ds_io.fitness("http://dsio/test.mem", path_type="url"))
+
+        self.assertEqual(0.0, ds_io.fitness("test.png", path_type=None))
+        self.assertEqual(0.0, ds_io.fitness("test.png", path_type="file"))
+        self.assertEqual(0.0, ds_io.fitness("test.png", path_type="dir"))
+        self.assertEqual(0.0, ds_io.fitness("http://dsio/test.png", path_type="url"))
+
+        ds_io.write(xr.Dataset(), "bibo.odod")
+        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type=None))
+        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type="file"))
+        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type="dir"))
+        self.assertEqual(1.0, ds_io.fitness("bibo.odod", path_type="url"))
+
     def test_read(self):
         ds_io = MemDatasetIO()
         with self.assertRaises(FileNotFoundError):
@@ -101,6 +123,30 @@ class Netcdf4DatasetIOTest(unittest.TestCase):
         self.assertEqual('NetCDF-4 file format', ds_io.description)
         self.assertEqual({'a', 'r', 'w'}, ds_io.modes)
 
+    def test_fitness(self):
+        ds_io = Netcdf4DatasetIO()
+
+        self.assertEqual(0.875, ds_io.fitness("test.nc", path_type=None))
+        self.assertEqual(0.875, ds_io.fitness("test.hdf", path_type=None))
+        self.assertEqual(0.875, ds_io.fitness("test.h5", path_type=None))
+
+        self.assertEqual(1.0, ds_io.fitness("test.nc", path_type="file"))
+        self.assertEqual(1.0, ds_io.fitness("test.hdf", path_type="file"))
+        self.assertEqual(1.0, ds_io.fitness("test.h5", path_type="file"))
+
+        self.assertEqual(0.0, ds_io.fitness("test.nc", path_type="dir"))
+        self.assertEqual(0.0, ds_io.fitness("test.hdf", path_type="dir"))
+        self.assertEqual(0.0, ds_io.fitness("test.h5", path_type="dir"))
+
+        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.nc", path_type="url"))
+        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.hdf", path_type="url"))
+        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.h5", path_type="url"))
+
+        self.assertEqual(0.125, ds_io.fitness("test.tif", path_type=None))
+        self.assertEqual(0.25, ds_io.fitness("test.tif", path_type="file"))
+        self.assertEqual(0.0, ds_io.fitness("test.tif", path_type="dir"))
+        self.assertEqual(0.0, ds_io.fitness("https://dsio/test.tif", path_type="url"))
+
     def test_read(self):
         ds_io = Netcdf4DatasetIO()
         with self.assertRaises(FileNotFoundError):
@@ -116,6 +162,24 @@ class ZarrDatasetIOTest(unittest.TestCase):
         self.assertEqual('Zarr file format (http://zarr.readthedocs.io)', ds_io.description)
         self.assertEqual({'a', 'r', 'w'}, ds_io.modes)
 
+    def test_fitness(self):
+        ds_io = ZarrDatasetIO()
+
+        self.assertEqual(0.875, ds_io.fitness("test.zarr", path_type=None))
+        self.assertEqual(0.0, ds_io.fitness("test.zarr", path_type="file"))
+        self.assertEqual(1.0, ds_io.fitness("test.zarr", path_type="dir"))
+        self.assertEqual(0.875, ds_io.fitness("http://dsio/test.zarr", path_type="url"))
+
+        self.assertEqual(0.875, ds_io.fitness("test.zarr.zip", path_type=None))
+        self.assertEqual(1.0, ds_io.fitness("test.zarr.zip", path_type="file"))
+        self.assertEqual(0.0, ds_io.fitness("test.zarr.zip", path_type="dir"))
+        self.assertEqual(0.0, ds_io.fitness("http://dsio/test.zarr.zip", path_type="url"))
+
+        self.assertEqual(0.0, ds_io.fitness("test.png", path_type=None))
+        self.assertEqual(0.0, ds_io.fitness("test.png", path_type="file"))
+        self.assertEqual(0.25, ds_io.fitness("test.png", path_type="dir"))
+        self.assertEqual(0.125, ds_io.fitness("http://dsio/test.png", path_type="url"))
+
     def test_read(self):
         ds_io = ZarrDatasetIO()
         with self.assertRaises(ValueError):